| Message ID | ba72fae1b0dff34ff4474a2cd53939a6c4fd3279.1720546299.git.jan.kiszka@siemens.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | Introduce container fetcher and pre-loader | expand |
On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > This allows to write dpkg-raw recipes which packages archived > container > images and load them into a local docker or podman registry on boot. > The > scenario behind this is to pre-fill local registries in a way that > still > permits live updates during runtime. > > The loader script only process images which are not yet available > under > the same name and tag in the local registry. Also after loading, the > archived images stay on the local file system. This allows to perform > reloading in case the local registry should be emptied (e.g. reset to > factory state). To reduce the space those original images need, they > are > compressed, by default with xz. Hi, can we switch to zstd please? That's much quicker and needs a fraction of the memory for decompression. On small devices, decompressing large xz files is sometimes simply not possible due to OOM. > > Separate include files are available to cater the main container > engines, one for docker and one for podman. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > .../container-loader/container-loader.inc | 76 > +++++++++++++++++++ > .../container-loader/docker-loader.inc | 10 +++ > .../files/container-loader.service.tmpl | 11 +++ > .../files/container-loader.sh.tmpl | 13 ++++ > .../container-loader/podman-loader.inc | 10 +++ > 5 files changed, 120 insertions(+) > create mode 100644 meta/recipes-support/container-loader/container- > loader.inc > create mode 100644 meta/recipes-support/container-loader/docker- > loader.inc > create mode 100644 meta/recipes-support/container- > loader/files/container-loader.service.tmpl > create mode 100755 meta/recipes-support/container- > loader/files/container-loader.sh.tmpl > create mode 100644 meta/recipes-support/container-loader/podman- > loader.inc > > diff --git a/meta/recipes-support/container-loader/container- > loader.inc b/meta/recipes-support/container-loader/container- > loader.inc > new file mode 100644 > index 00000000..8e352214 > --- /dev/null > +++ b/meta/recipes-support/container-loader/container-loader.inc > @@ -0,0 +1,76 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +FILESPATH:append := ":${FILE_DIRNAME}/files" > + > +inherit dpkg-raw > + > +SRC_URI += " \ > + file://container-loader.service.tmpl \ > + file://container-loader.sh.tmpl" > + > +TEMPLATE_FILES += " \ > + container-loader.service.tmpl \ > + container-loader.sh.tmpl" > +TEMPLATE_VARS += "CONTAINER_ENGINE" > + > +CONTAINER_COMPRESSION ?= "xz" > + > +DEBIAN_DEPENDS += " \ > + ${CONTAINER_ENGINE_PACKAGES} \ > + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' > else \ > + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +CONTAINER_COMPRESSOR = "${@ \ > + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +python do_install() { > + import os > + > + workdir = d.getVar('WORKDIR') > + D = d.getVar('D') > + PN= d.getVar('PN') > + > + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") > + > + src_uri = d.getVar('SRC_URI').split() > + for uri in src_uri: > + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) > + if scheme != "docker": > + continue > + > + image_name = host + (path if path != "/" else "") > + unpacked_image = workdir + "/" + image_name.replace('/', > '.') > + dest_dir = D + "/usr/share/" + PN + "/images" > + tar_image = dest_dir + "/" + image_name.replace('/', '.') + > ".tar" > + docker_ref = ":" + parm["tag"] if "tag" in parm else > "latest" > + > + cmd = f"skopeo copy dir:{unpacked_image} " \ > + f"docker-archive:{tar_image}:{image_name}{docker_ref}" Why not oci-archive? In oci-archive, the layers are compressed. Or do we explicitly not want to have the layers themselves compressed as we compress the whole artifact anyways? > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + line = > f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " > + \ > + image_name + docker_ref > + bb.note(f"adding '{line}' to image.list") > + image_list.write(line + "\n") > + > + image_list.close() > + > + bb.utils.copyfile(workdir + "/container-loader.sh", > + D + "/usr/share/" + PN + Shouldn't that be BPN instead of PN? > "/container-loader.sh") > +} > +do_install[cleandirs] += "${D}/usr/share/${PN}/images" Same here. > + > +do_prepare_build:append() { > + install -v -m 644 ${WORKDIR}/container-loader.service > ${S}/debian/${PN}.service > +} > diff --git a/meta/recipes-support/container-loader/docker-loader.inc > b/meta/recipes-support/container-loader/docker-loader.inc > new file mode 100644 > index 00000000..b864c854 > --- /dev/null > +++ b/meta/recipes-support/container-loader/docker-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "docker" > + > +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" > diff --git a/meta/recipes-support/container-loader/files/container- > loader.service.tmpl b/meta/recipes-support/container- > loader/files/container-loader.service.tmpl > new file mode 100644 > index 00000000..afde55d3 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container- > loader.service.tmpl > @@ -0,0 +1,11 @@ > +[Unit] > +Description=Load archived container images on boot > +After=${CONTAINER_ENGINE}.service We need an Requires=${CONTAINER_ENGINE}.service as well. Otherwise just the temporal order is defined, but not the logical order. > + > +[Service] > +Type=oneshot > +ExecStart=/usr/share/${PN}/container-loader.sh > +RemainAfterExit=true In case of rootless podman, the user that executes the script matters. But I don't know if we want to consider this use-case by now. > + > +[Install] > +WantedBy=multi-user.target > diff --git a/meta/recipes-support/container-loader/files/container- > loader.sh.tmpl b/meta/recipes-support/container- > loader/files/container-loader.sh.tmpl > new file mode 100755 > index 00000000..31d27865 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container- > loader.sh.tmpl > @@ -0,0 +1,13 @@ > +#!/bin/sh > +# > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +set -eu > + > +while read -r image ref; do > + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then > + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" That does not work with all compression formats. Better decompress the image manually and pass to stdin of the <tool> load. e.g. cat $image | unzstd | podman load - Also note, that podman needs tons of temporary space to load a compressed image (or compressed layers), so you might want to set e.g. TMPDIR=/var/tmp as /tmp could be a small memory backed fs. Felix > + fi > +done < /usr/share/${PN}/image.list > diff --git a/meta/recipes-support/container-loader/podman-loader.inc > b/meta/recipes-support/container-loader/podman-loader.inc > new file mode 100644 > index 00000000..d2c9a12d > --- /dev/null > +++ b/meta/recipes-support/container-loader/podman-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "podman" > + > +CONTAINER_ENGINE_PACKAGES ?= "podman"
On 10.07.24 14:46, Moessbauer, Felix (T CED OES-DE) wrote: > On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: >> From: Jan Kiszka <jan.kiszka@siemens.com> >> >> This allows to write dpkg-raw recipes which packages archived >> container >> images and load them into a local docker or podman registry on boot. >> The >> scenario behind this is to pre-fill local registries in a way that >> still >> permits live updates during runtime. >> >> The loader script only process images which are not yet available >> under >> the same name and tag in the local registry. Also after loading, the >> archived images stay on the local file system. This allows to perform >> reloading in case the local registry should be emptied (e.g. reset to >> factory state). To reduce the space those original images need, they >> are >> compressed, by default with xz. > > Hi, can we switch to zstd please? That's much quicker and needs a > fraction of the memory for decompression. On small devices, > decompressing large xz files is sometimes simply not possible due to > OOM. Can be added - once Debian is on a docker version that supports it :) > >> >> Separate include files are available to cater the main container >> engines, one for docker and one for podman. >> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >> --- >> .../container-loader/container-loader.inc | 76 >> +++++++++++++++++++ >> .../container-loader/docker-loader.inc | 10 +++ >> .../files/container-loader.service.tmpl | 11 +++ >> .../files/container-loader.sh.tmpl | 13 ++++ >> .../container-loader/podman-loader.inc | 10 +++ >> 5 files changed, 120 insertions(+) >> create mode 100644 meta/recipes-support/container-loader/container- >> loader.inc >> create mode 100644 meta/recipes-support/container-loader/docker- >> loader.inc >> create mode 100644 meta/recipes-support/container- >> loader/files/container-loader.service.tmpl >> create mode 100755 meta/recipes-support/container- >> loader/files/container-loader.sh.tmpl >> create mode 100644 meta/recipes-support/container-loader/podman- >> loader.inc >> >> diff --git a/meta/recipes-support/container-loader/container- >> loader.inc b/meta/recipes-support/container-loader/container- >> loader.inc >> new file mode 100644 >> index 00000000..8e352214 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/container-loader.inc >> @@ -0,0 +1,76 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +FILESPATH:append := ":${FILE_DIRNAME}/files" >> + >> +inherit dpkg-raw >> + >> +SRC_URI += " \ >> + file://container-loader.service.tmpl \ >> + file://container-loader.sh.tmpl" >> + >> +TEMPLATE_FILES += " \ >> + container-loader.service.tmpl \ >> + container-loader.sh.tmpl" >> +TEMPLATE_VARS += "CONTAINER_ENGINE" >> + >> +CONTAINER_COMPRESSION ?= "xz" >> + >> +DEBIAN_DEPENDS += " \ >> + ${CONTAINER_ENGINE_PACKAGES} \ >> + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' >> else \ >> + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +CONTAINER_COMPRESSOR = "${@ \ >> + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ >> + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +python do_install() { >> + import os >> + >> + workdir = d.getVar('WORKDIR') >> + D = d.getVar('D') >> + PN= d.getVar('PN') >> + >> + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") >> + >> + src_uri = d.getVar('SRC_URI').split() >> + for uri in src_uri: >> + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) >> + if scheme != "docker": >> + continue >> + >> + image_name = host + (path if path != "/" else "") >> + unpacked_image = workdir + "/" + image_name.replace('/', >> '.') >> + dest_dir = D + "/usr/share/" + PN + "/images" >> + tar_image = dest_dir + "/" + image_name.replace('/', '.') + >> ".tar" >> + docker_ref = ":" + parm["tag"] if "tag" in parm else >> "latest" >> + >> + cmd = f"skopeo copy dir:{unpacked_image} " \ >> + f"docker-archive:{tar_image}:{image_name}{docker_ref}" > > Why not oci-archive? In oci-archive, the layers are compressed. Or do > we explicitly not want to have the layers themselves compressed as we > compress the whole artifact anyways? Not tested if we aren't causing any conversion issues towards docker & friends - with versions of docker we currently use. Are you sure, for bookworm and bullseye? > >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + line = >> f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " >> + \ >> + image_name + docker_ref >> + bb.note(f"adding '{line}' to image.list") >> + image_list.write(line + "\n") >> + >> + image_list.close() >> + >> + bb.utils.copyfile(workdir + "/container-loader.sh", >> + D + "/usr/share/" + PN + > > Shouldn't that be BPN instead of PN? Oh, yes, thanks. > >> "/container-loader.sh") >> +} >> +do_install[cleandirs] += "${D}/usr/share/${PN}/images" > > Same here. > >> + >> +do_prepare_build:append() { >> + install -v -m 644 ${WORKDIR}/container-loader.service >> ${S}/debian/${PN}.service >> +} >> diff --git a/meta/recipes-support/container-loader/docker-loader.inc >> b/meta/recipes-support/container-loader/docker-loader.inc >> new file mode 100644 >> index 00000000..b864c854 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/docker-loader.inc >> @@ -0,0 +1,10 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +require container-loader.inc >> + >> +CONTAINER_ENGINE = "docker" >> + >> +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" >> diff --git a/meta/recipes-support/container-loader/files/container- >> loader.service.tmpl b/meta/recipes-support/container- >> loader/files/container-loader.service.tmpl >> new file mode 100644 >> index 00000000..afde55d3 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container- >> loader.service.tmpl >> @@ -0,0 +1,11 @@ >> +[Unit] >> +Description=Load archived container images on boot >> +After=${CONTAINER_ENGINE}.service > > We need an Requires=${CONTAINER_ENGINE}.service as well. Otherwise just > the temporal order is defined, but not the logical order. > True. >> + >> +[Service] >> +Type=oneshot >> +ExecStart=/usr/share/${PN}/container-loader.sh >> +RemainAfterExit=true > > In case of rootless podman, the user that executes the script matters. > But I don't know if we want to consider this use-case by now. Right, thought about that as well but didn't see a generic answer to that yet. > >> + >> +[Install] >> +WantedBy=multi-user.target >> diff --git a/meta/recipes-support/container-loader/files/container- >> loader.sh.tmpl b/meta/recipes-support/container- >> loader/files/container-loader.sh.tmpl >> new file mode 100755 >> index 00000000..31d27865 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container- >> loader.sh.tmpl >> @@ -0,0 +1,13 @@ >> +#!/bin/sh >> +# >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +set -eu >> + >> +while read -r image ref; do >> + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then >> + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" > > That does not work with all compression formats. Better decompress the > image manually and pass to stdin of the <tool> load. e.g. It does work for those we support, see above. And it avoids having to carry to specific tool also in this script. So, not completely nicer. > > cat $image | unzstd | podman load - > > Also note, that podman needs tons of temporary space to load a > compressed image (or compressed layers), so you might want to set e.g. > TMPDIR=/var/tmp as /tmp could be a small memory backed fs. Strange that open-coding this should actually be worse than letting the engine to it. Does this hold for oci-archives with compressed layers? Thanks, Jan
On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > This allows to write dpkg-raw recipes which packages archived container > images and load them into a local docker or podman registry on boot. The > scenario behind this is to pre-fill local registries in a way that still > permits live updates during runtime. > > The loader script only process images which are not yet available under > the same name and tag in the local registry. Also after loading, the > archived images stay on the local file system. This allows to perform > reloading in case the local registry should be emptied (e.g. reset to > factory state). To reduce the space those original images need, they are > compressed, by default with xz. > > Separate include files are available to cater the main container > engines, one for docker and one for podman. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > .../container-loader/container-loader.inc | 76 +++++++++++++++++++ > .../container-loader/docker-loader.inc | 10 +++ > .../files/container-loader.service.tmpl | 11 +++ > .../files/container-loader.sh.tmpl | 13 ++++ > .../container-loader/podman-loader.inc | 10 +++ > 5 files changed, 120 insertions(+) > create mode 100644 meta/recipes-support/container-loader/container-loader.inc > create mode 100644 meta/recipes-support/container-loader/docker-loader.inc > create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl > create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl > create mode 100644 meta/recipes-support/container-loader/podman-loader.inc > > diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes- > support/container-loader/container-loader.inc > new file mode 100644 > index 00000000..8e352214 > --- /dev/null > +++ b/meta/recipes-support/container-loader/container-loader.inc > @@ -0,0 +1,76 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +FILESPATH:append := ":${FILE_DIRNAME}/files" > + > +inherit dpkg-raw > + > +SRC_URI += " \ > + file://container-loader.service.tmpl \ > + file://container-loader.sh.tmpl" > + > +TEMPLATE_FILES += " \ > + container-loader.service.tmpl \ > + container-loader.sh.tmpl" > +TEMPLATE_VARS += "CONTAINER_ENGINE" > + > +CONTAINER_COMPRESSION ?= "xz" > + > +DEBIAN_DEPENDS += " \ > + ${CONTAINER_ENGINE_PACKAGES} \ > + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +CONTAINER_COMPRESSOR = "${@ \ > + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +python do_install() { > + import os > + > + workdir = d.getVar('WORKDIR') > + D = d.getVar('D') > + PN= d.getVar('PN') > + > + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") > + > + src_uri = d.getVar('SRC_URI').split() > + for uri in src_uri: > + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) > + if scheme != "docker": > + continue > + > + image_name = host + (path if path != "/" else "") > + unpacked_image = workdir + "/" + image_name.replace('/', '.') > + dest_dir = D + "/usr/share/" + PN + "/images" > + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" > + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" > + > + cmd = f"skopeo copy dir:{unpacked_image} " \ > + f"docker-archive:{tar_image}:{image_name}{docker_ref}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ > + image_name + docker_ref > + bb.note(f"adding '{line}' to image.list") > + image_list.write(line + "\n") > + > + image_list.close() > + > + bb.utils.copyfile(workdir + "/container-loader.sh", > + D + "/usr/share/" + PN + "/container-loader.sh") > +} > +do_install[cleandirs] += "${D}/usr/share/${PN}/images" > + > +do_prepare_build:append() { > + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${PN}.service > +} > diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes- > support/container-loader/docker-loader.inc > new file mode 100644 > index 00000000..b864c854 > --- /dev/null > +++ b/meta/recipes-support/container-loader/docker-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "docker" > + > +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" > diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl > b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > new file mode 100644 > index 00000000..afde55d3 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > @@ -0,0 +1,11 @@ > +[Unit] > +Description=Load archived container images on boot > +After=${CONTAINER_ENGINE}.service > + > +[Service] > +Type=oneshot > +ExecStart=/usr/share/${PN}/container-loader.sh > +RemainAfterExit=true > + > +[Install] > +WantedBy=multi-user.target > diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes- > support/container-loader/files/container-loader.sh.tmpl > new file mode 100755 > index 00000000..31d27865 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl > @@ -0,0 +1,13 @@ > +#!/bin/sh > +# > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +set -eu > + > +while read -r image ref; do > + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then > + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" > + fi > +done < /usr/share/${PN}/image.list Should the images be deleted after import? I see no reason to preserve them once they have been added to the local docker engine. An option to influence that would be good (e.g. CONTAINER_IMAGES_CLEAN). > diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes- > support/container-loader/podman-loader.inc > new file mode 100644 > index 00000000..d2c9a12d > --- /dev/null > +++ b/meta/recipes-support/container-loader/podman-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "podman" > + > +CONTAINER_ENGINE_PACKAGES ?= "podman" Benedikt
On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > This allows to write dpkg-raw recipes which packages archived container > images and load them into a local docker or podman registry on boot. The > scenario behind this is to pre-fill local registries in a way that still > permits live updates during runtime. > > The loader script only process images which are not yet available under > the same name and tag in the local registry. Also after loading, the > archived images stay on the local file system. This allows to perform > reloading in case the local registry should be emptied (e.g. reset to > factory state). To reduce the space those original images need, they are > compressed, by default with xz. > > Separate include files are available to cater the main container > engines, one for docker and one for podman. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > .../container-loader/container-loader.inc | 76 +++++++++++++++++++ > .../container-loader/docker-loader.inc | 10 +++ > .../files/container-loader.service.tmpl | 11 +++ > .../files/container-loader.sh.tmpl | 13 ++++ > .../container-loader/podman-loader.inc | 10 +++ > 5 files changed, 120 insertions(+) > create mode 100644 meta/recipes-support/container-loader/container-loader.inc > create mode 100644 meta/recipes-support/container-loader/docker-loader.inc > create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl > create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl > create mode 100644 meta/recipes-support/container-loader/podman-loader.inc > > diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes- > support/container-loader/container-loader.inc > new file mode 100644 > index 00000000..8e352214 > --- /dev/null > +++ b/meta/recipes-support/container-loader/container-loader.inc > @@ -0,0 +1,76 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +FILESPATH:append := ":${FILE_DIRNAME}/files" > + > +inherit dpkg-raw > + > +SRC_URI += " \ > + file://container-loader.service.tmpl \ > + file://container-loader.sh.tmpl" > + > +TEMPLATE_FILES += " \ > + container-loader.service.tmpl \ > + container-loader.sh.tmpl" > +TEMPLATE_VARS += "CONTAINER_ENGINE" > + > +CONTAINER_COMPRESSION ?= "xz" > + > +DEBIAN_DEPENDS += " \ > + ${CONTAINER_ENGINE_PACKAGES} \ > + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +CONTAINER_COMPRESSOR = "${@ \ > + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ''}" > + > +python do_install() { > + import os > + > + workdir = d.getVar('WORKDIR') > + D = d.getVar('D') > + PN= d.getVar('PN') > + > + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") > + > + src_uri = d.getVar('SRC_URI').split() > + for uri in src_uri: > + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) > + if scheme != "docker": > + continue > + > + image_name = host + (path if path != "/" else "") > + unpacked_image = workdir + "/" + image_name.replace('/', '.') > + dest_dir = D + "/usr/share/" + PN + "/images" > + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" > + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" > + > + cmd = f"skopeo copy dir:{unpacked_image} " \ > + f"docker-archive:{tar_image}:{image_name}{docker_ref}" regarding performance, using "xz -T0" or pigz instead of gzip would increase performance and reduce build time a lot. Benedikt > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ > + image_name + docker_ref > + bb.note(f"adding '{line}' to image.list") > + image_list.write(line + "\n") > + > + image_list.close() > + > + bb.utils.copyfile(workdir + "/container-loader.sh", > + D + "/usr/share/" + PN + "/container-loader.sh") > +} > +do_install[cleandirs] += "${D}/usr/share/${PN}/images" > + > +do_prepare_build:append() { > + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${PN}.service > +} > diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes- > support/container-loader/docker-loader.inc > new file mode 100644 > index 00000000..b864c854 > --- /dev/null > +++ b/meta/recipes-support/container-loader/docker-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "docker" > + > +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" > diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl > b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > new file mode 100644 > index 00000000..afde55d3 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > @@ -0,0 +1,11 @@ > +[Unit] > +Description=Load archived container images on boot > +After=${CONTAINER_ENGINE}.service > + > +[Service] > +Type=oneshot > +ExecStart=/usr/share/${PN}/container-loader.sh > +RemainAfterExit=true > + > +[Install] > +WantedBy=multi-user.target > diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes- > support/container-loader/files/container-loader.sh.tmpl > new file mode 100755 > index 00000000..31d27865 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl > @@ -0,0 +1,13 @@ > +#!/bin/sh > +# > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +set -eu > + > +while read -r image ref; do > + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then > + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" > + fi > +done < /usr/share/${PN}/image.list > diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes- > support/container-loader/podman-loader.inc > new file mode 100644 > index 00000000..d2c9a12d > --- /dev/null > +++ b/meta/recipes-support/container-loader/podman-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "podman" > + > +CONTAINER_ENGINE_PACKAGES ?= "podman"
On 12.07.24 11:19, Niedermayr, Benedikt (T CED OES-DE) wrote: > On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: >> From: Jan Kiszka <jan.kiszka@siemens.com> >> >> This allows to write dpkg-raw recipes which packages archived container >> images and load them into a local docker or podman registry on boot. The >> scenario behind this is to pre-fill local registries in a way that still >> permits live updates during runtime. >> >> The loader script only process images which are not yet available under >> the same name and tag in the local registry. Also after loading, the >> archived images stay on the local file system. This allows to perform >> reloading in case the local registry should be emptied (e.g. reset to >> factory state). To reduce the space those original images need, they are >> compressed, by default with xz. >> >> Separate include files are available to cater the main container >> engines, one for docker and one for podman. >> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >> --- >> .../container-loader/container-loader.inc | 76 +++++++++++++++++++ >> .../container-loader/docker-loader.inc | 10 +++ >> .../files/container-loader.service.tmpl | 11 +++ >> .../files/container-loader.sh.tmpl | 13 ++++ >> .../container-loader/podman-loader.inc | 10 +++ >> 5 files changed, 120 insertions(+) >> create mode 100644 meta/recipes-support/container-loader/container-loader.inc >> create mode 100644 meta/recipes-support/container-loader/docker-loader.inc >> create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl >> create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl >> create mode 100644 meta/recipes-support/container-loader/podman-loader.inc >> >> diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes- >> support/container-loader/container-loader.inc >> new file mode 100644 >> index 00000000..8e352214 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/container-loader.inc >> @@ -0,0 +1,76 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +FILESPATH:append := ":${FILE_DIRNAME}/files" >> + >> +inherit dpkg-raw >> + >> +SRC_URI += " \ >> + file://container-loader.service.tmpl \ >> + file://container-loader.sh.tmpl" >> + >> +TEMPLATE_FILES += " \ >> + container-loader.service.tmpl \ >> + container-loader.sh.tmpl" >> +TEMPLATE_VARS += "CONTAINER_ENGINE" >> + >> +CONTAINER_COMPRESSION ?= "xz" >> + >> +DEBIAN_DEPENDS += " \ >> + ${CONTAINER_ENGINE_PACKAGES} \ >> + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ >> + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +CONTAINER_COMPRESSOR = "${@ \ >> + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ >> + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +python do_install() { >> + import os >> + >> + workdir = d.getVar('WORKDIR') >> + D = d.getVar('D') >> + PN= d.getVar('PN') >> + >> + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") >> + >> + src_uri = d.getVar('SRC_URI').split() >> + for uri in src_uri: >> + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) >> + if scheme != "docker": >> + continue >> + >> + image_name = host + (path if path != "/" else "") >> + unpacked_image = workdir + "/" + image_name.replace('/', '.') >> + dest_dir = D + "/usr/share/" + PN + "/images" >> + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" >> + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" >> + >> + cmd = f"skopeo copy dir:{unpacked_image} " \ >> + f"docker-archive:{tar_image}:{image_name}{docker_ref}" >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ >> + image_name + docker_ref >> + bb.note(f"adding '{line}' to image.list") >> + image_list.write(line + "\n") >> + >> + image_list.close() >> + >> + bb.utils.copyfile(workdir + "/container-loader.sh", >> + D + "/usr/share/" + PN + "/container-loader.sh") >> +} >> +do_install[cleandirs] += "${D}/usr/share/${PN}/images" >> + >> +do_prepare_build:append() { >> + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${PN}.service >> +} >> diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes- >> support/container-loader/docker-loader.inc >> new file mode 100644 >> index 00000000..b864c854 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/docker-loader.inc >> @@ -0,0 +1,10 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +require container-loader.inc >> + >> +CONTAINER_ENGINE = "docker" >> + >> +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" >> diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> b/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> new file mode 100644 >> index 00000000..afde55d3 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> @@ -0,0 +1,11 @@ >> +[Unit] >> +Description=Load archived container images on boot >> +After=${CONTAINER_ENGINE}.service >> + >> +[Service] >> +Type=oneshot >> +ExecStart=/usr/share/${PN}/container-loader.sh >> +RemainAfterExit=true >> + >> +[Install] >> +WantedBy=multi-user.target >> diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes- >> support/container-loader/files/container-loader.sh.tmpl >> new file mode 100755 >> index 00000000..31d27865 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl >> @@ -0,0 +1,13 @@ >> +#!/bin/sh >> +# >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +set -eu >> + >> +while read -r image ref; do >> + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then >> + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" >> + fi >> +done < /usr/share/${PN}/image.list > > Should the images be deleted after import? I see no reason to preserve them once they have been > added to the local docker engine. You can't delete the images if they are part of a r/o rootfs. Furthermore, you may want to keep them for the purpose of factory resets. > An option to influence that would be good (e.g. CONTAINER_IMAGES_CLEAN). > Something like this is what I was thinking about already. Jan > >> diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes- >> support/container-loader/podman-loader.inc >> new file mode 100644 >> index 00000000..d2c9a12d >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/podman-loader.inc >> @@ -0,0 +1,10 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +require container-loader.inc >> + >> +CONTAINER_ENGINE = "podman" >> + >> +CONTAINER_ENGINE_PACKAGES ?= "podman" > > Benedikt
On 12.07.24 14:11, Niedermayr, Benedikt (T CED OES-DE) wrote: > On Tue, 2024-07-09 at 19:31 +0200, Jan Kiszka wrote: >> From: Jan Kiszka <jan.kiszka@siemens.com> >> >> This allows to write dpkg-raw recipes which packages archived container >> images and load them into a local docker or podman registry on boot. The >> scenario behind this is to pre-fill local registries in a way that still >> permits live updates during runtime. >> >> The loader script only process images which are not yet available under >> the same name and tag in the local registry. Also after loading, the >> archived images stay on the local file system. This allows to perform >> reloading in case the local registry should be emptied (e.g. reset to >> factory state). To reduce the space those original images need, they are >> compressed, by default with xz. >> >> Separate include files are available to cater the main container >> engines, one for docker and one for podman. >> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >> --- >> .../container-loader/container-loader.inc | 76 +++++++++++++++++++ >> .../container-loader/docker-loader.inc | 10 +++ >> .../files/container-loader.service.tmpl | 11 +++ >> .../files/container-loader.sh.tmpl | 13 ++++ >> .../container-loader/podman-loader.inc | 10 +++ >> 5 files changed, 120 insertions(+) >> create mode 100644 meta/recipes-support/container-loader/container-loader.inc >> create mode 100644 meta/recipes-support/container-loader/docker-loader.inc >> create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl >> create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl >> create mode 100644 meta/recipes-support/container-loader/podman-loader.inc >> >> diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes- >> support/container-loader/container-loader.inc >> new file mode 100644 >> index 00000000..8e352214 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/container-loader.inc >> @@ -0,0 +1,76 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +FILESPATH:append := ":${FILE_DIRNAME}/files" >> + >> +inherit dpkg-raw >> + >> +SRC_URI += " \ >> + file://container-loader.service.tmpl \ >> + file://container-loader.sh.tmpl" >> + >> +TEMPLATE_FILES += " \ >> + container-loader.service.tmpl \ >> + container-loader.sh.tmpl" >> +TEMPLATE_VARS += "CONTAINER_ENGINE" >> + >> +CONTAINER_COMPRESSION ?= "xz" >> + >> +DEBIAN_DEPENDS += " \ >> + ${CONTAINER_ENGINE_PACKAGES} \ >> + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ >> + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +CONTAINER_COMPRESSOR = "${@ \ >> + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ >> + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ >> + ''}" >> + >> +python do_install() { >> + import os >> + >> + workdir = d.getVar('WORKDIR') >> + D = d.getVar('D') >> + PN= d.getVar('PN') >> + >> + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") >> + >> + src_uri = d.getVar('SRC_URI').split() >> + for uri in src_uri: >> + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) >> + if scheme != "docker": >> + continue >> + >> + image_name = host + (path if path != "/" else "") >> + unpacked_image = workdir + "/" + image_name.replace('/', '.') >> + dest_dir = D + "/usr/share/" + PN + "/images" >> + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" >> + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" >> + >> + cmd = f"skopeo copy dir:{unpacked_image} " \ >> + f"docker-archive:{tar_image}:{image_name}{docker_ref}" > > regarding performance, > using "xz -T0" or pigz instead of gzip would increase performance and reduce build time a lot. > (looks like you wanted to comment below) > Benedikt > >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" We can tune CONTAINER_COMPRESSOR and its dependencies. I guess I will also follow Felix suggestion to open-code decompression so that we avoid shortcomings of podman's built-in decompression. Jan >> + bb.note(f"running: {cmd}") >> + bb.process.run(cmd) >> + >> + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ >> + image_name + docker_ref >> + bb.note(f"adding '{line}' to image.list") >> + image_list.write(line + "\n") >> + >> + image_list.close() >> + >> + bb.utils.copyfile(workdir + "/container-loader.sh", >> + D + "/usr/share/" + PN + "/container-loader.sh") >> +} >> +do_install[cleandirs] += "${D}/usr/share/${PN}/images" >> + >> +do_prepare_build:append() { >> + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${PN}.service >> +} >> diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes- >> support/container-loader/docker-loader.inc >> new file mode 100644 >> index 00000000..b864c854 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/docker-loader.inc >> @@ -0,0 +1,10 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +require container-loader.inc >> + >> +CONTAINER_ENGINE = "docker" >> + >> +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" >> diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> b/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> new file mode 100644 >> index 00000000..afde55d3 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl >> @@ -0,0 +1,11 @@ >> +[Unit] >> +Description=Load archived container images on boot >> +After=${CONTAINER_ENGINE}.service >> + >> +[Service] >> +Type=oneshot >> +ExecStart=/usr/share/${PN}/container-loader.sh >> +RemainAfterExit=true >> + >> +[Install] >> +WantedBy=multi-user.target >> diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes- >> support/container-loader/files/container-loader.sh.tmpl >> new file mode 100755 >> index 00000000..31d27865 >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl >> @@ -0,0 +1,13 @@ >> +#!/bin/sh >> +# >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +set -eu >> + >> +while read -r image ref; do >> + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then >> + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" >> + fi >> +done < /usr/share/${PN}/image.list >> diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes- >> support/container-loader/podman-loader.inc >> new file mode 100644 >> index 00000000..d2c9a12d >> --- /dev/null >> +++ b/meta/recipes-support/container-loader/podman-loader.inc >> @@ -0,0 +1,10 @@ >> +# This software is a part of ISAR. >> +# Copyright (c) Siemens AG, 2024 >> +# >> +# SPDX-License-Identifier: MIT >> + >> +require container-loader.inc >> + >> +CONTAINER_ENGINE = "podman" >> + >> +CONTAINER_ENGINE_PACKAGES ?= "podman" >
diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes-support/container-loader/container-loader.inc new file mode 100644 index 00000000..8e352214 --- /dev/null +++ b/meta/recipes-support/container-loader/container-loader.inc @@ -0,0 +1,76 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +FILESPATH:append := ":${FILE_DIRNAME}/files" + +inherit dpkg-raw + +SRC_URI += " \ + file://container-loader.service.tmpl \ + file://container-loader.sh.tmpl" + +TEMPLATE_FILES += " \ + container-loader.service.tmpl \ + container-loader.sh.tmpl" +TEMPLATE_VARS += "CONTAINER_ENGINE" + +CONTAINER_COMPRESSION ?= "xz" + +DEBIAN_DEPENDS += " \ + ${CONTAINER_ENGINE_PACKAGES} \ + ${@', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ + ', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ + ''}" + +CONTAINER_COMPRESSOR = "${@ \ + 'xz' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ + 'gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ + ''}" + +python do_install() { + import os + + workdir = d.getVar('WORKDIR') + D = d.getVar('D') + PN= d.getVar('PN') + + image_list = open(D + "/usr/share/" + PN +"/image.list", "w") + + src_uri = d.getVar('SRC_URI').split() + for uri in src_uri: + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) + if scheme != "docker": + continue + + image_name = host + (path if path != "/" else "") + unpacked_image = workdir + "/" + image_name.replace('/', '.') + dest_dir = D + "/usr/share/" + PN + "/images" + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" + + cmd = f"skopeo copy dir:{unpacked_image} " \ + f"docker-archive:{tar_image}:{image_name}{docker_ref}" + bb.note(f"running: {cmd}") + bb.process.run(cmd) + + cmd = f"{d.getVar('CONTAINER_COMPRESSOR')} {tar_image}" + bb.note(f"running: {cmd}") + bb.process.run(cmd) + + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ + image_name + docker_ref + bb.note(f"adding '{line}' to image.list") + image_list.write(line + "\n") + + image_list.close() + + bb.utils.copyfile(workdir + "/container-loader.sh", + D + "/usr/share/" + PN + "/container-loader.sh") +} +do_install[cleandirs] += "${D}/usr/share/${PN}/images" + +do_prepare_build:append() { + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${PN}.service +} diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes-support/container-loader/docker-loader.inc new file mode 100644 index 00000000..b864c854 --- /dev/null +++ b/meta/recipes-support/container-loader/docker-loader.inc @@ -0,0 +1,10 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +require container-loader.inc + +CONTAINER_ENGINE = "docker" + +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl b/meta/recipes-support/container-loader/files/container-loader.service.tmpl new file mode 100644 index 00000000..afde55d3 --- /dev/null +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl @@ -0,0 +1,11 @@ +[Unit] +Description=Load archived container images on boot +After=${CONTAINER_ENGINE}.service + +[Service] +Type=oneshot +ExecStart=/usr/share/${PN}/container-loader.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl new file mode 100755 index 00000000..31d27865 --- /dev/null +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +set -eu + +while read -r image ref; do + if [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then + ${CONTAINER_ENGINE} load -i /usr/share/${PN}/images/"$image" + fi +done < /usr/share/${PN}/image.list diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes-support/container-loader/podman-loader.inc new file mode 100644 index 00000000..d2c9a12d --- /dev/null +++ b/meta/recipes-support/container-loader/podman-loader.inc @@ -0,0 +1,10 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +require container-loader.inc + +CONTAINER_ENGINE = "podman" + +CONTAINER_ENGINE_PACKAGES ?= "podman"