| Message ID | 20210112103338.14712-2-silvano.cirujano-cuesta@siemens.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | support generation of sdk container images | expand |
Am Tue, 12 Jan 2021 11:33:37 +0100 schrieb "[ext] Silvano Cirujano Cuesta" <silvano.cirujano-cuesta@siemens.com>: > Extend task "populate_sdk" to support the creation of a container > image containing the SDK. > > Signed-off-by: Silvano Cirujano Cuesta > <silvano.cirujano-cuesta@siemens.com> --- > meta/classes/image-sdk-extension.bbclass | 99 > ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 > deletions(-) > > diff --git a/meta/classes/image-sdk-extension.bbclass > b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 > 100644 --- a/meta/classes/image-sdk-extension.bbclass > +++ b/meta/classes/image-sdk-extension.bbclass > @@ -6,10 +6,77 @@ > # This class extends the image.bbclass to supply the creation of a > sdk > SDK_INCLUDE_ISAR_APT ?= "0" > +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" I do not understand why there are two variables, maybe one is enough. And i think a ?= assignment would be a better choice here. > +sdk_tar() { I think this should be tar_xz or tar.xz > + # Copy mount_chroot.sh for convenience > + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} > + > + # Create SDK archive > + cd -P ${SDKCHROOT_DIR}/.. > + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ > + -c rootfs | xz -T0 > > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} > + > +sdk_container_images() { > + local cmd="/bin/dash" > + local empty_tag="empty" > + local full_tag="latest" > + local oci_img_dir="${WORKDIR}/oci-image" > + local sdk_container_formats="$1" > + > + # prepare OCI container image skeleton > + sudo umoci init --layout "${oci_img_dir}" > + sudo umoci new --image "${oci_img_dir}:${empty_tag}" > + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ > + --config.cmd="${cmd}" > + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ > + "${oci_img_dir}_unpacked" > + > + # add SDK root filesystem as the flesh of the skeleton > + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" > + > + # pack container image > + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ > + "${oci_img_dir}_unpacked" > + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" > + sudo rm -rf "${oci_img_dir}_unpacked" > + > + # no root needed anymore > + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" > + > + # convert the OCI container image to the desired format > + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" > + image_name="isar-${sdk_id}" > + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" > + for sdk_format in ${sdk_container_formats} ; do > + case "${sdk_format}" in > + "docker-archive" | "oci-archive") > + if [ "${sdk_format}" = "oci-archive" ] ; then > + target="${sdk_format}:${image_archive}:latest" > + else > + > target="${sdk_format}:${image_archive}:${image_name}:latest" > + fi > + skopeo --insecure-policy copy \ > + "oci:${oci_img_dir}:${full_tag}" "${target}" > + xz -T0 "${image_archive}" > + ;; > + "oci") > + tar --create --xz --directory "${oci_img_dir}" \ > + --file "${image_archive}.xz" . > + ;; do we not already have tar_xz code we can maybe reuse? > + "docker-daemon" | "containers-storage") > + skopeo --insecure-policy copy \ > + "oci:${oci_img_dir}:${full_tag}" \ > + "${sdk_format}:${image_name}:latest" > + ;; i really would not trust "skopeo" to generate valid docker images, the container world is full of incompatible stuff and compat fake news > + esac > + done This is using "umoci" and "skopeo", new runtime deps ... which might only be available/working in pretty bleeding edge debian. Henning > +} > > do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}" > do_populate_sdk[depends] = "sdkchroot:do_build" > -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT" > +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT > SDK_GENERATE_FORMATS" do_populate_sdk() { > if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then > # Copy isar-apt with deployed Isar packages > @@ -48,12 +115,30 @@ do_populate_sdk() { > done > done > > - # Copy mount_chroot.sh for convenience > - sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} > + # separate SDK formats: TAR and container formats > + container_formats="" > + for sdk_format in ${SDK_GENERATE_FORMATS} ; do > + case ${sdk_format} in > + tar) > + sdk_tar > + ;; > + "docker-archive" | "oci" | "oci-archive") > + container_formats="${container_formats} > ${sdk_format}" > + ;; > + "docker-daemon" | "containers-storage") > + if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; > then > + die "Adding the SDK container image to a > container runtime (${sdk_format}) not supported if running from a > container (e.g. 'kas-container')" > + fi > + ;; > + *) > + die "unsupported SDK format specified: ${sdk_format}" > + ;; > + esac > + done > > - # Create SDK archive > - cd -P ${SDKCHROOT_DIR}/.. > - sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ > - -c rootfs | xz -T0 > > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz > + # generate the SDK in all the desired container formats > + if [ -n "${container_formats}" ] ; then > + sdk_container_images "${container_formats}" > + fi > } > addtask populate_sdk after do_rootfs
On 12/01/2021 12:36, Henning Schild wrote: > Am Tue, 12 Jan 2021 11:33:37 +0100 > schrieb "[ext] Silvano Cirujano Cuesta" > <silvano.cirujano-cuesta@siemens.com>: > >> Extend task "populate_sdk" to support the creation of a container >> image containing the SDK. >> >> Signed-off-by: Silvano Cirujano Cuesta >> <silvano.cirujano-cuesta@siemens.com> --- >> meta/classes/image-sdk-extension.bbclass | 99 >> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >> deletions(-) >> >> diff --git a/meta/classes/image-sdk-extension.bbclass >> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >> 100644 --- a/meta/classes/image-sdk-extension.bbclass >> +++ b/meta/classes/image-sdk-extension.bbclass >> @@ -6,10 +6,77 @@ >> # This class extends the image.bbclass to supply the creation of a >> sdk >> SDK_INCLUDE_ISAR_APT ?= "0" >> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" > I do not understand why there are two variables, maybe one is enough. > And i think a ?= assignment would be a better choice here. Probably lack of ISAR expersite... I don't know why using SDK_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" fails. I've also tried SDK_FORMATS ?= "tar" and failed. > >> +sdk_tar() { > I think this should be tar_xz or tar.xz Why not... Will change it. > >> + # Copy mount_chroot.sh for convenience >> + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >> + >> + # Create SDK archive >> + cd -P ${SDKCHROOT_DIR}/.. >> + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >> + -c rootfs | xz -T0 > >> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} >> + >> +sdk_container_images() { >> + local cmd="/bin/dash" >> + local empty_tag="empty" >> + local full_tag="latest" >> + local oci_img_dir="${WORKDIR}/oci-image" >> + local sdk_container_formats="$1" >> + >> + # prepare OCI container image skeleton >> + sudo umoci init --layout "${oci_img_dir}" >> + sudo umoci new --image "${oci_img_dir}:${empty_tag}" >> + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ >> + --config.cmd="${cmd}" >> + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ >> + "${oci_img_dir}_unpacked" >> + >> + # add SDK root filesystem as the flesh of the skeleton >> + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" >> + >> + # pack container image >> + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ >> + "${oci_img_dir}_unpacked" >> + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" >> + sudo rm -rf "${oci_img_dir}_unpacked" >> + >> + # no root needed anymore >> + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" >> + >> + # convert the OCI container image to the desired format >> + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" >> + image_name="isar-${sdk_id}" >> + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" >> + for sdk_format in ${sdk_container_formats} ; do >> + case "${sdk_format}" in >> + "docker-archive" | "oci-archive") >> + if [ "${sdk_format}" = "oci-archive" ] ; then >> + target="${sdk_format}:${image_archive}:latest" >> + else >> + >> target="${sdk_format}:${image_archive}:${image_name}:latest" >> + fi >> + skopeo --insecure-policy copy \ >> + "oci:${oci_img_dir}:${full_tag}" "${target}" >> + xz -T0 "${image_archive}" >> + ;; >> + "oci") >> + tar --create --xz --directory "${oci_img_dir}" \ >> + --file "${image_archive}.xz" . >> + ;; > do we not already have tar_xz code we can maybe reuse? Do you mean the function "sdk_tar"? Not enough overlapping with the code in that function to make reuse meaningful. > >> + "docker-daemon" | "containers-storage") >> + skopeo --insecure-policy copy \ >> + "oci:${oci_img_dir}:${full_tag}" \ >> + "${sdk_format}:${image_name}:latest" >> + ;; > i really would not trust "skopeo" to generate valid docker images, the > container world is full of incompatible stuff and compat fake news Alternative? Docker-in-Docker? I would do that. Self-built? Not worth the effort, harder to keep compatibility. > >> + esac >> + done > This is using "umoci" and "skopeo", new runtime deps ... which might > only be available/working in pretty bleeding edge debian. Sorry, I forgot to mention it in the cover-letter. Bleeding edge? Nope: - Umoci is in Debian 10 (Buster, current stable). - Skopeo is in Debian 11 (Bullseye, upcoming stable), it can be easily backported. I've done it on a KAS container, and I'd contribute that addition to that project too. Silvano > > Henning > >> +} >> >> do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}" >> do_populate_sdk[depends] = "sdkchroot:do_build" >> -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT" >> +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT >> SDK_GENERATE_FORMATS" do_populate_sdk() { >> if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then >> # Copy isar-apt with deployed Isar packages >> @@ -48,12 +115,30 @@ do_populate_sdk() { >> done >> done >> >> - # Copy mount_chroot.sh for convenience >> - sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >> + # separate SDK formats: TAR and container formats >> + container_formats="" >> + for sdk_format in ${SDK_GENERATE_FORMATS} ; do >> + case ${sdk_format} in >> + tar) >> + sdk_tar >> + ;; >> + "docker-archive" | "oci" | "oci-archive") >> + container_formats="${container_formats} >> ${sdk_format}" >> + ;; >> + "docker-daemon" | "containers-storage") >> + if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; >> then >> + die "Adding the SDK container image to a >> container runtime (${sdk_format}) not supported if running from a >> container (e.g. 'kas-container')" >> + fi >> + ;; >> + *) >> + die "unsupported SDK format specified: ${sdk_format}" >> + ;; >> + esac >> + done >> >> - # Create SDK archive >> - cd -P ${SDKCHROOT_DIR}/.. >> - sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >> - -c rootfs | xz -T0 > >> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz >> + # generate the SDK in all the desired container formats >> + if [ -n "${container_formats}" ] ; then >> + sdk_container_images "${container_formats}" >> + fi >> } >> addtask populate_sdk after do_rootfs
On 12.01.21 12:50, [ext] Silvano Cirujano Cuesta wrote: > > On 12/01/2021 12:36, Henning Schild wrote: >> Am Tue, 12 Jan 2021 11:33:37 +0100 >> schrieb "[ext] Silvano Cirujano Cuesta" >> <silvano.cirujano-cuesta@siemens.com>: >> >>> Extend task "populate_sdk" to support the creation of a container >>> image containing the SDK. >>> >>> Signed-off-by: Silvano Cirujano Cuesta >>> <silvano.cirujano-cuesta@siemens.com> --- >>> meta/classes/image-sdk-extension.bbclass | 99 >>> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >>> deletions(-) >>> >>> diff --git a/meta/classes/image-sdk-extension.bbclass >>> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >>> 100644 --- a/meta/classes/image-sdk-extension.bbclass >>> +++ b/meta/classes/image-sdk-extension.bbclass >>> @@ -6,10 +6,77 @@ >>> # This class extends the image.bbclass to supply the creation of a >>> sdk >>> SDK_INCLUDE_ISAR_APT ?= "0" >>> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >> I do not understand why there are two variables, maybe one is enough. >> And i think a ?= assignment would be a better choice here. > > Probably lack of ISAR expersite... I don't know why using > > SDK_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" > > fails. I've also tried > > SDK_FORMATS ?= "tar" > > and failed. How did it fail - because this is how it should look like. Jan
On 12/01/2021 15:50, Jan Kiszka wrote: > On 12.01.21 12:50, [ext] Silvano Cirujano Cuesta wrote: >> On 12/01/2021 12:36, Henning Schild wrote: >>> Am Tue, 12 Jan 2021 11:33:37 +0100 >>> schrieb "[ext] Silvano Cirujano Cuesta" >>> <silvano.cirujano-cuesta@siemens.com>: >>> >>>> Extend task "populate_sdk" to support the creation of a container >>>> image containing the SDK. >>>> >>>> Signed-off-by: Silvano Cirujano Cuesta >>>> <silvano.cirujano-cuesta@siemens.com> --- >>>> meta/classes/image-sdk-extension.bbclass | 99 >>>> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >>>> deletions(-) >>>> >>>> diff --git a/meta/classes/image-sdk-extension.bbclass >>>> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >>>> 100644 --- a/meta/classes/image-sdk-extension.bbclass >>>> +++ b/meta/classes/image-sdk-extension.bbclass >>>> @@ -6,10 +6,77 @@ >>>> # This class extends the image.bbclass to supply the creation of a >>>> sdk >>>> SDK_INCLUDE_ISAR_APT ?= "0" >>>> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >>> I do not understand why there are two variables, maybe one is enough. >>> And i think a ?= assignment would be a better choice here. >> Probably lack of ISAR expersite... I don't know why using >> >> SDK_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >> >> fails. I've also tried >> >> SDK_FORMATS ?= "tar" >> >> and failed. > How did it fail - because this is how it should look like. Using `SDK_FORMATS ?= "tar"` and not providing any value for SDK_FORMATS (so using the default), `bitbake -e` doesn't show SDK_FORMATS at all and therefore I thought it was failing. I've retried with `SDK_FORMATS ?= "tar"` and it works. I'll fix it. Silvano > > Jan >
Am Tue, 12 Jan 2021 11:33:37 +0100 schrieb "[ext] Silvano Cirujano Cuesta" <silvano.cirujano-cuesta@siemens.com>: > Extend task "populate_sdk" to support the creation of a container > image containing the SDK. > > Signed-off-by: Silvano Cirujano Cuesta > <silvano.cirujano-cuesta@siemens.com> --- > meta/classes/image-sdk-extension.bbclass | 99 > ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 > deletions(-) > > diff --git a/meta/classes/image-sdk-extension.bbclass > b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 > 100644 --- a/meta/classes/image-sdk-extension.bbclass > +++ b/meta/classes/image-sdk-extension.bbclass > @@ -6,10 +6,77 @@ > # This class extends the image.bbclass to supply the creation of a > sdk > SDK_INCLUDE_ISAR_APT ?= "0" > +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" > + > +sdk_tar() { > + # Copy mount_chroot.sh for convenience > + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} > + > + # Create SDK archive > + cd -P ${SDKCHROOT_DIR}/.. > + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ > + -c rootfs | xz -T0 > > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} > + > +sdk_container_images() { > + local cmd="/bin/dash" > + local empty_tag="empty" > + local full_tag="latest" > + local oci_img_dir="${WORKDIR}/oci-image" > + local sdk_container_formats="$1" > + > + # prepare OCI container image skeleton > + sudo umoci init --layout "${oci_img_dir}" > + sudo umoci new --image "${oci_img_dir}:${empty_tag}" > + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ > + --config.cmd="${cmd}" > + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ > + "${oci_img_dir}_unpacked" > + > + # add SDK root filesystem as the flesh of the skeleton > + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" > + > + # pack container image > + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ > + "${oci_img_dir}_unpacked" > + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" > + sudo rm -rf "${oci_img_dir}_unpacked" > + > + # no root needed anymore > + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" > + > + # convert the OCI container image to the desired format > + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" > + image_name="isar-${sdk_id}" > + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" > + for sdk_format in ${sdk_container_formats} ; do > + case "${sdk_format}" in > + "docker-archive" | "oci-archive") > + if [ "${sdk_format}" = "oci-archive" ] ; then > + target="${sdk_format}:${image_archive}:latest" > + else > + > target="${sdk_format}:${image_archive}:${image_name}:latest" > + fi > + skopeo --insecure-policy copy \ > + "oci:${oci_img_dir}:${full_tag}" "${target}" > + xz -T0 "${image_archive}" > + ;; > + "oci") > + tar --create --xz --directory "${oci_img_dir}" \ > + --file "${image_archive}.xz" . > + ;; > + "docker-daemon" | "containers-storage") > + skopeo --insecure-policy copy \ > + "oci:${oci_img_dir}:${full_tag}" \ > + "${sdk_format}:${image_name}:latest" > + ;; Jan proposed to make the SDK class wider than x86(64). How is this going to affect docker? (i intentionally use "docker" as synonym for all sorts of ...) The proposed changes should be tested for riscv and arm64 as well. Henning > + esac > + done > +} > > do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}" > do_populate_sdk[depends] = "sdkchroot:do_build" > -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT" > +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT > SDK_GENERATE_FORMATS" do_populate_sdk() { > if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then > # Copy isar-apt with deployed Isar packages > @@ -48,12 +115,30 @@ do_populate_sdk() { > done > done > > - # Copy mount_chroot.sh for convenience > - sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} > + # separate SDK formats: TAR and container formats > + container_formats="" > + for sdk_format in ${SDK_GENERATE_FORMATS} ; do > + case ${sdk_format} in > + tar) > + sdk_tar > + ;; > + "docker-archive" | "oci" | "oci-archive") > + container_formats="${container_formats} > ${sdk_format}" > + ;; > + "docker-daemon" | "containers-storage") > + if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; > then > + die "Adding the SDK container image to a > container runtime (${sdk_format}) not supported if running from a > container (e.g. 'kas-container')" > + fi > + ;; > + *) > + die "unsupported SDK format specified: ${sdk_format}" > + ;; > + esac > + done > > - # Create SDK archive > - cd -P ${SDKCHROOT_DIR}/.. > - sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ > - -c rootfs | xz -T0 > > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz > + # generate the SDK in all the desired container formats > + if [ -n "${container_formats}" ] ; then > + sdk_container_images "${container_formats}" > + fi > } > addtask populate_sdk after do_rootfs
On 12/01/2021 18:36, Henning Schild wrote: > Am Tue, 12 Jan 2021 11:33:37 +0100 > schrieb "[ext] Silvano Cirujano Cuesta" > <silvano.cirujano-cuesta@siemens.com>: > >> Extend task "populate_sdk" to support the creation of a container >> image containing the SDK. >> >> Signed-off-by: Silvano Cirujano Cuesta >> <silvano.cirujano-cuesta@siemens.com> --- >> meta/classes/image-sdk-extension.bbclass | 99 >> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >> deletions(-) >> >> diff --git a/meta/classes/image-sdk-extension.bbclass >> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >> 100644 --- a/meta/classes/image-sdk-extension.bbclass >> +++ b/meta/classes/image-sdk-extension.bbclass >> @@ -6,10 +6,77 @@ >> # This class extends the image.bbclass to supply the creation of a >> sdk >> SDK_INCLUDE_ISAR_APT ?= "0" >> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >> + >> +sdk_tar() { >> + # Copy mount_chroot.sh for convenience >> + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >> + >> + # Create SDK archive >> + cd -P ${SDKCHROOT_DIR}/.. >> + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >> + -c rootfs | xz -T0 > >> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} >> + >> +sdk_container_images() { >> + local cmd="/bin/dash" >> + local empty_tag="empty" >> + local full_tag="latest" >> + local oci_img_dir="${WORKDIR}/oci-image" >> + local sdk_container_formats="$1" >> + >> + # prepare OCI container image skeleton >> + sudo umoci init --layout "${oci_img_dir}" >> + sudo umoci new --image "${oci_img_dir}:${empty_tag}" >> + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ >> + --config.cmd="${cmd}" >> + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ >> + "${oci_img_dir}_unpacked" >> + >> + # add SDK root filesystem as the flesh of the skeleton >> + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" >> + >> + # pack container image >> + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ >> + "${oci_img_dir}_unpacked" >> + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" >> + sudo rm -rf "${oci_img_dir}_unpacked" >> + >> + # no root needed anymore >> + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" >> + >> + # convert the OCI container image to the desired format >> + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" >> + image_name="isar-${sdk_id}" >> + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" >> + for sdk_format in ${sdk_container_formats} ; do >> + case "${sdk_format}" in >> + "docker-archive" | "oci-archive") >> + if [ "${sdk_format}" = "oci-archive" ] ; then >> + target="${sdk_format}:${image_archive}:latest" >> + else >> + >> target="${sdk_format}:${image_archive}:${image_name}:latest" >> + fi >> + skopeo --insecure-policy copy \ >> + "oci:${oci_img_dir}:${full_tag}" "${target}" >> + xz -T0 "${image_archive}" >> + ;; >> + "oci") >> + tar --create --xz --directory "${oci_img_dir}" \ >> + --file "${image_archive}.xz" . >> + ;; >> + "docker-daemon" | "containers-storage") >> + skopeo --insecure-policy copy \ >> + "oci:${oci_img_dir}:${full_tag}" \ >> + "${sdk_format}:${image_name}:latest" >> + ;; > Jan proposed to make the SDK class wider than x86(64). How is this > going to affect docker? (i intentionally use "docker" as synonym for all > sorts of ...) I intentionally use container instead of Docker where it's not docker-only. > > The proposed changes should be tested for riscv and arm64 as well. Do you mean hosts or targets? Assuming you're talking about RISC-V and ARM64 targets... This functionality is merely packaging the cross-buildchain that up-to-now could be generated with "populate_sdk", whatever was supported should be supported by this patch. Of course, the changes should be tested for different combinations. Feel free to test it for those architectures, as long as we are in the discussion round I'm fine with the "whatever was supported should be supported by this patch" that I've written above. I'd really test if only before contributing the final patches for integration. Silvano > > Henning > >> + esac >> + done >> +} >> >> do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}" >> do_populate_sdk[depends] = "sdkchroot:do_build" >> -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT" >> +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT >> SDK_GENERATE_FORMATS" do_populate_sdk() { >> if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then >> # Copy isar-apt with deployed Isar packages >> @@ -48,12 +115,30 @@ do_populate_sdk() { >> done >> done >> >> - # Copy mount_chroot.sh for convenience >> - sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >> + # separate SDK formats: TAR and container formats >> + container_formats="" >> + for sdk_format in ${SDK_GENERATE_FORMATS} ; do >> + case ${sdk_format} in >> + tar) >> + sdk_tar >> + ;; >> + "docker-archive" | "oci" | "oci-archive") >> + container_formats="${container_formats} >> ${sdk_format}" >> + ;; >> + "docker-daemon" | "containers-storage") >> + if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; >> then >> + die "Adding the SDK container image to a >> container runtime (${sdk_format}) not supported if running from a >> container (e.g. 'kas-container')" >> + fi >> + ;; >> + *) >> + die "unsupported SDK format specified: ${sdk_format}" >> + ;; >> + esac >> + done >> >> - # Create SDK archive >> - cd -P ${SDKCHROOT_DIR}/.. >> - sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >> - -c rootfs | xz -T0 > >> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz >> + # generate the SDK in all the desired container formats >> + if [ -n "${container_formats}" ] ; then >> + sdk_container_images "${container_formats}" >> + fi >> } >> addtask populate_sdk after do_rootfs
On 12.01.21 18:54, [ext] Silvano Cirujano Cuesta wrote: > > On 12/01/2021 18:36, Henning Schild wrote: >> Am Tue, 12 Jan 2021 11:33:37 +0100 >> schrieb "[ext] Silvano Cirujano Cuesta" >> <silvano.cirujano-cuesta@siemens.com>: >> >>> Extend task "populate_sdk" to support the creation of a container >>> image containing the SDK. >>> >>> Signed-off-by: Silvano Cirujano Cuesta >>> <silvano.cirujano-cuesta@siemens.com> --- >>> meta/classes/image-sdk-extension.bbclass | 99 >>> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >>> deletions(-) >>> >>> diff --git a/meta/classes/image-sdk-extension.bbclass >>> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >>> 100644 --- a/meta/classes/image-sdk-extension.bbclass >>> +++ b/meta/classes/image-sdk-extension.bbclass >>> @@ -6,10 +6,77 @@ >>> # This class extends the image.bbclass to supply the creation of a >>> sdk >>> SDK_INCLUDE_ISAR_APT ?= "0" >>> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >>> + >>> +sdk_tar() { >>> + # Copy mount_chroot.sh for convenience >>> + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >>> + >>> + # Create SDK archive >>> + cd -P ${SDKCHROOT_DIR}/.. >>> + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >>> + -c rootfs | xz -T0 > >>> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} >>> + >>> +sdk_container_images() { >>> + local cmd="/bin/dash" >>> + local empty_tag="empty" >>> + local full_tag="latest" >>> + local oci_img_dir="${WORKDIR}/oci-image" >>> + local sdk_container_formats="$1" >>> + >>> + # prepare OCI container image skeleton >>> + sudo umoci init --layout "${oci_img_dir}" >>> + sudo umoci new --image "${oci_img_dir}:${empty_tag}" >>> + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ >>> + --config.cmd="${cmd}" >>> + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ >>> + "${oci_img_dir}_unpacked" >>> + >>> + # add SDK root filesystem as the flesh of the skeleton >>> + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" >>> + >>> + # pack container image >>> + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ >>> + "${oci_img_dir}_unpacked" >>> + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" >>> + sudo rm -rf "${oci_img_dir}_unpacked" >>> + >>> + # no root needed anymore >>> + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" >>> + >>> + # convert the OCI container image to the desired format >>> + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" >>> + image_name="isar-${sdk_id}" >>> + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" >>> + for sdk_format in ${sdk_container_formats} ; do >>> + case "${sdk_format}" in >>> + "docker-archive" | "oci-archive") >>> + if [ "${sdk_format}" = "oci-archive" ] ; then >>> + target="${sdk_format}:${image_archive}:latest" >>> + else >>> + >>> target="${sdk_format}:${image_archive}:${image_name}:latest" >>> + fi >>> + skopeo --insecure-policy copy \ >>> + "oci:${oci_img_dir}:${full_tag}" "${target}" >>> + xz -T0 "${image_archive}" >>> + ;; >>> + "oci") >>> + tar --create --xz --directory "${oci_img_dir}" \ >>> + --file "${image_archive}.xz" . >>> + ;; >>> + "docker-daemon" | "containers-storage") >>> + skopeo --insecure-policy copy \ >>> + "oci:${oci_img_dir}:${full_tag}" \ >>> + "${sdk_format}:${image_name}:latest" >>> + ;; >> Jan proposed to make the SDK class wider than x86(64). How is this >> going to affect docker? (i intentionally use "docker" as synonym for all >> sorts of ...) > I intentionally use container instead of Docker where it's not docker-only. >> >> The proposed changes should be tested for riscv and arm64 as well. > > Do you mean hosts or targets? Assuming you're talking about RISC-V and ARM64 targets... > > This functionality is merely packaging the cross-buildchain that up-to-now could be generated with "populate_sdk", whatever was supported should be supported by this patch. > > Of course, the changes should be tested for different combinations. Feel free to test it for those architectures, as long as we are in the discussion round I'm fine with the "whatever was supported should be supported by this patch" that I've written above. I'd really test if only before contributing the final patches for integration. > Agreed. I don't see any arch-specific blocker here. Both skopeo and umoci are available for our "new" host arch, arm64. RISC-V is still no official Debian target, thus may break for any reason on any day. Jan
On 12/01/2021 19:04, Jan Kiszka wrote: > On 12.01.21 18:54, [ext] Silvano Cirujano Cuesta wrote: >> On 12/01/2021 18:36, Henning Schild wrote: >>> Am Tue, 12 Jan 2021 11:33:37 +0100 >>> schrieb "[ext] Silvano Cirujano Cuesta" >>> <silvano.cirujano-cuesta@siemens.com>: >>> >>>> Extend task "populate_sdk" to support the creation of a container >>>> image containing the SDK. >>>> >>>> Signed-off-by: Silvano Cirujano Cuesta >>>> <silvano.cirujano-cuesta@siemens.com> --- >>>> meta/classes/image-sdk-extension.bbclass | 99 >>>> ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 >>>> deletions(-) >>>> >>>> diff --git a/meta/classes/image-sdk-extension.bbclass >>>> b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 >>>> 100644 --- a/meta/classes/image-sdk-extension.bbclass >>>> +++ b/meta/classes/image-sdk-extension.bbclass >>>> @@ -6,10 +6,77 @@ >>>> # This class extends the image.bbclass to supply the creation of a >>>> sdk >>>> SDK_INCLUDE_ISAR_APT ?= "0" >>>> +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" >>>> + >>>> +sdk_tar() { >>>> + # Copy mount_chroot.sh for convenience >>>> + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} >>>> + >>>> + # Create SDK archive >>>> + cd -P ${SDKCHROOT_DIR}/.. >>>> + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ >>>> + -c rootfs | xz -T0 > >>>> ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} >>>> + >>>> +sdk_container_images() { >>>> + local cmd="/bin/dash" >>>> + local empty_tag="empty" >>>> + local full_tag="latest" >>>> + local oci_img_dir="${WORKDIR}/oci-image" >>>> + local sdk_container_formats="$1" >>>> + >>>> + # prepare OCI container image skeleton >>>> + sudo umoci init --layout "${oci_img_dir}" >>>> + sudo umoci new --image "${oci_img_dir}:${empty_tag}" >>>> + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ >>>> + --config.cmd="${cmd}" >>>> + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ >>>> + "${oci_img_dir}_unpacked" >>>> + >>>> + # add SDK root filesystem as the flesh of the skeleton >>>> + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" >>>> + >>>> + # pack container image >>>> + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ >>>> + "${oci_img_dir}_unpacked" >>>> + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" >>>> + sudo rm -rf "${oci_img_dir}_unpacked" >>>> + >>>> + # no root needed anymore >>>> + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" >>>> + >>>> + # convert the OCI container image to the desired format >>>> + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" >>>> + image_name="isar-${sdk_id}" >>>> + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" >>>> + for sdk_format in ${sdk_container_formats} ; do >>>> + case "${sdk_format}" in >>>> + "docker-archive" | "oci-archive") >>>> + if [ "${sdk_format}" = "oci-archive" ] ; then >>>> + target="${sdk_format}:${image_archive}:latest" >>>> + else >>>> + >>>> target="${sdk_format}:${image_archive}:${image_name}:latest" >>>> + fi >>>> + skopeo --insecure-policy copy \ >>>> + "oci:${oci_img_dir}:${full_tag}" "${target}" >>>> + xz -T0 "${image_archive}" >>>> + ;; >>>> + "oci") >>>> + tar --create --xz --directory "${oci_img_dir}" \ >>>> + --file "${image_archive}.xz" . >>>> + ;; >>>> + "docker-daemon" | "containers-storage") >>>> + skopeo --insecure-policy copy \ >>>> + "oci:${oci_img_dir}:${full_tag}" \ >>>> + "${sdk_format}:${image_name}:latest" >>>> + ;; >>> Jan proposed to make the SDK class wider than x86(64). How is this >>> going to affect docker? (i intentionally use "docker" as synonym for all >>> sorts of ...) >> I intentionally use container instead of Docker where it's not docker-only. >>> The proposed changes should be tested for riscv and arm64 as well. >> Do you mean hosts or targets? Assuming you're talking about RISC-V and ARM64 targets... >> >> This functionality is merely packaging the cross-buildchain that up-to-now could be generated with "populate_sdk", whatever was supported should be supported by this patch. >> >> Of course, the changes should be tested for different combinations. Feel free to test it for those architectures, as long as we are in the discussion round I'm fine with the "whatever was supported should be supported by this patch" that I've written above. I'd really test if only before contributing the final patches for integration. >> > Agreed. I don't see any arch-specific blocker here. Both skopeo and > umoci are available for our "new" host arch, arm64. RISC-V is still no > official Debian target, thus may break for any reason on any day. I have to admit I didn't expect Henning to mean ARM64 and RISC-V for the host... Anyway, as Jan says, ARM64 is supported for umoci and skopeo. Not only that, even the unofficial RISC-V port is providing umoci and I'd expect skopeo to become also supported in the upcoming future. The architecture of the host shouldn't be a big deal as long as it's supported by Debian, since neither umoci nor skopeo are using any "container-specific" technology. They simply build tarballed archives with a fixed format, write JSON documents, calculate SHA256 checksums and build file trees with a specified structure. Everything (except for the fixed TAR format) can be built with a shell script using very common tools like sed, jq, sha256sum,... Silvano > > Jan >
diff --git a/meta/classes/image-sdk-extension.bbclass b/meta/classes/image-sdk-extension.bbclass index a8c708a..9317256 100644 --- a/meta/classes/image-sdk-extension.bbclass +++ b/meta/classes/image-sdk-extension.bbclass @@ -6,10 +6,77 @@ # This class extends the image.bbclass to supply the creation of a sdk SDK_INCLUDE_ISAR_APT ?= "0" +SDK_GENERATE_FORMATS = "${@d.getVar("SDK_FORMATS", "tar")}" + +sdk_tar() { + # Copy mount_chroot.sh for convenience + sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} + + # Create SDK archive + cd -P ${SDKCHROOT_DIR}/.. + sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ + -c rootfs | xz -T0 > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz +} + +sdk_container_images() { + local cmd="/bin/dash" + local empty_tag="empty" + local full_tag="latest" + local oci_img_dir="${WORKDIR}/oci-image" + local sdk_container_formats="$1" + + # prepare OCI container image skeleton + sudo umoci init --layout "${oci_img_dir}" + sudo umoci new --image "${oci_img_dir}:${empty_tag}" + sudo umoci config --image "${oci_img_dir}:${empty_tag}" \ + --config.cmd="${cmd}" + sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \ + "${oci_img_dir}_unpacked" + + # add SDK root filesystem as the flesh of the skeleton + sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/" + + # pack container image + sudo umoci repack --image "${oci_img_dir}:${full_tag}" \ + "${oci_img_dir}_unpacked" + sudo umoci remove --image "${oci_img_dir}:${empty_tag}" + sudo rm -rf "${oci_img_dir}_unpacked" + + # no root needed anymore + sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}" + + # convert the OCI container image to the desired format + sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}" + image_name="isar-${sdk_id}" + image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar" + for sdk_format in ${sdk_container_formats} ; do + case "${sdk_format}" in + "docker-archive" | "oci-archive") + if [ "${sdk_format}" = "oci-archive" ] ; then + target="${sdk_format}:${image_archive}:latest" + else + target="${sdk_format}:${image_archive}:${image_name}:latest" + fi + skopeo --insecure-policy copy \ + "oci:${oci_img_dir}:${full_tag}" "${target}" + xz -T0 "${image_archive}" + ;; + "oci") + tar --create --xz --directory "${oci_img_dir}" \ + --file "${image_archive}.xz" . + ;; + "docker-daemon" | "containers-storage") + skopeo --insecure-policy copy \ + "oci:${oci_img_dir}:${full_tag}" \ + "${sdk_format}:${image_name}:latest" + ;; + esac + done +} do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}" do_populate_sdk[depends] = "sdkchroot:do_build" -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT" +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT SDK_GENERATE_FORMATS" do_populate_sdk() { if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then # Copy isar-apt with deployed Isar packages @@ -48,12 +115,30 @@ do_populate_sdk() { done done - # Copy mount_chroot.sh for convenience - sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR} + # separate SDK formats: TAR and container formats + container_formats="" + for sdk_format in ${SDK_GENERATE_FORMATS} ; do + case ${sdk_format} in + tar) + sdk_tar + ;; + "docker-archive" | "oci" | "oci-archive") + container_formats="${container_formats} ${sdk_format}" + ;; + "docker-daemon" | "containers-storage") + if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; then + die "Adding the SDK container image to a container runtime (${sdk_format}) not supported if running from a container (e.g. 'kas-container')" + fi + ;; + *) + die "unsupported SDK format specified: ${sdk_format}" + ;; + esac + done - # Create SDK archive - cd -P ${SDKCHROOT_DIR}/.. - sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \ - -c rootfs | xz -T0 > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz + # generate the SDK in all the desired container formats + if [ -n "${container_formats}" ] ; then + sdk_container_images "${container_formats}" + fi } addtask populate_sdk after do_rootfs
Extend task "populate_sdk" to support the creation of a container image containing the SDK. Signed-off-by: Silvano Cirujano Cuesta <silvano.cirujano-cuesta@siemens.com> --- meta/classes/image-sdk-extension.bbclass | 99 ++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 7 deletions(-)