diff mbox series

[v4,3/6] isar-bootstrap: Use tar output instead of directory

Message ID 20230927043320.6264-4-ubely@ilbers.de
State Superseded, archived
Headers show
Series Migrate to mmdebstrap | expand

Commit Message

Uladzimir Bely Sept. 27, 2023, 4:33 a.m. UTC 
From: Anton Mikanovich <amikan@ilbers.de>

Do not unpack tar with rootfs content inside bootstrap recipe.
This also requires moving chroot prepare step to the later unpack step.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 meta/classes/rootfs.bbclass                   | 11 +++++----
 .../isar-bootstrap/isar-bootstrap.inc         | 23 ++++---------------
 2 files changed, 12 insertions(+), 22 deletions(-)

Comments

Jan Kiszka Sept. 27, 2023, 10:07 a.m. UTC | #1 
On 27.09.23 06:33, Uladzimir Bely wrote:
> From: Anton Mikanovich <amikan@ilbers.de>
> 
> Do not unpack tar with rootfs content inside bootstrap recipe.

Motivating this change here would be nice.

> This also requires moving chroot prepare step to the later unpack step.
> 
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
>  meta/classes/rootfs.bbclass                   | 11 +++++----
>  .../isar-bootstrap/isar-bootstrap.inc         | 23 ++++---------------
>  2 files changed, 12 insertions(+), 22 deletions(-)
> 
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 7dec7b36..09d40563 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -73,12 +73,15 @@ rootfs_do_qemu() {
>      fi
>  }
>  
> -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
> -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
> +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"

zst is really slow on packing but fast on unpacking IIRC. Do we actually
benefit from that (significantly more unpack than pack)?

Or was it like that before already?

>  
>  rootfs_prepare[weight] = "25"
>  rootfs_prepare(){
> -    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> +    sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
> +
> +    # setup chroot
> +    sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>  }
>  
>  ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
> @@ -222,7 +225,7 @@ cache_deb_src() {
>      # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
>      # relies on that through out the build. Copy that state information instead of apt-get update
>      # which generates a new state from upstream.
> -    sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
> +    sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
>  
>      deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
>      debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index d8ac9d00..cb1fc91a 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -16,7 +16,6 @@ PV = "1.0"
>  BOOTSTRAP_FOR_HOST ?= "0"
>  
>  DEBOOTSTRAP = "mmdebstrap"
> -ROOTFSDIR = "${WORKDIR}/rootfs"
>  APTPREFS = "${WORKDIR}/apt-preferences"
>  APTSRCS = "${WORKDIR}/apt-sources"
>  APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
> @@ -287,9 +286,6 @@ do_bootstrap() {
>      fi
>      E="${@ isar_export_proxies(d)}"
>  
> -    sudo rm -rf --one-file-system "${ROOTFSDIR}"
> -    mkdir -p "${ROOTFSDIR}"
> -
>      if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>          base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
>          bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
> @@ -394,18 +390,9 @@ do_bootstrap() {
>                     "${WORKDIR}/rootfs.tar.zst" \
>                     "$bootstrap_list"
>  
> -    sudo -E -s <<'EOSUDO'
> -        set -e
> -
> -        tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
> -
> -        # setup chroot
> -        install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> -        "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> +    # Finalize debootstrap by setting the link in deploy
> +    sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>  
> -        # Finalize debootstrap by setting the link in deploy
> -        ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> -EOSUDO
>      if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
>          deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
>          sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
> @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>  
>  bootstrap_sstate_prepare() {
>      # this runs in SSTATE_BUILDDIR, which will be deleted automatically
> -    sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
> +    sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
>      sudo chown $(id -u):$(id -g) bootstrap.tar.zst
>  }
>  
>  bootstrap_sstate_finalize() {
>      # this runs in SSTATE_INSTDIR
>      if [ -f bootstrap.tar.zst ]; then
> -        mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> -        sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> +        mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
> +        sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
>                        "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>      fi
>  }

Jan
Anton Mikanovich Oct. 17, 2023, 2:05 p.m. UTC | #2 
27/09/2023 13:07, 'Jan Kiszka' via isar-users wrote:
> On 27.09.23 06:33, Uladzimir Bely wrote:
>> From: Anton Mikanovich <amikan@ilbers.de>
>>
>> Do not unpack tar with rootfs content inside bootstrap recipe.
> Motivating this change here would be nice.
>
>> This also requires moving chroot prepare step to the later unpack step.
>>
>> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
>> ---
>>   meta/classes/rootfs.bbclass                   | 11 +++++----
>>   .../isar-bootstrap/isar-bootstrap.inc         | 23 ++++---------------
>>   2 files changed, 12 insertions(+), 22 deletions(-)
>>
>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>> index 7dec7b36..09d40563 100644
>> --- a/meta/classes/rootfs.bbclass
>> +++ b/meta/classes/rootfs.bbclass
>> @@ -73,12 +73,15 @@ rootfs_do_qemu() {
>>       fi
>>   }
>>   
>> -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
>> -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
>> +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
> zst is really slow on packing but fast on unpacking IIRC. Do we actually
> benefit from that (significantly more unpack than pack)?
>
> Or was it like that before already?

Zstd was chosen because it was already used for sstate.
Packing counts will always be <= unpacks, so it should be ok.

Other feedback will be addressed in v5.

>>   rootfs_prepare[weight] = "25"
>>   rootfs_prepare(){
>> -    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>> +    sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
>> +
>> +    # setup chroot
>> +    sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>>   }
>>   
>>   ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>> @@ -222,7 +225,7 @@ cache_deb_src() {
>>       # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
>>       # relies on that through out the build. Copy that state information instead of apt-get update
>>       # which generates a new state from upstream.
>> -    sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
>> +    sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
>>   
>>       deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
>>       debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> index d8ac9d00..cb1fc91a 100644
>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> @@ -16,7 +16,6 @@ PV = "1.0"
>>   BOOTSTRAP_FOR_HOST ?= "0"
>>   
>>   DEBOOTSTRAP = "mmdebstrap"
>> -ROOTFSDIR = "${WORKDIR}/rootfs"
>>   APTPREFS = "${WORKDIR}/apt-preferences"
>>   APTSRCS = "${WORKDIR}/apt-sources"
>>   APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
>> @@ -287,9 +286,6 @@ do_bootstrap() {
>>       fi
>>       E="${@ isar_export_proxies(d)}"
>>   
>> -    sudo rm -rf --one-file-system "${ROOTFSDIR}"
>> -    mkdir -p "${ROOTFSDIR}"
>> -
>>       if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>           base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
>>           bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
>> @@ -394,18 +390,9 @@ do_bootstrap() {
>>                      "${WORKDIR}/rootfs.tar.zst" \
>>                      "$bootstrap_list"
>>   
>> -    sudo -E -s <<'EOSUDO'
>> -        set -e
>> -
>> -        tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
>> -
>> -        # setup chroot
>> -        install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
>> -        "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>> +    # Finalize debootstrap by setting the link in deploy
>> +    sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>>   
>> -        # Finalize debootstrap by setting the link in deploy
>> -        ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>> -EOSUDO
>>       if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
>>           deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
>>           sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
>> @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>>   
>>   bootstrap_sstate_prepare() {
>>       # this runs in SSTATE_BUILDDIR, which will be deleted automatically
>> -    sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
>> +    sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
>>       sudo chown $(id -u):$(id -g) bootstrap.tar.zst
>>   }
>>   
>>   bootstrap_sstate_finalize() {
>>       # this runs in SSTATE_INSTDIR
>>       if [ -f bootstrap.tar.zst ]; then
>> -        mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
>> -        sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
>> +        mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
>> +        sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
>>                         "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>>       fi
>>   }
> Jan
>
diff mbox series

Patch

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 7dec7b36..09d40563 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -73,12 +73,15 @@  rootfs_do_qemu() {
     fi
 }
 
-BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
-BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
+BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
 
 rootfs_prepare[weight] = "25"
 rootfs_prepare(){
-    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+    sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+    # setup chroot
+    sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
 }
 
 ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -222,7 +225,7 @@  cache_deb_src() {
     # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
     # relies on that through out the build. Copy that state information instead of apt-get update
     # which generates a new state from upstream.
-    sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+    sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
 
     deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
     debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index d8ac9d00..cb1fc91a 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -16,7 +16,6 @@  PV = "1.0"
 BOOTSTRAP_FOR_HOST ?= "0"
 
 DEBOOTSTRAP = "mmdebstrap"
-ROOTFSDIR = "${WORKDIR}/rootfs"
 APTPREFS = "${WORKDIR}/apt-preferences"
 APTSRCS = "${WORKDIR}/apt-sources"
 APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
@@ -287,9 +286,6 @@  do_bootstrap() {
     fi
     E="${@ isar_export_proxies(d)}"
 
-    sudo rm -rf --one-file-system "${ROOTFSDIR}"
-    mkdir -p "${ROOTFSDIR}"
-
     if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
         base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
         bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
@@ -394,18 +390,9 @@  do_bootstrap() {
                    "${WORKDIR}/rootfs.tar.zst" \
                    "$bootstrap_list"
 
-    sudo -E -s <<'EOSUDO'
-        set -e
-
-        tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
-
-        # setup chroot
-        install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
-        "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+    # Finalize debootstrap by setting the link in deploy
+    sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
 
-        # Finalize debootstrap by setting the link in deploy
-        ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
-EOSUDO
     if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
         deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
         sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
@@ -420,15 +407,15 @@  SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
 
 bootstrap_sstate_prepare() {
     # this runs in SSTATE_BUILDDIR, which will be deleted automatically
-    sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+    sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
     sudo chown $(id -u):$(id -g) bootstrap.tar.zst
 }
 
 bootstrap_sstate_finalize() {
     # this runs in SSTATE_INSTDIR
     if [ -f bootstrap.tar.zst ]; then
-        mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
-        sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+        mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+        sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
                       "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
     fi
 }