Message ID | 20230927043320.6264-4-ubely@ilbers.de |
---|---|
State | Superseded, archived |
Headers | show |
Series | Migrate to mmdebstrap | expand |
On 27.09.23 06:33, Uladzimir Bely wrote: > From: Anton Mikanovich <amikan@ilbers.de> > > Do not unpack tar with rootfs content inside bootstrap recipe. Motivating this change here would be nice. > This also requires moving chroot prepare step to the later unpack step. > > Signed-off-by: Anton Mikanovich <amikan@ilbers.de> > --- > meta/classes/rootfs.bbclass | 11 +++++---- > .../isar-bootstrap/isar-bootstrap.inc | 23 ++++--------------- > 2 files changed, 12 insertions(+), 22 deletions(-) > > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > index 7dec7b36..09d40563 100644 > --- a/meta/classes/rootfs.bbclass > +++ b/meta/classes/rootfs.bbclass > @@ -73,12 +73,15 @@ rootfs_do_qemu() { > fi > } > > -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}" > -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}" > +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst" > +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst" zst is really slow on packing but fast on unpacking IIRC. Do we actually benefit from that (significantly more unpack than pack)? Or was it like that before already? > > rootfs_prepare[weight] = "25" > rootfs_prepare(){ > - sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}' > + sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console" > + > + # setup chroot > + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" > } > > ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt" > @@ -222,7 +225,7 @@ cache_deb_src() { > # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and > # relies on that through out the build. Copy that state information instead of apt-get update > # which generates a new state from upstream. > - sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/" > + sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}" > > deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} > debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > index d8ac9d00..cb1fc91a 100644 > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > @@ -16,7 +16,6 @@ PV = "1.0" > BOOTSTRAP_FOR_HOST ?= "0" > > DEBOOTSTRAP = "mmdebstrap" > -ROOTFSDIR = "${WORKDIR}/rootfs" > APTPREFS = "${WORKDIR}/apt-preferences" > APTSRCS = "${WORKDIR}/apt-sources" > APTSRCS_INIT = "${WORKDIR}/apt-sources-init" > @@ -287,9 +286,6 @@ do_bootstrap() { > fi > E="${@ isar_export_proxies(d)}" > > - sudo rm -rf --one-file-system "${ROOTFSDIR}" > - mkdir -p "${ROOTFSDIR}" > - > if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then > base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)" > bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list" > @@ -394,18 +390,9 @@ do_bootstrap() { > "${WORKDIR}/rootfs.tar.zst" \ > "$bootstrap_list" > > - sudo -E -s <<'EOSUDO' > - set -e > - > - tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console" > - > - # setup chroot > - install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh" > - "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" > + # Finalize debootstrap by setting the link in deploy > + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" > > - # Finalize debootstrap by setting the link in deploy > - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}" > -EOSUDO > if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then > deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" > sudo rm -rf --one-file-system "${WORKDIR}/dl_dir" > @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize" > > bootstrap_sstate_prepare() { > # this runs in SSTATE_BUILDDIR, which will be deleted automatically > - sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst > + sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst > sudo chown $(id -u):$(id -g) bootstrap.tar.zst > } > > bootstrap_sstate_finalize() { > # this runs in SSTATE_INSTDIR > if [ -f bootstrap.tar.zst ]; then > - mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" > - sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \ > + mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst" > + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \ > "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" > fi > } Jan
27/09/2023 13:07, 'Jan Kiszka' via isar-users wrote: > On 27.09.23 06:33, Uladzimir Bely wrote: >> From: Anton Mikanovich <amikan@ilbers.de> >> >> Do not unpack tar with rootfs content inside bootstrap recipe. > Motivating this change here would be nice. > >> This also requires moving chroot prepare step to the later unpack step. >> >> Signed-off-by: Anton Mikanovich <amikan@ilbers.de> >> --- >> meta/classes/rootfs.bbclass | 11 +++++---- >> .../isar-bootstrap/isar-bootstrap.inc | 23 ++++--------------- >> 2 files changed, 12 insertions(+), 22 deletions(-) >> >> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass >> index 7dec7b36..09d40563 100644 >> --- a/meta/classes/rootfs.bbclass >> +++ b/meta/classes/rootfs.bbclass >> @@ -73,12 +73,15 @@ rootfs_do_qemu() { >> fi >> } >> >> -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}" >> -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}" >> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst" >> +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst" > zst is really slow on packing but fast on unpacking IIRC. Do we actually > benefit from that (significantly more unpack than pack)? > > Or was it like that before already? Zstd was chosen because it was already used for sstate. Packing counts will always be <= unpacks, so it should be ok. Other feedback will be addressed in v5. >> rootfs_prepare[weight] = "25" >> rootfs_prepare(){ >> - sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}' >> + sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console" >> + >> + # setup chroot >> + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" >> } >> >> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt" >> @@ -222,7 +225,7 @@ cache_deb_src() { >> # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and >> # relies on that through out the build. Copy that state information instead of apt-get update >> # which generates a new state from upstream. >> - sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/" >> + sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}" >> >> deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} >> debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} >> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >> index d8ac9d00..cb1fc91a 100644 >> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >> @@ -16,7 +16,6 @@ PV = "1.0" >> BOOTSTRAP_FOR_HOST ?= "0" >> >> DEBOOTSTRAP = "mmdebstrap" >> -ROOTFSDIR = "${WORKDIR}/rootfs" >> APTPREFS = "${WORKDIR}/apt-preferences" >> APTSRCS = "${WORKDIR}/apt-sources" >> APTSRCS_INIT = "${WORKDIR}/apt-sources-init" >> @@ -287,9 +286,6 @@ do_bootstrap() { >> fi >> E="${@ isar_export_proxies(d)}" >> >> - sudo rm -rf --one-file-system "${ROOTFSDIR}" >> - mkdir -p "${ROOTFSDIR}" >> - >> if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then >> base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)" >> bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list" >> @@ -394,18 +390,9 @@ do_bootstrap() { >> "${WORKDIR}/rootfs.tar.zst" \ >> "$bootstrap_list" >> >> - sudo -E -s <<'EOSUDO' >> - set -e >> - >> - tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console" >> - >> - # setup chroot >> - install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh" >> - "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" >> + # Finalize debootstrap by setting the link in deploy >> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" >> >> - # Finalize debootstrap by setting the link in deploy >> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}" >> -EOSUDO >> if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then >> deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" >> sudo rm -rf --one-file-system "${WORKDIR}/dl_dir" >> @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize" >> >> bootstrap_sstate_prepare() { >> # this runs in SSTATE_BUILDDIR, which will be deleted automatically >> - sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst >> + sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst >> sudo chown $(id -u):$(id -g) bootstrap.tar.zst >> } >> >> bootstrap_sstate_finalize() { >> # this runs in SSTATE_INSTDIR >> if [ -f bootstrap.tar.zst ]; then >> - mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" >> - sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \ >> + mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst" >> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \ >> "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" >> fi >> } > Jan >
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass index 7dec7b36..09d40563 100644 --- a/meta/classes/rootfs.bbclass +++ b/meta/classes/rootfs.bbclass @@ -73,12 +73,15 @@ rootfs_do_qemu() { fi } -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}" -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}" +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst" +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst" rootfs_prepare[weight] = "25" rootfs_prepare(){ - sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}' + sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console" + + # setup chroot + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" } ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt" @@ -222,7 +225,7 @@ cache_deb_src() { # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and # relies on that through out the build. Copy that state information instead of apt-get update # which generates a new state from upstream. - sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/" + sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}" deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index d8ac9d00..cb1fc91a 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -16,7 +16,6 @@ PV = "1.0" BOOTSTRAP_FOR_HOST ?= "0" DEBOOTSTRAP = "mmdebstrap" -ROOTFSDIR = "${WORKDIR}/rootfs" APTPREFS = "${WORKDIR}/apt-preferences" APTSRCS = "${WORKDIR}/apt-sources" APTSRCS_INIT = "${WORKDIR}/apt-sources-init" @@ -287,9 +286,6 @@ do_bootstrap() { fi E="${@ isar_export_proxies(d)}" - sudo rm -rf --one-file-system "${ROOTFSDIR}" - mkdir -p "${ROOTFSDIR}" - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)" bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list" @@ -394,18 +390,9 @@ do_bootstrap() { "${WORKDIR}/rootfs.tar.zst" \ "$bootstrap_list" - sudo -E -s <<'EOSUDO' - set -e - - tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console" - - # setup chroot - install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh" - "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" + # Finalize debootstrap by setting the link in deploy + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" - # Finalize debootstrap by setting the link in deploy - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}" -EOSUDO if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" sudo rm -rf --one-file-system "${WORKDIR}/dl_dir" @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize" bootstrap_sstate_prepare() { # this runs in SSTATE_BUILDDIR, which will be deleted automatically - sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst + sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst sudo chown $(id -u):$(id -g) bootstrap.tar.zst } bootstrap_sstate_finalize() { # this runs in SSTATE_INSTDIR if [ -f bootstrap.tar.zst ]; then - mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" - sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \ + mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst" + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst" fi }