diff mbox series

[v4,2/4] deb-dl-dir: optimize caching of source packages using apt natively

Message ID 20250925065433.4180883-3-cedric.hombourger@siemens.com
State Under Review
Headers show
Series non-privileged commands in chroot | expand

Commit Message

cedric.hombourger@siemens.com Sept. 25, 2025, 6:54 a.m. UTC 
From: "cedric.hombourger@siemens.com" <cedric.hombourger@siemens.com>

source package are downloaded by entering the target rootfs and run
apt there. For foreign architectures, this results in apt being
executed under QEMU and leads to poor performance. By using the
recently introduced rootfs_native_cmd command wrapper, apt will be
executed natively against the target rootfs and without elevated
privileges. For our test work-load, caching was reduced from more
than 10 hours to an hour. Performance is also more consistent as
it will no longer depend as to when bitbake kicks caching of
source packages for foreign architecture rootfs vs rootfs for the
host (in multiconfig builds).

Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
---
 meta/classes/deb-dl-dir.bbclass | 37 ++++++++-------------------------
 1 file changed, 9 insertions(+), 28 deletions(-)

Comments

Felix Moessbauer Sept. 25, 2025, 9:07 a.m. UTC | #1 
On Thu, 2025-09-25 at 08:54 +0200, 'Cedric Hombourger' via isar-users
wrote:
> From: "cedric.hombourger@siemens.com" <cedric.hombourger@siemens.com>
> 
> source package are downloaded by entering the target rootfs and run
> apt there. For foreign architectures, this results in apt being
> executed under QEMU and leads to poor performance. By using the
> recently introduced rootfs_native_cmd command wrapper, apt will be
> executed natively against the target rootfs and without elevated
> privileges. For our test work-load, caching was reduced from more
> than 10 hours to an hour. Performance is also more consistent as
> it will no longer depend as to when bitbake kicks caching of
> source packages for foreign architecture rootfs vs rootfs for the
> host (in multiconfig builds).
> 
> Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
> ---
>  meta/classes/deb-dl-dir.bbclass | 37 ++++++++-------------------------
>  1 file changed, 9 insertions(+), 28 deletions(-)
> 
> diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
> index 4780be20..ea0ed3d2 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -5,25 +5,6 @@
>  
>  inherit repository
>  
> -debsrc_do_mounts() {
> -    sudo -s <<EOSUDO
> -    set -e
> -    mkdir -p "${1}/deb-src"
> -    mountpoint -q "${1}/deb-src" || \
> -    mount -o bind,private "${DEBSRCDIR}" "${1}/deb-src"
> -EOSUDO
> -}
> -
> -debsrc_undo_mounts() {
> -    sudo -s <<EOSUDO
> -    set -e
> -    mkdir -p "${1}/deb-src"
> -    mountpoint -q "${1}/deb-src" && \
> -    umount "${1}/deb-src"
> -    rm -rf "${1}/deb-src"
> -EOSUDO
> -}
> -
>  debsrc_source_version_filter() {
>      # Filter the input to only consider Package, Version and Source lines
>      #
> @@ -51,11 +32,6 @@ debsrc_download() {
>      export rootfs_distro="$2"
>      mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
>  
> -    debsrc_do_mounts "${rootfs}"
> -
> -    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> -    trap 'debsrc_undo_mounts "${rootfs}"' EXIT
> -
>      ( flock 9
>      set -e
>      printenv | grep -q BB_VERBOSE_LOGS && set -x
> @@ -96,13 +72,18 @@ debsrc_download() {
>          dscname="${src}_${version#*:}.dsc"
>          [ -f "${DEBSRCDIR}"/"${rootfs_distro}"/"${src}"/"${dscname}" ] || {
>              # use apt-get source to download sources in DEBSRCDIR
> -            sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \
> -                sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
> +            mkdir -p "${DEBSRCDIR}/${rootfs_distro}"/"${src}"
> +            rootfs_cmd \
> +                --bind "${DEBSRCDIR}" "/deb-src" \
> +                --bind "${rootfs}" "${rootfs}" \
> +                --chdir "/deb-src/${rootfs_distro}/${src}" \
> +                -- \
> +                apt-get -o APT::Architecture=${DISTRO_ARCH} \
> +                        -o Dir="${rootfs}" -y --download-only \
> +                        --only-source source "${src}=${version}"
>          }
>      done
>      ) 9>"${DEBSRCDIR}/${rootfs_distro}.lock"

This lock was previously owned by root, hence it must be manually
cleaned when updating to this series. Otherwise the following error
happens:

/work/build/tmp/work/debian-bookworm-arm64/isar-mmdebstrap-host/1.0-
r0/temp/run.do_bootstrap.1528: 262: cannot create
/work/build/downloads/deb/debian-bookworm.lock: Permission denied

Apart from that, I'm wondering why we don't delete this lock in one of
the cleanup handlers. We anyways can only run a single bitbake instance
per build dir at a time.

Felix

> -
> -    debsrc_undo_mounts "${rootfs}"
>  }
>  
>  dbg_pkgs_download() {
> -- 
> 2.47.3
> 
> -- 
> You received this message because you are subscribed to the Google Groups "isar-users" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
> To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250925065433.4180883-3-cedric.hombourger%40siemens.com.
diff mbox series

Patch

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 4780be20..ea0ed3d2 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -5,25 +5,6 @@ 
 
 inherit repository
 
-debsrc_do_mounts() {
-    sudo -s <<EOSUDO
-    set -e
-    mkdir -p "${1}/deb-src"
-    mountpoint -q "${1}/deb-src" || \
-    mount -o bind,private "${DEBSRCDIR}" "${1}/deb-src"
-EOSUDO
-}
-
-debsrc_undo_mounts() {
-    sudo -s <<EOSUDO
-    set -e
-    mkdir -p "${1}/deb-src"
-    mountpoint -q "${1}/deb-src" && \
-    umount "${1}/deb-src"
-    rm -rf "${1}/deb-src"
-EOSUDO
-}
-
 debsrc_source_version_filter() {
     # Filter the input to only consider Package, Version and Source lines
     #
@@ -51,11 +32,6 @@  debsrc_download() {
     export rootfs_distro="$2"
     mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
 
-    debsrc_do_mounts "${rootfs}"
-
-    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
-    trap 'debsrc_undo_mounts "${rootfs}"' EXIT
-
     ( flock 9
     set -e
     printenv | grep -q BB_VERBOSE_LOGS && set -x
@@ -96,13 +72,18 @@  debsrc_download() {
         dscname="${src}_${version#*:}.dsc"
         [ -f "${DEBSRCDIR}"/"${rootfs_distro}"/"${src}"/"${dscname}" ] || {
             # use apt-get source to download sources in DEBSRCDIR
-            sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \
-                sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
+            mkdir -p "${DEBSRCDIR}/${rootfs_distro}"/"${src}"
+            rootfs_cmd \
+                --bind "${DEBSRCDIR}" "/deb-src" \
+                --bind "${rootfs}" "${rootfs}" \
+                --chdir "/deb-src/${rootfs_distro}/${src}" \
+                -- \
+                apt-get -o APT::Architecture=${DISTRO_ARCH} \
+                        -o Dir="${rootfs}" -y --download-only \
+                        --only-source source "${src}=${version}"
         }
     done
     ) 9>"${DEBSRCDIR}/${rootfs_distro}.lock"
-
-    debsrc_undo_mounts "${rootfs}"
 }
 
 dbg_pkgs_download() {