| Message ID | d93f0414-755d-4a77-9ebc-4514624194cf@siemens.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | [v6] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot | expand |
On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > During the migration to sbuild, support for using self-built packages > in > the build environment was lost if those were already part of the > sbuild-chroot. This restores it by adding --apt-distupgrade to the > sbuild call. But that is not enough because sbuild will only upgrade > packages from already configured sources, not those specified via > --extra-repository. We therefore have to switch back to configuring > isar-apt during sbuild-chroot creation. > > As rootfs_configure_isar_apt configures the isar repo under > /isar-apt, we bind-mount the one in /home/builder/${PN} to that > folder. > Another difference is that we now need to run apt-get update > explicitly, > but only for isar-apt. > > For apt fetching, we neither need nor want isar-apt to be available. > Rebuilding self-generated apt packages is generally pointless as the > needs are better addressed in the generating recipe itself. Exposing > isar-apt to the fetch may furthermore lead to fetching a previously > built source package of the same recipe, rather than pulling the > external version. > > And because the sbuild-chroots are now left behind with isar-apt > configured, the configuration in imager_run can be removed (credits > to > Srinuvasan Arjunan). > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > > Changes in v6: > - basically, going back to v4 but disabling isar-apt in do_apt_fetch > > I was able to resolve the scenario that Uladzimir was sharing this > way. > And I also realized that apt fetching as well as unpacking should > have > no deal with isar-apt. So, v5 was taking a wrong turn. > > meta/classes/dpkg-base.bbclass | 29 +++++++++++------ > -- > meta/classes/dpkg.bbclass | 5 ++-- > meta/classes/image-tools-extension.bbclass | 13 --------- > .../sbuild-chroot/sbuild-chroot.inc | 2 -- > 4 files changed, 20 insertions(+), 29 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > base.bbclass > index 7b054d3f..80686677 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild- > essential-riscv64' if d.getVar('ISAR_C > DEB_BUILD_PROFILES ?= "" > DEB_BUILD_OPTIONS ?= "" > > -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar- > apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" > - > python do_adjust_git() { > import subprocess > > @@ -115,6 +113,8 @@ do_apt_fetch() { > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > trap 'schroot_cleanup' EXIT > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > + rm /etc/apt/sources.list.d/isar-apt.list > /etc/apt/preferences.d/isar-apt The issue is still reproducible with the steps I posted below, as well as CI still fails with v6. I think, the steps above are not sufficient to completely stop dealing with isar-apt in do_apt_fetch() and we need some kind of 'apt update' here to consider removed list and preferences. The question is how to say apt to forget about isar-apt and do not run apt update for other source lists... maybe, simply remove isar-apt files from /var/lib/apt/lists/. > for uri in "${SRC_APT}"; do > schroot -d / -c ${SBUILD_CHROOT} -- \ > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- > source source "$2"' my_script "${BASE_DISTRO}- > ${BASE_DISTRO_CODENAME}" "${uri}" > @@ -221,8 +221,17 @@ def isar_export_build_settings(d): > os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) > os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) > > +dpkg_schroot_create_configs() { > + schroot_create_configs > + sudo -s <<'EOSUDO' > + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" > + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}- > ${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" > + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo > "${fstab_isarapt}" >> ${sbuild_fstab} > +EOSUDO > +} > + > python do_dpkg_build() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > try: > bb.build.exec_func("dpkg_runbuild", d) > finally: > @@ -288,7 +297,7 @@ do_deploy_deb[lockfiles] = > "${REPO_ISAR_DIR}/isar.lock" > do_deploy_deb[dirs] = "${S}" > > python do_devshell() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > > isar_export_proxies(d) > isar_export_ccache(d) > @@ -297,11 +306,9 @@ python do_devshell() { > bb.build.exec_func('schroot_configure_ccache', d) > > schroot = d.getVar('SBUILD_CHROOT') > - isar_apt = d.getVar('ISAR_APT_REPO') > pkg_arch = d.getVar('PACKAGE_ARCH') > build_arch = d.getVar('BUILD_ARCH') > pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > - debdistroname = d.getVar('DEBDISTRONAME') > > install_deps = ":" if d.getVar('BB_CURRENTTASK') == > "devshell_nodeps" else f"mk-build-deps -i \ > --host-arch {pkg_arch} --build-arch {build_arch} \ > @@ -310,15 +317,13 @@ python do_devshell() { > > termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ > cd {1}; \ > - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ > - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > > /etc/apt/preferences.d/isar-apt; \ > - echo \"APT::Get::allow-downgrades 1;\" > > /etc/apt/apt.conf.d/50isar-apt; \ > - apt-get -y -q update; \ > - {4}; \ > + apt-get -y -q update -o > Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o > Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ > + apt-get -y upgrade; \ > + {2}; \ > export PATH=$PATH_PREPEND:$PATH; \ > $SHELL -i \ > '" > - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, > debdistroname, install_deps), "Isar devshell", d) > + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar > devshell", d) > > bb.build.exec_func('schroot_delete_configs', d) > } > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index c699a84d..d1666f78 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -110,16 +110,17 @@ dpkg_runbuild() { > DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file > ${WORKDIR}/${PPS}/debian/changelog) > DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" - > maxdepth 1 -print) > > - sbuild -A -n -c ${SBUILD_CHROOT} --extra- > repository="${ISAR_APT_REPO}" \ > + sbuild -A -n -c ${SBUILD_CHROOT} \ > --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ > --no-run-lintian --no-run-piuparts --no-run-autopkgtest -- > resolve-alternatives \ > --bd-uninstallable-explainer=apt \ > - --no-apt-update \ > + --no-apt-update --apt-distupgrade \ > --chroot-setup-commands="echo \"Package: *\nPin: release > n=${DEBDISTRONAME}\nPin-Priority: 1000\" > > /etc/apt/preferences.d/isar-apt" \ > --chroot-setup-commands="echo \"APT::Get::allow-downgrades > 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ > --chroot-setup-commands="rm -f /var/log/dpkg.log" \ > --chroot-setup-commands="mkdir -p ${deb_dir}" \ > --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t > ${deb_dir}/" \ > + --chroot-setup-commands="apt-get update -o > Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o > Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ > --finished-build-commands="rm -f ${deb_dir}/sbuild-build- > depends-main-dummy_*.deb" \ > --finished-build-commands="[ -z \"\$(find ${deb_dir} - > maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} > ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ > --finished-build-commands="cp /var/log/dpkg.log > ${ext_root}/dpkg_partial.log" \ > diff --git a/meta/classes/image-tools-extension.bbclass > b/meta/classes/image-tools-extension.bbclass > index 58799fcf..46bdf78b 100644 > --- a/meta/classes/image-tools-extension.bbclass > +++ b/meta/classes/image-tools-extension.bbclass > @@ -44,19 +44,6 @@ imager_run() { > distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" > fi > > - # prepare isar-apt > - schroot -r -c ${session_id} -d / -u root -- sh -c " \ > - mkdir -p '/etc/apt/sources.list.d' > - echo 'deb [trusted=yes] file:///isar- > apt ${DEBDISTRONAME} main' > \ > - '/etc/apt/sources.list.d/isar-apt.list' > - > - mkdir -p '/etc/apt/preferences.d' > - cat << EOF > '/etc/apt/preferences.d/isar-apt' > -Package: * > -Pin: release n=${DEBDISTRONAME} > -Pin-Priority: 1000 > -EOF" > - > E="${@ isar_export_proxies(d)}" > deb_dl_dir_import ${schroot_dir} ${distro} > schroot -r -c ${session_id} -d / -u root -- sh -c " \ > diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > index 39ced80b..2f07de82 100644 > --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" > ROOTFSDIR = "${SBUILD_CHROOT_DIR}" > ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" > > -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's > handled by sbuild > -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" > ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" > > # Do not cleanup base-apt
On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: > On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: > > From: Jan Kiszka <jan.kiszka@siemens.com> > > > > During the migration to sbuild, support for using self-built > > packages > > in > > the build environment was lost if those were already part of the > > sbuild-chroot. This restores it by adding --apt-distupgrade to the > > sbuild call. But that is not enough because sbuild will only > > upgrade > > packages from already configured sources, not those specified via > > --extra-repository. We therefore have to switch back to configuring > > isar-apt during sbuild-chroot creation. > > > > As rootfs_configure_isar_apt configures the isar repo under > > /isar-apt, we bind-mount the one in /home/builder/${PN} to that > > folder. > > Another difference is that we now need to run apt-get update > > explicitly, > > but only for isar-apt. > > > > For apt fetching, we neither need nor want isar-apt to be > > available. > > Rebuilding self-generated apt packages is generally pointless as > > the > > needs are better addressed in the generating recipe itself. > > Exposing > > isar-apt to the fetch may furthermore lead to fetching a previously > > built source package of the same recipe, rather than pulling the > > external version. > > > > And because the sbuild-chroots are now left behind with isar-apt > > configured, the configuration in imager_run can be removed (credits > > to > > Srinuvasan Arjunan). > > > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > > --- > > > > Changes in v6: > > - basically, going back to v4 but disabling isar-apt in > > do_apt_fetch > > > > I was able to resolve the scenario that Uladzimir was sharing this > > way. > > And I also realized that apt fetching as well as unpacking should > > have > > no deal with isar-apt. So, v5 was taking a wrong turn. > > > > meta/classes/dpkg-base.bbclass | 29 +++++++++++---- > > -- > > -- > > meta/classes/dpkg.bbclass | 5 ++-- > > meta/classes/image-tools-extension.bbclass | 13 --------- > > .../sbuild-chroot/sbuild-chroot.inc | 2 -- > > 4 files changed, 20 insertions(+), 29 deletions(-) > > > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > > base.bbclass > > index 7b054d3f..80686677 100644 > > --- a/meta/classes/dpkg-base.bbclass > > +++ b/meta/classes/dpkg-base.bbclass > > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild- > > essential-riscv64' if d.getVar('ISAR_C > > DEB_BUILD_PROFILES ?= "" > > DEB_BUILD_OPTIONS ?= "" > > > > -ISAR_APT_REPO ?= "deb [trusted=yes] > > file:///home/builder/${PN}/isar- > > apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" > > - > > python do_adjust_git() { > > import subprocess > > > > @@ -115,6 +113,8 @@ do_apt_fetch() { > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > trap 'schroot_cleanup' EXIT > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > + rm /etc/apt/sources.list.d/isar-apt.list > > /etc/apt/preferences.d/isar-apt > > The issue is still reproducible with the steps I posted below, as > well > as CI still fails with v6. > I think, the steps above are not sufficient to completely stop > dealing > with isar-apt in do_apt_fetch() and we need some kind of 'apt update' > here to consider removed list and preferences. The question is how to > say apt to forget about isar-apt and do not run apt update for other > source lists... maybe, simply remove isar-apt files from > /var/lib/apt/lists/. An addition: It seems that files are not really removed from the session for some reason. I added some kind of "ls" near remove code: ``` + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ + sh -c 'ls -la /etc/apt/sources.list.d/' schroot -d / -u root -c ${SBUILD_CHROOT} -- \ rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ + sh -c 'ls -la /etc/apt/sources.list.d/' ``` , but still see the following in the log: ``` | DEBUG: Executing shell function do_apt_fetch | total 16 | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list | total 16 | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list | Reading package lists... ``` Fith the apt_fetch failure followed, because of existing of previously built hello source package in the isar-apt repo. > > > for uri in "${SRC_APT}"; do > > schroot -d / -c ${SBUILD_CHROOT} -- \ > > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- > > source source "$2"' my_script "${BASE_DISTRO}- > > ${BASE_DISTRO_CODENAME}" "${uri}" > > @@ -221,8 +221,17 @@ def isar_export_build_settings(d): > > os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) > > os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) > > > > +dpkg_schroot_create_configs() { > > + schroot_create_configs > > + sudo -s <<'EOSUDO' > > + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" > > + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}- > > ${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" > > + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo > > "${fstab_isarapt}" >> ${sbuild_fstab} > > +EOSUDO > > +} > > + > > python do_dpkg_build() { > > - bb.build.exec_func('schroot_create_configs', d) > > + bb.build.exec_func('dpkg_schroot_create_configs', d) > > try: > > bb.build.exec_func("dpkg_runbuild", d) > > finally: > > @@ -288,7 +297,7 @@ do_deploy_deb[lockfiles] = > > "${REPO_ISAR_DIR}/isar.lock" > > do_deploy_deb[dirs] = "${S}" > > > > python do_devshell() { > > - bb.build.exec_func('schroot_create_configs', d) > > + bb.build.exec_func('dpkg_schroot_create_configs', d) > > > > isar_export_proxies(d) > > isar_export_ccache(d) > > @@ -297,11 +306,9 @@ python do_devshell() { > > bb.build.exec_func('schroot_configure_ccache', d) > > > > schroot = d.getVar('SBUILD_CHROOT') > > - isar_apt = d.getVar('ISAR_APT_REPO') > > pkg_arch = d.getVar('PACKAGE_ARCH') > > build_arch = d.getVar('BUILD_ARCH') > > pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > > - debdistroname = d.getVar('DEBDISTRONAME') > > > > install_deps = ":" if d.getVar('BB_CURRENTTASK') == > > "devshell_nodeps" else f"mk-build-deps -i \ > > --host-arch {pkg_arch} --build-arch {build_arch} \ > > @@ -310,15 +317,13 @@ python do_devshell() { > > > > termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ > > cd {1}; \ > > - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ > > - echo \"Package: *\nPin: release n={3}\nPin-Priority: > > 1000\" > > > /etc/apt/preferences.d/isar-apt; \ > > - echo \"APT::Get::allow-downgrades 1;\" > > > /etc/apt/apt.conf.d/50isar-apt; \ > > - apt-get -y -q update; \ > > - {4}; \ > > + apt-get -y -q update -o > > Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o > > Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ > > + apt-get -y upgrade; \ > > + {2}; \ > > export PATH=$PATH_PREPEND:$PATH; \ > > $SHELL -i \ > > '" > > - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, > > debdistroname, install_deps), "Isar devshell", d) > > + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), > > "Isar > > devshell", d) > > > > bb.build.exec_func('schroot_delete_configs', d) > > } > > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > > index c699a84d..d1666f78 100644 > > --- a/meta/classes/dpkg.bbclass > > +++ b/meta/classes/dpkg.bbclass > > @@ -110,16 +110,17 @@ dpkg_runbuild() { > > DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source -- > > file > > ${WORKDIR}/${PPS}/debian/changelog) > > DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" - > > maxdepth 1 -print) > > > > - sbuild -A -n -c ${SBUILD_CHROOT} --extra- > > repository="${ISAR_APT_REPO}" \ > > + sbuild -A -n -c ${SBUILD_CHROOT} \ > > --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ > > --no-run-lintian --no-run-piuparts --no-run-autopkgtest -- > > resolve-alternatives \ > > --bd-uninstallable-explainer=apt \ > > - --no-apt-update \ > > + --no-apt-update --apt-distupgrade \ > > --chroot-setup-commands="echo \"Package: *\nPin: release > > n=${DEBDISTRONAME}\nPin-Priority: 1000\" > > > /etc/apt/preferences.d/isar-apt" \ > > --chroot-setup-commands="echo \"APT::Get::allow-downgrades > > 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ > > --chroot-setup-commands="rm -f /var/log/dpkg.log" \ > > --chroot-setup-commands="mkdir -p ${deb_dir}" \ > > --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t > > ${deb_dir}/" \ > > + --chroot-setup-commands="apt-get update -o > > Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o > > Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ > > --finished-build-commands="rm -f ${deb_dir}/sbuild-build- > > depends-main-dummy_*.deb" \ > > --finished-build-commands="[ -z \"\$(find ${deb_dir} - > > maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} > > ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ > > --finished-build-commands="cp /var/log/dpkg.log > > ${ext_root}/dpkg_partial.log" \ > > diff --git a/meta/classes/image-tools-extension.bbclass > > b/meta/classes/image-tools-extension.bbclass > > index 58799fcf..46bdf78b 100644 > > --- a/meta/classes/image-tools-extension.bbclass > > +++ b/meta/classes/image-tools-extension.bbclass > > @@ -44,19 +44,6 @@ imager_run() { > > distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" > > fi > > > > - # prepare isar-apt > > - schroot -r -c ${session_id} -d / -u root -- sh -c " \ > > - mkdir -p '/etc/apt/sources.list.d' > > - echo 'deb [trusted=yes] file:///isar- > > apt ${DEBDISTRONAME} main' > \ > > - '/etc/apt/sources.list.d/isar-apt.list' > > - > > - mkdir -p '/etc/apt/preferences.d' > > - cat << EOF > '/etc/apt/preferences.d/isar-apt' > > -Package: * > > -Pin: release n=${DEBDISTRONAME} > > -Pin-Priority: 1000 > > -EOF" > > - > > E="${@ isar_export_proxies(d)}" > > deb_dl_dir_import ${schroot_dir} ${distro} > > schroot -r -c ${session_id} -d / -u root -- sh -c " \ > > diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > > b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > > index 39ced80b..2f07de82 100644 > > --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > > +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > > @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" > > ROOTFSDIR = "${SBUILD_CHROOT_DIR}" > > ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" > > > > -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's > > handled by sbuild > > -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" > > ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" > > > > # Do not cleanup base-apt >
On 19.01.24 08:37, Uladzimir Bely wrote: > On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: >> On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: >>> From: Jan Kiszka <jan.kiszka@siemens.com> >>> >>> During the migration to sbuild, support for using self-built >>> packages >>> in >>> the build environment was lost if those were already part of the >>> sbuild-chroot. This restores it by adding --apt-distupgrade to the >>> sbuild call. But that is not enough because sbuild will only >>> upgrade >>> packages from already configured sources, not those specified via >>> --extra-repository. We therefore have to switch back to configuring >>> isar-apt during sbuild-chroot creation. >>> >>> As rootfs_configure_isar_apt configures the isar repo under >>> /isar-apt, we bind-mount the one in /home/builder/${PN} to that >>> folder. >>> Another difference is that we now need to run apt-get update >>> explicitly, >>> but only for isar-apt. >>> >>> For apt fetching, we neither need nor want isar-apt to be >>> available. >>> Rebuilding self-generated apt packages is generally pointless as >>> the >>> needs are better addressed in the generating recipe itself. >>> Exposing >>> isar-apt to the fetch may furthermore lead to fetching a previously >>> built source package of the same recipe, rather than pulling the >>> external version. >>> >>> And because the sbuild-chroots are now left behind with isar-apt >>> configured, the configuration in imager_run can be removed (credits >>> to >>> Srinuvasan Arjunan). >>> >>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>> --- >>> >>> Changes in v6: >>> - basically, going back to v4 but disabling isar-apt in >>> do_apt_fetch >>> >>> I was able to resolve the scenario that Uladzimir was sharing this >>> way. >>> And I also realized that apt fetching as well as unpacking should >>> have >>> no deal with isar-apt. So, v5 was taking a wrong turn. >>> >>> meta/classes/dpkg-base.bbclass | 29 +++++++++++---- >>> -- >>> -- >>> meta/classes/dpkg.bbclass | 5 ++-- >>> meta/classes/image-tools-extension.bbclass | 13 --------- >>> .../sbuild-chroot/sbuild-chroot.inc | 2 -- >>> 4 files changed, 20 insertions(+), 29 deletions(-) >>> >>> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- >>> base.bbclass >>> index 7b054d3f..80686677 100644 >>> --- a/meta/classes/dpkg-base.bbclass >>> +++ b/meta/classes/dpkg-base.bbclass >>> @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild- >>> essential-riscv64' if d.getVar('ISAR_C >>> DEB_BUILD_PROFILES ?= "" >>> DEB_BUILD_OPTIONS ?= "" >>> >>> -ISAR_APT_REPO ?= "deb [trusted=yes] >>> file:///home/builder/${PN}/isar- >>> apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" >>> - >>> python do_adjust_git() { >>> import subprocess >>> >>> @@ -115,6 +113,8 @@ do_apt_fetch() { >>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>> trap 'schroot_cleanup' EXIT >>> >>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>> + rm /etc/apt/sources.list.d/isar-apt.list >>> /etc/apt/preferences.d/isar-apt >> >> The issue is still reproducible with the steps I posted below, as >> well >> as CI still fails with v6. >> I think, the steps above are not sufficient to completely stop >> dealing >> with isar-apt in do_apt_fetch() and we need some kind of 'apt update' >> here to consider removed list and preferences. The question is how to >> say apt to forget about isar-apt and do not run apt update for other >> source lists... maybe, simply remove isar-apt files from >> /var/lib/apt/lists/. > > An addition: > > It seems that files are not really removed from the session for some > reason. I added some kind of "ls" near remove code: > > ``` > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > + sh -c 'ls -la /etc/apt/sources.list.d/' > schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > rm /etc/apt/sources.list.d/isar-apt.list > /etc/apt/preferences.d/isar-apt > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > + sh -c 'ls -la /etc/apt/sources.list.d/' > ``` > , but still see the following in the log: > ``` > | DEBUG: Executing shell function do_apt_fetch > | total 16 > | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > | total 16 > | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > | Reading package lists... > ``` > Fith the apt_fetch failure followed, because of existing of previously > built hello source package in the isar-apt repo. > OK... last-minute change of mine, splitting up the purging and the actual fetching into two schroot runs to avoid having to run the latter with "-u root". I didn't find a way to de-configure isar-apt unprivileged. Back to the drawing board. :( Jan
On 19.01.24 08:40, Jan Kiszka wrote: > On 19.01.24 08:37, Uladzimir Bely wrote: >> On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: >>> On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: >>>> From: Jan Kiszka <jan.kiszka@siemens.com> >>>> >>>> During the migration to sbuild, support for using self-built >>>> packages >>>> in >>>> the build environment was lost if those were already part of the >>>> sbuild-chroot. This restores it by adding --apt-distupgrade to the >>>> sbuild call. But that is not enough because sbuild will only >>>> upgrade >>>> packages from already configured sources, not those specified via >>>> --extra-repository. We therefore have to switch back to configuring >>>> isar-apt during sbuild-chroot creation. >>>> >>>> As rootfs_configure_isar_apt configures the isar repo under >>>> /isar-apt, we bind-mount the one in /home/builder/${PN} to that >>>> folder. >>>> Another difference is that we now need to run apt-get update >>>> explicitly, >>>> but only for isar-apt. >>>> >>>> For apt fetching, we neither need nor want isar-apt to be >>>> available. >>>> Rebuilding self-generated apt packages is generally pointless as >>>> the >>>> needs are better addressed in the generating recipe itself. >>>> Exposing >>>> isar-apt to the fetch may furthermore lead to fetching a previously >>>> built source package of the same recipe, rather than pulling the >>>> external version. >>>> >>>> And because the sbuild-chroots are now left behind with isar-apt >>>> configured, the configuration in imager_run can be removed (credits >>>> to >>>> Srinuvasan Arjunan). >>>> >>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>>> --- >>>> >>>> Changes in v6: >>>> - basically, going back to v4 but disabling isar-apt in >>>> do_apt_fetch >>>> >>>> I was able to resolve the scenario that Uladzimir was sharing this >>>> way. >>>> And I also realized that apt fetching as well as unpacking should >>>> have >>>> no deal with isar-apt. So, v5 was taking a wrong turn. >>>> >>>> meta/classes/dpkg-base.bbclass | 29 +++++++++++---- >>>> -- >>>> -- >>>> meta/classes/dpkg.bbclass | 5 ++-- >>>> meta/classes/image-tools-extension.bbclass | 13 --------- >>>> .../sbuild-chroot/sbuild-chroot.inc | 2 -- >>>> 4 files changed, 20 insertions(+), 29 deletions(-) >>>> >>>> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- >>>> base.bbclass >>>> index 7b054d3f..80686677 100644 >>>> --- a/meta/classes/dpkg-base.bbclass >>>> +++ b/meta/classes/dpkg-base.bbclass >>>> @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild- >>>> essential-riscv64' if d.getVar('ISAR_C >>>> DEB_BUILD_PROFILES ?= "" >>>> DEB_BUILD_OPTIONS ?= "" >>>> >>>> -ISAR_APT_REPO ?= "deb [trusted=yes] >>>> file:///home/builder/${PN}/isar- >>>> apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" >>>> - >>>> python do_adjust_git() { >>>> import subprocess >>>> >>>> @@ -115,6 +113,8 @@ do_apt_fetch() { >>>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>>> trap 'schroot_cleanup' EXIT >>>> >>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>> + rm /etc/apt/sources.list.d/isar-apt.list >>>> /etc/apt/preferences.d/isar-apt >>> >>> The issue is still reproducible with the steps I posted below, as >>> well >>> as CI still fails with v6. >>> I think, the steps above are not sufficient to completely stop >>> dealing >>> with isar-apt in do_apt_fetch() and we need some kind of 'apt update' >>> here to consider removed list and preferences. The question is how to >>> say apt to forget about isar-apt and do not run apt update for other >>> source lists... maybe, simply remove isar-apt files from >>> /var/lib/apt/lists/. >> >> An addition: >> >> It seems that files are not really removed from the session for some >> reason. I added some kind of "ls" near remove code: >> >> ``` >> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >> + sh -c 'ls -la /etc/apt/sources.list.d/' >> schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >> rm /etc/apt/sources.list.d/isar-apt.list >> /etc/apt/preferences.d/isar-apt >> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >> + sh -c 'ls -la /etc/apt/sources.list.d/' >> ``` >> , but still see the following in the log: >> ``` >> | DEBUG: Executing shell function do_apt_fetch >> | total 16 >> | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >> | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >> | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >> | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >> | total 16 >> | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >> | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >> | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >> | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >> | Reading package lists... >> ``` >> Fith the apt_fetch failure followed, because of existing of previously >> built hello source package in the isar-apt repo. >> > > OK... last-minute change of mine, splitting up the purging and the > actual fetching into two schroot runs to avoid having to run the latter > with "-u root". I didn't find a way to de-configure isar-apt > unprivileged. Back to the drawing board. :( > Comparing the split run of schroot with image-tool-extension, I think I'm just missing a proper session ID here. Jan
On 19.01.24 08:44, Jan Kiszka wrote: > On 19.01.24 08:40, Jan Kiszka wrote: >> On 19.01.24 08:37, Uladzimir Bely wrote: >>> On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: >>>> On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: >>>>> From: Jan Kiszka <jan.kiszka@siemens.com> >>>>> >>>>> During the migration to sbuild, support for using self-built >>>>> packages >>>>> in >>>>> the build environment was lost if those were already part of the >>>>> sbuild-chroot. This restores it by adding --apt-distupgrade to the >>>>> sbuild call. But that is not enough because sbuild will only >>>>> upgrade >>>>> packages from already configured sources, not those specified via >>>>> --extra-repository. We therefore have to switch back to configuring >>>>> isar-apt during sbuild-chroot creation. >>>>> >>>>> As rootfs_configure_isar_apt configures the isar repo under >>>>> /isar-apt, we bind-mount the one in /home/builder/${PN} to that >>>>> folder. >>>>> Another difference is that we now need to run apt-get update >>>>> explicitly, >>>>> but only for isar-apt. >>>>> >>>>> For apt fetching, we neither need nor want isar-apt to be >>>>> available. >>>>> Rebuilding self-generated apt packages is generally pointless as >>>>> the >>>>> needs are better addressed in the generating recipe itself. >>>>> Exposing >>>>> isar-apt to the fetch may furthermore lead to fetching a previously >>>>> built source package of the same recipe, rather than pulling the >>>>> external version. >>>>> >>>>> And because the sbuild-chroots are now left behind with isar-apt >>>>> configured, the configuration in imager_run can be removed (credits >>>>> to >>>>> Srinuvasan Arjunan). >>>>> >>>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>>>> --- >>>>> >>>>> Changes in v6: >>>>> - basically, going back to v4 but disabling isar-apt in >>>>> do_apt_fetch >>>>> >>>>> I was able to resolve the scenario that Uladzimir was sharing this >>>>> way. >>>>> And I also realized that apt fetching as well as unpacking should >>>>> have >>>>> no deal with isar-apt. So, v5 was taking a wrong turn. >>>>> >>>>> meta/classes/dpkg-base.bbclass | 29 +++++++++++---- >>>>> -- >>>>> -- >>>>> meta/classes/dpkg.bbclass | 5 ++-- >>>>> meta/classes/image-tools-extension.bbclass | 13 --------- >>>>> .../sbuild-chroot/sbuild-chroot.inc | 2 -- >>>>> 4 files changed, 20 insertions(+), 29 deletions(-) >>>>> >>>>> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- >>>>> base.bbclass >>>>> index 7b054d3f..80686677 100644 >>>>> --- a/meta/classes/dpkg-base.bbclass >>>>> +++ b/meta/classes/dpkg-base.bbclass >>>>> @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild- >>>>> essential-riscv64' if d.getVar('ISAR_C >>>>> DEB_BUILD_PROFILES ?= "" >>>>> DEB_BUILD_OPTIONS ?= "" >>>>> >>>>> -ISAR_APT_REPO ?= "deb [trusted=yes] >>>>> file:///home/builder/${PN}/isar- >>>>> apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" >>>>> - >>>>> python do_adjust_git() { >>>>> import subprocess >>>>> >>>>> @@ -115,6 +113,8 @@ do_apt_fetch() { >>>>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>>>> trap 'schroot_cleanup' EXIT >>>>> >>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>> + rm /etc/apt/sources.list.d/isar-apt.list >>>>> /etc/apt/preferences.d/isar-apt >>>> >>>> The issue is still reproducible with the steps I posted below, as >>>> well >>>> as CI still fails with v6. >>>> I think, the steps above are not sufficient to completely stop >>>> dealing >>>> with isar-apt in do_apt_fetch() and we need some kind of 'apt update' >>>> here to consider removed list and preferences. The question is how to >>>> say apt to forget about isar-apt and do not run apt update for other >>>> source lists... maybe, simply remove isar-apt files from >>>> /var/lib/apt/lists/. >>> >>> An addition: >>> >>> It seems that files are not really removed from the session for some >>> reason. I added some kind of "ls" near remove code: >>> >>> ``` >>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>> schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>> rm /etc/apt/sources.list.d/isar-apt.list >>> /etc/apt/preferences.d/isar-apt >>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>> ``` >>> , but still see the following in the log: >>> ``` >>> | DEBUG: Executing shell function do_apt_fetch >>> | total 16 >>> | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>> | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>> | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>> | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>> | total 16 >>> | drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>> | drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>> | -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>> | -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>> | Reading package lists... >>> ``` >>> Fith the apt_fetch failure followed, because of existing of previously >>> built hello source package in the isar-apt repo. >>> >> >> OK... last-minute change of mine, splitting up the purging and the >> actual fetching into two schroot runs to avoid having to run the latter >> with "-u root". I didn't find a way to de-configure isar-apt >> unprivileged. Back to the drawing board. :( >> > > Comparing the split run of schroot with image-tool-extension, I think > I'm just missing a proper session ID here. > Before sending v7: can you try this on top? diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 80686677..1a3333ac 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -113,10 +113,13 @@ do_apt_fetch() { trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) + echo "Started session: ${session_id}" + + schroot -r -c ${session_id} -d / -u root -- \ rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -r -c ${session_id} -d / -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done schroot_delete_configs Jan
On Fri, 2024-01-19 at 08:49 +0100, Jan Kiszka wrote: > On 19.01.24 08:44, Jan Kiszka wrote: > > On 19.01.24 08:40, Jan Kiszka wrote: > > > On 19.01.24 08:37, Uladzimir Bely wrote: > > > > On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: > > > > > On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: > > > > > > From: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > > > > > > > During the migration to sbuild, support for using self- > > > > > > built > > > > > > packages > > > > > > in > > > > > > the build environment was lost if those were already part > > > > > > of the > > > > > > sbuild-chroot. This restores it by adding --apt-distupgrade > > > > > > to the > > > > > > sbuild call. But that is not enough because sbuild will > > > > > > only > > > > > > upgrade > > > > > > packages from already configured sources, not those > > > > > > specified via > > > > > > --extra-repository. We therefore have to switch back to > > > > > > configuring > > > > > > isar-apt during sbuild-chroot creation. > > > > > > > > > > > > As rootfs_configure_isar_apt configures the isar repo under > > > > > > /isar-apt, we bind-mount the one in /home/builder/${PN} to > > > > > > that > > > > > > folder. > > > > > > Another difference is that we now need to run apt-get > > > > > > update > > > > > > explicitly, > > > > > > but only for isar-apt. > > > > > > > > > > > > For apt fetching, we neither need nor want isar-apt to be > > > > > > available. > > > > > > Rebuilding self-generated apt packages is generally > > > > > > pointless as > > > > > > the > > > > > > needs are better addressed in the generating recipe itself. > > > > > > Exposing > > > > > > isar-apt to the fetch may furthermore lead to fetching a > > > > > > previously > > > > > > built source package of the same recipe, rather than > > > > > > pulling the > > > > > > external version. > > > > > > > > > > > > And because the sbuild-chroots are now left behind with > > > > > > isar-apt > > > > > > configured, the configuration in imager_run can be removed > > > > > > (credits > > > > > > to > > > > > > Srinuvasan Arjunan). > > > > > > > > > > > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > --- > > > > > > > > > > > > Changes in v6: > > > > > > - basically, going back to v4 but disabling isar-apt in > > > > > > do_apt_fetch > > > > > > > > > > > > I was able to resolve the scenario that Uladzimir was > > > > > > sharing this > > > > > > way. > > > > > > And I also realized that apt fetching as well as unpacking > > > > > > should > > > > > > have > > > > > > no deal with isar-apt. So, v5 was taking a wrong turn. > > > > > > > > > > > > meta/classes/dpkg-base.bbclass | 29 > > > > > > +++++++++++---- > > > > > > -- > > > > > > -- > > > > > > meta/classes/dpkg.bbclass | 5 ++-- > > > > > > meta/classes/image-tools-extension.bbclass | 13 ------- > > > > > > -- > > > > > > .../sbuild-chroot/sbuild-chroot.inc | 2 -- > > > > > > 4 files changed, 20 insertions(+), 29 deletions(-) > > > > > > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass > > > > > > b/meta/classes/dpkg- > > > > > > base.bbclass > > > > > > index 7b054d3f..80686677 100644 > > > > > > --- a/meta/classes/dpkg-base.bbclass > > > > > > +++ b/meta/classes/dpkg-base.bbclass > > > > > > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' > > > > > > crossbuild- > > > > > > essential-riscv64' if d.getVar('ISAR_C > > > > > > DEB_BUILD_PROFILES ?= "" > > > > > > DEB_BUILD_OPTIONS ?= "" > > > > > > > > > > > > -ISAR_APT_REPO ?= "deb [trusted=yes] > > > > > > file:///home/builder/${PN}/isar- > > > > > > apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} > > > > > > main" > > > > > > - > > > > > > python do_adjust_git() { > > > > > > import subprocess > > > > > > > > > > > > @@ -115,6 +113,8 @@ do_apt_fetch() { > > > > > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > > > > > trap 'schroot_cleanup' EXIT > > > > > > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > + rm /etc/apt/sources.list.d/isar-apt.list > > > > > > /etc/apt/preferences.d/isar-apt > > > > > > > > > > The issue is still reproducible with the steps I posted > > > > > below, as > > > > > well > > > > > as CI still fails with v6. > > > > > I think, the steps above are not sufficient to completely > > > > > stop > > > > > dealing > > > > > with isar-apt in do_apt_fetch() and we need some kind of 'apt > > > > > update' > > > > > here to consider removed list and preferences. The question > > > > > is how to > > > > > say apt to forget about isar-apt and do not run apt update > > > > > for other > > > > > source lists... maybe, simply remove isar-apt files from > > > > > /var/lib/apt/lists/. > > > > > > > > An addition: > > > > > > > > It seems that files are not really removed from the session for > > > > some > > > > reason. I added some kind of "ls" near remove code: > > > > > > > > ``` > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > rm /etc/apt/sources.list.d/isar-apt.list > > > > /etc/apt/preferences.d/isar-apt > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > ``` > > > > , but still see the following in the log: > > > > ``` > > > > > DEBUG: Executing shell function do_apt_fetch > > > > > total 16 > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > > > > > total 16 > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > > > > > Reading package lists... > > > > ``` > > > > Fith the apt_fetch failure followed, because of existing of > > > > previously > > > > built hello source package in the isar-apt repo. > > > > > > > > > > OK... last-minute change of mine, splitting up the purging and > > > the > > > actual fetching into two schroot runs to avoid having to run the > > > latter > > > with "-u root". I didn't find a way to de-configure isar-apt > > > unprivileged. Back to the drawing board. :( > > > > > > > Comparing the split run of schroot with image-tool-extension, I > > think > > I'm just missing a proper session ID here. > > > > Before sending v7: can you try this on top? > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > base.bbclass > index 80686677..1a3333ac 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -113,10 +113,13 @@ do_apt_fetch() { > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > trap 'schroot_cleanup' EXIT > > - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > + echo "Started session: ${session_id}" > + > + schroot -r -c ${session_id} -d / -u root -- \ > rm /etc/apt/sources.list.d/isar-apt.list > /etc/apt/preferences.d/isar-apt > for uri in "${SRC_APT}"; do > - schroot -d / -c ${SBUILD_CHROOT} -- \ > + schroot -r -c ${session_id} -d / -- \ > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- > source source "$2"' my_script "${BASE_DISTRO}- > ${BASE_DISTRO_CODENAME}" "${uri}" > done > schroot_delete_configs > > Jan > Yes, with persistent session it works, at least with reproduce steps I used. I just added session cleanup (e.g, "schroot -e -c ${session_id}") at the end. Before sending v7 I could also check if it is not reproduced anymore in CI. Also, due to using schroot session we probably also need to care about proper session removing in case commands under schroot fail, similar to how it's done in image-tools-extention by imager_cleanup().
On 19.01.24 10:15, Uladzimir Bely wrote: > On Fri, 2024-01-19 at 08:49 +0100, Jan Kiszka wrote: >> On 19.01.24 08:44, Jan Kiszka wrote: >>> On 19.01.24 08:40, Jan Kiszka wrote: >>>> On 19.01.24 08:37, Uladzimir Bely wrote: >>>>> On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: >>>>>> On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: >>>>>>> From: Jan Kiszka <jan.kiszka@siemens.com> >>>>>>> >>>>>>> During the migration to sbuild, support for using self- >>>>>>> built >>>>>>> packages >>>>>>> in >>>>>>> the build environment was lost if those were already part >>>>>>> of the >>>>>>> sbuild-chroot. This restores it by adding --apt-distupgrade >>>>>>> to the >>>>>>> sbuild call. But that is not enough because sbuild will >>>>>>> only >>>>>>> upgrade >>>>>>> packages from already configured sources, not those >>>>>>> specified via >>>>>>> --extra-repository. We therefore have to switch back to >>>>>>> configuring >>>>>>> isar-apt during sbuild-chroot creation. >>>>>>> >>>>>>> As rootfs_configure_isar_apt configures the isar repo under >>>>>>> /isar-apt, we bind-mount the one in /home/builder/${PN} to >>>>>>> that >>>>>>> folder. >>>>>>> Another difference is that we now need to run apt-get >>>>>>> update >>>>>>> explicitly, >>>>>>> but only for isar-apt. >>>>>>> >>>>>>> For apt fetching, we neither need nor want isar-apt to be >>>>>>> available. >>>>>>> Rebuilding self-generated apt packages is generally >>>>>>> pointless as >>>>>>> the >>>>>>> needs are better addressed in the generating recipe itself. >>>>>>> Exposing >>>>>>> isar-apt to the fetch may furthermore lead to fetching a >>>>>>> previously >>>>>>> built source package of the same recipe, rather than >>>>>>> pulling the >>>>>>> external version. >>>>>>> >>>>>>> And because the sbuild-chroots are now left behind with >>>>>>> isar-apt >>>>>>> configured, the configuration in imager_run can be removed >>>>>>> (credits >>>>>>> to >>>>>>> Srinuvasan Arjunan). >>>>>>> >>>>>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>>>>>> --- >>>>>>> >>>>>>> Changes in v6: >>>>>>> - basically, going back to v4 but disabling isar-apt in >>>>>>> do_apt_fetch >>>>>>> >>>>>>> I was able to resolve the scenario that Uladzimir was >>>>>>> sharing this >>>>>>> way. >>>>>>> And I also realized that apt fetching as well as unpacking >>>>>>> should >>>>>>> have >>>>>>> no deal with isar-apt. So, v5 was taking a wrong turn. >>>>>>> >>>>>>> meta/classes/dpkg-base.bbclass | 29 >>>>>>> +++++++++++---- >>>>>>> -- >>>>>>> -- >>>>>>> meta/classes/dpkg.bbclass | 5 ++-- >>>>>>> meta/classes/image-tools-extension.bbclass | 13 ------- >>>>>>> -- >>>>>>> .../sbuild-chroot/sbuild-chroot.inc | 2 -- >>>>>>> 4 files changed, 20 insertions(+), 29 deletions(-) >>>>>>> >>>>>>> diff --git a/meta/classes/dpkg-base.bbclass >>>>>>> b/meta/classes/dpkg- >>>>>>> base.bbclass >>>>>>> index 7b054d3f..80686677 100644 >>>>>>> --- a/meta/classes/dpkg-base.bbclass >>>>>>> +++ b/meta/classes/dpkg-base.bbclass >>>>>>> @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' >>>>>>> crossbuild- >>>>>>> essential-riscv64' if d.getVar('ISAR_C >>>>>>> DEB_BUILD_PROFILES ?= "" >>>>>>> DEB_BUILD_OPTIONS ?= "" >>>>>>> >>>>>>> -ISAR_APT_REPO ?= "deb [trusted=yes] >>>>>>> file:///home/builder/${PN}/isar- >>>>>>> apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} >>>>>>> main" >>>>>>> - >>>>>>> python do_adjust_git() { >>>>>>> import subprocess >>>>>>> >>>>>>> @@ -115,6 +113,8 @@ do_apt_fetch() { >>>>>>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>>>>>> trap 'schroot_cleanup' EXIT >>>>>>> >>>>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>>>> + rm /etc/apt/sources.list.d/isar-apt.list >>>>>>> /etc/apt/preferences.d/isar-apt >>>>>> >>>>>> The issue is still reproducible with the steps I posted >>>>>> below, as >>>>>> well >>>>>> as CI still fails with v6. >>>>>> I think, the steps above are not sufficient to completely >>>>>> stop >>>>>> dealing >>>>>> with isar-apt in do_apt_fetch() and we need some kind of 'apt >>>>>> update' >>>>>> here to consider removed list and preferences. The question >>>>>> is how to >>>>>> say apt to forget about isar-apt and do not run apt update >>>>>> for other >>>>>> source lists... maybe, simply remove isar-apt files from >>>>>> /var/lib/apt/lists/. >>>>> >>>>> An addition: >>>>> >>>>> It seems that files are not really removed from the session for >>>>> some >>>>> reason. I added some kind of "ls" near remove code: >>>>> >>>>> ``` >>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>>>> schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>> rm /etc/apt/sources.list.d/isar-apt.list >>>>> /etc/apt/preferences.d/isar-apt >>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>>>> ``` >>>>> , but still see the following in the log: >>>>> ``` >>>>>> DEBUG: Executing shell function do_apt_fetch >>>>>> total 16 >>>>>> drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>>>>> drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>>>>> -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>>>>> -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>>>>> total 16 >>>>>> drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>>>>> drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>>>>> -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>>>>> -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>>>>> Reading package lists... >>>>> ``` >>>>> Fith the apt_fetch failure followed, because of existing of >>>>> previously >>>>> built hello source package in the isar-apt repo. >>>>> >>>> >>>> OK... last-minute change of mine, splitting up the purging and >>>> the >>>> actual fetching into two schroot runs to avoid having to run the >>>> latter >>>> with "-u root". I didn't find a way to de-configure isar-apt >>>> unprivileged. Back to the drawing board. :( >>>> >>> >>> Comparing the split run of schroot with image-tool-extension, I >>> think >>> I'm just missing a proper session ID here. >>> >> >> Before sending v7: can you try this on top? >> >> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- >> base.bbclass >> index 80686677..1a3333ac 100644 >> --- a/meta/classes/dpkg-base.bbclass >> +++ b/meta/classes/dpkg-base.bbclass >> @@ -113,10 +113,13 @@ do_apt_fetch() { >> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >> trap 'schroot_cleanup' EXIT >> >> - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >> + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) >> + echo "Started session: ${session_id}" >> + >> + schroot -r -c ${session_id} -d / -u root -- \ >> rm /etc/apt/sources.list.d/isar-apt.list >> /etc/apt/preferences.d/isar-apt >> for uri in "${SRC_APT}"; do >> - schroot -d / -c ${SBUILD_CHROOT} -- \ >> + schroot -r -c ${session_id} -d / -- \ >> sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd >> /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- >> source source "$2"' my_script "${BASE_DISTRO}- >> ${BASE_DISTRO_CODENAME}" "${uri}" >> done >> schroot_delete_configs >> >> Jan >> > > Yes, with persistent session it works, at least with reproduce steps I > used. I just added session cleanup (e.g, "schroot -e -c ${session_id}") > at the end. > > Before sending v7 I could also check if it is not reproduced anymore in > CI. > > Also, due to using schroot session we probably also need to care about > proper session removing in case commands under schroot fail, similar to > how it's done in image-tools-extention by imager_cleanup(). Yeah, still need to check that - schroot is also still new to me, learned a lot about it these days. Jan
On 19.01.24 10:42, Jan Kiszka wrote: > On 19.01.24 10:15, Uladzimir Bely wrote: >> On Fri, 2024-01-19 at 08:49 +0100, Jan Kiszka wrote: >>> On 19.01.24 08:44, Jan Kiszka wrote: >>>> On 19.01.24 08:40, Jan Kiszka wrote: >>>>> On 19.01.24 08:37, Uladzimir Bely wrote: >>>>>> On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: >>>>>>> On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: >>>>>>>> From: Jan Kiszka <jan.kiszka@siemens.com> >>>>>>>> >>>>>>>> During the migration to sbuild, support for using self- >>>>>>>> built >>>>>>>> packages >>>>>>>> in >>>>>>>> the build environment was lost if those were already part >>>>>>>> of the >>>>>>>> sbuild-chroot. This restores it by adding --apt-distupgrade >>>>>>>> to the >>>>>>>> sbuild call. But that is not enough because sbuild will >>>>>>>> only >>>>>>>> upgrade >>>>>>>> packages from already configured sources, not those >>>>>>>> specified via >>>>>>>> --extra-repository. We therefore have to switch back to >>>>>>>> configuring >>>>>>>> isar-apt during sbuild-chroot creation. >>>>>>>> >>>>>>>> As rootfs_configure_isar_apt configures the isar repo under >>>>>>>> /isar-apt, we bind-mount the one in /home/builder/${PN} to >>>>>>>> that >>>>>>>> folder. >>>>>>>> Another difference is that we now need to run apt-get >>>>>>>> update >>>>>>>> explicitly, >>>>>>>> but only for isar-apt. >>>>>>>> >>>>>>>> For apt fetching, we neither need nor want isar-apt to be >>>>>>>> available. >>>>>>>> Rebuilding self-generated apt packages is generally >>>>>>>> pointless as >>>>>>>> the >>>>>>>> needs are better addressed in the generating recipe itself. >>>>>>>> Exposing >>>>>>>> isar-apt to the fetch may furthermore lead to fetching a >>>>>>>> previously >>>>>>>> built source package of the same recipe, rather than >>>>>>>> pulling the >>>>>>>> external version. >>>>>>>> >>>>>>>> And because the sbuild-chroots are now left behind with >>>>>>>> isar-apt >>>>>>>> configured, the configuration in imager_run can be removed >>>>>>>> (credits >>>>>>>> to >>>>>>>> Srinuvasan Arjunan). >>>>>>>> >>>>>>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>>>>>>> --- >>>>>>>> >>>>>>>> Changes in v6: >>>>>>>> - basically, going back to v4 but disabling isar-apt in >>>>>>>> do_apt_fetch >>>>>>>> >>>>>>>> I was able to resolve the scenario that Uladzimir was >>>>>>>> sharing this >>>>>>>> way. >>>>>>>> And I also realized that apt fetching as well as unpacking >>>>>>>> should >>>>>>>> have >>>>>>>> no deal with isar-apt. So, v5 was taking a wrong turn. >>>>>>>> >>>>>>>> meta/classes/dpkg-base.bbclass | 29 >>>>>>>> +++++++++++---- >>>>>>>> -- >>>>>>>> -- >>>>>>>> meta/classes/dpkg.bbclass | 5 ++-- >>>>>>>> meta/classes/image-tools-extension.bbclass | 13 ------- >>>>>>>> -- >>>>>>>> .../sbuild-chroot/sbuild-chroot.inc | 2 -- >>>>>>>> 4 files changed, 20 insertions(+), 29 deletions(-) >>>>>>>> >>>>>>>> diff --git a/meta/classes/dpkg-base.bbclass >>>>>>>> b/meta/classes/dpkg- >>>>>>>> base.bbclass >>>>>>>> index 7b054d3f..80686677 100644 >>>>>>>> --- a/meta/classes/dpkg-base.bbclass >>>>>>>> +++ b/meta/classes/dpkg-base.bbclass >>>>>>>> @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' >>>>>>>> crossbuild- >>>>>>>> essential-riscv64' if d.getVar('ISAR_C >>>>>>>> DEB_BUILD_PROFILES ?= "" >>>>>>>> DEB_BUILD_OPTIONS ?= "" >>>>>>>> >>>>>>>> -ISAR_APT_REPO ?= "deb [trusted=yes] >>>>>>>> file:///home/builder/${PN}/isar- >>>>>>>> apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} >>>>>>>> main" >>>>>>>> - >>>>>>>> python do_adjust_git() { >>>>>>>> import subprocess >>>>>>>> >>>>>>>> @@ -115,6 +113,8 @@ do_apt_fetch() { >>>>>>>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>>>>>>> trap 'schroot_cleanup' EXIT >>>>>>>> >>>>>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>>>>> + rm /etc/apt/sources.list.d/isar-apt.list >>>>>>>> /etc/apt/preferences.d/isar-apt >>>>>>> >>>>>>> The issue is still reproducible with the steps I posted >>>>>>> below, as >>>>>>> well >>>>>>> as CI still fails with v6. >>>>>>> I think, the steps above are not sufficient to completely >>>>>>> stop >>>>>>> dealing >>>>>>> with isar-apt in do_apt_fetch() and we need some kind of 'apt >>>>>>> update' >>>>>>> here to consider removed list and preferences. The question >>>>>>> is how to >>>>>>> say apt to forget about isar-apt and do not run apt update >>>>>>> for other >>>>>>> source lists... maybe, simply remove isar-apt files from >>>>>>> /var/lib/apt/lists/. >>>>>> >>>>>> An addition: >>>>>> >>>>>> It seems that files are not really removed from the session for >>>>>> some >>>>>> reason. I added some kind of "ls" near remove code: >>>>>> >>>>>> ``` >>>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>>>>> schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>>> rm /etc/apt/sources.list.d/isar-apt.list >>>>>> /etc/apt/preferences.d/isar-apt >>>>>> + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>>>>> + sh -c 'ls -la /etc/apt/sources.list.d/' >>>>>> ``` >>>>>> , but still see the following in the log: >>>>>> ``` >>>>>>> DEBUG: Executing shell function do_apt_fetch >>>>>>> total 16 >>>>>>> drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>>>>>> drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>>>>>> -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>>>>>> -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>>>>>> total 16 >>>>>>> drwxr-xr-x 2 root root 4096 Jan 19 07:09 . >>>>>>> drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. >>>>>>> -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list >>>>>>> -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list >>>>>>> Reading package lists... >>>>>> ``` >>>>>> Fith the apt_fetch failure followed, because of existing of >>>>>> previously >>>>>> built hello source package in the isar-apt repo. >>>>>> >>>>> >>>>> OK... last-minute change of mine, splitting up the purging and >>>>> the >>>>> actual fetching into two schroot runs to avoid having to run the >>>>> latter >>>>> with "-u root". I didn't find a way to de-configure isar-apt >>>>> unprivileged. Back to the drawing board. :( >>>>> >>>> >>>> Comparing the split run of schroot with image-tool-extension, I >>>> think >>>> I'm just missing a proper session ID here. >>>> >>> >>> Before sending v7: can you try this on top? >>> >>> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- >>> base.bbclass >>> index 80686677..1a3333ac 100644 >>> --- a/meta/classes/dpkg-base.bbclass >>> +++ b/meta/classes/dpkg-base.bbclass >>> @@ -113,10 +113,13 @@ do_apt_fetch() { >>> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 >>> trap 'schroot_cleanup' EXIT >>> >>> - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ >>> + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) >>> + echo "Started session: ${session_id}" >>> + >>> + schroot -r -c ${session_id} -d / -u root -- \ >>> rm /etc/apt/sources.list.d/isar-apt.list >>> /etc/apt/preferences.d/isar-apt >>> for uri in "${SRC_APT}"; do >>> - schroot -d / -c ${SBUILD_CHROOT} -- \ >>> + schroot -r -c ${session_id} -d / -- \ >>> sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd >>> /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- >>> source source "$2"' my_script "${BASE_DISTRO}- >>> ${BASE_DISTRO_CODENAME}" "${uri}" >>> done >>> schroot_delete_configs >>> >>> Jan >>> >> >> Yes, with persistent session it works, at least with reproduce steps I >> used. I just added session cleanup (e.g, "schroot -e -c ${session_id}") >> at the end. >> >> Before sending v7 I could also check if it is not reproduced anymore in >> CI. >> >> Also, due to using schroot session we probably also need to care about >> proper session removing in case commands under schroot fail, similar to >> how it's done in image-tools-extention by imager_cleanup(). > > Yeah, still need to check that - schroot is also still new to me, > learned a lot about it these days. > > Jan > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 80686677..85d0a495 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -107,18 +107,24 @@ do_apt_fetch() { E="${@ isar_export_proxies(d)}" schroot_create_configs + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) + echo "Started session: ${session_id}" + schroot_cleanup() { + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 schroot_delete_configs } trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ + schroot -r -c ${session_id} -d / -u root -- \ rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -r -c ${session_id} -d / -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done + + schroot -e -c ${session_id} schroot_delete_configs } Jan
On Fri, 2024-01-19 at 10:49 +0100, Jan Kiszka wrote: > On 19.01.24 10:42, Jan Kiszka wrote: > > On 19.01.24 10:15, Uladzimir Bely wrote: > > > On Fri, 2024-01-19 at 08:49 +0100, Jan Kiszka wrote: > > > > On 19.01.24 08:44, Jan Kiszka wrote: > > > > > On 19.01.24 08:40, Jan Kiszka wrote: > > > > > > On 19.01.24 08:37, Uladzimir Bely wrote: > > > > > > > On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely wrote: > > > > > > > > On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: > > > > > > > > > From: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > > > > > > > > > > > > > During the migration to sbuild, support for using > > > > > > > > > self- > > > > > > > > > built > > > > > > > > > packages > > > > > > > > > in > > > > > > > > > the build environment was lost if those were already > > > > > > > > > part > > > > > > > > > of the > > > > > > > > > sbuild-chroot. This restores it by adding --apt- > > > > > > > > > distupgrade > > > > > > > > > to the > > > > > > > > > sbuild call. But that is not enough because sbuild > > > > > > > > > will > > > > > > > > > only > > > > > > > > > upgrade > > > > > > > > > packages from already configured sources, not those > > > > > > > > > specified via > > > > > > > > > --extra-repository. We therefore have to switch back > > > > > > > > > to > > > > > > > > > configuring > > > > > > > > > isar-apt during sbuild-chroot creation. > > > > > > > > > > > > > > > > > > As rootfs_configure_isar_apt configures the isar repo > > > > > > > > > under > > > > > > > > > /isar-apt, we bind-mount the one in > > > > > > > > > /home/builder/${PN} to > > > > > > > > > that > > > > > > > > > folder. > > > > > > > > > Another difference is that we now need to run apt-get > > > > > > > > > update > > > > > > > > > explicitly, > > > > > > > > > but only for isar-apt. > > > > > > > > > > > > > > > > > > For apt fetching, we neither need nor want isar-apt > > > > > > > > > to be > > > > > > > > > available. > > > > > > > > > Rebuilding self-generated apt packages is generally > > > > > > > > > pointless as > > > > > > > > > the > > > > > > > > > needs are better addressed in the generating recipe > > > > > > > > > itself. > > > > > > > > > Exposing > > > > > > > > > isar-apt to the fetch may furthermore lead to > > > > > > > > > fetching a > > > > > > > > > previously > > > > > > > > > built source package of the same recipe, rather than > > > > > > > > > pulling the > > > > > > > > > external version. > > > > > > > > > > > > > > > > > > And because the sbuild-chroots are now left behind > > > > > > > > > with > > > > > > > > > isar-apt > > > > > > > > > configured, the configuration in imager_run can be > > > > > > > > > removed > > > > > > > > > (credits > > > > > > > > > to > > > > > > > > > Srinuvasan Arjunan). > > > > > > > > > > > > > > > > > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > > > > --- > > > > > > > > > > > > > > > > > > Changes in v6: > > > > > > > > > - basically, going back to v4 but disabling isar-apt > > > > > > > > > in > > > > > > > > > do_apt_fetch > > > > > > > > > > > > > > > > > > I was able to resolve the scenario that Uladzimir was > > > > > > > > > sharing this > > > > > > > > > way. > > > > > > > > > And I also realized that apt fetching as well as > > > > > > > > > unpacking > > > > > > > > > should > > > > > > > > > have > > > > > > > > > no deal with isar-apt. So, v5 was taking a wrong > > > > > > > > > turn. > > > > > > > > > > > > > > > > > > meta/classes/dpkg-base.bbclass | 29 > > > > > > > > > +++++++++++---- > > > > > > > > > -- > > > > > > > > > -- > > > > > > > > > meta/classes/dpkg.bbclass | 5 > > > > > > > > > ++-- > > > > > > > > > meta/classes/image-tools-extension.bbclass | 13 - > > > > > > > > > ------ > > > > > > > > > -- > > > > > > > > > .../sbuild-chroot/sbuild-chroot.inc | 2 - > > > > > > > > > - > > > > > > > > > 4 files changed, 20 insertions(+), 29 deletions(-) > > > > > > > > > > > > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass > > > > > > > > > b/meta/classes/dpkg- > > > > > > > > > base.bbclass > > > > > > > > > index 7b054d3f..80686677 100644 > > > > > > > > > --- a/meta/classes/dpkg-base.bbclass > > > > > > > > > +++ b/meta/classes/dpkg-base.bbclass > > > > > > > > > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' > > > > > > > > > crossbuild- > > > > > > > > > essential-riscv64' if d.getVar('ISAR_C > > > > > > > > > DEB_BUILD_PROFILES ?= "" > > > > > > > > > DEB_BUILD_OPTIONS ?= "" > > > > > > > > > > > > > > > > > > -ISAR_APT_REPO ?= "deb [trusted=yes] > > > > > > > > > file:///home/builder/${PN}/isar- > > > > > > > > > apt/${DISTRO}- > > > > > > > > > ${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} > > > > > > > > > main" > > > > > > > > > - > > > > > > > > > python do_adjust_git() { > > > > > > > > > import subprocess > > > > > > > > > > > > > > > > > > @@ -115,6 +113,8 @@ do_apt_fetch() { > > > > > > > > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > > > > > > > > trap 'schroot_cleanup' EXIT > > > > > > > > > > > > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > > > + rm /etc/apt/sources.list.d/isar-apt.list > > > > > > > > > /etc/apt/preferences.d/isar-apt > > > > > > > > > > > > > > > > The issue is still reproducible with the steps I posted > > > > > > > > below, as > > > > > > > > well > > > > > > > > as CI still fails with v6. > > > > > > > > I think, the steps above are not sufficient to > > > > > > > > completely > > > > > > > > stop > > > > > > > > dealing > > > > > > > > with isar-apt in do_apt_fetch() and we need some kind > > > > > > > > of 'apt > > > > > > > > update' > > > > > > > > here to consider removed list and preferences. The > > > > > > > > question > > > > > > > > is how to > > > > > > > > say apt to forget about isar-apt and do not run apt > > > > > > > > update > > > > > > > > for other > > > > > > > > source lists... maybe, simply remove isar-apt files > > > > > > > > from > > > > > > > > /var/lib/apt/lists/. > > > > > > > > > > > > > > An addition: > > > > > > > > > > > > > > It seems that files are not really removed from the > > > > > > > session for > > > > > > > some > > > > > > > reason. I added some kind of "ls" near remove code: > > > > > > > > > > > > > > ``` > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > > > > schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > rm /etc/apt/sources.list.d/isar-apt.list > > > > > > > /etc/apt/preferences.d/isar-apt > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > > > > ``` > > > > > > > , but still see the following in the log: > > > > > > > ``` > > > > > > > > DEBUG: Executing shell function do_apt_fetch > > > > > > > > total 16 > > > > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > > > > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > > > > > > > > total 16 > > > > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 bootstrap.list > > > > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar-apt.list > > > > > > > > Reading package lists... > > > > > > > ``` > > > > > > > Fith the apt_fetch failure followed, because of existing > > > > > > > of > > > > > > > previously > > > > > > > built hello source package in the isar-apt repo. > > > > > > > > > > > > > > > > > > > OK... last-minute change of mine, splitting up the purging > > > > > > and > > > > > > the > > > > > > actual fetching into two schroot runs to avoid having to > > > > > > run the > > > > > > latter > > > > > > with "-u root". I didn't find a way to de-configure isar- > > > > > > apt > > > > > > unprivileged. Back to the drawing board. :( > > > > > > > > > > > > > > > > Comparing the split run of schroot with image-tool-extension, > > > > > I > > > > > think > > > > > I'm just missing a proper session ID here. > > > > > > > > > > > > > Before sending v7: can you try this on top? > > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass > > > > b/meta/classes/dpkg- > > > > base.bbclass > > > > index 80686677..1a3333ac 100644 > > > > --- a/meta/classes/dpkg-base.bbclass > > > > +++ b/meta/classes/dpkg-base.bbclass > > > > @@ -113,10 +113,13 @@ do_apt_fetch() { > > > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > > > trap 'schroot_cleanup' EXIT > > > > > > > > - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > > > > + echo "Started session: ${session_id}" > > > > + > > > > + schroot -r -c ${session_id} -d / -u root -- \ > > > > rm /etc/apt/sources.list.d/isar-apt.list > > > > /etc/apt/preferences.d/isar-apt > > > > for uri in "${SRC_APT}"; do > > > > - schroot -d / -c ${SBUILD_CHROOT} -- \ > > > > + schroot -r -c ${session_id} -d / -- \ > > > > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > > > > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only -- > > > > only- > > > > source source "$2"' my_script "${BASE_DISTRO}- > > > > ${BASE_DISTRO_CODENAME}" "${uri}" > > > > done > > > > schroot_delete_configs > > > > > > > > Jan > > > > > > > > > > Yes, with persistent session it works, at least with reproduce > > > steps I > > > used. I just added session cleanup (e.g, "schroot -e -c > > > ${session_id}") > > > at the end. > > > > > > Before sending v7 I could also check if it is not reproduced > > > anymore in > > > CI. > > > > > > Also, due to using schroot session we probably also need to care > > > about > > > proper session removing in case commands under schroot fail, > > > similar to > > > how it's done in image-tools-extention by imager_cleanup(). > > > > Yeah, still need to check that - schroot is also still new to me, > > learned a lot about it these days. > > > > Jan > > > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > base.bbclass > index 80686677..85d0a495 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -107,18 +107,24 @@ do_apt_fetch() { > E="${@ isar_export_proxies(d)}" > schroot_create_configs > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > + echo "Started session: ${session_id}" > + > schroot_cleanup() { > + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 > schroot_delete_configs > } > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > trap 'schroot_cleanup' EXIT > > - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > + schroot -r -c ${session_id} -d / -u root -- \ > rm /etc/apt/sources.list.d/isar-apt.list > /etc/apt/preferences.d/isar-apt > for uri in "${SRC_APT}"; do > - schroot -d / -c ${SBUILD_CHROOT} -- \ > + schroot -r -c ${session_id} -d / -- \ > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- > source source "$2"' my_script "${BASE_DISTRO}- > ${BASE_DISTRO_CODENAME}" "${uri}" > done > + > + schroot -e -c ${session_id} > schroot_delete_configs > } > > > Jan > Hello. Checked in CI during the weekend - the issue seems not to be reproduced anymore. So, v7 should be OK.
On Mon, 2024-01-22 at 10:59 +0300, Uladzimir Bely wrote: > On Fri, 2024-01-19 at 10:49 +0100, Jan Kiszka wrote: > > On 19.01.24 10:42, Jan Kiszka wrote: > > > On 19.01.24 10:15, Uladzimir Bely wrote: > > > > On Fri, 2024-01-19 at 08:49 +0100, Jan Kiszka wrote: > > > > > On 19.01.24 08:44, Jan Kiszka wrote: > > > > > > On 19.01.24 08:40, Jan Kiszka wrote: > > > > > > > On 19.01.24 08:37, Uladzimir Bely wrote: > > > > > > > > On Fri, 2024-01-19 at 10:20 +0300, Uladzimir Bely > > > > > > > > wrote: > > > > > > > > > On Thu, 2024-01-18 at 14:50 +0100, Jan Kiszka wrote: > > > > > > > > > > From: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > > > > > > > > > > > > > > > During the migration to sbuild, support for using > > > > > > > > > > self- > > > > > > > > > > built > > > > > > > > > > packages > > > > > > > > > > in > > > > > > > > > > the build environment was lost if those were > > > > > > > > > > already > > > > > > > > > > part > > > > > > > > > > of the > > > > > > > > > > sbuild-chroot. This restores it by adding --apt- > > > > > > > > > > distupgrade > > > > > > > > > > to the > > > > > > > > > > sbuild call. But that is not enough because sbuild > > > > > > > > > > will > > > > > > > > > > only > > > > > > > > > > upgrade > > > > > > > > > > packages from already configured sources, not those > > > > > > > > > > specified via > > > > > > > > > > --extra-repository. We therefore have to switch > > > > > > > > > > back > > > > > > > > > > to > > > > > > > > > > configuring > > > > > > > > > > isar-apt during sbuild-chroot creation. > > > > > > > > > > > > > > > > > > > > As rootfs_configure_isar_apt configures the isar > > > > > > > > > > repo > > > > > > > > > > under > > > > > > > > > > /isar-apt, we bind-mount the one in > > > > > > > > > > /home/builder/${PN} to > > > > > > > > > > that > > > > > > > > > > folder. > > > > > > > > > > Another difference is that we now need to run apt- > > > > > > > > > > get > > > > > > > > > > update > > > > > > > > > > explicitly, > > > > > > > > > > but only for isar-apt. > > > > > > > > > > > > > > > > > > > > For apt fetching, we neither need nor want isar-apt > > > > > > > > > > to be > > > > > > > > > > available. > > > > > > > > > > Rebuilding self-generated apt packages is generally > > > > > > > > > > pointless as > > > > > > > > > > the > > > > > > > > > > needs are better addressed in the generating recipe > > > > > > > > > > itself. > > > > > > > > > > Exposing > > > > > > > > > > isar-apt to the fetch may furthermore lead to > > > > > > > > > > fetching a > > > > > > > > > > previously > > > > > > > > > > built source package of the same recipe, rather > > > > > > > > > > than > > > > > > > > > > pulling the > > > > > > > > > > external version. > > > > > > > > > > > > > > > > > > > > And because the sbuild-chroots are now left behind > > > > > > > > > > with > > > > > > > > > > isar-apt > > > > > > > > > > configured, the configuration in imager_run can be > > > > > > > > > > removed > > > > > > > > > > (credits > > > > > > > > > > to > > > > > > > > > > Srinuvasan Arjunan). > > > > > > > > > > > > > > > > > > > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > > > > > > > > > > --- > > > > > > > > > > > > > > > > > > > > Changes in v6: > > > > > > > > > > - basically, going back to v4 but disabling isar- > > > > > > > > > > apt > > > > > > > > > > in > > > > > > > > > > do_apt_fetch > > > > > > > > > > > > > > > > > > > > I was able to resolve the scenario that Uladzimir > > > > > > > > > > was > > > > > > > > > > sharing this > > > > > > > > > > way. > > > > > > > > > > And I also realized that apt fetching as well as > > > > > > > > > > unpacking > > > > > > > > > > should > > > > > > > > > > have > > > > > > > > > > no deal with isar-apt. So, v5 was taking a wrong > > > > > > > > > > turn. > > > > > > > > > > > > > > > > > > > > meta/classes/dpkg-base.bbclass | 29 > > > > > > > > > > +++++++++++---- > > > > > > > > > > -- > > > > > > > > > > -- > > > > > > > > > > meta/classes/dpkg.bbclass | 5 > > > > > > > > > > ++-- > > > > > > > > > > meta/classes/image-tools-extension.bbclass | 13 > > > > > > > > > > - > > > > > > > > > > ------ > > > > > > > > > > -- > > > > > > > > > > .../sbuild-chroot/sbuild-chroot.inc | 2 > > > > > > > > > > - > > > > > > > > > > - > > > > > > > > > > 4 files changed, 20 insertions(+), 29 deletions(-) > > > > > > > > > > > > > > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass > > > > > > > > > > b/meta/classes/dpkg- > > > > > > > > > > base.bbclass > > > > > > > > > > index 7b054d3f..80686677 100644 > > > > > > > > > > --- a/meta/classes/dpkg-base.bbclass > > > > > > > > > > +++ b/meta/classes/dpkg-base.bbclass > > > > > > > > > > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' > > > > > > > > > > crossbuild- > > > > > > > > > > essential-riscv64' if d.getVar('ISAR_C > > > > > > > > > > DEB_BUILD_PROFILES ?= "" > > > > > > > > > > DEB_BUILD_OPTIONS ?= "" > > > > > > > > > > > > > > > > > > > > -ISAR_APT_REPO ?= "deb [trusted=yes] > > > > > > > > > > file:///home/builder/${PN}/isar- > > > > > > > > > > apt/${DISTRO}- > > > > > > > > > > ${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} > > > > > > > > > > main" > > > > > > > > > > - > > > > > > > > > > python do_adjust_git() { > > > > > > > > > > import subprocess > > > > > > > > > > > > > > > > > > > > @@ -115,6 +113,8 @@ do_apt_fetch() { > > > > > > > > > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > > > > > > > > > trap 'schroot_cleanup' EXIT > > > > > > > > > > > > > > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > > > > + rm /etc/apt/sources.list.d/isar-apt.list > > > > > > > > > > /etc/apt/preferences.d/isar-apt > > > > > > > > > > > > > > > > > > The issue is still reproducible with the steps I > > > > > > > > > posted > > > > > > > > > below, as > > > > > > > > > well > > > > > > > > > as CI still fails with v6. > > > > > > > > > I think, the steps above are not sufficient to > > > > > > > > > completely > > > > > > > > > stop > > > > > > > > > dealing > > > > > > > > > with isar-apt in do_apt_fetch() and we need some kind > > > > > > > > > of 'apt > > > > > > > > > update' > > > > > > > > > here to consider removed list and preferences. The > > > > > > > > > question > > > > > > > > > is how to > > > > > > > > > say apt to forget about isar-apt and do not run apt > > > > > > > > > update > > > > > > > > > for other > > > > > > > > > source lists... maybe, simply remove isar-apt files > > > > > > > > > from > > > > > > > > > /var/lib/apt/lists/. > > > > > > > > > > > > > > > > An addition: > > > > > > > > > > > > > > > > It seems that files are not really removed from the > > > > > > > > session for > > > > > > > > some > > > > > > > > reason. I added some kind of "ls" near remove code: > > > > > > > > > > > > > > > > ``` > > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > > > > > schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > > rm /etc/apt/sources.list.d/isar-apt.list > > > > > > > > /etc/apt/preferences.d/isar-apt > > > > > > > > + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > > > > + sh -c 'ls -la /etc/apt/sources.list.d/' > > > > > > > > ``` > > > > > > > > , but still see the following in the log: > > > > > > > > ``` > > > > > > > > > DEBUG: Executing shell function do_apt_fetch > > > > > > > > > total 16 > > > > > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 > > > > > > > > > bootstrap.list > > > > > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar- > > > > > > > > > apt.list > > > > > > > > > total 16 > > > > > > > > > drwxr-xr-x 2 root root 4096 Jan 19 07:09 . > > > > > > > > > drwxr-xr-x 8 root root 4096 Jan 19 07:09 .. > > > > > > > > > -rw-r--r-- 1 root root 569 Jan 19 07:09 > > > > > > > > > bootstrap.list > > > > > > > > > -rw-r--r-- 1 root root 94 Jan 19 07:09 isar- > > > > > > > > > apt.list > > > > > > > > > Reading package lists... > > > > > > > > ``` > > > > > > > > Fith the apt_fetch failure followed, because of > > > > > > > > existing > > > > > > > > of > > > > > > > > previously > > > > > > > > built hello source package in the isar-apt repo. > > > > > > > > > > > > > > > > > > > > > > OK... last-minute change of mine, splitting up the > > > > > > > purging > > > > > > > and > > > > > > > the > > > > > > > actual fetching into two schroot runs to avoid having to > > > > > > > run the > > > > > > > latter > > > > > > > with "-u root". I didn't find a way to de-configure isar- > > > > > > > apt > > > > > > > unprivileged. Back to the drawing board. :( > > > > > > > > > > > > > > > > > > > Comparing the split run of schroot with image-tool- > > > > > > extension, > > > > > > I > > > > > > think > > > > > > I'm just missing a proper session ID here. > > > > > > > > > > > > > > > > Before sending v7: can you try this on top? > > > > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass > > > > > b/meta/classes/dpkg- > > > > > base.bbclass > > > > > index 80686677..1a3333ac 100644 > > > > > --- a/meta/classes/dpkg-base.bbclass > > > > > +++ b/meta/classes/dpkg-base.bbclass > > > > > @@ -113,10 +113,13 @@ do_apt_fetch() { > > > > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > > > > trap 'schroot_cleanup' EXIT > > > > > > > > > > - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > > > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > > > > > + echo "Started session: ${session_id}" > > > > > + > > > > > + schroot -r -c ${session_id} -d / -u root -- \ > > > > > rm /etc/apt/sources.list.d/isar-apt.list > > > > > /etc/apt/preferences.d/isar-apt > > > > > for uri in "${SRC_APT}"; do > > > > > - schroot -d / -c ${SBUILD_CHROOT} -- \ > > > > > + schroot -r -c ${session_id} -d / -- \ > > > > > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && > > > > > cd > > > > > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only -- > > > > > only- > > > > > source source "$2"' my_script "${BASE_DISTRO}- > > > > > ${BASE_DISTRO_CODENAME}" "${uri}" > > > > > done > > > > > schroot_delete_configs > > > > > > > > > > Jan > > > > > > > > > > > > > Yes, with persistent session it works, at least with reproduce > > > > steps I > > > > used. I just added session cleanup (e.g, "schroot -e -c > > > > ${session_id}") > > > > at the end. > > > > > > > > Before sending v7 I could also check if it is not reproduced > > > > anymore in > > > > CI. > > > > > > > > Also, due to using schroot session we probably also need to > > > > care > > > > about > > > > proper session removing in case commands under schroot fail, > > > > similar to > > > > how it's done in image-tools-extention by imager_cleanup(). > > > > > > Yeah, still need to check that - schroot is also still new to me, > > > learned a lot about it these days. > > > > > > Jan > > > > > > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > > base.bbclass > > index 80686677..85d0a495 100644 > > --- a/meta/classes/dpkg-base.bbclass > > +++ b/meta/classes/dpkg-base.bbclass > > @@ -107,18 +107,24 @@ do_apt_fetch() { > > E="${@ isar_export_proxies(d)}" > > schroot_create_configs > > > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > > + echo "Started session: ${session_id}" > > + > > schroot_cleanup() { > > + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 > > schroot_delete_configs > > } > > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > > trap 'schroot_cleanup' EXIT > > > > - schroot -d / -u root -c ${SBUILD_CHROOT} -- \ > > + schroot -r -c ${session_id} -d / -u root -- \ > > rm /etc/apt/sources.list.d/isar-apt.list > > /etc/apt/preferences.d/isar-apt > > for uri in "${SRC_APT}"; do > > - schroot -d / -c ${SBUILD_CHROOT} -- \ > > + schroot -r -c ${session_id} -d / -- \ > > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only- > > source source "$2"' my_script "${BASE_DISTRO}- > > ${BASE_DISTRO_CODENAME}" "${uri}" > > done > > + > > + schroot -e -c ${session_id} > > schroot_delete_configs > > } > > > > > > Jan > > > > Hello. > > Checked in CI during the weekend - the issue seems not to be > reproduced > anymore. So, v7 should be OK. > Since there is still no v7 on list, I'll send it on my own. The patch is checked in CI as well as in the downstream that requires downgrade for linux-libc-dev package. Everything looks OK.
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 7b054d3f..80686677 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C DEB_BUILD_PROFILES ?= "" DEB_BUILD_OPTIONS ?= "" -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" - python do_adjust_git() { import subprocess @@ -115,6 +113,8 @@ do_apt_fetch() { trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT + schroot -d / -u root -c ${SBUILD_CHROOT} -- \ + rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt for uri in "${SRC_APT}"; do schroot -d / -c ${SBUILD_CHROOT} -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" @@ -221,8 +221,17 @@ def isar_export_build_settings(d): os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) +dpkg_schroot_create_configs() { + schroot_create_configs + sudo -s <<'EOSUDO' + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab} +EOSUDO +} + python do_dpkg_build() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) try: bb.build.exec_func("dpkg_runbuild", d) finally: @@ -288,7 +297,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" do_deploy_deb[dirs] = "${S}" python do_devshell() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) isar_export_proxies(d) isar_export_ccache(d) @@ -297,11 +306,9 @@ python do_devshell() { bb.build.exec_func('schroot_configure_ccache', d) schroot = d.getVar('SBUILD_CHROOT') - isar_apt = d.getVar('ISAR_APT_REPO') pkg_arch = d.getVar('PACKAGE_ARCH') build_arch = d.getVar('BUILD_ARCH') pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - debdistroname = d.getVar('DEBDISTRONAME') install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \ --host-arch {pkg_arch} --build-arch {build_arch} \ @@ -310,15 +317,13 @@ python do_devshell() { termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ cd {1}; \ - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \ - echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \ - apt-get -y -q update; \ - {4}; \ + apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ + apt-get -y upgrade; \ + {2}; \ export PATH=$PATH_PREPEND:$PATH; \ $SHELL -i \ '" - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d) + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d) bb.build.exec_func('schroot_delete_configs', d) } diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index c699a84d..d1666f78 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -110,16 +110,17 @@ dpkg_runbuild() { DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog) DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print) - sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ + sbuild -A -n -c ${SBUILD_CHROOT} \ --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ --bd-uninstallable-explainer=apt \ - --no-apt-update \ + --no-apt-update --apt-distupgrade \ --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ --chroot-setup-commands="rm -f /var/log/dpkg.log" \ --chroot-setup-commands="mkdir -p ${deb_dir}" \ --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \ + --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass index 58799fcf..46bdf78b 100644 --- a/meta/classes/image-tools-extension.bbclass +++ b/meta/classes/image-tools-extension.bbclass @@ -44,19 +44,6 @@ imager_run() { distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" fi - # prepare isar-apt - schroot -r -c ${session_id} -d / -u root -- sh -c " \ - mkdir -p '/etc/apt/sources.list.d' - echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \ - '/etc/apt/sources.list.d/isar-apt.list' - - mkdir -p '/etc/apt/preferences.d' - cat << EOF > '/etc/apt/preferences.d/isar-apt' -Package: * -Pin: release n=${DEBDISTRONAME} -Pin-Priority: 1000 -EOF" - E="${@ isar_export_proxies(d)}" deb_dl_dir_import ${schroot_dir} ${distro} schroot -r -c ${session_id} -d / -u root -- sh -c " \ diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc index 39ced80b..2f07de82 100644 --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" ROOTFSDIR = "${SBUILD_CHROOT_DIR}" ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" # Do not cleanup base-apt