| Message ID | 20240127074253.24570-1-ubely@ilbers.de |
|---|---|
| State | Accepted, archived |
| Headers | show |
| Series | [v7] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot | expand |
On 27.01.24 08:42, Uladzimir Bely wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > During the migration to sbuild, support for using self-built packages in > the build environment was lost if those were already part of the > sbuild-chroot. This restores it by adding --apt-distupgrade to the > sbuild call. But that is not enough because sbuild will only upgrade > packages from already configured sources, not those specified via > --extra-repository. We therefore have to switch back to configuring > isar-apt during sbuild-chroot creation. > > As rootfs_configure_isar_apt configures the isar repo under > /isar-apt, we bind-mount the one in /home/builder/${PN} to that folder. > Another difference is that we now need to run apt-get update explicitly, > but only for isar-apt. > > For apt fetching, we neither need nor want isar-apt to be available. > Rebuilding self-generated apt packages is generally pointless as the > needs are better addressed in the generating recipe itself. Exposing > isar-apt to the fetch may furthermore lead to fetching a previously > built source package of the same recipe, rather than pulling the > external version. > > And because the sbuild-chroots are now left behind with isar-apt > configured, the configuration in imager_run can be removed (credits to > Srinuvasan Arjunan). > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > Reviewed-by: Uladzimir Bely <ubely@ilbers.de> > --- > > Changes in v7: Merged v6 with fix disabling isar-apt for do_apt_fetch > > meta/classes/dpkg-base.bbclass | 37 ++++++++++++------- > meta/classes/dpkg.bbclass | 5 ++- > meta/classes/image-tools-extension.bbclass | 13 ------- > .../sbuild-chroot/sbuild-chroot.inc | 2 - > 4 files changed, 27 insertions(+), 30 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass > index 95ca3db8..86466b57 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C > DEB_BUILD_PROFILES ?= "" > DEB_BUILD_OPTIONS ?= "" > > -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" > - > python do_adjust_git() { > import subprocess > > @@ -109,16 +107,24 @@ do_apt_fetch() { > E="${@ isar_export_proxies(d)}" > schroot_create_configs > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > + echo "Started session: ${session_id}" > + > schroot_cleanup() { > + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 > schroot_delete_configs > } > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > trap 'schroot_cleanup' EXIT > > + schroot -r -c ${session_id} -d / -u root -- \ > + rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt > for uri in "${SRC_APT}"; do > - schroot -d / -c ${SBUILD_CHROOT} -- \ > + schroot -r -c ${session_id} -d / -- \ > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" > done > + > + schroot -e -c ${session_id} > schroot_delete_configs > } > > @@ -221,8 +227,17 @@ def isar_export_build_settings(d): > os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) > os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) > > +dpkg_schroot_create_configs() { > + schroot_create_configs > + sudo -s <<'EOSUDO' > + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" > + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" > + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab} > +EOSUDO > +} > + > python do_dpkg_build() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > try: > bb.build.exec_func("dpkg_runbuild", d) > finally: > @@ -288,7 +303,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" > do_deploy_deb[dirs] = "${S}" > > python do_devshell() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > > isar_export_proxies(d) > isar_export_ccache(d) > @@ -297,11 +312,9 @@ python do_devshell() { > bb.build.exec_func('schroot_configure_ccache', d) > > schroot = d.getVar('SBUILD_CHROOT') > - isar_apt = d.getVar('ISAR_APT_REPO') > pkg_arch = d.getVar('PACKAGE_ARCH') > build_arch = d.getVar('BUILD_ARCH') > pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > - debdistroname = d.getVar('DEBDISTRONAME') > > install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \ > --host-arch {pkg_arch} --build-arch {build_arch} \ > @@ -310,15 +323,13 @@ python do_devshell() { > > termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ > cd {1}; \ > - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ > - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \ > - echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \ > - apt-get -y -q update; \ > - {4}; \ > + apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ > + apt-get -y upgrade; \ > + {2}; \ > export PATH=$PATH_PREPEND:$PATH; \ > $SHELL -i \ > '" > - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d) > + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d) > > bb.build.exec_func('schroot_delete_configs', d) > } > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index c699a84d..d1666f78 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -110,16 +110,17 @@ dpkg_runbuild() { > DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog) > DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print) > > - sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ > + sbuild -A -n -c ${SBUILD_CHROOT} \ > --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ > --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ > --bd-uninstallable-explainer=apt \ > - --no-apt-update \ > + --no-apt-update --apt-distupgrade \ > --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ > --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ > --chroot-setup-commands="rm -f /var/log/dpkg.log" \ > --chroot-setup-commands="mkdir -p ${deb_dir}" \ > --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \ > + --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ > --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ > --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ > --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ > diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass > index 58799fcf..46bdf78b 100644 > --- a/meta/classes/image-tools-extension.bbclass > +++ b/meta/classes/image-tools-extension.bbclass > @@ -44,19 +44,6 @@ imager_run() { > distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" > fi > > - # prepare isar-apt > - schroot -r -c ${session_id} -d / -u root -- sh -c " \ > - mkdir -p '/etc/apt/sources.list.d' > - echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \ > - '/etc/apt/sources.list.d/isar-apt.list' > - > - mkdir -p '/etc/apt/preferences.d' > - cat << EOF > '/etc/apt/preferences.d/isar-apt' > -Package: * > -Pin: release n=${DEBDISTRONAME} > -Pin-Priority: 1000 > -EOF" > - > E="${@ isar_export_proxies(d)}" > deb_dl_dir_import ${schroot_dir} ${distro} > schroot -r -c ${session_id} -d / -u root -- sh -c " \ > diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > index 39ced80b..2f07de82 100644 > --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" > ROOTFSDIR = "${SBUILD_CHROOT_DIR}" > ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" > > -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild > -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" > ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" > > # Do not cleanup base-apt Thanks for sending - thought I did, but I only prepared locally, never triggered the submission. Jan
On Sat, 2024-01-27 at 08:42 +0100, Uladzimir Bely wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > During the migration to sbuild, support for using self-built packages > in > the build environment was lost if those were already part of the > sbuild-chroot. This restores it by adding --apt-distupgrade to the > sbuild call. But that is not enough because sbuild will only upgrade > packages from already configured sources, not those specified via > --extra-repository. We therefore have to switch back to configuring > isar-apt during sbuild-chroot creation. > > As rootfs_configure_isar_apt configures the isar repo under > /isar-apt, we bind-mount the one in /home/builder/${PN} to that > folder. > Another difference is that we now need to run apt-get update > explicitly, > but only for isar-apt. > > For apt fetching, we neither need nor want isar-apt to be available. > Rebuilding self-generated apt packages is generally pointless as the > needs are better addressed in the generating recipe itself. Exposing > isar-apt to the fetch may furthermore lead to fetching a previously > built source package of the same recipe, rather than pulling the > external version. > > And because the sbuild-chroots are now left behind with isar-apt > configured, the configuration in imager_run can be removed (credits > to > Srinuvasan Arjunan). > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > Reviewed-by: Uladzimir Bely <ubely@ilbers.de> > --- > > Changes in v7: Merged v6 with fix disabling isar-apt for do_apt_fetch > > meta/classes/dpkg-base.bbclass | 37 ++++++++++++----- > -- > meta/classes/dpkg.bbclass | 5 ++- > meta/classes/image-tools-extension.bbclass | 13 ------- > .../sbuild-chroot/sbuild-chroot.inc | 2 - > 4 files changed, 27 insertions(+), 30 deletions(-) > Applied to next, thanks.
29/01/2024 09:59, 'Jan Kiszka' via isar-users wrote: > On 27.01.24 08:42, Uladzimir Bely wrote: >> From: Jan Kiszka <jan.kiszka@siemens.com> >> >> During the migration to sbuild, support for using self-built packages in >> the build environment was lost if those were already part of the >> sbuild-chroot. This restores it by adding --apt-distupgrade to the >> sbuild call. But that is not enough because sbuild will only upgrade >> packages from already configured sources, not those specified via >> --extra-repository. We therefore have to switch back to configuring >> isar-apt during sbuild-chroot creation. >> >> As rootfs_configure_isar_apt configures the isar repo under >> /isar-apt, we bind-mount the one in /home/builder/${PN} to that folder. >> Another difference is that we now need to run apt-get update explicitly, >> but only for isar-apt. >> >> For apt fetching, we neither need nor want isar-apt to be available. >> Rebuilding self-generated apt packages is generally pointless as the >> needs are better addressed in the generating recipe itself. Exposing >> isar-apt to the fetch may furthermore lead to fetching a previously >> built source package of the same recipe, rather than pulling the >> external version. >> >> And because the sbuild-chroots are now left behind with isar-apt >> configured, the configuration in imager_run can be removed (credits to >> Srinuvasan Arjunan). >> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >> Reviewed-by: Uladzimir Bely <ubely@ilbers.de> > Thanks for sending - thought I did, but I only prepared locally, never > triggered the submission. > > Jan > Hello Jan, It looks like this change creates one more issue. Here are simple steps to reproduce: Prepare test recipe: $ cat meta-isar/recipes-app/hello/pcre2.bb inherit dpkg SRC_URI = "apt://pcre2" CHANGELOG_V = "<orig-version>+isar" do_prepare_build() { deb_add_changelog } Go to kas shell (qemuamd64 bookworm): $ kas-container shell Perform those steps: builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay builder@b5422048da4a:/build$ bitbake -c build pcre2 builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target builder@b5422048da4a:/build$ bitbake -c build cowsay Result: | Reading package lists... | Building dependency tree... | Reading state information... | Calculating upgrade... | The following packages will be upgraded: | libpcre2-8-0 | 1 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. | Need to get 0 B/261 kB of archives. | After this operation, 0 B of additional disk space will be used. | Get:1 file:/isar-apt isar/main amd64 libpcre2-8-0 amd64 10.42-1+isar [261 kB] | Err:1 file:/isar-apt isar/main amd64 libpcre2-8-0 amd64 10.42-1+isar | File not found - /isar-apt/pool/main/p/pcre2/libpcre2-8-0_10.42-1+isar_amd64.deb (2: No such file or directory) | E: Failed to fetch file:/isar-apt/pool/main/p/pcre2/libpcre2-8-0_10.42-1%2bisar_amd64.deb File not found - /isar-apt/pool/main/p/pcre2/libpcre2-8-0_10.42-1+isar_amd64.deb (2: No such file or directory) | E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? | E: apt-get dist-upgrade failed Cowsay do not depend on libpcre2-8-0, but dist-upgrade makes apt to install updated packages. The reason is /var/lib/apt/lists in sbuild chroot rootfs has the information about newer libpcre2-8-0_10.42-1+isar_amd64.deb while it is not exists in local isar-apt copy of cowsay package. And even calling apt-get update of isar-apt do not help. And now I have the question about which use case does your commit fix? I've tried 2 scenarios: 1) Upgraded package needed. meta-isar/recipes-app/hello/pcre2.bb: CHANGELOG_V = "<orig-version>+isar" meta-isar/recipes-app/example-raw/example-raw_0.3.bb: DEBIAN_BUILD_DEPENDS = "libpcre2-8-0 (= 10.42-1+isar)" DEPENDS += "pcre2" 2) Downgraded package needed. meta-isar/recipes-app/hello/pcre2.bb: CHANGELOG_V = "10.40-1" meta-isar/recipes-app/example-raw/example-raw_0.3.bb: DEBIAN_BUILD_DEPENDS = "libpcre2-8-0 (= 10.40-1)" DEPENDS += "pcre2" And with your commit reverted both scenarios work ok (example-raw is built ok). Am I missing something?
On 07.02.25 16:57, Anton Mikanovich wrote: > 29/01/2024 09:59, 'Jan Kiszka' via isar-users wrote: >> On 27.01.24 08:42, Uladzimir Bely wrote: >>> From: Jan Kiszka <jan.kiszka@siemens.com> >>> >>> During the migration to sbuild, support for using self-built packages in >>> the build environment was lost if those were already part of the >>> sbuild-chroot. This restores it by adding --apt-distupgrade to the >>> sbuild call. But that is not enough because sbuild will only upgrade >>> packages from already configured sources, not those specified via >>> --extra-repository. We therefore have to switch back to configuring >>> isar-apt during sbuild-chroot creation. >>> >>> As rootfs_configure_isar_apt configures the isar repo under >>> /isar-apt, we bind-mount the one in /home/builder/${PN} to that folder. >>> Another difference is that we now need to run apt-get update explicitly, >>> but only for isar-apt. >>> >>> For apt fetching, we neither need nor want isar-apt to be available. >>> Rebuilding self-generated apt packages is generally pointless as the >>> needs are better addressed in the generating recipe itself. Exposing >>> isar-apt to the fetch may furthermore lead to fetching a previously >>> built source package of the same recipe, rather than pulling the >>> external version. >>> >>> And because the sbuild-chroots are now left behind with isar-apt >>> configured, the configuration in imager_run can be removed (credits to >>> Srinuvasan Arjunan). >>> >>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> >>> Reviewed-by: Uladzimir Bely <ubely@ilbers.de> >> Thanks for sending - thought I did, but I only prepared locally, never >> triggered the submission. >> >> Jan >> > Hello Jan, > > It looks like this change creates one more issue. > Here are simple steps to reproduce: > > Prepare test recipe: > > $ cat meta-isar/recipes-app/hello/pcre2.bb > inherit dpkg > > SRC_URI = "apt://pcre2" > > CHANGELOG_V = "<orig-version>+isar" > > do_prepare_build() { > deb_add_changelog > } > > Go to kas shell (qemuamd64 bookworm): > > $ kas-container shell > > Perform those steps: > > builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay > builder@b5422048da4a:/build$ bitbake -c build pcre2 > builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target > builder@b5422048da4a:/build$ bitbake -c build cowsay The outcome is expected: local_isarapt is incorrectly run prior to deploy_deb of pcre2. This won't happen during a normal build when cowsay DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = "do_deploy_deb". Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we do have problem because your patch 21a05608d85b forgot to adjust meta/classes/essential.bbclass. I'll send a fix. Jan
09/02/2025 11:19, Jan Kiszka wrote: > On 07.02.25 16:57, Anton Mikanovich wrote: >> Hello Jan, >> >> It looks like this change creates one more issue. >> Here are simple steps to reproduce: >> >> Prepare test recipe: >> >> $ cat meta-isar/recipes-app/hello/pcre2.bb >> inherit dpkg >> >> SRC_URI = "apt://pcre2" >> >> CHANGELOG_V = "<orig-version>+isar" >> >> do_prepare_build() { >> deb_add_changelog >> } >> >> Go to kas shell (qemuamd64 bookworm): >> >> $ kas-container shell >> >> Perform those steps: >> >> builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay >> builder@b5422048da4a:/build$ bitbake -c build pcre2 >> builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target >> builder@b5422048da4a:/build$ bitbake -c build cowsay > The outcome is expected: local_isarapt is incorrectly run prior to > deploy_deb of pcre2. This won't happen during a normal build when cowsay > DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = > "do_deploy_deb". > > Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we > do have problem because your patch 21a05608d85b forgot to adjust > meta/classes/essential.bbclass. I'll send a fix. > > Jan > But cowsay doesn't depend on pcre2, libpcre2 got installed before dependencies installation but during distupgrade stage. So DEPENDS is not an option (or how can we depend on every package?). Probably ISAR_REBUILD_ESSENTIAL_PKGS with your fix will resolve it. Speaking about ISAR_REBUILD_ESSENTIAL_PKGS, we should add a testcase (or better even a real usage in meta-isar/meta-test) for this feature.
On 10.02.25 08:47, Anton Mikanovich wrote: > 09/02/2025 11:19, Jan Kiszka wrote: >> On 07.02.25 16:57, Anton Mikanovich wrote: >>> Hello Jan, >>> >>> It looks like this change creates one more issue. >>> Here are simple steps to reproduce: >>> >>> Prepare test recipe: >>> >>> $ cat meta-isar/recipes-app/hello/pcre2.bb >>> inherit dpkg >>> >>> SRC_URI = "apt://pcre2" >>> >>> CHANGELOG_V = "<orig-version>+isar" >>> >>> do_prepare_build() { >>> deb_add_changelog >>> } >>> >>> Go to kas shell (qemuamd64 bookworm): >>> >>> $ kas-container shell >>> >>> Perform those steps: >>> >>> builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay >>> builder@b5422048da4a:/build$ bitbake -c build pcre2 >>> builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target >>> builder@b5422048da4a:/build$ bitbake -c build cowsay >> The outcome is expected: local_isarapt is incorrectly run prior to >> deploy_deb of pcre2. This won't happen during a normal build when cowsay >> DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = >> "do_deploy_deb". >> >> Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we >> do have problem because your patch 21a05608d85b forgot to adjust >> meta/classes/essential.bbclass. I'll send a fix. >> >> Jan >> > But cowsay doesn't depend on pcre2, libpcre2 got installed before > dependencies > installation but during distupgrade stage. So DEPENDS is not an option > (or how can we depend on every package?). > Probably ISAR_REBUILD_ESSENTIAL_PKGS with your fix will resolve it. That is why we introduced ISAR_REBUILD_ESSENTIAL_PKGS. > > Speaking about ISAR_REBUILD_ESSENTIAL_PKGS, we should add a testcase (or > better > even a real usage in meta-isar/meta-test) for this feature. > I don't disagree. Jan
09/02/2025 11:19, Jan Kiszka wrote: > On 07.02.25 16:57, Anton Mikanovich wrote: >> Hello Jan, >> >> It looks like this change creates one more issue. >> Here are simple steps to reproduce: >> >> Prepare test recipe: >> >> $ cat meta-isar/recipes-app/hello/pcre2.bb >> inherit dpkg >> >> SRC_URI = "apt://pcre2" >> >> CHANGELOG_V = "<orig-version>+isar" >> >> do_prepare_build() { >> deb_add_changelog >> } >> >> Go to kas shell (qemuamd64 bookworm): >> >> $ kas-container shell >> >> Perform those steps: >> >> builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay >> builder@b5422048da4a:/build$ bitbake -c build pcre2 >> builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target >> builder@b5422048da4a:/build$ bitbake -c build cowsay > The outcome is expected: local_isarapt is incorrectly run prior to > deploy_deb of pcre2. This won't happen during a normal build when cowsay > DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = > "do_deploy_deb". > > Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we > do have problem because your patch 21a05608d85b forgot to adjust > meta/classes/essential.bbclass. I'll send a fix. > > Jan > After some testing with ISAR_REBUILD_ESSENTIAL_PKGS="pcre2" the issue is still there. Steps to reproduce (clean build inside kas, ISAR_REBUILD_ESSENTIAL_PKGS is set to "pcre2"): builder@a1f979144422:/build$ bitbake -c build pcre2 builder@a1f979144422:/build$ echo -e "\npython do_rootfs_install:append() {\n\n}" >> /work/meta/classes/rootfs.bbclass builder@a1f979144422:/build$ bitbake -c build pcre2 So the question is the same: which use case does your original commit suppose to fix?
On 10.02.25 11:59, Anton Mikanovich wrote: > 09/02/2025 11:19, Jan Kiszka wrote: >> On 07.02.25 16:57, Anton Mikanovich wrote: >>> Hello Jan, >>> >>> It looks like this change creates one more issue. >>> Here are simple steps to reproduce: >>> >>> Prepare test recipe: >>> >>> $ cat meta-isar/recipes-app/hello/pcre2.bb >>> inherit dpkg >>> >>> SRC_URI = "apt://pcre2" >>> >>> CHANGELOG_V = "<orig-version>+isar" >>> >>> do_prepare_build() { >>> deb_add_changelog >>> } >>> >>> Go to kas shell (qemuamd64 bookworm): >>> >>> $ kas-container shell >>> >>> Perform those steps: >>> >>> builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay >>> builder@b5422048da4a:/build$ bitbake -c build pcre2 >>> builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target >>> builder@b5422048da4a:/build$ bitbake -c build cowsay >> The outcome is expected: local_isarapt is incorrectly run prior to >> deploy_deb of pcre2. This won't happen during a normal build when cowsay >> DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = >> "do_deploy_deb". >> >> Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we >> do have problem because your patch 21a05608d85b forgot to adjust >> meta/classes/essential.bbclass. I'll send a fix. >> >> Jan >> > After some testing with ISAR_REBUILD_ESSENTIAL_PKGS="pcre2" the issue is > still > there. Steps to reproduce (clean build inside kas, > ISAR_REBUILD_ESSENTIAL_PKGS > is set to "pcre2"): > > builder@a1f979144422:/build$ bitbake -c build pcre2 > builder@a1f979144422:/build$ echo -e "\npython > do_rootfs_install:append() {\n\n}" >> /work/meta/classes/rootfs.bbclass > builder@a1f979144422:/build$ bitbake -c build pcre2 > Let me try. Maybe we have more regressions piled up since the patch was merged. > So the question is the same: which use case does your original commit > suppose > to fix? A) the one from above B) when you are rebuilding some non-essential package that happens to be in the sbuild-chroot already Jan
10/02/2025 13:13, Jan Kiszka wrote: > On 10.02.25 11:59, Anton Mikanovich wrote: >> After some testing with ISAR_REBUILD_ESSENTIAL_PKGS="pcre2" the issue is >> still >> there. Steps to reproduce (clean build inside kas, >> ISAR_REBUILD_ESSENTIAL_PKGS >> is set to "pcre2"): >> >> builder@a1f979144422:/build$ bitbake -c build pcre2 >> builder@a1f979144422:/build$ echo -e "\npython >> do_rootfs_install:append() {\n\n}" >> /work/meta/classes/rootfs.bbclass >> builder@a1f979144422:/build$ bitbake -c build pcre2 >> > Let me try. Maybe we have more regressions piled up since the patch was > merged. The same issue can be easily reproduced on 0e846829 itself. And it was ok on previous commit (c44c088c). >> So the question is the same: which use case does your original commit >> suppose >> to fix? > A) the one from above > B) when you are rebuilding some non-essential package that happens to be > in the sbuild-chroot already > > Jan > As I've already said both downgrade and upgrade cases are ok on the latest next with reverted 0e846829. And they were also ok on c44c088c (previous commit). Are there any opensource downstream effected? I just want to make sure that my next fix for this pcre2 issue will not cause any more regressions.
On 10.02.25 12:34, Anton Mikanovich wrote: > 10/02/2025 13:13, Jan Kiszka wrote: >> On 10.02.25 11:59, Anton Mikanovich wrote: >>> After some testing with ISAR_REBUILD_ESSENTIAL_PKGS="pcre2" the issue is >>> still >>> there. Steps to reproduce (clean build inside kas, >>> ISAR_REBUILD_ESSENTIAL_PKGS >>> is set to "pcre2"): >>> >>> builder@a1f979144422:/build$ bitbake -c build pcre2 >>> builder@a1f979144422:/build$ echo -e "\npython >>> do_rootfs_install:append() {\n\n}" >> /work/meta/classes/rootfs.bbclass >>> builder@a1f979144422:/build$ bitbake -c build pcre2 >>> >> Let me try. Maybe we have more regressions piled up since the patch was >> merged. > The same issue can be easily reproduced on 0e846829 itself. > And it was ok on previous commit (c44c088c). >>> So the question is the same: which use case does your original commit >>> suppose >>> to fix? >> A) the one from above >> B) when you are rebuilding some non-essential package that happens to be >> in the sbuild-chroot already >> >> Jan >> > As I've already said both downgrade and upgrade cases are ok on the > latest next > with reverted 0e846829. And they were also ok on c44c088c (previous > commit). Without 0e846829, packages that are already in the sbuild-chroot are not updated. That is what the patch fixes. If things apparently "work", then you are not using the self-built package. Please read my length commit message to understand the issue better. Jan > Are there any opensource downstream effected? > I just want to make sure that my next fix for this pcre2 issue will not > cause > any more regressions. >
On 10.02.25 12:13, 'Jan Kiszka' via isar-users wrote: > On 10.02.25 11:59, Anton Mikanovich wrote: >> 09/02/2025 11:19, Jan Kiszka wrote: >>> On 07.02.25 16:57, Anton Mikanovich wrote: >>>> Hello Jan, >>>> >>>> It looks like this change creates one more issue. >>>> Here are simple steps to reproduce: >>>> >>>> Prepare test recipe: >>>> >>>> $ cat meta-isar/recipes-app/hello/pcre2.bb >>>> inherit dpkg >>>> >>>> SRC_URI = "apt://pcre2" >>>> >>>> CHANGELOG_V = "<orig-version>+isar" >>>> >>>> do_prepare_build() { >>>> deb_add_changelog >>>> } >>>> >>>> Go to kas shell (qemuamd64 bookworm): >>>> >>>> $ kas-container shell >>>> >>>> Perform those steps: >>>> >>>> builder@b5422048da4a:/build$ bitbake -c local_isarapt cowsay >>>> builder@b5422048da4a:/build$ bitbake -c build pcre2 >>>> builder@b5422048da4a:/build$ bitbake -c cleansstate sbuild-chroot-target >>>> builder@b5422048da4a:/build$ bitbake -c build cowsay >>> The outcome is expected: local_isarapt is incorrectly run prior to >>> deploy_deb of pcre2. This won't happen during a normal build when cowsay >>> DEPENDS on pcre2, though, because of do_local_isarapt[deptask] = >>> "do_deploy_deb". >>> >>> Now, if you used ISAR_REBUILD_ESSENTIAL_PKGS in your actual scenario, we >>> do have problem because your patch 21a05608d85b forgot to adjust >>> meta/classes/essential.bbclass. I'll send a fix. >>> >>> Jan >>> >> After some testing with ISAR_REBUILD_ESSENTIAL_PKGS="pcre2" the issue is >> still >> there. Steps to reproduce (clean build inside kas, >> ISAR_REBUILD_ESSENTIAL_PKGS >> is set to "pcre2"): >> >> builder@a1f979144422:/build$ bitbake -c build pcre2 >> builder@a1f979144422:/build$ echo -e "\npython >> do_rootfs_install:append() {\n\n}" >> /work/meta/classes/rootfs.bbclass >> builder@a1f979144422:/build$ bitbake -c build pcre2 >> > > Let me try. Maybe we have more regressions piled up since the patch was > merged. > Invalid partial rebuild scenario: If you are enforcing a rebuild of the sbuild-chroot, you also need to wipe all ISAR_REBUILD_ESSENTIAL_PKGS first. Can you explain what you actually did to trigger this? If you sit down for a moment, it should become clear what makes rebuilding complex when essential packages are replaced: They can become part of their own sbuild-chroot. Simply excluding isar-apt from the sbuild-chroots of essential packages could help - but only as long as we are not rebuilding a dependency chain of multiple packages. Granted, the latter is even less likely to happen than already having to rebuild essential packages. We are going down the rabbit hole of distro boot strapping here... Jan
10/02/2025 14:02, Jan Kiszka wrote: > Invalid partial rebuild scenario: If you are enforcing a rebuild of the > sbuild-chroot, you also need to wipe all ISAR_REBUILD_ESSENTIAL_PKGS > first. Can you explain what you actually did to trigger this? > > If you sit down for a moment, it should become clear what makes > rebuilding complex when essential packages are replaced: They can become > part of their own sbuild-chroot. > > Simply excluding isar-apt from the sbuild-chroots of essential packages > could help - but only as long as we are not rebuilding a dependency > chain of multiple packages. Granted, the latter is even less likely to > happen than already having to rebuild essential packages. We are going > down the rabbit hole of distro boot strapping here... > > Jan > Ok, it looks like I've found the case that was fixed: If package A (already installed in sbuild-chroot) gets rebuilt with an updated version, package B depends on any version of A (not just =new version like in my case), when B is built, apt would keep the old version of A and not pull the newly built one. The issues were triggered downstream after updating Isar version. I've changed a bbclass and bitbake has failed. As a user, I'd like to be able to perform rebuilds without wiping packages. I'll work on a testcase and a fix for this issue.
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 95ca3db8..86466b57 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C DEB_BUILD_PROFILES ?= "" DEB_BUILD_OPTIONS ?= "" -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" - python do_adjust_git() { import subprocess @@ -109,16 +107,24 @@ do_apt_fetch() { E="${@ isar_export_proxies(d)}" schroot_create_configs + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) + echo "Started session: ${session_id}" + schroot_cleanup() { + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 schroot_delete_configs } trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT + schroot -r -c ${session_id} -d / -u root -- \ + rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -r -c ${session_id} -d / -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done + + schroot -e -c ${session_id} schroot_delete_configs } @@ -221,8 +227,17 @@ def isar_export_build_settings(d): os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) +dpkg_schroot_create_configs() { + schroot_create_configs + sudo -s <<'EOSUDO' + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab} +EOSUDO +} + python do_dpkg_build() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) try: bb.build.exec_func("dpkg_runbuild", d) finally: @@ -288,7 +303,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" do_deploy_deb[dirs] = "${S}" python do_devshell() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) isar_export_proxies(d) isar_export_ccache(d) @@ -297,11 +312,9 @@ python do_devshell() { bb.build.exec_func('schroot_configure_ccache', d) schroot = d.getVar('SBUILD_CHROOT') - isar_apt = d.getVar('ISAR_APT_REPO') pkg_arch = d.getVar('PACKAGE_ARCH') build_arch = d.getVar('BUILD_ARCH') pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - debdistroname = d.getVar('DEBDISTRONAME') install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \ --host-arch {pkg_arch} --build-arch {build_arch} \ @@ -310,15 +323,13 @@ python do_devshell() { termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ cd {1}; \ - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \ - echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \ - apt-get -y -q update; \ - {4}; \ + apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ + apt-get -y upgrade; \ + {2}; \ export PATH=$PATH_PREPEND:$PATH; \ $SHELL -i \ '" - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d) + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d) bb.build.exec_func('schroot_delete_configs', d) } diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index c699a84d..d1666f78 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -110,16 +110,17 @@ dpkg_runbuild() { DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog) DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print) - sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ + sbuild -A -n -c ${SBUILD_CHROOT} \ --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ --bd-uninstallable-explainer=apt \ - --no-apt-update \ + --no-apt-update --apt-distupgrade \ --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ --chroot-setup-commands="rm -f /var/log/dpkg.log" \ --chroot-setup-commands="mkdir -p ${deb_dir}" \ --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \ + --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass index 58799fcf..46bdf78b 100644 --- a/meta/classes/image-tools-extension.bbclass +++ b/meta/classes/image-tools-extension.bbclass @@ -44,19 +44,6 @@ imager_run() { distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" fi - # prepare isar-apt - schroot -r -c ${session_id} -d / -u root -- sh -c " \ - mkdir -p '/etc/apt/sources.list.d' - echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \ - '/etc/apt/sources.list.d/isar-apt.list' - - mkdir -p '/etc/apt/preferences.d' - cat << EOF > '/etc/apt/preferences.d/isar-apt' -Package: * -Pin: release n=${DEBDISTRONAME} -Pin-Priority: 1000 -EOF" - E="${@ isar_export_proxies(d)}" deb_dl_dir_import ${schroot_dir} ${distro} schroot -r -c ${session_id} -d / -u root -- sh -c " \ diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc index 39ced80b..2f07de82 100644 --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" ROOTFSDIR = "${SBUILD_CHROOT_DIR}" ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" # Do not cleanup base-apt