Message ID | 20240127074253.24570-1-ubely@ilbers.de |
---|---|
State | Accepted, archived |
Headers | show |
Series | [v7] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot | expand |
On 27.01.24 08:42, Uladzimir Bely wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > During the migration to sbuild, support for using self-built packages in > the build environment was lost if those were already part of the > sbuild-chroot. This restores it by adding --apt-distupgrade to the > sbuild call. But that is not enough because sbuild will only upgrade > packages from already configured sources, not those specified via > --extra-repository. We therefore have to switch back to configuring > isar-apt during sbuild-chroot creation. > > As rootfs_configure_isar_apt configures the isar repo under > /isar-apt, we bind-mount the one in /home/builder/${PN} to that folder. > Another difference is that we now need to run apt-get update explicitly, > but only for isar-apt. > > For apt fetching, we neither need nor want isar-apt to be available. > Rebuilding self-generated apt packages is generally pointless as the > needs are better addressed in the generating recipe itself. Exposing > isar-apt to the fetch may furthermore lead to fetching a previously > built source package of the same recipe, rather than pulling the > external version. > > And because the sbuild-chroots are now left behind with isar-apt > configured, the configuration in imager_run can be removed (credits to > Srinuvasan Arjunan). > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > Reviewed-by: Uladzimir Bely <ubely@ilbers.de> > --- > > Changes in v7: Merged v6 with fix disabling isar-apt for do_apt_fetch > > meta/classes/dpkg-base.bbclass | 37 ++++++++++++------- > meta/classes/dpkg.bbclass | 5 ++- > meta/classes/image-tools-extension.bbclass | 13 ------- > .../sbuild-chroot/sbuild-chroot.inc | 2 - > 4 files changed, 27 insertions(+), 30 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass > index 95ca3db8..86466b57 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C > DEB_BUILD_PROFILES ?= "" > DEB_BUILD_OPTIONS ?= "" > > -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" > - > python do_adjust_git() { > import subprocess > > @@ -109,16 +107,24 @@ do_apt_fetch() { > E="${@ isar_export_proxies(d)}" > schroot_create_configs > > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > + echo "Started session: ${session_id}" > + > schroot_cleanup() { > + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 > schroot_delete_configs > } > trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > trap 'schroot_cleanup' EXIT > > + schroot -r -c ${session_id} -d / -u root -- \ > + rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt > for uri in "${SRC_APT}"; do > - schroot -d / -c ${SBUILD_CHROOT} -- \ > + schroot -r -c ${session_id} -d / -- \ > sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" > done > + > + schroot -e -c ${session_id} > schroot_delete_configs > } > > @@ -221,8 +227,17 @@ def isar_export_build_settings(d): > os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) > os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) > > +dpkg_schroot_create_configs() { > + schroot_create_configs > + sudo -s <<'EOSUDO' > + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" > + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" > + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab} > +EOSUDO > +} > + > python do_dpkg_build() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > try: > bb.build.exec_func("dpkg_runbuild", d) > finally: > @@ -288,7 +303,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" > do_deploy_deb[dirs] = "${S}" > > python do_devshell() { > - bb.build.exec_func('schroot_create_configs', d) > + bb.build.exec_func('dpkg_schroot_create_configs', d) > > isar_export_proxies(d) > isar_export_ccache(d) > @@ -297,11 +312,9 @@ python do_devshell() { > bb.build.exec_func('schroot_configure_ccache', d) > > schroot = d.getVar('SBUILD_CHROOT') > - isar_apt = d.getVar('ISAR_APT_REPO') > pkg_arch = d.getVar('PACKAGE_ARCH') > build_arch = d.getVar('BUILD_ARCH') > pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > - debdistroname = d.getVar('DEBDISTRONAME') > > install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \ > --host-arch {pkg_arch} --build-arch {build_arch} \ > @@ -310,15 +323,13 @@ python do_devshell() { > > termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ > cd {1}; \ > - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ > - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \ > - echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \ > - apt-get -y -q update; \ > - {4}; \ > + apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ > + apt-get -y upgrade; \ > + {2}; \ > export PATH=$PATH_PREPEND:$PATH; \ > $SHELL -i \ > '" > - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d) > + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d) > > bb.build.exec_func('schroot_delete_configs', d) > } > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index c699a84d..d1666f78 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -110,16 +110,17 @@ dpkg_runbuild() { > DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog) > DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print) > > - sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ > + sbuild -A -n -c ${SBUILD_CHROOT} \ > --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ > --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ > --bd-uninstallable-explainer=apt \ > - --no-apt-update \ > + --no-apt-update --apt-distupgrade \ > --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ > --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ > --chroot-setup-commands="rm -f /var/log/dpkg.log" \ > --chroot-setup-commands="mkdir -p ${deb_dir}" \ > --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \ > + --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ > --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ > --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ > --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ > diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass > index 58799fcf..46bdf78b 100644 > --- a/meta/classes/image-tools-extension.bbclass > +++ b/meta/classes/image-tools-extension.bbclass > @@ -44,19 +44,6 @@ imager_run() { > distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" > fi > > - # prepare isar-apt > - schroot -r -c ${session_id} -d / -u root -- sh -c " \ > - mkdir -p '/etc/apt/sources.list.d' > - echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \ > - '/etc/apt/sources.list.d/isar-apt.list' > - > - mkdir -p '/etc/apt/preferences.d' > - cat << EOF > '/etc/apt/preferences.d/isar-apt' > -Package: * > -Pin: release n=${DEBDISTRONAME} > -Pin-Priority: 1000 > -EOF" > - > E="${@ isar_export_proxies(d)}" > deb_dl_dir_import ${schroot_dir} ${distro} > schroot -r -c ${session_id} -d / -u root -- sh -c " \ > diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > index 39ced80b..2f07de82 100644 > --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc > @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" > ROOTFSDIR = "${SBUILD_CHROOT_DIR}" > ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" > > -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild > -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" > ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" > > # Do not cleanup base-apt Thanks for sending - thought I did, but I only prepared locally, never triggered the submission. Jan
On Sat, 2024-01-27 at 08:42 +0100, Uladzimir Bely wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > During the migration to sbuild, support for using self-built packages > in > the build environment was lost if those were already part of the > sbuild-chroot. This restores it by adding --apt-distupgrade to the > sbuild call. But that is not enough because sbuild will only upgrade > packages from already configured sources, not those specified via > --extra-repository. We therefore have to switch back to configuring > isar-apt during sbuild-chroot creation. > > As rootfs_configure_isar_apt configures the isar repo under > /isar-apt, we bind-mount the one in /home/builder/${PN} to that > folder. > Another difference is that we now need to run apt-get update > explicitly, > but only for isar-apt. > > For apt fetching, we neither need nor want isar-apt to be available. > Rebuilding self-generated apt packages is generally pointless as the > needs are better addressed in the generating recipe itself. Exposing > isar-apt to the fetch may furthermore lead to fetching a previously > built source package of the same recipe, rather than pulling the > external version. > > And because the sbuild-chroots are now left behind with isar-apt > configured, the configuration in imager_run can be removed (credits > to > Srinuvasan Arjunan). > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > Reviewed-by: Uladzimir Bely <ubely@ilbers.de> > --- > > Changes in v7: Merged v6 with fix disabling isar-apt for do_apt_fetch > > meta/classes/dpkg-base.bbclass | 37 ++++++++++++----- > -- > meta/classes/dpkg.bbclass | 5 ++- > meta/classes/image-tools-extension.bbclass | 13 ------- > .../sbuild-chroot/sbuild-chroot.inc | 2 - > 4 files changed, 27 insertions(+), 30 deletions(-) > Applied to next, thanks.
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 95ca3db8..86466b57 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C DEB_BUILD_PROFILES ?= "" DEB_BUILD_OPTIONS ?= "" -ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main" - python do_adjust_git() { import subprocess @@ -109,16 +107,24 @@ do_apt_fetch() { E="${@ isar_export_proxies(d)}" schroot_create_configs + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) + echo "Started session: ${session_id}" + schroot_cleanup() { + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 schroot_delete_configs } trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT + schroot -r -c ${session_id} -d / -u root -- \ + rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -r -c ${session_id} -d / -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done + + schroot -e -c ${session_id} schroot_delete_configs } @@ -221,8 +227,17 @@ def isar_export_build_settings(d): os.environ['DEB_BUILD_OPTIONS'] = isar_deb_build_options(d) os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d) +dpkg_schroot_create_configs() { + schroot_create_configs + sudo -s <<'EOSUDO' + sbuild_fstab="${SBUILD_CONF_DIR}/fstab" + fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0" + grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab} +EOSUDO +} + python do_dpkg_build() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) try: bb.build.exec_func("dpkg_runbuild", d) finally: @@ -288,7 +303,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" do_deploy_deb[dirs] = "${S}" python do_devshell() { - bb.build.exec_func('schroot_create_configs', d) + bb.build.exec_func('dpkg_schroot_create_configs', d) isar_export_proxies(d) isar_export_ccache(d) @@ -297,11 +312,9 @@ python do_devshell() { bb.build.exec_func('schroot_configure_ccache', d) schroot = d.getVar('SBUILD_CHROOT') - isar_apt = d.getVar('ISAR_APT_REPO') pkg_arch = d.getVar('PACKAGE_ARCH') build_arch = d.getVar('BUILD_ARCH') pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - debdistroname = d.getVar('DEBDISTRONAME') install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \ --host-arch {pkg_arch} --build-arch {build_arch} \ @@ -310,15 +323,13 @@ python do_devshell() { termcmd = "schroot -d / -c {0} -u root -- sh -c ' \ cd {1}; \ - echo {2} > /etc/apt/sources.list.d/isar_apt.list; \ - echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \ - echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \ - apt-get -y -q update; \ - {4}; \ + apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \ + apt-get -y upgrade; \ + {2}; \ export PATH=$PATH_PREPEND:$PATH; \ $SHELL -i \ '" - oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d) + oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d) bb.build.exec_func('schroot_delete_configs', d) } diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index c699a84d..d1666f78 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -110,16 +110,17 @@ dpkg_runbuild() { DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog) DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print) - sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ + sbuild -A -n -c ${SBUILD_CHROOT} \ --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \ --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ --bd-uninstallable-explainer=apt \ - --no-apt-update \ + --no-apt-update --apt-distupgrade \ --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ --chroot-setup-commands="rm -f /var/log/dpkg.log" \ --chroot-setup-commands="mkdir -p ${deb_dir}" \ --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \ + --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \ --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \ --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass index 58799fcf..46bdf78b 100644 --- a/meta/classes/image-tools-extension.bbclass +++ b/meta/classes/image-tools-extension.bbclass @@ -44,19 +44,6 @@ imager_run() { distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" fi - # prepare isar-apt - schroot -r -c ${session_id} -d / -u root -- sh -c " \ - mkdir -p '/etc/apt/sources.list.d' - echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \ - '/etc/apt/sources.list.d/isar-apt.list' - - mkdir -p '/etc/apt/preferences.d' - cat << EOF > '/etc/apt/preferences.d/isar-apt' -Package: * -Pin: release n=${DEBDISTRONAME} -Pin-Priority: 1000 -EOF" - E="${@ isar_export_proxies(d)}" deb_dl_dir_import ${schroot_dir} ${distro} schroot -r -c ${session_id} -d / -u root -- sh -c " \ diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc index 39ced80b..2f07de82 100644 --- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc +++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc @@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs" ROOTFSDIR = "${SBUILD_CHROOT_DIR}" ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}" -# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild -ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt" ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt" # Do not cleanup base-apt