| Message ID | 1f7d8469de5afbad8d9172bdc5022b46c52f846e.1715963742.git.jan.kiszka@siemens.com |
|---|---|
| State | Accepted, archived |
| Headers | show |
| Series | More kbuild improvements, single-build source packages, cross profile fix | expand |
On Fri, 2024-05-17 at 18:35 +0200, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > Avoid building the source package multiple times, possibly even > inconsistently. This is achieved by delegating this task to to the > base > package and installing the source package from isar-apt in the native > and compat package variants. Those derived packages will also no > longer > trigger fetching, unpacking or prepare_build as everything is > supposed > to be in the source package. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > meta/classes/dpkg-base.bbclass | 4 +-- > meta/classes/dpkg-prebuilt.bbclass | 7 ++---- > meta/classes/dpkg-source.bbclass | 40 > ++++++++++++++++++++++++++++-- > meta/classes/dpkg.bbclass | 3 ++- > 4 files changed, 44 insertions(+), 10 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg- > base.bbclass > index 93321976..789d6c74 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -78,7 +78,7 @@ addtask adjust_git after do_unpack before do_patch > do_adjust_git[lockfiles] += "${DL_DIR}/git/isar.lock" > > inherit patch > -addtask patch after do_adjust_git before do_dpkg_build > +addtask patch after do_adjust_git > > SRC_APT ?= "" > > @@ -191,7 +191,7 @@ do_prepare_build() { > true > } > > -addtask prepare_build after do_patch do_transform_template before > do_dpkg_build > +addtask prepare_build after do_patch do_transform_template > # If Isar recipes depend on each other, they typically need the > package > # deployed to isar-apt > do_local_isarapt[depends] += "isar-apt:do_cache_config" > diff --git a/meta/classes/dpkg-prebuilt.bbclass b/meta/classes/dpkg- > prebuilt.bbclass > index 8135fc81..7092670b 100644 > --- a/meta/classes/dpkg-prebuilt.bbclass > +++ b/meta/classes/dpkg-prebuilt.bbclass > @@ -16,11 +16,8 @@ python do_unpack:prepend() { > d.setVar('SRC_URI', ' '.join(src_uri)) > } > > -# break dependencies on do_patch, etc... but still support sstate > caching > -deltask dpkg_build > -addtask dpkg_build after do_unpack before do_deploy_deb > -# break inherited (from dpkg-base) dependency on sbuild_chroot > -do_dpkg_build[depends] = "" > +# also breaks inherited (from dpkg-base) dependency on sbuild_chroot > +do_dpkg_build[depends] = "${PN}:do_unpack" > do_dpkg_build() { > true > } > diff --git a/meta/classes/dpkg-source.bbclass b/meta/classes/dpkg- > source.bbclass > index 560f536b..d6ab5aad 100644 > --- a/meta/classes/dpkg-source.bbclass > +++ b/meta/classes/dpkg-source.bbclass > @@ -18,7 +18,7 @@ do_dpkg_source() { > find ${WORKDIR} -maxdepth 1 -name "${DEBIAN_SOURCE}_*.dsc" - > delete > sh -c "cd ${WORKDIR}; dpkg-source ${DPKG_SOURCE_EXTRA_ARGS} -b > ${PPS}" > } > -addtask dpkg_source after do_prepare_build before do_dpkg_build > +addtask dpkg_source after do_prepare_build > > do_deploy_source[depends] += "isar-apt:do_cache_config" > do_deploy_source[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" > @@ -34,4 +34,40 @@ do_deploy_source() { > "${DSC_FILE}" > fi > } > -addtask deploy_source after do_dpkg_source before do_dpkg_build > +addtask deploy_source after do_dpkg_source > + > +do_dpkg_build[depends] += "${BPN}:do_deploy_source" > + > +SCHROOT_MOUNTS = "${WORKDIR}:/work ${REPO_ISAR_DIR}/${DISTRO}:/isar- > apt" > + > +do_fetch_common_source[depends] += "${SCHROOT_DEP} > ${BPN}:do_deploy_source" > +do_fetch_common_source[network] = "${TASK_USE_SUDO}" > +do_fetch_common_source() { > + schroot_create_configs > + insert_mounts > + > + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) > + echo "Started session: ${session_id}" > + > + schroot_cleanup() { > + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 > + remove_mounts > /dev/null 2>&1 > + schroot_delete_configs > + } > + trap 'exit 1' INT HUP QUIT TERM ALRM USR1 > + trap 'schroot_cleanup' EXIT > + > + schroot -r -c ${session_id} -d / -u root -- \ > + apt-get update -o Dir::Etc::SourceList="sources.list.d/isar- > apt.list" -o Dir::Etc::SourceParts="-" -o APT::Get::List-Cleanup="0" > + schroot -r -c ${session_id} -d / -- \ > + sh -c ' > + cd /work > + apt-get -y --download-only --only-source -o > Acquire::Source-Symlinks="false" source ${DEBIAN_SOURCE}' > + > + schroot -e -c ${session_id} > + remove_mounts > + schroot_delete_configs > +} > +addtask fetch_common_source > + > +do_dpkg_build[depends] += "${@'${PN}:do_dpkg_source' if '${PN}' == > '${BPN}' else '${PN}:do_fetch_common_source'}" Hi, while debugging another issue related to DPKG_ARCH=all, I noticed that the do_fetch_common_source task has a hidden dependency to having the source package in isar apt. While this is modeled in bitbake (via depends to ${BPN}:do_deploy_source), the source package is not there anymore when running repo_del_package on the -native package. This happens because the clean task on the -native package calls out to repo_del_package, which also removes the source package. This deletion remains hidden to bitbake, hence the ${PN}:do_dpkg_source is not executed again and the do_fetch_common_source fails, as the source package is not there anymore. I'm wondering if there is a way to tell reprepro to not delete source packages. This could be used to then only delete the source package in the non-native case. An alternative solution would be to let the - native clean task depend on the non-native clean task. Felix > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index 804fe5a6..d92ff68c 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -24,9 +24,10 @@ def expand_sbuild_pt_additions(d): > cmds += 'sbuild_export ' + var + ' "' + varval + '"\n' > return cmds > > -do_prepare_build:append() { > +do_get_reference_env() { > env > ${DPKG_PREBUILD_ENV_FILE} > } > +addtask get_reference_env before do_dpkg_build > > # cp -n results in nonzero exit code starting from coreutils 9.2 > # and starting from 9.3 we can use --update=none for the same > behaviour
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 93321976..789d6c74 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -78,7 +78,7 @@ addtask adjust_git after do_unpack before do_patch do_adjust_git[lockfiles] += "${DL_DIR}/git/isar.lock" inherit patch -addtask patch after do_adjust_git before do_dpkg_build +addtask patch after do_adjust_git SRC_APT ?= "" @@ -191,7 +191,7 @@ do_prepare_build() { true } -addtask prepare_build after do_patch do_transform_template before do_dpkg_build +addtask prepare_build after do_patch do_transform_template # If Isar recipes depend on each other, they typically need the package # deployed to isar-apt do_local_isarapt[depends] += "isar-apt:do_cache_config" diff --git a/meta/classes/dpkg-prebuilt.bbclass b/meta/classes/dpkg-prebuilt.bbclass index 8135fc81..7092670b 100644 --- a/meta/classes/dpkg-prebuilt.bbclass +++ b/meta/classes/dpkg-prebuilt.bbclass @@ -16,11 +16,8 @@ python do_unpack:prepend() { d.setVar('SRC_URI', ' '.join(src_uri)) } -# break dependencies on do_patch, etc... but still support sstate caching -deltask dpkg_build -addtask dpkg_build after do_unpack before do_deploy_deb -# break inherited (from dpkg-base) dependency on sbuild_chroot -do_dpkg_build[depends] = "" +# also breaks inherited (from dpkg-base) dependency on sbuild_chroot +do_dpkg_build[depends] = "${PN}:do_unpack" do_dpkg_build() { true } diff --git a/meta/classes/dpkg-source.bbclass b/meta/classes/dpkg-source.bbclass index 560f536b..d6ab5aad 100644 --- a/meta/classes/dpkg-source.bbclass +++ b/meta/classes/dpkg-source.bbclass @@ -18,7 +18,7 @@ do_dpkg_source() { find ${WORKDIR} -maxdepth 1 -name "${DEBIAN_SOURCE}_*.dsc" -delete sh -c "cd ${WORKDIR}; dpkg-source ${DPKG_SOURCE_EXTRA_ARGS} -b ${PPS}" } -addtask dpkg_source after do_prepare_build before do_dpkg_build +addtask dpkg_source after do_prepare_build do_deploy_source[depends] += "isar-apt:do_cache_config" do_deploy_source[lockfiles] = "${REPO_ISAR_DIR}/isar.lock" @@ -34,4 +34,40 @@ do_deploy_source() { "${DSC_FILE}" fi } -addtask deploy_source after do_dpkg_source before do_dpkg_build +addtask deploy_source after do_dpkg_source + +do_dpkg_build[depends] += "${BPN}:do_deploy_source" + +SCHROOT_MOUNTS = "${WORKDIR}:/work ${REPO_ISAR_DIR}/${DISTRO}:/isar-apt" + +do_fetch_common_source[depends] += "${SCHROOT_DEP} ${BPN}:do_deploy_source" +do_fetch_common_source[network] = "${TASK_USE_SUDO}" +do_fetch_common_source() { + schroot_create_configs + insert_mounts + + session_id=$(schroot -q -b -c ${SBUILD_CHROOT}) + echo "Started session: ${session_id}" + + schroot_cleanup() { + schroot -q -f -e -c ${session_id} > /dev/null 2>&1 + remove_mounts > /dev/null 2>&1 + schroot_delete_configs + } + trap 'exit 1' INT HUP QUIT TERM ALRM USR1 + trap 'schroot_cleanup' EXIT + + schroot -r -c ${session_id} -d / -u root -- \ + apt-get update -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" -o Dir::Etc::SourceParts="-" -o APT::Get::List-Cleanup="0" + schroot -r -c ${session_id} -d / -- \ + sh -c ' + cd /work + apt-get -y --download-only --only-source -o Acquire::Source-Symlinks="false" source ${DEBIAN_SOURCE}' + + schroot -e -c ${session_id} + remove_mounts + schroot_delete_configs +} +addtask fetch_common_source + +do_dpkg_build[depends] += "${@'${PN}:do_dpkg_source' if '${PN}' == '${BPN}' else '${PN}:do_fetch_common_source'}" diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index 804fe5a6..d92ff68c 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -24,9 +24,10 @@ def expand_sbuild_pt_additions(d): cmds += 'sbuild_export ' + var + ' "' + varval + '"\n' return cmds -do_prepare_build:append() { +do_get_reference_env() { env > ${DPKG_PREBUILD_ENV_FILE} } +addtask get_reference_env before do_dpkg_build # cp -n results in nonzero exit code starting from coreutils 9.2 # and starting from 9.3 we can use --update=none for the same behaviour