@@ -30,6 +30,7 @@ python __anonymous() {
schroot_dir = d.getVar('SCHROOT_HOST_DIR', False)
sbuild_dep = "sbuild-chroot-host" + flavor_suffix + ":do_build"
sdk_toolchain = "crossbuild-essential-" + distro_arch
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR'))
else:
d.setVar('BUILD_ARCH', distro_arch)
schroot_dir = d.getVar('SCHROOT_TARGET_DIR', False)
@@ -11,6 +11,7 @@ inherit terminal
inherit repository
inherit deb-dl-dir
inherit essential
+inherit debrepo
# Local (WORKDIR-internal) dir where do_dpkg_build collects the built debs.
# Its content is exported into the shared, sstate-tracked DEPLOY_DIR_DEB so
@@ -120,6 +120,12 @@ dpkg_runbuild() {
echo '$stalled_pkg_timeout = ${DPKG_BUILD_TIMEOUT};' >> ${SBUILD_CONFIG}
DSC_FILE=$(find ${DEPLOY_DIR_SRC} -maxdepth 1 -name "${DEBIAN_SOURCE}_*.dsc" -print)
+ debrepo_parse_dscfile "${DSC_FILE}"
+
+ locked_update_cmd=":"
+ if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+ locked_update_cmd="flock -x /base-apt/repo.lock -c 'apt-get -y update'"
+ fi
# networking is automatically enabled on older versions of sbuild
sbuild_network_option=""
@@ -143,9 +149,11 @@ dpkg_runbuild() {
--chroot-setup-commands="rm -f /var/log/dpkg.log" \
--chroot-setup-commands="mkdir -p ${deb_dir}" \
--chroot-setup-commands="find ${ext_deb_dir} -maxdepth 1 -name '*.deb' -exec ln -t ${deb_dir}/ -sf {} +" \
+ --chroot-setup-commands="${locked_update_cmd}" \
--chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \
--finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-*-dummy_*.deb" \
--finished-build-commands="find ${deb_dir} -maxdepth 1 -type f -name '*.deb' -print -exec cp ${CP_FLAGS} -t ${ext_deb_dir}/ {} +" \
+ --finished-build-commands="mkdir -p ${ext_root}" \
${@ '--finished-build-commands="cp /var/log/dpkg.log $ext_root/dpkg_partial.log"' if d.getVar('ISAR_CHROOT_MODE') == 'schroot' else '' } \
--build-path="" --build-dir=${WORKDIR} --dist="${DEBDISTRONAME}" ${DSC_FILE}
new file mode 100644
@@ -0,0 +1,159 @@
+# This software is a part of Isar.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+#
+# This class extends the image.bbclass for setting locales and purging unneeded
+# ones.
+
+inherit debrepo
+
+LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
+ en_US ISO-8859-1\n"
+LOCALE_DEFAULT ?= "en_US.UTF-8"
+# set locale used during package installation, which is visible to
+# maintainer scripts (e.g. used in console-setup)
+export LANG = "${LOCALE_DEFAULT}"
+export LANGUAGE = "${LOCALE_DEFAULT}"
+export LC_ALL = "${LOCALE_DEFAULT}"
+
+def get_locale_gen(d, sep='\n'):
+ locale_gen = d.getVar("LOCALE_GEN") or ""
+ return sep.join(sorted(set(i.strip()
+ for i in locale_gen.split('\\n')
+ if i.strip())))
+
+def get_locale_dirs(d):
+ # Directory names below /usr/share/locale that shall be kept, derived from
+ # the language and language_territory parts of LOCALE_GEN.
+ locale_gen = d.getVar("LOCALE_GEN") or ""
+ return sorted(set(i
+ for j in locale_gen.split('\\n')
+ if j.strip()
+ for i in (j.split()[0].split("_")[0],
+ j.split()[0].split(".")[0])))
+
+<<<<<<< HEAD
+def get_locale_path_include(d):
+ lines = []
+ for name in get_locale_dirs(d):
+ lines.append("path-include=/usr/share/locale/%s" % name)
+ lines.append("path-include=/usr/share/locale/%s/*" % name)
+ return "\n".join(lines)
+
+# Configure dpkg to only unpack the requested locales, so unneeded locale files
+# are never installed. This runs before any package is installed into the image.
+# Locale files that the bootstrap already installed are cleaned up here as well.
+ROOTFS_CONFIGURE_COMMAND += "image_configure_locale_filter"
+image_configure_locale_filter[weight] = "5"
+image_configure_locale_filter() {
+ cat<<__EOF__ > ${WORKDIR}/locale.dpkg-filter
+path-exclude=/usr/share/locale/*
+path-include=/usr/share/locale/locale.alias
+${@get_locale_path_include(d)}
+__EOF__
+
+ run_privileged_heredoc <<'EOSUDO'
+ set -e
+
+ mkdir -p '${ROOTFSDIR}/etc/dpkg/dpkg.cfg.d'
+ cat '${WORKDIR}/locale.dpkg-filter' \
+ > '${ROOTFSDIR}/etc/dpkg/dpkg.cfg.d/50isar-locales'
+
+ # Drop locale files the bootstrap installed that are not requested.
+ if [ -d '${ROOTFSDIR}/usr/share/locale' ]; then
+ keep=' locale.alias ${@' '.join(get_locale_dirs(d))} '
+ for entry in '${ROOTFSDIR}'/usr/share/locale/*; do
+ [ -e "$entry" ] || continue
+ name=$(basename "$entry")
+ case "$keep" in
+ *" $name "*) ;;
+ *) rm -rf "$entry" ;;
+ esac
+ done
+ fi
+EOSUDO
+=======
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
+image_install_localepurge_download[weight] = "40"
+image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+image_install_localepurge_download() {
+ debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge"
+ debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
+ run_privileged_heredoc <<'EOF'
+ set -e
+ ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), d.getVar('ROOTFS_MOUNTS') if d.getVar('ISAR_CHROOT_MODE') == 'unshare' else '')}
+ chroot ${ROOTFSDIR} \
+ /usr/bin/apt-get ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 --download-only localepurge
+EOF
+>>>>>>> 3308a3c4 (base-apt: Predownload packages to base-apt before install)
+}
+
+# Preseed the debconf selection and generate the requested locales up front,
+# before any package is installed, so package maintainer scripts find working
+# locales.
+ROOTFS_CONFIGURE_COMMAND += "image_configure_locale_debconf"
+image_configure_locale_debconf[weight] = "5"
+image_configure_locale_debconf() {
+ cat<<__EOF__ > ${WORKDIR}/locale.debconf
+locales locales/locales_to_be_generated multiselect ${@get_locale_gen(d, ', ')}
+locales locales/default_environment_locale select ${LOCALE_DEFAULT}
+__EOF__
+ cat<<__EOF__ > ${WORKDIR}/locale.gen
+${@get_locale_gen(d)}
+__EOF__
+ cat<<__EOF__ > ${WORKDIR}/locale.default
+LANG=${LOCALE_DEFAULT}
+__EOF__
+
+ run_privileged_heredoc <<'EOSUDO'
+ set -e
+
+ ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), '')}
+
+ cat '${WORKDIR}/locale.default' > '${ROOTFSDIR}/etc/default/locale'
+ cat '${WORKDIR}/locale.debconf' > '${ROOTFSDIR}/tmp/locale.debconf'
+
+ # Enable the requested locales by uncommenting them in /etc/locale.gen
+ while read -r locale; do
+ [ -n "$locale" ] || continue
+ sed -i "/$locale/s/^# *//" '${ROOTFSDIR}/etc/locale.gen'
+ done < '${WORKDIR}/locale.gen'
+
+ chroot '${ROOTFSDIR}' /bin/sh <<'EOSH'
+ set -e
+
+ debconf-set-selections /tmp/locale.debconf
+ rm -f /tmp/locale.debconf
+EOSH
+EOSUDO
+}
+
+# The systemd locale.conf symlink can only be created once systemd is installed.
+ROOTFS_INSTALL_COMMAND += "image_configure_locales"
+image_configure_locales[weight] = "100"
+image_configure_locales() {
+ run_privileged_heredoc <<'EOSUDO'
+ set -e
+
+ ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), '')}
+
+ chroot '${ROOTFSDIR}' /bin/sh <<'EOSH'
+ set -e
+
+ SYSTEMD_VERSION=$(dpkg-query \
+ --showformat='${source:Upstream-Version}' \
+ --show systemd || echo "0" )
+
+ if dpkg --compare-versions "$SYSTEMD_VERSION" "ge" "251"; then
+ if dpkg --compare-versions "$SYSTEMD_VERSION" "lt" "253"; then
+ ln -s /etc/default/locale /etc/locale.conf
+ fi
+ fi
+
+ echo 'reconfigure locales'
+ dpkg-reconfigure -f noninteractive locales
+EOSH
+EOSUDO
+}
@@ -6,6 +6,11 @@
# This file extends the image.bbclass to supply tools for futher imager functions
inherit sbuild
+inherit debrepo
+
+python __anonymous() {
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR'))
+}
IMAGER_INSTALL ??= ""
IMAGER_BUILD_DEPS ??= ""
@@ -60,12 +65,20 @@ imager_run_schroot() {
echo "Installing imager deps: ${local_install}"
distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ debrepo_workdir=${DEBREPO_TARGET_DIR}
if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+ debrepo_workdir=${DEBREPO_HOST_DIR}
+ fi
fi
E="${@ isar_export_proxies(d)}"
deb_dl_dir_import ${schroot_dir} ${distro}
+
+ debrepo_add_packages --isarapt "${debrepo_workdir}" "${local_install}"
+ debrepo_update_apt_source_list "${schroot_dir}" "base-apt"
+
${SCRIPTSDIR}/lockrun.py -r -f "${REPO_ISAR_DIR}/isar.lock" -s <<EOAPT
schroot -r -c ${session_id} -d / -u root -- sh -c " \
apt-get update \
@@ -2,7 +2,7 @@
# Copyright (c) Siemens AG, 2020
inherit deb-dl-dir
-
+inherit debrepo
inherit sbom
ROOTFS_ARCH ?= "${DISTRO_ARCH}"
@@ -366,6 +366,9 @@ rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadPro
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK}"
rootfs_install_pkgs_download() {
+ debrepo_add_packages --isarapt "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}"
+ debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
# download packages using apt in a non-privileged namespace
rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
${ROOTFSDIR} \
@@ -43,18 +43,29 @@ class AptSrcSchroot(AptSrc):
repo_isar_dir = d.getVar('REPO_ISAR_DIR')
lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
+ debrepo_target_dir = d.getVar('DEBREPO_TARGET_DIR')
+ isar_prefetch_base_apt = d.getVar('ISAR_PREFETCH_BASE_APT')
+ repo_base_dir = d.getVar('REPO_BASE_DIR')
+ scriptsdir = d.getVar('SCRIPTSDIR')
+
try:
runfetchcmd(f'''
set -e
+
schroot -r -c {session_id} -d / -u root -- \
rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
- schroot -r -c {session_id} -d / -- \
- sh -c '
- set -e
- mkdir -p /downloads/{ud.localfile}
- cd /downloads/{ud.localfile}
- apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
- '
+
+ if [ "{isar_prefetch_base_apt}" = "1" ]; then
+ {scriptsdir}/deb-repo --workdir={debrepo_target_dir} --srcmode "{ud.src_package}"
+ else
+ schroot -r -c {session_id} -d / -- \
+ sh -c '
+ set -e
+ mkdir -p /downloads/{ud.localfile}
+ cd /downloads/{ud.localfile}
+ apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
+ '
+ fi
''', d)
except (OSError, FetchError):
raise
@@ -72,9 +83,19 @@ class AptSrcSchroot(AptSrc):
pp = d.getVar('PP')
pps = d.getVar('PPS')
+
+ isar_prefetch_base_apt = d.getVar('ISAR_PREFETCH_BASE_APT')
+ repo_base_dir = d.getVar('REPO_BASE_DIR')
+
try:
runfetchcmd(f'''
set -e
+ if [ "{isar_prefetch_base_apt}" = "1" ]; then
+ flock -x "{repo_base_dir}/repo.lock" -c "
+ schroot -r -c {session_id} -d / -u root -- \
+ sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+ "
+ fi
schroot -r -c {session_id} -d / -u root -- \
rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
schroot -r -c {session_id} -d / -- \
@@ -82,7 +103,16 @@ class AptSrcSchroot(AptSrc):
set -e
dscfile=$(apt-get -y -qq --print-uris --only-source source {ud.src_package} | \
cut -d " " -f2 | grep -E "\.dsc")
- cp /downloads/{ud.localfile}/* {pp}
+
+ if [ "{isar_prefetch_base_apt}" = "1" ]; then
+ temp_dir=$(mktemp -d)
+ cd "$temp_dir"
+ apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
+ # Getting rid of links
+ cp -L "$temp_dir"/* {pp}
+ else
+ cp /downloads/{ud.localfile}/* {pp}
+ fi
cd {pp}
mv -f {pps} {pps}.prev
dpkg-source -x "$dscfile" {pps}
@@ -26,6 +26,8 @@ SBOM_IMAGE_INSTALL = "python3-debsbom python3-spdx-tools python3-cyclonedx-lib"
ROOTFSDIR = "${WORKDIR}/rootfs"
ROOTFS_PACKAGES = "${SBOM_IMAGE_INSTALL}"
+DEBREPO_WORKDIR = "${@ '${DEBREPO_HOST_DIR}' if d.getVar("DISTRO_ARCH") != d.getVar("HOST_ARCH") else '${DEBREPO_TARGET_DIR}'}"
+
do_sbomchroot_deploy[dirs] = "${SBOM_DIR}"
do_sbomchroot_deploy[network] = "${TASK_USE_SUDO}"
do_sbomchroot_deploy() {
@@ -18,3 +18,5 @@ SBUILD_CHROOT_PREINSTALL ?= " \
${@ 'libc6-dev:${DISTRO_ARCH} crossbuild-essential-${DISTRO_ARCH}' if bb.utils.to_boolean(d.getVar('ISAR_CROSS_COMPILE')) else ''} \
apt-utils \
"
+
+DEBREPO_WORKDIR = "${DEBREPO_HOST_DIR}"