From patchwork Wed Nov 6 08:21:10 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Anton Mikanovich X-Patchwork-Id: 3894 Return-Path: Received: from shymkent.ilbers.de ([unix socket]) by shymkent (Cyrus 2.5.10-Debian-2.5.10-3+deb9u2) with LMTPA; Wed, 06 Nov 2024 09:21:37 +0100 X-Sieve: CMU Sieve 2.4 Received: from mail-wr1-f64.google.com (mail-wr1-f64.google.com [209.85.221.64]) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPS id 4A68LZWH017008 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT) for ; Wed, 6 Nov 2024 09:21:36 +0100 Received: by mail-wr1-f64.google.com with SMTP id ffacd0b85a97d-37d4cf04be1sf3393308f8f.2 for ; Wed, 06 Nov 2024 00:21:36 -0800 (PST) ARC-Seal: i=2; a=rsa-sha256; t=1730881289; cv=pass; d=google.com; s=arc-20240605; b=W+Em9DcdOMdzDW4FWk/Sz6K+UkCwDZQ7jgQC3ztB0IfSY5puNB/6XG+nqhHgFytvbO 1N1DvxMcScMujdx0g4AahB9tfRH+Wta1ZN6VLCfLBFhWtCNaAupyKOKAlox1/Fbo9qwI +6qh0CDJvqH3ZQrm25LBMTildLmvb3AKKSyeATBtrIfrz22xAWN2rCAHG+GuxGIZ8EMU x8b12kO7ggxStDFdkIEsBt5at3vvpI8HLh2n/tlVpJkUraMhR8o3aaZLS/nqxoki1xiM ce1SXEmM+8xprg1wXCtESj4Yq5ML3QWgpKByDOpGI25zcgqi/bwfS2lSZhlEgPj1FR6o zIlQ== ARC-Message-Signature: i=2; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605; h=list-unsubscribe:list-subscribe:list-archive:list-help:list-post :list-id:mailing-list:precedence:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:sender:dkim-signature; bh=EuRTUONUeJ/2BNLp9TxuE5zZassyloqOuCbb3TCEFMI=; fh=1L5HGb6QbC1WZBHmPhspVdHwCQa78w1vLbiWn1E+Mk8=; b=DhRnvvafj9PHgb00KtoRp2cLVnfPlJNyBCROM07PXUUL3QgKpZlCHfPtZeoZJ/V75T IZ95ouGSvt/j3GigTGXFq5TPeBsZJFa9hQ+R82nezEzsGe7e7mDdrUqgJNC/UMDAYFCY qlhBt/MlSpDOcvrtbgXU8ATA258mIPNUA7VpG7hXd16cX9ZrqpevqtSUQxN0kizevYLw flJuRc6FTHs5BQ0OME8+fzaPu7vW9hm2uimRg5WVfSWSNfzKO84IPvWiyoCJmjkuX4+f gh5+5jA/kNHeZm2nMC5hW9eMRfezbsFfqMjyKXl0MqKaa/GgeHgad/uw/FwBrFBqHmmg meqg==; darn=isar-build.org ARC-Authentication-Results: i=2; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=googlegroups.com; s=20230601; t=1730881289; x=1731486089; darn=isar-build.org; h=list-unsubscribe:list-subscribe:list-archive:list-help:list-post :list-id:mailing-list:precedence:x-original-authentication-results :x-original-sender:mime-version:references:in-reply-to:message-id :date:subject:cc:to:from:sender:from:to:cc:subject:date:message-id :reply-to; bh=EuRTUONUeJ/2BNLp9TxuE5zZassyloqOuCbb3TCEFMI=; b=rX69d/qbxHPL7tK6gFzc8I2R4lE9NJ5L3yam/C5MQqN8Z3dleypMkPmDMHE1e6m+IC iOWpaFG4XLgguekzBW5Iw66tSddzhyuRsF3keZ5cyJOhM15fz//5u+E2gA0p05N+iyx4 wBxWyeHGqDekbpSQE1jQwTeDIDkfWFZ6YIhmPtTZvXNkhXpfXjy2liIcQ93nsLQlVZtp t08o2dL2tBGOonx8Ev3nYlsBEcGDsveAchwdPZMyobnN7m3/kNsOBxD1ajGwFqMufKGq M/WwhSRE5bltXRBdomRDnbn7a6hYO5tiE2DRPK6JXuXNON6/BDrfs85Fu9Bpr/0dZGh+ 2aEQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1730881289; x=1731486089; h=list-unsubscribe:list-subscribe:list-archive:list-help:list-post :x-spam-checked-in-group:list-id:mailing-list:precedence :x-original-authentication-results:x-original-sender:mime-version :references:in-reply-to:message-id:date:subject:cc:to:from :x-beenthere:x-gm-message-state:sender:from:to:cc:subject:date :message-id:reply-to; bh=EuRTUONUeJ/2BNLp9TxuE5zZassyloqOuCbb3TCEFMI=; b=Lj0sorPjxklz2g26hNXnkoebN9N3u4xWT2g671XlxvFVtlHOyWh85f8faWca2YltHy So7rxywCO1ll6YgKJSPbk4XrmHR7UHhTQIggYQ4i2ao8oRzpx6K0UnpETv6EBdh9UUzQ JN45QLP/Ibmk3yWo0n477M0DPwAAdRoPv5KevQcVJ5Y97/N8iJy2GOVN/4/HsxeFutGN A5pgP98BYa0E7SJ715Tjl0d/MK0mRIVIwEBU1+0lmBR0UfJIr6QpEbCszjNI0cWG6yyR 4G3qadteVdz0QNz5Zf1R26fG10hX4quQuvCC7AN55DmIEOLSeeycZnm74sxLi2o+D4Kt C6Jg== Sender: isar-users@googlegroups.com X-Forwarded-Encrypted: i=2; AJvYcCW6PxKGZP9KyEtGanPYPDnIhZlhg2YW/GrWMYyrVvHV0xp4FYWsGhl1rs0Z4YFjs3wCbi1xPJc=@isar-build.org X-Gm-Message-State: AOJu0YyRWr0HvYzLPDriYDzTXfst9PZUt2md3ubEOBnKM0MHUXkSqn7a HxV+rmGg5YR3XR/iXhqIOCxyq+6R/uQyuziZiI9Ajdiamwst2p+F X-Google-Smtp-Source: AGHT+IHnKuwfH+LOGpeGvyE/HiwlqHzXiabHpLR/Bs1vwrtVh49s1pWrlrpV8U8PaxhL0baL5MS6zQ== X-Received: by 2002:a5d:6c68:0:b0:381:b68f:d14b with SMTP id ffacd0b85a97d-381bea1bfc4mr18994930f8f.45.1730881288945; Wed, 06 Nov 2024 00:21:28 -0800 (PST) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:600c:1c81:b0:431:405a:f96f with SMTP id 5b1f17b1804b1-4327b6d53d0ls34604405e9.0.-pod-prod-02-eu; Wed, 06 Nov 2024 00:21:27 -0800 (PST) X-Received: by 2002:a05:600c:a09:b0:42c:b52b:4335 with SMTP id 5b1f17b1804b1-4327b6fda39mr185824965e9.10.1730881286641; Wed, 06 Nov 2024 00:21:26 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1730881286; cv=none; d=google.com; s=arc-20240605; b=D/Fo53aMXCtfKVyijoXEyZTcN/L7Dq9v3wFYjiTjeBfOAeBpIUS/NnTZO1EWPYxTsb QcSvdLhaOtbMjOwHzuXu65YZHRTFjPGRoYMmjMWpNe4uubqOW84r4IoGSmPUr1yXK9SK yBEWMkJDB+3+MEgrTxRYEfYQ5wlxWhkzSTFxZmu7yAwmDBHcK+uIYOt4+gNPRWHyUBYE l019DJyuEt0VuqzhSCRVObJF3Gf5xjOf4ucNNCqMQXzfJk1j3F9K9lcPWvgTxYI3M7LH dsAkondAmzzNBOZXr+ATn9IQA4wLxjg9jg19J+Z09ht6xorDtjezrvQVBXjkbTiDgovM MLyw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=c6mda56arW427J628WW74gm4Nr7s//H874fbor8aOGc=; fh=/h9QQkzJ8EboVkWg45aWwpaUro6WMavIVd2OhN45RtE=; b=huRpDvxYwK4BiBzVmA21Fg5Ze37b3aLgUw7gw+Aits1hZPAGzXqa0Dc2Idadd123Zy lWar/l6Wiyaxhv0RZlf1Q4Sjzu0HD/nm6YCWTj1XmPTfCrHsy23bz/+yWhoHkQJ14Bwd fQFJ+lWD1b3bYeeCbfgWFHLk9pJbfm4kJJRUxUR9X95EPvmXhwArSdK/zMF+j3R7Q6Vf Jtk92PnzEbbNCWukT1Iq2Sp2iMooWGsOcbkXARLgP8HdtP1VIJ+sglS3IRchsAfjVzPM sTr9CO2lR/X10AXjYWeVIkGwPsxtGTbchHd5tkYq1y1p4yy4Bm6N3kq8lHejNqFjPHEc Bb1A==; dara=google.com ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id 5b1f17b1804b1-432a368745esi1572995e9.1.2024.11.06.00.21.26 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Wed, 06 Nov 2024 00:21:26 -0800 (PST) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Received: from user-B660.promwad.corp ([159.148.83.114]) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPSA id 4A68LOjq016956 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Wed, 6 Nov 2024 09:21:25 +0100 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH v11 1/8] isar-bootstrap: Move common parts to bbclass Date: Wed, 6 Nov 2024 10:21:10 +0200 Message-Id: <20241106082117.1089554-2-amikan@ilbers.de> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20241106082117.1089554-1-amikan@ilbers.de> References: <20241106082117.1089554-1-amikan@ilbers.de> MIME-Version: 1.0 X-Spam-Status: No, score=-4.6 required=5.0 tests=DKIMWL_WL_MED,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_EF,HEADER_FROM_DIFFERENT_DOMAINS, MAILING_LIST_MULTI,RCVD_IN_DNSWL_BLOCKED,RCVD_IN_MSPIKE_H2, RCVD_IN_RP_CERTIFIED,RCVD_IN_RP_RNBL,RCVD_IN_RP_SAFE,SPF_PASS autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-Original-Sender: amikan@ilbers.de X-Original-Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Precedence: list Mailing-list: list isar-users@googlegroups.com; contact isar-users+owners@googlegroups.com List-ID: X-Spam-Checked-In-Group: isar-users@googlegroups.com X-Google-Group-Id: 914930254986 List-Post: , List-Help: , List-Archive: , List-Unsubscribe: , X-getmail-retrieved-from-mailbox: =?utf-8?q?INBOX?= To allow implementing other bootstrap recipes move some functionality supposed to be reusable to bbclass. Signed-off-by: Anton Mikanovich --- meta/classes/bootstrap.bbclass | 225 ++++++++++++++++++ .../isar-bootstrap/isar-bootstrap.inc | 219 +---------------- 2 files changed, 227 insertions(+), 217 deletions(-) create mode 100644 meta/classes/bootstrap.bbclass diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass new file mode 100644 index 00000000..870465a9 --- /dev/null +++ b/meta/classes/bootstrap.bbclass @@ -0,0 +1,225 @@ +# This software is a part of Isar. +# Copyright (c) Siemens AG, 2018-2021 +# Copyright (C) 2024 ilbers GmbH +# +# SPDX-License-Identifier: MIT + +FILESEXTRAPATHS:append = ":${BBPATH}" + +SRC_URI = " \ + file://locale \ + file://chroot-setup.sh \ + ${DISTRO_BOOTSTRAP_KEYS} \ + ${THIRD_PARTY_APT_KEYS}" + +BOOTSTRAP_FOR_HOST ?= "0" + +APTPREFS = "${WORKDIR}/apt-preferences" +APTSRCS = "${WORKDIR}/apt-sources" +APTSRCS_INIT = "${WORKDIR}/apt-sources-init" +DISTRO_BOOTSTRAP_KEYFILES = "" +THIRD_PARTY_APT_KEYFILES = "" +DISTRO_BOOTSTRAP_KEYS ?= "" +THIRD_PARTY_APT_KEYS ?= "" +DEPLOY_ISAR_BOOTSTRAP ?= "" +DISTRO_BOOTSTRAP_BASE_PACKAGES ??= "" +DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}" +BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}" +BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}" +ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}" + +python () { + distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split() + third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split() + topdir = d.getVar("TOPDIR") + + # The cached repo key can be both for bootstrapping and apt package + # installation afterwards. However, bootstrap will include the key into + # the rootfs automatically thus the right place is distro_bootstrap_keys. + + if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): + own_pub_key = d.getVar("BASE_REPO_KEY") + if own_pub_key: + distro_bootstrap_keys += own_pub_key.split() + + for key in distro_bootstrap_keys: + fetcher = bb.fetch2.Fetch([key], d) + filename = os.path.relpath(fetcher.localpath(key), topdir) + d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename) + + for key in third_party_apt_keys: + fetcher = bb.fetch2.Fetch([key], d) + filename = os.path.relpath(fetcher.localpath(key), topdir) + d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename) + + distro_apt_sources = get_aptsources_list(d) + for file in distro_apt_sources: + d.appendVar("SRC_URI", " file://%s" % file) + + distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" + for file in distro_apt_preferences.split(): + d.appendVar("SRC_URI", " file://%s" % file) +} + +def aggregate_files(d, file_list, file_out): + import shutil + + with open(file_out, "wb") as out_fd: + for entry in file_list: + entry_real = bb.parse.resolve_file(entry, d) + with open(entry_real, "rb") as in_fd: + shutil.copyfileobj(in_fd, out_fd, 1024*1024*10) + out_fd.write("\n".encode()) + +def parse_aptsources_list_line(source_list_line): + import re + + s = source_list_line.strip() + + if not s or s.startswith("#"): + return None + + type, s = re.split("\s+", s, maxsplit=1) + if type not in ["deb", "deb-src"]: + return None + + options = "" + options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s) + if options_match: + options = options_match.group(0).strip() + s = s[options_match.end():] + + source, s = re.split("\s+", s, maxsplit=1) + + if s.startswith("/"): + suite = "" + else: + suite, s = re.split("\s+", s, maxsplit=1) + + components = " ".join(s.split()) + + return [type, options, source, suite, components] + +def get_isar_apt_snapshot_date(d): + import time + source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP') + return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch))) + +def get_apt_source_mirror(d, aptsources_entry_list): + import re + + # this is executed during parsing. No error checking possible + use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) + snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR') + if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): + premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n" + elif use_snapshot and snapshot_mirror: + premirrors = snapshot_mirror + else: + premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or "" + mirror_list = [entry.split() + for entry in premirrors.split('\\n') + if any(entry)] + + for regex, replace in mirror_list: + match = re.search(regex, aptsources_entry_list[2]) + + if match: + new_aptsources_entry_list = aptsources_entry_list.copy() + new_aptsources_entry_list[2] = re.sub(regex, replace, + aptsources_entry_list[2], + count = 1) + if use_snapshot: + new_aptsources_entry_list[1] = "[check-valid-until=no]" + return new_aptsources_entry_list + + return aptsources_entry_list + +def aggregate_aptsources_list(d, file_list, file_out): + import shutil + + with open(file_out, "wb") as out_fd: + for entry in file_list: + entry_real = bb.parse.resolve_file(entry, d) + with open(entry_real, "r") as in_fd: + for line in in_fd: + parsed = parse_aptsources_list_line(line) + if parsed: + parsed = get_apt_source_mirror(d, parsed) + out_fd.write(" ".join(parsed).encode()) + else: + out_fd.write(line.encode()) + out_fd.write("\n".encode()) + out_fd.write("\n".encode()) + +def get_aptsources_list(d): + import errno + from collections import OrderedDict + apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES" + apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split())) + for p in apt_sources_list: + try: + bb.parse.resolve_file(p, d) + except FileNotFoundError as e: + bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"') + return apt_sources_list + +def generate_distro_sources(d): + apt_sources_list = get_aptsources_list(d) + for entry in apt_sources_list: + with open(bb.parse.resolve_file(entry, d), "r") as in_fd: + for line in in_fd: + parsed = parse_aptsources_list_line(line) + if parsed: + parsed = get_apt_source_mirror(d, parsed) + yield parsed + +def get_distro_primary_source_entry(d): + for source in generate_distro_sources(d): + if source[0] == "deb": + return source[2:] + bb.fatal('Invalid apt sources list') + +def get_distro_source(d): + return get_distro_primary_source_entry(d)[0] + +def get_distro_suite(d): + return get_distro_primary_source_entry(d)[1] + +def get_distro_components_argument(d): + components = get_distro_primary_source_entry(d)[2] + if components and components.strip(): + return "--components=" + ",".join(components.split()) + else: + return "" + +do_apt_config_prepare[dirs] = "${WORKDIR}" +do_apt_config_prepare[vardeps] += " \ + APTPREFS \ + ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \ + DEBDISTRONAME \ + APTSRCS \ + ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \ + DEPLOY_ISAR_BOOTSTRAP \ + ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \ + " +python do_apt_config_prepare() { + apt_preferences_out = d.getVar("APTPREFS") + apt_preferences_list = ( + d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" + ).split() + aggregate_files(d, apt_preferences_list, apt_preferences_out) + + apt_sources_out = d.getVar("APTSRCS") + apt_sources_init_out = d.getVar("APTSRCS_INIT") + apt_sources_list = get_aptsources_list(d) + + aggregate_files(d, apt_sources_list, apt_sources_init_out) + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) +} +addtask apt_config_prepare before do_bootstrap after do_unpack + +CLEANFUNCS = "clean_deploy" +clean_deploy() { + rm -f "${DEPLOY_ISAR_BOOTSTRAP}" +} diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index e155aade..0e5ab57d 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -5,191 +5,20 @@ # # SPDX-License-Identifier: MIT +inherit bootstrap + LICENSE = "gpl-2.0" LIC_FILES_CHKSUM = "file://${LAYERDIR_core}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe" -FILESPATH:prepend := "${THISDIR}/files:" -SRC_URI = " \ - file://locale \ - file://chroot-setup.sh \ - ${DISTRO_BOOTSTRAP_KEYS} \ - ${THIRD_PARTY_APT_KEYS}" PV = "1.0" -BOOTSTRAP_FOR_HOST ?= "0" - DEBOOTSTRAP ?= "qemu-debootstrap" ROOTFSDIR = "${WORKDIR}/rootfs" -APTPREFS = "${WORKDIR}/apt-preferences" -APTSRCS = "${WORKDIR}/apt-sources" -APTSRCS_INIT = "${WORKDIR}/apt-sources-init" -DISTRO_BOOTSTRAP_KEYFILES = "" -THIRD_PARTY_APT_KEYFILES = "" -DISTRO_BOOTSTRAP_KEYS ?= "" -THIRD_PARTY_APT_KEYS ?= "" -DEPLOY_ISAR_BOOTSTRAP ?= "" DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales" DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg" DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates" -DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}" -BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}" -BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}" -FILESEXTRAPATHS:append = ":${BBPATH}" -ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}" inherit deb-dl-dir -python () { - distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split() - third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split() - topdir = d.getVar("TOPDIR") - - # The cached repo key can be both for bootstrapping and apt package - # installation afterwards. However, debootstrap will include the key into - # the rootfs automatically thus the right place is distro_bootstrap_keys. - - if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): - own_pub_key = d.getVar("BASE_REPO_KEY") - if own_pub_key: - distro_bootstrap_keys += own_pub_key.split() - - for key in distro_bootstrap_keys: - fetcher = bb.fetch2.Fetch([key], d) - filename = os.path.relpath(fetcher.localpath(key), topdir) - d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename) - - for key in third_party_apt_keys: - fetcher = bb.fetch2.Fetch([key], d) - filename = os.path.relpath(fetcher.localpath(key), topdir) - d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename) - - distro_apt_sources = get_aptsources_list(d) - for file in distro_apt_sources: - d.appendVar("SRC_URI", " file://%s" % file) - - distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" - for file in distro_apt_preferences.split(): - d.appendVar("SRC_URI", " file://%s" % file) -} - -def aggregate_files(d, file_list, file_out): - import shutil - - with open(file_out, "wb") as out_fd: - for entry in file_list: - entry_real = bb.parse.resolve_file(entry, d) - with open(entry_real, "rb") as in_fd: - shutil.copyfileobj(in_fd, out_fd, 1024*1024*10) - out_fd.write("\n".encode()) - -def parse_aptsources_list_line(source_list_line): - import re - - s = source_list_line.strip() - - if not s or s.startswith("#"): - return None - - type, s = re.split("\s+", s, maxsplit=1) - if type not in ["deb", "deb-src"]: - return None - - options = "" - options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s) - if options_match: - options = options_match.group(0).strip() - s = s[options_match.end():] - - source, s = re.split("\s+", s, maxsplit=1) - - if s.startswith("/"): - suite = "" - else: - suite, s = re.split("\s+", s, maxsplit=1) - - components = " ".join(s.split()) - - return [type, options, source, suite, components] - -def get_isar_apt_snapshot_date(d): - import time - source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP') - return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch))) - -def get_apt_source_mirror(d, aptsources_entry_list): - import re - - # this is executed during parsing. No error checking possible - use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) - snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR') - if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): - premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n" - elif use_snapshot and snapshot_mirror: - premirrors = snapshot_mirror - else: - premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or "" - mirror_list = [entry.split() - for entry in premirrors.split('\\n') - if any(entry)] - - for regex, replace in mirror_list: - match = re.search(regex, aptsources_entry_list[2]) - - if match: - new_aptsources_entry_list = aptsources_entry_list.copy() - new_aptsources_entry_list[2] = re.sub(regex, replace, - aptsources_entry_list[2], - count = 1) - if use_snapshot: - new_aptsources_entry_list[1] = "[check-valid-until=no]" - return new_aptsources_entry_list - - return aptsources_entry_list - -def aggregate_aptsources_list(d, file_list, file_out): - import shutil - - with open(file_out, "wb") as out_fd: - for entry in file_list: - entry_real = bb.parse.resolve_file(entry, d) - with open(entry_real, "r") as in_fd: - for line in in_fd: - parsed = parse_aptsources_list_line(line) - if parsed: - parsed = get_apt_source_mirror(d, parsed) - out_fd.write(" ".join(parsed).encode()) - else: - out_fd.write(line.encode()) - out_fd.write("\n".encode()) - out_fd.write("\n".encode()) - -def get_aptsources_list(d): - import errno - from collections import OrderedDict - apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES" - apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split())) - for p in apt_sources_list: - try: - bb.parse.resolve_file(p, d) - except FileNotFoundError as e: - bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"') - return apt_sources_list - -def generate_distro_sources(d): - apt_sources_list = get_aptsources_list(d) - for entry in apt_sources_list: - with open(bb.parse.resolve_file(entry, d), "r") as in_fd: - for line in in_fd: - parsed = parse_aptsources_list_line(line) - if parsed: - parsed = get_apt_source_mirror(d, parsed) - yield parsed - -def get_distro_primary_source_entry(d): - for source in generate_distro_sources(d): - if source[0] == "deb": - return source[2:] - bb.fatal('Invalid apt sources list') - def get_distro_have_https_source(d): return any(source[2].startswith("https://") for source in generate_distro_sources(d)) @@ -211,19 +40,6 @@ def get_distro_needs_gpg_support(d): OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}" -def get_distro_source(d): - return get_distro_primary_source_entry(d)[0] - -def get_distro_suite(d): - return get_distro_primary_source_entry(d)[1] - -def get_distro_components_argument(d): - components = get_distro_primary_source_entry(d)[2] - if components and components.strip(): - return "--components=" + ",".join(components.split()) - else: - return "" - APT_KEYS_DIR = "${WORKDIR}/aptkeys" DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg" @@ -247,32 +63,6 @@ do_generate_keyrings() { } addtask generate_keyrings before do_build after do_unpack -do_apt_config_prepare[dirs] = "${WORKDIR}" -do_apt_config_prepare[vardeps] += " \ - APTPREFS \ - ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \ - DEBDISTRONAME \ - APTSRCS \ - ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \ - DEPLOY_ISAR_BOOTSTRAP \ - ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \ - " -python do_apt_config_prepare() { - apt_preferences_out = d.getVar("APTPREFS") - apt_preferences_list = ( - d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" - ).split() - aggregate_files(d, apt_preferences_list, apt_preferences_out) - - apt_sources_out = d.getVar("APTSRCS") - apt_sources_init_out = d.getVar("APTSRCS_INIT") - apt_sources_list = get_aptsources_list(d) - - aggregate_files(d, apt_sources_list, apt_sources_init_out) - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) -} -addtask apt_config_prepare before do_bootstrap after do_unpack - def get_host_release(): import platform rel = platform.release() @@ -448,8 +238,3 @@ python do_bootstrap_setscene() { addtask do_bootstrap_setscene do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}" - -CLEANFUNCS = "clean_deploy" -clean_deploy() { - rm -f "${DEPLOY_ISAR_BOOTSTRAP}" -}