| Message ID | 20230726061457.2297344-1-felix.moessbauer@siemens.com |
|---|---|
| State | Accepted, archived |
| Headers | show |
| Series | [1/1] deduplicate apt sources.list based on filenames | expand |
On 7/26/23 08:14, 'Felix Moessbauer' via isar-users wrote: > When combining layers, it can happen that the same DISTRO_APT_SOURCES > entry is added multiple times. This creates duplicate entries in the > bootstrap.list, which is considered invalid by apt. To solve this, we > deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable. > In addition, this also avoids adding the same SRC_URI entry twice. > > Note, that this only solves the trivial case that the duplicate entries > come from the same file. However in case duplicate entries are added > from multiple files with different names, the :remove syntax can be > used to remove these for the affected targets. > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> > --- > meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++------- > 1 file changed, 6 insertions(+), 7 deletions(-) > > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > index b94ae0bd..8af73a9b 100644 > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > @@ -59,8 +59,8 @@ python () { > filename = os.path.relpath(fetcher.localpath(key), topdir) > d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename) > > - distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or "" > - for file in distro_apt_sources.split(): > + distro_apt_sources = get_aptsources_list(d) > + for file in distro_apt_sources: > d.appendVar("SRC_URI", " file://%s" % file) > > distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" > @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, file_out): > > def get_aptsources_list(d): > import errno > + from collections import OrderedDict > apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES" > - list = (d.getVar(apt_sources_var) or "").split() > + apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split())) Why not use `set()` if the goal is to remove all duplicates? Quirin > ret = [] > - for p in list: > + for p in apt_sources_list: > try: > f = bb.parse.resolve_file(p, d) > ret.append(f) > @@ -251,9 +252,7 @@ python do_apt_config_prepare() { > > apt_sources_out = d.getVar("APTSRCS") > apt_sources_init_out = d.getVar("APTSRCS_INIT") > - apt_sources_list = ( > - d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or "" > - ).split() > + apt_sources_list = get_aptsources_list(d) > > aggregate_files(d, apt_sources_list, apt_sources_init_out) > aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
On Wed, 2023-07-26 at 10:59 +0200, 'Gylstorff Quirin' via isar-users wrote: > > > On 7/26/23 08:14, 'Felix Moessbauer' via isar-users wrote: > > When combining layers, it can happen that the same > > DISTRO_APT_SOURCES > > entry is added multiple times. This creates duplicate entries in > > the > > bootstrap.list, which is considered invalid by apt. To solve this, > > we > > deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable. > > In addition, this also avoids adding the same SRC_URI entry twice. > > > > Note, that this only solves the trivial case that the duplicate > > entries > > come from the same file. However in case duplicate entries are > > added > > from multiple files with different names, the :remove syntax can be > > used to remove these for the affected targets. > > > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> > > --- > > meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++-- > > ----- > > 1 file changed, 6 insertions(+), 7 deletions(-) > > > > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > > index b94ae0bd..8af73a9b 100644 > > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > > @@ -59,8 +59,8 @@ python () { > > filename = os.path.relpath(fetcher.localpath(key), > > topdir) > > d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % > > filename) > > > > - distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + > > "DISTRO_APT_SOURCES") or "" > > - for file in distro_apt_sources.split(): > > + distro_apt_sources = get_aptsources_list(d) > > + for file in distro_apt_sources: > > d.appendVar("SRC_URI", " file://%s" % file) > > > > distro_apt_preferences = > > d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") > > or "" > > @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, > > file_out): > > > > def get_aptsources_list(d): > > import errno > > + from collections import OrderedDict > > apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + > > "DISTRO_APT_SOURCES" > > - list = (d.getVar(apt_sources_var) or "").split() > > + apt_sources_list = > > list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or > > "").split())) > Why not use `set()` if the goal is to remove all duplicates? The set() does not preserve the order of the entries. However, the order might be relevant as it influences the order of the entries in the apt sources list. With the OrderedDict approach, the order is preserved. Felix > > Quirin > > ret = [] > > - for p in list: > > + for p in apt_sources_list: > > try: > > f = bb.parse.resolve_file(p, d) > > ret.append(f) > > @@ -251,9 +252,7 @@ python do_apt_config_prepare() { > > > > apt_sources_out = d.getVar("APTSRCS") > > apt_sources_init_out = d.getVar("APTSRCS_INIT") > > - apt_sources_list = ( > > - d.getVar(d.getVar("DISTRO_VARS_PREFIX") + > > "DISTRO_APT_SOURCES") or "" > > - ).split() > > + apt_sources_list = get_aptsources_list(d) > > > > aggregate_files(d, apt_sources_list, apt_sources_init_out) > > aggregate_aptsources_list(d, apt_sources_list, > > apt_sources_out) >
On 7/27/23 05:26, MOESSBAUER, Felix (T CED INW-CN) wrote: > On Wed, 2023-07-26 at 10:59 +0200, 'Gylstorff Quirin' via isar-users > wrote: >> >> >> On 7/26/23 08:14, 'Felix Moessbauer' via isar-users wrote: >>> When combining layers, it can happen that the same >>> DISTRO_APT_SOURCES >>> entry is added multiple times. This creates duplicate entries in >>> the >>> bootstrap.list, which is considered invalid by apt. To solve this, >>> we >>> deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable. >>> In addition, this also avoids adding the same SRC_URI entry twice. >>> >>> Note, that this only solves the trivial case that the duplicate >>> entries >>> come from the same file. However in case duplicate entries are >>> added >>> from multiple files with different names, the :remove syntax can be >>> used to remove these for the affected targets. >>> >>> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> >>> --- >>> meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++-- >>> ----- >>> 1 file changed, 6 insertions(+), 7 deletions(-) >>> >>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >>> index b94ae0bd..8af73a9b 100644 >>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc >>> @@ -59,8 +59,8 @@ python () { >>> filename = os.path.relpath(fetcher.localpath(key), >>> topdir) >>> d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % >>> filename) >>> >>> - distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + >>> "DISTRO_APT_SOURCES") or "" >>> - for file in distro_apt_sources.split(): >>> + distro_apt_sources = get_aptsources_list(d) >>> + for file in distro_apt_sources: >>> d.appendVar("SRC_URI", " file://%s" % file) >>> >>> distro_apt_preferences = >>> d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") >>> or "" >>> @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, >>> file_out): >>> >>> def get_aptsources_list(d): >>> import errno >>> + from collections import OrderedDict >>> apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + >>> "DISTRO_APT_SOURCES" >>> - list = (d.getVar(apt_sources_var) or "").split() >>> + apt_sources_list = >>> list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or >>> "").split())) >> Why not use `set()` if the goal is to remove all duplicates? > > The set() does not preserve the order of the entries. However, the > order might be relevant as it influences the order of the entries in > the apt sources list. With the OrderedDict approach, the order is > preserved. > > Felix Thanks, the patch looks good to me. Quirin
On Wed, 2023-07-26 at 06:14 +0000, 'Felix Moessbauer' via isar-users wrote: > When combining layers, it can happen that the same DISTRO_APT_SOURCES > entry is added multiple times. This creates duplicate entries in the > bootstrap.list, which is considered invalid by apt. To solve this, we > deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable. > In addition, this also avoids adding the same SRC_URI entry twice. > > Note, that this only solves the trivial case that the duplicate > entries > come from the same file. However in case duplicate entries are added > from multiple files with different names, the :remove syntax can be > used to remove these for the affected targets. > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> > --- > meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++----- > -- > 1 file changed, 6 insertions(+), 7 deletions(-) > > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > index b94ae0bd..8af73a9b 100644 > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > @@ -59,8 +59,8 @@ python () { > filename = os.path.relpath(fetcher.localpath(key), topdir) > d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % > filename) > > - distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + > "DISTRO_APT_SOURCES") or "" > - for file in distro_apt_sources.split(): > + distro_apt_sources = get_aptsources_list(d) > + for file in distro_apt_sources: > d.appendVar("SRC_URI", " file://%s" % file) > > distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") > + "DISTRO_APT_PREFERENCES") or "" > @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, > file_out): > > def get_aptsources_list(d): > import errno > + from collections import OrderedDict > apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + > "DISTRO_APT_SOURCES" > - list = (d.getVar(apt_sources_var) or "").split() > + apt_sources_list = > list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split())) > ret = [] > - for p in list: > + for p in apt_sources_list: > try: > f = bb.parse.resolve_file(p, d) > ret.append(f) > @@ -251,9 +252,7 @@ python do_apt_config_prepare() { > > apt_sources_out = d.getVar("APTSRCS") > apt_sources_init_out = d.getVar("APTSRCS_INIT") > - apt_sources_list = ( > - d.getVar(d.getVar("DISTRO_VARS_PREFIX") + > "DISTRO_APT_SOURCES") or "" > - ).split() > + apt_sources_list = get_aptsources_list(d) > > aggregate_files(d, apt_sources_list, apt_sources_init_out) > aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) > -- > 2.34.1 > Applied to next, thanks.
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index b94ae0bd..8af73a9b 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -59,8 +59,8 @@ python () { filename = os.path.relpath(fetcher.localpath(key), topdir) d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename) - distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or "" - for file in distro_apt_sources.split(): + distro_apt_sources = get_aptsources_list(d) + for file in distro_apt_sources: d.appendVar("SRC_URI", " file://%s" % file) distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or "" @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, file_out): def get_aptsources_list(d): import errno + from collections import OrderedDict apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES" - list = (d.getVar(apt_sources_var) or "").split() + apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split())) ret = [] - for p in list: + for p in apt_sources_list: try: f = bb.parse.resolve_file(p, d) ret.append(f) @@ -251,9 +252,7 @@ python do_apt_config_prepare() { apt_sources_out = d.getVar("APTSRCS") apt_sources_init_out = d.getVar("APTSRCS_INIT") - apt_sources_list = ( - d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or "" - ).split() + apt_sources_list = get_aptsources_list(d) aggregate_files(d, apt_sources_list, apt_sources_init_out) aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
When combining layers, it can happen that the same DISTRO_APT_SOURCES entry is added multiple times. This creates duplicate entries in the bootstrap.list, which is considered invalid by apt. To solve this, we deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable. In addition, this also avoids adding the same SRC_URI entry twice. Note, that this only solves the trivial case that the duplicate entries come from the same file. However in case duplicate entries are added from multiple files with different names, the :remove syntax can be used to remove these for the affected targets. Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> --- meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)