diff mbox series

[v3] Convert apt source fetcher into native bitbake variant

Message ID e20926f4-4da7-44b9-81d2-b1b19e707e80@siemens.com
State Accepted, archived
Headers show
Series [v3] Convert apt source fetcher into native bitbake variant | expand

Commit Message

Jan Kiszka Nov. 29, 2024, 2:05 p.m. UTC 
From: Jan Kiszka <jan.kiszka@siemens.com>

There is no major functional difference, but we no longer have to
manipulate SRC_URI by registering an official fetcher for apt://.

As the fetching no longer takes place in separate tasks, do_fetch and
do_unpack need to gain the extra flags that were so far assigned to
apt_fetch and apt_unpack. That happens conditionally, i.e. only if
SRC_URI actually contains an apt URL.

One difference to the original version is the possibility - even if
practically of minor relevance - to unpack multiple apt sources into S.
The old version contained a loop but was directing dpkg-source to a
pre-existing dir which would have failed on the second iteration. The
new version now folds the results together after each step.

There is no progress report realized because dpkg-source unfortunately
does not provide information upfront to make this predictable, thus
expressible in form of percentage.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Changes in v3:
 - keep split by distro codename in downloads/deb-src

 meta/classes/dpkg-base.bbclass | 104 ++++-----------------------------
 meta/lib/aptsrc_fetcher.py     |  88 ++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 93 deletions(-)
 create mode 100644 meta/lib/aptsrc_fetcher.py

Comments

Uladzimir Bely Dec. 6, 2024, 7:30 a.m. UTC | #1 
On Fri, 2024-11-29 at 22:05 +0800, Jan Kiszka wrote:
> From: Jan Kiszka <jan.kiszka@siemens.com>
> 
> There is no major functional difference, but we no longer have to
> manipulate SRC_URI by registering an official fetcher for apt://.
> 
> As the fetching no longer takes place in separate tasks, do_fetch and
> do_unpack need to gain the extra flags that were so far assigned to
> apt_fetch and apt_unpack. That happens conditionally, i.e. only if
> SRC_URI actually contains an apt URL.
> 
> One difference to the original version is the possibility - even if
> practically of minor relevance - to unpack multiple apt sources into
> S.
> The old version contained a loop but was directing dpkg-source to a
> pre-existing dir which would have failed on the second iteration. The
> new version now folds the results together after each step.
> 
> There is no progress report realized because dpkg-source
> unfortunately
> does not provide information upfront to make this predictable, thus
> expressible in form of percentage.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> Changes in v3:
>  - keep split by distro codename in downloads/deb-src
> 
>  meta/classes/dpkg-base.bbclass | 104 ++++---------------------------
> --
>  meta/lib/aptsrc_fetcher.py     |  88 ++++++++++++++++++++++++++++
>  2 files changed, 99 insertions(+), 93 deletions(-)
>  create mode 100644 meta/lib/aptsrc_fetcher.py
> 
> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-
> base.bbclass
> index b4ea8e17..c02c07a8 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -79,110 +79,28 @@ do_adjust_git[lockfiles] +=
> "${DL_DIR}/git/isar.lock"
>  inherit patch
>  addtask patch after do_adjust_git
>  
> -SRC_APT ?= ""
> -
> -# filter out all "apt://" URIs out of SRC_URI and stick them into
> SRC_APT
>  python() {
> -    src_uri = (d.getVar('SRC_URI', False) or "").split()
> +    from bb.fetch2 import methods
>  
> -    prefix = "apt://"
> -    src_apt = []
> -    for u in src_uri:
> -        if u.startswith(prefix):
> -            src_apt.append(u[len(prefix) :])
> -            d.setVar('SRC_URI:remove', u)
> +    # apt-src fetcher
> +    import aptsrc_fetcher
> +    methods.append(aptsrc_fetcher.AptSrc())
>  
> -    d.prependVar('SRC_APT', ' '.join(src_apt))
> +    src_uri = (d.getVar('SRC_URI', False) or "").split()
> +    for u in src_uri:
> +        if u.startswith("apt://"):
> +            d.appendVarFlag('do_fetch', 'depends',
> d.getVar('SCHROOT_DEP'))
>  
> -    if len(d.getVar('SRC_APT').strip()) > 0:
> -        bb.build.addtask('apt_unpack', 'do_patch', '', d)
> -        bb.build.addtask('cleanall_apt', 'do_cleanall', '', d)
> +            d.appendVarFlag('do_unpack', 'cleandirs', d.getVar('S'))
> +            d.setVarFlag('do_unpack', 'network',
> d.getVar('TASK_USE_SUDO'))
> +            break
>  
>      # container docker fetcher
>      import container_fetcher
> -    from bb.fetch2 import methods
>  
>      methods.append(container_fetcher.Container())
>  }
>  
> -do_apt_fetch() {
> -    E="${@ isar_export_proxies(d)}"
> -    schroot_create_configs
> -
> -    session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
> -    echo "Started session: ${session_id}"
> -
> -    schroot_cleanup() {
> -        schroot -q -f -e -c ${session_id} > /dev/null 2>&1
> -        schroot_delete_configs
> -    }
> -    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> -    trap 'schroot_cleanup' EXIT
> -
> -    schroot -r -c ${session_id} -d / -u root -- \
> -        rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> -    schroot -r -c ${session_id} -d / -- \
> -        sh -c '
> -            set -e
> -            for uri in $2; do
> -                mkdir -p /downloads/deb-src/"$1"/${uri}
> -                cd /downloads/deb-src/"$1"/${uri}
> -                apt-get -y --download-only --only-source source
> ${uri}
> -            done' \
> -                my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> "${SRC_APT}"
> -
> -    schroot -e -c ${session_id}
> -    schroot_delete_configs
> -}
> -
> -addtask apt_fetch
> -do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
> -do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
> -
> -# Add dependency from the correct schroot: host or target
> -do_apt_fetch[depends] += "${SCHROOT_DEP}"
> -
> -do_apt_unpack() {
> -    rm -rf ${S}
> -    schroot_create_configs
> -
> -    session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
> -    echo "Started session: ${session_id}"
> -
> -    schroot_cleanup() {
> -        schroot -q -f -e -c ${session_id} > /dev/null 2>&1
> -        schroot_delete_configs
> -    }
> -    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> -    trap 'schroot_cleanup' EXIT
> -
> -    schroot -r -c ${session_id} -d / -u root -- \
> -        rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> -    schroot -r -c ${session_id} -d / -- \
> -        sh -c '
> -            set -e
> -            for uri in $2; do
> -                dscfile="$(apt-get -y -qq --print-uris --only-source
> source $uri | cut -d " " -f2 | grep -E "*.dsc")"
> -                cd ${PP}
> -                cp /downloads/deb-src/"${1}"/${uri}/* ${PP}
> -                dpkg-source -x "${dscfile}" "${PPS}"
> -            done' \
> -                my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> "${SRC_APT}"
> -
> -    schroot -e -c ${session_id}
> -    schroot_delete_configs
> -}
> -do_apt_unpack[network] = "${TASK_USE_SUDO}"
> -
> -addtask apt_unpack after do_apt_fetch
> -
> -do_cleanall_apt[nostamp] = "1"
> -do_cleanall_apt() {
> -    for uri in "${SRC_APT}"; do
> -        rm -rf "${DEBSRCDIR}/${BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}/$uri"
> -    done
> -}
> -
>  def get_package_srcdir(d):
>      s = os.path.abspath(d.getVar("S"))
>      workdir = os.path.abspath(d.getVar("WORKDIR"))
> diff --git a/meta/lib/aptsrc_fetcher.py b/meta/lib/aptsrc_fetcher.py
> new file mode 100644
> index 00000000..041f5445
> --- /dev/null
> +++ b/meta/lib/aptsrc_fetcher.py
> @@ -0,0 +1,88 @@
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2024
> +#
> +# SPDX-License-Identifier: MIT
> +
> +from bb.fetch2 import FetchError
> +from bb.fetch2 import FetchMethod
> +from bb.fetch2 import logger
> +from bb.fetch2 import runfetchcmd
> +
> +class AptSrc(FetchMethod):
> +    def supports(self, ud, d):
> +        return ud.type in ['apt']
> +
> +    def urldata_init(self, ud, d):
> +        ud.src_package = ud.url[len('apt://'):]
> +        ud.host = ud.host.replace('=', '_')
> +
> +        base_distro = d.getVar('BASE_DISTRO')
> +        codename = d.getVar('BASE_DISTRO_CODENAME')
> +        ud.localfile='deb-src/' + base_distro + '-' + codename + '/'
> + ud.host
> +
> +    def download(self, ud, d):
> +        bb.utils.exec_flat_python_func('isar_export_proxies', d)
> +        bb.build.exec_func('schroot_create_configs', d)
> +
> +        sbuild_chroot = d.getVar('SBUILD_CHROOT')
> +        session_id = runfetchcmd(f'schroot -q -b -c
> {sbuild_chroot}', d).strip()
> +        logger.info(f'Started session: {session_id}')
> +
> +        repo_isar_dir = d.getVar('REPO_ISAR_DIR')
> +        lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
> +
> +        try:
> +            runfetchcmd(f'''
> +                set -e
> +                schroot -r -c {session_id} -d / -u root -- \
> +                    rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> +                schroot -r -c {session_id} -d / -- \
> +                    sh -c '
> +                        set -e
> +                        mkdir -p /downloads/{ud.localfile}
> +                        cd /downloads/{ud.localfile}
> +                        apt-get -y --download-only --only-source
> source {ud.src_package}
> +                        '
> +                ''', d)
> +        except (OSError, FetchError):
> +            raise
> +        finally:
> +            bb.utils.unlockfile(lockfile)
> +            runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
> +            bb.build.exec_func('schroot_delete_configs', d)
> +
> +    def unpack(self, ud, rootdir, d):
> +        bb.build.exec_func('schroot_create_configs', d)
> +
> +        sbuild_chroot = d.getVar('SBUILD_CHROOT')
> +        session_id = runfetchcmd(f'schroot -q -b -c
> {sbuild_chroot}', d).strip()
> +        logger.info(f'Started session: {session_id}')
> +
> +        pp = d.getVar('PP')
> +        pps = d.getVar('PPS')
> +        try:
> +            runfetchcmd(f'''
> +                set -e
> +                schroot -r -c {session_id} -d / -u root -- \
> +                    rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> +                schroot -r -c {session_id} -d / -- \
> +                    sh -c '
> +                        set -e
> +                        dscfile=$(apt-get -y -qq --print-uris --
> only-source source {ud.src_package} | \
> +                                  cut -d " " -f2 | grep -E "\.dsc")
> +                        cp /downloads/{ud.localfile}/* {pp}
> +                        cd {pp}
> +                        mv -f {pps} {pps}.prev
> +                        dpkg-source -x "$dscfile" {pps}
> +                        find {pps}.prev -mindepth 1 -maxdepth 1 -
> exec mv {{}} {pps}/ \;
> +                        rmdir {pps}.prev
> +                        '
> +                ''', d)
> +        except (OSError, FetchError):
> +            raise
> +        finally:
> +            runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
> +            bb.build.exec_func('schroot_delete_configs', d)
> +
> +    def clean(self, ud, d):
> +        bb.utils.remove(ud.localpath, recurse=True)

Applied to next, thanks.
diff mbox series

Patch

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index b4ea8e17..c02c07a8 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -79,110 +79,28 @@  do_adjust_git[lockfiles] += "${DL_DIR}/git/isar.lock"
 inherit patch
 addtask patch after do_adjust_git
 
-SRC_APT ?= ""
-
-# filter out all "apt://" URIs out of SRC_URI and stick them into SRC_APT
 python() {
-    src_uri = (d.getVar('SRC_URI', False) or "").split()
+    from bb.fetch2 import methods
 
-    prefix = "apt://"
-    src_apt = []
-    for u in src_uri:
-        if u.startswith(prefix):
-            src_apt.append(u[len(prefix) :])
-            d.setVar('SRC_URI:remove', u)
+    # apt-src fetcher
+    import aptsrc_fetcher
+    methods.append(aptsrc_fetcher.AptSrc())
 
-    d.prependVar('SRC_APT', ' '.join(src_apt))
+    src_uri = (d.getVar('SRC_URI', False) or "").split()
+    for u in src_uri:
+        if u.startswith("apt://"):
+            d.appendVarFlag('do_fetch', 'depends', d.getVar('SCHROOT_DEP'))
 
-    if len(d.getVar('SRC_APT').strip()) > 0:
-        bb.build.addtask('apt_unpack', 'do_patch', '', d)
-        bb.build.addtask('cleanall_apt', 'do_cleanall', '', d)
+            d.appendVarFlag('do_unpack', 'cleandirs', d.getVar('S'))
+            d.setVarFlag('do_unpack', 'network', d.getVar('TASK_USE_SUDO'))
+            break
 
     # container docker fetcher
     import container_fetcher
-    from bb.fetch2 import methods
 
     methods.append(container_fetcher.Container())
 }
 
-do_apt_fetch() {
-    E="${@ isar_export_proxies(d)}"
-    schroot_create_configs
-
-    session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
-    echo "Started session: ${session_id}"
-
-    schroot_cleanup() {
-        schroot -q -f -e -c ${session_id} > /dev/null 2>&1
-        schroot_delete_configs
-    }
-    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
-    trap 'schroot_cleanup' EXIT
-
-    schroot -r -c ${session_id} -d / -u root -- \
-        rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
-    schroot -r -c ${session_id} -d / -- \
-        sh -c '
-            set -e
-            for uri in $2; do
-                mkdir -p /downloads/deb-src/"$1"/${uri}
-                cd /downloads/deb-src/"$1"/${uri}
-                apt-get -y --download-only --only-source source ${uri}
-            done' \
-                my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${SRC_APT}"
-
-    schroot -e -c ${session_id}
-    schroot_delete_configs
-}
-
-addtask apt_fetch
-do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
-do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
-
-# Add dependency from the correct schroot: host or target
-do_apt_fetch[depends] += "${SCHROOT_DEP}"
-
-do_apt_unpack() {
-    rm -rf ${S}
-    schroot_create_configs
-
-    session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
-    echo "Started session: ${session_id}"
-
-    schroot_cleanup() {
-        schroot -q -f -e -c ${session_id} > /dev/null 2>&1
-        schroot_delete_configs
-    }
-    trap 'exit 1' INT HUP QUIT TERM ALRM USR1
-    trap 'schroot_cleanup' EXIT
-
-    schroot -r -c ${session_id} -d / -u root -- \
-        rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
-    schroot -r -c ${session_id} -d / -- \
-        sh -c '
-            set -e
-            for uri in $2; do
-                dscfile="$(apt-get -y -qq --print-uris --only-source source $uri | cut -d " " -f2 | grep -E "*.dsc")"
-                cd ${PP}
-                cp /downloads/deb-src/"${1}"/${uri}/* ${PP}
-                dpkg-source -x "${dscfile}" "${PPS}"
-            done' \
-                my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${SRC_APT}"
-
-    schroot -e -c ${session_id}
-    schroot_delete_configs
-}
-do_apt_unpack[network] = "${TASK_USE_SUDO}"
-
-addtask apt_unpack after do_apt_fetch
-
-do_cleanall_apt[nostamp] = "1"
-do_cleanall_apt() {
-    for uri in "${SRC_APT}"; do
-        rm -rf "${DEBSRCDIR}/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}/$uri"
-    done
-}
-
 def get_package_srcdir(d):
     s = os.path.abspath(d.getVar("S"))
     workdir = os.path.abspath(d.getVar("WORKDIR"))
diff --git a/meta/lib/aptsrc_fetcher.py b/meta/lib/aptsrc_fetcher.py
new file mode 100644
index 00000000..041f5445
--- /dev/null
+++ b/meta/lib/aptsrc_fetcher.py
@@ -0,0 +1,88 @@ 
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+from bb.fetch2 import FetchError
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
+
+class AptSrc(FetchMethod):
+    def supports(self, ud, d):
+        return ud.type in ['apt']
+
+    def urldata_init(self, ud, d):
+        ud.src_package = ud.url[len('apt://'):]
+        ud.host = ud.host.replace('=', '_')
+
+        base_distro = d.getVar('BASE_DISTRO')
+        codename = d.getVar('BASE_DISTRO_CODENAME')
+        ud.localfile='deb-src/' + base_distro + '-' + codename + '/' + ud.host
+
+    def download(self, ud, d):
+        bb.utils.exec_flat_python_func('isar_export_proxies', d)
+        bb.build.exec_func('schroot_create_configs', d)
+
+        sbuild_chroot = d.getVar('SBUILD_CHROOT')
+        session_id = runfetchcmd(f'schroot -q -b -c {sbuild_chroot}', d).strip()
+        logger.info(f'Started session: {session_id}')
+
+        repo_isar_dir = d.getVar('REPO_ISAR_DIR')
+        lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
+
+        try:
+            runfetchcmd(f'''
+                set -e
+                schroot -r -c {session_id} -d / -u root -- \
+                    rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
+                schroot -r -c {session_id} -d / -- \
+                    sh -c '
+                        set -e
+                        mkdir -p /downloads/{ud.localfile}
+                        cd /downloads/{ud.localfile}
+                        apt-get -y --download-only --only-source source {ud.src_package}
+                        '
+                ''', d)
+        except (OSError, FetchError):
+            raise
+        finally:
+            bb.utils.unlockfile(lockfile)
+            runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
+            bb.build.exec_func('schroot_delete_configs', d)
+
+    def unpack(self, ud, rootdir, d):
+        bb.build.exec_func('schroot_create_configs', d)
+
+        sbuild_chroot = d.getVar('SBUILD_CHROOT')
+        session_id = runfetchcmd(f'schroot -q -b -c {sbuild_chroot}', d).strip()
+        logger.info(f'Started session: {session_id}')
+
+        pp = d.getVar('PP')
+        pps = d.getVar('PPS')
+        try:
+            runfetchcmd(f'''
+                set -e
+                schroot -r -c {session_id} -d / -u root -- \
+                    rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
+                schroot -r -c {session_id} -d / -- \
+                    sh -c '
+                        set -e
+                        dscfile=$(apt-get -y -qq --print-uris --only-source source {ud.src_package} | \
+                                  cut -d " " -f2 | grep -E "\.dsc")
+                        cp /downloads/{ud.localfile}/* {pp}
+                        cd {pp}
+                        mv -f {pps} {pps}.prev
+                        dpkg-source -x "$dscfile" {pps}
+                        find {pps}.prev -mindepth 1 -maxdepth 1 -exec mv {{}} {pps}/ \;
+                        rmdir {pps}.prev
+                        '
+                ''', d)
+        except (OSError, FetchError):
+            raise
+        finally:
+            runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
+            bb.build.exec_func('schroot_delete_configs', d)
+
+    def clean(self, ud, d):
+        bb.utils.remove(ud.localpath, recurse=True)