diff mbox series

rootfs: Improve progress reporting

Message ID 92422bd2-f3a6-40e1-be76-149caefe8a6a@siemens.com
State Superseded, archived
Headers show
Series rootfs: Improve progress reporting | expand

Commit Message

Jan Kiszka Aug. 24, 2024, 6:55 a.m. UTC 
From: Jan Kiszka <jan.kiszka@siemens.com>

Add a custom progress reporter that visualizes the state of the
rootfs_install_pkgs_download and rootfs_install_pkgs_install functions
by parsing dpkg output. This is achieved by reading the number of
upgraded and new installed packages first and then distributing the
effort equally over each package.

There is another specialty: As do_rootfs_install already created a
MultiStageProgressReporter, we forward it via the metadata object and
report from functions as stage progress so that the bar moves forward
consistently.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Just to keep us entertained while waiting for builds to complete...

 meta/classes/rootfs.bbclass |  9 +++++
 meta/lib/rootfs_progress.py | 69 +++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 meta/lib/rootfs_progress.py

Comments

MOESSBAUER, Felix Sept. 3, 2024, 7:30 a.m. UTC | #1 
On Sat, 2024-08-24 at 08:55 +0200, 'Jan Kiszka' via isar-users wrote:
> From: Jan Kiszka <jan.kiszka@siemens.com>
> 
> Add a custom progress reporter that visualizes the state of the
> rootfs_install_pkgs_download and rootfs_install_pkgs_install
> functions
> by parsing dpkg output. This is achieved by reading the number of
> upgraded and new installed packages first and then distributing the
> effort equally over each package.
> 
> There is another specialty: As do_rootfs_install already created a
> MultiStageProgressReporter, we forward it via the metadata object and
> report from functions as stage progress so that the bar moves forward
> consistently.

Many thanks for implementing this. Now I know if I have time to get a
coffee or not ;)

I just hope the output of apt-get is stable enough.

Acked-by: Felix Moessbauer <felix.moessbauer@siemens.com>

Felix

> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> Just to keep us entertained while waiting for builds to complete...
> 
>  meta/classes/rootfs.bbclass |  9 +++++
>  meta/lib/rootfs_progress.py | 69
> +++++++++++++++++++++++++++++++++++++
>  2 files changed, 78 insertions(+)
>  create mode 100644 meta/lib/rootfs_progress.py
> 
> diff --git a/meta/classes/rootfs.bbclass
> b/meta/classes/rootfs.bbclass
> index fdb25eaa..f0abd795 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -76,6 +76,12 @@ rootfs_do_qemu() {
>  BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-
> host_${DISTRO}-${DISTRO_ARCH}"
>  BOOTSTRAP_SRC:${ROOTFS_ARCH} =
> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>  
> +def rootfs_extra_import(d):
> +    bb.utils._context["rootfs_progress"] =
> __import__("rootfs_progress")
> +    return ""
> +
> +ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}"
> +
>  rootfs_prepare[weight] = "25"
>  rootfs_prepare(){
>      sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> @@ -153,6 +159,7 @@ rootfs_import_package_cache() {
>  
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>  rootfs_install_pkgs_download[weight] = "600"
> +rootfs_install_pkgs_download[progress] =
> "custom:rootfs_progress.PkgsDownloadProgressHandler"
>  rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
>  rootfs_install_pkgs_download[network] =
> "${TASK_USE_NETWORK_AND_SUDO}"
>  rootfs_install_pkgs_download() {
> @@ -178,6 +185,7 @@ rootfs_install_clean_files() {
>  
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>  rootfs_install_pkgs_install[weight] = "8000"
> +rootfs_install_pkgs_install[progress] =
> "custom:rootfs_progress.PkgsInstallProgressHandler"
>  rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}"
>  rootfs_install_pkgs_install() {
>      sudo -E chroot "${ROOTFSDIR}" \
> @@ -206,6 +214,7 @@ python do_rootfs_install() {
>                       for i in cmds]
>  
>      progress_reporter = bb.progress.MultiStageProgressReporter(d,
> stage_weights)
> +    d.rootfs_progress = progress_reporter
>  
>      for cmd in cmds:
>          progress_reporter.next_stage()
> diff --git a/meta/lib/rootfs_progress.py
> b/meta/lib/rootfs_progress.py
> new file mode 100644
> index 00000000..88299fd7
> --- /dev/null
> +++ b/meta/lib/rootfs_progress.py
> @@ -0,0 +1,69 @@
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2024
> +#
> +# SPDX-License-Identifier: MIT
> +
> +import bb.progress
> +import re
> +
> +
> +class PkgsProgressHandler(bb.progress.ProgressHandler):
> +    def __init__(self, d, outfile):
> +        self._outfile = outfile
> +        self._progress = d.rootfs_progress
> +        self._progress.update(0)
> +        self._linebuffer = ''
> +        self._num_pkgs = 0
> +        self._pkg = 0
> +        self._stage = 'prepare'
> +
> +    def write(self, string):
> +        self._outfile.write(string)
> +        self._linebuffer += string
> +        while True:
> +            breakpos = self._linebuffer.find('\n') + 1
> +            if breakpos == 0:
> +                break
> +            line = self._linebuffer[:breakpos]
> +            self._linebuffer = self._linebuffer[breakpos:]
> +
> +            if self._stage == 'prepare':
> +                match = re.search(
> +                    r'^([0-9]+) upgraded, ([0-9]+) newly installed',
> line)
> +                if match:
> +                    self._num_pkgs = int(match.group(1)) +
> int(match.group(2))
> +                    self._stage = 'post-prepare'
> +                    self._progress.update(1)
> +            else:
> +                self.process_line(line)
> +
> +    def process_line(self, line):
> +        return
> +
> +
> +class PkgsDownloadProgressHandler(PkgsProgressHandler):
> +    def __init__(self, d, outfile, otherargs=None):
> +        super().__init__(d, outfile)
> +
> +    def process_line(self, line):
> +        if line.startswith('Get:'):
> +            self._pkg += 1
> +            self._progress.update(1 + 99 * self._pkg /
> self._num_pkgs)
> +
> +
> +class PkgsInstallProgressHandler(PkgsProgressHandler):
> +    def __init__(self, d, outfile, otherargs=None):
> +        super().__init__(d, outfile)
> +
> +    def process_line(self, line):
> +        if self._stage == 'post-prepare':
> +            if line.startswith('Preparing to unpack'):
> +                self._pkg += 1
> +                self._progress.update(1 + 49 * self._pkg /
> self._num_pkgs)
> +                if self._pkg == self._num_pkgs:
> +                    self._stage = 'setup'
> +                    self._pkg = 0
> +        else:
> +            if line.startswith('Setting up'):
> +                self._pkg += 1
> +                self._progress.update(50 + 50 * self._pkg /
> self._num_pkgs)
> -- 
> 2.43.0
>
diff mbox series

Patch

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index fdb25eaa..f0abd795 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -76,6 +76,12 @@  rootfs_do_qemu() {
 BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
 BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
 
+def rootfs_extra_import(d):
+    bb.utils._context["rootfs_progress"] = __import__("rootfs_progress")
+    return ""
+
+ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}"
+
 rootfs_prepare[weight] = "25"
 rootfs_prepare(){
     sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
@@ -153,6 +159,7 @@  rootfs_import_package_cache() {
 
 ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
 rootfs_install_pkgs_download[weight] = "600"
+rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadProgressHandler"
 rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
 rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 rootfs_install_pkgs_download() {
@@ -178,6 +185,7 @@  rootfs_install_clean_files() {
 
 ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
 rootfs_install_pkgs_install[weight] = "8000"
+rootfs_install_pkgs_install[progress] = "custom:rootfs_progress.PkgsInstallProgressHandler"
 rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}"
 rootfs_install_pkgs_install() {
     sudo -E chroot "${ROOTFSDIR}" \
@@ -206,6 +214,7 @@  python do_rootfs_install() {
                      for i in cmds]
 
     progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights)
+    d.rootfs_progress = progress_reporter
 
     for cmd in cmds:
         progress_reporter.next_stage()
diff --git a/meta/lib/rootfs_progress.py b/meta/lib/rootfs_progress.py
new file mode 100644
index 00000000..88299fd7
--- /dev/null
+++ b/meta/lib/rootfs_progress.py
@@ -0,0 +1,69 @@ 
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+import bb.progress
+import re
+
+
+class PkgsProgressHandler(bb.progress.ProgressHandler):
+    def __init__(self, d, outfile):
+        self._outfile = outfile
+        self._progress = d.rootfs_progress
+        self._progress.update(0)
+        self._linebuffer = ''
+        self._num_pkgs = 0
+        self._pkg = 0
+        self._stage = 'prepare'
+
+    def write(self, string):
+        self._outfile.write(string)
+        self._linebuffer += string
+        while True:
+            breakpos = self._linebuffer.find('\n') + 1
+            if breakpos == 0:
+                break
+            line = self._linebuffer[:breakpos]
+            self._linebuffer = self._linebuffer[breakpos:]
+
+            if self._stage == 'prepare':
+                match = re.search(
+                    r'^([0-9]+) upgraded, ([0-9]+) newly installed', line)
+                if match:
+                    self._num_pkgs = int(match.group(1)) + int(match.group(2))
+                    self._stage = 'post-prepare'
+                    self._progress.update(1)
+            else:
+                self.process_line(line)
+
+    def process_line(self, line):
+        return
+
+
+class PkgsDownloadProgressHandler(PkgsProgressHandler):
+    def __init__(self, d, outfile, otherargs=None):
+        super().__init__(d, outfile)
+
+    def process_line(self, line):
+        if line.startswith('Get:'):
+            self._pkg += 1
+            self._progress.update(1 + 99 * self._pkg / self._num_pkgs)
+
+
+class PkgsInstallProgressHandler(PkgsProgressHandler):
+    def __init__(self, d, outfile, otherargs=None):
+        super().__init__(d, outfile)
+
+    def process_line(self, line):
+        if self._stage == 'post-prepare':
+            if line.startswith('Preparing to unpack'):
+                self._pkg += 1
+                self._progress.update(1 + 49 * self._pkg / self._num_pkgs)
+                if self._pkg == self._num_pkgs:
+                    self._stage = 'setup'
+                    self._pkg = 0
+        else:
+            if line.startswith('Setting up'):
+                self._pkg += 1
+                self._progress.update(50 + 50 * self._pkg / self._num_pkgs)