| Message ID | fc1f610b-b174-4696-8ad5-b974f4d4bdc4@siemens.com |
|---|---|
| State | Accepted, archived |
| Headers | show |
| Series | [v2] rootfs: Improve progress reporting | expand |
On Sat, 2024-08-24 at 09:24 +0200, 'Jan Kiszka' via isar-users wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > Add a custom progress reporter that visualizes the state of the > rootfs_install_pkgs_download and rootfs_install_pkgs_install > functions > by parsing dpkg output. This is achieved by reading the number of > upgraded and new installed packages first and then distributing the > effort equally over each package. > > There is another specialty: As do_rootfs_install already created a > MultiStageProgressReporter, we forward it via the metadata object and > report from functions as stage progress so that the bar moves forward > consistently. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > > Changes in v2: > - catch all "Setting up" > - scale down to 99% max to account for that we already progress > before > a package step is done > > meta/classes/rootfs.bbclass | 9 +++++ > meta/lib/rootfs_progress.py | 67 > +++++++++++++++++++++++++++++++++++++ > 2 files changed, 76 insertions(+) > create mode 100644 meta/lib/rootfs_progress.py > > diff --git a/meta/classes/rootfs.bbclass > b/meta/classes/rootfs.bbclass > index fdb25eaa..f0abd795 100644 > --- a/meta/classes/rootfs.bbclass > +++ b/meta/classes/rootfs.bbclass > @@ -76,6 +76,12 @@ rootfs_do_qemu() { > BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}- > host_${DISTRO}-${DISTRO_ARCH}" > BOOTSTRAP_SRC:${ROOTFS_ARCH} = > "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}" > > +def rootfs_extra_import(d): > + bb.utils._context["rootfs_progress"] = > __import__("rootfs_progress") > + return "" > + > +ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}" > + > rootfs_prepare[weight] = "25" > rootfs_prepare(){ > sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}' > @@ -153,6 +159,7 @@ rootfs_import_package_cache() { > > ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download" > rootfs_install_pkgs_download[weight] = "600" > +rootfs_install_pkgs_download[progress] = > "custom:rootfs_progress.PkgsDownloadProgressHandler" > rootfs_install_pkgs_download[isar-apt-lock] = "release-after" > rootfs_install_pkgs_download[network] = > "${TASK_USE_NETWORK_AND_SUDO}" > rootfs_install_pkgs_download() { > @@ -178,6 +185,7 @@ rootfs_install_clean_files() { > > ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install" > rootfs_install_pkgs_install[weight] = "8000" > +rootfs_install_pkgs_install[progress] = > "custom:rootfs_progress.PkgsInstallProgressHandler" > rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}" > rootfs_install_pkgs_install() { > sudo -E chroot "${ROOTFSDIR}" \ > @@ -206,6 +214,7 @@ python do_rootfs_install() { > for i in cmds] > > progress_reporter = bb.progress.MultiStageProgressReporter(d, > stage_weights) > + d.rootfs_progress = progress_reporter > > for cmd in cmds: > progress_reporter.next_stage() > diff --git a/meta/lib/rootfs_progress.py > b/meta/lib/rootfs_progress.py > new file mode 100644 > index 00000000..f808852f > --- /dev/null > +++ b/meta/lib/rootfs_progress.py > @@ -0,0 +1,67 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +import bb.progress > +import re > + > + > +class PkgsProgressHandler(bb.progress.ProgressHandler): > + def __init__(self, d, outfile): > + self._outfile = outfile > + self._progress = d.rootfs_progress > + self._progress.update(0) > + self._linebuffer = '' > + self._num_pkgs = 0 > + self._pkg = 0 > + self._stage = 'prepare' > + > + def write(self, string): > + self._outfile.write(string) > + self._linebuffer += string > + while True: > + breakpos = self._linebuffer.find('\n') + 1 > + if breakpos == 0: > + break > + line = self._linebuffer[:breakpos] > + self._linebuffer = self._linebuffer[breakpos:] > + > + if self._stage == 'prepare': > + match = re.search( > + r'^([0-9]+) upgraded, ([0-9]+) newly installed', > line) > + if match: > + self._num_pkgs = int(match.group(1)) + > int(match.group(2)) > + self._stage = 'post-prepare' > + else: > + self.process_line(line) > + > + def process_line(self, line): > + return > + > + > +class PkgsDownloadProgressHandler(PkgsProgressHandler): > + def __init__(self, d, outfile, otherargs=None): > + super().__init__(d, outfile) > + > + def process_line(self, line): > + if line.startswith('Get:'): > + self._pkg += 1 > + self._progress.update(99 * self._pkg / self._num_pkgs) > + > + > +class PkgsInstallProgressHandler(PkgsProgressHandler): > + def __init__(self, d, outfile, otherargs=None): > + self._pkg_set_up = 0 > + super().__init__(d, outfile) > + > + def process_line(self, line): > + if line.startswith('Preparing to unpack'): > + self._pkg += 1 > + elif line.startswith('Setting up'): > + self._pkg_set_up += 1 > + else: > + return > + > + progress = 99 * (self._pkg + self._pkg_set_up) / > (self._num_pkgs * 2) > + self._progress.update(progress) > -- > 2.43.0 > Applied to next, thanks.
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass index fdb25eaa..f0abd795 100644 --- a/meta/classes/rootfs.bbclass +++ b/meta/classes/rootfs.bbclass @@ -76,6 +76,12 @@ rootfs_do_qemu() { BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}" BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}" +def rootfs_extra_import(d): + bb.utils._context["rootfs_progress"] = __import__("rootfs_progress") + return "" + +ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}" + rootfs_prepare[weight] = "25" rootfs_prepare(){ sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}' @@ -153,6 +159,7 @@ rootfs_import_package_cache() { ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download" rootfs_install_pkgs_download[weight] = "600" +rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadProgressHandler" rootfs_install_pkgs_download[isar-apt-lock] = "release-after" rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}" rootfs_install_pkgs_download() { @@ -178,6 +185,7 @@ rootfs_install_clean_files() { ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install" rootfs_install_pkgs_install[weight] = "8000" +rootfs_install_pkgs_install[progress] = "custom:rootfs_progress.PkgsInstallProgressHandler" rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}" rootfs_install_pkgs_install() { sudo -E chroot "${ROOTFSDIR}" \ @@ -206,6 +214,7 @@ python do_rootfs_install() { for i in cmds] progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights) + d.rootfs_progress = progress_reporter for cmd in cmds: progress_reporter.next_stage() diff --git a/meta/lib/rootfs_progress.py b/meta/lib/rootfs_progress.py new file mode 100644 index 00000000..f808852f --- /dev/null +++ b/meta/lib/rootfs_progress.py @@ -0,0 +1,67 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +import bb.progress +import re + + +class PkgsProgressHandler(bb.progress.ProgressHandler): + def __init__(self, d, outfile): + self._outfile = outfile + self._progress = d.rootfs_progress + self._progress.update(0) + self._linebuffer = '' + self._num_pkgs = 0 + self._pkg = 0 + self._stage = 'prepare' + + def write(self, string): + self._outfile.write(string) + self._linebuffer += string + while True: + breakpos = self._linebuffer.find('\n') + 1 + if breakpos == 0: + break + line = self._linebuffer[:breakpos] + self._linebuffer = self._linebuffer[breakpos:] + + if self._stage == 'prepare': + match = re.search( + r'^([0-9]+) upgraded, ([0-9]+) newly installed', line) + if match: + self._num_pkgs = int(match.group(1)) + int(match.group(2)) + self._stage = 'post-prepare' + else: + self.process_line(line) + + def process_line(self, line): + return + + +class PkgsDownloadProgressHandler(PkgsProgressHandler): + def __init__(self, d, outfile, otherargs=None): + super().__init__(d, outfile) + + def process_line(self, line): + if line.startswith('Get:'): + self._pkg += 1 + self._progress.update(99 * self._pkg / self._num_pkgs) + + +class PkgsInstallProgressHandler(PkgsProgressHandler): + def __init__(self, d, outfile, otherargs=None): + self._pkg_set_up = 0 + super().__init__(d, outfile) + + def process_line(self, line): + if line.startswith('Preparing to unpack'): + self._pkg += 1 + elif line.startswith('Setting up'): + self._pkg_set_up += 1 + else: + return + + progress = 99 * (self._pkg + self._pkg_set_up) / (self._num_pkgs * 2) + self._progress.update(progress)