diff mbox series

[1/1] testsuite: speedup execution by limiting mc parsing to selected targets

Message ID 20260717150012.1670718-1-felix.moessbauer@siemens.com
State Under Review
Headers show
Series [1/1] testsuite: speedup execution by limiting mc parsing to selected targets | expand

Commit Message

MOESSBAUER, Felix July 17, 2026, 3 p.m. UTC 
The global isar mc.conf has over 70 multiconfig targets defined. This
makes initial parsing super slow, leading to parse times of 5-10 minutes
on testsuite tasks (per avocado task execution). To speed this up, We now
set the BBMULTICONFIG bitbake variable to exactly the targets we want to
test. This reduces the parsing time of tests to ~10s per test.

When running with sstate cache, this massively speeds up repeated runs,
as the parse time was the biggest single contributor.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 testsuite/cibase.py           | 20 +++++++++++---------
 testsuite/cibuilder.py        | 17 +++++++++++++++++
 testsuite/repro-build-test.py |  1 +
 3 files changed, 29 insertions(+), 9 deletions(-)

Comments

MOESSBAUER, Felix July 17, 2026, 3:49 p.m. UTC | #1 
On Fri, 2026-07-17 at 17:00 +0200, Felix Moessbauer wrote:
> The global isar mc.conf has over 70 multiconfig targets defined. This
> makes initial parsing super slow, leading to parse times of 5-10 minutes
> on testsuite tasks (per avocado task execution). To speed this up, We now
> set the BBMULTICONFIG bitbake variable to exactly the targets we want to
> test. This reduces the parsing time of tests to ~10s per test.
> 
> When running with sstate cache, this massively speeds up repeated runs,
> as the parse time was the biggest single contributor.

Do not merge yet.

While the results are quite impressive, this seems to expose further
hidden dependencies between the build and run_vm parts. I'm currently
looking into it.

PS:

On a repeated build, these runtimes can be achieved (the error is also
visible):

./scripts/run-tests.sh -t dev -p rootless=1 -p sstate=1
+ avocado run --max-parallel-tasks=1 --disable-sysinfo -t dev -p
rootless=1 -p sstate=1 testsuite/citest.py
JOB ID     : 5f51d20d167dcaf5fccd414d273c5680bc018f08
JOB LOG    : /work/build/testsuite/job-results/job-2026-07-17T17.43-
5f51d20/job.log
 (1/6) testsuite/citest.py:DevTest.test_dev: STARTED
 (1/6) testsuite/citest.py:DevTest.test_dev: PASS (38.71 s)
 (2/6) testsuite/citest.py:DevTest.test_dev_apps: STARTED
 (2/6) testsuite/citest.py:DevTest.test_dev_apps: PASS (35.92 s)
 (3/6) testsuite/citest.py:DevTest.test_dev_rebuild: STARTED
 (3/6) testsuite/citest.py:DevTest.test_dev_rebuild: PASS (36.69 s)
 (4/6) testsuite/citest.py:DevTest.test_dev_run_amd64_bookworm: STARTED
 (4/6) testsuite/citest.py:DevTest.test_dev_run_amd64_bookworm: PASS
(30.71 s)
 (5/6) testsuite/citest.py:DevTest.test_dev_run_arm64_bookworm: STARTED
 (5/6) testsuite/citest.py:DevTest.test_dev_run_arm64_bookworm: ERROR:
Traceback (most recent call last):\n  File
"/isar/bitbake/lib/bb/command.py", line 90, in runCommand\n    result =
command_method(self, commandline)\n  File
"/isar/bitbake/lib/bb/command.py", line 470, in findBestProvider\n   
return command.cooker.findBestPr... (0.52 s)
 (6/6) testsuite/citest.py:DevTest.test_dev_run_arm_bookworm: STARTED
 (6/6) testsuite/citest.py:DevTest.test_dev_run_arm_bookworm: ERROR:
Traceback (most recent call last):\n  File
"/isar/bitbake/lib/bb/command.py", line 90, in runCommand\n    result =
command_method(self, commandline)\n  File
"/isar/bitbake/lib/bb/command.py", line 470, in findBestProvider\n   
return command.cooker.findBestPr... (0.54 s)
RESULTS    : PASS 4 | ERROR 2 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0
| CANCEL 0
JOB TIME   : 146.54 s

Test summary:
5-testsuite/citest.py:DevTest.test_dev_run_arm64_bookworm: ERROR
6-testsuite/citest.py:DevTest.test_dev_run_arm_bookworm: ERROR

Felix

> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  testsuite/cibase.py           | 20 +++++++++++---------
>  testsuite/cibuilder.py        | 17 +++++++++++++++++
>  testsuite/repro-build-test.py |  1 +
>  3 files changed, 29 insertions(+), 9 deletions(-)
> 
> diff --git a/testsuite/cibase.py b/testsuite/cibase.py
> index 4a6308d0..52748004 100755
> --- a/testsuite/cibase.py
> +++ b/testsuite/cibase.py
> @@ -15,7 +15,7 @@ from avocado.utils import process
>  
>  class CIBaseTest(CIBuilder):
>      def perform_build_test(self, targets, should_fail=False, **kwargs):
> -        self.configure(**kwargs)
> +        self.configure(targets=targets, **kwargs)
>  
>          if bool(int(self.params.get('depgraph', default=0))):
>              self.generate_dependency_graph(targets, reconfigure=False, **kwargs)
> @@ -31,7 +31,7 @@ class CIBaseTest(CIBuilder):
>                                    **kwargs):
>          """Debug helper to better understand test task graphs."""
>          if reconfigure:
> -            self.configure(**kwargs)
> +            self.configure(targets=targets, **kwargs)
>  
>          self.log.info("Generating dependency graph...")
>  
> @@ -41,7 +41,7 @@ class CIBaseTest(CIBuilder):
>          self.move_in_build_dir('pn-buildlist', f"pn-buildlist-{self.name}")
>  
>      def perform_wic_partition_test(self, targets, wic_deploy_parts, **kwargs):
> -        self.configure(wic_deploy_parts=wic_deploy_parts, **kwargs)
> +        self.configure(targets=targets, wic_deploy_parts=wic_deploy_parts, **kwargs)
>          self.bitbake(targets, **kwargs)
>  
>          wic_path = f"{self.build_dir}/tmp/deploy/images/*/*.wic.p1"
> @@ -57,6 +57,7 @@ class CIBaseTest(CIBuilder):
>          gpg_priv_key = os.path.join(keys_dir, 'test_priv.key')
>  
>          self.configure(
> +            targets=targets,
>              gpg_pub_key=gpg_pub_key if signed else None,
>              sstate_dir='',
>              **kwargs,
> @@ -76,6 +77,7 @@ class CIBaseTest(CIBuilder):
>              repro_type = 'signed' if signed else 'unsigned'
>              self.move_in_build_dir('tmp', f"tmp_middle_repro_{repro_type}")
>              self.configure(
> +                targets=targets,
>                  gpg_pub_key=gpg_pub_key if signed else None,
>                  offline=True,
>                  sstate_dir='',
> @@ -89,7 +91,7 @@ class CIBaseTest(CIBuilder):
>  
>              if not signed:
>                  # Try to build with changed configuration with no cleanup
> -                self.configure(**kwargs)
> +                self.configure(targets=targets, **kwargs)
>                  self.bitbake(targets, **kwargs)
>  
>          finally:
> @@ -109,7 +111,7 @@ class CIBaseTest(CIBuilder):
>                              count += int(content[field])
>              return count
>  
> -        self.configure(ccache=True, sstate_dir='', **kwargs)
> +        self.configure(targets=targets, ccache=True, sstate_dir='', **kwargs)
>  
>          # Field that stores direct ccache hits
>          direct_cache_hit = 22
> @@ -173,7 +175,7 @@ class CIBaseTest(CIBuilder):
>          process.run(f"git --work-tree={isar_sstate} checkout HEAD -- .")
>  
>          self.init('../build-sstate', isar_dir=isar_sstate)
> -        self.configure(sstate=True, sstate_dir='', **kwargs)
> +        self.configure(targets=image_target, sstate=True, sstate_dir='', **kwargs)
>  
>          # Cleanup sstate and tmp before test
>          self.delete_from_build_dir('sstate-cache')
> @@ -196,7 +198,7 @@ class CIBaseTest(CIBuilder):
>          """
>          Generate signature data for target(s) and check for cacheability issues
>          """
> -        self.configure(**kwargs)
> +        self.configure(targets=targets, **kwargs)
>          self.delete_from_build_dir('tmp_before_sstate')
>          self.move_in_build_dir('tmp', 'tmp_before_sstate')
>          self.bitbake(targets, sig_handler='none')
> @@ -246,7 +248,7 @@ class CIBaseTest(CIBuilder):
>                      return False
>              return True
>  
> -        self.configure(sstate=True, sstate_dir='', **kwargs)
> +        self.configure(targets=[image_target, package_target], sstate=True, sstate_dir='', **kwargs)
>  
>          deploy_dir = f"{self.build_dir}/tmp/deploy"
>  
> @@ -380,7 +382,7 @@ class CIBaseTest(CIBuilder):
>                         sfiles[target][fname]  = CIUtils.get_tar_content(fname)
>              return sfiles
>  
> -        self.configure(**kwargs)
> +        self.configure(targets=targets, **kwargs)
>  
>          tmp_layer_dir = self.create_tmp_layer()
>          try:
> diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py
> index d42e8631..1d415084 100755
> --- a/testsuite/cibuilder.py
> +++ b/testsuite/cibuilder.py
> @@ -104,8 +104,21 @@ class CIBuilder(Test):
>          if not hasattr(self, 'build_dir'):
>              self.error("Broken test implementation: need to call init().")
>  
> +    @staticmethod
> +    def _extract_multiconfigs(targets):
> +        if isinstance(targets, str):
> +            targets = [targets]
> +        mc_names = set()
> +        for target in targets:
> +            if target.startswith('mc:'):
> +                parts = target.split(':')
> +                if len(parts) >= 2:
> +                    mc_names.add(parts[1])
> +        return mc_names
> +
>      def configure(
>          self,
> +        targets=None,
>          compat_arch=True,
>          cross=True,
>          debsrc_cache=False,
> @@ -196,6 +209,10 @@ class CIBuilder(Test):
>  
>          # write ci_build.conf
>          with open(self.build_dir + '/conf/ci_build.conf', 'w') as f:
> +            if targets:
> +                mc_names = self._extract_multiconfigs(targets)
> +                if mc_names:
> +                    f.write('BBMULTICONFIG = "%s"\n\n' % ' '.join(sorted(mc_names)))
>              if compat_arch:
>                  f.write(
>                      'ISAR_ENABLE_COMPAT_ARCH:amd64 = "1"\n'
> diff --git a/testsuite/repro-build-test.py b/testsuite/repro-build-test.py
> index c06269f5..c1b51aff 100755
> --- a/testsuite/repro-build-test.py
> +++ b/testsuite/repro-build-test.py
> @@ -50,6 +50,7 @@ class ReproBuild(CIBuilder):
>          # Build
>          self.log.info("Started Build " + image_name)
>          self.configure(
> +            targets=target,
>              source_date_epoch=source_date_epoch, use_apt_snapshot=True
>          )
>          self.bitbake(target)
> -- 
> 2.53.0
diff mbox series

Patch

diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index 4a6308d0..52748004 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -15,7 +15,7 @@  from avocado.utils import process
 
 class CIBaseTest(CIBuilder):
     def perform_build_test(self, targets, should_fail=False, **kwargs):
-        self.configure(**kwargs)
+        self.configure(targets=targets, **kwargs)
 
         if bool(int(self.params.get('depgraph', default=0))):
             self.generate_dependency_graph(targets, reconfigure=False, **kwargs)
@@ -31,7 +31,7 @@  class CIBaseTest(CIBuilder):
                                   **kwargs):
         """Debug helper to better understand test task graphs."""
         if reconfigure:
-            self.configure(**kwargs)
+            self.configure(targets=targets, **kwargs)
 
         self.log.info("Generating dependency graph...")
 
@@ -41,7 +41,7 @@  class CIBaseTest(CIBuilder):
         self.move_in_build_dir('pn-buildlist', f"pn-buildlist-{self.name}")
 
     def perform_wic_partition_test(self, targets, wic_deploy_parts, **kwargs):
-        self.configure(wic_deploy_parts=wic_deploy_parts, **kwargs)
+        self.configure(targets=targets, wic_deploy_parts=wic_deploy_parts, **kwargs)
         self.bitbake(targets, **kwargs)
 
         wic_path = f"{self.build_dir}/tmp/deploy/images/*/*.wic.p1"
@@ -57,6 +57,7 @@  class CIBaseTest(CIBuilder):
         gpg_priv_key = os.path.join(keys_dir, 'test_priv.key')
 
         self.configure(
+            targets=targets,
             gpg_pub_key=gpg_pub_key if signed else None,
             sstate_dir='',
             **kwargs,
@@ -76,6 +77,7 @@  class CIBaseTest(CIBuilder):
             repro_type = 'signed' if signed else 'unsigned'
             self.move_in_build_dir('tmp', f"tmp_middle_repro_{repro_type}")
             self.configure(
+                targets=targets,
                 gpg_pub_key=gpg_pub_key if signed else None,
                 offline=True,
                 sstate_dir='',
@@ -89,7 +91,7 @@  class CIBaseTest(CIBuilder):
 
             if not signed:
                 # Try to build with changed configuration with no cleanup
-                self.configure(**kwargs)
+                self.configure(targets=targets, **kwargs)
                 self.bitbake(targets, **kwargs)
 
         finally:
@@ -109,7 +111,7 @@  class CIBaseTest(CIBuilder):
                             count += int(content[field])
             return count
 
-        self.configure(ccache=True, sstate_dir='', **kwargs)
+        self.configure(targets=targets, ccache=True, sstate_dir='', **kwargs)
 
         # Field that stores direct ccache hits
         direct_cache_hit = 22
@@ -173,7 +175,7 @@  class CIBaseTest(CIBuilder):
         process.run(f"git --work-tree={isar_sstate} checkout HEAD -- .")
 
         self.init('../build-sstate', isar_dir=isar_sstate)
-        self.configure(sstate=True, sstate_dir='', **kwargs)
+        self.configure(targets=image_target, sstate=True, sstate_dir='', **kwargs)
 
         # Cleanup sstate and tmp before test
         self.delete_from_build_dir('sstate-cache')
@@ -196,7 +198,7 @@  class CIBaseTest(CIBuilder):
         """
         Generate signature data for target(s) and check for cacheability issues
         """
-        self.configure(**kwargs)
+        self.configure(targets=targets, **kwargs)
         self.delete_from_build_dir('tmp_before_sstate')
         self.move_in_build_dir('tmp', 'tmp_before_sstate')
         self.bitbake(targets, sig_handler='none')
@@ -246,7 +248,7 @@  class CIBaseTest(CIBuilder):
                     return False
             return True
 
-        self.configure(sstate=True, sstate_dir='', **kwargs)
+        self.configure(targets=[image_target, package_target], sstate=True, sstate_dir='', **kwargs)
 
         deploy_dir = f"{self.build_dir}/tmp/deploy"
 
@@ -380,7 +382,7 @@  class CIBaseTest(CIBuilder):
                        sfiles[target][fname]  = CIUtils.get_tar_content(fname)
             return sfiles
 
-        self.configure(**kwargs)
+        self.configure(targets=targets, **kwargs)
 
         tmp_layer_dir = self.create_tmp_layer()
         try:
diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py
index d42e8631..1d415084 100755
--- a/testsuite/cibuilder.py
+++ b/testsuite/cibuilder.py
@@ -104,8 +104,21 @@  class CIBuilder(Test):
         if not hasattr(self, 'build_dir'):
             self.error("Broken test implementation: need to call init().")
 
+    @staticmethod
+    def _extract_multiconfigs(targets):
+        if isinstance(targets, str):
+            targets = [targets]
+        mc_names = set()
+        for target in targets:
+            if target.startswith('mc:'):
+                parts = target.split(':')
+                if len(parts) >= 2:
+                    mc_names.add(parts[1])
+        return mc_names
+
     def configure(
         self,
+        targets=None,
         compat_arch=True,
         cross=True,
         debsrc_cache=False,
@@ -196,6 +209,10 @@  class CIBuilder(Test):
 
         # write ci_build.conf
         with open(self.build_dir + '/conf/ci_build.conf', 'w') as f:
+            if targets:
+                mc_names = self._extract_multiconfigs(targets)
+                if mc_names:
+                    f.write('BBMULTICONFIG = "%s"\n\n' % ' '.join(sorted(mc_names)))
             if compat_arch:
                 f.write(
                     'ISAR_ENABLE_COMPAT_ARCH:amd64 = "1"\n'
diff --git a/testsuite/repro-build-test.py b/testsuite/repro-build-test.py
index c06269f5..c1b51aff 100755
--- a/testsuite/repro-build-test.py
+++ b/testsuite/repro-build-test.py
@@ -50,6 +50,7 @@  class ReproBuild(CIBuilder):
         # Build
         self.log.info("Started Build " + image_name)
         self.configure(
+            targets=target,
             source_date_epoch=source_date_epoch, use_apt_snapshot=True
         )
         self.bitbake(target)