diff mbox series

[RFC,1/1] mark ci tests that need attention

Message ID 20251212163448.2743851-1-felix.moessbauer@siemens.com
State New
Headers show
Series [RFC,1/1] mark ci tests that need attention | expand

Commit Message

MOESSBAUER, Felix Dec. 12, 2025, 4:34 p.m. UTC 
We mark the tests that are either suboptimal from a performance
PoV or that otherwise need attention. In general, the following things
need to be improved:

- use smaller baseline images for feature tests
- avoid repeated builds of kernels (especially with fast tag)

No functional change.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 testsuite/citest.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

Comments

Jan Kiszka Dec. 12, 2025, 4:42 p.m. UTC | #1 
On 12.12.25 17:34, Felix Moessbauer wrote:
> We mark the tests that are either suboptimal from a performance
> PoV or that otherwise need attention. In general, the following things
> need to be improved:
> 
> - use smaller baseline images for feature tests
> - avoid repeated builds of kernels (especially with fast tag)
> 
> No functional change.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  testsuite/citest.py | 24 ++++++++++++++++++++++++
>  1 file changed, 24 insertions(+)
> 
> diff --git a/testsuite/citest.py b/testsuite/citest.py
> index 27cc9ff6..23cd06e9 100755
> --- a/testsuite/citest.py
> +++ b/testsuite/citest.py
> @@ -199,11 +199,14 @@ class CrossTest(CIBaseTest):
>          targets = [
>              'mc:qemuarm-buster:isar-image-ci',
>              'mc:qemuarm-bullseye:isar-image-ci',
> +            # TODO: avoid kernel build (or at least build with ccache)
>              'mc:de0-nano-soc-bullseye:isar-image-base',
> +            # TODO: avoid kernel build
>              'mc:stm32mp15x-bullseye:isar-image-base',
>              'mc:qemuarm-bookworm:isar-image-ci',
>              'mc:qemuarm64-focal:isar-image-base',
>              'mc:nanopi-neo-efi-bookworm:isar-image-base',
> +            # TODO: avoid kernel build
>              'mc:phyboard-mira-bookworm:isar-image-base',

Most arm boards, specifically 32-bit, still have issues with debian
kernels. So, if you want to test the image, you must build the kernel.
But using caches makes a lot of sense.

>          ]
>  
> @@ -212,6 +215,7 @@ class CrossTest(CIBaseTest):
>  
>      def test_cross_debsrc(self):
>          targets = [
> +            # TODO: speedup task by removing prebuilt container added in 407a91322
>              'mc:qemuarm64-bookworm:isar-image-ci',

...provided the prebuild container in a cross scenario is already tested
elsewhere. Provide reasoning, please, not just speed-up suggestions, or
we cannot execute them.

>          ]
>  
> @@ -220,6 +224,7 @@ class CrossTest(CIBaseTest):
>  
>      def test_cross_trixie(self):
>          targets = [
> +            # TODO: avoid sbuild-chroot-target-db2m

See above.

>              'mc:qemuamd64-trixie:isar-image-base',
>              'mc:qemuarm64-trixie:isar-image-base',
>          ]
> @@ -271,6 +276,9 @@ class KernelTests(CIBaseTest):
>      """
>      Tests associated with kernel builds and development.
>      :avocado: tags=kernel,full
> +
> +    TODO: as we have dedicated kernel tests here, can we avoid the
> +    kernel builds in test_cross?

Only if those other kernel builds do not need to generate bootable images.

>      """
>  
>      def test_per_kernel(self):
> @@ -306,6 +314,7 @@ class InitRdBaseTest(CIBaseTest):
>      def deploy_dir_image(self, machine):
>          return os.path.join(self.deploy_dir, 'images', machine)
>  
> +    # TODO: can we use a smaller baseline than isar-image-ci?
>      def dracut_in_image(self, targets):
>          machine = 'qemuamd64'
>          distro = 'bookworm'
> @@ -326,6 +335,7 @@ class InitRdBaseTest(CIBaseTest):
>                  cmd="systemctl is-active dracut-shutdown"
>              )
>  
> +    # TODO: use smaller baseline, currently we include containers
>      def build_image_with_dependent_initrd(self, image, initrd,
>                                            distro="debian-bookworm",
>                                            machine="qemuamd64",
> @@ -365,6 +375,7 @@ class InitRdTest(InitRdBaseTest):
>  
>      def test_dracut_in_image(self):
>          """Test switch to dracut in an image recipe."""
> +        # TODO: smaller baseline is sufficient to test this
>          self.dracut_in_image(['mc:qemuamd64-bookworm:isar-image-ci'])
>  
>      def test_dracut_build_initrd(self):
> @@ -377,6 +388,7 @@ class InitRdTest(InitRdBaseTest):
>          lines = InitRdBaseTest.DRACUT_CONF
>          lines.append("ROOTFS_INITRAMFS_GENERATOR_CMDLINE:append = ' --unknown-option'")
>          self.init()
> +        # TODO: smaller baseline is sufficient to test this
>          self.perform_build_test('mc:qemuamd64-bookworm:isar-image-ci',
>                                  should_fail=True, lines=lines)
>  
> @@ -390,6 +402,7 @@ class InitRdTest(InitRdBaseTest):
>              f"INITRD_IMAGE = '{initrd}-{distro}-{machine}.initrd.img'",
>              f"do_image[depends] += '{initrd}:do_build'"
>          ]
> +        # TODO: either use smaller baseline, or better a parsing-only test
>  
>          self.init()
>          self.build_image_with_dependent_initrd('isar-image-ci', initrd,
> @@ -400,6 +413,7 @@ class InitRdTest(InitRdBaseTest):
>          initrd = 'isar-initramfs'
>          lines = [f"IMAGE_INITRD = '{initrd}'"]
>  
> +        # TODO: use smaller baseline
>          self.init()
>          self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
>  
> @@ -424,6 +438,7 @@ class InitRdTest(InitRdBaseTest):
>              f"IMAGE_INITRD = '{initrd}'",
>              "INITRD_IMAGE = 'not-a-valid-initrd-file'"
>          ]
> +        # TODO: smaller baseline is sufficient to test this
>          self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
>  
>  
> @@ -435,6 +450,7 @@ class InitRdCrossTests(InitRdBaseTest):
>  
>      def test_dracut_in_image(self):
>          """Test switch to dracut in an image recipe."""
> +        # TODO: can be tested with bare-minimal images
>          self.dracut_in_image([
>              'mc:qemuamd64-bookworm:isar-image-ci',
>              'mc:qemuamd64-trixie:isar-image-ci',
> @@ -503,8 +519,11 @@ class NoCrossTest(CIBaseTest):
>              'mc:qemuamd64-buster:isar-image-ci',
>              'mc:qemuamd64-bullseye:isar-initramfs',
>              'mc:qemumipsel-bullseye:isar-image-base',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:imx6-sabrelite-bullseye:isar-image-base',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:phyboard-mira-bullseye:isar-image-base',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:hikey-bullseye:isar-image-base',
>              'mc:virtualbox-bullseye:isar-image-base',
>              'mc:virtualbox-bookworm:isar-image-base',
> @@ -517,6 +536,7 @@ class NoCrossTest(CIBaseTest):
>              'mc:qemuamd64-iso-bookworm:isar-image-ci',
>              'mc:qemui386-bookworm:isar-image-base',
>              'mc:qemumipsel-bookworm:isar-image-ci',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:hikey-bookworm:isar-image-base',
>              'mc:beagleplay-bookworm:isar-image-base',
>              'mc:qemuarm64-noble:isar-image-base',
> @@ -532,6 +552,7 @@ class NoCrossTest(CIBaseTest):
>      def test_nocross_debsrc(self):
>          targets = [
>              'mc:qemuarm-bookworm:isar-image-ci',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:stm32mp15x-bullseye:isar-image-base',
>              'mc:de0-nano-soc-bookworm:isar-image-base',
>          ]
> @@ -565,6 +586,7 @@ class NoCrossTest(CIBaseTest):
>              'mc:qemuarm64-trixie:isar-image-base',
>              'mc:qemuarm-trixie:isar-image-base',
>              'mc:qemuriscv64-trixie:isar-image-base',
> +            # TODO: avoid targets with kernel builds or use ccache
>              'mc:sifive-fu540-trixie:isar-image-base',
>              'mc:starfive-visionfive2-trixie:isar-image-base',
>          ]
> @@ -598,6 +620,7 @@ class ContainerImageTest(CIBaseTest):
>  
>      @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
>      def test_container_image(self):
> +        # TODO: add trixie target
>          targets = [
>              'mc:container-amd64-buster:isar-image-base',
>              'mc:container-amd64-bullseye:isar-image-base',
> @@ -618,6 +641,7 @@ class ContainerSdkTest(CIBaseTest):
>  
>      @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
>      def test_container_sdk(self):
> +        # TODO: why is this only tested on bullseye? Consider moving to trixie
>          targets = ['mc:container-amd64-bullseye:isar-image-base']
>  
>          self.init()

A good share of you remarks point out that testcases are also aging and
need frequent maintenance to keep them focused, fresh and still
sufficiently broad. Tough work. I just spent >3 weeks on getting
isar-cip-core tests in a better shape again, and we are constantly
struggling how to avoid test matrix explosion in isar-cip-core or xenomai.

Jan
MOESSBAUER, Felix Dec. 12, 2025, 4:58 p.m. UTC | #2 
On Fri, 2025-12-12 at 17:42 +0100, Jan Kiszka wrote:
> On 12.12.25 17:34, Felix Moessbauer wrote:
> > We mark the tests that are either suboptimal from a performance
> > PoV or that otherwise need attention. In general, the following things
> > need to be improved:
> > 
> > - use smaller baseline images for feature tests
> > - avoid repeated builds of kernels (especially with fast tag)
> > 
> > No functional change.
> > 
> > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > 
> 
> A good share of you remarks point out that testcases are also aging and
> need frequent maintenance to keep them focused, fresh and still
> sufficiently broad. 
> 

I just noticed, that the test_cross_kselftest is not cross anymore,
because the kselftest class explicitly disables it via
ISAR_CROSS_COMPILE = "0". This is a prime example of the test aging
pattern.

Monitoring the CI execution time helps to spot slow time increases, as
well as spikes due to unintended changes (like this one).
This at least gives an indication.

Felix
diff mbox series

Patch

diff --git a/testsuite/citest.py b/testsuite/citest.py
index 27cc9ff6..23cd06e9 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -199,11 +199,14 @@  class CrossTest(CIBaseTest):
         targets = [
             'mc:qemuarm-buster:isar-image-ci',
             'mc:qemuarm-bullseye:isar-image-ci',
+            # TODO: avoid kernel build (or at least build with ccache)
             'mc:de0-nano-soc-bullseye:isar-image-base',
+            # TODO: avoid kernel build
             'mc:stm32mp15x-bullseye:isar-image-base',
             'mc:qemuarm-bookworm:isar-image-ci',
             'mc:qemuarm64-focal:isar-image-base',
             'mc:nanopi-neo-efi-bookworm:isar-image-base',
+            # TODO: avoid kernel build
             'mc:phyboard-mira-bookworm:isar-image-base',
         ]
 
@@ -212,6 +215,7 @@  class CrossTest(CIBaseTest):
 
     def test_cross_debsrc(self):
         targets = [
+            # TODO: speedup task by removing prebuilt container added in 407a91322
             'mc:qemuarm64-bookworm:isar-image-ci',
         ]
 
@@ -220,6 +224,7 @@  class CrossTest(CIBaseTest):
 
     def test_cross_trixie(self):
         targets = [
+            # TODO: avoid sbuild-chroot-target-db2m
             'mc:qemuamd64-trixie:isar-image-base',
             'mc:qemuarm64-trixie:isar-image-base',
         ]
@@ -271,6 +276,9 @@  class KernelTests(CIBaseTest):
     """
     Tests associated with kernel builds and development.
     :avocado: tags=kernel,full
+
+    TODO: as we have dedicated kernel tests here, can we avoid the
+    kernel builds in test_cross?
     """
 
     def test_per_kernel(self):
@@ -306,6 +314,7 @@  class InitRdBaseTest(CIBaseTest):
     def deploy_dir_image(self, machine):
         return os.path.join(self.deploy_dir, 'images', machine)
 
+    # TODO: can we use a smaller baseline than isar-image-ci?
     def dracut_in_image(self, targets):
         machine = 'qemuamd64'
         distro = 'bookworm'
@@ -326,6 +335,7 @@  class InitRdBaseTest(CIBaseTest):
                 cmd="systemctl is-active dracut-shutdown"
             )
 
+    # TODO: use smaller baseline, currently we include containers
     def build_image_with_dependent_initrd(self, image, initrd,
                                           distro="debian-bookworm",
                                           machine="qemuamd64",
@@ -365,6 +375,7 @@  class InitRdTest(InitRdBaseTest):
 
     def test_dracut_in_image(self):
         """Test switch to dracut in an image recipe."""
+        # TODO: smaller baseline is sufficient to test this
         self.dracut_in_image(['mc:qemuamd64-bookworm:isar-image-ci'])
 
     def test_dracut_build_initrd(self):
@@ -377,6 +388,7 @@  class InitRdTest(InitRdBaseTest):
         lines = InitRdBaseTest.DRACUT_CONF
         lines.append("ROOTFS_INITRAMFS_GENERATOR_CMDLINE:append = ' --unknown-option'")
         self.init()
+        # TODO: smaller baseline is sufficient to test this
         self.perform_build_test('mc:qemuamd64-bookworm:isar-image-ci',
                                 should_fail=True, lines=lines)
 
@@ -390,6 +402,7 @@  class InitRdTest(InitRdBaseTest):
             f"INITRD_IMAGE = '{initrd}-{distro}-{machine}.initrd.img'",
             f"do_image[depends] += '{initrd}:do_build'"
         ]
+        # TODO: either use smaller baseline, or better a parsing-only test
 
         self.init()
         self.build_image_with_dependent_initrd('isar-image-ci', initrd,
@@ -400,6 +413,7 @@  class InitRdTest(InitRdBaseTest):
         initrd = 'isar-initramfs'
         lines = [f"IMAGE_INITRD = '{initrd}'"]
 
+        # TODO: use smaller baseline
         self.init()
         self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
 
@@ -424,6 +438,7 @@  class InitRdTest(InitRdBaseTest):
             f"IMAGE_INITRD = '{initrd}'",
             "INITRD_IMAGE = 'not-a-valid-initrd-file'"
         ]
+        # TODO: smaller baseline is sufficient to test this
         self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
 
 
@@ -435,6 +450,7 @@  class InitRdCrossTests(InitRdBaseTest):
 
     def test_dracut_in_image(self):
         """Test switch to dracut in an image recipe."""
+        # TODO: can be tested with bare-minimal images
         self.dracut_in_image([
             'mc:qemuamd64-bookworm:isar-image-ci',
             'mc:qemuamd64-trixie:isar-image-ci',
@@ -503,8 +519,11 @@  class NoCrossTest(CIBaseTest):
             'mc:qemuamd64-buster:isar-image-ci',
             'mc:qemuamd64-bullseye:isar-initramfs',
             'mc:qemumipsel-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:imx6-sabrelite-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:phyboard-mira-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:hikey-bullseye:isar-image-base',
             'mc:virtualbox-bullseye:isar-image-base',
             'mc:virtualbox-bookworm:isar-image-base',
@@ -517,6 +536,7 @@  class NoCrossTest(CIBaseTest):
             'mc:qemuamd64-iso-bookworm:isar-image-ci',
             'mc:qemui386-bookworm:isar-image-base',
             'mc:qemumipsel-bookworm:isar-image-ci',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:hikey-bookworm:isar-image-base',
             'mc:beagleplay-bookworm:isar-image-base',
             'mc:qemuarm64-noble:isar-image-base',
@@ -532,6 +552,7 @@  class NoCrossTest(CIBaseTest):
     def test_nocross_debsrc(self):
         targets = [
             'mc:qemuarm-bookworm:isar-image-ci',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:stm32mp15x-bullseye:isar-image-base',
             'mc:de0-nano-soc-bookworm:isar-image-base',
         ]
@@ -565,6 +586,7 @@  class NoCrossTest(CIBaseTest):
             'mc:qemuarm64-trixie:isar-image-base',
             'mc:qemuarm-trixie:isar-image-base',
             'mc:qemuriscv64-trixie:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:sifive-fu540-trixie:isar-image-base',
             'mc:starfive-visionfive2-trixie:isar-image-base',
         ]
@@ -598,6 +620,7 @@  class ContainerImageTest(CIBaseTest):
 
     @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
     def test_container_image(self):
+        # TODO: add trixie target
         targets = [
             'mc:container-amd64-buster:isar-image-base',
             'mc:container-amd64-bullseye:isar-image-base',
@@ -618,6 +641,7 @@  class ContainerSdkTest(CIBaseTest):
 
     @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
     def test_container_sdk(self):
+        # TODO: why is this only tested on bullseye? Consider moving to trixie
         targets = ['mc:container-amd64-bullseye:isar-image-base']
 
         self.init()