add test for upgrade

Add a basic test to check that systemd works after an upgrade

.github/workflows/coverage.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@7e4ec15aee6b98300b2ee14265bc647a716a9f8a
+      - uses: systemd/mkosi@dbb4020beee2cdf250f93a425794f1cf8b0fe693
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location
@@ -90,7 +90,6 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
-            -Dintegration-tests=true \
             build
 
       - name: Build image
@@ -120,7 +119,8 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --suite integration-tests \
+            --setup=integration \
+            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

.github/workflows/mkosi.yml

@@ -120,7 +120,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@7e4ec15aee6b98300b2ee14265bc647a716a9f8a
+      - uses: systemd/mkosi@dbb4020beee2cdf250f93a425794f1cf8b0fe693
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location
@@ -197,7 +197,6 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
-            -Dintegration-tests=true \
             -Dbpf-framework=disabled \
             build
 
@@ -233,7 +232,8 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --suite integration-tests \
+            --setup=integration \
+            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

meson.build

@@ -13,6 +13,12 @@ project('systemd', 'c',
         meson_version : '>= 0.62.0',
 )
 
+add_test_setup(
+        'default',
+        exclude_suites : ['integration-tests'],
+        is_default : true,
+)
+
 project_major_version = meson.project_version().split('.')[0].split('~')[0]
 if meson.project_version().contains('.')
         project_minor_version = meson.project_version().split('.')[-1].split('~')[0]
@@ -339,7 +345,6 @@ meson_build_sh = find_program('tools/meson-build.sh')
 want_tests = get_option('tests')
 want_slow_tests = want_tests != 'false' and get_option('slow-tests')
 want_fuzz_tests = want_tests != 'false' and get_option('fuzz-tests')
-want_integration_tests = want_tests != 'false' and get_option('integration-tests')
 install_tests = want_tests != 'false' and get_option('install-tests')
 
 if add_languages('cpp', native : false, required : fuzzer_build)
@@ -2661,10 +2666,6 @@ endif
 #####################################################################
 
 mkosi = find_program('mkosi', required : false)
-if want_integration_tests and not mkosi.found()
-        error('Could not find mkosi which is required to run the integration tests')
-endif
-
 mkosi_depends = public_programs
 
 foreach executable : ['systemd-journal-remote', 'systemd-sbsign', 'systemd-keyutil']

meson_options.txt

@@ -509,7 +509,7 @@ option('install-tests', type : 'boolean', value : false,
        description : 'install test executables')
 option('log-message-verification', type : 'feature', deprecated : { 'true' : 'enabled', 'false' : 'disabled' },
        description : 'do fake printf() calls to verify format strings')
-option('integration-tests', type : 'boolean', value : false,
+option('integration-tests', type : 'boolean', value : false, deprecated : true,
        description : 'run the integration tests')
 
 option('ok-color', type : 'combo',

mkosi/mkosi.conf

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Config]
-MinimumVersion=commit:7e4ec15aee6b98300b2ee14265bc647a716a9f8a
+MinimumVersion=commit:dbb4020beee2cdf250f93a425794f1cf8b0fe693
 Dependencies=
         exitrd
         initrd
@@ -78,8 +78,7 @@ KernelCommandLine=
         oops=panic
         panic=-1
         softlockup_panic=1
-        # Disabled due to BTRFS issue, waiting for the fix to become available
-        panic_on_warn=0
+        panic_on_warn=1
         psi=1
         mitigations=off
 

mkosi/mkosi.conf.d/centos-fedora/mkosi.prepare

@@ -0,0 +1,29 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+if [[ "$1" == "build" ]]; then
+    exit 0
+fi
+
+# Get the systemd version installed in the host assuming it's recent enough
+# but a major version older than the one in the image.
+if dnf --version |grep -q '^4\.'; then
+    disable='--disablerepo'
+elif dnf --version |grep -q '^dnf5\s'; then
+    disable='--disable-repo'
+else
+    echo 'Unknown dnf version!'
+    exit 1
+fi
+
+pkgdir=/usr/host-pkgs
+distrover=$(dnf repoquery "$disable" mkosi --latest-limit=1 --queryformat='%{evr}' systemd)
+develver=$(rpm -q --queryformat='%{evr}' systemd)
+
+# Filter only noarch and arch packages otherwise on CentOS 9 we could end having i686 packages too
+distropkgs=$(dnf repoquery 'systemd*' |grep -F "$distrover" |grep -e noarch -e "$DISTRIBUTION_ARCHITECTURE" |grep -v -e -debuginfo -e -debugsource)
+develpkgs=$(dnf repoquery 'systemd*' |grep -F "$develver" |grep -e noarch -e "$DISTRIBUTION_ARCHITECTURE" |grep -v -e -debuginfo -e -debugsource)
+
+dnf download --arch noarch --arch "$DISTRIBUTION_ARCHITECTURE" --destdir "$BUILDROOT$pkgdir/distro" $distropkgs
+dnf download --arch noarch --arch "$DISTRIBUTION_ARCHITECTURE" --destdir "$BUILDROOT$pkgdir/devel" $develpkgs

src/libsystemd/sd-netlink/netlink-socket.c

@@ -161,6 +161,10 @@ static int socket_recv_message(int fd, void *buf, size_t buf_size, uint32_t *ret
         assert(fd >= 0);
         assert(peek || (buf && buf_size > 0));
 
+        /* Note: this might return successfully, but with a zero size under some transient conditions, such
+         * as the reception of a non-kernel message. In such a case the passed buffer might or might not be
+         * modified. Caller must treat a zero return as "no message, but also not an error". */
+
         n = recvmsg_safe(fd, &msg, peek ? (MSG_PEEK|MSG_TRUNC) : 0);
         if (ERRNO_IS_NEG_TRANSIENT(n))
                 goto transient;
@@ -179,8 +183,9 @@ static int socket_recv_message(int fd, void *buf, size_t buf_size, uint32_t *ret
 
                 if (peek) {
                         /* Drop the message. Note that we ignore ECHRNG/EXFULL errors here, which
-                         * recvmsg_safe() returns in case the payload or cdata is truncated. Here it's quite
-                         * likely it is truncated, because we pass a zero-sized buffer. */
+                         * recvmsg_safe() returns in case the payload or cdata is truncated. Given we just
+                         * want to drop the message we also don't care if its payload or cdata was
+                         * truncated. */
                         n = recvmsg_safe(fd, &msg, 0);
                         if (n < 0 && !IN_SET(n, -ECHRNG, -EXFULL))
                                 return (int) n;

test/integration-tests/README.md

@@ -38,14 +38,14 @@ directory (`OutputDirectory=`) to point to the other directory using `mkosi/mkos
 After the image has been built, the integration tests can be run with:
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build --suite integration-tests --num-processes "$(($(nproc) / 4))"
+$ mkosi -f sandbox -- meson test -C build --setup=integration --suite integration-tests --num-processes "$(($(nproc) / 4))"
 ```
 
 As usual, specific tests can be run in meson by appending the name of the test
 which is usually the name of the directory e.g.
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
+$ mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
 ```
 
 See `mkosi -f sandbox -- meson introspect build --tests` for a list of tests.
@@ -55,7 +55,7 @@ To interactively debug a failing integration test, the `--interactive` option
 newer:
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
+$ mkosi -f sandbox -- meson test -C build --setup=integration -i TEST-01-BASIC
 ```
 
 Due to limitations in meson, the integration tests do not yet depend on the
@@ -64,7 +64,7 @@ running the integration tests. To rebuild the image and rerun a test, the
 following command can be used:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
 ```
 
 The integration tests use the same mkosi configuration that's used when you run
@@ -78,7 +78,7 @@ To iterate on an integration test, let's first get a shell in the integration te
 the following:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=shell -i TEST-01-BASIC
 ```
 
 This will get us a shell in the integration test environment after booting the machine without running the
@@ -107,7 +107,7 @@ re-running the test will first install the new packages we just built, make a ne
 the test again. You can keep running the loop of `mkosi -R`, `systemctl soft-reboot` and
 `systemctl start ...` until the changes to the integration test are working.
 
-If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`),
+If you're debugging a failing integration test (running `meson test --interactive`),
 there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to
 rerun the test.
 
@@ -120,10 +120,6 @@ rerun the test.
 `TEST_NO_KVM=1`: Disable qemu KVM auto-detection (may be necessary when you're
 trying to run the *vanilla* qemu and have both qemu and qemu-kvm installed)
 
-`TEST_SHELL=1`: Configure the machine to be more *user-friendly* for
-interactive debugging (e.g. by setting a usable default terminal, suppressing
-the shutdown after the test, etc.).
-
 `TEST_MATCH_SUBTEST=subtest`:  If the test makes use of `run_subtests` use this
 variable to provide a POSIX extended regex to run only subtests matching the
 expression.

test/integration-tests/TEST-88-UPGRADE/meson.build

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+integration_tests += [
+        integration_test_template + {
+                'name' : fs.name(meson.current_source_dir()),
+        },
+]

test/integration-tests/integration-test-wrapper.py

@@ -361,7 +361,7 @@ def statfs(path: Path) -> str:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--mkosi', required=True)
+    parser.add_argument('--mkosi', default=None)
     parser.add_argument('--meson-source-dir', required=True, type=Path)
     parser.add_argument('--meson-build-dir', required=True, type=Path)
     parser.add_argument('--name', required=True)
@@ -379,6 +379,12 @@ def main() -> None:
     parser.add_argument('mkosi_args', nargs='*')
     args = parser.parse_args()
 
+    if not args.mkosi:
+        args.mkosi = shutil.which('mkosi')
+        if not args.mkosi:
+            print('Could not find mkosi which is required to run the integration tests', file=sys.stderr)
+            sys.exit(1)
+
     # The meson source directory can either be the top-level repository directory or the
     # test/integration-tests/standalone subdirectory in the repository directory. The mkosi configuration
     # will always be a parent directory of one of these directories and at most 4 levels upwards, so don't
@@ -395,13 +401,6 @@ def main() -> None:
         )
         exit(1)
 
-    if not bool(int(os.getenv('SYSTEMD_INTEGRATION_TESTS', '0'))):
-        print(
-            f'SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.name}',
-            file=sys.stderr,
-        )
-        exit(77)
-
     if args.slow and not bool(int(os.getenv('SYSTEMD_SLOW_TESTS', '0'))):
         print(
             f'SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}',

test/integration-tests/meson.build

@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+# We'd give these more descriptive names but only alphanumeric characters are allowed.
+add_test_setup('integration')
+add_test_setup('shell', env : {'TEST_SHELL' : '1'})
+
 integration_test_wrapper = find_program('integration-test-wrapper.py')
 integration_tests = []
 integration_test_template = {
@@ -96,6 +100,7 @@ foreach dirname : [
         'TEST-85-NETWORK',
         'TEST-86-MULTI-PROFILE-UKI',
         'TEST-87-AUX-UTILS-VM',
+        'TEST-88-UPGRADE',
 ]
         subdir(dirname)
 endforeach
@@ -129,11 +134,11 @@ foreach integration_test : integration_tests
                 integration_test_args += ['--skip']
         endif
 
-        if not mkosi.found()
-                continue
+        if mkosi.found()
+                integration_test_args += ['--mkosi', mkosi.full_path()]
         endif
 
-        integration_test_args += ['--mkosi', mkosi.full_path(), '--']
+        integration_test_args += ['--']
 
         if integration_test['cmdline'].length() > 0
                 integration_test_args += [
@@ -151,19 +156,12 @@ foreach integration_test : integration_tests
 
         integration_test_args += integration_test['mkosi-args']
 
-        integration_test_env = {}
-
-        if want_integration_tests
-                integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'}
-        endif
-
         # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt on every
         # "ninja -C build". Instead, the mkosi target has to be rebuilt manually before running the
         # integration tests with mkosi.
         test(
                 integration_test['name'],
                 integration_test_wrapper,
-                env : integration_test_env,
                 args : integration_test_args,
                 timeout : integration_test['timeout'],
                 priority : integration_test['priority'],

test/integration-tests/standalone/meson.build

@@ -16,7 +16,6 @@ project('systemd-testsuite',
 
 fs = import('fs')
 mkosi = find_program('mkosi', required : true)
-want_integration_tests = true
 
 # meson refuses .. in subdir() so we use a symlink to trick it into accepting it anyway.
 subdir('integration-tests')

test/units/TEST-88-UPGRADE.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+set -eux
+set -o pipefail
+
+pkgdir=/usr/host-pkgs
+
+if ! [ -d "$pkgdir" ]; then
+    echo "Distro packages not found in $pkgdir" >/skipped
+    exit 77
+fi
+
+minor=$(systemctl --version |awk '/^systemd/{print$2}')
+if which networkctl >/dev/null; then
+    networkd=1
+fi
+
+if dnf --version |grep -q '^4\.'; then
+    disable='--disablerepo'
+elif dnf --version |grep -q '^dnf5\s'; then
+    disable='--disable-repo'
+else
+    echo 'Unknown dnf version!'
+    exit 1
+fi
+
+dnf downgrade -y --allowerasing "$disable" '*' "$pkgdir"/distro/*.rpm
+
+newminor=$(systemctl --version |awk '/^systemd/{print$2}')
+
+if [ "$newminor" -lt "$minor" ]; then
+    echo "Downgrade to $newminor was successful."
+else
+    echo "Downgrade failed. Current version is still $newminor."
+    exit 1
+fi
+
+# TODO: sanity checks
+
+systemctl --failed |grep -Fqx '0 loaded units listed.'
+[ -n "$networkd" ] && networkctl status
+loginctl list-sessions
+
+# Finally test the upgrade
+dnf -y upgrade "$disable" '*' "$pkgdir"/devel/*.rpm
+
+# TODO: sanity checks
+
+systemctl --failed |grep -Fqx '0 loaded units listed.'
+[ -n "$networkd" ] && networkctl status
+loginctl list-sessions