TEST-17: add reproducer for issue #35329

Without the previous commit, the test case will fail.

.github/workflows/linter.yml

@@ -37,7 +37,7 @@ jobs:
           VALIDATE_GITHUB_ACTIONS: true
 
       - name: Check that tabs are not used in Python code
-        run: sh -c '! git grep -P "\\t" -- src/ukify/ukify.py'
+        run: sh -c '! git grep -P "\\t" -- src/ukify/ukify.py test/integration-test-wrapper.py'
 
       - name: Install ruff and mypy
         run: |
@@ -47,14 +47,14 @@ jobs:
       - name: Run mypy
         run: |
           python3 -m mypy --version
-          python3 -m mypy src/ukify/ukify.py
+          python3 -m mypy src/ukify/ukify.py test/integration-test-wrapper.py
 
       - name: Run ruff check
         run: |
           ruff --version
-          ruff check src/ukify/ukify.py
+          ruff check src/ukify/ukify.py test/integration-test-wrapper.py
 
       - name: Run ruff format
         run: |
           ruff --version
-          ruff format --check src/ukify/ukify.py
+          ruff format --check src/ukify/ukify.py test/integration-test-wrapper.py

.github/workflows/mkosi.yml

@@ -105,7 +105,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@8976a0abb19221e65300222f2d33067970cca0f1
+      - uses: systemd/mkosi@0825cca8084674ec8fa27502134b1bc601f79e0c
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location

NEWS

@@ -764,6 +764,9 @@ CHANGES WITH 257 in spe:
           other cases EnterNamespace= might be an suitable approach to acquire
           symbolized backtraces.)
 
+        Special thanks to Nick Owens for bringing attention to and testing
+        fixes for issue #34516.
+
         Contributions from: 12paper, A. Wilcox, Abderrahim Kitouni,
         Adrian Vovk, Alain Greppin, Allison Karlitskaya, Alyssa Ross,
         Anders Jonsson, Andika Triwidada, Andres Beltran, Anouk Ceyssens,

hwdb.d/60-sensor.hwdb

@@ -295,6 +295,10 @@ sensor:modalias:acpi:MXC6655*:dmi:*:svnCHUWIInnovationAndTechnology*:pnHi10X:*
 sensor:modalias:acpi:KIOX000A*:dmi:*:svnCHUWIInnovationAndTechnology*:pnHi10X:*
  ACCEL_MOUNT_MATRIX=0, -1, 0; -1, 0, 0; 0, 0, 1
 
+# Chuwi Hi10 X1
+sensor:modalias:acpi:NSA2513*:dmi:*:svnCHUWIInnovationAndTechnology*:pnHi10X1:*
+ ACCEL_MOUNT_MATRIX=0, 1, 0; -1, 0, 0; 0, 0, 1
+
 # Chuwi Hi10 Go
 sensor:modalias:acpi:MXC6655*:dmi:*:svnCHUWIINNOVATIONLIMITED:pnHi10Go:*
  ACCEL_MOUNT_MATRIX=-1, 0, 0; 0,-1, 0; 0, 0, 1

mkosi.clangd

@@ -1,12 +1,18 @@
 #!/bin/bash
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-MKOSI_CONFIG="$(mkosi --json summary | jq -r .Images[-1])"
+if command -v flatpak-spawn >/dev/null; then
+    SPAWN=(flatpak-spawn --host)
+else
+    SPAWN=()
+fi
+
+MKOSI_CONFIG="$("${SPAWN[@]}" --host mkosi --json summary | jq -r .Images[-1])"
 DISTRIBUTION="$(jq -r .Distribution <<< "$MKOSI_CONFIG")"
 RELEASE="$(jq -r .Release <<< "$MKOSI_CONFIG")"
 ARCH="$(jq -r .Architecture <<< "$MKOSI_CONFIG")"
 
-exec mkosi \
+exec "${SPAWN[@]}" mkosi \
     --incremental=strict \
     --build-sources-ephemeral=no \
     --format=none \

mkosi.conf

@@ -38,9 +38,8 @@ SignExpectedPcr=yes
 
 [Content]
 ExtraTrees=
+        mkosi.extra.common
         mkosi.crt:/usr/lib/verity.d/mkosi.crt # sysext verification key
-        mkosi.leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions
-        mkosi.coredump-journal-storage.conf:/usr/lib/systemd/coredump.conf.d/10-coredump-journal-storage.conf
         %O/minimal-0.root-%a.raw:/usr/share/minimal_0.raw
         %O/minimal-0.root-%a-verity.raw:/usr/share/minimal_0.verity
         %O/minimal-0.root-%a-verity-sig.raw:/usr/share/minimal_0.verity.sig

mkosi.conf.d/05-tools/mkosi.conf.d/arch.conf

@@ -6,10 +6,12 @@ ToolsTreeDistribution=arch
 [Build]
 ToolsTreePackages=
         cryptsetup
+        github-cli
         libcap
         libmicrohttpd
         python-jinja
         python-pytest
         ruff
+        shellcheck
         tpm2-tss
         util-linux-libs

mkosi.conf.d/05-tools/mkosi.conf.d/centos-fedora.conf

@@ -16,3 +16,4 @@ ToolsTreePackages=
         tpm2-tss-devel
         python3-jinja2
         python3-pytest
+        shellcheck

mkosi.conf.d/05-tools/mkosi.conf.d/debian-ubuntu.conf

@@ -6,6 +6,7 @@ ToolsTreeDistribution=|ubuntu
 
 [Build]
 ToolsTreePackages=
+        gh
         libblkid-dev
         libcap-dev
         libcryptsetup-dev
@@ -16,3 +17,4 @@ ToolsTreePackages=
         libtss2-dev
         python3-jinja2
         python3-pytest
+        shellcheck

mkosi.conf.d/05-tools/mkosi.conf.d/fedora.conf

@@ -5,4 +5,5 @@ ToolsTreeDistribution=fedora
 
 [Build]
 ToolsTreePackages=
+        gh
         ruff

mkosi.conf.d/05-tools/mkosi.conf.d/opensuse.conf

@@ -5,6 +5,7 @@ ToolsTreeDistribution=opensuse
 
 [Build]
 ToolsTreePackages=
+        gh
         pkgconfig(blkid)
         pkgconfig(libcap)
         pkgconfig(libcryptsetup)
@@ -16,3 +17,4 @@ ToolsTreePackages=
         tss2-devel
         python3-jinja2
         python3-pytest
+        ShellCheck

mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.conf

@@ -13,6 +13,7 @@ Environment=
 
 [Content]
 Packages=
+        clang-devel
         compiler-rt
         gdb
         git-core

mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.conf

@@ -15,6 +15,7 @@ Environment=
 [Content]
 Packages=
         apt
+        clangd
         erofs-utils
         git-core
         libclang-rt-dev

mkosi.images/build/mkosi.conf.d/opensuse/mkosi.conf

@@ -12,6 +12,7 @@ Environment=
 
 [Content]
 Packages=
+        clang
         diffutils
         erofs-utils
         gcc-c++

mkosi.images/initrd/mkosi.conf

@@ -6,9 +6,7 @@ Include=
         %D/mkosi.sanitizers
 
 [Content]
-ExtraTrees=
+ExtraTrees=%D/mkosi.extra.common
-        %D/mkosi.leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions
-        %D/mkosi.coredump-journal-storage.conf:/usr/lib/systemd/coredump.conf.d/10-coredump-journal-storage.conf
 
 Packages=
         findutils

mkosi.sanitizers/mkosi.postinst

@@ -57,6 +57,8 @@ wrap=(
     delv
     dhcpd
     dig
+    dnf
+    dnf5
     dmsetup
     dnsmasq
     findmnt
@@ -93,7 +95,7 @@ wrap=(
 )
 
 for bin in "${wrap[@]}"; do
-    if ! mkosi-chroot command -v "$bin" >/dev/null; then
+    if ! mkosi-chroot bash -c "command -v $bin" >/dev/null; then
         continue
     fi
 
@@ -103,7 +105,7 @@ for bin in "${wrap[@]}"; do
         enable_lsan=0
     fi
 
-    target="$(mkosi-chroot command -v "$bin")"
+    target="$(mkosi-chroot bash -c "command -v $bin")"
 
     mv "$BUILDROOT/$target" "$BUILDROOT/$target.orig"
 

src/core/device.c

@@ -1048,9 +1048,6 @@ static void device_enumerate(Manager *m) {
                 _cleanup_set_free_ Set *ready_units = NULL, *not_ready_units = NULL;
                 Device *d;
 
-                if (device_is_processed(dev) <= 0)
-                        continue;
-
                 if (device_setup_units(m, dev, &ready_units, &not_ready_units) < 0)
                         continue;
 

src/ukify/ukify.py

@@ -467,7 +467,7 @@ class SignTool:
         raise NotImplementedError()
 
     @staticmethod
-    def from_string(name) -> type['SignTool']:
+    def from_string(name: str) -> type['SignTool']:
         if name == 'pesign':
             return PeSign
         elif name == 'sbsign':

test/integration-test-wrapper.py

@@ -1,8 +1,7 @@
 #!/usr/bin/python3
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-'''Test wrapper command for driving integration tests.
+"""Test wrapper command for driving integration tests."""
-'''
 
 import argparse
 import json
@@ -13,7 +12,6 @@ import sys
 import textwrap
 from pathlib import Path
 
-
 EMERGENCY_EXIT_DROPIN = """\
 [Unit]
 Wants=emergency-exit.service
@@ -34,7 +32,7 @@ ExecStart=false
 """
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--mkosi', required=True)
     parser.add_argument('--meson-source-dir', required=True, type=Path)
@@ -46,34 +44,43 @@ def main():
     parser.add_argument('--slow', action=argparse.BooleanOptionalAction)
     parser.add_argument('--vm', action=argparse.BooleanOptionalAction)
     parser.add_argument('--exit-code', required=True, type=int)
-    parser.add_argument('mkosi_args', nargs="*")
+    parser.add_argument('mkosi_args', nargs='*')
     args = parser.parse_args()
 
-    if not bool(int(os.getenv("SYSTEMD_INTEGRATION_TESTS", "0"))):
+    if not bool(int(os.getenv('SYSTEMD_INTEGRATION_TESTS', '0'))):
-        print(f"SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.name}", file=sys.stderr)
+        print(
+            f'SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.name}',
+            file=sys.stderr,
+        )
         exit(77)
 
-    if args.slow and not bool(int(os.getenv("SYSTEMD_SLOW_TESTS", "0"))):
+    if args.slow and not bool(int(os.getenv('SYSTEMD_SLOW_TESTS', '0'))):
-        print(f"SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}", file=sys.stderr)
+        print(
+            f'SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}',
+            file=sys.stderr,
+        )
         exit(77)
 
-    if args.vm and bool(int(os.getenv("TEST_NO_QEMU", "0"))):
+    if args.vm and bool(int(os.getenv('TEST_NO_QEMU', '0'))):
-        print(f"TEST_NO_QEMU=1, skipping {args.name}", file=sys.stderr)
+        print(f'TEST_NO_QEMU=1, skipping {args.name}', file=sys.stderr)
         exit(77)
 
-    for s in os.getenv("TEST_SKIP", "").split():
+    for s in os.getenv('TEST_SKIP', '').split():
         if s in args.name:
-            print(f"Skipping {args.name} due to TEST_SKIP", file=sys.stderr)
+            print(f'Skipping {args.name} due to TEST_SKIP', file=sys.stderr)
             exit(77)
 
-    keep_journal = os.getenv("TEST_SAVE_JOURNAL", "fail")
+    keep_journal = os.getenv('TEST_SAVE_JOURNAL', 'fail')
-    shell = bool(int(os.getenv("TEST_SHELL", "0")))
+    shell = bool(int(os.getenv('TEST_SHELL', '0')))
 
     if shell and not sys.stderr.isatty():
-        print(f"--interactive must be passed to meson test to use TEST_SHELL=1", file=sys.stderr)
+        print(
+            '--interactive must be passed to meson test to use TEST_SHELL=1',
+            file=sys.stderr,
+        )
         exit(1)
 
-    name = args.name + (f"-{i}" if (i := os.getenv("MESON_TEST_ITERATION")) else "")
+    name = args.name + (f'-{i}' if (i := os.getenv('MESON_TEST_ITERATION')) else '')
 
     dropin = textwrap.dedent(
         """\
@@ -84,14 +91,14 @@ def main():
 
     if not shell:
         dropin += textwrap.dedent(
-            f"""
+            """
             [Unit]
             SuccessAction=exit
             SuccessActionExitStatus=123
             """
         )
 
-    if os.getenv("TEST_MATCH_SUBTEST"):
+    if os.getenv('TEST_MATCH_SUBTEST'):
         dropin += textwrap.dedent(
             f"""
             [Service]
@@ -99,7 +106,7 @@ def main():
             """
         )
 
-    if os.getenv("TEST_MATCH_TESTCASE"):
+    if os.getenv('TEST_MATCH_TESTCASE'):
         dropin += textwrap.dedent(
             f"""
             [Service]
@@ -116,7 +123,7 @@ def main():
             """
         )
 
-        journal_file = (args.meson_build_dir / (f"test/journal/{name}.journal")).absolute()
+        journal_file = (args.meson_build_dir / (f'test/journal/{name}.journal')).absolute()
         journal_file.unlink(missing_ok=True)
     elif not shell:
         dropin += textwrap.dedent(
@@ -136,54 +143,60 @@ def main():
         *(['--forward-journal', journal_file] if journal_file else []),
         *(
             [
-                '--credential',
+                '--credential', f'systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}',  # noqa: E501
-                f"systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}",
+                '--credential', f'systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}',
-                '--credential',
-                f"systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}",
             ]
             if not sys.stderr.isatty()
             else []
         ),
-        '--credential',
+        '--credential', f'systemd.unit-dropin.{args.unit}={shlex.quote(dropin)}',
-        f"systemd.unit-dropin.{args.unit}={shlex.quote(dropin)}",
         '--runtime-network=none',
         '--runtime-scratch=no',
         *args.mkosi_args,
-        '--qemu-firmware', args.firmware,
+        '--qemu-firmware',
-        *(['--qemu-kvm', 'no'] if int(os.getenv("TEST_NO_KVM", "0")) else []),
+        args.firmware,
+        *(['--qemu-kvm', 'no'] if int(os.getenv('TEST_NO_KVM', '0')) else []),
         '--kernel-command-line-extra',
-        ' '.join([
+        ' '.join(
+            [
                 'systemd.hostname=H',
-            f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/{args.name}.units:/usr/lib/systemd/tests/testdata/units:",
+                f'SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/{args.name}.units:/usr/lib/systemd/tests/testdata/units:',
-            *([f"systemd.unit={args.unit}"] if not shell else []),
+                *([f'systemd.unit={args.unit}'] if not shell else []),
                 'systemd.mask=systemd-networkd-wait-online.service',
                 *(
                     [
-                    "systemd.mask=serial-getty@.service",
+                        'systemd.mask=serial-getty@.service',
-                    "systemd.show_status=error",
+                        'systemd.show_status=error',
-                    "systemd.crash_shell=0",
+                        'systemd.crash_shell=0',
-                    "systemd.crash_action=poweroff",
+                        'systemd.crash_action=poweroff',
                     ]
                     if not sys.stderr.isatty()
                     else []
                 ),
-        ]),
+            ]
+        ),
         '--credential', f"journal.storage={'persistent' if sys.stderr.isatty() else args.storage}",
         *(['--runtime-build-sources=no'] if not sys.stderr.isatty() else []),
         'qemu' if args.vm or os.getuid() != 0 else 'boot',
-    ]
+    ]  # fmt: skip
 
     result = subprocess.run(cmd)
 
     # On Debian/Ubuntu we get a lot of random QEMU crashes. Retry once, and then skip if it fails again.
     if args.vm and result.returncode == 247 and args.exit_code != 247:
+        if journal_file:
             journal_file.unlink(missing_ok=True)
         result = subprocess.run(cmd)
         if args.vm and result.returncode == 247 and args.exit_code != 247:
-            print(f"Test {args.name} failed due to QEMU crash (error 247), ignoring", file=sys.stderr)
+            print(
+                f'Test {args.name} failed due to QEMU crash (error 247), ignoring',
+                file=sys.stderr,
+            )
             exit(77)
 
-    if journal_file and (keep_journal == "0" or (result.returncode in (args.exit_code, 77) and keep_journal == "fail")):
+    if journal_file and (
+        keep_journal == '0' or (result.returncode in (args.exit_code, 77) and keep_journal == 'fail')
+    ):
         journal_file.unlink(missing_ok=True)
 
     if shell or result.returncode in (args.exit_code, 77):
@@ -192,31 +205,33 @@ def main():
     if journal_file:
         ops = []
 
-        if os.getenv("GITHUB_ACTIONS"):
+        if os.getenv('GITHUB_ACTIONS'):
-            id = os.environ["GITHUB_RUN_ID"]
+            id = os.environ['GITHUB_RUN_ID']
-            iteration = os.environ["GITHUB_RUN_ATTEMPT"]
+            iteration = os.environ['GITHUB_RUN_ATTEMPT']
             j = json.loads(
                 subprocess.run(
                     [
                         args.mkosi,
-                        "--directory", os.fspath(args.meson_source_dir),
+                        '--directory', os.fspath(args.meson_source_dir),
-                        "--json",
+                        '--json',
-                        "summary",
+                        'summary',
                     ],
                     stdout=subprocess.PIPE,
                     text=True,
                 ).stdout
+            )  # fmt: skip
+            distribution = j['Images'][-1]['Distribution']
+            release = j['Images'][-1]['Release']
+            artifact = f'ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals'
+            ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}']
+            journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal')
+
+        ops += [f'journalctl --file {journal_file} --no-hostname -o short-monotonic -u {args.unit} -p info']
+
+        print(
+            "Test failed, relevant logs can be viewed with: \n\n" f"{(' && '.join(ops))}\n",
+            file=sys.stderr,
         )
-            distribution = j["Images"][-1]["Distribution"]
-            release = j["Images"][-1]["Release"]
-            artifact = f"ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals"
-            ops += [f"gh run download {id} --name {artifact} -D ci/{artifact}"]
-            journal_file = Path(f"ci/{artifact}/test/journal/{name}.journal")
-
-        ops += [f"journalctl --file {journal_file} --no-hostname -o short-monotonic -u {args.unit} -p info"]
-
-        print("Test failed, relevant logs can be viewed with: \n\n"
-              f"{(' && '.join(ops))}\n", file=sys.stderr)
 
     # 0 also means we failed so translate that to a non-zero exit code to mark the test as failed.
     exit(result.returncode or 1)

test/units/TEST-17-UDEV.device_is_processing.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# shellcheck disable=SC2317
+set -ex
+set -o pipefail
+
+# This is a reproducer of issue #35329,
+# which is a regression caused by 405be62f05d76f1845f347737b5972158c79dd3e.
+
+IFNAME=udevtestnetif
+
+at_exit() {
+    set +e
+
+    systemctl stop testsleep.service
+    rm -f /run/udev/udev.conf.d/timeout.conf
+    rm -f /run/udev/rules.d/99-testsuite.rules
+    # Forcibly kills sleep command invoked by the udev rule before restarting,
+    # otherwise systemctl restart below will takes longer.
+    killall -KILL sleep
+    systemctl restart systemd-udevd.service
+    ip link del "$IFNAME"
+}
+
+trap at_exit EXIT
+
+udevadm settle
+
+mkdir -p /run/udev/udev.conf.d/
+cat >/run/udev/udev.conf.d/timeout.conf <<EOF
+event_timeout=1h
+EOF
+
+mkdir -p /run/udev/rules.d/
+cat >/run/udev/rules.d/99-testsuite.rules <<EOF
+SUBSYSTEM=="net", ACTION=="change", KERNEL=="${IFNAME}", OPTIONS="log_level=debug", RUN+="/usr/bin/sleep 1000"
+EOF
+
+systemctl restart systemd-udevd.service
+
+ip link add "$IFNAME" type dummy
+IFINDEX=$(ip -json link show "$IFNAME" | jq '.[].ifindex')
+udevadm wait --timeout 10 "/sys/class/net/${IFNAME}"
+# Check if the database file is created.
+[[ -e "/run/udev/data/n${IFINDEX}" ]]
+
+systemd-run \
+    -p After="sys-subsystem-net-devices-${IFNAME}.device" \
+    -p BindsTo="sys-subsystem-net-devices-${IFNAME}.device" \
+    -u testsleep.service \
+    sleep 1h
+
+timeout 10 bash -c 'until systemctl is-active testsleep.service; do sleep .5; done'
+
+udevadm trigger "/sys/class/net/${IFNAME}"
+timeout 30 bash -c "until grep -F 'ID_PROCESSING=1' /run/udev/data/n${IFINDEX}; do sleep .5; done"
+
+for _ in {1..3}; do
+    systemctl daemon-reexec
+    systemctl is-active testsleep.service
+done
+
+for _ in {1..3}; do
+    systemctl daemon-reload
+    systemctl is-active testsleep.service
+done
+
+# Check if the reexec and reload have finished during processing the event.
+grep -F 'ID_PROCESSING=1' "/run/udev/data/n${IFINDEX}"
+
+exit 0

test/units/TEST-22-TMPFILES.sh

@@ -6,6 +6,14 @@ set -o pipefail
 # shellcheck source=test/units/test-control.sh
 . "$(dirname "$0")"/test-control.sh
 
+if systemd-detect-virt --quiet --container; then
+    # This comes from the selinux package and tries to write
+    # some files under sysfs, which will be read-only in a container,
+    # so mask it. It's not our tmpfiles.d file anyway.
+    mkdir -p /run/tmpfiles.d/
+    ln -s /dev/null /run/tmpfiles.d/selinux-policy.conf
+fi
+
 run_subtests
 
 touch /testok