ask-password-api: Add more debug logging

Enable nspawn job, as there's no nested kvm so VMs are too slow. Fix
some tests that fail in a VM anyway, might add a nightly job later that
runs them.

.github/workflows/mkosi.yml

@@ -46,9 +46,9 @@ permissions:
 
 jobs:
   ci:
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
     concurrency:
-      group: ${{ github.workflow }}-${{ matrix.distro }}-${{ matrix.release }}-${{ github.ref }}
+      group: ${{ github.workflow }}-${{ matrix.distro }}-${{ matrix.release }}-${{ github.ref }}-${{ matrix.runner }}
       cancel-in-progress: true
     strategy:
       fail-fast: false
@@ -56,60 +56,94 @@ jobs:
         include:
           - distro: arch
             release: rolling
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-O2 -D_FORTIFY_SOURCE=3"
             relabel: no
             vm: 1
+            no_qemu: 0
+            no_kvm: 0
           - distro: debian
             release: testing
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: no
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
+          - distro: debian
+            release: testing
+            runner: ubuntu-24.04-arm
+            sanitizers: ""
+            llvm: 0
+            cflags: "-Og"
+            relabel: no
+            vm: 0
+            no_qemu: 1
+            no_kvm: 1
           - distro: ubuntu
             release: noble
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: no
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
           - distro: fedora
             release: "42"
+            runner: ubuntu-24.04
             sanitizers: address,undefined
             llvm: 1
             cflags: "-Og"
             relabel: yes
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
           - distro: fedora
             release: rawhide
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: yes
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
           - distro: opensuse
             release: tumbleweed
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: no
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
           - distro: centos
             release: "9"
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: yes
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
           - distro: centos
             release: "10"
+            runner: ubuntu-24.04
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
             relabel: yes
             vm: 0
+            no_qemu: 0
+            no_kvm: 0
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
@@ -215,14 +249,22 @@ jobs:
               MAX_LINES=()
           fi
 
+          if [ "${{ matrix.no_kvm }}" = 1 ] && [ "${{ matrix.no_qemu }}" = 0 ]; then
+              timeout_multiplier=4
+          fi
+
           # --preserve-env makes sure all the github actions environment variables are propagated which are
           # used in integration-test-wrapper.py to construct the `gh` command line to download the journals
           # of failed tests.
           sudo --preserve-env mkosi sandbox -- \
             env \
             TEST_PREFER_QEMU=${{ matrix.vm }} \
+            TEST_NO_QEMU=${{ matrix.no_qemu }} \
+            TEST_NO_KVM=${{ matrix.no_kvm }} \
+            TEST_RUNNER=${{ matrix.runner }} \
             meson test \
             -C build \
+            --timeout-multiplier="${timeout_multiplier:-1}" \
             --no-rebuild \
             --setup=integration \
             --suite=integration-tests \
@@ -235,7 +277,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure() && (github.repository == 'systemd/systemd' || github.repository == 'systemd/systemd-stable')
         with:
-          name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-failed-test-journals
+          name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-${{ matrix.runner }}-failed-test-journals
           path: |
             build/test/journal/*.journal
             build/meson-logs/*

src/shared/ask-password-api.c

@@ -241,14 +241,14 @@ static int ask_password_keyring(const AskPasswordRequest *req, AskPasswordFlags
                 return -EUNATCH;
 
         r = lookup_key(req->keyring, &serial);
-        if (ERRNO_IS_NEG_NOT_SUPPORTED(r) || r == -EPERM)
+        if (ERRNO_IS_NEG_NOT_SUPPORTED(r) || IN_SET(r, -EPERM, -ENOKEY))
                 /* When retrieving, the distinction between "kernel or container manager don't support or
                  * allow this" and "no matching key known" doesn't matter. Note that we propagate EACCESS
                  * here (even if EPERM not) since that is used if the keyring is available, but we lack
                  * access to the key. */
                 return -ENOKEY;
         if (r < 0)
-                return r;
+                return log_debug_errno(r, "Failed to look up key %s in keyring: %m", req->keyring);
 
         _cleanup_strv_free_erase_ char **l = NULL;
         r = retrieve_key(serial, &l);

test/integration-tests/integration-test-wrapper.py

@@ -672,7 +672,10 @@ def main() -> None:
             id = os.environ['GITHUB_RUN_ID']
             wf = os.environ['GITHUB_WORKFLOW']
             iter = os.environ['GITHUB_RUN_ATTEMPT']
-            artifact = f'ci-{wf}-{id}-{iter}-{summary.distribution}-{summary.release}-failed-test-journals'
+            runner = os.environ['TEST_RUNNER']
+            artifact = (
+                f'ci-{wf}-{id}-{iter}-{summary.distribution}-{summary.release}-{runner}-failed-test-journals'  # noqa: E501
+            )
             ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}']
             journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal')
 

test/networkd-test.py

@@ -64,10 +64,11 @@ def setUpModule():
     # Ensure we don't mess with an existing networkd config
     for u in [
         'systemd-networkd.socket',
-        'systemd-networkd',
+        'systemd-networkd-varlink.socket',
+        'systemd-networkd.service',
         'systemd-resolved-varlink.socket',
         'systemd-resolved-monitor.socket',
-        'systemd-resolved',
+        'systemd-resolved.service',
     ]:
         if subprocess.call(['systemctl', 'is-active', '--quiet', u]) == 0:
             subprocess.call(['systemctl', 'stop', u])

test/units/TEST-13-NSPAWN.machined.sh

@@ -13,7 +13,7 @@ at_exit() {
     set +e
 
     machinectl status long-running &>/dev/null && machinectl kill --signal=KILL long-running
-    mountpoint -q /var/lib/machines && timeout 10 sh -c "until umount /var/lib/machines; do sleep .5; done"
+    mountpoint -q /var/lib/machines && timeout 30 sh -c "until umount /var/lib/machines; do sleep .5; done"
     [[ -n "${NSPAWN_FRAGMENT:-}" ]] && rm -f "/etc/systemd/nspawn/$NSPAWN_FRAGMENT" "/var/lib/machines/$NSPAWN_FRAGMENT"
     rm -f /run/systemd/nspawn/*.nspawn
 }
@@ -120,22 +120,22 @@ machinectl disable long-running long-running long-running container1
 # Equivalent to machinectl kill --signal=SIGRTMIN+4 --kill-whom=leader
 rm -f /var/lib/machines/long-running/poweroff
 machinectl poweroff long-running
-timeout 10 bash -c "until test -e /var/lib/machines/long-running/poweroff; do sleep .5; done"
+timeout 30 bash -c "until test -e /var/lib/machines/long-running/poweroff; do sleep .5; done"
 # Equivalent to machinectl kill --signal=SIGINT --kill-whom=leader
 rm -f /var/lib/machines/long-running/reboot
 machinectl reboot long-running
-timeout 10 bash -c "until test -e /var/lib/machines/long-running/reboot; do sleep .5; done"
+timeout 30 bash -c "until test -e /var/lib/machines/long-running/reboot; do sleep .5; done"
 # Test for 'machinectl terminate'
 rm -f /var/lib/machines/long-running/terminate
 machinectl terminate long-running
-timeout 10 bash -c "until test -e /var/lib/machines/long-running/terminate; do sleep .5; done"
+timeout 30 bash -c "until test -e /var/lib/machines/long-running/terminate; do sleep .5; done"
-timeout 10 bash -c "while machinectl status long-running &>/dev/null; do sleep .5; done"
+timeout 30 bash -c "while machinectl status long-running &>/dev/null; do sleep .5; done"
 # Restart container
 long_running_machine_start
 # Test for 'machinectl kill'
 rm -f /var/lib/machines/long-running/trap
 machinectl kill --signal=SIGTRAP --kill-whom=leader long-running
-timeout 10 bash -c "until test -e /var/lib/machines/long-running/trap; do sleep .5; done"
+timeout 30 bash -c "until test -e /var/lib/machines/long-running/trap; do sleep .5; done"
 # Multiple machines at once
 machinectl poweroff long-running long-running long-running
 machinectl reboot long-running long-running long-running
@@ -223,7 +223,7 @@ machinectl import-fs /var/tmp/container.dir container-dir
 machinectl start container-dir
 rm -fr /var/tmp/container.dir
 
-timeout 10 bash -c "until machinectl clean --all; do sleep .5; done"
+timeout 30 bash -c "until machinectl clean --all; do sleep .5; done"
 
 NSPAWN_FRAGMENT="machinectl-test-$RANDOM.nspawn"
 cat >"/var/lib/machines/$NSPAWN_FRAGMENT" <<EOF
@@ -442,7 +442,7 @@ varlinkctl call /run/systemd/machine/io.systemd.Machine io.systemd.Machine.OpenR
 # Terminating machine, otherwise acquiring image metadata by io.systemd.MachineImage.List may fail in the below.
 machinectl terminate long-running
 # wait for the container being stopped, otherwise acquiring image metadata by io.systemd.MachineImage.List may fail in the below.
-timeout 10 bash -c "while machinectl status long-running &>/dev/null; do sleep .5; done"
+timeout 30 bash -c "while machinectl status long-running &>/dev/null; do sleep .5; done"
 systemctl kill --signal=KILL systemd-nspawn@long-running.service || :
 
 # test io.systemd.MachineImage.List

test/units/TEST-13-NSPAWN.nspawn.sh

@@ -1126,7 +1126,10 @@ testcase_check_os_release() {
     base="$(mktemp -d /var/lib/machines/TEST-13-NSPAWN.check_os_release_base.XXX)"
     root="$(mktemp -d /var/lib/machines/TEST-13-NSPAWN.check_os_release.XXX)"
     create_dummy_container "$base"
-    cp -d "$base"/{bin,sbin,lib,lib64} "$root/"
+    cp -d "$base"/{bin,sbin,lib} "$root/"
+    if [ -d "$base"/lib64 ]; then
+        cp -d "$base"/lib64 "$root/"
+    fi
     common_opts=(
         --boot
         --register=no

test/units/TEST-13-NSPAWN.nss-mymachines.sh

@@ -11,7 +11,7 @@ at_exit() {
     set +e
 
     machinectl kill --signal=KILL nss-mymachines-{noip,singleip,manyips}
-    mountpoint -q /var/lib/machines && timeout 10 sh -c "until umount /var/lib/machines; do sleep .5; done"
+    mountpoint -q /var/lib/machines && timeout 30 sh -c "until umount /var/lib/machines; do sleep .5; done"
     rm -f /run/systemd/nspawn/*.nspawn
 }
 

test/units/TEST-17-UDEV.sanity-check.sh

@@ -190,8 +190,8 @@ udevadm test-builtin "factory_reset status" "$loopdev"
 # systemd-hwdb update is extremely slow when combined with sanitizers and run
 # in a VM without acceleration, so let's just skip the one particular test
 # if we detect this combination
-if ! [[ -v ASAN_OPTIONS && "$(systemd-detect-virt -v)" == "qemu" ]]; then
+# scsi_debug is not available in all architectures/kernels combinations
-    modprobe scsi_debug
+if ! [[ -v ASAN_OPTIONS && "$(systemd-detect-virt -v)" == "qemu" ]] && modprobe scsi_debug; then
     scsidev=$(readlink -f /sys/bus/pseudo/drivers/scsi_debug/adapter*/host*/target*/[0-9]*)
     mkdir -p /etc/udev/hwdb.d
     cat >/etc/udev/hwdb.d/99-test.hwdb <<EOF

test/units/TEST-35-LOGIN.sh

@@ -298,7 +298,7 @@ teardown_session() (
 
     rm -f /run/udev/rules.d/70-logindtest-scsi_debug-user.rules
     udevadm control --reload
-    rmmod scsi_debug
+    rmmod scsi_debug || true
 
     return 0
 )
@@ -446,7 +446,11 @@ EOF
 
     # coldplug: logind started with existing device
     systemctl stop systemd-logind.service
-    modprobe scsi_debug
+    if ! modprobe scsi_debug; then
+        echo "scsi_debug module not available, skipping test ${FUNCNAME[0]}."
+        systemctl start systemd-logind.service
+        return
+    fi
     timeout 30 bash -c 'until ls /sys/bus/pseudo/drivers/scsi_debug/adapter*/host*/target*/*:*/block 2>/dev/null; do sleep 1; done'
     dev=/dev/$(ls /sys/bus/pseudo/drivers/scsi_debug/adapter*/host*/target*/*:*/block 2>/dev/null)
     if [[ ! -b "$dev" ]]; then