tests: some more logs to try and get info about VM spurious reboots (#40028)

https://github.com/systemd/systemd/issues/38240 keep happening, trying
to get more info

.github/workflows/coverage.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@10544812b35a668d4aac9834c78ee8166e99bc78
+      - uses: systemd/mkosi@5a476a92deca8ad54869e5d416217aa1bb137b25
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location

.github/workflows/linter.yml

@@ -38,7 +38,7 @@ jobs:
           LINTER_RULES_PATH: .github/linters
           GITHUB_ACTIONS_CONFIG_FILE: actionlint.yml
 
-      - uses: systemd/mkosi@10544812b35a668d4aac9834c78ee8166e99bc78
+      - uses: systemd/mkosi@5a476a92deca8ad54869e5d416217aa1bb137b25
 
       - name: Check that tabs are not used in Python code
         run: sh -c '! git grep -P "\\t" -- src/core/generate-bpf-delegate-configs.py src/boot/generate-hwids-section.py src/ukify/ukify.py test/integration-tests/integration-test-wrapper.py'

.github/workflows/mkosi.yml

@@ -167,7 +167,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@10544812b35a668d4aac9834c78ee8166e99bc78
+      - uses: systemd/mkosi@5a476a92deca8ad54869e5d416217aa1bb137b25
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location

mkosi/mkosi.conf

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Config]
-MinimumVersion=commit:10544812b35a668d4aac9834c78ee8166e99bc78
+MinimumVersion=commit:5a476a92deca8ad54869e5d416217aa1bb137b25
 Dependencies=
         exitrd
         initrd
@@ -147,7 +147,6 @@ Credentials=
         tty.virtual.tty1.agetty.autologin=root
         tty.virtual.tty1.login.noauth=yes
 RuntimeBuildSources=yes
-RuntimeScratch=no
 CPUs=2
 TPM=yes
 VSock=yes

src/import/importd.c

@@ -27,7 +27,6 @@
 #include "import-common.h"
 #include "import-util.h"
 #include "json-util.h"
-#include "machine-pool.h"
 #include "main-func.h"
 #include "notify-recv.h"
 #include "os-util.h"
@@ -820,9 +819,9 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_
                                          "Local image name %s is invalid", local);
 
         if (class == IMAGE_MACHINE) {
-                r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota);
+                r = image_setup_pool(m->runtime_scope, class, m->use_btrfs_subvol, m->use_btrfs_quota);
                 if (r < 0)
-                        return r;
+                        return sd_bus_error_set_errnof(error, r, "Failed to set up machine pool: %m");
         }
 
         type = startswith(sd_bus_message_get_member(msg), "ImportTar") ?
@@ -921,9 +920,9 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e
                                          "Local image name %s is invalid", local);
 
         if (class == IMAGE_MACHINE) {
-                r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota);
+                r = image_setup_pool(m->runtime_scope, class, m->use_btrfs_subvol, m->use_btrfs_quota);
                 if (r < 0)
-                        return r;
+                        return sd_bus_error_set_errnof(error, r, "Failed to set up machine pool: %m");
         }
 
         r = transfer_new(m, &t);
@@ -1126,9 +1125,9 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er
                                          "Unknown verification mode %s", verify);
 
         if (class == IMAGE_MACHINE) {
-                r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota);
+                r = image_setup_pool(m->runtime_scope, class, m->use_btrfs_subvol, m->use_btrfs_quota);
                 if (r < 0)
-                        return r;
+                        return sd_bus_error_set_errnof(error, r, "Failed to set up machine pool: %m");
         }
 
         type = startswith(sd_bus_message_get_member(msg), "PullTar") ?

src/machine/image-varlink.c

@@ -14,7 +14,6 @@
 #include "image-varlink.h"
 #include "io-util.h"
 #include "json-util.h"
-#include "machine-pool.h"
 #include "machined.h"
 #include "operation.h"
 #include "process-util.h"
@@ -280,7 +279,11 @@ int vl_method_set_pool_limit(sd_varlink *link, sd_json_variant *parameters, sd_v
         }
 
         /* Set up the machine directory if necessary */
-        r = setup_machine_directory(/* error = */ NULL, /* use_btrfs_subvol= */ true, /* use_btrfs_quota= */ true);
+        r = image_setup_pool(
+                        manager->runtime_scope,
+                        IMAGE_MACHINE,
+                        /* use_btrfs_subvol= */ true,
+                        /* use_btrfs_quota= */ true);
         if (r < 0)
                 return r;
 

src/machine/machined-dbus.c

@@ -6,9 +6,7 @@
 #include "sd-id128.h"
 
 #include "alloc-util.h"
-#include "btrfs-util.h"
 #include "bus-common-errors.h"
-#include "bus-get-properties.h"
 #include "bus-locator.h"
 #include "bus-message-util.h"
 #include "bus-object.h"
@@ -27,7 +25,6 @@
 #include "io-util.h"
 #include "machine.h"
 #include "machine-dbus.h"
-#include "machine-pool.h"
 #include "machined.h"
 #include "namespace-util.h"
 #include "operation.h"
@@ -39,7 +36,25 @@
 #include "unit-def.h"
 #include "user-util.h"
 
-static BUS_DEFINE_PROPERTY_GET_GLOBAL(property_get_pool_path, "s", "/var/lib/machines");
+static int property_get_pool_path(
+                sd_bus *bus,
+                const char *path,
+                const char *interface,
+                const char *property,
+                sd_bus_message *reply,
+                void *userdata,
+                sd_bus_error *error) {
+
+        _cleanup_free_ char *poolpath = NULL;
+        Manager *m = ASSERT_PTR(userdata);
+
+        assert(bus);
+        assert(reply);
+
+        (void) image_get_pool_path(m->runtime_scope, IMAGE_MACHINE, &poolpath);
+
+        return sd_bus_message_append(reply, "s", strempty(poolpath));
+}
 
 static int property_get_pool_usage(
                 sd_bus *bus,
@@ -50,19 +65,13 @@ static int property_get_pool_usage(
                 void *userdata,
                 sd_bus_error *error) {
 
-        _cleanup_close_ int fd = -EBADF;
+        Manager *m = ASSERT_PTR(userdata);
         uint64_t usage = UINT64_MAX;
 
         assert(bus);
         assert(reply);
 
-        fd = open("/var/lib/machines", O_RDONLY|O_CLOEXEC|O_DIRECTORY);
+        (void) image_get_pool_usage(m->runtime_scope, IMAGE_MACHINE, &usage);
-        if (fd >= 0) {
-                BtrfsQuotaInfo q;
-
-                if (btrfs_subvol_get_subtree_quota_fd(fd, 0, &q) >= 0)
-                        usage = q.referenced;
-        }
 
         return sd_bus_message_append(reply, "t", usage);
 }
@@ -76,19 +85,13 @@ static int property_get_pool_limit(
                 void *userdata,
                 sd_bus_error *error) {
 
-        _cleanup_close_ int fd = -EBADF;
+        Manager *m = ASSERT_PTR(userdata);
         uint64_t size = UINT64_MAX;
 
         assert(bus);
         assert(reply);
 
-        fd = open("/var/lib/machines", O_RDONLY|O_CLOEXEC|O_DIRECTORY);
+        (void) image_get_pool_limit(m->runtime_scope, IMAGE_MACHINE, &size);
-        if (fd >= 0) {
-                BtrfsQuotaInfo q;
-
-                if (btrfs_subvol_get_subtree_quota_fd(fd, 0, &q) >= 0)
-                        size = q.referenced_max;
-        }
 
         return sd_bus_message_append(reply, "t", size);
 }
@@ -1077,9 +1080,13 @@ static int method_set_pool_limit(sd_bus_message *message, void *userdata, sd_bus
         }
 
         /* Set up the machine directory if necessary */
-        r = setup_machine_directory(error, /* use_btrfs_subvol= */ true, /* use_btrfs_quota= */ true);
+        r = image_setup_pool(
+                        m->runtime_scope,
+                        IMAGE_MACHINE,
+                        /* use_btrfs_subvol= */ true,
+                        /* use_btrfs_quota= */ true);
         if (r < 0)
-                return r;
+                return sd_bus_error_set_errnof(error, r, "Failed to set up machine pool: %m");
 
         r = image_set_pool_limit(m->runtime_scope, IMAGE_MACHINE, limit);
         if (ERRNO_IS_NEG_NOT_SUPPORTED(r))

src/shared/discover-image.c

@@ -31,6 +31,7 @@
 #include "hashmap.h"
 #include "hostname-setup.h"
 #include "id128-util.h"
+#include "label-util.h"
 #include "lock-util.h"
 #include "log.h"
 #include "loop-util.h"
@@ -1919,6 +1920,119 @@ int image_set_pool_limit(RuntimeScope scope, ImageClass class, uint64_t referenc
         return 0;
 }
 
+int image_get_pool_path(RuntimeScope scope, ImageClass class, char **ret) {
+        assert(scope >= 0 && scope < _RUNTIME_SCOPE_MAX);
+        assert(class >= 0 && class < _IMAGE_CLASS_MAX);
+        assert(ret);
+
+        return get_pool_directory(scope, class, /* fname= */ NULL, /* suffix= */ NULL, ret);
+}
+
+int image_get_pool_usage(RuntimeScope scope, ImageClass class, uint64_t *ret) {
+        int r;
+
+        assert(scope >= 0 && scope < _RUNTIME_SCOPE_MAX);
+        assert(class >= 0 && class < _IMAGE_CLASS_MAX);
+        assert(ret);
+
+        _cleanup_free_ char *pool = NULL;
+        r = get_pool_directory(scope, class, /* fname= */ NULL, /* suffix= */ NULL, &pool);
+        if (r < 0)
+                return r;
+
+        _cleanup_close_ int fd = open(pool, O_RDONLY|O_CLOEXEC|O_DIRECTORY);
+        if (fd < 0)
+                return -errno;
+
+        BtrfsQuotaInfo q;
+        r = btrfs_subvol_get_subtree_quota_fd(fd, /* subvol_id= */ 0, &q);
+        if (r < 0)
+                return r;
+
+        *ret = q.referenced;
+        return 0;
+}
+
+int image_get_pool_limit(RuntimeScope scope, ImageClass class, uint64_t *ret) {
+        int r;
+
+        assert(scope >= 0 && scope < _RUNTIME_SCOPE_MAX);
+        assert(class >= 0 && class < _IMAGE_CLASS_MAX);
+        assert(ret);
+
+        _cleanup_free_ char *pool = NULL;
+        r = get_pool_directory(scope, class, /* fname= */ NULL, /* suffix= */ NULL, &pool);
+        if (r < 0)
+                return r;
+
+        _cleanup_close_ int fd = open(pool, O_RDONLY|O_CLOEXEC|O_DIRECTORY);
+        if (fd < 0)
+                return -errno;
+
+        BtrfsQuotaInfo q;
+        r = btrfs_subvol_get_subtree_quota_fd(fd, /* subvol_id= */ 0, &q);
+        if (r < 0)
+                return r;
+
+        *ret = q.referenced_max;
+        return 0;
+}
+
+static int check_btrfs(const char *path) {
+        struct statfs sfs;
+        int r;
+
+        if (statfs(path, &sfs) < 0) {
+                if (errno != ENOENT)
+                        return -errno;
+
+                _cleanup_free_ char *parent = NULL;
+                r = path_extract_directory(path, &parent);
+                if (r < 0)
+                        return r;
+
+                if (statfs(parent, &sfs) < 0)
+                        return -errno;
+        }
+
+        return F_TYPE_EQUAL(sfs.f_type, BTRFS_SUPER_MAGIC);
+}
+
+int image_setup_pool(RuntimeScope scope, ImageClass class, bool use_btrfs_subvol, bool use_btrfs_quota) {
+        int r;
+
+        assert(class >= 0 && class < _IMAGE_CLASS_MAX);
+
+        _cleanup_free_ char *pool = NULL;
+        r = image_get_pool_path(scope, class, &pool);
+        if (r < 0)
+                return r;
+
+        r = check_btrfs(pool);
+        if (r < 0)
+                return r;
+        if (r == 0)
+                return 0;
+
+        if (!use_btrfs_subvol)
+                return 0;
+
+        (void) btrfs_subvol_make_label(pool);
+
+        if (!use_btrfs_quota)
+                return 0;
+
+        r = btrfs_quota_enable(pool, /* b= */ true);
+        if (r < 0)
+                log_warning_errno(r, "Failed to enable quota for %s, ignoring: %m", pool);
+
+        r = btrfs_subvol_auto_qgroup(pool, /* subvol_id= */ 0, /* create_intermediary_qgroup= */ true);
+        if (r < 0)
+                log_warning_errno(r, "Failed to set up default quota hierarchy for %s, ignoring: %m", pool);
+
+        return 0;
+}
+
 int image_read_metadata(Image *i, const ImagePolicy *image_policy, RuntimeScope scope) {
         _cleanup_(release_lock_file) LockFile global_lock = LOCK_FILE_INIT, local_lock = LOCK_FILE_INIT;
         int r;

src/shared/discover-image.h

@@ -69,6 +69,10 @@ int image_name_lock(RuntimeScope scope, const char *name, int operation, LockFil
 
 int image_set_limit(Image *i, uint64_t referenced_max);
 int image_set_pool_limit(RuntimeScope scope, ImageClass class, uint64_t referenced_max);
+int image_get_pool_path(RuntimeScope scope, ImageClass class, char **ret);
+int image_get_pool_usage(RuntimeScope scope, ImageClass class, uint64_t *ret);
+int image_get_pool_limit(RuntimeScope scope, ImageClass class, uint64_t *ret);
+int image_setup_pool(RuntimeScope scope, ImageClass class, bool use_btrfs_subvol, bool use_btrfs_quota);
 
 int image_read_metadata(Image *i, const ImagePolicy *image_policy, RuntimeScope scope);
 

src/shared/machine-pool.c

@@ -1,53 +0,0 @@
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-
-#include <linux/magic.h>
-
-#include "sd-bus.h"
-
-#include "btrfs-util.h"
-#include "label-util.h"
-#include "log.h"
-#include "machine-pool.h"
-#include "stat-util.h"
-
-static int check_btrfs(void) {
-        struct statfs sfs;
-
-        if (statfs("/var/lib/machines", &sfs) < 0) {
-                if (errno != ENOENT)
-                        return -errno;
-
-                if (statfs("/var/lib", &sfs) < 0)
-                        return -errno;
-        }
-
-        return F_TYPE_EQUAL(sfs.f_type, BTRFS_SUPER_MAGIC);
-}
-
-int setup_machine_directory(sd_bus_error *error, bool use_btrfs_subvol, bool use_btrfs_quota) {
-        int r;
-
-        r = check_btrfs();
-        if (r < 0)
-                return sd_bus_error_set_errnof(error, r, "Failed to determine whether /var/lib/machines is located on btrfs: %m");
-        if (r == 0)
-                return 0;
-
-        if (!use_btrfs_subvol)
-                return 0;
-
-        (void) btrfs_subvol_make_label("/var/lib/machines");
-
-        if (!use_btrfs_quota)
-                return 0;
-
-        r = btrfs_quota_enable("/var/lib/machines", true);
-        if (r < 0)
-                log_warning_errno(r, "Failed to enable quota for /var/lib/machines, ignoring: %m");
-
-        r = btrfs_subvol_auto_qgroup("/var/lib/machines", 0, true);
-        if (r < 0)
-                log_warning_errno(r, "Failed to set up default quota hierarchy for /var/lib/machines, ignoring: %m");
-
-        return 0;
-}

src/shared/machine-pool.h

@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-#pragma once
-
-#include "shared-forward.h"
-
-int setup_machine_directory(sd_bus_error *error, bool use_btrfs_subvol, bool use_btrfs_quota);

src/shared/meson.build

@@ -125,7 +125,6 @@ shared_sources = files(
         'machine-bind-user.c',
         'machine-credential.c',
         'machine-id-setup.c',
-        'machine-pool.c',
         'macvlan-util.c',
         'main-func.c',
         'mkdir-label.c',

test/integration-tests/integration-test-wrapper.py

@@ -584,7 +584,6 @@ def main() -> None:
         ),
         '--credential', f'systemd.unit-dropin.{args.unit}={shlex.quote(dropin)}',
         '--runtime-network=none',
-        '--runtime-scratch=no',
         *([f'--qemu-args=-rtc base={rtc}'] if rtc else []),
         *args.mkosi_args,
         '--firmware', firmware,
@@ -616,6 +615,10 @@ def main() -> None:
         *(['--', '--capability=CAP_BPF'] if not vm else []),
     ]  # fmt: skip
 
+    # XXX: debug for https://github.com/systemd/systemd/issues/38240
+    if vm:
+        cmd += ['--qemu-args=-d cpu_reset,guest_errors -D /dev/stderr']
+
     try:
         result = subprocess.run(cmd)
 
@@ -623,6 +626,10 @@ def main() -> None:
         if args.vm and result.returncode == 247 and args.exit_code != 247:
             if journal_file:
                 journal_file.unlink(missing_ok=True)
+            print(
+                f'Test {args.name} failed due to QEMU crash (error 247), retrying...',
+                file=sys.stderr,
+            )
             result = subprocess.run(cmd)
             if args.vm and result.returncode == 247 and args.exit_code != 247:
                 print(
@@ -630,6 +637,10 @@ def main() -> None:
                     file=sys.stderr,
                 )
                 exit(77)
+            print(
+                f'Test {args.name} worked on re-run after QEMU crash (error 247)',
+                file=sys.stderr,
+            )
     except KeyboardInterrupt:
         result = subprocess.CompletedProcess(args=cmd, returncode=-signal.SIGINT)