Merge fd1803144c into a035eaa227

.github/workflows/coverage.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@dbb4020beee2cdf250f93a425794f1cf8b0fe693
+      - uses: systemd/mkosi@7e4ec15aee6b98300b2ee14265bc647a716a9f8a
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location
@@ -90,6 +90,7 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
+            -Dintegration-tests=true \
             build
 
       - name: Build image
@@ -119,8 +120,7 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --setup=integration \
+            --suite integration-tests \
-            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

.github/workflows/mkosi.yml

@@ -120,7 +120,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@dbb4020beee2cdf250f93a425794f1cf8b0fe693
+      - uses: systemd/mkosi@7e4ec15aee6b98300b2ee14265bc647a716a9f8a
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location
@@ -197,6 +197,7 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
+            -Dintegration-tests=true \
             -Dbpf-framework=disabled \
             build
 
@@ -232,8 +233,7 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --setup=integration \
+            --suite integration-tests \
-            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

meson.build

@@ -13,12 +13,6 @@ project('systemd', 'c',
         meson_version : '>= 0.62.0',
 )
 
-add_test_setup(
-        'default',
-        exclude_suites : ['integration-tests'],
-        is_default : true,
-)
-
 project_major_version = meson.project_version().split('.')[0].split('~')[0]
 if meson.project_version().contains('.')
         project_minor_version = meson.project_version().split('.')[-1].split('~')[0]
@@ -345,6 +339,7 @@ meson_build_sh = find_program('tools/meson-build.sh')
 want_tests = get_option('tests')
 want_slow_tests = want_tests != 'false' and get_option('slow-tests')
 want_fuzz_tests = want_tests != 'false' and get_option('fuzz-tests')
+want_integration_tests = want_tests != 'false' and get_option('integration-tests')
 install_tests = want_tests != 'false' and get_option('install-tests')
 
 if add_languages('cpp', native : false, required : fuzzer_build)
@@ -2666,6 +2661,10 @@ endif
 #####################################################################
 
 mkosi = find_program('mkosi', required : false)
+if want_integration_tests and not mkosi.found()
+        error('Could not find mkosi which is required to run the integration tests')
+endif
+
 mkosi_depends = public_programs
 
 foreach executable : ['systemd-journal-remote', 'systemd-sbsign', 'systemd-keyutil']

meson_options.txt

@@ -509,7 +509,7 @@ option('install-tests', type : 'boolean', value : false,
        description : 'install test executables')
 option('log-message-verification', type : 'feature', deprecated : { 'true' : 'enabled', 'false' : 'disabled' },
        description : 'do fake printf() calls to verify format strings')
-option('integration-tests', type : 'boolean', value : false, deprecated : true,
+option('integration-tests', type : 'boolean', value : false,
        description : 'run the integration tests')
 
 option('ok-color', type : 'combo',

mkosi/mkosi.conf

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Config]
-MinimumVersion=commit:dbb4020beee2cdf250f93a425794f1cf8b0fe693
+MinimumVersion=commit:7e4ec15aee6b98300b2ee14265bc647a716a9f8a
 Dependencies=
         exitrd
         initrd
@@ -78,7 +78,8 @@ KernelCommandLine=
         oops=panic
         panic=-1
         softlockup_panic=1
-        panic_on_warn=1
+        # Disabled due to BTRFS issue, waiting for the fix to become available
+        panic_on_warn=0
         psi=1
         mitigations=off
 

src/basic/fd-util.c

@@ -1001,13 +1001,13 @@ int fd_verify_safe_flags_full(int fd, int extra_flags) {
         if (flags < 0)
                 return -errno;
 
-        unexpected_flags = flags & ~(O_ACCMODE_STRICT|O_NOFOLLOW|RAW_O_LARGEFILE|extra_flags);
+        unexpected_flags = flags & ~(O_ACCMODE|O_NOFOLLOW|RAW_O_LARGEFILE|extra_flags);
         if (unexpected_flags != 0)
                 return log_debug_errno(SYNTHETIC_ERRNO(EREMOTEIO),
                                        "Unexpected flags set for extrinsic fd: 0%o",
                                        (unsigned) unexpected_flags);
 
-        return flags & (O_ACCMODE_STRICT | extra_flags); /* return the flags variable, but remove the noise */
+        return flags & (O_ACCMODE | extra_flags); /* return the flags variable, but remove the noise */
 }
 
 int read_nr_open(void) {
@@ -1132,7 +1132,7 @@ int fds_are_same_mount(int fd1, int fd2) {
 }
 
 const char* accmode_to_string(int flags) {
-        switch (flags & O_ACCMODE_STRICT) {
+        switch (flags & O_ACCMODE) {
         case O_RDONLY:
                 return "ro";
         case O_WRONLY:

src/basic/fs-util.c

@@ -1036,7 +1036,7 @@ int open_mkdir_at_full(int dirfd, const char *path, int flags, XOpenFlags xopen_
 
         if (flags & ~(O_RDONLY|O_CLOEXEC|O_DIRECTORY|O_EXCL|O_NOATIME|O_NOFOLLOW|O_PATH))
                 return -EINVAL;
-        if ((flags & O_ACCMODE_STRICT) != O_RDONLY)
+        if ((flags & O_ACCMODE) != O_RDONLY)
                 return -EINVAL;
 
         /* Note that O_DIRECTORY|O_NOFOLLOW is implied, but we allow specifying it anyway. The following

src/basic/missing_fcntl.h

@@ -43,9 +43,3 @@
 #ifndef AT_HANDLE_FID
 #define AT_HANDLE_FID AT_REMOVEDIR
 #endif
-
-/* On musl, O_ACCMODE is defined as (03|O_SEARCH), unlike glibc which defines it as
- * (O_RDONLY|O_WRONLY|O_RDWR). Additionally, O_SEARCH is simply defined as O_PATH. This changes the behaviour
- * of O_ACCMODE in certain situations, which we don't want. This definition is copied from glibc and works
- * around the problems with musl's definition. */
-#define O_ACCMODE_STRICT (O_RDONLY|O_WRONLY|O_RDWR)

src/core/exec-invoke.c

@@ -267,7 +267,7 @@ static int acquire_path(const char *path, int flags, mode_t mode) {
 
         assert(path);
 
-        if (IN_SET(flags & O_ACCMODE_STRICT, O_WRONLY, O_RDWR))
+        if (IN_SET(flags & O_ACCMODE, O_WRONLY, O_RDWR))
                 flags |= O_CREAT;
 
         fd = open(path, flags|O_NOCTTY, mode);
@@ -291,9 +291,9 @@ static int acquire_path(const char *path, int flags, mode_t mode) {
         if (r < 0)
                 return r;
 
-        if ((flags & O_ACCMODE_STRICT) == O_RDONLY)
+        if ((flags & O_ACCMODE) == O_RDONLY)
                 r = shutdown(fd, SHUT_WR);
-        else if ((flags & O_ACCMODE_STRICT) == O_WRONLY)
+        else if ((flags & O_ACCMODE) == O_WRONLY)
                 r = shutdown(fd, SHUT_RD);
         else
                 r = 0;

src/libsystemd/sd-journal/journal-file.c

@@ -98,7 +98,7 @@ DEFINE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
                 JournalFile, journal_file_close);
 
 static int mmap_prot_from_open_flags(int flags) {
-        switch (flags & O_ACCMODE_STRICT) {
+        switch (flags & O_ACCMODE) {
         case O_RDONLY:
                 return PROT_READ;
         case O_WRONLY:
@@ -4075,10 +4075,10 @@ int journal_file_open(
         assert(mmap_cache);
         assert(ret);
 
-        if (!IN_SET((open_flags & O_ACCMODE_STRICT), O_RDONLY, O_RDWR))
+        if (!IN_SET((open_flags & O_ACCMODE), O_RDONLY, O_RDWR))
                 return -EINVAL;
 
-        if ((open_flags & O_ACCMODE_STRICT) == O_RDONLY && FLAGS_SET(open_flags, O_CREAT))
+        if ((open_flags & O_ACCMODE) == O_RDONLY && FLAGS_SET(open_flags, O_CREAT))
                 return -EINVAL;
 
         if (fname && (open_flags & O_CREAT) && !endswith(fname, ".journal"))

src/libsystemd/sd-journal/journal-file.h

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 #pragma once
 
+#include <fcntl.h>
 #include <inttypes.h>
 #include <sys/uio.h>
 
@@ -14,7 +15,6 @@
 #include "compress.h"
 #include "hashmap.h"
 #include "journal-def.h"
-#include "missing_fcntl.h"
 #include "mmap-cache.h"
 #include "sparse-endian.h"
 #include "time-util.h"
@@ -391,5 +391,5 @@ static inline uint32_t COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(Compression c) {
 
 static inline bool journal_file_writable(JournalFile *f) {
         assert(f);
-        return (f->open_flags & O_ACCMODE_STRICT) != O_RDONLY;
+        return (f->open_flags & O_ACCMODE) != O_RDONLY;
 }

src/login/logind-session-dbus.c

@@ -536,7 +536,7 @@ static int method_set_tty(sd_bus_message *message, void *userdata, sd_bus_error
         flags = fcntl(fd, F_GETFL, 0);
         if (flags < 0)
                 return -errno;
-        if ((flags & O_ACCMODE_STRICT) != O_RDWR)
+        if ((flags & O_ACCMODE) != O_RDWR)
                 return -EACCES;
         if (FLAGS_SET(flags, O_PATH))
                 return -ENOTTY;

src/mountfsd/mountwork.c

@@ -99,7 +99,7 @@ static int validate_image_fd(int fd, MountImageParameters *p) {
         if (fl < 0)
                 return log_debug_errno(fl, "Image file descriptor has unsafe flags set: %m");
 
-        switch (fl & O_ACCMODE_STRICT) {
+        switch (fl & O_ACCMODE) {
 
         case O_RDONLY:
                 p->read_only = true;

src/shared/data-fd-util.c

@@ -146,13 +146,13 @@ int memfd_clone_fd(int fd, const char *name, int mode) {
 
         assert(fd >= 0);
         assert(name);
-        assert(IN_SET(mode & O_ACCMODE_STRICT, O_RDONLY, O_RDWR));
+        assert(IN_SET(mode & O_ACCMODE, O_RDONLY, O_RDWR));
         assert((mode & ~(O_RDONLY|O_RDWR|O_CLOEXEC)) == 0);
 
         if (fstat(fd, &st) < 0)
                 return -errno;
 
-        ro = (mode & O_ACCMODE_STRICT) == O_RDONLY;
+        ro = (mode & O_ACCMODE) == O_RDONLY;
         exec = st.st_mode & 0111;
 
         mfd = memfd_create_wrapper(name,

src/shared/journal-file-util.c

@@ -504,7 +504,7 @@ int journal_file_open_reliably(
                     -EIDRM))            /* File has been deleted */
                 return r;
 
-        if ((open_flags & O_ACCMODE_STRICT) == O_RDONLY)
+        if ((open_flags & O_ACCMODE) == O_RDONLY)
                 return r;
 
         if (!(open_flags & O_CREAT))
@@ -519,7 +519,7 @@ int journal_file_open_reliably(
         /* The file is corrupted. Try opening it read-only as the template before rotating to inherit its
          * sequence number and ID. */
         r = journal_file_open(-EBADF, fname,
-                              (open_flags & ~(O_ACCMODE_STRICT|O_CREAT|O_EXCL)) | O_RDONLY,
+                              (open_flags & ~(O_ACCMODE|O_CREAT|O_EXCL)) | O_RDONLY,
                               file_flags, 0, compress_threshold_bytes, NULL,
                               mmap_cache, /* template = */ NULL, &old_file);
         if (r < 0)

src/shared/loop-util.c

@@ -500,7 +500,7 @@ static int loop_device_make_internal(
                 .block_size = sector_size,
                 .info = {
                         /* Use the specified flags, but configure the read-only flag from the open flags, and force autoclear */
-                        .lo_flags = (loop_flags & ~LO_FLAGS_READ_ONLY) | ((open_flags & O_ACCMODE_STRICT) == O_RDONLY ? LO_FLAGS_READ_ONLY : 0) | LO_FLAGS_AUTOCLEAR,
+                        .lo_flags = (loop_flags & ~LO_FLAGS_READ_ONLY) | ((open_flags & O_ACCMODE) == O_RDONLY ? LO_FLAGS_READ_ONLY : 0) | LO_FLAGS_AUTOCLEAR,
                         .lo_offset = offset,
                         .lo_sizelimit = size == UINT64_MAX ? 0 : size,
                 },

src/test/test-fileio.c

@@ -1104,7 +1104,7 @@ TEST(fdopen_independent) {
         zero(buf);
         assert_se(fread(buf, 1, sizeof(buf), f) == strlen(TEST_TEXT));
         ASSERT_STREQ(buf, TEST_TEXT);
-        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE_STRICT) == O_RDONLY);
+        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE) == O_RDONLY);
         assert_se(FLAGS_SET(fcntl(fileno(f), F_GETFD), FD_CLOEXEC));
         f = safe_fclose(f);
 
@@ -1112,7 +1112,7 @@ TEST(fdopen_independent) {
         zero(buf);
         assert_se(fread(buf, 1, sizeof(buf), f) == strlen(TEST_TEXT));
         ASSERT_STREQ(buf, TEST_TEXT);
-        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE_STRICT) == O_RDONLY);
+        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE) == O_RDONLY);
         assert_se(!FLAGS_SET(fcntl(fileno(f), F_GETFD), FD_CLOEXEC));
         f = safe_fclose(f);
 
@@ -1120,7 +1120,7 @@ TEST(fdopen_independent) {
         zero(buf);
         assert_se(fread(buf, 1, sizeof(buf), f) == strlen(TEST_TEXT));
         ASSERT_STREQ(buf, TEST_TEXT);
-        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE_STRICT) == O_RDWR);
+        assert_se((fcntl(fileno(f), F_GETFL) & O_ACCMODE) == O_RDWR);
         assert_se(FLAGS_SET(fcntl(fileno(f), F_GETFD), FD_CLOEXEC));
         f = safe_fclose(f);
 }

test/integration-tests/README.md

@@ -38,14 +38,14 @@ directory (`OutputDirectory=`) to point to the other directory using `mkosi/mkos
 After the image has been built, the integration tests can be run with:
 
 ```shell
-$ mkosi -f sandbox -- meson test -C build --setup=integration --suite integration-tests --num-processes "$(($(nproc) / 4))"
+$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build --suite integration-tests --num-processes "$(($(nproc) / 4))"
 ```
 
 As usual, specific tests can be run in meson by appending the name of the test
 which is usually the name of the directory e.g.
 
 ```shell
-$ mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
+$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
 ```
 
 See `mkosi -f sandbox -- meson introspect build --tests` for a list of tests.
@@ -55,7 +55,7 @@ To interactively debug a failing integration test, the `--interactive` option
 newer:
 
 ```shell
-$ mkosi -f sandbox -- meson test -C build --setup=integration -i TEST-01-BASIC
+$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
 ```
 
 Due to limitations in meson, the integration tests do not yet depend on the
@@ -64,7 +64,7 @@ running the integration tests. To rebuild the image and rerun a test, the
 following command can be used:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
 ```
 
 The integration tests use the same mkosi configuration that's used when you run
@@ -78,7 +78,7 @@ To iterate on an integration test, let's first get a shell in the integration te
 the following:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=shell -i TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
 ```
 
 This will get us a shell in the integration test environment after booting the machine without running the
@@ -107,7 +107,7 @@ re-running the test will first install the new packages we just built, make a ne
 the test again. You can keep running the loop of `mkosi -R`, `systemctl soft-reboot` and
 `systemctl start ...` until the changes to the integration test are working.
 
-If you're debugging a failing integration test (running `meson test --interactive`),
+If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`),
 there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to
 rerun the test.
 
@@ -120,6 +120,10 @@ rerun the test.
 `TEST_NO_KVM=1`: Disable qemu KVM auto-detection (may be necessary when you're
 trying to run the *vanilla* qemu and have both qemu and qemu-kvm installed)
 
+`TEST_SHELL=1`: Configure the machine to be more *user-friendly* for
+interactive debugging (e.g. by setting a usable default terminal, suppressing
+the shutdown after the test, etc.).
+
 `TEST_MATCH_SUBTEST=subtest`:  If the test makes use of `run_subtests` use this
 variable to provide a POSIX extended regex to run only subtests matching the
 expression.

test/integration-tests/integration-test-wrapper.py

@@ -361,7 +361,7 @@ def statfs(path: Path) -> str:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--mkosi', default=None)
+    parser.add_argument('--mkosi', required=True)
     parser.add_argument('--meson-source-dir', required=True, type=Path)
     parser.add_argument('--meson-build-dir', required=True, type=Path)
     parser.add_argument('--name', required=True)
@@ -379,12 +379,6 @@ def main() -> None:
     parser.add_argument('mkosi_args', nargs='*')
     args = parser.parse_args()
 
-    if not args.mkosi:
-        args.mkosi = shutil.which('mkosi')
-        if not args.mkosi:
-            print('Could not find mkosi which is required to run the integration tests', file=sys.stderr)
-            sys.exit(1)
-
     # The meson source directory can either be the top-level repository directory or the
     # test/integration-tests/standalone subdirectory in the repository directory. The mkosi configuration
     # will always be a parent directory of one of these directories and at most 4 levels upwards, so don't
@@ -401,6 +395,13 @@ def main() -> None:
         )
         exit(1)
 
+    if not bool(int(os.getenv('SYSTEMD_INTEGRATION_TESTS', '0'))):
+        print(
+            f'SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.name}',
+            file=sys.stderr,
+        )
+        exit(77)
+
     if args.slow and not bool(int(os.getenv('SYSTEMD_SLOW_TESTS', '0'))):
         print(
             f'SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}',

test/integration-tests/meson.build

@@ -1,9 +1,5 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-# We'd give these more descriptive names but only alphanumeric characters are allowed.
-add_test_setup('integration')
-add_test_setup('shell', env : {'TEST_SHELL' : '1'})
-
 integration_test_wrapper = find_program('integration-test-wrapper.py')
 integration_tests = []
 integration_test_template = {
@@ -133,11 +129,11 @@ foreach integration_test : integration_tests
                 integration_test_args += ['--skip']
         endif
 
-        if mkosi.found()
+        if not mkosi.found()
-                integration_test_args += ['--mkosi', mkosi.full_path()]
+                continue
         endif
 
-        integration_test_args += ['--']
+        integration_test_args += ['--mkosi', mkosi.full_path(), '--']
 
         if integration_test['cmdline'].length() > 0
                 integration_test_args += [
@@ -155,12 +151,19 @@ foreach integration_test : integration_tests
 
         integration_test_args += integration_test['mkosi-args']
 
+        integration_test_env = {}
+
+        if want_integration_tests
+                integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'}
+        endif
+
         # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt on every
         # "ninja -C build". Instead, the mkosi target has to be rebuilt manually before running the
         # integration tests with mkosi.
         test(
                 integration_test['name'],
                 integration_test_wrapper,
+                env : integration_test_env,
                 args : integration_test_args,
                 timeout : integration_test['timeout'],
                 priority : integration_test['priority'],

test/integration-tests/standalone/meson.build

@@ -16,6 +16,7 @@ project('systemd-testsuite',
 
 fs = import('fs')
 mkosi = find_program('mkosi', required : true)
+want_integration_tests = true
 
 # meson refuses .. in subdir() so we use a symlink to trick it into accepting it anyway.
 subdir('integration-tests')