Merge 3e2b6fd389 into 2b60615a41

Basically, distros that maintain more than one release stream (i.e.
multiple stable versions, a beta channel, etc) can put the others'
transfer definitions into /usr/lib/sysupdate@$STREAM.d/, and the admin
can pass --stream= on the CLI to switch to this new stream.

Part of https://github.com/systemd/systemd/issues/33345

SQUASHME: s/--next/--streams=<stream>/

man/systemd-sysupdate.xml

@@ -190,6 +190,17 @@
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>streams</option></term>
+
+        <listitem><para>Lists streams that can be updated. This enumerates the
+        <filename>/var/cache/systemd/sysupdate@*.d/</filename> and <filename>/usr/lib/sysupdate@*.d/</filename>
+        directories that contain transfer definitions. This command is useful to list possible parameters
+        for <option>--stream=</option> (see below).</para>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
+
       <xi:include href="standard-options.xml" xpointer="help" />
       <xi:include href="standard-options.xml" xpointer="version" />
     </variablelist>
@@ -225,7 +236,8 @@
         updated together in a synchronous fashion. Simply define multiple transfer files within the same
         <filename>sysupdate.d/</filename> directory for these cases.</para>
 
-        <para>This option may not be combined with <option>--definitions=</option>.</para>
+        <para>This option may not be combined with <option>--definitions=</option> or
+        <option>--stream=</option>.</para>
 
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
@@ -237,11 +249,29 @@
         are read from this directory instead of <filename>/usr/lib/sysupdate.d/*.conf</filename>,
         <filename>/etc/sysupdate.d/*.conf</filename>, and <filename>/run/sysupdate.d/*.conf</filename>.</para>
 
-        <para>This option may not be combined with <option>--component=</option>.</para>
+        <para>This option may not be combined with <option>--component=</option> or
+        <option>--stream=</option>.</para>
 
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>--stream=</option></term>
+
+        <listitem><para>Selects the update stream to use. Takes a stream name as argument. This alters
+        the search logic for transfer definitions to look in
+        <filename>/usr/lib/sysupdate@<replaceable>stream</replaceable>.d/</filename> and
+        <filename>/var/cache/systemd/sysupdate@<replaceable>stream</replaceable>.d/</filename> instead of
+        <filename>/usr/lib/sysupdate.d</filename>.
+        Note that administrator-controlled directories (i.e. <filename>/etc/sysupdate.d/</filename>, etc) are
+        still loaded as usual.</para>
+
+        <para>This option may not be combined with <option>--definitions=</option> or
+        <option>--component=</option>.</para>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>--root=</option></term>
 

man/sysupdate.d.xml

@@ -65,6 +65,52 @@
     <citerefentry><refentrytitle>sysupdate.features</refentrytitle><manvolnum>5</manvolnum></citerefentry>.
     </para>
 
+    <para>Sometimes, distributions need to update certain parts of themselves independently from the normal
+    update cycle.
+    For example, the <ulink url="https://github.com/rhboot/shim">UEFI Shim loader</ulink> (necessary for
+    UEFI Secure Boot support in many cases) has its own release cycle, requires code signatures from a
+    third-party, and in general is not tied to a distribution's update cycle.
+    Support for this scenario is provided by "components", which allow distributions to define transfer
+    definitions that receive updates independently from the base OS.
+    Components are defined in <filename>/usr/lib/sysupdate.<replaceable>component</replaceable>.d/</filename>,
+    and have corresponding override directories for administrators (i.e.
+    <filename>/etc/sysupdate.<replaceable>component</replaceable>.d/</filename>, etc).</para>
+
+    <para>Some distributions may wish to maintain multiple update "streams" at a time, for example to offer
+    a beta/nightly update channel, or to distribute security updates to multiple major versions at a time.
+    Users of such distributions may wish to remain on their current stream, and switch streams at some future
+    point in time.
+    A distribution with multiple update streams should ship the transfer definitions for each stream in the
+    <filename>/usr/lib/sysupdate@<replaceable>stream</replaceable>.d/</filename> or
+    <filename>/var/cache/systemd/sysupdate@<replaceable>stream</replaceable>.d/</filename> directories.
+    For example, a distribution with multiple stable branches can ship the next major release's transfer
+    definitions in the current release's
+    <filename>/usr/lib/sysupdate@foobarOS-<replaceable>next</replaceable>.d/</filename> directory, and users
+    can switch to it by updating the <literal>foobarOS-<replaceable>next</replaceable></literal> stream.
+    How exactly these stream definition directories are delivered is up to distributions: they can stabilize
+    transfer definitions a version in advance and ship the stream definitions from day one, or they can ship
+    these files as part of a regular security patch that users will install anyway, or they can use a
+    component as described above to update the stream definitions under <filename>/var/cache/</filename>
+    independently from the host system.
+    Note that the presence of a stream definition directory does not imply the availability of an upgrade on
+    that stream; it just defines where to look and if an update is found on the remote how to install it.
+    Also note that the normal administrator override files (i.e. transfer definitions, feature definitions,
+    or drop-ins found in <filname>/etc/sysupdate.d/</filname>, <filename>/run/sysupdate.d/</filename>, etc)
+    are applied over top of the definitions found in the stream definition directory.
+    This is done because a the stream definition directory turns into the normal definition directory
+    (<filename>/usr/lib/sysupdate.d/</filename>) when that stream is switched to.</para>
+
+    <para>System Administrators must take <emphasis>extreme</emphasis> care when overriding any transfer or
+    optional feature definitions, other than to turn on or off features!
+    As with any configuration defined in <filename>/usr</filename> and overridden in
+    <filename>/etc</filename>, an update to the host system can break the administrator overrides.
+    However, <command>systemd-sysupdate</command> is uniquely destructive: a broken configuration could
+    prevent the system from updating (best case), or completely destroy an installation by wiping the wrong
+    partition.
+    Distributions must take care to avoid breaking systems where overrides exist only to turn on or off
+    optional features; supporting (or choosing not to) everything else is up to distribution policy.
+    <emphasis>You have been warned.</emphasis></para>
+
     <para>Each <filename>*.transfer</filename> file contains three sections: [Transfer], [Source] and [Target].</para>
   </refsect1>
 

src/basic/constants.h

@@ -64,12 +64,18 @@
         "/usr/local/lib/" n "\0"                \
         "/usr/lib/" n "\0"
 
-#define CONF_PATHS(n)                           \
+#define CONF_PATHS_ADMIN(n)                     \
         "/etc/" n,                              \
-        "/run/" n,                              \
+        "/run/" n
+
+#define CONF_PATHS_SYSTEM(n)                    \
         "/usr/local/lib/" n,                    \
         "/usr/lib/" n
 
+#define CONF_PATHS(n)                           \
+        CONF_PATHS_ADMIN(n),                    \
+        CONF_PATHS_SYSTEM(n)
+
 #define CONF_PATHS_STRV(n)                      \
         STRV_MAKE(CONF_PATHS(n))
 

src/sysupdate/sysupdate.c

@@ -47,6 +47,7 @@ char *arg_root = NULL;
 static char *arg_image = NULL;
 static bool arg_reboot = false;
 static char *arg_component = NULL;
+static char *arg_stream = NULL;
 static int arg_verify = -1;
 static ImagePolicy *arg_image_policy = NULL;
 static bool arg_offline = false;
@@ -56,6 +57,7 @@ STATIC_DESTRUCTOR_REGISTER(arg_definitions, freep);
 STATIC_DESTRUCTOR_REGISTER(arg_root, freep);
 STATIC_DESTRUCTOR_REGISTER(arg_image, freep);
 STATIC_DESTRUCTOR_REGISTER(arg_component, freep);
+STATIC_DESTRUCTOR_REGISTER(arg_stream, freep);
 STATIC_DESTRUCTOR_REGISTER(arg_image_policy, image_policy_freep);
 STATIC_DESTRUCTOR_REGISTER(arg_transfer_source, freep);
 
@@ -180,7 +182,54 @@ static int context_read_definitions(Context *c, const char* node) {
 
         if (arg_definitions)
                 dirs = strv_new(arg_definitions);
-        else if (arg_component) {
+        else if (arg_stream) {
+                /* Ultimately we end up with a search path along the lines of: /etc/sysupdate.d/,
+                 * /run/sysupdate.d/, /var/cache/systemd/sysupdate@<stream>.d, /usr/lib/sysupdate@<stream>.d.
+                 * This is very unusual! It seems wrong! But this is the correct behavior. When a
+                 * `systemd-sysupdate --stream=` update is completed, /usr/lib/sysupdate@<stream>.d (or its
+                 * /var/cache alternative) turns into /usr/lib/sysupdate.d, but the admin overrides remain
+                 * untouched. So if we did this any differently, we'd end up in a situation where the admin's
+                 * settings are ignored when first installing a major upgrade but then suddenly considered
+                 * again once the update is completed. In my opinion, that behavior would be more unexpected
+                 * and dangerous than what is implemented here!
+                 *
+                 * Is this a big and surprising footgun for the admin? Yes. But frankly, so is overriding
+                 * anything relating to sysupdate. If an admin has overrides that do anything other than
+                 * turning on/off optional features, they've already aimed a ballistic missile at their
+                 * installation. It'll detonate either immediately when trying to switch streams (as
+                 * implemented now), or when updating to the first patch of the new stream (the alternative);
+                 * the installation is doomed either way. And failing immediately during a major OS upgrade
+                 * seems a lot more preferable, and something that admins will be more prepared for, than a
+                 * subsequent security patch suddenly bricking installations. */
+
+                char **admin = STRV_MAKE(CONF_PATHS_ADMIN("sysupdate.d"));
+                char **system = STRV_MAKE("/var/cache/systemd/", CONF_PATHS_SYSTEM(""));
+                size_t i = 0;
+
+                dirs = new0(char*, strv_length(admin) + strv_length(system) + 1);
+                if (!dirs)
+                        return log_oom();
+
+                STRV_FOREACH(dir, admin) {
+                        char *d;
+
+                        d = strdup(*dir);
+                        if (!d)
+                                return log_oom();
+
+                        dirs[i++] = d;
+                }
+
+                STRV_FOREACH(dir, system) {
+                        char *j;
+
+                        j = strjoin(*dir, "sysupdate@", arg_stream, ".d");
+                        if (!j)
+                                return log_oom();
+
+                        dirs[i++] = j;
+                }
+        } else if (arg_component) {
                 char **l = CONF_PATHS_STRV("");
                 size_t i = 0;
 
@@ -247,6 +296,11 @@ static int context_read_definitions(Context *c, const char* node) {
                                                "No transfer definitions for component '%s' found.",
                                                arg_component);
 
+                if (arg_stream)
+                        return log_error_errno(SYNTHETIC_ERRNO(ENOENT),
+                                               "No transfer definitions for stream '%s' found.",
+                                               arg_stream);
+
                 return log_error_errno(SYNTHETIC_ERRNO(ENOENT),
                                        "No transfer definitions found.");
         }
@@ -1532,22 +1586,45 @@ static int component_name_valid(const char *c) {
         return filename_is_valid(j);
 }
 
-static int verb_components(int argc, char **argv, void *userdata) {
+static int stream_name_valid(const char *s) {
+        _cleanup_free_ char *j = NULL;
+
+        /* See if the specified string enclosed in the directory prefix+suffix would be a valid file name */
+
+        if (isempty(s))
+                return false;
+
+        if (string_has_cc(s, NULL))
+                return false;
+
+        if (!utf8_is_valid(s))
+                return false;
+
+        j = strjoin("sysupdate@", s, ".d");
+        if (!j)
+                return -ENOMEM;
+
+        return filename_is_valid(j);
+}
+
+static int walk_search_paths(char **paths, bool component, char ***ret, bool *ret_has_default) {
         _cleanup_(loop_device_unrefp) LoopDevice *loop_device = NULL;
         _cleanup_(umount_and_rmdir_and_freep) char *mounted_dir = NULL;
         _cleanup_set_free_ Set *names = NULL;
-        _cleanup_free_ char **z = NULL; /* We use simple free() rather than strv_free() here, since set_free() will free the strings for us */
+        _cleanup_free_ char **names_strv = NULL; /* free() b/c the set still owns the values */
-        char **l = CONF_PATHS_STRV("");
+        _cleanup_strv_free_ char **names_dup = NULL;
-        bool has_default_component = false;
+        bool has_default = false;
         int r;
 
-        assert(argc <= 1);
+        assert(paths);
+        assert(ret);
+        assert(ret_has_default);
 
         r = process_image(/* ro= */ false, &mounted_dir, &loop_device);
         if (r < 0)
                 return r;
 
-        STRV_FOREACH(i, l) {
+        STRV_FOREACH(i, paths) {
                 _cleanup_closedir_ DIR *d = NULL;
                 _cleanup_free_ char *p = NULL;
 
@@ -1577,11 +1654,11 @@ static int verb_components(int argc, char **argv, void *userdata) {
                                 continue;
 
                         if (streq(de->d_name, "sysupdate.d")) {
-                                has_default_component = true;
+                                has_default = true;
                                 continue;
                         }
 
-                        e = startswith(de->d_name, "sysupdate.");
+                        e = startswith(de->d_name, component ? "sysupdate." : "sysupdate@");
                         if (!e)
                                 continue;
 
@@ -1593,26 +1670,51 @@ static int verb_components(int argc, char **argv, void *userdata) {
                         if (!n)
                                 return log_oom();
 
+                        if (component)
                                 r = component_name_valid(n);
+                        else
+                                r = stream_name_valid(n);
                         if (r < 0)
-                                return log_error_errno(r, "Unable to validate component name: %m");
+                                return log_error_errno(r, "Unable to validate %s name: %m",
+                                                       component ? "component" : "stream");
                         if (r == 0)
                                 continue;
 
                         r = set_ensure_consume(&names, &string_hash_ops_free, TAKE_PTR(n));
                         if (r < 0 && r != -EEXIST)
-                                return log_error_errno(r, "Failed to add component to set: %m");
+                                return log_error_errno(r, "Failed to add %s to set: %m",
+                                                       component ? "component" : "stream");
                 }
         }
 
-        z = set_get_strv(names);
+        names_strv = set_get_strv(names);
-        if (!z)
+        if (!names_strv)
                 return log_oom();
 
-        strv_sort(z);
+        names_dup = strv_copy(names_strv);
+        if (!names_dup)
+                return log_oom();
+
+        strv_sort(names_dup);
+
+        *ret = TAKE_PTR(names_dup);
+        *ret_has_default = has_default;
+        return 0;
+}
+
+static int verb_components(int argc, char **argv, void *userdata) {
+        _cleanup_strv_free_ char **names = NULL;
+        bool has_default_component = false;
+        int r;
+
+        assert(argc <= 1);
+
+        r = walk_search_paths(CONF_PATHS_STRV(""), true, &names, &has_default_component);
+        if (r < 0)
+                return r;
 
         if (!sd_json_format_enabled(arg_json_format_flags)) {
-                if (!has_default_component && set_isempty(names)) {
+                if (!has_default_component && strv_isempty(names)) {
                         log_info("No components defined.");
                         return 0;
                 }
@@ -1621,13 +1723,53 @@ static int verb_components(int argc, char **argv, void *userdata) {
                         printf("%s<default>%s\n",
                                ansi_highlight(), ansi_normal());
 
-                STRV_FOREACH(i, z)
+                STRV_FOREACH(i, names)
                         puts(*i);
         } else {
                 _cleanup_(sd_json_variant_unrefp) sd_json_variant *json = NULL;
 
                 r = sd_json_buildo(&json, SD_JSON_BUILD_PAIR_BOOLEAN("default", has_default_component),
-                                          SD_JSON_BUILD_PAIR_STRV("components", z));
+                                          SD_JSON_BUILD_PAIR_STRV("components", names));
+                if (r < 0)
+                        return log_error_errno(r, "Failed to create JSON: %m");
+
+                r = sd_json_variant_dump(json, arg_json_format_flags, stdout, NULL);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to print JSON: %m");
+        }
+
+        return 0;
+}
+
+static int verb_streams(int argc, char **argv, void *userdata) {
+        char **dirs = STRV_MAKE("/var/cache/systemd/", CONF_PATHS_SYSTEM(""));
+        _cleanup_strv_free_ char **names = NULL;
+        bool has_default_stream = false;
+        int r;
+
+        assert(argc <= 1);
+
+        r = walk_search_paths(dirs, false, &names, &has_default_stream);
+        if (r < 0)
+                return r;
+
+        if (FLAGS_SET(arg_json_format_flags, SD_JSON_FORMAT_OFF)) {
+                if (!has_default_stream && strv_isempty(names)) {
+                        log_info("No streams defined.");
+                        return 0;
+                }
+
+                if (has_default_stream)
+                        printf("%s<default>%s\n",
+                               ansi_highlight(), ansi_normal());
+
+                STRV_FOREACH(i, names)
+                        puts(*i);
+        } else {
+                _cleanup_(sd_json_variant_unrefp) sd_json_variant *json = NULL;
+
+                r = sd_json_buildo(&json, SD_JSON_BUILD_PAIR_BOOLEAN("default", has_default_stream),
+                                          SD_JSON_BUILD_PAIR_STRV("streams", names));
                 if (r < 0)
                         return log_error_errno(r, "Failed to create JSON: %m");
 
@@ -1659,11 +1801,13 @@ static int verb_help(int argc, char **argv, void *userdata) {
                "                          currently booted\n"
                "  reboot                  Reboot if a newer version is installed than booted\n"
                "  components              Show list of components\n"
+               "  streams                 Show list of streams\n"
                "  -h --help               Show this help\n"
                "     --version            Show package version\n"
                "\n%3$sOptions:%4$s\n"
                "  -C --component=NAME     Select component to update\n"
                "     --definitions=DIR    Find transfer definitions in specified directory\n"
+               "     --stream=STREAM      Select stream to switch to\n"
                "     --root=PATH          Operate on an alternate filesystem root\n"
                "     --image=PATH         Operate on disk image as filesystem root\n"
                "     --image-policy=POLICY\n"
@@ -1698,6 +1842,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_NO_LEGEND,
                 ARG_SYNC,
                 ARG_DEFINITIONS,
+                ARG_STREAM,
                 ARG_JSON,
                 ARG_ROOT,
                 ARG_IMAGE,
@@ -1714,6 +1859,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "no-pager",          no_argument,       NULL, ARG_NO_PAGER          },
                 { "no-legend",         no_argument,       NULL, ARG_NO_LEGEND         },
                 { "definitions",       required_argument, NULL, ARG_DEFINITIONS       },
+                { "stream",            required_argument, NULL, ARG_STREAM            },
                 { "instances-max",     required_argument, NULL, 'm'                   },
                 { "sync",              required_argument, NULL, ARG_SYNC              },
                 { "json",              required_argument, NULL, ARG_JSON              },
@@ -1770,6 +1916,24 @@ static int parse_argv(int argc, char *argv[]) {
                                 return r;
                         break;
 
+                case ARG_STREAM:
+                        if (isempty(optarg)) {
+                                arg_stream = mfree(arg_stream);
+                                break;
+                        }
+
+                        r = stream_name_valid(optarg);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to determine if stream name is valid: %m");
+                        if (r == 0)
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Stream name invalid: %s", optarg);
+
+                        r = free_and_strdup_warn(&arg_stream, optarg);
+                        if (r < 0)
+                                return r;
+
+                        break;
+
                 case ARG_JSON:
                         r = parse_json_argument(optarg, &arg_json_format_flags);
                         if (r <= 0)
@@ -1856,6 +2020,12 @@ static int parse_argv(int argc, char *argv[]) {
         if (arg_definitions && arg_component)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "The --definitions= and --component= switches may not be combined.");
 
+        if (arg_definitions && arg_stream)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "The --definitions= and --stream= switches may not be combined.");
+
+        if (arg_component && arg_stream)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "The --component= and --stream= switches may not be combined.");
+
         return 1;
 }
 
@@ -1864,6 +2034,7 @@ static int sysupdate_main(int argc, char *argv[]) {
         static const Verb verbs[] = {
                 { "list",       VERB_ANY, 2, VERB_DEFAULT, verb_list              },
                 { "components", VERB_ANY, 1, 0,            verb_components        },
+                { "streams",    VERB_ANY, 1, 0,            verb_streams           },
                 { "features",   VERB_ANY, 2, 0,            verb_features          },
                 { "check-new",  VERB_ANY, 1, 0,            verb_check_new         },
                 { "update",     VERB_ANY, 2, 0,            verb_update            },