[metrics] Introduce metrics API (#39202)

See the [design
doc](https://github.com/systemd/systemd/issues/38023#issue-3192567450)
by @ikruglov

This PR introduces the metrics API framework, adds some basic system
wide/per unit/service metrics, and a basic CLI. The PR is broken into
two commits as described below.

### Deviations from the original design
- Introduced top level field "object" for ease of filtering. Instead of
having `fields: { unit: "foo", unit_type: "service" }`, we now have
`object: foo.service` as the top level field.


### First commit

The first commit includes:
- Metrics API definitions
- Code to set up the varlink server
- The describe method which shows all the metrics families
- The list method which lists all the metrics
- Type definitions related to MetricFamily
- Common code to build json objects

### Second commit
The second commit adds some basic metrics, a basic CLI (systemd-report)
which
lists the metrics, and integration tests.

**System wide metrics:**
- units_by_type_total
- units_by_state_total

**Two per unit metrics:**
- unit_active_state
- unit_load_state

**A service state metric:**
- nrestarts

### Sample outputs

**units_by_type_total**:

```
{
        "name" : "io.systemd.Manager.units_by_type_total",
        "value" : 52,
        "fields" : {
                "type" : "target"
        }
}
{
        "name" : "io.systemd.Manager.units_by_type_total",
        "value" : 82,
        "fields" : {
                "type" : "device"
        }
}
{
        "name" : "io.systemd.Manager.units_by_type_total",
        "value" : 2,
        "fields" : {
                "type" : "automount"
        }
}
```
**units_by_state_total**:
```
{
        "name" : "io.systemd.Manager.units_by_state_total",
        "value" : 216,
        "fields" : {
                "state" : "active"
        }
}
{
        "name" : "io.systemd.Manager.units_by_state_total",
        "value" : 0,
        "fields" : {
                "state" : "reloading"
        }
}
{
        "name" : "io.systemd.Manager.units_by_state_total",
        "value" : 120,
        "fields" : {
                "state" : "inactive"
        }
}
```
**unit_active_state**:
```
{
        "name" : "io.systemd.Manager.unit_active_state",
        "object" : "multi-user.target",
        "value" : "active"
}
{
        "name" : "io.systemd.Manager.unit_active_state",
        "object" : "systemd-sysusers.service",
        "value" : "inactive"
}
```
**unit_load_state**:
```
{
        "name" : "io.systemd.Manager.unit_load_state",
        "object" : "multi-user.target",
        "value" : "loaded"
}

```
**nrestarts**:
```
{
        "name" : "io.systemd.Manager.nrestarts",
        "object" : "user@0.service",
        "value" : 0
}
{
        "name" : "io.systemd.Manager.nrestarts",
        "object" : "user-runtime-dir@0.service",
        "value" : 0
}
```

meson.build

@@ -2398,6 +2398,7 @@ subdir('src/quotacheck')
 subdir('src/random-seed')
 subdir('src/remount-fs')
 subdir('src/repart')
+subdir('src/report')
 subdir('src/reply-password')
 subdir('src/resolve')
 subdir('src/rfkill')

src/boot/efi-string.c

@@ -497,7 +497,8 @@ char* line_get_key_value(char *s, const char *sep, size_t *pos, char **ret_key,
                         value++;
 
                 /* unquote */
-                if (value[0] == '"' && line[linelen - 1] == '"') {
+                if ((value[0] == '"' && line[linelen - 1] == '"') ||
+                    (value[0] == '\'' && line[linelen - 1] == '\'')) {
                         value++;
                         line[linelen - 1] = '\0';
                 }

src/core/manager.h

@@ -469,6 +469,8 @@ typedef struct Manager {
          * systemd-oomd to report changes in ManagedOOM settings (systemd client - oomd server). */
         sd_varlink *managed_oom_varlink;
 
+        sd_varlink_server *metrics_varlink_server;
+
         /* Reference to RestrictFileSystems= BPF program */
         struct restrict_fs_bpf *restrict_fs;
 

src/core/meson.build

@@ -69,6 +69,7 @@ libcore_sources = files(
         'varlink-dynamic-user.c',
         'varlink-execute.c',
         'varlink-manager.c',
+        'varlink-metrics.c',
         'varlink-unit.c',
 )
 

src/core/varlink-metrics.c

@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "hashmap.h"
+#include "manager.h"
+#include "metrics.h"
+#include "service.h"
+#include "unit-def.h"
+#include "unit.h"
+#include "varlink-metrics.h"
+
+static int unit_active_state_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                r = metric_build_send_string(
+                                context,
+                                unit->id,
+                                unit_active_state_to_string(unit_active_state(unit)),
+                                /* field_pairs= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int unit_load_state_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                r = metric_build_send_string(
+                                context,
+                                unit->id,
+                                unit_load_state_to_string(unit->load_state),
+                                /* field_pairs= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int nrestarts_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        int r;
+
+        assert(context);
+
+        LIST_FOREACH(units_by_type, unit, manager->units_by_type[UNIT_SERVICE]) {
+                r = metric_build_send_unsigned(
+                                context, unit->id, SERVICE(unit)->n_restarts, /* field_pairs= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int units_by_type_total_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        int r;
+
+        assert(context);
+
+        for (UnitType type = 0; type < _UNIT_TYPE_MAX; type++) {
+                uint64_t counter = 0;
+
+                LIST_FOREACH(units_by_type, _u, manager->units_by_type[type])
+                        counter++;
+
+                r = metric_build_send_unsigned(
+                                context,
+                                /* object= */ NULL,
+                                counter,
+                                STRV_MAKE("type", unit_type_to_string(type)));
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int units_by_state_total_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        UnitActiveState counters[_UNIT_ACTIVE_STATE_MAX] = {};
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        /* TODO need a rework probably with state counter */
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                counters[unit_active_state(unit)]++;
+        }
+
+        for (UnitActiveState state = 0; state < _UNIT_ACTIVE_STATE_MAX; state++) {
+                r = metric_build_send_unsigned(
+                                context,
+                                /* object= */ NULL,
+                                counters[state],
+                                STRV_MAKE("state", unit_active_state_to_string(state)));
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+const MetricFamily metric_family_table[] = {
+        // Keep metrics ordered alphabetically
+        {
+         .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "nrestarts",
+         .description = "Per unit metric: number of restarts",
+         .type = METRIC_FAMILY_TYPE_COUNTER,
+         .generate_cb = nrestarts_build_json,
+        },
+        {
+         .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "unit_active_state",
+         .description = "Per unit metric: active state",
+         .type = METRIC_FAMILY_TYPE_STRING,
+         .generate_cb = unit_active_state_build_json,
+        },
+        {
+         .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "unit_load_state",
+         .description = "Per unit metric: load state",
+         .type = METRIC_FAMILY_TYPE_STRING,
+         .generate_cb = unit_load_state_build_json,
+        },
+        {
+         .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "units_by_state_total",
+         .description = "Total number of units of different state",
+         .type = METRIC_FAMILY_TYPE_GAUGE,
+         .generate_cb = units_by_state_total_build_json,
+        },
+        {
+         .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "units_by_type_total",
+         .description = "Total number of units of different types",
+         .type = METRIC_FAMILY_TYPE_GAUGE,
+         .generate_cb = units_by_type_total_build_json,
+        },
+        {}
+};
+
+int vl_method_describe(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) {
+        return metrics_method_describe(metric_family_table, link, parameters, flags, userdata);
+}
+
+int vl_method_list(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) {
+        return metrics_method_list(metric_family_table, link, parameters, flags, userdata);
+}

src/core/varlink-metrics.h

@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "sd-varlink.h"
+#include "sd-json.h"
+
+#define METRIC_IO_SYSTEMD_MANAGER_PREFIX "io.systemd.Manager."
+
+int vl_method_list(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata);
+int vl_method_describe(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata);

src/core/varlink.c

@@ -5,6 +5,7 @@
 #include "constants.h"
 #include "errno-util.h"
 #include "manager.h"
+#include "metrics.h"
 #include "path-util.h"
 #include "pidref.h"
 #include "string-util.h"
@@ -18,6 +19,7 @@
 #include "varlink-io.systemd.UserDatabase.h"
 #include "varlink-io.systemd.service.h"
 #include "varlink-manager.h"
+#include "varlink-metrics.h"
 #include "varlink-serialize.h"
 #include "varlink-unit.h"
 #include "varlink-util.h"
@@ -423,8 +425,26 @@ int manager_setup_varlink_server(Manager *m) {
         return 1;
 }
 
+static int manager_setup_varlink_metrics_server(Manager *m) {
+        sd_varlink_server_flags_t flags = SD_VARLINK_SERVER_INHERIT_USERDATA;
+        int r;
+
+        assert(m);
+
+        if (MANAGER_IS_SYSTEM(m))
+                flags |= SD_VARLINK_SERVER_ACCOUNT_UID;
+
+        r = metrics_setup_varlink_server(
+                        &m->metrics_varlink_server, flags, m->event, vl_method_list, vl_method_describe, m);
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
 static int manager_varlink_init_system(Manager *m) {
         int r;
+        _cleanup_free_ char *metrics_address = NULL;
 
         assert(m);
 
@@ -433,16 +453,29 @@ static int manager_varlink_init_system(Manager *m) {
                 return log_error_errno(r, "Failed to set up varlink server: %m");
         bool fresh = r > 0;
 
+        r = manager_setup_varlink_metrics_server(m);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set up metrics varlink server: %m");
+        bool metrics_fresh = r > 0;
+
+        r = runtime_directory_generic(m->runtime_scope, "systemd/report/io.systemd.Manager", &metrics_address);
+        if (r < 0)
+                return r;
+
         if (!MANAGER_IS_TEST_RUN(m)) {
                 FOREACH_STRING(address,
                                "/run/systemd/userdb/io.systemd.DynamicUser",
                                VARLINK_PATH_MANAGED_OOM_SYSTEM,
-                               "/run/systemd/io.systemd.Manager") {
+                               "/run/systemd/io.systemd.Manager",
+                               metrics_address) {
+
+                        sd_varlink_server *server = streq(address, metrics_address) ? m->metrics_varlink_server : m->varlink_server;
+                        fresh = streq(address, metrics_address) ? metrics_fresh : fresh;
                         /* We might have got sockets through deserialization. Do not bind to them twice. */
-                        if (!fresh && varlink_server_contains_socket(m->varlink_server, address))
+                        if (!fresh && varlink_server_contains_socket(server, address))
                                 continue;
 
-                        r = sd_varlink_server_listen_address(m->varlink_server, address, 0666 | SD_VARLINK_SERVER_MODE_MKDIR_0755);
+                        r = sd_varlink_server_listen_address(server, address, 0666 | SD_VARLINK_SERVER_MODE_MKDIR_0755);
                         if (r < 0)
                                 return log_error_errno(r, "Failed to bind to varlink socket '%s': %m", address);
                 }
@@ -479,6 +512,10 @@ static int manager_varlink_init_user(Manager *m) {
                         return log_error_errno(r, "Failed to bind to varlink socket '%s': %m", address);
         }
 
+        r = manager_setup_varlink_metrics_server(m);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set up metrics varlink server: %m");
+
         return manager_varlink_managed_oom_connect(m);
 }
 
@@ -497,6 +534,7 @@ void manager_varlink_done(Manager *m) {
 
         m->varlink_server = sd_varlink_server_unref(m->varlink_server);
         m->managed_oom_varlink = sd_varlink_close_unref(m->managed_oom_varlink);
+        m->metrics_varlink_server = sd_varlink_server_unref(m->metrics_varlink_server);
 }
 
 void manager_varlink_send_pending_reload_message(Manager *m) {

src/report/meson.build

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+executables += [
+        executable_template + {
+                'name' : 'systemd-report',
+                'public' : true,
+                'sources' : files('report.c'),
+        },
+]

src/report/report.c

@@ -0,0 +1,328 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <getopt.h>
+
+#include "sd-event.h"
+#include "sd-varlink.h"
+
+#include "alloc-util.h"
+#include "ansi-color.h"
+#include "build.h"
+#include "dirent-util.h"
+#include "fd-util.h"
+#include "log.h"
+#include "main-func.h"
+#include "path-lookup.h"
+#include "pretty-print.h"
+#include "runtime-scope.h"
+#include "sort-util.h"
+#include "string-util.h"
+#include "time-util.h"
+
+#define MAX_CONCURRENT_METRICS_SOCKETS 20
+#define TIMEOUT_USEC (30 * USEC_PER_SEC) /* 30 seconds */
+
+static RuntimeScope arg_runtime_scope = RUNTIME_SCOPE_SYSTEM;
+
+typedef struct Context {
+        unsigned n_open_connections;
+        sd_json_variant **metrics;  /* Collected metrics for sorting */
+        size_t n_metrics;
+} Context;
+
+static int metric_compare(sd_json_variant *const *a, sd_json_variant *const *b) {
+        const char *name_a, *name_b, *object_a, *object_b;
+        sd_json_variant *fields_a, *fields_b;
+        _cleanup_free_ char *fields_str_a = NULL, *fields_str_b = NULL;
+        int r;
+
+        assert(a && *a);
+        assert(b && *b);
+
+        name_a = sd_json_variant_string(sd_json_variant_by_key(*a, "name"));
+        name_b = sd_json_variant_string(sd_json_variant_by_key(*b, "name"));
+        r = strcmp_ptr(name_a, name_b);
+        if (r != 0)
+                return r;
+
+        object_a = sd_json_variant_string(sd_json_variant_by_key(*a, "object"));
+        object_b = sd_json_variant_string(sd_json_variant_by_key(*b, "object"));
+        r = strcmp_ptr(object_a, object_b);
+        if (r != 0)
+                return r;
+
+        fields_a = sd_json_variant_by_key(*a, "fields");
+        fields_b = sd_json_variant_by_key(*b, "fields");
+        if (fields_a)
+                (void) sd_json_variant_format(fields_a, 0, &fields_str_a);
+        if (fields_b)
+                (void) sd_json_variant_format(fields_b, 0, &fields_str_b);
+
+        return strcmp_ptr(fields_str_a, fields_str_b);
+}
+
+static int metrics_on_query_reply(
+                sd_varlink *link,
+                sd_json_variant *parameters,
+                const char *error_id,
+                sd_varlink_reply_flags_t flags,
+                void *userdata) {
+
+        assert(link);
+
+        Context *context = ASSERT_PTR(userdata);
+
+        if (error_id) {
+                if (streq(error_id, SD_VARLINK_ERROR_DISCONNECTED))
+                        log_info("Varlink disconnected");
+                else if (streq(error_id, SD_VARLINK_ERROR_TIMEOUT))
+                        log_info("Varlink timed out");
+                else
+                        log_error("Varlink error: %s", error_id);
+
+                goto finish;
+        }
+
+        /* Collect metrics for later sorting */
+        if (!GREEDY_REALLOC(context->metrics, context->n_metrics + 1))
+                return log_oom();
+        context->metrics[context->n_metrics++] = sd_json_variant_ref(parameters);
+
+finish:
+        if (!FLAGS_SET(flags, SD_VARLINK_REPLY_CONTINUES)) {
+                assert(context->n_open_connections > 0);
+                context->n_open_connections--;
+
+                if (context->n_open_connections == 0)
+                        (void) sd_event_exit(ASSERT_PTR(sd_varlink_get_event(link)), EXIT_SUCCESS);
+        }
+
+        return 0;
+}
+
+static int metrics_call(const char *path, sd_event *event, sd_varlink **ret, Context *context) {
+        _cleanup_(sd_varlink_unrefp) sd_varlink *vl = NULL;
+        int r;
+
+        assert(path);
+        assert(event);
+        assert(ret);
+        assert(context);
+
+        r = sd_varlink_connect_address(&vl, path);
+        if (r < 0)
+                return log_debug_errno(r, "Unable to connect to %s: %m", path);
+
+        (void) sd_varlink_set_userdata(vl, context);
+
+        r = sd_varlink_set_relative_timeout(vl, TIMEOUT_USEC);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set varlink timeout: %m");
+
+        r = sd_varlink_attach_event(vl, event, SD_EVENT_PRIORITY_NORMAL);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to attach varlink connection to event loop: %m");
+
+        r = sd_varlink_bind_reply(vl, metrics_on_query_reply);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to bind reply callback: %m");
+
+        r = sd_varlink_observe(vl, "io.systemd.Metrics.List", /* parameters= */ NULL);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to issue io.systemd.Metrics.List call: %m");
+
+        *ret = TAKE_PTR(vl);
+
+        return 0;
+}
+
+static void sd_varlink_unref_many(sd_varlink **array, size_t n) {
+        assert(array);
+
+        FOREACH_ARRAY(v, array, n)
+                sd_varlink_unref(*v);
+
+        free(array);
+}
+
+static void context_done(Context *context) {
+        assert(context);
+
+        for (size_t i = 0; i < context->n_metrics; i++)
+                sd_json_variant_unref(context->metrics[i]);
+        free(context->metrics);
+}
+
+static void metrics_output_sorted(Context *context) {
+        assert(context);
+
+        if (context->n_metrics == 0)
+                return;
+
+        typesafe_qsort(context->metrics, context->n_metrics, metric_compare);
+
+        FOREACH_ARRAY(m, context->metrics, context->n_metrics)
+                sd_json_variant_dump(
+                                *m,
+                                SD_JSON_FORMAT_PRETTY_AUTO | SD_JSON_FORMAT_COLOR_AUTO | SD_JSON_FORMAT_FLUSH,
+                                stdout,
+                                NULL);
+}
+
+static int metrics_query(void) {
+        _cleanup_closedir_ DIR *d = NULL;
+        _cleanup_(sd_event_unrefp) sd_event *event = NULL;
+        _cleanup_free_ char *metrics_path = NULL;
+        int r;
+
+        r = runtime_directory_generic(arg_runtime_scope, "systemd/report", &metrics_path);
+        if (r < 0)
+                return log_error_errno(r, "Failed to determine metrics directory path: %m");
+
+        d = opendir(metrics_path);
+        if (!d) {
+                if (errno == ENOENT)
+                        return 0;
+
+                return log_error_errno(errno, "Failed to open directory %s: %m", metrics_path);
+        }
+
+        r = sd_event_default(&event);
+        if (r < 0)
+                return log_error_errno(r, "Failed to get event loop: %m");
+
+        r = sd_event_set_signal_exit(event, true);
+        if (r < 0)
+                return log_error_errno(r, "Failed to enable exit on SIGINT/SIGTERM: %m");
+
+        size_t n_varlinks = MAX_CONCURRENT_METRICS_SOCKETS;
+        sd_varlink **varlinks = new0(sd_varlink *, n_varlinks);
+        if (!varlinks)
+                return log_error_errno(ENOMEM, "Failed to allocate varlinks array: %m");
+
+        CLEANUP_ARRAY(varlinks, n_varlinks, sd_varlink_unref_many);
+
+        Context context = {};
+
+        FOREACH_DIRENT(de, d, return -errno) {
+                _cleanup_free_ char *p = NULL;
+
+                if (!IN_SET(de->d_type, DT_SOCK, DT_UNKNOWN))
+                        continue;
+
+                p = path_join(metrics_path, de->d_name);
+                if (!p)
+                        return log_oom();
+
+                r = metrics_call(p, event, &varlinks[context.n_open_connections], &context);
+                if (r < 0) {
+                        log_error_errno(r, "Failed to connect to %s: %m", p);
+                        continue;
+                }
+
+                if (++context.n_open_connections >= MAX_CONCURRENT_METRICS_SOCKETS) {
+                        log_warning("Too many concurrent metrics sockets, stop iterating");
+                        break;
+                }
+        }
+
+        r = sd_event_loop(event);
+        if (r < 0) {
+                context_done(&context);
+                return log_error_errno(r, "Failed to run event loop: %m");
+        }
+
+        metrics_output_sorted(&context);
+
+        context_done(&context);
+
+        return 0;
+}
+
+static int help(void) {
+        _cleanup_free_ char *link = NULL;
+        int r;
+
+        r = terminal_urlify_man("systemd-report", "1", &link);
+        if (r < 0)
+                return log_oom();
+
+        printf("%s [OPTIONS...] \n\n"
+               "%sPrint metrics for all system components.%s\n\n"
+               "  -h --help             Show this help\n"
+               "     --version          Show package version\n"
+               "     --user             Connect to user service manager\n"
+               "     --system           Connect to system service manager (default)\n"
+               "\nSee the %s for details.\n",
+               program_invocation_short_name,
+               ansi_highlight(),
+               ansi_normal(),
+               link);
+
+        return 0;
+}
+
+static int parse_argv(int argc, char *argv[]) {
+        enum {
+                ARG_VERSION = 0x100,
+                ARG_USER,
+                ARG_SYSTEM,
+        };
+
+        static const struct option options[] = {
+                { "help",    no_argument, NULL, 'h'         },
+                { "version", no_argument, NULL, ARG_VERSION },
+                { "user",    no_argument, NULL, ARG_USER    },
+                { "system",  no_argument, NULL, ARG_SYSTEM  },
+                {}
+        };
+
+        int c;
+
+        assert(argc >= 0);
+        assert(argv);
+
+        while ((c = getopt_long(argc, argv, "hp", options, NULL)) >= 0)
+                switch (c) {
+                case 'h':
+                        return help();
+                case ARG_VERSION:
+                        return version();
+                case ARG_USER:
+                        arg_runtime_scope = RUNTIME_SCOPE_USER;
+                        break;
+                case ARG_SYSTEM:
+                        arg_runtime_scope = RUNTIME_SCOPE_SYSTEM;
+                        break;
+                case '?':
+                        return -EINVAL;
+                default:
+                        assert_not_reached();
+                }
+
+        if (optind < argc)
+                return log_error_errno(
+                                SYNTHETIC_ERRNO(EINVAL),
+                                "%s takes no arguments.",
+                                program_invocation_short_name);
+
+        return 1;
+}
+
+static int run(int argc, char *argv[]) {
+        int r;
+
+        log_setup();
+
+        r = parse_argv(argc, argv);
+        if (r <= 0)
+                return r;
+
+        r = metrics_query();
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
+DEFINE_MAIN_FUNCTION_WITH_POSITIVE_FAILURE(run);

src/shared/meson.build

@@ -127,6 +127,7 @@ shared_sources = files(
         'machine-id-setup.c',
         'macvlan-util.c',
         'main-func.c',
+        'metrics.c',
         'mkdir-label.c',
         'mkfs-util.c',
         'module-util.c',
@@ -208,6 +209,7 @@ shared_sources = files(
         'varlink-io.systemd.MachineImage.c',
         'varlink-io.systemd.ManagedOOM.c',
         'varlink-io.systemd.Manager.c',
+        'varlink-io.systemd.Metrics.c',
         'varlink-io.systemd.MountFileSystem.c',
         'varlink-io.systemd.MuteConsole.c',
         'varlink-io.systemd.NamespaceResource.c',

src/shared/metrics.c

@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "json-util.h"
+#include "log.h"
+#include "metrics.h"
+#include "string-table.h"
+#include "strv.h"
+#include "varlink-io.systemd.Metrics.h"
+#include "varlink-util.h"
+
+static void metric_family_context_done(MetricFamilyContext *ctx) {
+        assert(ctx);
+
+        sd_json_variant_unref(ctx->previous);
+}
+
+int metrics_setup_varlink_server(
+                sd_varlink_server **server, /* in and out param */
+                sd_varlink_server_flags_t flags,
+                sd_event *event,
+                sd_varlink_method_t vl_method_list_cb,
+                sd_varlink_method_t vl_method_describe_cb,
+                void *userdata) {
+
+        _cleanup_(sd_varlink_server_unrefp) sd_varlink_server *s = NULL;
+        int r;
+
+        assert(server);
+        assert(event);
+
+        if (*server)
+                return 0;
+
+        r = varlink_server_new(&s, flags, userdata);
+        if (r < 0)
+                return log_error_errno(r, "Failed to allocate Varlink server: %m");
+
+        r = sd_varlink_server_add_interface(s, &vl_interface_io_systemd_Metrics);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to add varlink metrics interface to varlink server: %m");
+
+        r = sd_varlink_server_bind_method_many(
+                        s,
+                        "io.systemd.Metrics.List",
+                        vl_method_list_cb,
+                        "io.systemd.Metrics.Describe",
+                        vl_method_describe_cb);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to register varlink metrics methods: %m");
+
+        r = sd_varlink_server_set_description(s, "systemd varlink metrics server");
+        if (r < 0)
+                return log_debug_errno(r, "Failed to set varlink metrics server description: %m");
+
+        r = sd_varlink_server_attach_event(s, event, SD_EVENT_PRIORITY_NORMAL);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to attach varlink metrics connection to event loop: %m");
+
+        *server = TAKE_PTR(s);
+
+        return 0;
+}
+
+static const char * const metric_family_type_table[_METRIC_FAMILY_TYPE_MAX] = {
+        [METRIC_FAMILY_TYPE_COUNTER] = "counter",
+        [METRIC_FAMILY_TYPE_GAUGE]   = "gauge",
+        [METRIC_FAMILY_TYPE_STRING]  = "string",
+};
+
+DEFINE_STRING_TABLE_LOOKUP_TO_STRING(metric_family_type, MetricFamilyType);
+
+static int metric_family_build_send(sd_varlink *link, const MetricFamily *mf, bool more) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL;
+        int r;
+
+        assert(link);
+        assert(mf);
+
+        r = sd_json_buildo(
+                        &v,
+                        SD_JSON_BUILD_PAIR_STRING("name", mf->name),
+                        SD_JSON_BUILD_PAIR_STRING("description", mf->description),
+                        SD_JSON_BUILD_PAIR_STRING("type", metric_family_type_to_string(mf->type)));
+        if (r < 0)
+                return r;
+
+        if (more)
+                return sd_varlink_notify(link, v);
+
+        return sd_varlink_reply(link, v);
+}
+
+int metrics_method_describe(
+                const MetricFamily metric_family_table[],
+                sd_varlink *link,
+                sd_json_variant *parameters,
+                sd_varlink_method_flags_t flags,
+                void *userdata) {
+
+        int r;
+
+        assert(metric_family_table);
+        assert(link);
+        assert(parameters);
+
+        r = sd_varlink_dispatch(link, parameters, /* dispatch_table= */ NULL, /* userdata= */ NULL);
+        if (r != 0)
+                return r;
+
+        if (!FLAGS_SET(flags, SD_VARLINK_METHOD_MORE))
+                return sd_varlink_error(link, SD_VARLINK_ERROR_EXPECTED_MORE, NULL);
+
+        const MetricFamily *previous = NULL;
+        for (const MetricFamily *mf = metric_family_table; mf && mf->name; mf++) {
+                if (previous) {
+                        r = metric_family_build_send(link, previous, /* more= */ true);
+                        if (r < 0)
+                                return log_debug_errno(
+                                                r, "Failed to describe metric family '%s': %m", previous->name);
+                }
+
+                previous = mf;
+        }
+
+        if (!previous)
+                return sd_varlink_error(link, "io.systemd.Metrics.NoSuchMetric", NULL);
+
+        r = metric_family_build_send(link, previous, /* more= */ false);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to describe metric family '%s': %m", previous->name);
+
+        return 0;
+}
+
+int metrics_method_list(
+                const MetricFamily metric_family_table[],
+                sd_varlink *link,
+                sd_json_variant *parameters,
+                sd_varlink_method_flags_t flags,
+                void *userdata) {
+
+        int r;
+
+        assert(metric_family_table);
+        assert(link);
+        assert(parameters);
+
+        r = sd_varlink_dispatch(link, parameters, /* dispatch_table= */ NULL, /* userdata= */ NULL);
+        if (r != 0)
+                return r;
+
+        if (!FLAGS_SET(flags, SD_VARLINK_METHOD_MORE))
+                return sd_varlink_error(link, SD_VARLINK_ERROR_EXPECTED_MORE, NULL);
+
+        _cleanup_(metric_family_context_done) MetricFamilyContext ctx = { .link = link };
+        for (const MetricFamily *mf = metric_family_table; mf && mf->name; mf++) {
+                assert(mf->generate_cb);
+
+                ctx.metric_family = mf;
+                r = mf->generate_cb(&ctx, userdata);
+                if (r < 0)
+                        return log_debug_errno(
+                                        r, "Failed to list metrics for metric family '%s': %m", mf->name);
+        }
+
+        if (!ctx.previous)
+                return sd_varlink_error(link, "io.systemd.Metrics.NoSuchMetric", NULL);
+
+        /* produce the last metric */
+        return sd_varlink_reply(link, ctx.previous);
+}
+
+static int metric_set_fields(sd_json_variant **v, char **field_pairs) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *w = NULL;
+        size_t n;
+        int r;
+
+        assert(v);
+
+        n = strv_length(field_pairs);
+        if (n == 0)
+                return 0;
+
+        if (n % 2 != 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(ERANGE), "Odd number of field pairs: %zu", n);
+
+        sd_json_variant **array = new0(sd_json_variant *, n);
+        if (!array)
+                return log_oom();
+
+        CLEANUP_ARRAY(array, n, sd_json_variant_unref_many);
+
+        int i = 0;
+        STRV_FOREACH_PAIR(key, value, field_pairs) {
+                r = sd_json_variant_new_string(&array[i++], *key);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to create key variant: %m");
+
+                r = sd_json_variant_new_string(&array[i++], *value);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to create value variant: %m");
+        }
+
+        r = sd_json_variant_new_object(&w, array, n);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to allocate JSON object: %m");
+
+        return sd_json_variant_set_field(v, "fields", w);
+}
+
+static int metric_build_send(MetricFamilyContext *context, const char *object, sd_json_variant *value, char **field_pairs) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL;
+        int r;
+
+        assert(context);
+        assert(value);
+        assert(context->link);
+        assert(context->metric_family);
+
+        r = sd_json_buildo(
+                        &v,
+                        SD_JSON_BUILD_PAIR_STRING("name", context->metric_family->name),
+                        JSON_BUILD_PAIR_STRING_NON_EMPTY("object", object),
+                        SD_JSON_BUILD_PAIR("value", SD_JSON_BUILD_VARIANT(value)));
+        /* TODO JSON_BUILD_PAIR_OBJECT_STRV_NOT_NULL */
+        if (r < 0)
+                return r;
+
+        if (field_pairs) { /* NULL => no fields object, empty strv => fields:{} */
+                r = metric_set_fields(&v, field_pairs);
+                if (r < 0)
+                        return r;
+        }
+
+        if (context->previous) {
+                r = sd_varlink_notify(context->link, context->previous);
+                if (r < 0)
+                        return r;
+
+                context->previous = sd_json_variant_unref(context->previous);
+        }
+
+        context->previous = TAKE_PTR(v);
+
+        return 0;
+}
+
+int metric_build_send_string(MetricFamilyContext *context, const char *object, const char *value, char **field_pairs) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL;
+        int r;
+
+        assert(value);
+
+        r = sd_json_variant_new_string(&v, value);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to allocate JSON string: %m");
+
+        return metric_build_send(context, object, v, field_pairs);
+}
+
+int metric_build_send_unsigned(MetricFamilyContext *context, const char *object, uint64_t value, char **field_pairs) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL;
+        int r;
+
+        r = sd_json_variant_new_unsigned(&v, value);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to allocate JSON unsigned: %m");
+
+        return metric_build_send(context, object, v, field_pairs);
+}

src/shared/metrics.h

@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "sd-varlink.h"
+
+#include "macro-fundamental.h"
+
+typedef enum MetricFamilyType {
+        METRIC_FAMILY_TYPE_COUNTER,
+        METRIC_FAMILY_TYPE_GAUGE,
+        METRIC_FAMILY_TYPE_STRING,
+        _METRIC_FAMILY_TYPE_MAX,
+        _METRIC_FAMILY_TYPE_INVALID = -EINVAL,
+} MetricFamilyType;
+
+typedef struct MetricFamily MetricFamily;
+
+typedef struct MetricFamilyContext {
+        const MetricFamily* metric_family;
+        sd_varlink *link;
+        sd_json_variant *previous;
+} MetricFamilyContext;
+
+typedef int (*metric_family_generate_cb_t) (MetricFamilyContext *mfc, void *userdata);
+
+typedef struct MetricFamily {
+        const char *name;
+        const char *description;
+        MetricFamilyType type;
+        metric_family_generate_cb_t generate_cb;
+} MetricFamily;
+
+int metrics_setup_varlink_server(
+                sd_varlink_server **server, /* in and out param */
+                sd_varlink_server_flags_t flags,
+                sd_event *event,
+                sd_varlink_method_t vl_method_list_cb,
+                sd_varlink_method_t vl_method_describe_cb,
+                void *userdata);
+
+const char* metric_family_type_to_string(MetricFamilyType i) _const_;
+int metrics_method_describe(const MetricFamily metric_family_table[], sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata);
+int metrics_method_list(const MetricFamily metric_family_table[], sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata);
+
+int metric_build_send_string(MetricFamilyContext* context, const char *object, const char *value, char **field_pairs);
+int metric_build_send_unsigned(MetricFamilyContext* context, const char *object, uint64_t value, char **field_pairs);

src/shared/varlink-io.systemd.Metrics.c

@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "sd-varlink-idl.h"
+
+#include "varlink-io.systemd.Metrics.h"
+
+static SD_VARLINK_DEFINE_ENUM_TYPE(
+                MetricFamilyType,
+                SD_VARLINK_FIELD_COMMENT("A counter metric family type which is a monotonically increasing value"),
+                SD_VARLINK_DEFINE_ENUM_VALUE(counter),
+                SD_VARLINK_FIELD_COMMENT("A gauge metric family type which is a value that can go up and down"),
+                SD_VARLINK_DEFINE_ENUM_VALUE(gauge),
+                SD_VARLINK_FIELD_COMMENT("A string metric family type"),
+                SD_VARLINK_DEFINE_ENUM_VALUE(string));
+
+static SD_VARLINK_DEFINE_ERROR(NoSuchMetric);
+
+static SD_VARLINK_DEFINE_METHOD_FULL(
+                List,
+                SD_VARLINK_REQUIRES_MORE,
+                SD_VARLINK_FIELD_COMMENT("Metric name, e.g. io.systemd.Manager.units_by_type_total or io.systemd.Manager.unit_active_state"),
+                SD_VARLINK_DEFINE_OUTPUT(name, SD_VARLINK_STRING, 0),
+                /* metric value has various types depending on MetricFamilyType and actual data double/int/uint */
+                SD_VARLINK_FIELD_COMMENT("Metric value"),
+                SD_VARLINK_DEFINE_OUTPUT(value, SD_VARLINK_ANY, 0),
+                SD_VARLINK_FIELD_COMMENT("Metric object name can be unit name, process name, etc, e.g. dev-hvc0.device"),
+                SD_VARLINK_DEFINE_OUTPUT(object, SD_VARLINK_STRING, SD_VARLINK_NULLABLE),
+                SD_VARLINK_FIELD_COMMENT("Metric fields are values to deferentiate between different metrics in the same metric family"),
+                SD_VARLINK_DEFINE_OUTPUT(fields, SD_VARLINK_OBJECT, SD_VARLINK_NULLABLE));
+
+static SD_VARLINK_DEFINE_METHOD_FULL(
+                Describe,
+                SD_VARLINK_REQUIRES_MORE,
+                SD_VARLINK_FIELD_COMMENT("Metric family name"),
+                SD_VARLINK_DEFINE_OUTPUT(name, SD_VARLINK_STRING, 0),
+                SD_VARLINK_FIELD_COMMENT("Metric family description"),
+                SD_VARLINK_DEFINE_OUTPUT(description, SD_VARLINK_STRING, 0),
+                SD_VARLINK_FIELD_COMMENT("Metric family type"),
+                SD_VARLINK_DEFINE_OUTPUT_BY_TYPE(type, MetricFamilyType, 0));
+
+SD_VARLINK_DEFINE_INTERFACE(
+                io_systemd_Metrics,
+                "io.systemd.Metrics",
+                SD_VARLINK_INTERFACE_COMMENT("Metrics APIs"),
+                SD_VARLINK_SYMBOL_COMMENT("An enum representing various metric family types"),
+                &vl_type_MetricFamilyType,
+                SD_VARLINK_SYMBOL_COMMENT("Method to get a list of metrics among which the collection of related metrics forms a metric family"),
+                &vl_method_List,
+                SD_VARLINK_SYMBOL_COMMENT("Method to get the metric families"),
+                &vl_method_Describe,
+                SD_VARLINK_SYMBOL_COMMENT("No such metric found"),
+                &vl_error_NoSuchMetric);

src/shared/varlink-io.systemd.Metrics.h

@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "sd-varlink-idl.h"
+
+extern const sd_varlink_interface vl_interface_io_systemd_Metrics;

test/units/TEST-74-AUX-UTILS.varlinkctl.sh

@@ -210,6 +210,18 @@ varlinkctl call /run/systemd/io.systemd.Manager io.systemd.Unit.List '{"cgroup":
 invocation_id="$(systemctl show -P InvocationID systemd-journald.service)"
 varlinkctl call /run/systemd/io.systemd.Manager io.systemd.Unit.List "{\"invocationID\": \"$invocation_id\"}"
 
+# test io.systemd.Metrics
+varlinkctl info /run/systemd/report/io.systemd.Manager
+
+varlinkctl list-methods /run/systemd/report/io.systemd.Manager
+varlinkctl list-methods -j /run/systemd/report/io.systemd.Manager io.systemd.Metrics | jq .
+
+varlinkctl introspect /run/systemd/report/io.systemd.Manager
+varlinkctl introspect -j /run/systemd/report/io.systemd.Manager io.systemd.Metrics | jq .
+
+varlinkctl --more call /run/systemd/report/io.systemd.Manager io.systemd.Metrics.List {}
+varlinkctl --more call /run/systemd/report/io.systemd.Manager io.systemd.Metrics.Describe {}
+
 # test io.systemd.Manager in user manager
 testuser_uid=$(id -u testuser)
 systemd-run --wait --pipe --user --machine testuser@ \
@@ -222,3 +234,6 @@ systemd-run --wait --pipe --user --machine testuser@ \
 # test io.systemd.Unit in user manager
 systemd-run --wait --pipe --user --machine testuser@ \
         varlinkctl --more call "/run/user/$testuser_uid/systemd/io.systemd.Manager" io.systemd.Unit.List '{}'
+
+# test report
+systemd-report