env-util: add unsetenv_erase() helper

Let's unify how we remove secrets from the env block.

docs/ENVIRONMENT.md

@@ -316,3 +316,22 @@ fuzzers:
 
 Note that is may be also useful to set `$SYSTEMD_LOG_LEVEL`, since all logging
 is suppressed by default.
+
+systemd-importd:
+
+* `SYSTEMD_IMPORT_BTRFS_SUBVOL` – takes a boolean, which controls whether to
+  prefer creating btrfs subvolumes over plain directories for machine
+  images. Has no effect on non-btrfs file systems where subvolumes are not
+  available anyway. If not set, defaults to true.
+
+* `SYSTEMD_IMPORT_BTRFS_QUOTA` – takes a boolean, which controls whether to set
+  up quota automatically for created btrfs subvolumes for machine images. If
+  not set, defaults to true. Has no effect if machines are placed in regular
+  directories, because btrfs subvolumes are not supported or disabled. If
+  enabled, the quota group of the subvolume is automatically added to a
+  combined quota group for all such machine subvolumes.
+
+* `SYSTEMD_IMPORT_SYNC` – takes a boolean, which controls whether to
+  synchronize images to disk after installing them, before completing the
+  operation. If not set, defaults to true. If disabled installation of images
+  will be quicker, but not as safe.

po/pt_BR.po

@@ -10,7 +10,7 @@ msgstr ""
 "Project-Id-Version: systemd\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2021-01-08 17:48+0100\n"
-"PO-Revision-Date: 2021-02-25 18:41+0000\n"
+"PO-Revision-Date: 2021-08-17 07:04+0000\n"
 "Last-Translator: Gustavo Costa <xfgusta@gmail.com>\n"
 "Language-Team: Portuguese (Brazil) <https://translate.fedoraproject.org/"
 "projects/systemd/master/pt_BR/>\n"
@@ -19,7 +19,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n > 1;\n"
-"X-Generator: Weblate 4.4.2\n"
+"X-Generator: Weblate 4.7.2\n"
 
 #: src/core/org.freedesktop.systemd1.policy.in:22
 msgid "Send passphrase back to system"
@@ -316,7 +316,7 @@ msgstr ""
 #: src/login/org.freedesktop.login1.policy:117
 msgid "Allow applications to inhibit system handling of the reboot key"
 msgstr ""
-"Permitir que aplicativos inibam o sistema de gerenciar o botão de "
+"Permitir que os aplicativos inibam o gerenciamento do sistema da tecla de "
 "reinicialização"
 
 #: src/login/org.freedesktop.login1.policy:118
@@ -324,8 +324,8 @@ msgid ""
 "Authentication is required for an application to inhibit system handling of "
 "the reboot key."
 msgstr ""
-"É necessária autenticação para que um aplicativo iniba o sistema de "
+"A autenticação é necessária para um aplicativo para inibir o gerenciamento "
-"gerenciar o botão de reinicialização."
+"do sistema da tecla de reinicialização."
 
 #: src/login/org.freedesktop.login1.policy:128
 msgid "Allow non-logged-in user to run programs"

src/basic/env-util.c

@@ -870,3 +870,20 @@ int getenv_path_list(const char *name, char ***ret_paths) {
         *ret_paths = TAKE_PTR(l);
         return 1;
 }
+
+int unsetenv_erase(const char *name) {
+        char *p;
+
+        assert(name);
+
+        p = getenv(name);
+        if (!p)
+                return 0;
+
+        string_erase(p);
+
+        if (unsetenv(name) < 0)
+                return -errno;
+
+        return 1;
+}

src/basic/env-util.h

@@ -68,3 +68,5 @@ int setenv_systemd_exec_pid(bool update_only);
 /* Parses and does sanity checks on an environment variable containing
  * PATH-like colon-separated absolute paths */
 int getenv_path_list(const char *name, char ***ret_paths);
+
+int unsetenv_erase(const char *name);

src/cryptenroll/cryptenroll-password.c

@@ -2,6 +2,7 @@
 
 #include "ask-password-api.h"
 #include "cryptenroll-password.h"
+#include "env-util.h"
 #include "escape.h"
 #include "memory-util.h"
 #include "pwquality-util.h"
@@ -27,8 +28,7 @@ int enroll_password(
                 if (!new_password)
                         return log_oom();
 
-                string_erase(e);
+                assert_se(unsetenv_erase("NEWPASSWORD") >= 0);
-                assert_se(unsetenv("NEWPASSWORD") == 0);
 
         } else {
                 _cleanup_free_ char *disk_path = NULL;

src/cryptenroll/cryptenroll.c

@@ -12,6 +12,7 @@
 #include "cryptenroll-wipe.h"
 #include "cryptenroll.h"
 #include "cryptsetup-util.h"
+#include "env-util.h"
 #include "escape.h"
 #include "libfido2-util.h"
 #include "main-func.h"
@@ -426,8 +427,7 @@ static int prepare_luks(
                 if (!password)
                         return log_oom();
 
-                string_erase(e);
+                assert_se(unsetenv_erase("PASSWORD") >= 0);
-                assert_se(unsetenv("PASSWORD") >= 0);
 
                 r = crypt_volume_key_get(
                                 cd,

src/cryptsetup/cryptsetup-fido2.c

@@ -2,6 +2,7 @@
 
 #include "ask-password-api.h"
 #include "cryptsetup-fido2.h"
+#include "env-util.h"
 #include "fileio.h"
 #include "hexdecoct.h"
 #include "json.h"
@@ -70,9 +71,7 @@ int acquire_fido2_key(
                 if (!pins)
                         return log_oom();
 
-                string_erase(e);
+                assert_se(unsetenv_erase("PIN") >= 0);
-                if (unsetenv("PIN") < 0)
-                        return log_error_errno(errno, "Failed to unset $PIN: %m");
         }
 
         for (;;) {

src/home/homectl.c

@@ -215,9 +215,7 @@ static int acquire_existing_password(
                 if (r < 0)
                         return log_error_errno(r, "Failed to store password: %m");
 
-                string_erase(e);
+                assert_se(unsetenv_erase("PASSWORD") >= 0);
-                assert_se(unsetenv("PASSWORD") == 0);
-
                 return 1;
         }
 
@@ -273,9 +271,7 @@ static int acquire_token_pin(
                 if (r < 0)
                         return log_error_errno(r, "Failed to store token PIN: %m");
 
-                string_erase(e);
+                assert_se(unsetenv_erase("PIN") >= 0);
-                assert_se(unsetenv("PIN") == 0);
-
                 return 1;
         }
 
@@ -1097,8 +1093,7 @@ static int acquire_new_password(
                 if (r < 0)
                         return log_error_errno(r, "Failed to store password: %m");
 
-                string_erase(e);
+                assert_se(unsetenv_erase("NEWPASSWORD") >= 0);
-                assert_se(unsetenv("NEWPASSWORD") == 0);
 
                 if (ret)
                         *ret = TAKE_PTR(copy);

src/import/import-common.c

@@ -13,6 +13,7 @@
 #include "fd-util.h"
 #include "fileio.h"
 #include "fs-util.h"
+#include "hostname-util.h"
 #include "import-common.h"
 #include "os-util.h"
 #include "process-util.h"
@@ -22,57 +23,6 @@
 #include "tmpfile-util.h"
 #include "util.h"
 
-int import_make_read_only_fd(int fd) {
-        struct stat st;
-        int r;
-
-        assert(fd >= 0);
-
-        /* First, let's make this a read-only subvolume if it refers
-         * to a subvolume */
-        r = btrfs_subvol_set_read_only_fd(fd, true);
-        if (r >= 0)
-                return 0;
-
-        if (!ERRNO_IS_NOT_SUPPORTED(r) && !IN_SET(r, -ENOTDIR, -EINVAL))
-                return log_error_errno(r, "Failed to make subvolume read-only: %m");
-
-        /* This doesn't refer to a subvolume, or the file system isn't even btrfs. In that, case fall back to
-         * chmod()ing */
-
-        r = fstat(fd, &st);
-        if (r < 0)
-                return log_error_errno(errno, "Failed to stat image: %m");
-
-        if (S_ISDIR(st.st_mode)) {
-                /* For directories set the immutable flag on the dir itself */
-
-                r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL);
-                if (r < 0)
-                        return log_error_errno(r, "Failed to set +i attribute on directory image: %m");
-
-        } else if (S_ISREG(st.st_mode)) {
-                /* For regular files drop "w" flags */
-
-                if ((st.st_mode & 0222) != 0)
-                        if (fchmod(fd, st.st_mode & 07555) < 0)
-                                return log_error_errno(errno, "Failed to chmod() image: %m");
-        } else
-                return log_error_errno(SYNTHETIC_ERRNO(EBADFD), "Image of unexpected type");
-
-        return 0;
-}
-
-int import_make_read_only(const char *path) {
-        _cleanup_close_ int fd = 1;
-
-        fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
-        if (fd < 0)
-                return log_error_errno(errno, "Failed to open %s: %m", path);
-
-        return import_make_read_only_fd(fd);
-}
-
 int import_fork_tar_x(const char *path, pid_t *ret) {
         _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
         bool use_selinux;
@@ -327,3 +277,38 @@ int import_mangle_os_tree(const char *path) {
 
         return 0;
 }
+
+bool import_validate_local(const char *name, ImportFlags flags) {
+
+        /* By default we insist on a valid hostname for naming images. But optionally we relax that, in which
+         * case it can be any path name */
+
+        if (FLAGS_SET(flags, IMPORT_DIRECT))
+                return path_is_valid(name);
+
+        return hostname_is_valid(name, 0);
+}
+
+static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
+        log_notice("Transfer aborted.");
+        sd_event_exit(sd_event_source_get_event(s), EINTR);
+        return 0;
+}
+
+int import_allocate_event_with_signals(sd_event **ret) {
+        _cleanup_(sd_event_unrefp) sd_event *event = NULL;
+        int r;
+
+        assert(ret);
+
+        r = sd_event_default(&event);
+        if (r < 0)
+                return log_error_errno(r, "Failed to allocate event loop: %m");
+
+        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
+        (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler,  NULL);
+        (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+
+        *ret = TAKE_PTR(event);
+        return 0;
+}

src/import/import-common.h

@@ -3,17 +3,26 @@
 
 #include <sys/types.h>
 
+#include "sd-event.h"
+
 typedef enum ImportFlags {
-        IMPORT_FORCE     = 1 << 0, /* replace existing image */
+        IMPORT_FORCE          = 1 << 0, /* replace existing image */
-        IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */
+        IMPORT_READ_ONLY      = 1 << 1, /* make generated image read-only */
+        IMPORT_BTRFS_SUBVOL   = 1 << 2, /* tar: preferably create images as btrfs subvols */
+        IMPORT_BTRFS_QUOTA    = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
+        IMPORT_CONVERT_QCOW2  = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
+        IMPORT_DIRECT         = 1 << 5, /* import without rename games */
+        IMPORT_SYNC           = 1 << 6, /* fsync() right before we are done */
 
-        IMPORT_FLAGS_MASK = IMPORT_FORCE|IMPORT_READ_ONLY,
+        IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC,
+        IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC,
 } ImportFlags;
 
-int import_make_read_only_fd(int fd);
-int import_make_read_only(const char *path);
-
 int import_fork_tar_c(const char *path, pid_t *ret);
 int import_fork_tar_x(const char *path, pid_t *ret);
 
 int import_mangle_os_tree(const char *path);
+
+bool import_validate_local(const char *name, ImportFlags flags);
+
+int import_allocate_event_with_signals(sd_event **ret);

src/import/import-fs.c

@@ -12,15 +12,24 @@
 #include "hostname-util.h"
 #include "import-common.h"
 #include "import-util.h"
+#include "install-file.h"
+#include "main-func.h"
 #include "mkdir.h"
+#include "parse-argument.h"
 #include "ratelimit.h"
 #include "rm-rf.h"
+#include "signal-util.h"
 #include "string-util.h"
+#include "terminal-util.h"
 #include "tmpfile-util.h"
 #include "verbs.h"
 
 static bool arg_force = false;
 static bool arg_read_only = false;
+static bool arg_btrfs_subvol = true;
+static bool arg_btrfs_quota = true;
+static bool arg_sync = true;
+static bool arg_direct = false;
 static const char *arg_image_root = "/var/lib/machines";
 
 typedef struct ProgressInfo {
@@ -31,12 +40,6 @@ typedef struct ProgressInfo {
         bool logged_incomplete;
 } ProgressInfo;
 
-static volatile sig_atomic_t cancelled = false;
-
-static void sigterm_sigint(int sig) {
-        cancelled = true;
-}
-
 static void progress_info_free(ProgressInfo *p) {
         free(p->path);
 }
@@ -56,7 +59,7 @@ static void progress_show(ProgressInfo *p) {
 
         /* Mention the list is incomplete before showing first output. */
         if (!p->logged_incomplete) {
-                log_notice("(Note, file list shown below is incomplete, and is intended as sporadic progress report only.)");
+                log_notice("(Note: file list shown below is incomplete, and is intended as sporadic progress report only.)");
                 p->logged_incomplete = true;
         }
 
@@ -72,9 +75,6 @@ static int progress_path(const char *path, const struct stat *st, void *userdata
 
         assert(p);
 
-        if (cancelled)
-                return -EOWNERDEAD;
-
         r = free_and_strdup(&p->path, path);
         if (r < 0)
                 return r;
@@ -91,9 +91,6 @@ static int progress_bytes(uint64_t nbytes, void *userdata) {
         assert(p);
         assert(p->size != UINT64_MAX);
 
-        if (cancelled)
-                return -EOWNERDEAD;
-
         p->size += nbytes;
 
         progress_show(p);
@@ -103,44 +100,60 @@ static int progress_bytes(uint64_t nbytes, void *userdata) {
 static int import_fs(int argc, char *argv[], void *userdata) {
         _cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL;
         _cleanup_(progress_info_free) ProgressInfo progress = {};
-        const char *path = NULL, *local = NULL, *final_path;
+        _cleanup_free_ char *l = NULL, *final_path = NULL;
+        const char *path = NULL, *local = NULL, *dest = NULL;
         _cleanup_close_ int open_fd = -1;
-        struct sigaction old_sigint_sa, old_sigterm_sa;
-        static const struct sigaction sa = {
-                .sa_handler = sigterm_sigint,
-                .sa_flags = SA_RESTART,
-        };
         int r, fd;
 
         if (argc >= 2)
-                path = argv[1];
+                path = empty_or_dash_to_null(argv[1]);
-        path = empty_or_dash_to_null(path);
 
         if (argc >= 3)
-                local = argv[2];
+                local = empty_or_dash_to_null(argv[2]);
-        else if (path)
+        else if (path) {
-                local = basename(path);
+                r = path_extract_filename(path, &l);
-        local = empty_or_dash_to_null(local);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
 
-        if (local) {
+                local = l;
-                if (!hostname_is_valid(local, 0))
+        }
+
+        if (arg_direct) {
+                if (!local)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified.");
+
+                if (path_is_absolute(local))
+                        final_path = strdup(local);
+                else
+                        final_path = path_join(arg_image_root, local);
+                if (!final_path)
+                        return log_oom();
+
+                if (!path_is_valid(final_path))
                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
-                                               "Local image name '%s' is not valid.",
+                                               "Local path name '%s' is not valid.", final_path);
-                                               local);
+        } else {
+                if (local) {
+                        if (!hostname_is_valid(local, 0))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                       "Local image name '%s' is not valid.", local);
+                } else
+                        local = "imported";
+
+                final_path = path_join(arg_image_root, local);
+                if (!final_path)
+                        return log_oom();
 
                 if (!arg_force) {
                         r = image_find(IMAGE_MACHINE, local, NULL, NULL);
                         if (r < 0) {
                                 if (r != -ENOENT)
                                         return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
-                        } else {
+                        } else
                                 return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
-                                                       "Image '%s' already exists.",
+                                                       "Image '%s' already exists.", local);
-                                                       local);
-                        }
                 }
-        } else
+        }
-                local = "imported";
 
         if (path) {
                 open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
@@ -159,84 +172,107 @@ static int import_fs(int argc, char *argv[], void *userdata) {
                 log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
         }
 
-        final_path = prefix_roota(arg_image_root, local);
+        if (!arg_sync)
+                log_info("File system synchronization on completion is off.");
 
-        r = tempfn_random(final_path, NULL, &temp_path);
+        if (arg_direct) {
-        if (r < 0)
+                if (arg_force)
-                return log_oom();
+                        (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 
-        (void) mkdir_parents_label(temp_path, 0700);
+                dest = final_path;
+        } else {
+                r = tempfn_random(final_path, NULL, &temp_path);
+                if (r < 0)
+                        return log_oom();
+
+                (void) mkdir_parents_label(temp_path, 0700);
+                dest = temp_path;
+        }
 
         progress.limit = (RateLimit) { 200*USEC_PER_MSEC, 1 };
 
-        /* Hook into SIGINT/SIGTERM, so that we can cancel things then */
+        {
-        assert_se(sigaction(SIGINT, &sa, &old_sigint_sa) >= 0);
+                BLOCK_SIGNALS(SIGINT, SIGTERM);
-        assert_se(sigaction(SIGTERM, &sa, &old_sigterm_sa) >= 0);
 
-        r = btrfs_subvol_snapshot_fd_full(
+                if (arg_btrfs_subvol)
-                        fd,
+                        r = btrfs_subvol_snapshot_fd_full(
-                        temp_path,
+                                        fd,
-                        BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_RECURSIVE|BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|BTRFS_SNAPSHOT_QUOTA,
+                                        dest,
-                        progress_path,
+                                        BTRFS_SNAPSHOT_FALLBACK_COPY|
-                        progress_bytes,
+                                        BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
-                        &progress);
+                                        BTRFS_SNAPSHOT_RECURSIVE|
-        if (r == -EOWNERDEAD) { /* SIGINT + SIGTERM cause this, see signal handler above */
+                                        BTRFS_SNAPSHOT_SIGINT|
-                log_error("Copy cancelled.");
+                                        BTRFS_SNAPSHOT_SIGTERM,
-                goto finish;
+                                        progress_path,
-        }
+                                        progress_bytes,
-        if (r < 0) {
+                                        &progress);
-                log_error_errno(r, "Failed to copy directory: %m");
+                else
-                goto finish;
+                        r = copy_directory_fd_full(
+                                        fd,
+                                        dest,
+                                        COPY_REFLINK|
+                                        COPY_SAME_MOUNT|
+                                        COPY_HARDLINKS|
+                                        COPY_SIGINT|
+                                        COPY_SIGTERM|
+                                        (arg_direct ? COPY_MERGE_EMPTY : 0),
+                                        progress_path,
+                                        progress_bytes,
+                                        &progress);
+                if (r == -EINTR) /* SIGINT/SIGTERM hit */
+                        return log_error_errno(r, "Copy cancelled.");
+                if (r < 0)
+                        return log_error_errno(r, "Failed to copy directory: %m");
         }
 
-        r = import_mangle_os_tree(temp_path);
+        r = import_mangle_os_tree(dest);
         if (r < 0)
-                goto finish;
+                return r;
 
-        (void) import_assign_pool_quota_and_warn(arg_image_root);
+        if (arg_btrfs_quota) {
-        (void) import_assign_pool_quota_and_warn(temp_path);
+                if (!arg_direct)
-
+                        (void) import_assign_pool_quota_and_warn(arg_image_root);
-        if (arg_read_only) {
+                (void) import_assign_pool_quota_and_warn(dest);
-                r = import_make_read_only(temp_path);
-                if (r < 0) {
-                        log_error_errno(r, "Failed to make directory read-only: %m");
-                        goto finish;
-                }
         }
 
-        if (arg_force)
+        r = install_file(AT_FDCWD, dest,
-                (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
+                         AT_FDCWD, arg_direct ? NULL : final_path, /* pass NULL as target in case of direct
-
+                                                                    * mode since file is already in place */
-        r = rename_noreplace(AT_FDCWD, temp_path, AT_FDCWD, final_path);
+                         (arg_force ? INSTALL_REPLACE : 0) |
-        if (r < 0) {
+                         (arg_read_only ? INSTALL_READ_ONLY : 0) |
-                log_error_errno(r, "Failed to move image into place: %m");
+                         (arg_sync ? INSTALL_SYNCFS : 0));
-                goto finish;
+        if (r < 0)
-        }
+                return log_error_errno(r, "Failed install directory as '%s': %m", final_path);
 
         temp_path = mfree(temp_path);
 
-        log_info("Exiting.");
+        log_info("Directory '%s successfully installed. Exiting.", final_path);
-
-finish:
-        /* Put old signal handlers into place */
-        assert_se(sigaction(SIGINT, &old_sigint_sa, NULL) >= 0);
-        assert_se(sigaction(SIGTERM, &old_sigterm_sa, NULL) >= 0);
-
         return 0;
 }
 
 static int help(int argc, char *argv[], void *userdata) {
 
-        printf("%s [OPTIONS...] {COMMAND} ...\n\n"
+        printf("%1$s [OPTIONS...] {COMMAND} ...\n"
-               "Import container images from a file system.\n\n"
+               "\n%4$sImport container images from a file system directories.%5$s\n"
+               "\n%2$sCommands:%3$s\n"
+               "  run DIRECTORY [NAME]        Import a directory\n"
+               "\n%2$sOptions:%3$s\n"
                "  -h --help                   Show this help\n"
                "     --version                Show package version\n"
                "     --force                  Force creation of image\n"
                "     --image-root=PATH        Image root directory\n"
-               "     --read-only              Create a read-only image\n\n"
+               "     --read-only              Create a read-only image\n"
-               "Commands:\n"
+               "     --direct                 Import directly to specified directory\n"
-               "  run DIRECTORY [NAME]             Import a directory\n",
+               "     --btrfs-subvol=BOOL      Controls whether to create a btrfs subvolume\n"
-               program_invocation_short_name);
+               "                              instead of a directory\n"
+               "     --btrfs-quota=BOOL       Controls whether to set up quota for btrfs\n"
+               "                              subvolume\n"
+               "     --sync=BOOL              Controls whether to sync() before completing\n",
+               program_invocation_short_name,
+               ansi_underline(),
+               ansi_normal(),
+               ansi_highlight(),
+               ansi_normal());
 
         return 0;
 }
@@ -248,6 +284,10 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_FORCE,
                 ARG_IMAGE_ROOT,
                 ARG_READ_ONLY,
+                ARG_DIRECT,
+                ARG_BTRFS_SUBVOL,
+                ARG_BTRFS_QUOTA,
+                ARG_SYNC,
         };
 
         static const struct option options[] = {
@@ -256,10 +296,14 @@ static int parse_argv(int argc, char *argv[]) {
                 { "force",           no_argument,       NULL, ARG_FORCE           },
                 { "image-root",      required_argument, NULL, ARG_IMAGE_ROOT      },
                 { "read-only",       no_argument,       NULL, ARG_READ_ONLY       },
+                { "direct",          no_argument,       NULL, ARG_DIRECT          },
+                { "btrfs-subvol",    required_argument, NULL, ARG_BTRFS_SUBVOL    },
+                { "btrfs-quota",     required_argument, NULL, ARG_BTRFS_QUOTA     },
+                { "sync",            required_argument, NULL, ARG_SYNC            },
                 {}
         };
 
-        int c;
+        int c, r;
 
         assert(argc >= 0);
         assert(argv);
@@ -286,6 +330,31 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_read_only = true;
                         break;
 
+                case ARG_DIRECT:
+                        arg_direct = true;
+                        break;
+
+                case ARG_BTRFS_SUBVOL:
+                        r = parse_boolean_argument("--btrfs-subvol=", optarg, &arg_btrfs_subvol);
+                        if (r < 0)
+                                return r;
+
+                        break;
+
+                case ARG_BTRFS_QUOTA:
+                        r = parse_boolean_argument("--btrfs-quota=", optarg, &arg_btrfs_quota);
+                        if (r < 0)
+                                return r;
+
+                        break;
+
+                case ARG_SYNC:
+                        r = parse_boolean_argument("--sync=", optarg, &arg_sync);
+                        if (r < 0)
+                                return r;
+
+                        break;
+
                 case '?':
                         return -EINVAL;
 
@@ -307,7 +376,7 @@ static int import_fs_main(int argc, char *argv[]) {
         return dispatch_verb(argc, argv, verbs, NULL);
 }
 
-int main(int argc, char *argv[]) {
+static int run(int argc, char *argv[]) {
         int r;
 
         setlocale(LC_ALL, "");
@@ -316,10 +385,9 @@ int main(int argc, char *argv[]) {
 
         r = parse_argv(argc, argv);
         if (r <= 0)
-                goto finish;
+                return r;
 
-        r = import_fs_main(argc, argv);
+        return import_fs_main(argc, argv);
-
-finish:
-        return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
 }
+
+DEFINE_MAIN_FUNCTION(run);

src/import/import-raw.c

@@ -14,6 +14,7 @@
 #include "import-common.h"
 #include "import-compress.h"
 #include "import-raw.h"
+#include "install-file.h"
 #include "io-util.h"
 #include "machine-pool.h"
 #include "mkdir.h"
@@ -52,23 +53,27 @@ struct RawImport {
         uint64_t written_compressed;
         uint64_t written_uncompressed;
 
-        struct stat st;
+        struct stat input_stat;
+        struct stat output_stat;
 
         unsigned last_percent;
         RateLimit progress_ratelimit;
+
+        uint64_t offset;
+        uint64_t size_max;
 };
 
 RawImport* raw_import_unref(RawImport *i) {
         if (!i)
                 return NULL;
 
-        sd_event_unref(i->event);
+        sd_event_source_unref(i->input_event_source);
 
         unlink_and_free(i->temp_path);
 
         import_compress_free(&i->compress);
 
-        sd_event_source_unref(i->input_event_source);
+        sd_event_unref(i->event);
 
         safe_close(i->output_fd);
 
@@ -107,6 +112,8 @@ int raw_import_new(
                 .last_percent = UINT_MAX,
                 .image_root = TAKE_PTR(root),
                 .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 },
+                .offset = UINT64_MAX,
+                .size_max = UINT64_MAX,
         };
 
         if (event)
@@ -118,7 +125,6 @@ int raw_import_new(
         }
 
         *ret = TAKE_PTR(i);
-
         return 0;
 }
 
@@ -127,13 +133,13 @@ static void raw_import_report_progress(RawImport *i) {
         assert(i);
 
         /* We have no size information, unless the source is a regular file */
-        if (!S_ISREG(i->st.st_mode))
+        if (!S_ISREG(i->input_stat.st_mode))
                 return;
 
-        if (i->written_compressed >= (uint64_t) i->st.st_size)
+        if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
                 percent = 100;
         else
-                percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size);
+                percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
 
         if (percent == i->last_percent)
                 return;
@@ -149,11 +155,18 @@ static void raw_import_report_progress(RawImport *i) {
 
 static int raw_import_maybe_convert_qcow2(RawImport *i) {
         _cleanup_close_ int converted_fd = -1;
-        _cleanup_free_ char *t = NULL;
+        _cleanup_(unlink_and_freep) char *t = NULL;
+        _cleanup_free_ char *f = NULL;
         int r;
 
         assert(i);
 
+        /* Do QCOW2 conversion if enabled and not in direct mode */
+        if ((i->flags & (IMPORT_CONVERT_QCOW2|IMPORT_DIRECT)) != IMPORT_CONVERT_QCOW2)
+                return 0;
+
+        assert(i->final_path);
+
         r = qcow2_detect(i->output_fd);
         if (r < 0)
                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
@@ -161,26 +174,26 @@ static int raw_import_maybe_convert_qcow2(RawImport *i) {
                 return 0;
 
         /* This is a QCOW2 image, let's convert it */
-        r = tempfn_random(i->final_path, NULL, &t);
+        r = tempfn_random(i->final_path, NULL, &f);
         if (r < 0)
                 return log_oom();
 
-        converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+        converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
         if (converted_fd < 0)
-                return log_error_errno(errno, "Failed to create %s: %m", t);
+                return log_error_errno(errno, "Failed to create %s: %m", f);
+
+        t = TAKE_PTR(f);
 
         (void) import_set_nocow_and_log(converted_fd, t);
 
         log_info("Unpacking QCOW2 file.");
 
         r = qcow2_convert(i->output_fd, converted_fd);
-        if (r < 0) {
+        if (r < 0)
-                (void) unlink(t);
                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
-        }
 
-        (void) unlink(i->temp_path);
+        unlink_and_free(i->temp_path);
-        free_and_replace(i->temp_path, t);
+        i->temp_path = TAKE_PTR(t);
         CLOSE_AND_REPLACE(i->output_fd, converted_fd);
 
         return 1;
@@ -191,34 +204,45 @@ static int raw_import_finish(RawImport *i) {
 
         assert(i);
         assert(i->output_fd >= 0);
-        assert(i->temp_path);
-        assert(i->final_path);
 
-        /* In case this was a sparse file, make sure the file system is right */
+        /* Nothing of what is below applies to block devices */
-        if (i->written_uncompressed > 0) {
+        if (S_ISBLK(i->output_stat.st_mode)) {
-                if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
+
-                        return log_error_errno(errno, "Failed to truncate file: %m");
+                if (i->flags & IMPORT_SYNC) {
+                        if (fsync(i->output_fd) < 0)
+                                return log_error_errno(errno, "Failed to synchronize block device: %m");
+                }
+
+                return 0;
         }
 
-        r = raw_import_maybe_convert_qcow2(i);
+        assert(S_ISREG(i->output_stat.st_mode));
-        if (r < 0)
-                return r;
 
-        if (S_ISREG(i->st.st_mode)) {
+        /* If an offset is specified we only are supposed to affect part of an existing output file or block
-                (void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME);
+         * device, thus don't manipulate file properties in that case */
-                (void) copy_xattr(i->input_fd, i->output_fd, 0);
-        }
 
-        if (i->flags & IMPORT_READ_ONLY) {
+        if (i->offset == UINT64_MAX) {
-                r = import_make_read_only_fd(i->output_fd);
+                /* In case this was a sparse file, make sure the file size is right */
+                if (i->written_uncompressed > 0) {
+                        if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
+                                return log_error_errno(errno, "Failed to truncate file: %m");
+                }
+
+                r = raw_import_maybe_convert_qcow2(i);
                 if (r < 0)
                         return r;
+
+                if (S_ISREG(i->input_stat.st_mode)) {
+                        (void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME);
+                        (void) copy_xattr(i->input_fd, i->output_fd, 0);
+                }
         }
 
-        if (i->flags & IMPORT_FORCE)
+        r = install_file(AT_FDCWD, i->temp_path ?: i->local,
-                (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
+                         AT_FDCWD, i->final_path,
-
+                         (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
-        r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
+                         (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
+                         (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0));
         if (r < 0)
                 return log_error_errno(r, "Failed to move image into place: %m");
 
@@ -231,26 +255,56 @@ static int raw_import_open_disk(RawImport *i) {
         int r;
 
         assert(i);
-
+        assert(i->local);
         assert(!i->final_path);
         assert(!i->temp_path);
         assert(i->output_fd < 0);
 
-        i->final_path = strjoin(i->image_root, "/", i->local, ".raw");
+        if (i->flags & IMPORT_DIRECT) {
-        if (!i->final_path)
+                (void) mkdir_parents_label(i->local, 0700);
-                return log_oom();
 
-        r = tempfn_random(i->final_path, NULL, &i->temp_path);
+                /* In direct mode we just open/create the local path and truncate it (like shell >
-        if (r < 0)
+                 * redirection would do it) — except if an offset was passed, in which case we are supposed
-                return log_oom();
+                 * to operate on a section of the file only, in which case we apparently work on an some
+                 * existing thing (i.e. are not the sole thing stored in the file), in which case we will
+                 * neither truncate nor create. */
 
-        (void) mkdir_parents_label(i->temp_path, 0700);
+                i->output_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
+                if (i->output_fd < 0)
+                        return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
 
-        i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+                if (i->offset == UINT64_MAX)
-        if (i->output_fd < 0)
+                        (void) import_set_nocow_and_log(i->output_fd, i->local);
-                return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path);
+        } else {
+                i->final_path = strjoin(i->image_root, "/", i->local, ".raw");
+                if (!i->final_path)
+                        return log_oom();
+
+                r = tempfn_random(i->final_path, NULL, &i->temp_path);
+                if (r < 0)
+                        return log_oom();
+
+                (void) mkdir_parents_label(i->temp_path, 0700);
+
+                i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+                if (i->output_fd < 0)
+                        return log_error_errno(errno, "Failed to open destination '%s': %m", i->temp_path);
+
+                (void) import_set_nocow_and_log(i->output_fd, i->temp_path);
+        }
+
+        if (fstat(i->output_fd, &i->output_stat) < 0)
+                return log_error_errno(errno, "Failed to stat() output file: %m");
+
+        if (!S_ISREG(i->output_stat.st_mode) && !S_ISBLK(i->output_stat.st_mode))
+                return log_error_errno(SYNTHETIC_ERRNO(EBADFD),
+                                       "Target file is not a regular file or block device");
+
+        if (i->offset != UINT64_MAX) {
+                if (lseek(i->output_fd, i->offset, SEEK_SET) == (off_t) -1)
+                        return log_error_errno(errno, "Failed to seek to offset: %m");
+        }
 
-        (void) import_set_nocow_and_log(i->output_fd, i->temp_path);
         return 0;
 }
 
@@ -265,7 +319,7 @@ static int raw_import_try_reflink(RawImport *i) {
         if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED)
                 return 0;
 
-        if (!S_ISREG(i->st.st_mode))
+        if (!S_ISREG(i->input_stat.st_mode) || !S_ISREG(i->output_stat.st_mode))
                 return 0;
 
         p = lseek(i->input_fd, 0, SEEK_CUR);
@@ -280,21 +334,57 @@ static int raw_import_try_reflink(RawImport *i) {
         if (r >= 0)
                 return 1;
 
+        log_debug_errno(r, "Couldn't establish reflink, using copy: %m");
         return 0;
 }
 
 static int raw_import_write(const void *p, size_t sz, void *userdata) {
         RawImport *i = userdata;
-        ssize_t n;
+        bool too_much = false;
+        int r;
 
-        n = sparse_write(i->output_fd, p, sz, 64);
+        assert(i);
-        if (n < 0)
+        assert(p);
-                return (int) n;
+        assert(sz > 0);
-        if ((size_t) n < sz)
+
-                return -EIO;
+        if (i->written_uncompressed >= UINT64_MAX - sz)
+                return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
+
+        if (i->size_max != UINT64_MAX) {
+                if (i->written_uncompressed >= i->size_max) {
+                        too_much = true;
+                        goto finish;
+                }
+
+                if (i->written_uncompressed + sz > i->size_max) {
+                        too_much = true;
+                        sz = i->size_max - i->written_uncompressed; /* since we have the data in memory
+                                                                     * already, we might as well write it to
+                                                                     * disk to the max */
+                }
+        }
+
+        /* Generate sparse file if we created/truncated the file */
+        if (S_ISREG(i->output_stat.st_mode)) {
+                ssize_t n;
+
+                n = sparse_write(i->output_fd, p, sz, 64);
+                if (n < 0)
+                        return log_error_errno((int) n, "Failed to write file: %m");
+                if ((size_t) n < sz)
+                        return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
+        } else {
+                r = loop_write(i->output_fd, p, sz, false);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to write file: %m");
+        }
 
         i->written_uncompressed += sz;
 
+finish:
+        if (too_much)
+                return log_error_errno(SYNTHETIC_ERRNO(E2BIG), "File too large");
+
         return 0;
 }
 
@@ -382,15 +472,22 @@ static int raw_import_on_defer(sd_event_source *s, void *userdata) {
         return raw_import_process(i);
 }
 
-int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) {
+int raw_import_start(
+                RawImport *i,
+                int fd,
+                const char *local,
+                uint64_t offset,
+                uint64_t size_max,
+                ImportFlags flags) {
         int r;
 
         assert(i);
         assert(fd >= 0);
         assert(local);
-        assert(!(flags & ~IMPORT_FLAGS_MASK));
+        assert(!(flags & ~IMPORT_FLAGS_MASK_RAW));
+        assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
 
-        if (!hostname_is_valid(local, 0))
+        if (!import_validate_local(local, flags))
                 return -EINVAL;
 
         if (i->input_fd >= 0)
@@ -405,8 +502,10 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
                 return r;
 
         i->flags = flags;
+        i->offset = offset;
+        i->size_max = size_max;
 
-        if (fstat(fd, &i->st) < 0)
+        if (fstat(fd, &i->input_stat) < 0)
                 return -errno;
 
         r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i);
@@ -422,5 +521,5 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
                 return r;
 
         i->input_fd = fd;
-        return r;
+        return 0;
 }

src/import/import-raw.h

@@ -16,4 +16,4 @@ RawImport* raw_import_unref(RawImport *import);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref);
 
-int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags);
+int raw_import_start(RawImport *i, int fd, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags);

src/import/import-tar.c

@@ -15,6 +15,7 @@
 #include "import-common.h"
 #include "import-compress.h"
 #include "import-tar.h"
+#include "install-file.h"
 #include "io-util.h"
 #include "machine-pool.h"
 #include "mkdir.h"
@@ -54,7 +55,7 @@ struct TarImport {
         uint64_t written_compressed;
         uint64_t written_uncompressed;
 
-        struct stat st;
+        struct stat input_stat;
 
         pid_t tar_pid;
 
@@ -136,13 +137,13 @@ static void tar_import_report_progress(TarImport *i) {
         assert(i);
 
         /* We have no size information, unless the source is a regular file */
-        if (!S_ISREG(i->st.st_mode))
+        if (!S_ISREG(i->input_stat.st_mode))
                 return;
 
-        if (i->written_compressed >= (uint64_t) i->st.st_size)
+        if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
                 percent = 100;
         else
-                percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size);
+                percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
 
         if (percent == i->last_percent)
                 return;
@@ -157,12 +158,11 @@ static void tar_import_report_progress(TarImport *i) {
 }
 
 static int tar_import_finish(TarImport *i) {
+        const char *d;
         int r;
 
         assert(i);
         assert(i->tar_fd >= 0);
-        assert(i->temp_path);
-        assert(i->final_path);
 
         i->tar_fd = safe_close(i->tar_fd);
 
@@ -175,22 +175,20 @@ static int tar_import_finish(TarImport *i) {
                         return -EPROTO;
         }
 
-        r = import_mangle_os_tree(i->temp_path);
+        assert_se(d = i->temp_path ?: i->local);
+
+        r = import_mangle_os_tree(d);
         if (r < 0)
                 return r;
 
-        if (i->flags & IMPORT_READ_ONLY) {
+        r = install_file(
-                r = import_make_read_only(i->temp_path);
+                        AT_FDCWD, d,
-                if (r < 0)
+                        AT_FDCWD, i->final_path,
-                        return r;
+                        (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
-        }
+                        (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
-
+                        (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
-        if (i->flags & IMPORT_FORCE)
-                (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
-
-        r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
         if (r < 0)
-                return log_error_errno(r, "Failed to move image into place: %m");
+                return log_error_errno(r, "Failed to move '%s' into place: %m", i->final_path ?: i->local);
 
         i->temp_path = mfree(i->temp_path);
 
@@ -198,33 +196,54 @@ static int tar_import_finish(TarImport *i) {
 }
 
 static int tar_import_fork_tar(TarImport *i) {
+        const char *d, *root;
         int r;
 
         assert(i);
-
+        assert(i->local);
         assert(!i->final_path);
         assert(!i->temp_path);
         assert(i->tar_fd < 0);
 
-        i->final_path = path_join(i->image_root, i->local);
+        if (i->flags & IMPORT_DIRECT) {
-        if (!i->final_path)
+                d = i->local;
-                return log_oom();
+                root = NULL;
+        } else {
+                i->final_path = path_join(i->image_root, i->local);
+                if (!i->final_path)
+                        return log_oom();
 
-        r = tempfn_random(i->final_path, NULL, &i->temp_path);
+                r = tempfn_random(i->final_path, NULL, &i->temp_path);
-        if (r < 0)
+                if (r < 0)
-                return log_oom();
+                        return log_oom();
 
-        (void) mkdir_parents_label(i->temp_path, 0700);
+                d = i->temp_path;
-
+                root = i->image_root;
-        r = btrfs_subvol_make_fallback(i->temp_path, 0755);
-        if (r < 0)
-                return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
-        if (r > 0) { /* actually btrfs subvol */
-                (void) import_assign_pool_quota_and_warn(i->image_root);
-                (void) import_assign_pool_quota_and_warn(i->temp_path);
         }
 
-        i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
+        assert(d);
+
+        (void) mkdir_parents_label(d, 0700);
+
+        if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE))
+                (void) rm_rf(d, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
+
+        if (i->flags & IMPORT_BTRFS_SUBVOL)
+                r = btrfs_subvol_make_fallback(d, 0755);
+        else
+                r = mkdir(d, 0755) < 0 ? -errno : 0;
+        if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
+                                                         * because in that case our temporary path collided */
+                r = 0;
+        if (r < 0)
+                return log_error_errno(r, "Failed to create directory/subvolume %s: %m", d);
+        if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */
+                if (!(i->flags & IMPORT_DIRECT))
+                        (void) import_assign_pool_quota_and_warn(root);
+                (void) import_assign_pool_quota_and_warn(d);
+        }
+
+        i->tar_fd = import_fork_tar_x(d, &i->tar_pid);
         if (i->tar_fd < 0)
                 return i->tar_fd;
 
@@ -327,9 +346,9 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
         assert(i);
         assert(fd >= 0);
         assert(local);
-        assert(!(flags & ~IMPORT_FLAGS_MASK));
+        assert(!(flags & ~IMPORT_FLAGS_MASK_TAR));
 
-        if (!hostname_is_valid(local, 0))
+        if (!import_validate_local(local, flags))
                 return -EINVAL;
 
         if (i->input_fd >= 0)
@@ -345,7 +364,7 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
 
         i->flags = flags;
 
-        if (fstat(fd, &i->st) < 0)
+        if (fstat(fd, &i->input_stat) < 0)
                 return -errno;
 
         r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i);
@@ -361,5 +380,5 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
                 return r;
 
         i->input_fd = fd;
-        return r;
+        return 0;
 }

src/import/import.c

@@ -8,26 +8,114 @@
 
 #include "alloc-util.h"
 #include "discover-image.h"
+#include "env-util.h"
 #include "fd-util.h"
 #include "fs-util.h"
 #include "hostname-util.h"
 #include "import-raw.h"
 #include "import-tar.h"
 #include "import-util.h"
+#include "io-util.h"
 #include "main-func.h"
+#include "parse-argument.h"
+#include "parse-util.h"
 #include "signal-util.h"
 #include "string-util.h"
+#include "terminal-util.h"
 #include "verbs.h"
 
 static const char *arg_image_root = "/var/lib/machines";
-static ImportFlags arg_import_flags = 0;
+static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
+static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
 
-static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
+static int normalize_local(const char *local, char **ret) {
-        log_notice("Transfer aborted.");
+        _cleanup_free_ char *ll = NULL;
-        sd_event_exit(sd_event_source_get_event(s), EINTR);
+        int r;
+
+        assert(ret);
+
+        if (arg_import_flags & IMPORT_DIRECT) {
+
+                if (!local)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified.");
+
+                if (!path_is_absolute(local))  {
+                        ll = path_join(arg_image_root, local);
+                        if (!ll)
+                                return log_oom();
+
+                        local = ll;
+                }
+
+                if (!path_is_valid(local))
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                               "Local path name '%s' is not valid.", local);
+        } else {
+                if (local) {
+                        if (!hostname_is_valid(local, 0))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                       "Local image name '%s' is not valid.",
+                                                       local);
+                } else
+                        local = "imported";
+
+                if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
+                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
+                        if (r < 0) {
+                                if (r != -ENOENT)
+                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
+                        } else
+                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
+                                                       "Image '%s' already exists.",
+                                                       local);
+                }
+        }
+
+        if (!ll) {
+                ll = strdup(local);
+                if (!ll)
+                        return log_oom();
+        }
+
+        *ret = TAKE_PTR(ll);
         return 0;
 }
 
+static int open_source(const char *path, const char *local, int *ret_open_fd) {
+        _cleanup_close_ int open_fd = -1;
+        int retval;
+
+        assert(local);
+        assert(ret_open_fd);
+
+        if (path) {
+                open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
+                if (open_fd < 0)
+                        return log_error_errno(errno, "Failed to open raw image to import: %m");
+
+                retval = open_fd;
+
+                if (arg_offset != UINT64_MAX)
+                        log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", path, arg_offset, local);
+                else
+                        log_info("Importing '%s', saving as '%s'.", path, local);
+        } else {
+                _cleanup_free_ char *pretty = NULL;
+
+                retval = STDIN_FILENO;
+
+                (void) fd_get_path(STDIN_FILENO, &pretty);
+
+                if (arg_offset != UINT64_MAX)
+                        log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", strempty(pretty), arg_offset, local);
+                else
+                        log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
+        }
+
+        *ret_open_fd = TAKE_FD(open_fd);
+        return retval;
+}
+
 static void on_tar_finished(TarImport *import, int error, void *userdata) {
         sd_event *event = userdata;
         assert(import);
@@ -40,9 +128,9 @@ static void on_tar_finished(TarImport *import, int error, void *userdata) {
 
 static int import_tar(int argc, char *argv[], void *userdata) {
         _cleanup_(tar_import_unrefp) TarImport *import = NULL;
+        _cleanup_free_ char *ll = NULL, *normalized = NULL;
         _cleanup_(sd_event_unrefp) sd_event *event = NULL;
         const char *path = NULL, *local = NULL;
-        _cleanup_free_ char *ll = NULL;
         _cleanup_close_ int open_fd = -1;
         int r, fd;
 
@@ -51,64 +139,44 @@ static int import_tar(int argc, char *argv[], void *userdata) {
 
         if (argc >= 3)
                 local = empty_or_dash_to_null(argv[2]);
-        else if (path)
+        else if (path) {
-                local = basename(path);
+                _cleanup_free_ char *l = NULL;
 
-        if (local) {
+                r = path_extract_filename(path, &l);
-                r = tar_strip_suffixes(local, &ll);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
+
+                r = tar_strip_suffixes(l, &ll);
                 if (r < 0)
                         return log_oom();
 
                 local = ll;
-
-                if (!hostname_is_valid(local, 0))
-                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
-                                               "Local image name '%s' is not valid.",
-                                               local);
-
-                if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
-                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
-                        if (r < 0) {
-                                if (r != -ENOENT)
-                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
-                        } else
-                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
-                                                       "Image '%s' already exists.",
-                                                       local);
-                }
-        } else
-                local = "imported";
-
-        if (path) {
-                open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
-                if (open_fd < 0)
-                        return log_error_errno(errno, "Failed to open tar image to import: %m");
-
-                fd = open_fd;
-
-                log_info("Importing '%s', saving as '%s'.", path, local);
-        } else {
-                _cleanup_free_ char *pretty = NULL;
-
-                fd = STDIN_FILENO;
-
-                (void) fd_get_path(fd, &pretty);
-                log_info("Importing '%s', saving as '%s'.", strna(pretty), local);
         }
 
-        r = sd_event_default(&event);
+        r = normalize_local(local, &normalized);
         if (r < 0)
-                return log_error_errno(r, "Failed to allocate event loop: %m");
+                return r;
 
-        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
+        fd = open_source(path, normalized, &open_fd);
-        (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler,  NULL);
+        if (fd < 0)
-        (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+                return r;
+
+        r = import_allocate_event_with_signals(&event);
+        if (r < 0)
+                return r;
+
+        if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
+                log_info("File system synchronization on completion is off.");
 
         r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event);
         if (r < 0)
                 return log_error_errno(r, "Failed to allocate importer: %m");
 
-        r = tar_import_start(import, fd, local, arg_import_flags);
+        r = tar_import_start(
+                        import,
+                        fd,
+                        normalized,
+                        arg_import_flags & IMPORT_FLAGS_MASK_TAR);
         if (r < 0)
                 return log_error_errno(r, "Failed to import image: %m");
 
@@ -132,9 +200,9 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
 
 static int import_raw(int argc, char *argv[], void *userdata) {
         _cleanup_(raw_import_unrefp) RawImport *import = NULL;
+        _cleanup_free_ char *ll = NULL, *normalized = NULL;
         _cleanup_(sd_event_unrefp) sd_event *event = NULL;
         const char *path = NULL, *local = NULL;
-        _cleanup_free_ char *ll = NULL;
         _cleanup_close_ int open_fd = -1;
         int r, fd;
 
@@ -143,64 +211,46 @@ static int import_raw(int argc, char *argv[], void *userdata) {
 
         if (argc >= 3)
                 local = empty_or_dash_to_null(argv[2]);
-        else if (path)
+        else if (path) {
-                local = basename(path);
+                _cleanup_free_ char *l = NULL;
 
-        if (local) {
+                r = path_extract_filename(path, &l);
-                r = raw_strip_suffixes(local, &ll);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
+
+                r = raw_strip_suffixes(l, &ll);
                 if (r < 0)
                         return log_oom();
 
                 local = ll;
-
-                if (!hostname_is_valid(local, 0))
-                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
-                                               "Local image name '%s' is not valid.",
-                                               local);
-
-                if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
-                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
-                        if (r < 0) {
-                                if (r != -ENOENT)
-                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
-                        } else
-                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
-                                                       "Image '%s' already exists.",
-                                                       local);
-                }
-        } else
-                local = "imported";
-
-        if (path) {
-                open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
-                if (open_fd < 0)
-                        return log_error_errno(errno, "Failed to open raw image to import: %m");
-
-                fd = open_fd;
-
-                log_info("Importing '%s', saving as '%s'.", path, local);
-        } else {
-                _cleanup_free_ char *pretty = NULL;
-
-                fd = STDIN_FILENO;
-
-                (void) fd_get_path(fd, &pretty);
-                log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
         }
 
-        r = sd_event_default(&event);
+        r = normalize_local(local, &normalized);
         if (r < 0)
-                return log_error_errno(r, "Failed to allocate event loop: %m");
+                return r;
 
-        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
+        fd = open_source(path, normalized, &open_fd);
-        (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler,  NULL);
+        if (fd < 0)
-        (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+                return fd;
+
+        r = import_allocate_event_with_signals(&event);
+        if (r < 0)
+                return r;
+
+        if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
+                log_info("File system synchronization on completion is off.");
 
         r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event);
         if (r < 0)
                 return log_error_errno(r, "Failed to allocate importer: %m");
 
-        r = raw_import_start(import, fd, local, arg_import_flags);
+        r = raw_import_start(
+                        import,
+                        fd,
+                        normalized,
+                        arg_offset,
+                        arg_size_max,
+                        arg_import_flags & IMPORT_FLAGS_MASK_RAW);
         if (r < 0)
                 return log_error_errno(r, "Failed to import image: %m");
 
@@ -214,17 +264,32 @@ static int import_raw(int argc, char *argv[], void *userdata) {
 
 static int help(int argc, char *argv[], void *userdata) {
 
-        printf("%s [OPTIONS...] {COMMAND} ...\n\n"
+        printf("%1$s [OPTIONS...] {COMMAND} ...\n"
-               "Import container or virtual machine images.\n\n"
+               "\n%4$sImport container or virtual machine images.%5$s\n"
+               "\n%2$sCommands:%3$s\n"
+               "  tar FILE [NAME]             Import a TAR image\n"
+               "  raw FILE [NAME]             Import a RAW image\n"
+               "\n%2$sOptions:%3$s\n"
                "  -h --help                   Show this help\n"
                "     --version                Show package version\n"
                "     --force                  Force creation of image\n"
                "     --image-root=PATH        Image root directory\n"
-               "     --read-only              Create a read-only image\n\n"
+               "     --read-only              Create a read-only image\n"
-               "Commands:\n"
+               "     --direct                 Import directly to specified file\n"
-               "  tar FILE [NAME]             Import a TAR image\n"
+               "     --btrfs-subvol=BOOL      Controls whether to create a btrfs subvolume\n"
-               "  raw FILE [NAME]             Import a RAW image\n",
+               "                              instead of a directory\n"
-               program_invocation_short_name);
+               "     --btrfs-quota=BOOL       Controls whether to set up quota for btrfs\n"
+               "                              subvolume\n"
+               "     --convert-qcow2=BOOL     Controls whether to convert QCOW2 images to\n"
+               "                              regular disk images\n"
+               "     --sync=BOOL              Controls whether to sync() before completing\n"
+               "     --offset=BYTES           Offset to seek to in destination\n"
+               "     --size-max=BYTES         Maximum number of bytes to write to destination\n",
+               program_invocation_short_name,
+               ansi_underline(),
+               ansi_normal(),
+               ansi_highlight(),
+               ansi_normal());
 
         return 0;
 }
@@ -236,6 +301,13 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_FORCE,
                 ARG_IMAGE_ROOT,
                 ARG_READ_ONLY,
+                ARG_DIRECT,
+                ARG_BTRFS_SUBVOL,
+                ARG_BTRFS_QUOTA,
+                ARG_CONVERT_QCOW2,
+                ARG_SYNC,
+                ARG_OFFSET,
+                ARG_SIZE_MAX,
         };
 
         static const struct option options[] = {
@@ -244,10 +316,17 @@ static int parse_argv(int argc, char *argv[]) {
                 { "force",           no_argument,       NULL, ARG_FORCE           },
                 { "image-root",      required_argument, NULL, ARG_IMAGE_ROOT      },
                 { "read-only",       no_argument,       NULL, ARG_READ_ONLY       },
+                { "direct",          no_argument,       NULL, ARG_DIRECT          },
+                { "btrfs-subvol",    required_argument, NULL, ARG_BTRFS_SUBVOL    },
+                { "btrfs-quota",     required_argument, NULL, ARG_BTRFS_QUOTA     },
+                { "convert-qcow2",   required_argument, NULL, ARG_CONVERT_QCOW2   },
+                { "sync",            required_argument, NULL, ARG_SYNC            },
+                { "offset",          required_argument, NULL, ARG_OFFSET          },
+                { "size-max",        required_argument, NULL, ARG_SIZE_MAX        },
                 {}
         };
 
-        int c;
+        int r, c;
 
         assert(argc >= 0);
         assert(argv);
@@ -274,6 +353,68 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_import_flags |= IMPORT_READ_ONLY;
                         break;
 
+                case ARG_DIRECT:
+                        arg_import_flags |= IMPORT_DIRECT;
+                        break;
+
+                case ARG_BTRFS_SUBVOL:
+                        r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
+                        break;
+
+                case ARG_BTRFS_QUOTA:
+                        r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
+                        break;
+
+                case ARG_CONVERT_QCOW2:
+                        r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r);
+                        break;
+
+                case ARG_SYNC:
+                        r = parse_boolean_argument("--sync=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
+                        break;
+
+                case ARG_OFFSET: {
+                        uint64_t u;
+
+                        r = safe_atou64(optarg, &u);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
+                        if (!FILE_SIZE_VALID(u))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
+
+                        arg_offset = u;
+                        break;
+                }
+
+                case ARG_SIZE_MAX: {
+                        uint64_t u;
+
+                        r = parse_size(optarg, 1024, &u);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
+                        if (!FILE_SIZE_VALID(u))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
+
+                        arg_size_max = u;
+                        break;
+                }
+
                 case '?':
                         return -EINVAL;
 
@@ -281,6 +422,15 @@ static int parse_argv(int argc, char *argv[]) {
                         assert_not_reached();
                 }
 
+        /* Make sure offset+size is still in the valid range if both set */
+        if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
+            ((arg_size_max > (UINT64_MAX - arg_offset)) ||
+             !FILE_SIZE_VALID(arg_offset + arg_size_max)))
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
+
+        if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
+
         return 1;
 }
 
@@ -295,6 +445,31 @@ static int import_main(int argc, char *argv[]) {
         return dispatch_verb(argc, argv, verbs, NULL);
 }
 
+static void parse_env(void) {
+        int r;
+
+        /* Let's make these relatively low-level settings also controllable via env vars. User can then set
+         * them to systemd-import if they like to tweak behaviour */
+
+        r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
+        if (r >= 0)
+                SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
+
+        r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
+        if (r >= 0)
+                SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
+
+        r = getenv_bool("SYSTEMD_IMPORT_SYNC");
+        if (r >= 0)
+                SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
+}
+
 static int run(int argc, char *argv[]) {
         int r;
 
@@ -302,9 +477,11 @@ static int run(int argc, char *argv[]) {
         log_parse_environment();
         log_open();
 
+        parse_env();
+
         r = parse_argv(argc, argv);
         if (r <= 0)
-                return 0;
+                return r;
 
         (void) ignore_signals(SIGPIPE);
 

src/import/pull-common.c

@@ -9,6 +9,7 @@
 #include "dirent-util.h"
 #include "escape.h"
 #include "fd-util.h"
+#include "hostname-util.h"
 #include "io-util.h"
 #include "memory-util.h"
 #include "path-util.h"
@@ -112,34 +113,6 @@ int pull_find_old_etags(
         return 0;
 }
 
-int pull_make_local_copy(const char *final, const char *image_root, const char *local, PullFlags flags) {
-        const char *p;
-        int r;
-
-        assert(final);
-        assert(local);
-
-        if (!image_root)
-                image_root = "/var/lib/machines";
-
-        p = prefix_roota(image_root, local);
-
-        if (FLAGS_SET(flags, PULL_FORCE))
-                (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
-
-        r = btrfs_subvol_snapshot(final, p,
-                                  BTRFS_SNAPSHOT_QUOTA|
-                                  BTRFS_SNAPSHOT_FALLBACK_COPY|
-                                  BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
-                                  BTRFS_SNAPSHOT_RECURSIVE);
-        if (r < 0)
-                return log_error_errno(r, "Failed to create local image: %m");
-
-        log_info("Created new local image '%s'.", local);
-
-        return 0;
-}
-
 static int hash_url(const char *url, char **ret) {
         uint64_t h;
         static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f);
@@ -206,7 +179,9 @@ int pull_make_auxiliary_job(
                 const char *url,
                 int (*strip_suffixes)(const char *name, char **ret),
                 const char *suffix,
+                ImportVerify verify,
                 CurlGlue *glue,
+                PullJobOpenDisk on_open_disk,
                 PullJobFinished on_finished,
                 void *userdata) {
 
@@ -238,45 +213,82 @@ int pull_make_auxiliary_job(
         if (r < 0)
                 return r;
 
+        job->on_open_disk = on_open_disk;
         job->on_finished = on_finished;
         job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL;
+        job->calc_checksum = IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
 
         *ret = TAKE_PTR(job);
-
         return 0;
 }
 
+static bool is_checksum_file(const char *fn) {
+        /* Returns true if the specified filename refers to a checksum file we grok */
+
+        if (!fn)
+                return false;
+
+        return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256");
+}
+
+static bool is_signature_file(const char *fn) {
+        /* Returns true if the specified filename refers to a signature file we grok (reminder:
+         * suse-style .sha256 files are inline signed) */
+
+        if (!fn)
+                return false;
+
+        return streq(fn, "SHA256SUMS.gpg") || endswith(fn, ".sha256");
+}
+
 int pull_make_verification_jobs(
                 PullJob **ret_checksum_job,
                 PullJob **ret_signature_job,
                 ImportVerify verify,
+                const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
                 const char *url,
                 CurlGlue *glue,
                 PullJobFinished on_finished,
                 void *userdata) {
 
         _cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL;
+        _cleanup_free_ char *fn = NULL;
         int r;
 
         assert(ret_checksum_job);
         assert(ret_signature_job);
-        assert(verify >= 0);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
-        assert(verify < _IMPORT_VERIFY_MAX);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+        assert((verify < 0) || !checksum);
         assert(url);
         assert(glue);
 
-        if (verify != IMPORT_VERIFY_NO) {
+        /* If verification is turned off, or if the checksum to validate is already specified we don't need
-                _cleanup_free_ char *checksum_url = NULL, *fn = NULL;
+         * to download a checksum file or signature, hence shortcut things */
-                const char *chksums = NULL;
+        if (verify == IMPORT_VERIFY_NO || checksum) {
+                *ret_checksum_job = *ret_signature_job = NULL;
+                return 0;
+        }
+
+        r = import_url_last_component(url, &fn);
+        if (r < 0 && r != -EADDRNOTAVAIL) /* EADDRNOTAVAIL means there was no last component, which is OK for
+                                           * us, we'll just assume it's not a checksum/signature file */
+                return r;
+
+        /* Acquire the checksum file if verification or signature verification is requested and the main file
+         * to acquire isn't a checksum or signature file anyway */
+        if (verify != IMPORT_VERIFY_NO && !is_checksum_file(fn) && !is_signature_file(fn)) {
+                _cleanup_free_ char *checksum_url = NULL;
+                const char *suffixed = NULL;
 
                 /* Queue jobs for the checksum file for the image. */
-                r = import_url_last_component(url, &fn);
-                if (r < 0)
-                        return r;
 
-                chksums = strjoina(fn, ".sha256");
+                if (fn)
+                        suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */
+                else
+                        suffixed = "SHA256SUMS";
 
-                r = import_url_change_last_component(url, chksums, &checksum_url);
+                r = import_url_change_last_component(url, suffixed, &checksum_url);
                 if (r < 0)
                         return r;
 
@@ -286,9 +298,10 @@ int pull_make_verification_jobs(
 
                 checksum_job->on_finished = on_finished;
                 checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
+                checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */
         }
 
-        if (verify == IMPORT_VERIFY_SIGNATURE) {
+        if (verify == IMPORT_VERIFY_SIGNATURE && !is_signature_file(fn)) {
                 _cleanup_free_ char *signature_url = NULL;
 
                 /* Queue job for the SHA256SUMS.gpg file for the image. */
@@ -306,7 +319,6 @@ int pull_make_verification_jobs(
 
         *ret_checksum_job = TAKE_PTR(checksum_job);
         *ret_signature_job = TAKE_PTR(signature_job);
-
         return 0;
 }
 
@@ -335,31 +347,33 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
 
         r = import_url_last_component(job->url, &fn);
         if (r < 0)
-                return log_oom();
+                return log_error_errno(r, "Failed to extract filename from URL '%s': %m", job->url);
 
         if (!filename_is_valid(fn))
                 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
                                        "Cannot verify checksum, could not determine server-side file name.");
 
-        line = strjoina(job->checksum, " *", fn, "\n");
+        if (is_checksum_file(fn) || is_signature_file(fn)) /* We cannot verify checksum files or signature files with a checksum file */
+                return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
+                                       "Cannot verify checksum/signature files via themselves.");
 
+        line = strjoina(job->checksum, " *", fn, "\n"); /* string for binary mode */
         p = memmem_safe(checksum_job->payload,
                         checksum_job->payload_size,
                         line,
                         strlen(line));
-
         if (!p) {
-                line = strjoina(job->checksum, "  ", fn, "\n");
+                line = strjoina(job->checksum, "  ", fn, "\n"); /* string for text mode */
-
                 p = memmem_safe(checksum_job->payload,
                                 checksum_job->payload_size,
                                 line,
                                 strlen(line));
         }
 
+        /* Only counts if found at beginning of a line */
         if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n'))
                 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
-                                       "DOWNLOAD INVALID: Checksum of %s file did not checkout, file has been tampered with.", fn);
+                                       "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", fn);
 
         log_info("SHA256 checksum of %s is valid.", job->url);
         return 1;
@@ -472,10 +486,10 @@ static int verify_gpg(
         pid = 0;
         if (r < 0)
                 goto finish;
-        if (r != EXIT_SUCCESS) {
+        if (r != EXIT_SUCCESS)
-                log_error("DOWNLOAD INVALID: Signature verification failed.");
+                r = log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
-                r = -EBADMSG;
+                                    "DOWNLOAD INVALID: Signature verification failed.");
-        } else {
+        else {
                 log_info("Signature verification succeeded.");
                 r = 0;
         }
@@ -491,6 +505,7 @@ finish:
 }
 
 int pull_verify(ImportVerify verify,
+                const char *checksum, /* Verify with literal checksum */
                 PullJob *main_job,
                 PullJob *checksum_job,
                 PullJob *signature_job,
@@ -499,37 +514,79 @@ int pull_verify(ImportVerify verify,
                 PullJob *roothash_signature_job,
                 PullJob *verity_job) {
 
+        _cleanup_free_ char *fn = NULL;
         VerificationStyle style;
-        PullJob *j;
+        PullJob *verify_job;
         int r;
 
+        assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+        assert((verify < 0) || !checksum);
         assert(main_job);
         assert(main_job->state == PULL_JOB_DONE);
 
-        if (verify == IMPORT_VERIFY_NO)
+        if (verify == IMPORT_VERIFY_NO) /* verification turned off */
                 return 0;
 
-        assert(main_job->calc_checksum);
+        if (checksum) {
-        assert(main_job->checksum);
+                /* Verification by literal checksum */
-        assert(checksum_job);
+                assert(!checksum_job);
-        assert(checksum_job->state == PULL_JOB_DONE);
+                assert(!signature_job);
+                assert(!settings_job);
+                assert(!roothash_job);
+                assert(!roothash_signature_job);
+                assert(!verity_job);
 
-        if (!checksum_job->payload || checksum_job->payload_size <= 0)
+                assert(main_job->calc_checksum);
-                return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+                assert(main_job->checksum);
-                                       "Checksum is empty, cannot verify.");
 
-        FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
+                if (!strcaseeq(checksum, main_job->checksum))
-                r = verify_one(checksum_job, j);
+                        return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
-                if (r < 0)
+                                               "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.",
-                        return r;
+                                               main_job->url);
+
+                return 0;
         }
 
-        if (verify == IMPORT_VERIFY_CHECKSUM)
+        r = import_url_last_component(main_job->url, &fn);
+        if (r < 0)
+                return log_error_errno(r, "Failed to extract filename from URL '%s': %m", main_job->url);
+
+        if (is_signature_file(fn))
+                return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
+                                       "Main download is a signature file, can't verify it.");
+
+        if (is_checksum_file(fn)) {
+                log_debug("Main download is a checksum file, can't validate its checksum with itself, skipping.");
+                verify_job = main_job;
+        } else {
+                PullJob *j;
+                assert(main_job->calc_checksum);
+                assert(main_job->checksum);
+                assert(checksum_job);
+                assert(checksum_job->state == PULL_JOB_DONE);
+
+                if (!checksum_job->payload || checksum_job->payload_size <= 0)
+                        return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                               "Checksum is empty, cannot verify.");
+
+                FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
+                        r = verify_one(checksum_job, j);
+                        if (r < 0)
+                                return r;
+                }
+
+                verify_job = checksum_job;
+        }
+
+        if (verify != IMPORT_VERIFY_SIGNATURE)
                 return 0;
 
-        r = verification_style_from_url(checksum_job->url, &style);
+        assert(verify_job);
+
+        r = verification_style_from_url(verify_job->url, &style);
         if (r < 0)
-                return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", checksum_job->url);
+                return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", verify_job->url);
 
         if (style == VERIFICATION_PER_DIRECTORY) {
                 assert(signature_job);
@@ -539,9 +596,9 @@ int pull_verify(ImportVerify verify,
                         return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
                                                "Signature is empty, cannot verify.");
 
-                return verify_gpg(checksum_job->payload, checksum_job->payload_size, signature_job->payload, signature_job->payload_size);
+                return verify_gpg(verify_job->payload, verify_job->payload_size, signature_job->payload, signature_job->payload_size);
         } else
-                return verify_gpg(checksum_job->payload, checksum_job->payload_size, NULL, 0);
+                return verify_gpg(verify_job->payload, verify_job->payload_size, NULL, 0);
 }
 
 int verification_style_from_url(const char *url, VerificationStyle *ret) {
@@ -595,3 +652,28 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) {
 
         return 1;
 }
+
+bool pull_validate_local(const char *name, PullFlags flags) {
+
+        if (FLAGS_SET(flags, PULL_DIRECT))
+                return path_is_valid(name);
+
+        return hostname_is_valid(name, 0);
+}
+
+int pull_url_needs_checksum(const char *url) {
+        _cleanup_free_ char *fn = NULL;
+        int r;
+
+        /* Returns true if we need to validate this resource via a hash value. This returns true for all
+         * files — except for gpg signature files and SHA256SUMS files and the like, which are validated with
+         * a validation tool like gpg. */
+
+        r = import_url_last_component(url, &fn);
+        if (r == -EADDRNOTAVAIL) /* no last component? then let's assume it's not a signature/checksum file */
+                return false;
+        if (r < 0)
+                return r;
+
+        return !is_checksum_file(fn) && !is_signature_file(fn);
+}

src/import/pull-common.h

@@ -7,27 +7,31 @@
 #include "pull-job.h"
 
 typedef enum PullFlags {
-        PULL_FORCE              = 1 << 0,  /* replace existing image */
+        PULL_FORCE              = 1 << 0, /* replace existing image */
-        PULL_SETTINGS           = 1 << 1,  /* .nspawn settings file */
+        PULL_READ_ONLY          = 1 << 1, /* make generated image read-only */
-        PULL_ROOTHASH           = 1 << 2,  /* only for raw: .roothash file for verity */
+        PULL_SETTINGS           = 1 << 1, /* download .nspawn settings file */
-        PULL_ROOTHASH_SIGNATURE = 1 << 3,  /* only for raw: .roothash.p7s file for verity */
+        PULL_ROOTHASH           = 1 << 2, /* only for raw: download .roothash file for verity */
-        PULL_VERITY             = 1 << 4,  /* only for raw: .verity file for verity */
+        PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: download .roothash.p7s file for verity */
+        PULL_VERITY             = 1 << 4, /* only for raw: download .verity file for verity */
+        PULL_BTRFS_SUBVOL       = 1 << 2, /* tar: preferably create images as btrfs subvols */
+        PULL_BTRFS_QUOTA        = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
+        PULL_CONVERT_QCOW2      = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
+        PULL_DIRECT             = 1 << 5, /* download without rename games */
+        PULL_SYNC               = 1 << 6, /* fsync() right before we are done */
 
         /* The supported flags for the tar and the raw pulling */
-        PULL_FLAGS_MASK_TAR     = PULL_FORCE|PULL_SETTINGS,
+        PULL_FLAGS_MASK_TAR     = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC,
-        PULL_FLAGS_MASK_RAW     = PULL_FORCE|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY,
+        PULL_FLAGS_MASK_RAW     = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC,
 } PullFlags;
 
-int pull_make_local_copy(const char *final, const char *root, const char *local, PullFlags flags);
-
 int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags);
 
 int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
 
-int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
+int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata);
-int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
+int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
 
-int pull_verify(ImportVerify verify, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
+int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
 
 typedef enum VerificationStyle {
         VERIFICATION_PER_FILE,      /* SuSE-style ".sha256" files with inline gpg signature */
@@ -39,3 +43,7 @@ typedef enum VerificationStyle {
 int verification_style_from_url(const char *url, VerificationStyle *style);
 
 int pull_job_restart_with_sha256sum(PullJob *job, char **ret);
+
+bool pull_validate_local(const char *name, PullFlags flags);
+
+int pull_url_needs_checksum(const char *url);

src/import/pull-job.c

@@ -7,6 +7,7 @@
 #include "alloc-util.h"
 #include "fd-util.h"
 #include "format-util.h"
+#include "fs-util.h"
 #include "gcrypt-util.h"
 #include "hexdecoct.h"
 #include "import-util.h"
@@ -19,15 +20,25 @@
 #include "strv.h"
 #include "xattr-util.h"
 
+void pull_job_close_disk_fd(PullJob *j) {
+        if (!j)
+                return;
+
+        if (j->close_disk_fd)
+                safe_close(j->disk_fd);
+
+        j->disk_fd = -1;
+}
+
 PullJob* pull_job_unref(PullJob *j) {
         if (!j)
                 return NULL;
 
+        pull_job_close_disk_fd(j);
+
         curl_glue_remove_and_free(j->glue, j->curl);
         curl_slist_free_all(j->request_header);
 
-        safe_close(j->disk_fd);
-
         import_compress_free(&j->compress);
 
         if (j->checksum_context)
@@ -116,15 +127,13 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
                 return;
 
         if (result != CURLE_OK) {
-                log_error("Transfer failed: %s", curl_easy_strerror(result));
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Transfer failed: %s", curl_easy_strerror(result));
-                r = -EIO;
                 goto finish;
         }
 
         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
         if (code != CURLE_OK) {
-                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
-                r = -EIO;
                 goto finish;
         } else if (status == 304) {
                 log_info("Image already downloaded. Skipping download.");
@@ -150,8 +159,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 
                                 code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
                                 if (code != CURLE_OK) {
-                                        log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
+                                        r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
-                                        r = -EIO;
                                         goto finish;
                                 }
 
@@ -160,25 +168,23 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
                         }
                 }
 
-                log_error("HTTP request to %s failed with code %li.", j->url, status);
+                r = log_error_errno(
-                r = -EIO;
+                                status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */
+                                "HTTP request to %s failed with code %li.", j->url, status);
                 goto finish;
         } else if (status < 200) {
-                log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status);
-                r = -EIO;
                 goto finish;
         }
 
         if (j->state != PULL_JOB_RUNNING) {
-                log_error("Premature connection termination.");
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.");
-                r = -EIO;
                 goto finish;
         }
 
         if (j->content_length != UINT64_MAX &&
             j->content_length != j->written_compressed) {
-                log_error("Download truncated.");
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated.");
-                r = -EIO;
                 goto finish;
         }
 
@@ -187,8 +193,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 
                 k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
                 if (!k) {
-                        log_error("Failed to get checksum.");
+                        r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
-                        r = -EIO;
                         goto finish;
                 }
 
@@ -201,28 +206,57 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
                 log_debug("SHA256 of %s is %s.", j->url, j->checksum);
         }
 
-        if (j->disk_fd >= 0 && j->allow_sparse) {
+        /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no
-                /* Make sure the file size is right, in case the file was
+         * offset is specified, which indicates we only own the file partially) */
-                 * sparse and we just seeked for the last part */
 
-                if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
+        if (j->disk_fd >= 0) {
-                        r = log_error_errno(errno, "Failed to truncate file: %m");
-                        goto finish;
-                }
 
-                if (j->etag)
+                if (S_ISREG(j->disk_stat.st_mode)) {
-                        (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
-                if (j->url)
-                        (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
 
-                if (j->mtime != 0) {
+                        if (j->offset == UINT64_MAX) {
-                        struct timespec ut[2];
 
-                        timespec_store(&ut[0], j->mtime);
+                                if (j->written_compressed > 0) {
-                        ut[1] = ut[0];
+                                        /* Make sure the file size is right, in case the file was sparse and we just seeked
-                        (void) futimens(j->disk_fd, ut);
+                                         * for the last part */
+                                        if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
+                                                r = log_error_errno(errno, "Failed to truncate file: %m");
+                                                goto finish;
+                                        }
+                                }
 
-                        (void) fd_setcrtime(j->disk_fd, j->mtime);
+                                if (j->etag)
+                                        (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
+                                if (j->url)
+                                        (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
+
+                                if (j->mtime != 0) {
+                                        struct timespec ut;
+
+                                        timespec_store(&ut, j->mtime);
+
+                                        if (futimens(j->disk_fd, (struct timespec[]) { ut, ut }) < 0)
+                                                log_debug_errno(errno, "Failed to adjust atime/mtime of created image, ignoring: %m");
+
+                                        r = fd_setcrtime(j->disk_fd, j->mtime);
+                                        if (r < 0)
+                                                log_debug_errno(r, "Failed to adjust crtime of created image, ignoring: %m");
+                                }
+                        }
+
+                        if (j->sync) {
+                                r = fsync_full(j->disk_fd);
+                                if (r < 0) {
+                                        log_error_errno(r, "Failed to synchronize file to disk: %m");
+                                        goto finish;
+                                }
+                        }
+
+                } else if (S_ISBLK(j->disk_stat.st_mode) && j->sync) {
+
+                        if (fsync(j->disk_fd) < 0) {
+                                r = log_error_errno(errno, "Failed to synchronize block device: %m");
+                                goto finish;
+                        }
                 }
         }
 
@@ -234,37 +268,46 @@ finish:
 
 static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
         PullJob *j = userdata;
-        ssize_t n;
+        bool too_much = false;
+        int r;
 
         assert(j);
         assert(p);
+        assert(sz > 0);
 
-        if (sz <= 0)
+        if (j->written_uncompressed > UINT64_MAX - sz)
-                return 0;
+                return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
 
-        if (j->written_uncompressed + sz < j->written_uncompressed)
+        if (j->written_uncompressed >= j->uncompressed_max) {
-                return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW),
+                too_much = true;
-                                       "File too large, overflow");
+                goto finish;
+        }
 
-        if (j->written_uncompressed + sz > j->uncompressed_max)
+        if (j->written_uncompressed + sz > j->uncompressed_max) {
-                return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
+                too_much = true;
-                                       "File overly large, refusing");
+                sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory
+                                                                     * already, we might as well write it to
+                                                                     * disk to the max */
+        }
 
         if (j->disk_fd >= 0) {
 
-                if (j->allow_sparse)
+                if (S_ISREG(j->disk_stat.st_mode)) {
-                        n = sparse_write(j->disk_fd, p, sz, 64);
+                        ssize_t n;
-                else {
-                        n = write(j->disk_fd, p, sz);
-                        if (n < 0)
-                                n = -errno;
-                }
-                if (n < 0)
-                        return log_error_errno((int) n, "Failed to write file: %m");
-                if ((size_t) n < sz)
-                        return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
-        } else {
 
+                        n = sparse_write(j->disk_fd, p, sz, 64);
+                        if (n < 0)
+                                return log_error_errno((int) n, "Failed to write file: %m");
+                        if ((size_t) n < sz)
+                                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
+                } else {
+                        r = loop_write(j->disk_fd, p, sz, false);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to write file: %m");
+                }
+        }
+
+        if (j->disk_fd < 0 || j->force_memory) {
                 if (!GREEDY_REALLOC(j->payload, j->payload_size + sz))
                         return log_oom();
 
@@ -274,6 +317,10 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata)
 
         j->written_uncompressed += sz;
 
+finish:
+        if (too_much)
+                return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing.");
+
         return 0;
 }
 
@@ -321,15 +368,12 @@ static int pull_job_open_disk(PullJob *j) {
         }
 
         if (j->disk_fd >= 0) {
-                /* Check if we can do sparse files */
+                if (fstat(j->disk_fd, &j->disk_stat) < 0)
+                        return log_error_errno(errno, "Failed to stat disk file: %m");
 
-                if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
+                if (j->offset != UINT64_MAX) {
-                        j->allow_sparse = true;
+                        if (lseek(j->disk_fd, j->offset, SEEK_SET) == (off_t) -1)
-                else {
-                        if (errno != ESPIPE)
                                 return log_error_errno(errno, "Failed to seek on file descriptor: %m");
-
-                        j->allow_sparse = false;
                 }
         }
 
@@ -461,8 +505,7 @@ static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb
 
         code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
         if (code != CURLE_OK) {
-                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
+                r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
-                r = -EIO;
                 goto fail;
         }
 
@@ -576,7 +619,12 @@ static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_o
         return 0;
 }
 
-int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
+int pull_job_new(
+                PullJob **ret,
+                const char *url,
+                CurlGlue *glue,
+                void *userdata) {
+
         _cleanup_(pull_job_unrefp) PullJob *j = NULL;
         _cleanup_free_ char *u = NULL;
 
@@ -595,6 +643,7 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata)
         *j = (PullJob) {
                 .state = PULL_JOB_INIT,
                 .disk_fd = -1,
+                .close_disk_fd = true,
                 .userdata = userdata,
                 .glue = glue,
                 .content_length = UINT64_MAX,
@@ -602,6 +651,8 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata)
                 .compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
                 .uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
                 .url = TAKE_PTR(u),
+                .offset = UINT64_MAX,
+                .sync = true,
         };
 
         *ret = TAKE_PTR(j);

src/import/pull-job.h

@@ -2,10 +2,12 @@
 #pragma once
 
 #include <gcrypt.h>
+#include <sys/stat.h>
 
 #include "curl-util.h"
 #include "import-compress.h"
 #include "macro.h"
+#include "pull-common.h"
 
 typedef struct PullJob PullJob;
 
@@ -51,6 +53,7 @@ struct PullJob {
         uint64_t content_length;
         uint64_t written_compressed;
         uint64_t written_uncompressed;
+        uint64_t offset;
 
         uint64_t uncompressed_max;
         uint64_t compressed_max;
@@ -59,6 +62,8 @@ struct PullJob {
         size_t payload_size;
 
         int disk_fd;
+        bool close_disk_fd;
+        struct stat disk_stat;
 
         usec_t mtime;
 
@@ -68,12 +73,12 @@ struct PullJob {
         usec_t start_usec;
         usec_t last_status_usec;
 
-        bool allow_sparse;
-
         bool calc_checksum;
         gcry_md_hd_t checksum_context;
 
         char *checksum;
+        bool sync;
+        bool force_memory;
 };
 
 int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata);
@@ -83,4 +88,6 @@ int pull_job_begin(PullJob *j);
 
 void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result);
 
+void pull_job_close_disk_fd(PullJob *j);
+
 DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref);

src/import/pull-raw.c

@@ -15,6 +15,7 @@
 #include "hostname-util.h"
 #include "import-common.h"
 #include "import-util.h"
+#include "install-file.h"
 #include "macro.h"
 #include "mkdir.h"
 #include "path-util.h"
@@ -46,6 +47,8 @@ struct RawPull {
         ImportVerify verify;
         char *image_root;
 
+        uint64_t offset;
+
         PullJob *raw_job;
         PullJob *checksum_job;
         PullJob *signature_job;
@@ -57,7 +60,8 @@ struct RawPull {
         RawPullFinished on_finished;
         void *userdata;
 
-        char *local;
+        char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the
+                      * machine name of the final copy we make */
 
         char *final_path;
         char *temp_path;
@@ -73,6 +77,8 @@ struct RawPull {
 
         char *verity_path;
         char *verity_temp_path;
+
+        char *checksum;
 };
 
 RawPull* raw_pull_unref(RawPull *i) {
@@ -103,6 +109,7 @@ RawPull* raw_pull_unref(RawPull *i) {
         free(i->verity_path);
         free(i->image_root);
         free(i->local);
+        free(i->checksum);
 
         return mfree(i);
 }
@@ -148,6 +155,7 @@ int raw_pull_new(
                 .image_root = TAKE_PTR(root),
                 .event = TAKE_PTR(e),
                 .glue = TAKE_PTR(g),
+                .offset = UINT64_MAX,
         };
 
         i->glue->on_finished = pull_job_curl_on_finished;
@@ -230,12 +238,20 @@ static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 }
 
 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
+        _cleanup_(unlink_and_freep) char *t = NULL;
         _cleanup_close_ int converted_fd = -1;
-        _cleanup_free_ char *t = NULL;
+        _cleanup_free_ char *f = NULL;
         int r;
 
         assert(i);
         assert(i->raw_job);
+        assert(!FLAGS_SET(i->flags, PULL_DIRECT));
+
+        if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2))
+                return 0;
+
+        assert(i->final_path);
+        assert(i->raw_job->close_disk_fd);
 
         r = qcow2_detect(i->raw_job->disk_fd);
         if (r < 0)
@@ -244,32 +260,35 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) {
                 return 0;
 
         /* This is a QCOW2 image, let's convert it */
-        r = tempfn_random(i->final_path, NULL, &t);
+        r = tempfn_random(i->final_path, NULL, &f);
         if (r < 0)
                 return log_oom();
 
-        converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+        converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
         if (converted_fd < 0)
-                return log_error_errno(errno, "Failed to create %s: %m", t);
+                return log_error_errno(errno, "Failed to create %s: %m", f);
+
+        t = TAKE_PTR(f);
 
         (void) import_set_nocow_and_log(converted_fd, t);
 
         log_info("Unpacking QCOW2 file.");
 
         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
-        if (r < 0) {
+        if (r < 0)
-                (void) unlink(t);
                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
-        }
 
-        (void) unlink(i->temp_path);
+        unlink_and_free(i->temp_path);
-        free_and_replace(i->temp_path, t);
+        i->temp_path = TAKE_PTR(t);
         CLOSE_AND_REPLACE(i->raw_job->disk_fd, converted_fd);
 
         return 1;
 }
 
-static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
+static int raw_pull_determine_path(
+                RawPull *i,
+                const char *suffix,
+                char **field /* input + output (!) */) {
         int r;
 
         assert(i);
@@ -290,7 +309,7 @@ static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field)
 static int raw_pull_copy_auxiliary_file(
                 RawPull *i,
                 const char *suffix,
-                char **path) {
+                char **path /* input + output (!) */) {
 
         const char *local;
         int r;
@@ -305,7 +324,14 @@ static int raw_pull_copy_auxiliary_file(
 
         local = strjoina(i->image_root, "/", i->local, suffix);
 
-        r = copy_file_atomic(*path, local, 0644, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
+        r = copy_file_atomic(
+                        *path,
+                        local,
+                        0644,
+                        0, 0,
+                        COPY_REFLINK |
+                        (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
+                        (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
         if (r == -EEXIST)
                 log_warning_errno(r, "File %s already exists, not replacing.", local);
         else if (r == -ENOENT)
@@ -319,13 +345,15 @@ static int raw_pull_copy_auxiliary_file(
 }
 
 static int raw_pull_make_local_copy(RawPull *i) {
-        _cleanup_free_ char *tp = NULL;
+        _cleanup_(unlink_and_freep) char *tp = NULL;
+        _cleanup_free_ char *f = NULL;
         _cleanup_close_ int dfd = -1;
         const char *p;
         int r;
 
         assert(i);
         assert(i->raw_job);
+        assert(!FLAGS_SET(i->flags, PULL_DIRECT));
 
         if (!i->local)
                 return 0;
@@ -342,6 +370,7 @@ static int raw_pull_make_local_copy(RawPull *i) {
                 /* We freshly downloaded the image, use it */
 
                 assert(i->raw_job->disk_fd >= 0);
+                assert(i->offset == UINT64_MAX);
 
                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
@@ -349,38 +378,38 @@ static int raw_pull_make_local_copy(RawPull *i) {
 
         p = strjoina(i->image_root, "/", i->local, ".raw");
 
-        if (FLAGS_SET(i->flags, PULL_FORCE))
+        r = tempfn_random(p, NULL, &f);
-                (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
-
-        r = tempfn_random(p, NULL, &tp);
         if (r < 0)
                 return log_oom();
 
-        dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+        dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
         if (dfd < 0)
                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 
+        tp = TAKE_PTR(f);
+
         /* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs,
          * since it reduces fragmentation caused by not allowing in-place writes. */
         (void) import_set_nocow_and_log(dfd, tp);
 
         r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK);
-        if (r < 0) {
+        if (r < 0)
-                (void) unlink(tp);
                 return log_error_errno(r, "Failed to make writable copy of image: %m");
-        }
 
         (void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME);
         (void) copy_xattr(i->raw_job->disk_fd, dfd, 0);
 
         dfd = safe_close(dfd);
 
-        r = rename(tp, p);
+        r = install_file(AT_FDCWD, tp,
-        if (r < 0)  {
+                         AT_FDCWD, p,
-                r = log_error_errno(errno, "Failed to move writable image into place: %m");
+                         (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
-                (void) unlink(tp);
+                         (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
-                return r;
+                         (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
-        }
+        if (r < 0)
+                return log_error_errno(errno, "Failed to move local image into place '%s': %m", p);
+
+        tp = mfree(tp);
 
         log_info("Created new local image '%s'.", i->local);
 
@@ -442,9 +471,10 @@ static int raw_pull_rename_auxiliary_file(
         int r;
 
         assert(i);
-        assert(temp_path);
-        assert(suffix);
         assert(path);
+        assert(temp_path);
+        assert(*temp_path);
+        assert(suffix);
 
         /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
          * incorporate it in the file name if we can */
@@ -453,61 +483,78 @@ static int raw_pull_rename_auxiliary_file(
         if (r < 0)
                 return r;
 
-        r = import_make_read_only(*temp_path);
+        r = install_file(
+                        AT_FDCWD, *temp_path,
+                        AT_FDCWD, *path,
+                        INSTALL_READ_ONLY|
+                        (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
         if (r < 0)
-                return r;
+                return log_error_errno(r, "Failed to move '%s' into place: %m", *path);
-
-        r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
-        if (r < 0)
-                return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
 
         *temp_path = mfree(*temp_path);
-
         return 1;
 }
 
 static void raw_pull_job_on_finished(PullJob *j) {
         RawPull *i;
+        PullJob *jj;
         int r;
 
         assert(j);
         assert(j->userdata);
 
         i = j->userdata;
-        if (j == i->settings_job) {
-                if (j->error != 0)
-                        log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
-        } else if (j == i->roothash_job) {
-                if (j->error != 0)
-                        log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
-        } else if (j == i->roothash_signature_job) {
-                if (j->error != 0)
-                        log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without.");
-        } else if (j == i->verity_job) {
-                if (j->error != 0)
-                        log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without. %s", j->url);
-        } else if (j->error != 0 && j != i->signature_job) {
-                if (j == i->checksum_job)
-                        log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
-                else
-                        log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 
-                r = j->error;
+        if (j->error != 0) {
-                goto finish;
+                /* Only the main job and the checksum job are fatal if they fail. The other fails are just
+                 * "decoration", that we'll download if we can. The signature job isn't fatal here because we
+                 * might not actually need it in case Suse style signatures are used, that are inline in the
+                 * checksum file. */
+
+                if (j == i->raw_job) {
+                        if (j->error == ENOMEDIUM) /* HTTP 404 */
+                                r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
+                        else
+                                r = log_error_errno(j->error, "Failed to retrieve image file.");
+                        goto finish;
+                } else if (j == i->checksum_job) {
+                        r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
+                        goto finish;
+                } else if (j == i->signature_job)
+                        log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
+                else if (j == i->settings_job)
+                        log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
+                else if (j == i->roothash_job)
+                        log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
+                else if (j == i->roothash_signature_job)
+                        log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without.");
+                else if (j == i->verity_job)
+                        log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without.");
+                else
+                        assert_not_reached();
         }
 
         /* This is invoked if either the download completed successfully, or the download was skipped because
          * we already have the etag. In this case ->etag_exists is true.
          *
-         * We only do something when we got all three files */
+         * We only do something when we got all files */
 
         if (!raw_pull_is_done(i))
                 return;
 
         if (i->signature_job && i->signature_job->error != 0) {
                 VerificationStyle style;
+                PullJob *verify_job;
 
-                r = verification_style_from_url(i->checksum_job->url, &style);
+                /* The signature job failed. Let's see if we actually need it */
+
+                verify_job = i->checksum_job ?: i->raw_job; /* if the checksum job doesn't exist this must be
+                                                             * because the main job is the checksum file
+                                                             * itself */
+
+                assert(verify_job);
+
+                r = verification_style_from_url(verify_job->url, &style);
                 if (r < 0) {
                         log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
                         goto finish;
@@ -517,32 +564,21 @@ static void raw_pull_job_on_finished(PullJob *j) {
                                                             * in per-directory verification mode, since only
                                                             * then the signature is detached, and thus a file
                                                             * of its own. */
-                        log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
+                        r = log_error_errno(i->signature_job->error,
-                        r = i->signature_job->error;
+                                            "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
                         goto finish;
                 }
         }
 
-        if (i->settings_job)
+        /* Let's close these auxiliary files now, we don't need access to them anymore. */
-                i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
+        FOREACH_POINTER(jj, i->settings_job, i->roothash_job, i->roothash_signature_job, i->verity_job)
-        if (i->roothash_job)
+                pull_job_close_disk_fd(jj);
-                i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
-        if (i->roothash_signature_job)
-                i->roothash_signature_job->disk_fd = safe_close(i->roothash_signature_job->disk_fd);
-        if (i->verity_job)
-                i->verity_job->disk_fd = safe_close(i->verity_job->disk_fd);
-
-        r = raw_pull_determine_path(i, ".raw", &i->final_path);
-        if (r < 0)
-                goto finish;
 
         if (!i->raw_job->etag_exists) {
-                /* This is a new download, verify it, and move it into place */
-                assert(i->raw_job->disk_fd >= 0);
-
                 raw_pull_report_progress(i, RAW_VERIFYING);
 
                 r = pull_verify(i->verify,
+                                i->checksum,
                                 i->raw_job,
                                 i->checksum_job,
                                 i->signature_job,
@@ -552,51 +588,92 @@ static void raw_pull_job_on_finished(PullJob *j) {
                                 i->verity_job);
                 if (r < 0)
                         goto finish;
+        }
 
-                raw_pull_report_progress(i, RAW_UNPACKING);
+        if (i->flags & PULL_DIRECT) {
-
+                assert(!i->settings_job);
-                r = raw_pull_maybe_convert_qcow2(i);
+                assert(!i->roothash_job);
-                if (r < 0)
+                assert(!i->roothash_signature_job);
-                        goto finish;
+                assert(!i->verity_job);
 
                 raw_pull_report_progress(i, RAW_FINALIZING);
 
-                if (i->raw_job->etag) {
+                if (i->local) {
-                        /* Only make a read-only copy if ETag header is set. */
+                        r = install_file(AT_FDCWD, i->local,
-                        r = import_make_read_only_fd(i->raw_job->disk_fd);
+                                         AT_FDCWD, NULL,
-                        if (r < 0)
+                                         ((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) |
-                                goto finish;
+                                         (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
-
-                        r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
                         if (r < 0) {
-                                log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path);
+                                log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local);
                                 goto finish;
                         }
                 }
+        } else {
+                r = raw_pull_determine_path(i, ".raw", &i->final_path);
+                if (r < 0)
+                        goto finish;
 
-                i->temp_path = mfree(i->temp_path);
+                if (!i->raw_job->etag_exists) {
+                        /* This is a new download, verify it, and move it into place */
 
-                if (i->roothash_job &&
+                        assert(i->temp_path);
-                    i->roothash_job->error == 0) {
+                        assert(i->final_path);
-                        r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
+
+                        raw_pull_report_progress(i, RAW_UNPACKING);
+
+                        r = raw_pull_maybe_convert_qcow2(i);
                         if (r < 0)
                                 goto finish;
+
+                        raw_pull_report_progress(i, RAW_FINALIZING);
+
+                        r = install_file(AT_FDCWD, i->temp_path,
+                                         AT_FDCWD, i->final_path,
+                                         INSTALL_READ_ONLY|
+                                         (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
+                        if (r < 0) {
+                                log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path);
+                                goto finish;
+                        }
+
+                        i->temp_path = mfree(i->temp_path);
+
+                        if (i->settings_job &&
+                            i->settings_job->error == 0) {
+                                r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
+                                if (r < 0)
+                                        goto finish;
+                        }
+
+                        if (i->roothash_job &&
+                            i->roothash_job->error == 0) {
+                                r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
+                                if (r < 0)
+                                        goto finish;
+                        }
+
+                        if (i->roothash_signature_job &&
+                            i->roothash_signature_job->error == 0) {
+                                r = raw_pull_rename_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_temp_path, &i->roothash_signature_path);
+                                if (r < 0)
+                                        goto finish;
+                        }
+
+                        if (i->verity_job &&
+                            i->verity_job->error == 0) {
+                                r = raw_pull_rename_auxiliary_file(i, ".verity", &i->verity_temp_path, &i->verity_path);
+                                if (r < 0)
+                                        goto finish;
+                        }
                 }
 
-                if (i->settings_job &&
+                raw_pull_report_progress(i, RAW_COPYING);
-                    i->settings_job->error == 0) {
+
-                        r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
+                r = raw_pull_make_local_copy(i);
-                        if (r < 0)
+                if (r < 0)
-                                goto finish;
+                        goto finish;
-                }
         }
 
-        raw_pull_report_progress(i, RAW_COPYING);
-
-        r = raw_pull_make_local_copy(i);
-        if (r < 0)
-                goto finish;
-
         r = 0;
 
 finish:
@@ -610,7 +687,7 @@ static int raw_pull_job_on_open_disk_generic(
                 RawPull *i,
                 PullJob *j,
                 const char *extra,
-                char **temp_path) {
+                char **temp_path /* input + output */) {
 
         int r;
 
@@ -619,6 +696,8 @@ static int raw_pull_job_on_open_disk_generic(
         assert(extra);
         assert(temp_path);
 
+        assert(!FLAGS_SET(i->flags, PULL_DIRECT));
+
         if (!*temp_path) {
                 r = tempfn_random_child(i->image_root, extra, temp_path);
                 if (r < 0)
@@ -643,12 +722,34 @@ static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 
         i = j->userdata;
         assert(i->raw_job == j);
+        assert(j->disk_fd < 0);
 
-        r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
+        if (i->flags & PULL_DIRECT) {
-        if (r < 0)
+
-                return r;
+                if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */
+                        j->disk_fd = STDOUT_FILENO;
+                        j->close_disk_fd = false;
+                        return 0;
+                }
+
+                (void) mkdir_parents_label(i->local, 0700);
+
+                j->disk_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
+                if (j->disk_fd < 0)
+                        return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
+
+                if (i->offset == UINT64_MAX)
+                        (void) import_set_nocow_and_log(j->disk_fd, i->local);
+
+        } else {
+                r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
+                if (r < 0)
+                        return r;
+
+                assert(i->offset == UINT64_MAX);
+                (void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
+        }
 
-        (void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
         return 0;
 }
 
@@ -715,20 +816,29 @@ int raw_pull_start(
                 RawPull *i,
                 const char *url,
                 const char *local,
+                uint64_t offset,
+                uint64_t size_max,
                 PullFlags flags,
-                ImportVerify verify) {
+                ImportVerify verify,
+                const char *checksum) {
 
+        PullJob *j;
         int r;
 
         assert(i);
-        assert(verify < _IMPORT_VERIFY_MAX);
+        assert(url);
-        assert(verify >= 0);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+        assert((verify < 0) || !checksum);
         assert(!(flags & ~PULL_FLAGS_MASK_RAW));
+        assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT));
+        assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT));
+        assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum);
 
         if (!http_url_is_valid(url))
                 return -EINVAL;
 
-        if (local && !hostname_is_valid(local, 0))
+        if (local && !pull_validate_local(local, flags))
                 return -EINVAL;
 
         if (i->raw_job)
@@ -738,6 +848,10 @@ int raw_pull_start(
         if (r < 0)
                 return r;
 
+        r = free_and_strdup(&i->checksum, checksum);
+        if (r < 0)
+                return r;
+
         i->flags = flags;
         i->verify = verify;
 
@@ -748,98 +862,121 @@ int raw_pull_start(
 
         i->raw_job->on_finished = raw_pull_job_on_finished;
         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
-        i->raw_job->on_progress = raw_pull_job_on_progress;
-        i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 
-        r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
+        if (checksum)
-        if (r < 0)
+                i->raw_job->calc_checksum = true;
-                return r;
+        else if (verify != IMPORT_VERIFY_NO) {
+                /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its
+                 * signature, which we let gpg verify instead. */
 
-        r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
+                r = pull_url_needs_checksum(url);
+                if (r < 0)
+                        return r;
+
+                i->raw_job->calc_checksum = r;
+                i->raw_job->force_memory = true; /* make sure this is both written to disk if that's
+                                                  * requested and into memory, since we need to verify it */
+        }
+
+        if (size_max != UINT64_MAX)
+                i->raw_job->uncompressed_max = size_max;
+        if (offset != UINT64_MAX)
+                i->raw_job->offset = i->offset = offset;
+
+        if (!FLAGS_SET(flags, PULL_DIRECT)) {
+                r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
+                if (r < 0)
+                        return r;
+        }
+
+        r = pull_make_verification_jobs(
+                        &i->checksum_job,
+                        &i->signature_job,
+                        verify,
+                        i->checksum,
+                        url,
+                        i->glue,
+                        raw_pull_job_on_finished,
+                        i);
         if (r < 0)
                 return r;
 
         if (FLAGS_SET(flags, PULL_SETTINGS)) {
-                r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
+                r = pull_make_auxiliary_job(
+                                &i->settings_job,
+                                url,
+                                raw_strip_suffixes,
+                                ".nspawn",
+                                verify,
+                                i->glue,
+                                raw_pull_job_on_open_disk_settings,
+                                raw_pull_job_on_finished,
+                                i);
                 if (r < 0)
                         return r;
-
-                i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
-                i->settings_job->on_progress = raw_pull_job_on_progress;
-                i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
         }
 
         if (FLAGS_SET(flags, PULL_ROOTHASH)) {
-                r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
+                r = pull_make_auxiliary_job(
+                                &i->roothash_job,
+                                url,
+                                raw_strip_suffixes,
+                                ".roothash",
+                                verify,
+                                i->glue,
+                                raw_pull_job_on_open_disk_roothash,
+                                raw_pull_job_on_finished,
+                                i);
                 if (r < 0)
                         return r;
-
-                i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
-                i->roothash_job->on_progress = raw_pull_job_on_progress;
-                i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
         }
 
         if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) {
-                r = pull_make_auxiliary_job(&i->roothash_signature_job, url, raw_strip_suffixes, ".roothash.p7s", i->glue, raw_pull_job_on_finished, i);
+                r = pull_make_auxiliary_job(
+                                &i->roothash_signature_job,
+                                url,
+                                raw_strip_suffixes,
+                                ".roothash.p7s",
+                                verify,
+                                i->glue,
+                                raw_pull_job_on_open_disk_roothash_signature,
+                                raw_pull_job_on_finished,
+                                i);
                 if (r < 0)
                         return r;
-
-                i->roothash_signature_job->on_open_disk = raw_pull_job_on_open_disk_roothash_signature;
-                i->roothash_signature_job->on_progress = raw_pull_job_on_progress;
-                i->roothash_signature_job->calc_checksum = verify != IMPORT_VERIFY_NO;
         }
 
         if (FLAGS_SET(flags, PULL_VERITY)) {
-                r = pull_make_auxiliary_job(&i->verity_job, url, raw_strip_suffixes, ".verity", i->glue, raw_pull_job_on_finished, i);
+                r = pull_make_auxiliary_job(
-                if (r < 0)
+                                &i->verity_job,
-                        return r;
+                                url,
-
+                                raw_strip_suffixes,
-                i->verity_job->on_open_disk = raw_pull_job_on_open_disk_verity;
+                                ".verity",
-                i->verity_job->on_progress = raw_pull_job_on_progress;
+                                verify,
-                i->verity_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+                                i->glue,
-        }
+                                raw_pull_job_on_open_disk_verity,
-
+                                raw_pull_job_on_finished,
-        r = pull_job_begin(i->raw_job);
+                                i);
-        if (r < 0)
-                return r;
-
-        if (i->checksum_job) {
-                i->checksum_job->on_progress = raw_pull_job_on_progress;
-                i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
-
-                r = pull_job_begin(i->checksum_job);
                 if (r < 0)
                         return r;
         }
 
-        if (i->signature_job) {
+        FOREACH_POINTER(j,
-                i->signature_job->on_progress = raw_pull_job_on_progress;
+                        i->raw_job,
+                        i->checksum_job,
+                        i->signature_job,
+                        i->settings_job,
+                        i->roothash_job,
+                        i->roothash_signature_job,
+                        i->verity_job) {
 
-                r = pull_job_begin(i->signature_job);
+                if (!j)
-                if (r < 0)
+                        continue;
-                        return r;
-        }
 
-        if (i->settings_job) {
+                j->on_progress = raw_pull_job_on_progress;
-                r = pull_job_begin(i->settings_job);
+                j->sync = FLAGS_SET(flags, PULL_SYNC);
-                if (r < 0)
-                        return r;
-        }
 
-        if (i->roothash_job) {
+                r = pull_job_begin(j);
-                r = pull_job_begin(i->roothash_job);
-                if (r < 0)
-                        return r;
-        }
-
-        if (i->roothash_signature_job) {
-                r = pull_job_begin(i->roothash_signature_job);
-                if (r < 0)
-                        return r;
-        }
-
-        if (i->verity_job) {
-                r = pull_job_begin(i->verity_job);
                 if (r < 0)
                         return r;
         }

src/import/pull-raw.h

@@ -16,4 +16,4 @@ RawPull* raw_pull_unref(RawPull *pull);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
 
-int raw_pull_start(RawPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
+int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum);

src/import/pull-tar.c

@@ -14,6 +14,7 @@
 #include "hostname-util.h"
 #include "import-common.h"
 #include "import-util.h"
+#include "install-file.h"
 #include "macro.h"
 #include "mkdir.h"
 #include "path-util.h"
@@ -25,6 +26,7 @@
 #include "string-util.h"
 #include "strv.h"
 #include "tmpfile-util.h"
+#include "user-util.h"
 #include "utf8.h"
 #include "util.h"
 #include "web-util.h"
@@ -61,6 +63,8 @@ struct TarPull {
 
         char *settings_path;
         char *settings_temp_path;
+
+        char *checksum;
 };
 
 TarPull* tar_pull_unref(TarPull *i) {
@@ -87,6 +91,7 @@ TarPull* tar_pull_unref(TarPull *i) {
         free(i->settings_path);
         free(i->image_root);
         free(i->local);
+        free(i->checksum);
 
         return mfree(i);
 }
@@ -194,7 +199,10 @@ static void tar_pull_report_progress(TarPull *i, TarProgress p) {
         log_debug("Combined progress %u%%", percent);
 }
 
-static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
+static int tar_pull_determine_path(
+                TarPull *i,
+                const char *suffix,
+                char **field /* input + output (!) */) {
         int r;
 
         assert(i);
@@ -213,6 +221,8 @@ static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field)
 }
 
 static int tar_pull_make_local_copy(TarPull *i) {
+        _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
+        const char *p;
         int r;
 
         assert(i);
@@ -221,9 +231,38 @@ static int tar_pull_make_local_copy(TarPull *i) {
         if (!i->local)
                 return 0;
 
-        r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->flags);
+        assert(i->final_path);
+
+        p = prefix_roota(i->image_root, i->local);
+
+        r = tempfn_random(p, NULL, &t);
         if (r < 0)
-                return r;
+                return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
+
+        if (i->flags & PULL_BTRFS_SUBVOL)
+                r = btrfs_subvol_snapshot(
+                                i->final_path,
+                                t,
+                                (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
+                                BTRFS_SNAPSHOT_FALLBACK_COPY|
+                                BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
+                                BTRFS_SNAPSHOT_RECURSIVE);
+        else
+                r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS);
+        if (r < 0)
+                return log_error_errno(r, "Failed to create local image: %m");
+
+        r = install_file(AT_FDCWD, t,
+                         AT_FDCWD, p,
+                         (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
+                         (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
+                         (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
+        if (r < 0)
+                return log_error_errno(r, "Failed to install local image '%s': %m", p);
+
+        t = mfree(t);
+
+        log_info("Created new local image '%s'.", i->local);
 
         if (FLAGS_SET(i->flags, PULL_SETTINGS)) {
                 const char *local_settings;
@@ -235,7 +274,14 @@ static int tar_pull_make_local_copy(TarPull *i) {
 
                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 
-                r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
+                r = copy_file_atomic(
+                                i->settings_path,
+                                local_settings,
+                                0664,
+                                0, 0,
+                                COPY_REFLINK |
+                                (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
+                                (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
                 if (r == -EEXIST)
                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
                 else if (r == -ENOENT)
@@ -274,17 +320,22 @@ static void tar_pull_job_on_finished(PullJob *j) {
 
         i = j->userdata;
 
-        if (j == i->settings_job) {
+        if (j->error != 0) {
-                if (j->error != 0)
+                if (j == i->tar_job) {
+                        if (j->error == ENOMEDIUM) /* HTTP 404 */
+                                r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
+                        else
+                                r = log_error_errno(j->error, "Failed to retrieve image file.");
+                        goto finish;
+                } else if (j == i->checksum_job) {
+                        r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
+                        goto finish;
+                } else if (j == i->signature_job)
+                        log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
+                else if (j == i->settings_job)
                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
-        } else if (j->error != 0 && j != i->signature_job) {
-                if (j == i->checksum_job)
-                        log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
                 else
-                        log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
+                        assert("unexpected job");
-
-                r = j->error;
-                goto finish;
         }
 
         /* This is invoked if either the download completed successfully, or the download was skipped because
@@ -296,6 +347,8 @@ static void tar_pull_job_on_finished(PullJob *j) {
         if (i->signature_job && i->signature_job->error != 0) {
                 VerificationStyle style;
 
+                assert(i->checksum_job);
+
                 r = verification_style_from_url(i->checksum_job->url, &style);
                 if (r < 0) {
                         log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
@@ -306,19 +359,14 @@ static void tar_pull_job_on_finished(PullJob *j) {
                                                             * in per-directory verification mode, since only
                                                             * then the signature is detached, and thus a file
                                                             * of its own. */
-                        log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
+                        r = log_error_errno(i->signature_job->error,
-                        r = i->signature_job->error;
+                                            "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
                         goto finish;
                 }
         }
 
-        i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
+        pull_job_close_disk_fd(i->tar_job);
-        if (i->settings_job)
+        pull_job_close_disk_fd(i->settings_job);
-                i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
-
-        r = tar_pull_determine_path(i, NULL, &i->final_path);
-        if (r < 0)
-                goto finish;
 
         if (i->tar_pid > 0) {
                 r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG);
@@ -337,6 +385,7 @@ static void tar_pull_job_on_finished(PullJob *j) {
                 tar_pull_report_progress(i, TAR_VERIFYING);
 
                 r = pull_verify(i->verify,
+                                i->checksum,
                                 i->tar_job,
                                 i->checksum_job,
                                 i->signature_job,
@@ -346,60 +395,93 @@ static void tar_pull_job_on_finished(PullJob *j) {
                                 /* verity_job = */ NULL);
                 if (r < 0)
                         goto finish;
+        }
+
+        if (i->flags & PULL_DIRECT) {
+                assert(!i->settings_job);
+                assert(i->local);
+                assert(!i->temp_path);
 
                 tar_pull_report_progress(i, TAR_FINALIZING);
 
-                r = import_mangle_os_tree(i->temp_path);
+                r = import_mangle_os_tree(i->local);
                 if (r < 0)
                         goto finish;
 
-                r = import_make_read_only(i->temp_path);
+                r = install_file(
-                if (r < 0)
+                                AT_FDCWD, i->local,
-                        goto finish;
+                                AT_FDCWD, NULL,
-
+                                (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 |
-                r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
+                                (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
                 if (r < 0) {
-                        log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
+                        log_error_errno(r, "Failed to finalize '%s': %m", i->local);
                         goto finish;
                 }
+        } else {
+                r = tar_pull_determine_path(i, NULL, &i->final_path);
+                if (r < 0)
+                        goto finish;
 
-                i->temp_path = mfree(i->temp_path);
+                if (!i->tar_job->etag_exists) {
+                        /* This is a new download, verify it, and move it into place */
 
-                if (i->settings_job &&
+                        assert(i->temp_path);
-                    i->settings_job->error == 0) {
+                        assert(i->final_path);
 
-                        /* Also move the settings file into place, if it exists. Note that we do so only if we also
+                        tar_pull_report_progress(i, TAR_FINALIZING);
-                         * moved the tar file in place, to keep things strictly in sync. */
-                        assert(i->settings_temp_path);
 
-                        /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
+                        r = import_mangle_os_tree(i->temp_path);
-                         * we should incorporate it in the file name if we can */
-                        i->settings_path = mfree(i->settings_path);
-
-                        r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
                         if (r < 0)
                                 goto finish;
 
-                        r = import_make_read_only(i->settings_temp_path);
+                        r = install_file(
-                        if (r < 0)
+                                        AT_FDCWD, i->temp_path,
-                                goto finish;
+                                        AT_FDCWD, i->final_path,
-
+                                        INSTALL_READ_ONLY|
-                        r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
+                                        (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
                         if (r < 0) {
-                                log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
+                                log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
                                 goto finish;
                         }
 
-                        i->settings_temp_path = mfree(i->settings_temp_path);
+                        i->temp_path = mfree(i->temp_path);
+
+                        if (i->settings_job &&
+                            i->settings_job->error == 0) {
+
+                                /* Also move the settings file into place, if it exists. Note that we do so only if we also
+                                 * moved the tar file in place, to keep things strictly in sync. */
+                                assert(i->settings_temp_path);
+
+                                /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
+                                 * we should incorporate it in the file name if we can */
+                                i->settings_path = mfree(i->settings_path);
+
+                                r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
+                                if (r < 0)
+                                        goto finish;
+
+                                r = install_file(
+                                                AT_FDCWD, i->settings_temp_path,
+                                                AT_FDCWD, i->settings_path,
+                                                INSTALL_READ_ONLY|
+                                                (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
+                                if (r < 0) {
+                                        log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
+                                        goto finish;
+                                }
+
+                                i->settings_temp_path = mfree(i->settings_temp_path);
+                        }
                 }
+
+                tar_pull_report_progress(i, TAR_COPYING);
+
+                r = tar_pull_make_local_copy(i);
+                if (r < 0)
+                        goto finish;
         }
 
-        tar_pull_report_progress(i, TAR_COPYING);
-
-        r = tar_pull_make_local_copy(i);
-        if (r < 0)
-                goto finish;
-
         r = 0;
 
 finish:
@@ -410,6 +492,7 @@ finish:
 }
 
 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
+        const char *where;
         TarPull *i;
         int r;
 
@@ -420,23 +503,39 @@ static int tar_pull_job_on_open_disk_tar(PullJob *j) {
         assert(i->tar_job == j);
         assert(i->tar_pid <= 0);
 
-        if (!i->temp_path) {
+        if (i->flags & PULL_DIRECT)
-                r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
+                where = i->local;
-                if (r < 0)
+        else {
-                        return log_oom();
+                if (!i->temp_path) {
+                        r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
+                        if (r < 0)
+                                return log_oom();
+                }
+
+                where = i->temp_path;
         }
 
-        mkdir_parents_label(i->temp_path, 0700);
+        (void) mkdir_parents_label(where, 0700);
 
-        r = btrfs_subvol_make_fallback(i->temp_path, 0755);
+        if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE))
+                (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
+
+        if (i->flags & PULL_BTRFS_SUBVOL)
+                r = btrfs_subvol_make_fallback(where, 0755);
+        else
+                r = mkdir(where, 0755) < 0 ? -errno : 0;
+        if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
+                                                       * because in that case our temporary path collided */
+                r = 0;
         if (r < 0)
-                return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
+                return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
-        if (r > 0) { /* actually btrfs subvol */
+        if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */
-                (void) import_assign_pool_quota_and_warn(i->image_root);
+                if (!(i->flags & PULL_DIRECT))
-                (void) import_assign_pool_quota_and_warn(i->temp_path);
+                        (void) import_assign_pool_quota_and_warn(i->image_root);
+                (void) import_assign_pool_quota_and_warn(where);
         }
 
-        j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
+        j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
         if (j->disk_fd < 0)
                 return j->disk_fd;
 
@@ -459,7 +558,7 @@ static int tar_pull_job_on_open_disk_settings(PullJob *j) {
                         return log_oom();
         }
 
-        mkdir_parents_label(i->settings_temp_path, 0700);
+        (void) mkdir_parents_label(i->settings_temp_path, 0700);
 
         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
         if (j->disk_fd < 0)
@@ -484,19 +583,24 @@ int tar_pull_start(
                 const char *url,
                 const char *local,
                 PullFlags flags,
-                ImportVerify verify) {
+                ImportVerify verify,
+                const char *checksum) {
 
+        PullJob *j;
         int r;
 
         assert(i);
-        assert(verify < _IMPORT_VERIFY_MAX);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
-        assert(verify >= 0);
+        assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+        assert((verify < 0) || !checksum);
         assert(!(flags & ~PULL_FLAGS_MASK_TAR));
+        assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT));
+        assert(!(flags & PULL_SETTINGS) || !checksum);
 
         if (!http_url_is_valid(url))
                 return -EINVAL;
 
-        if (local && !hostname_is_valid(local, 0))
+        if (local && !pull_validate_local(local, flags))
                 return -EINVAL;
 
         if (i->tar_job)
@@ -506,6 +610,10 @@ int tar_pull_start(
         if (r < 0)
                 return r;
 
+        r = free_and_strdup(&i->checksum, checksum);
+        if (r < 0)
+                return r;
+
         i->flags = flags;
         i->verify = verify;
 
@@ -516,52 +624,56 @@ int tar_pull_start(
 
         i->tar_job->on_finished = tar_pull_job_on_finished;
         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
-        i->tar_job->on_progress = tar_pull_job_on_progress;
+        i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
-        i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 
-        r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
+        if (!FLAGS_SET(flags, PULL_DIRECT)) {
-        if (r < 0)
+                r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
-                return r;
+                if (r < 0)
+                        return r;
+        }
 
         /* Set up download of checksum/signature files */
-        r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
+        r = pull_make_verification_jobs(
+                        &i->checksum_job,
+                        &i->signature_job,
+                        verify,
+                        checksum,
+                        url,
+                        i->glue,
+                        tar_pull_job_on_finished,
+                        i);
         if (r < 0)
                 return r;
 
         /* Set up download job for the settings file (.nspawn) */
         if (FLAGS_SET(flags, PULL_SETTINGS)) {
-                r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
+                r = pull_make_auxiliary_job(
-                if (r < 0)
+                                &i->settings_job,
-                        return r;
+                                url,
-
+                                tar_strip_suffixes,
-                i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
+                                ".nspawn",
-                i->settings_job->on_progress = tar_pull_job_on_progress;
+                                verify,
-                i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+                                i->glue,
-        }
+                                tar_pull_job_on_open_disk_settings,
-
+                                tar_pull_job_on_finished,
-        r = pull_job_begin(i->tar_job);
+                                i);
-        if (r < 0)
-                return r;
-
-        if (i->checksum_job) {
-                i->checksum_job->on_progress = tar_pull_job_on_progress;
-                i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
-
-                r = pull_job_begin(i->checksum_job);
                 if (r < 0)
                         return r;
         }
 
-        if (i->signature_job) {
+        FOREACH_POINTER(j,
-                i->signature_job->on_progress = tar_pull_job_on_progress;
+                        i->tar_job,
+                        i->checksum_job,
+                        i->signature_job,
+                        i->settings_job) {
 
-                r = pull_job_begin(i->signature_job);
+                if (!j)
-                if (r < 0)
+                        continue;
-                        return r;
-        }
 
-        if (i->settings_job) {
+                j->on_progress = tar_pull_job_on_progress;
-                r = pull_job_begin(i->settings_job);
+                j->sync = FLAGS_SET(flags, PULL_SYNC);
+
+                r = pull_job_begin(j);
                 if (r < 0)
                         return r;
         }

src/import/pull-tar.h

@@ -16,4 +16,4 @@ TarPull* tar_pull_unref(TarPull *pull);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref);
 
-int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
+int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum);

src/import/pull.c

@@ -8,24 +8,87 @@
 
 #include "alloc-util.h"
 #include "discover-image.h"
+#include "env-util.h"
+#include "hexdecoct.h"
 #include "hostname-util.h"
+#include "import-common.h"
 #include "import-util.h"
+#include "io-util.h"
 #include "main-func.h"
+#include "parse-argument.h"
 #include "parse-util.h"
 #include "pull-raw.h"
 #include "pull-tar.h"
 #include "signal-util.h"
 #include "string-util.h"
+#include "terminal-util.h"
 #include "verbs.h"
 #include "web-util.h"
 
 static const char *arg_image_root = "/var/lib/machines";
 static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
-static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY;
+static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC;
+static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
+static char *arg_checksum = NULL;
 
-static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
+STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep);
-        log_notice("Transfer aborted.");
+
-        sd_event_exit(sd_event_source_get_event(s), EINTR);
+static int normalize_local(const char *local, const char *url, char **ret) {
+        _cleanup_free_ char *ll = NULL;
+        int r;
+
+        if (arg_pull_flags & PULL_DIRECT) {
+
+                if (!local)
+                        log_debug("Writing downloaded data to STDOUT.");
+                else {
+                        if (!path_is_absolute(local)) {
+                                ll = path_join(arg_image_root, local);
+                                if (!ll)
+                                        return log_oom();
+
+                                local = ll;
+                        }
+
+                        if (!path_is_valid(local))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                               "Local path name '%s' is not valid.", local);
+                }
+
+        } else if (local) {
+
+                if (!hostname_is_valid(local, 0))
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                               "Local image name '%s' is not valid.",
+                                               local);
+
+                if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
+                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
+                        if (r < 0) {
+                                if (r != -ENOENT)
+                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
+                        } else
+                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
+                                                       "Image '%s' already exists.",
+                                                       local);
+                }
+        }
+
+        if (!ll && local) {
+                ll = strdup(local);
+                if (!ll)
+                        return log_oom();
+        }
+
+        if (ll) {
+                if (arg_offset != UINT64_MAX)
+                        log_info("Pulling '%s', saving at offset %" PRIu64 " in '%s'.", url, arg_offset, ll);
+                else
+                        log_info("Pulling '%s', saving as '%s'.", url, ll);
+        } else
+                log_info("Pulling '%s'.", url);
+
+        *ret = TAKE_PTR(ll);
         return 0;
 }
 
@@ -40,70 +103,57 @@ static void on_tar_finished(TarPull *pull, int error, void *userdata) {
 }
 
 static int pull_tar(int argc, char *argv[], void *userdata) {
-        _cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
+        _cleanup_free_ char *ll = NULL, *normalized = NULL;
         _cleanup_(sd_event_unrefp) sd_event *event = NULL;
+        _cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
         const char *url, *local;
-        _cleanup_free_ char *l = NULL, *ll = NULL;
         int r;
 
         url = argv[1];
         if (!http_url_is_valid(url))
-                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
-                                       "URL '%s' is not valid.", url);
 
         if (argc >= 3)
-                local = argv[2];
+                local = empty_or_dash_to_null(argv[2]);
         else {
+                _cleanup_free_ char *l = NULL;
+
                 r = import_url_last_component(url, &l);
                 if (r < 0)
-                        return log_error_errno(r, "Failed get final component of URL: %m");
+                        return log_error_errno(r, "Failed to get final component of URL: %m");
 
-                local = l;
+                r = tar_strip_suffixes(l, &ll);
-        }
-
-        local = empty_or_dash_to_null(local);
-
-        if (local) {
-                r = tar_strip_suffixes(local, &ll);
                 if (r < 0)
                         return log_oom();
 
                 local = ll;
+        }
 
-                if (!hostname_is_valid(local, 0))
+        if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT))
-                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported.");
-                                               "Local image name '%s' is not valid.",
-                                               local);
 
-                if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
+        r = normalize_local(local, url, &normalized);
-                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
-                        if (r < 0) {
-                                if (r != -ENOENT)
-                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
-                        } else {
-                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
-                                                       "Image '%s' already exists.",
-                                                       local);
-                        }
-                }
-
-                log_info("Pulling '%s', saving as '%s'.", url, local);
-        } else
-                log_info("Pulling '%s'.", url);
-
-        r = sd_event_default(&event);
         if (r < 0)
-                return log_error_errno(r, "Failed to allocate event loop: %m");
+                return r;
 
-        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
+        r = import_allocate_event_with_signals(&event);
-        (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler,  NULL);
+        if (r < 0)
-        (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+                return r;
+
+        if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
+                log_info("File system synchronization on completion is off.");
 
         r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event);
         if (r < 0)
                 return log_error_errno(r, "Failed to allocate puller: %m");
 
-        r = tar_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_TAR, arg_verify);
+        r = tar_pull_start(
+                        pull,
+                        url,
+                        normalized,
+                        arg_pull_flags & PULL_FLAGS_MASK_TAR,
+                        arg_verify,
+                        arg_checksum);
         if (r < 0)
                 return log_error_errno(r, "Failed to pull image: %m");
 
@@ -126,70 +176,55 @@ static void on_raw_finished(RawPull *pull, int error, void *userdata) {
 }
 
 static int pull_raw(int argc, char *argv[], void *userdata) {
-        _cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
+        _cleanup_free_ char *ll = NULL, *normalized = NULL;
         _cleanup_(sd_event_unrefp) sd_event *event = NULL;
+        _cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
         const char *url, *local;
-        _cleanup_free_ char *l = NULL, *ll = NULL;
         int r;
 
         url = argv[1];
         if (!http_url_is_valid(url))
-                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
-                                       "URL '%s' is not valid.", url);
 
         if (argc >= 3)
-                local = argv[2];
+                local = empty_or_dash_to_null(argv[2]);
         else {
+                _cleanup_free_ char *l = NULL;
+
                 r = import_url_last_component(url, &l);
                 if (r < 0)
-                        return log_error_errno(r, "Failed get final component of URL: %m");
+                        return log_error_errno(r, "Failed to get final component of URL: %m");
 
-                local = l;
+                r = raw_strip_suffixes(l, &ll);
-        }
-
-        local = empty_or_dash_to_null(local);
-
-        if (local) {
-                r = raw_strip_suffixes(local, &ll);
                 if (r < 0)
                         return log_oom();
 
                 local = ll;
+        }
 
-                if (!hostname_is_valid(local, 0))
+        r = normalize_local(local, url, &normalized);
-                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
-                                               "Local image name '%s' is not valid.",
-                                               local);
-
-                if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
-                        r = image_find(IMAGE_MACHINE, local, NULL, NULL);
-                        if (r < 0) {
-                                if (r != -ENOENT)
-                                        return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
-                        } else {
-                                return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
-                                                       "Image '%s' already exists.",
-                                                       local);
-                        }
-                }
-
-                log_info("Pulling '%s', saving as '%s'.", url, local);
-        } else
-                log_info("Pulling '%s'.", url);
-
-        r = sd_event_default(&event);
         if (r < 0)
-                return log_error_errno(r, "Failed to allocate event loop: %m");
+                return r;
 
-        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
+        r = import_allocate_event_with_signals(&event);
-        (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler,  NULL);
+        if (r < 0)
-        (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+                return r;
 
-        r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
+        if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
+                log_info("File system synchronization on completion is off.");
+         r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
         if (r < 0)
                 return log_error_errno(r, "Failed to allocate puller: %m");
 
-        r = raw_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_RAW, arg_verify);
+        r = raw_pull_start(
+                        pull,
+                        url,
+                        normalized,
+                        arg_offset,
+                        arg_size_max,
+                        arg_pull_flags & PULL_FLAGS_MASK_RAW,
+                        arg_verify,
+                        arg_checksum);
         if (r < 0)
                 return log_error_errno(r, "Failed to pull image: %m");
 
@@ -203,23 +238,39 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
 
 static int help(int argc, char *argv[], void *userdata) {
 
-        printf("%s [OPTIONS...] {COMMAND} ...\n\n"
+        printf("%1$s [OPTIONS...] {COMMAND} ...\n"
-               "Download container or virtual machine images.\n\n"
+               "\n%4$sDownload container or virtual machine images.%5$s\n"
+               "\n%2$sCommands:%3$s\n"
+               "  tar URL [NAME]              Download a TAR image\n"
+               "  raw URL [NAME]              Download a RAW image\n"
+               "\n%2$sOptions:%3$s\n"
                "  -h --help                   Show this help\n"
                "     --version                Show package version\n"
                "     --force                  Force creation of image\n"
                "     --verify=MODE            Verify downloaded image, one of: 'no',\n"
-               "                              'checksum', 'signature'\n"
+               "                              'checksum', 'signature' or literal SHA256 hash\n"
                "     --settings=BOOL          Download settings file with image\n"
                "     --roothash=BOOL          Download root hash file with image\n"
                "     --roothash-signature=BOOL\n"
                "                              Download root hash signature file with image\n"
                "     --verity=BOOL            Download verity file with image\n"
                "     --image-root=PATH        Image root directory\n\n"
-               "Commands:\n"
+               "     --read-only              Create a read-only image\n"
-               "  tar URL [NAME]              Download a TAR image\n"
+               "     --direct                 Download directly to specified file\n"
-               "  raw URL [NAME]              Download a RAW image\n",
+               "     --btrfs-subvol=BOOL      Controls whether to create a btrfs subvolume\n"
-               program_invocation_short_name);
+               "                              instead of a directory\n"
+               "     --btrfs-quota=BOOL       Controls whether to set up quota for btrfs\n"
+               "                              subvolume\n"
+               "     --convert-qcow2=BOOL     Controls whether to convert QCOW2 images to\n"
+               "                              regular disk images\n"
+               "     --sync=BOOL              Controls whether to sync() before completing\n"
+               "     --offset=BYTES           Offset to seek to in destination\n"
+               "     --size-max=BYTES         Maximum number of bytes to write to destination\n",
+               program_invocation_short_name,
+               ansi_underline(),
+               ansi_normal(),
+               ansi_highlight(),
+               ansi_normal());
 
         return 0;
 }
@@ -235,6 +286,14 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_ROOTHASH,
                 ARG_ROOTHASH_SIGNATURE,
                 ARG_VERITY,
+                ARG_READ_ONLY,
+                ARG_DIRECT,
+                ARG_BTRFS_SUBVOL,
+                ARG_BTRFS_QUOTA,
+                ARG_CONVERT_QCOW2,
+                ARG_SYNC,
+                ARG_OFFSET,
+                ARG_SIZE_MAX,
         };
 
         static const struct option options[] = {
@@ -247,6 +306,14 @@ static int parse_argv(int argc, char *argv[]) {
                 { "roothash",           required_argument, NULL, ARG_ROOTHASH           },
                 { "roothash-signature", required_argument, NULL, ARG_ROOTHASH_SIGNATURE },
                 { "verity",             required_argument, NULL, ARG_VERITY             },
+                { "read-only",          no_argument,       NULL, ARG_READ_ONLY          },
+                { "direct",             no_argument,       NULL, ARG_DIRECT             },
+                { "btrfs-subvol",       required_argument, NULL, ARG_BTRFS_SUBVOL       },
+                { "btrfs-quota",        required_argument, NULL, ARG_BTRFS_QUOTA        },
+                { "convert-qcow2",      required_argument, NULL, ARG_CONVERT_QCOW2      },
+                { "sync",               required_argument, NULL, ARG_SYNC               },
+                { "offset",             required_argument, NULL, ARG_OFFSET             },
+                { "size-max",           required_argument, NULL, ARG_SIZE_MAX           },
                 {}
         };
 
@@ -273,26 +340,51 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_image_root = optarg;
                         break;
 
-                case ARG_VERIFY:
+                case ARG_VERIFY: {
-                        arg_verify = import_verify_from_string(optarg);
+                        ImportVerify v;
-                        if (arg_verify < 0)
+
-                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                        v = import_verify_from_string(optarg);
-                                                       "Invalid verification setting '%s'", optarg);
+                        if (v < 0) {
+                                _cleanup_free_ void *h = NULL;
+                                char *hh;
+                                size_t n;
+
+                                /* If this is not a vaid verification mode, maybe it's a literally specified
+                                 * SHA256 hash? We can handle that too... */
+
+                                r = unhexmem(optarg, (size_t) -1, &h, &n);
+                                if (r < 0 || n == 0)
+                                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                               "Invalid verification setting: %s", optarg);
+                                if (n != 32)
+                                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                               "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2);
+
+                                hh = hexmem(h, n); /* bring into canonical (lowercase) form */
+                                if (!hh)
+                                        return log_oom();
+
+                                free_and_replace(arg_checksum, hh);
+                                arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
+                                arg_verify = _IMPORT_VERIFY_INVALID;
+                        } else
+                                arg_verify = v;
 
                         break;
+                }
 
                 case ARG_SETTINGS:
-                        r = parse_boolean(optarg);
+                        r = parse_boolean_argument("--settings=", optarg, NULL);
                         if (r < 0)
-                                return log_error_errno(r, "Failed to parse --settings= parameter '%s': %m", optarg);
+                                return r;
 
                         SET_FLAG(arg_pull_flags, PULL_SETTINGS, r);
                         break;
 
                 case ARG_ROOTHASH:
-                        r = parse_boolean(optarg);
+                        r = parse_boolean_argument("--roothash=", optarg, NULL);
                         if (r < 0)
-                                return log_error_errno(r, "Failed to parse --roothash= parameter '%s': %m", optarg);
+                                return r;
 
                         SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r);
 
@@ -302,21 +394,88 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
 
                 case ARG_ROOTHASH_SIGNATURE:
-                        r = parse_boolean(optarg);
+                        r = parse_boolean_argument("--roothash-signature=", optarg, NULL);
                         if (r < 0)
-                                return log_error_errno(r, "Failed to parse --roothash-signature= parameter '%s': %m", optarg);
+                                return r;
 
                         SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r);
                         break;
 
                 case ARG_VERITY:
-                        r = parse_boolean(optarg);
+                        r = parse_boolean_argument("--verity=", optarg, NULL);
                         if (r < 0)
-                                return log_error_errno(r, "Failed to parse --verity= parameter '%s': %m", optarg);
+                                return r;
 
                         SET_FLAG(arg_pull_flags, PULL_VERITY, r);
                         break;
 
+                case ARG_READ_ONLY:
+                        arg_pull_flags |= PULL_READ_ONLY;
+                        break;
+
+                case ARG_DIRECT:
+                        arg_pull_flags |= PULL_DIRECT;
+                        arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
+                        break;
+
+                case ARG_BTRFS_SUBVOL:
+                        r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
+                        break;
+
+                case ARG_BTRFS_QUOTA:
+                        r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
+                        break;
+
+                case ARG_CONVERT_QCOW2:
+                        r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r);
+                        break;
+
+                case ARG_SYNC:
+                        r = parse_boolean_argument("--sync=", optarg, NULL);
+                        if (r < 0)
+                                return r;
+
+                        SET_FLAG(arg_pull_flags, PULL_SYNC, r);
+                        break;
+
+                case ARG_OFFSET: {
+                        uint64_t u;
+
+                        r = safe_atou64(optarg, &u);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
+                        if (!FILE_SIZE_VALID(u))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
+
+                        arg_offset = u;
+                        break;
+                }
+
+                case ARG_SIZE_MAX: {
+                        uint64_t u;
+
+                        r = parse_size(optarg, 1024, &u);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
+                        if (!FILE_SIZE_VALID(u))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
+
+                        arg_size_max = u;
+                        break;
+                }
+
                 case '?':
                         return -EINVAL;
 
@@ -324,9 +483,46 @@ static int parse_argv(int argc, char *argv[]) {
                         assert_not_reached();
                 }
 
+        /* Make sure offset+size is still in the valid range if both set */
+        if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
+            ((arg_size_max > (UINT64_MAX - arg_offset)) ||
+             !FILE_SIZE_VALID(arg_offset + arg_size_max)))
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
+
+        if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT))
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
+
+        if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded.");
+
         return 1;
 }
 
+static void parse_env(void) {
+        int r;
+
+        /* Let's make these relatively low-level settings also controllable via env vars. User can then set
+         * them for systemd-importd.service if they like to tweak behaviour */
+
+        r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
+        if (r >= 0)
+                SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
+
+        r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
+        if (r >= 0)
+                SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
+
+        r = getenv_bool("SYSTEMD_IMPORT_SYNC");
+        if (r >= 0)
+                SET_FLAG(arg_pull_flags, PULL_SYNC, r);
+        else if (r != -ENXIO)
+                log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
+}
+
 static int pull_main(int argc, char *argv[]) {
         static const Verb verbs[] = {
                 { "help", VERB_ANY, VERB_ANY, 0, help     },
@@ -345,6 +541,8 @@ static int run(int argc, char *argv[]) {
         log_parse_environment();
         log_open();
 
+        parse_env();
+
         r = parse_argv(argc, argv);
         if (r <= 0)
                 return r;

src/shared/install-file.c

@@ -0,0 +1,269 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <sys/ioctl.h>
+
+#include "btrfs-util.h"
+#include "chattr-util.h"
+#include "errno-util.h"
+#include "fd-util.h"
+#include "fs-util.h"
+#include "install-file.h"
+#include "missing_syscall.h"
+#include "rm-rf.h"
+
+int fs_make_very_read_only(int fd) {
+        struct stat st;
+        int r;
+
+        assert(fd >= 0);
+
+        /* Tries to make the specified fd "comprehensively" read-only. Primary usecase for this is OS images,
+         * i.e. either loopback files or larger directory hierarchies. Depending on the inode type and
+         * backing file system this means something different:
+         *
+         * 1. If the fd refers to a btrfs subvolume we'll mark it read-only as a whole
+         * 2. If the fd refers to any other directory we'll set the FS_IMMUTABLE_FL flag on it
+         * 3. If the fd refers to a regular file we'll drop the w bits.
+         * 4. If the fd refers to a block device, use BLKROSET to set read-only state
+         *
+         * You might wonder why not drop the x bits for directories. That's because we want to guarantee that
+         * everything "inside" the image remains largely the way it is, in case you mount it. And since the
+         * mode of the root dir of the image is pretty visible we don't want to modify it. btrfs subvol flags
+         * and the FS_IMMUTABLE_FL otoh are much less visible. Changing the mode of regular files should be
+         * OK though, since after all this is supposed to be used for disk images, i.e. the fs in the disk
+         * image doesn't make the mode of the loopback file it is stored in visible. */
+
+        if (fstat(fd, &st) < 0)
+                return -errno;
+
+        switch (st.st_mode & S_IFMT) {
+
+        case S_IFDIR:
+                if (btrfs_might_be_subvol(&st)) {
+                        r = btrfs_subvol_set_read_only_fd(fd, true);
+                        if (r >= 0)
+                                return 0;
+
+                        if (!ERRNO_IS_NOT_SUPPORTED(r) && r != -EINVAL)
+                                return r;
+                }
+
+                r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL);
+                if (r < 0)
+                        return r;
+
+                break;
+
+        case S_IFREG:
+                if ((st.st_mode & 0222) != 0)
+                        if (fchmod(fd, st.st_mode & 07555) < 0)
+                                return -errno;
+
+                break;
+
+        case S_IFBLK: {
+                int ro = 1;
+
+                if (ioctl(fd, BLKROSET, &ro) < 0)
+                        return -errno;
+
+                break;
+        }
+
+        default:
+                return -EBADFD;
+        }
+
+        return 0;
+}
+
+static int unlinkat_maybe_dir(int dirfd, const char *pathname) {
+
+        /* Invokes unlinkat() for regular files first, and if this fails with EISDIR tries again with
+         * AT_REMOVEDIR */
+
+        if (unlinkat(dirfd, pathname, 0) < 0) {
+                if (errno != EISDIR)
+                        return -errno;
+
+                if (unlinkat(dirfd, pathname, AT_REMOVEDIR) < 0)
+                        return -errno;
+        }
+
+        return 0;
+}
+
+int install_file(int source_atfd, const char *source_name,
+                 int target_atfd, const char *target_name,
+                 InstallFileFlags flags) {
+
+        _cleanup_close_ int rofd = -1;
+        int r;
+
+        /* Moves a file or directory tree into place, with some bells and whistles:
+         *
+         * 1. Optionally syncs before/after to ensure file installation can be used as barrier
+         * 2. Optionally marks the file/directory read-only using fs_make_very_read_only()
+         * 3. Optionally operates in replacing or in non-replacing mode.
+         * 4. If it replaces will remove the old tree if needed.
+         */
+
+        assert(source_atfd >= 0 || source_atfd == AT_FDCWD);
+        assert(source_name);
+        assert(target_atfd >= 0 || target_atfd == AT_FDCWD);
+
+        /* If target_name is specified as NULL no renaming takes place. Instead it is assumed the file is
+         * already in place, and only the syncing/read-only marking shall be applied. Note that with
+         * target_name=NULL and flags=0 this call is a NOP */
+
+        if ((flags & (INSTALL_FSYNC|INSTALL_FSYNC_FULL|INSTALL_SYNCFS|INSTALL_READ_ONLY)) != 0) {
+                _cleanup_close_ int pfd = -1;
+                struct stat st;
+
+                /* Open an O_PATH fd for the source if we need to sync things or mark things read only. */
+
+                pfd = openat(source_atfd, source_name, O_PATH|O_CLOEXEC|O_NOFOLLOW);
+                if (pfd < 0)
+                        return -errno;
+
+                if (fstat(pfd, &st) < 0)
+                        return -errno;
+
+                switch (st.st_mode & S_IFMT) {
+
+                case S_IFREG: {
+                        _cleanup_close_ int regfd = -1;
+
+                        regfd = fd_reopen(pfd, O_RDONLY|O_CLOEXEC);
+                        if (regfd < 0)
+                                return regfd;
+
+                        if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
+                                /* If this is just a regular file (as oppose to a fully populated directory)
+                                 * let's downgrade INSTALL_SYNCFS to INSTALL_FSYNC_FULL, after all this is
+                                 * going to be a single inode we install */
+                                r = fsync_full(regfd);
+                                if (r < 0)
+                                        return r;
+                        } else if (flags & INSTALL_FSYNC) {
+                                if (fsync(regfd) < 0)
+                                        return -errno;
+                        }
+
+                        if (flags & INSTALL_READ_ONLY)
+                                rofd = TAKE_FD(regfd);
+
+                        break;
+                }
+
+                case S_IFDIR: {
+                        _cleanup_close_ int dfd = -1;
+
+                        dfd = fd_reopen(pfd, O_RDONLY|O_DIRECTORY|O_CLOEXEC);
+                        if (dfd < 0)
+                                return dfd;
+
+                        if (flags & INSTALL_SYNCFS) {
+                                if (syncfs(dfd) < 0)
+                                        return -errno;
+                        } else if (flags & INSTALL_FSYNC_FULL) {
+                                r = fsync_full(dfd);
+                                if (r < 0)
+                                        return r;
+                        } else if (flags & INSTALL_FSYNC) {
+                                if (fsync(dfd) < 0)
+                                        return -errno;
+                        }
+
+                        if (flags & INSTALL_READ_ONLY)
+                                rofd = TAKE_FD(dfd);
+
+                        break;
+                }
+
+                default:
+                        /* Other inodes: char/block device inodes, fifos, symlinks, sockets don't need
+                         * syncing themselves, as they only exist in the directory, and have no contents on
+                         * disk */
+
+                        if (target_name && (flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
+                                r = fsync_directory_of_file(pfd);
+                                if (r < 0)
+                                        return r;
+                        }
+
+                        break;
+                }
+        }
+
+        if (target_name) {
+                /* Rename the file */
+
+                if (flags & INSTALL_REPLACE) {
+                        /* First, try a simple renamat(), maybe that's enough */
+                        if (renameat(source_atfd, source_name, target_atfd, target_name) < 0) {
+                                _cleanup_close_ int dfd = -1;
+
+                                if (!IN_SET(errno, EEXIST, ENOTDIR, ENOTEMPTY, EISDIR, EBUSY))
+                                        return -errno;
+
+                                /* Hmm, the target apparently existed already. Let's try to use
+                                 * RENAME_EXCHANGE. But let's first open the inode if it's a directory, so
+                                 * that we can later remove its contents if it's a directory. Why do this
+                                 * before the rename()? Mostly because if we have trouble opening the thing
+                                 * we want to know before we start actually modifying the file system. */
+
+                                dfd = openat(target_atfd, target_name, O_RDONLY|O_DIRECTORY|O_CLOEXEC, 0);
+                                if (dfd < 0 && errno != ENOTDIR)
+                                        return -errno;
+
+                                if (renameat2(source_atfd, source_name, target_atfd, target_name, RENAME_EXCHANGE) < 0) {
+
+                                        if (!ERRNO_IS_NOT_SUPPORTED(errno) && errno != EINVAL)
+                                                return -errno;
+
+                                        /* The exchange didn't work, let's remove the target first, and try again */
+
+                                        if (dfd >= 0)
+                                                (void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL);
+
+                                        r = unlinkat_maybe_dir(target_atfd, target_name);
+                                        if (r < 0)
+                                                return log_debug_errno(r, "Failed to remove target directory: %m");
+
+                                        if (renameat(source_atfd, source_name, target_atfd, target_name) < 0)
+                                                return -errno;
+                                } else {
+                                        /* The exchange worked, hence let's remove the source (i.e. the old target) */
+                                        if (dfd >= 0)
+                                                (void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL);
+
+                                        r = unlinkat_maybe_dir(source_atfd, source_name);
+                                        if (r < 0)
+                                                return log_debug_errno(r, "Failed to remove replaced target directory: %m");
+                                }
+                        }
+                } else {
+                        r = rename_noreplace(source_atfd, source_name, target_atfd, target_name);
+                        if (r < 0)
+                                return r;
+                }
+        }
+
+        if (rofd >= 0) {
+                r = fs_make_very_read_only(rofd);
+                if (r < 0)
+                        return r;
+        }
+
+        if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
+                if (target_name)
+                        r = fsync_parent_at(target_atfd, target_name);
+                else
+                        r = fsync_parent_at(source_atfd, source_name);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}

src/shared/install-file.h

@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+int fs_make_very_read_only(int fd);
+
+typedef enum InstallFileFlags {
+        INSTALL_REPLACE    = 1 << 0, /* Replace an existing inode */
+        INSTALL_READ_ONLY  = 1 << 1, /* Call fs_make_very_read_only() to make the inode comprehensively read-only */
+        INSTALL_FSYNC      = 1 << 2, /* fsync() file contents before moving file in */
+        INSTALL_FSYNC_FULL = 1 << 3, /* like INSTALL_FSYNC, but also fsync() parent dir before+after moving file in */
+        INSTALL_SYNCFS     = 1 << 4, /* syncfs() before moving file in, fsync() parent dir after moving file in */
+} InstallFileFlags;
+
+int install_file(int source_atfd, const char *source_name, int target_atfd, const char *target_name, InstallFileFlags flags);

src/shared/meson.build

@@ -152,6 +152,8 @@ shared_sources = files('''
         import-util.c
         import-util.h
         initreq.h
+        install-file.c
+        install-file.h
         install-printf.c
         install-printf.h
         install.c

src/shared/pkcs11-util.c

@@ -3,6 +3,7 @@
 #include <fcntl.h>
 
 #include "ask-password-api.h"
+#include "env-util.h"
 #include "escape.h"
 #include "fd-util.h"
 #include "format-table.h"
@@ -245,9 +246,7 @@ int pkcs11_token_login(
                         if (!passwords)
                                 return log_oom();
 
-                        string_erase(e);
+                        assert_se(unsetenv_erase("PIN") >= 0);
-                        if (unsetenv("PIN") < 0)
-                                return log_error_errno(errno, "Failed to unset $PIN: %m");
                 } else if (headless)
                         return log_error_errno(SYNTHETIC_ERRNO(ENOPKG), "PIN querying disabled via 'headless' option. Use the 'PIN' environment variable.");
                 else {

src/test/meson.build

@@ -215,6 +215,8 @@ tests += [
 
         [['src/test/test-fs-util.c']],
 
+        [['src/test/test-install-file.c']],
+
         [['src/test/test-umask-util.c']],
 
         [['src/test/test-proc-cmdline.c']],

src/test/test-env-util.c

@@ -428,6 +428,52 @@ static void test_setenv_systemd_exec_pid(void) {
         assert_se(set_unset_env("SYSTEMD_EXEC_PID", saved, 1) >= 0);
 }
 
+static void test_unsetenv_erase(void) {
+        int r;
+
+        log_info("/* %s */", __func__);
+
+        r = safe_fork("(sd-unsetenverase)", FORK_DEATHSIG|FORK_LOG|FORK_WAIT, NULL);
+        if (r == 0) {
+                _cleanup_strv_free_ char **l = NULL;
+                char **e;
+
+                /* child */
+
+                assert_se(unsetenv_erase("thisenvvardefinitelywontexist") == 0);
+
+                l = strv_new("FOO=BAR", "QUUX=PIFF", "ONE=TWO", "A=B");
+                assert_se(strv_length(l) == 4);
+
+                environ = l;
+
+                STRV_FOREACH(e, environ) {
+                        _cleanup_free_ char *n = NULL;
+                        char *eq;
+
+                        eq = strchr(*e, '=');
+                        if (!eq)
+                                continue;
+
+                        n = strndup(*e, eq - *e);
+                        assert_se(n);
+
+                        assert_se(streq_ptr(getenv(n), eq + 1));
+                        assert_se(getenv(n) == eq + 1);
+                        assert_se(unsetenv_erase(n) > 0);
+                        assert_se(isempty(eq + 1));
+                        assert_se(!getenv(n));
+                }
+
+                environ = NULL;
+                l = strv_free(l);
+
+                _exit(EXIT_SUCCESS);
+        }
+
+        assert_se(r > 0);
+}
+
 int main(int argc, char *argv[]) {
         test_setup_logging(LOG_DEBUG);
 
@@ -451,6 +497,7 @@ int main(int argc, char *argv[]) {
         test_env_assignment_is_valid();
         test_putenv_dup();
         test_setenv_systemd_exec_pid();
+        test_unsetenv_erase();
 
         return 0;
 }

src/test/test-install-file.c

@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "fileio.h"
+#include "install-file.h"
+#include "path-util.h"
+#include "rm-rf.h"
+#include "tests.h"
+#include "tmpfile-util.h"
+#include "umask-util.h"
+
+static void test_install_file(void) {
+        _cleanup_(rm_rf_physical_and_freep) char *p = NULL;
+        _cleanup_free_ char *a = NULL, *b = NULL, *c = NULL;
+        struct stat stat1, stat2;
+
+        log_info("/* %s */", __func__);
+
+        assert_se(mkdtemp_malloc(NULL, &p) >= 0);
+        assert_se(a = path_join(p, "foo"));
+        assert_se(b = path_join(p, "bar"));
+
+        RUN_WITH_UMASK(0077)
+                assert_se(write_string_file(a, "wups", WRITE_STRING_FILE_CREATE) >= 0);
+
+        assert_se(lstat(a, &stat1) >= 0);
+        assert_se(S_ISREG(stat1.st_mode));
+
+        assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, 0) >= 0);
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC) >= 0);
+
+        assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0);
+        assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL) == -EEXIST);
+        assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL|INSTALL_REPLACE) >= 0);
+
+        assert_se(stat(b, &stat2) >= 0);
+        assert_se(stat1.st_dev == stat2.st_dev);
+        assert_se(stat1.st_ino == stat2.st_ino);
+        assert_se((stat2.st_mode & 0222) != 0); /* writable */
+
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE|INSTALL_READ_ONLY) >= 0);
+
+        assert_se(stat(a, &stat2) >= 0);
+        assert_se(stat1.st_dev == stat2.st_dev);
+        assert_se(stat1.st_ino == stat2.st_ino);
+        assert_se((stat2.st_mode & 0222) == 0); /* read-only */
+
+        assert_se(mkdir(b, 0755) >= 0);
+        assert_se(c = path_join(b, "dir"));
+        assert_se(mkdir(c, 0755) >= 0);
+        free(c);
+        assert_se(c = path_join(b, "reg"));
+        assert_se(mknod(c, S_IFREG|0755, 0) >= 0);
+        free(c);
+        assert_se(c = path_join(b, "fifo"));
+        assert_se(mknod(c, S_IFIFO|0755, 0) >= 0);
+
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST);
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0);
+
+        assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0);
+
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST);
+        assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0);
+}
+
+int main(int argc, char *argv[]) {
+        test_setup_logging(LOG_INFO);
+
+        test_install_file();
+
+        return 0;
+}