test/syscalls,g3doc: add journaled overlay test variants and user guide

g3doc/user_guide/checkpoint_restore.md

@@ -5,6 +5,142 @@
 gVisor has the ability to checkpoint a process, save its current state in a
 state file, and restore into a new container using the state file.
 
+## Choosing a way to save sandbox state
+
+gVisor has four ways to save state and use it in a later sandbox. They differ
+in what they save, when they save it, and what the new sandbox starts with.
+
+Method                                              | Saves                                                                                | When                                  | New sandbox gets
+--------------------------------------------------- | ------------------------------------------------------------------------------------ | ------------------------------------- | ----------------
+Checkpoint/restore (this page)                      | Processes, memory, open files and all filesystems                                    | When you run `runsc checkpoint`       | The same processes, resumed
+[Filesystem snapshot](fs_snapshot.md)               | Rootfs upper layers of all containers, and optionally other disk-backed tmpfs mounts | When you run `runsc fscheckpoint`     | The saved files, with new processes
+[Rootfs tar snapshot](rootfs_snapshot.md)           | Rootfs upper layer of one container, as a tar file                                   | When you run `runsc tar rootfs-upper` | The saved files, with new processes
+[Journaling rootfs upper](#journaling-rootfs-upper) | Rootfs upper layer of each journaled container                                       | At each filesystem change             | The files as of the last change, with new processes
+
+### Comparison
+
+*   **Recovery point.** The first three methods save a point in time, so
+    changes made after the last save are lost. The journaling rootfs upper
+    records every change as it happens. After a sentry crash or a clean stop,
+    the next sandbox gets the upper layer as of the last completed file
+    operation. An operation that was in progress during a crash can be missing
+    or partly applied, as after a `SIGKILL`.
+
+*   **Crash recovery.** Only the journaling rootfs upper recovers from a crash
+    without an earlier save. It does not cover a host crash: it relies on the
+    host page cache, and it does not `fsync` its files. The other methods can
+    recover from any crash, but only to their last saved image.
+
+*   **Processes.** Only checkpoint/restore resumes processes, memory and open
+    file descriptors. The other methods restore files only, and the new sandbox
+    starts its processes from the beginning.
+
+*   **Cost.** Checkpoint/restore cost grows with application memory and
+    filesystem size, and the sandbox pauses while the checkpoint is taken.
+    Filesystem and tar snapshots cost grows with the size of the saved upper
+    layers. The journaling rootfs upper adds a small, synchronous metadata
+    record to each filesystem change, and data is never copied: a restart
+    reuses the pages already in the filestore, so its cost grows with the
+    number of files, not the amount of data. While the journal is compacted,
+    files can't be created, deleted or renamed, and the start of a new journal
+    after a restore pauses the sandbox. Both grow with the number of files.
+
+*   **Restore speed.** Checkpoint/restore and filesystem snapshots can load
+    data in the background while the application runs. A rootfs tar snapshot
+    must be applied completely before the container starts. The journaling
+    rootfs upper only rebuilds metadata at start.
+
+*   **Format and portability.** Rootfs tar snapshots are standard tar files
+    that other tools can read and change. Checkpoint images, filesystem
+    snapshots and journaling rootfs upper files can only be used by the same
+    runsc version that wrote them. The journaling rootfs upper can optionally
+    record and verify the container's lower-layer image ID on import.
+
+*   **Scope.** Checkpoint/restore and filesystem snapshots cover all
+    containers in a sandbox. A rootfs tar snapshot covers one container. The
+    journaling rootfs upper is enabled per container.
+
+*   **File management.** Checkpoint images, filesystem snapshots and tar files
+    are written where you choose, and you delete them when you no longer need
+    them. With `--overlay2=root:self` (the default), the journaling rootfs
+    upper stores both file data and journal records inside
+    `<rootfs>/.gvisor.filestore.<sandbox-id>`, and `runsc delete` keeps that
+    file so external tooling can extract or re-import it.
+
+### Combining methods
+
+The journaling rootfs upper works with the other methods:
+
+*   You can checkpoint, filesystem-checkpoint or tar-snapshot a sandbox that
+    uses a journaling rootfs upper. These saves only read the upper layer
+    (omitting journal metadata from checkpoints), and with `--leave-running`
+    the journal stays valid.
+
+*   A restore, a filesystem snapshot restore or a rootfs tar snapshot can
+    populate a new container with `--journaling-rootfs-upper=on`, starting
+    a fresh journal in the container's filestore once population completes.
+    `--journaling-rootfs-upper-import` cannot be combined with any of those
+    restore methods.
+
+*   Images saved from a sandbox that uses a journaling rootfs upper restore
+    either with `--journaling-rootfs-upper=on` or without it on any
+    file-backed rootfs overlay (such as `--overlay2=root:self` or
+    `--overlay2=root:dir=<dir>`), and vice versa.
+
+### Journaling rootfs upper
+
+> Note: This feature is experimental.
+
+The journaling rootfs upper logs metadata changes inside the container's
+file-backed rootfs overlay upper layer `tmpfs` so that file data pages and
+journal records live together in the same backing filestore file. Runtime
+journaling and startup replay/import are configured independently:
+
+*   **Runtime journaling**: Pass `--journaling-rootfs-upper=on` for the root
+    container, or pass `--journaling-rootfs-upper-annotations=on` and set
+    `dev.gvisor.journal.rootfs.upper: "true"` for any container (append
+    `.<container-name>` in multi-container pods, as for
+    `dev.gvisor.tar.rootfs.upper`). Requires a file-backed rootfs overlay such
+    as `--overlay2=root:self` (the default). With `root:self`, `runsc delete`
+    preserves `<rootfs>/.gvisor.filestore.<sandbox-id>` on the host's rootfs
+    upper layer so external tooling can extract or re-import it before tearing
+    down the container snapshot.
+
+*   **Startup import / replay**: Pass
+    `--journaling-rootfs-upper-import=<path-or-/proc/self/fd/N>` for the root
+    container, or pass `--journaling-rootfs-upper-annotations=on` and set
+    `dev.gvisor.journal.rootfs.upper.import` to a regular host file path (with
+    `.<container-name>` in multi-container pods; `/proc/self/fd/N` is only
+    accepted via the `--journaling-rootfs-upper-import` flag) to replay a
+    previously journaled filestore file and adopt its data pages in place
+    without copying. Startup import works with or without runtime journaling
+    enabled on the new container.
+
+*   **Optional image verification**: Pass `--journaling-rootfs-upper-image-id`
+    or (with `--journaling-rootfs-upper-annotations=on`) set
+    `dev.gvisor.journal.rootfs.upper.image-id` to record the lower-layer image
+    identity when creating a journal and verify it when importing one. If
+    omitted on import (the default), image ID verification is skipped.
+
+```bash
+# Run with runtime journaling enabled (leaves .gvisor.filestore.<id> in rootfs on delete):
+runsc --journaling-rootfs-upper=on run <container id>
+
+# Start a new container replaying an extracted filestore in place:
+runsc --journaling-rootfs-upper-import=/var/lib/sandboxes/app/upper.filestore \
+    run <new container id>
+```
+
+*   `runsc` takes an exclusive `flock` lock on journaled and imported filestore
+    files so two sandboxes cannot use the same file concurrently.
+
+*   `runsc restore`, `--fs-restore-image-path`, and `dev.gvisor.tar.rootfs.upper`
+    can populate a new container with `--journaling-rootfs-upper=on`,
+    initializing a fresh journal after population completes. They cannot be
+    combined with `--journaling-rootfs-upper-import`. If journal activation
+    fails during restore, the sandbox stops and leaves the filestore marked
+    incomplete so a subsequent import refuses it.
+
 ## How to use checkpoint/restore
 
 Checkpoint/restore functionality is currently available via raw `runsc`

g3doc/user_guide/fs_snapshot.md

@@ -5,6 +5,9 @@
 gVisor has the ability to save changes made to root filesystems in a *filesystem
 snapshot*, and subsequently restore those changes in new sandboxes.
 
+For a comparison of all the ways to save sandbox state, see
+[Choosing a way to save sandbox state](checkpoint_restore.md#choosing-a-way-to-save-sandbox-state).
+
 Compared to [rootfs tar snapshots](rootfs_snapshot.md):
 
 *   Filesystem snapshots include root filesystem changes in all containers in a

g3doc/user_guide/rootfs_snapshot.md

@@ -6,6 +6,9 @@ gVisor allows users to snapshot changes made to the root filesystem and save
 them to a tar file. These changes in the tar file can then be applied to a new
 sandbox upon creation.
 
+For a comparison of all the ways to save sandbox state, see
+[Choosing a way to save sandbox state](checkpoint_restore.md#choosing-a-way-to-save-sandbox-state).
+
 Compared to [filesystem snapshots](fs_snapshot.md):
 
 *   Rootfs tar snapshots include root filesystem changes from a single

test/runner/defs.bzl

@@ -84,6 +84,7 @@ def _syscall_test(
         limit_ephemeral_ports = False,
         file_access = "exclusive",
         overlay = False,
+        journaled_overlay = False,
         add_host_uds = False,
         add_host_connector = False,
         add_host_fifo = False,
@@ -114,6 +115,8 @@ def _syscall_test(
         name += "_shared"
     if overlay:
         name += "_overlay"
+    if journaled_overlay:
+        name += "_overlay_journal"
     if network != "none":
         name += "_" + network + "net"
     if fusefs:
@@ -184,6 +187,7 @@ def _syscall_test(
         "--fusefs=" + str(fusefs),
         "--file-access=" + file_access,
         "--overlay=" + str(overlay),
+        "--journaled-overlay=" + str(journaled_overlay),
         "--add-host-uds=" + str(add_host_uds),
         "--add-host-connector=" + str(add_host_connector),
         "--add-host-fifo=" + str(add_host_fifo),
@@ -254,6 +258,7 @@ def syscall_test_variants(
         nftables = False,
         kvm_use_cpu_nums = False,
         in_sandbox_cgroup = "v1",
+        add_journaled_overlay = None,
         **kwargs):
     """Generates syscall tests for all variants.
 
@@ -283,8 +288,14 @@ def syscall_test_variants(
       nftables: if nftables is true, enable nftables.
       kvm_use_cpu_nums: use cpu numbers in kvm platform.
       in_sandbox_cgroup: cgroup version to use inside the sandbox.
+      add_journaled_overlay: add an overlay test whose rootfs upper layer is
+        journaled (--journaling-rootfs-upper). None means the same as
+        add_overlay. Each restore of a save variant populates a new
+        journaling upper.
       **kwargs: Additional test arguments; requires_atime enables host atime updates.
     """
+    if add_journaled_overlay == None:
+        add_journaled_overlay = add_overlay
     for platform, platform_tags in all_platforms():
         # Add directfs to the default platform variant.
         directfs = add_directfs and platform == default_platform
@@ -340,6 +351,32 @@ def syscall_test_variants(
             **kwargs
         )
 
+    if add_journaled_overlay:
+        _syscall_test(
+            test = test,
+            platform = default_platform,
+            use_tmpfs = use_tmpfs,
+            add_host_uds = add_host_uds,
+            add_host_connector = add_host_connector,
+            add_host_fifo = add_host_fifo,
+            add_host_tty = add_host_tty,
+            tags = platforms.get(default_platform, []) + tags,
+            debug = debug,
+            iouring = iouring,
+            container = container,
+            one_sandbox = one_sandbox,
+            journaled_overlay = True,
+            leak_check = leak_check,
+            save = save,
+            save_resume = save_resume,
+            size = size,
+            timeout = timeout,
+            nftables = nftables,
+            kvm_use_cpu_nums = kvm_use_cpu_nums,
+            in_sandbox_cgroup = in_sandbox_cgroup,
+            **kwargs
+        )
+
     # Connected host sockets do not survive restore, so the save tests are
     # skipped and only the save_resume tests are generated.
     if add_hostinet and not save:
@@ -443,6 +480,7 @@ def syscall_test(
         perf = False,
         kvm_use_cpu_nums = False,
         in_sandbox_cgroup = "v1",
+        add_journaled_overlay = None,
         **kwargs):
     """syscall_test is a macro that will create targets for all platforms.
 
@@ -471,6 +509,8 @@ def syscall_test(
       perf: test is a benchmark.
       kvm_use_cpu_nums: use cpu numbers in kvm platform.
       in_sandbox_cgroup: cgroup version to use inside the sandbox.
+      add_journaled_overlay: add a journaled overlay test. None means the same
+        as add_overlay.
       **kwargs: Additional test arguments; requires_atime enables host atime updates.
     """
     if not tags:
@@ -530,6 +570,7 @@ def syscall_test(
         nftables = nftables,
         kvm_use_cpu_nums = kvm_use_cpu_nums,
         in_sandbox_cgroup = in_sandbox_cgroup,
+        add_journaled_overlay = add_journaled_overlay,
         **kwargs
     )
 
@@ -562,6 +603,7 @@ def syscall_test(
             nftables = nftables,
             kvm_use_cpu_nums = kvm_use_cpu_nums,
             in_sandbox_cgroup = in_sandbox_cgroup,
+            add_journaled_overlay = add_journaled_overlay,
             **kwargs
         )
 
@@ -591,5 +633,6 @@ def syscall_test(
             nftables = nftables,
             kvm_use_cpu_nums = kvm_use_cpu_nums,
             in_sandbox_cgroup = in_sandbox_cgroup,
+            add_journaled_overlay = add_journaled_overlay,
             **kwargs
         )

test/runner/main.go

@@ -62,6 +62,7 @@ var (
 	fusefs             = flag.Bool("fusefs", false, "mounts a fusefs for /tmp")
 	fileAccess         = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode")
 	overlay            = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay")
+	journaledOverlay   = flag.Bool("journaled-overlay", false, "like --overlay, and also journal the rootfs overlay upper layer with --journaling-rootfs-upper")
 	container          = flag.Bool("container", false, "run tests in their own namespaces (user ns, network ns, etc), pretending to be root. Implicitly enabled if network=host, or if using network namespaces")
 	setupContainerPath = flag.String("setup-container", "", "path to setup_container binary (for use with --container)")
 	trace              = flag.Bool("trace", false, "enables all trace points")
@@ -377,7 +378,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 		spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("TEST_HOST_PTY_FD=%d", hostPTYGuestFD))
 	}
 
-	if *overlay {
+	if *overlay || *journaledOverlay {
 		for _, kv := range spec.Process.Env {
 			xmlPath, ok := strings.CutPrefix(kv, "XML_OUTPUT_FILE=")
 			if !ok || xmlPath == "" {
@@ -442,11 +443,16 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 	} else {
 		args = append(args, "-net-raw")
 	}
-	if *overlay {
+	if *overlay || *journaledOverlay {
+		// The journaling rootfs upper requires a root overlay, which
+		// dir=/tmp provides.
 		args = append(args, "-overlay2=all:dir=/tmp")
 	} else {
 		args = append(args, "-overlay2=none")
 	}
+	if *journaledOverlay {
+		args = append(args, "-journaling-rootfs-upper=on")
+	}
 	if *debug {
 		args = append(args, "-debug", "-log-packets=true")
 	}

test/syscalls/BUILD

@@ -471,6 +471,7 @@ syscall_test(
 
 syscall_test(
     size = "medium",
+    add_journaled_overlay = True,
     shard_count = more_shards,
     test = "//test/syscalls/linux:mmap_test",
 )
@@ -1323,6 +1324,7 @@ syscall_test(
 )
 
 syscall_test(
+    add_journaled_overlay = True,
     test = "//test/syscalls/linux:utimes_test",
 )