From ada9a427494db28133d3ab42c70f721c451e9a4a Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 20:05:53 +0200 Subject: [PATCH 01/26] Extract elfuse_launch into src/core/launch.c elfuse_launch owns guest bring-up: guest_bootstrap_prepare, the FUSE-temp unlink, the sysroot casefold probe, vCPU creation, GDB init/sync/wait, the run loop, gdb_stub_shutdown, the shim counter and syscall histogram dumps, and guest_destroy. main() retains the original CLI argv (proctitle rewriting), option parsing, sysroot provisioning, the shebang loop, the --gdb x86_64 guard, host cwd, and the heap resource cleanup, and now hands off through launch_args_t so other launchers (the OCI run helper) can share one bring-up path. The launch_args_t envp field generalizes the old hard-coded environ: NULL keeps the host environment, so main()'s behavior is unchanged. Bring-up failures unwind through a single fail label instead of repeating the guest_destroy-plus-unlink tail at every error site. Ownership of the FUSE-materialized temp ELF moves with the bring-up: elfuse_launch owns the unlink from the prepare call onward (teardown and the post-prepare error paths), and main() drops its claim before handing off, so main's shared goto unwind cannot double-unlink a path whose ownership has been transferred. The embedded shim blob include moves along with its only consumer, so shim_bin has a single object definition site. With the guest's whole lifetime inside elfuse_launch, main() no longer tracks one: the guest_t and its initialized flag are gone, and cleanup_main_resources drops the guest_destroy branch that could never fire from that call site. It now releases only what main() owns. --- Makefile | 1 + src/core/launch.c | 196 ++++++++++++++++++++++++++++++++++++++++++++++ src/core/launch.h | 91 +++++++++++++++++++++ src/main.c | 184 +++++++++++-------------------------------- 4 files changed, 335 insertions(+), 137 deletions(-) create mode 100644 src/core/launch.c create mode 100644 src/core/launch.h diff --git a/Makefile b/Makefile index a03b38d3..049863bd 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ SRCS := \ core/vdso.c \ core/shim-globals.c \ core/bootstrap.c \ + core/launch.c \ core/rosetta.c \ core/sysroot.c \ runtime/thread.c \ diff --git a/src/core/launch.c b/src/core/launch.c new file mode 100644 index 00000000..f766e559 --- /dev/null +++ b/src/core/launch.c @@ -0,0 +1,196 @@ +/* elfuse VM launch: bring-up + GDB + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implementation of elfuse_launch (contract and caller/callee ownership in + * launch.h). It lives apart from src/main.c so bring-up is one path behind a + * struct, leaving CLI concerns (option parsing, sysroot provisioning, the + * shebang loop) in main(). + * + * shim_blob.h is included here, not in src/main.c, so the static + * shim_bin / shim_bin_len blob has a single object definition site. + */ + +#include "launch.h" + +#include +#include +#include +#include +#include +#include + +#include "core/bootstrap.h" +#include "core/guest.h" +#include "core/shim-globals.h" +#include "core/sysroot.h" + +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" +#include "syscall/poll.h" /* wakeup_pipe_signal */ +#include "syscall/proc.h" + +#include "debug/gdbstub.h" +#include "debug/log.h" +#include "debug/syscall-hist.h" + +/* Embedded shim binary (generated by xxd -i from shim.bin). */ +#include "shim_blob.h" + +/* The shim code slot in the infra reserve is sized tight (INFRA_SHIM_SLOT, a + * few x the current blob) so the rest of the reserve goes to the page-table + * pool. If the shim ever outgrows the slot it would overlap the shim-data + * block; fail the build loudly rather than corrupt memory at boot. Enlarge + * INFRA_SHIM_SLOT (and shrink the pool to match) if this fires. + */ +_Static_assert(sizeof(shim_bin) <= INFRA_SHIM_SLOT, + "shim blob exceeds its infra slot; bump INFRA_SHIM_SLOT"); + +int elfuse_launch(const launch_args_t *args) +{ + if (!args) { + log_error("elfuse_launch: NULL args"); + return 1; + } + + extern char **environ; + char **envp_use = args->envp ? args->envp : environ; + + guest_t g; + bool guest_initialized = false; + guest_bootstrap_t boot; + /* Track the temp flag locally: elfuse_launch owns the post-prepare + * unlink of a FUSE-materialized temp, but the caller's launch_args_t + * is const and its copy is discarded after the call anyway. + */ + bool elf_host_temp = args->elf_host_temp; + /* The guest-visible entrypoint path is argv[0]; elf_path is the + * resolved host path to that binary. They differ when path + * translation or a FUSE-materialized temp is involved. + */ + const char *elf_guest_path = (args->guest_argc > 0 && args->guest_argv) + ? args->guest_argv[0] + : args->elf_path; + + if (guest_bootstrap_prepare( + &g, args->elf_path, elf_host_temp, elf_guest_path, args->sysroot, + args->guest_argc, args->guest_argv, envp_use, shim_bin, + shim_bin_len, args->verbose, &guest_initialized, &boot) < 0) + goto fail; + + /* A FUSE-materialized temp has been loaded; drop it once the guest + * has its own mapping, unless Rosetta still needs the reopenable + * host path. + */ + if (elf_host_temp && !g.is_rosetta) { + unlink(args->elf_path); + elf_host_temp = false; + } + + if (args->sysroot) { + bool case_sensitive = true; + bool case_preserving = true; + if (sysroot_probe_case_sensitivity(args->sysroot, &case_sensitive, + &case_preserving) == 0) + proc_set_sysroot_casefold(case_preserving && !case_sensitive); + else + proc_set_sysroot_casefold(false); + } else { + proc_set_sysroot_casefold(false); + } + + hv_vcpu_t vcpu; + hv_vcpu_exit_t *vexit; + if (guest_bootstrap_create_vcpu(&g, &boot, args->verbose, &vcpu, &vexit) < + 0) + goto fail; + + /* GDB setup must happen before the first run so entry-stop and + * hardware breakpoints can affect the initial vCPU. + */ + if (args->gdb_port > 0) { + if (gdb_stub_init(args->gdb_port, &g) < 0) { + log_error("failed to initialize GDB stub"); + goto fail; + } + gdb_stub_sync_debug_regs(vcpu); + if (args->gdb_stop_on_entry) + gdb_stub_wait_for_attach(); + } + + /* vcpu_run_loop owns guest execution until exit, fatal signal, or timeout. + */ + int exit_code = + vcpu_run_loop(vcpu, vexit, &g, args->verbose, args->timeout_sec, NULL); + + /* Tear down debugger state before joining workers: a worker parked in + * gdb_stub_handle_stop() stays active (not deactivated) until this + * broadcasts resume_cond, so joining first would just time out and + * detach it while it is still paused. + */ + gdb_stub_shutdown(); + + /* Join worker vCPU threads before guest_destroy unmaps the guest slab: a + * sibling still mid-iteration in its own run loop would fault on freed + * guest memory and crash the host with SIGSEGV, masking the real exit + * code. The join is a no-op once workers have wound down (the common + * single-threaded case). + * + * vcpu_run_loop can also return via a bare break (alarm timeout 124, a + * fatal default-disposition signal, or ELR_EL1==0) with no one having + * requested exit_group or kicked the siblings out of hv_vcpu_run. Mirror + * guest_destroy's request-interrupt prefix here first; otherwise this join + * burns its full poll cap and detaches every worker, and guest_destroy's + * own interrupt-join skips them (it honors join_abandoned), leaving live + * pthreads to fault on the imminent unmap. + */ + if (!proc_exit_group_requested()) + proc_request_exit_group(0); + futex_interrupt_request(); + wakeup_pipe_signal(); + thread_interrupt_all(); + /* Workers parked on internal condvars (fork barrier, ptrace stop/wait) + * see neither the pipe nor the vCPU kick; broadcast so they re-check the + * exit-group flag and terminate before the join below gives up on them. + */ + thread_wake_exit_waiters(); + thread_join_workers(); + + /* Diagnostic counter dump runs before guest_destroy so the + * shim_data mapping is still valid. ELFUSE_SHIM_STATS is the gate; + * an unset variable produces no output. + */ + if (shim_globals_stats_enabled()) + shim_globals_counters_dump(&g); + + /* Dump the startup histogram before guest_destroy so any + * cleanup-path syscalls (closing host fds, unmapping the slab) do + * not appear in the captured set. The dump is a no-op when + * ELFUSE_STARTUP_TRACE=syscalls was not requested. + */ + syscall_hist_dump(); + + if (guest_initialized) + guest_destroy(&g); + + /* Rosetta guests keep the FUSE-materialized temp alive for the whole run + * (the translator reopens the host path); drop it now that the guest is + * gone so repeated Rosetta launches do not accumulate temp files. + */ + if (elf_host_temp) + unlink(args->elf_path); + + return exit_code; + +fail: + /* Bring-up failed: unwind whatever exists so far. The caller owns + * pre-prepare failures; from the prepare call onward the temp unlink + * is ours. + */ + if (guest_initialized) + guest_destroy(&g); + if (elf_host_temp) + unlink(args->elf_path); + return 1; +} diff --git a/src/core/launch.h b/src/core/launch.h new file mode 100644 index 00000000..35499c4d --- /dev/null +++ b/src/core/launch.h @@ -0,0 +1,91 @@ +/* elfuse VM launch entry: post-CLI bring-up + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse_launch is the single entry point for "run a guest binary in a + * fresh HVF VM until it exits". main() is its only caller: every launcher + * reaches it through the CLI, including `elfuse-oci run`, which execs elfuse + * with the flags that fill launch_args_t rather than linking against it. + * Keeping bring-up behind one struct is what lets a front end select the + * guest identity, cwd, and environment without a second bring-up path. + * + * The function owns the guest_t, the vCPU, the GDB stub, the run loop, + * the diagnostic dumps, and guest teardown; it does NOT own the + * elf_path / sysroot / guest_argv heap copies or the sysroot_mount the + * host CLI may have provisioned; those stay with the caller so + * behaviors that need the original CLI argv (proctitle rewriting, + * --create-sysroot detach on exit, host cwd save+restore) remain + * coherent regardless of how the launch was kicked off. + * + * Lifetime / ownership contract: + * + * - The caller owns every pointer in launch_args_t. elfuse_launch reads + * them and does not free them; const-qualified pointers stay valid + * for the duration of the call. + * - envp may be NULL; the host process environ is used in that case. + * - guest_argv is the string array the guest sees as its argv. + * guest_argv[0] is the guest-visible entrypoint path (what the guest + * reads back via /proc/self/exe and argv[0]); elf_path is the + * resolved host path to that binary. The two differ when path + * translation or a FUSE-materialized temp is involved. + * - elf_host_temp is true when elf_path is a FUSE-materialized temp + * that must be unlinked once guest_bootstrap_prepare has loaded it + * (skipped for Rosetta guests, which reopen the path). The caller + * owns the unlink for any pre-prepare failure; elfuse_launch owns it + * from the prepare call onward. + * - fork_child_fd / vfork_notify_fd are forwarded for fork-child-routed + * launches; main() dispatches the fork-child path before reaching + * elfuse_launch and so passes -1. + */ + +#pragma once + +#include +#include + +typedef struct { + /* Host filesystem path to the guest ELF (resolved; may be a + * FUSE-materialized temp when elf_host_temp is true). + */ + const char *elf_path; + + /* True when elf_path is a temp to unlink after guest_bootstrap_prepare + * loads it. + */ + bool elf_host_temp; + + /* Host filesystem path to the sysroot the guest sees as / (absolute), + * or NULL when the guest runs without a sysroot. + */ + const char *sysroot; + + /* String array the guest sees as its argv. guest_argv[0] is the + * guest-visible entrypoint path. + */ + int guest_argc; + const char **guest_argv; + + /* NULL-terminated guest environ. NULL means "use host environ". envp is + * char** (not const) to match the environ/guest_bootstrap_prepare + * convention: guest programs may mutate their environment. + */ + char **envp; + + /* GDB Remote Serial Protocol port (0 disables the stub) and whether + * to halt before the first guest instruction. + */ + int gdb_port; + bool gdb_stop_on_entry; + + /* Per-iteration vCPU run timeout. 0 disables (no alarm()). */ + int timeout_sec; + + bool verbose; +} launch_args_t; + +/* Bring up the guest VM, run it to exit / signal / timeout, tear down, + * return the exit code. Returns 1 on bring-up failure (with a log + * message) and the guest's exit status otherwise. + */ +int elfuse_launch(const launch_args_t *args); diff --git a/src/main.c b/src/main.c index 6c410c1f..dd1c8e7c 100644 --- a/src/main.c +++ b/src/main.c @@ -31,21 +31,17 @@ #include "core/bootstrap.h" #include "core/guest.h" +#include "core/launch.h" #include "core/rosetta.h" -#include "core/shim-globals.h" #include "core/sysroot.h" #include "runtime/forkipc.h" -#include "runtime/futex.h" /* futex_interrupt_request */ #include "runtime/proctitle.h" -#include "runtime/thread.h" #include "syscall/fuse.h" #include "syscall/path.h" -#include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" -#include "debug/gdbstub.h" #include "debug/log.h" #include "debug/syscall-hist.h" @@ -108,24 +104,20 @@ static void free_guest_argv(const char **guest_argv, int guest_argc) free((void *) guest_argv); } -static void cleanup_main_resources(guest_t *g, - bool guest_initialized, - sysroot_mount_t *sysroot_mount, +/* Releases the host state main() owns: the sysroot mount, the host cwd, and the + * heap copies of argv. The guest itself belongs to elfuse_launch, which + * destroys it on every exit path, so nothing here touches HVF or guest memory. + * This must still run after a bring-up whose HVF teardown guest_destroy + * deferred to process exit, because process exit reclaims neither the sysroot + * mount nor the FUSE-materialized temp ELF. + */ +static void cleanup_main_resources(sysroot_mount_t *sysroot_mount, const char *host_cwd, const char **guest_argv, int guest_argc, char *elf_path, char *sysroot_path) { - /* guest_destroy may defer HVF teardown to process exit when a worker vCPU - * is still live, but the remaining cleanup below touches only host state - * (mounts, cwd, heap) -- never HVF or guest memory -- so it is safe to run - * regardless and must run so a deferred teardown does not orphan the - * sysroot mount or the FUSE-materialized temp ELF, which process exit will - * not reclaim. - */ - if (guest_initialized) - guest_destroy(g); rosettad_clear_binary_path(); if (host_cwd && host_cwd[0] != '\0' && chdir(host_cwd) < 0) (void) chdir("/"); @@ -135,17 +127,11 @@ static void cleanup_main_resources(guest_t *g, free((void *) sysroot_path); } -/* Embedded shim binary (generated by xxd -i from shim.bin) */ -#include "shim_blob.h" - -/* The shim code slot in the infra reserve is sized tight (INFRA_SHIM_SLOT, a - * few x the current blob) so the rest of the reserve goes to the page-table - * pool. If the shim ever outgrows the slot it would overlap the shim-data - * block; fail the build loudly rather than corrupt memory at boot. Enlarge - * INFRA_SHIM_SLOT (and shrink the pool to match) if this fires. +/* The embedded shim binary (shim_blob.h, generated by xxd -i from shim.bin) + * is now included by src/core/launch.c, the single site that hands the blob to + * guest_bootstrap_prepare. main() no longer references shim_bin/shim_bin_len + * directly, so the static blob has one object definition site. */ -_Static_assert(sizeof(shim_bin) <= INFRA_SHIM_SLOT, - "shim blob exceeds its infra slot; bump INFRA_SHIM_SLOT"); /* The infra-reserve layout invariants documented in guest.h are derived from * raw offset constants, so a future edit that grows the pool by shifting one @@ -502,8 +488,6 @@ int main(int argc, char **argv) int guest_argc = argc - arg_start; const char **guest_argv = (const char **) calloc((size_t) guest_argc, sizeof(char *)); - guest_t g; - bool guest_initialized = false; sysroot_mount_t sysroot_mount; char host_cwd[LINUX_PATH_MAX]; char elf_host_path[LINUX_PATH_MAX]; @@ -646,125 +630,51 @@ int main(int argc, char **argv) } } - guest_bootstrap_t boot; - extern char **environ; - - if (guest_bootstrap_prepare(&g, elf_host_path, elf_host_temp, elf_path, - sysroot, guest_argc, guest_argv, environ, - shim_bin, shim_bin_len, verbose, - &guest_initialized, &boot) < 0) - goto fail; - if (elf_host_temp && !g.is_rosetta) { - unlink(elf_host_path); - elf_host_temp = false; - } - - if (have_sysroot) { - bool case_sensitive = true; - bool case_preserving = true; - if (sysroot_probe_case_sensitivity(sysroot, &case_sensitive, - &case_preserving) == 0) { - proc_set_sysroot_casefold(case_preserving && !case_sensitive); - } else { - proc_set_sysroot_casefold(false); - } - } else { - proc_set_sysroot_casefold(false); - } - - runtime_set_process_title(argc, argv, elf_path); - - hv_vcpu_t vcpu; - hv_vcpu_exit_t *vexit; - if (guest_bootstrap_create_vcpu(&g, &boot, verbose, &vcpu, &vexit) < 0) - goto fail; - - /* GDB setup must happen before the first run so entry-stop and hardware - * breakpoints can affect the initial vCPU. - */ - if (gdb_port > 0) { - if (gdb_stub_init(gdb_port, &g) < 0) { - log_error("failed to initialize GDB stub"); - goto fail; - } - /* Mirror any preconfigured breakpoints/watchpoints into this vCPU. */ - gdb_stub_sync_debug_regs(vcpu); - - if (gdb_stop_on_entry) - gdb_stub_wait_for_attach(); - } - - /* vcpu_run_loop owns guest execution until exit, fatal signal, or timeout. + /* Rewrite the host-visible process title from the guest entrypoint. This + * clobbers the original argv block (already snapshotted into the heap + * elf_path / guest_argv above), so it must run before elfuse_launch hands + * control to the guest but after the shebang loop has fixed elf_path. The + * call only touches the original argv and the kernel procname; it does not + * depend on guest_bootstrap_prepare having run, so hoisting it out of the + * bring-up (where it previously sat between prepare and create_vcpu) is + * behavior-preserving. */ - exit_code = vcpu_run_loop(vcpu, vexit, &g, verbose, timeout_sec, NULL); - - /* Tear down debugger state before joining workers: a worker parked in - * gdb_stub_handle_stop() stays active (not deactivated) until this - * broadcasts resume_cond, so joining first would just time out and detach - * it while it is still paused. - */ - gdb_stub_shutdown(); - - /* Wait for worker vCPU threads to stop before tearing down guest memory. - * The main thread leaves the run loop as soon as it observes the exit_group - * flag, but sibling vCPU threads may still be mid-iteration in their own - * run loops (e.g. touching shim_globals). cleanup_main_resources unmaps the - * guest slab via guest_destroy, so a still-running worker would fault on - * freed guest memory and crash the host with SIGSEGV, masking the real exit - * code. thread_join_workers() is a no-op once the workers have already - * wound down (the common single-threaded case). - * - * vcpu_run_loop can also return here without anyone having requested - * exit_group or kicked the siblings out of hv_vcpu_run: the alarm timeout - * (exit_code 124), a fatal default-disposition signal, or ELR_EL1==0 all - * bail out with a bare break. On those paths siblings are still spinning in - * the guest, so mirror guest_destroy's request-interrupt prefix before - * joining -- otherwise this call burns its full poll cap, detaches every - * worker, and guest_destroy's own request-interrupt-join (which honors - * join_abandoned) skips them, leaving live pthreads to fault on the - * imminent unmap. - */ - if (!proc_exit_group_requested()) - proc_request_exit_group(0); - futex_interrupt_request(); - wakeup_pipe_signal(); - thread_interrupt_all(); - - /* Workers parked on internal condvars (fork barrier, ptrace stop/wait) see - * neither the pipe nor the vCPU kick; broadcast so they re-check the - * exit-group flag and terminate before the join below gives up on them. - */ - thread_wake_exit_waiters(); - thread_join_workers(); + runtime_set_process_title(argc, argv, elf_path); - /* Diagnostic counter dump runs before guest_destroy so the shim_data - * mapping is still valid. ELFUSE_SHIM_STATS is the gate; an unset variable - * produces no output. + /* Hand the bring-up, run loop, and guest teardown to elfuse_launch. main() + * retains ownership of the original argv (proctitle above), the sysroot + * mount (detached in cleanup_main_resources after the guest exits so the + * mount stays live for the whole run), host cwd, and the heap elf_path / + * sysroot_path / guest_argv copies. */ - if (shim_globals_stats_enabled()) - shim_globals_counters_dump(&g); - - /* Dump the startup histogram before guest_destroy so any cleanup-path - * syscalls (closing host fds, unmapping the slab) do not appear in the - * captured set. The dump is a no-op when ELFUSE_STARTUP_TRACE=syscalls was - * not requested. + launch_args_t largs = { + .elf_path = elf_host_path, + .elf_host_temp = elf_host_temp, + .sysroot = sysroot, + .guest_argc = guest_argc, + .guest_argv = guest_argv, + .gdb_port = gdb_port, + .gdb_stop_on_entry = gdb_stop_on_entry, + .timeout_sec = timeout_sec, + .verbose = verbose, + }; + /* elfuse_launch owns the temp unlink from the prepare call onward, so + * drop main()'s claim before handing off: the shared cleanup below must + * not unlink a path whose ownership has been transferred. */ - syscall_hist_dump(); + elf_host_temp = false; + exit_code = elfuse_launch(&largs); goto cleanup; fail: exit_code = 1; cleanup: /* Single unwind for every exit past the heap-copy allocations: frees the - * caller-owned heap copies (guest_destroy included, via - * cleanup_main_resources, once the guest came up), detaches the sysroot - * mount, restores the host cwd, and drops a still-owned FUSE-materialized - * temp ELF, which the post-prepare error paths and a Rosetta guest's - * teardown previously leaked. + * caller-owned heap copies, detaches the sysroot mount, restores the host + * cwd, and drops a still-owned FUSE-materialized temp ELF. */ - cleanup_main_resources(&g, guest_initialized, &sysroot_mount, - have_host_cwd ? host_cwd : NULL, guest_argv, - guest_argc, elf_path, sysroot_path); + cleanup_main_resources(&sysroot_mount, have_host_cwd ? host_cwd : NULL, + guest_argv, guest_argc, elf_path, sysroot_path); if (elf_host_temp) unlink(elf_host_path); From 5692cba2c3c6086fff8c510a13cc7d01bad0260e Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:10:43 +0200 Subject: [PATCH 02/26] Add --user, --workdir, and --env launch flags An OCI image front end needs to set the guest identity, working directory, and environment without patching the runtime; the new flags map onto launch_args_t fields and `elfuse-oci run` drives exactly this interface. The --user identity is staged before bring-up (proc_set_initial_ids) so the auxv AT_UID/AT_GID snapshot taken by build_linux_stack matches what getuid()/getgid() later report. --workdir rejects non-absolute paths up front instead of silently resolving them against the host cwd, and is applied by elfuse_launch after the casefold probe so the translation sees the sysroot's real case behavior. --env/--clear-env build the guest environment with env(1) semantics: KEY=VAL sets, bare KEY inherits from the host environ, --clear-env starts empty; with neither flag given envp stays NULL and the host environ is used unchanged. The new heap resources join main()'s shared goto unwind: envp, workdir, and the raw --env override array are released at the single cleanup label on every exit path. --fakeroot and a non-root --user are refused together. Fakeroot means the guest starts as uid/gid 0, and the setuid permission check grants every id switch on that basis; a non-root --user would leave that grant in place while the guest reported an unprivileged uid, so the guest could raise itself back to root at will. tests/test-launch-flags.sh covers the refusal along with the --workdir and --user parse rules. The parse-error unwind frees the ELF and sysroot path copies too, so the sysroot-too-long branch reaches it instead of repeating the frees. --- mk/tests.mk | 10 +- src/core/launch.c | 49 +++++ src/core/launch.h | 67 +++---- src/main.c | 353 ++++++++++++++++++++++++++++++++---- src/syscall/proc-identity.c | 24 +++ src/syscall/proc.h | 8 + tests/test-launch-flags.sh | 79 ++++++++ 7 files changed, 521 insertions(+), 69 deletions(-) create mode 100755 tests/test-launch-flags.sh diff --git a/mk/tests.mk b/mk/tests.mk index 2f610375..9be5f593 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -15,7 +15,8 @@ test-sysroot-create-paths test-fork-ipc-protocol-host \ test-vcpu-run-hooks-host test-identity-override-host \ test-proctitle-host test-proctitle-low-stack \ - test-sysroot-procfs-exec test-timeout-disable test-fuse-alpine \ + test-sysroot-procfs-exec test-timeout-disable test-launch-flags \ + test-fuse-alpine \ test-sysroot-nofollow test-sysroot-chdir test-sysroot-symlink-escape \ test-sysroot-dotdot test-sysroot-openat2-walk \ test-sysroot-inotify-names test-sysroot-exec-names \ @@ -225,6 +226,8 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @$(MAKE) --no-print-directory test-fuse-alpine @printf "\n$(BLUE)━━━ timeout=0 validation ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-timeout-disable + @printf "\n$(BLUE)━━━ launch flag rejection ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-launch-flags @printf "\n$(BLUE)━━━ rosetta CLI gating ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-rosetta-cli @printf "\n$(BLUE)━━━ hot-syscall guardrail ━━━$(RESET)\n" @@ -1041,6 +1044,11 @@ test-sysroot-procfs-exec: $(ELFUSE_BIN) $(BUILD_DIR)/test-procfs-exec test-timeout-disable: $(ELFUSE_BIN) $(TEST_HELLO_DEP) @$(ELFUSE_BIN) --timeout 0 $(TEST_DIR)/test-hello > /dev/null +## Verify --user / --workdir / --fakeroot reject contradictory requests during +## option parsing, before any VM is created. +test-launch-flags: $(ELFUSE_BIN) $(TEST_HELLO_DEP) + @bash tests/test-launch-flags.sh $(ELFUSE_BIN) $(TEST_DIR)/test-hello + ## Run GDB stub integration tests (LLDB <-> elfuse gdbstub) test-gdbstub: $(ELFUSE_BIN) $(TEST_DIR)/test-hello @bash tests/test-gdbstub.sh -e $(ELFUSE_BIN) -v diff --git a/src/core/launch.c b/src/core/launch.c index f766e559..fc6de2e6 100644 --- a/src/core/launch.c +++ b/src/core/launch.c @@ -16,9 +16,11 @@ #include #include +#include #include #include #include +#include #include #include "core/bootstrap.h" @@ -28,6 +30,7 @@ #include "runtime/futex.h" /* futex_interrupt_request */ #include "runtime/thread.h" +#include "syscall/path.h" #include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" @@ -73,6 +76,14 @@ int elfuse_launch(const launch_args_t *args) ? args->guest_argv[0] : args->elf_path; + /* Stage --user before bring-up: prepare's proc_init re-seeds the identity + * state, and build_linux_stack snapshots it into auxv AT_UID/AT_GID. + * Setting the ids after prepare would leave getauxval() reporting the + * default identity while getuid() reports the requested one. + */ + if (args->has_creds) + proc_set_initial_ids(args->uid, args->gid); + if (guest_bootstrap_prepare( &g, args->elf_path, elf_host_temp, elf_guest_path, args->sysroot, args->guest_argc, args->guest_argv, envp_use, shim_bin, @@ -88,6 +99,18 @@ int elfuse_launch(const launch_args_t *args) elf_host_temp = false; } + /* Reject GDB for a Rosetta (x86_64) guest here, not just in main(): the + * stub exposes the aarch64 shim's register/memory view, which is the wrong + * architecture for a Rosetta-translated x86_64 guest. main() rejects it up + * front via a static ELF probe, but enforcing it in elfuse_launch (once + * bring-up has set g.is_rosetta) makes every caller inherit the constraint, + * including the planned OCI run helper. + */ + if (args->gdb_port > 0 && g.is_rosetta) { + log_error("--gdb is not supported for x86_64 (Rosetta) guests"); + goto fail; + } + if (args->sysroot) { bool case_sensitive = true; bool case_preserving = true; @@ -100,6 +123,32 @@ int elfuse_launch(const launch_args_t *args) proc_set_sysroot_casefold(false); } + /* Apply the guest's initial working directory. The guest cwd IS the host + * process cwd (sys_chdir translates a guest path and calls host chdir), + * so --workdir DIR does the same: translate DIR against the sysroot (now + * that casefold is configured above) and chdir to the resulting host path, + * then refresh the cached guest-visible cwd so the first getcwd sees DIR. + * This mirrors the plain real-directory branch of sys_chdir; FUSE-mounted + * or /proc-virtual workdirs are not supported through this flag (neither + * is a realistic image WorkingDir). + */ + if (args->cwd_guest && args->cwd_guest[0] != '\0') { + path_translation_t tx; + if (path_translate_at(LINUX_AT_FDCWD, args->cwd_guest, PATH_TR_NONE, + &tx) < 0) { + log_error("failed to resolve working directory %s: %s", + args->cwd_guest, strerror(errno)); + goto fail; + } + if (chdir(tx.host_path) < 0) { + log_error("failed to set working directory %s: %s", args->cwd_guest, + strerror(errno)); + goto fail; + } + if (proc_cwd_refresh() < 0) + proc_cwd_invalidate(); + } + hv_vcpu_t vcpu; hv_vcpu_exit_t *vexit; if (guest_bootstrap_create_vcpu(&g, &boot, args->verbose, &vcpu, &vexit) < diff --git a/src/core/launch.h b/src/core/launch.h index 35499c4d..27ebd450 100644 --- a/src/core/launch.h +++ b/src/core/launch.h @@ -10,33 +10,16 @@ * Keeping bring-up behind one struct is what lets a front end select the * guest identity, cwd, and environment without a second bring-up path. * - * The function owns the guest_t, the vCPU, the GDB stub, the run loop, - * the diagnostic dumps, and guest teardown; it does NOT own the - * elf_path / sysroot / guest_argv heap copies or the sysroot_mount the - * host CLI may have provisioned; those stay with the caller so - * behaviors that need the original CLI argv (proctitle rewriting, - * --create-sysroot detach on exit, host cwd save+restore) remain - * coherent regardless of how the launch was kicked off. + * The function owns the guest_t, the vCPU, the GDB stub, the run loop, the + * diagnostic dumps, and guest teardown; it does NOT own the elf_path / + * sysroot / guest_argv heap copies or the sysroot_mount the host CLI may + * have provisioned. Those stay with the caller so behaviors that need the + * original CLI argv (proctitle rewriting, --create-sysroot detach on exit, + * host cwd save+restore) stay coherent however the launch was kicked off. * - * Lifetime / ownership contract: - * - * - The caller owns every pointer in launch_args_t. elfuse_launch reads - * them and does not free them; const-qualified pointers stay valid - * for the duration of the call. - * - envp may be NULL; the host process environ is used in that case. - * - guest_argv is the string array the guest sees as its argv. - * guest_argv[0] is the guest-visible entrypoint path (what the guest - * reads back via /proc/self/exe and argv[0]); elf_path is the - * resolved host path to that binary. The two differ when path - * translation or a FUSE-materialized temp is involved. - * - elf_host_temp is true when elf_path is a FUSE-materialized temp - * that must be unlinked once guest_bootstrap_prepare has loaded it - * (skipped for Rosetta guests, which reopen the path). The caller - * owns the unlink for any pre-prepare failure; elfuse_launch owns it - * from the prepare call onward. - * - fork_child_fd / vfork_notify_fd are forwarded for fork-child-routed - * launches; main() dispatches the fork-child path before reaching - * elfuse_launch and so passes -1. + * The caller owns every pointer in launch_args_t for the duration of the + * call; elfuse_launch reads but never frees them. Per-field lifetime and + * ownership notes live on the struct members below. */ #pragma once @@ -45,13 +28,16 @@ #include typedef struct { - /* Host filesystem path to the guest ELF (resolved; may be a - * FUSE-materialized temp when elf_host_temp is true). + /* Host path to the guest ELF; may be a FUSE-materialized temp when + * elf_host_temp is set. */ const char *elf_path; - /* True when elf_path is a temp to unlink after guest_bootstrap_prepare - * loads it. + /* elf_path is a FUSE-materialized temp to unlink once + * guest_bootstrap_prepare has loaded it (kept for Rosetta guests, which + * reopen the path). The caller owns the unlink on any pre-prepare + * failure; elfuse_launch owns it from the prepare call onward, + * including a prepare that fails. */ bool elf_host_temp; @@ -60,8 +46,10 @@ typedef struct { */ const char *sysroot; - /* String array the guest sees as its argv. guest_argv[0] is the - * guest-visible entrypoint path. + /* Argv the guest sees. guest_argv[0] is the guest-visible entrypoint + * path (what the guest reads back via /proc/self/exe and argv[0]); it + * differs from elf_path (the resolved host path) under path translation + * or a FUSE-materialized temp. */ int guest_argc; const char **guest_argv; @@ -72,6 +60,21 @@ typedef struct { */ char **envp; + /* When true, stage uid/gid as the guest identity before bring-up so the + * auxv AT_UID/AT_GID snapshot and getuid()/getgid() agree. When false, + * uid/gid are ignored and the guest runs under the compile-time default + * GUEST_UID/GUEST_GID (0 under fakeroot), NOT the host identity; a + * launcher that wants the host identity must set has_creds and pass + * getuid()/getgid(). + */ + bool has_creds; + uint32_t uid, gid; + + /* Guest-absolute initial working directory. NULL inherits the host + * cwd (the caller may chdir first to control it). + */ + const char *cwd_guest; + /* GDB Remote Serial Protocol port (0 disables the stub) and whether * to halt before the first guest instruction. */ diff --git a/src/main.c b/src/main.c index dd1c8e7c..763d5438 100644 --- a/src/main.c +++ b/src/main.c @@ -12,7 +12,8 @@ * - Guest memory identity-mapped at GVA=GPA with 2MiB block page tables. * - Syscall handlers that translate Linux syscalls to macOS equivalents. * - * Usage: elfuse [--verbose] [--timeout N] [--sysroot PATH] [args...] + * Usage: elfuse [options] [args...]; `elfuse --help` lists the + * options, which ELFUSE_USAGE below defines so the two cannot drift. */ #include @@ -104,6 +105,127 @@ static void free_guest_argv(const char **guest_argv, int guest_argc) free((void *) guest_argv); } +/* Free a guest envp vector produced by build_guest_env. Each entry is a + * heap "KEY=VAL" string owned by us (never a borrowed environ pointer), so + * free every slot then the array. A NULL envp (meaning "use host environ") + * is a no-op. + */ +static void free_envp(char **envp) +{ + if (!envp) + return; + for (char **e = envp; *e; e++) + free(*e); + free((void *) envp); +} + +/* Free the raw --env override array collected during option parsing. These + * strings are distinct from build_guest_env's output (which strdups its own + * copies), so both must be freed. + */ +static void free_env_overrides(char **env_overrides, int n) +{ + free_guest_argv((const char **) env_overrides, n); +} + +/* Build the guest environment vector, mirroring `env(1)` semantics. Returns 0 + * and sets *out_envp to either: + * - NULL when no --env/--clear-env was given, meaning "use the host environ + * as-is" (the pre-flag behavior, preserved exactly), or + * - a malloc'd, NULL-terminated char** of strdup'd "KEY=VAL" strings (caller + * frees with free_envp): the base is the host environ, or empty under + * clear_env, and each override replaces a matching KEY= in place or + * appends. "KEY=VAL" sets; a bare "KEY" inherits KEY from the host environ, + * skipped when unset so it can never create an empty-string variable. Returns + * -1 on allocation failure (out_envp untouched). + */ +static int build_guest_env(char *const *overrides, + int n_overrides, + bool clear_env, + char ***out_envp) +{ + if (n_overrides == 0 && !clear_env) { + *out_envp = NULL; + return 0; + } + + extern char **environ; + int cap = 1; /* NULL terminator */ + if (!clear_env) + for (char **e = environ; *e; e++) + cap++; + cap += n_overrides; + + char **envp = (char **) calloc((size_t) cap, sizeof(char *)); + if (!envp) + return -1; + int n = 0; + + if (!clear_env) { + for (char **e = environ; *e; e++) { + envp[n] = strdup(*e); + if (!envp[n]) + goto fail; + n++; + } + } + + for (int i = 0; i < n_overrides; i++) { + const char *ov = overrides[i]; + const char *eq = strchr(ov, '='); + /* Reject an empty variable name ("--env =VAL", or a bare "--env ""): + * eq == ov (or an empty ov) means a zero-length key. setenv(3), whose + * semantics --env mirrors, rejects an empty name, and appending + * "=VAL" verbatim would hand the guest a malformed environ entry the + * dedup scan cannot match. */ + if ((size_t) (eq ? eq - ov : strlen(ov)) == 0) { + log_error("invalid --env entry \"%s\": empty variable name", ov); + goto fail; + } + char *entry; + if (eq) { + entry = strdup(ov); + } else { + const char *val = getenv(ov); + if (!val) + continue; /* bare KEY, unset on host: skip */ + size_t need = strlen(ov) + 1 + strlen(val) + 1; + entry = (char *) malloc(need); + if (entry) + snprintf(entry, need, "%s=%s", ov, val); + } + if (!entry) + goto fail; + + size_t klen = (size_t) (eq ? eq - ov : strlen(ov)); + int found = -1; + for (int j = 0; j < n; j++) { + if (envp[j] && strncmp(envp[j], ov, klen) == 0 && + envp[j][klen] == '=') { + found = j; + break; + } + } + if (found >= 0) { + free(envp[found]); + envp[found] = entry; + } else { + /* cap is an exact upper bound (host environ + n_overrides + the + * NULL slot) and each override appends at most once, so the + * append can never outgrow the allocation. + */ + envp[n++] = entry; + } + } + envp[n] = NULL; + *out_envp = envp; + return 0; + +fail: + free_envp(envp); + return -1; +} + /* Releases the host state main() owns: the sysroot mount, the host cwd, and the * heap copies of argv. The guest itself belongs to elfuse_launch, which * destroys it on every exit path, so nothing here touches HVF or guest memory. @@ -127,12 +249,6 @@ static void cleanup_main_resources(sysroot_mount_t *sysroot_mount, free((void *) sysroot_path); } -/* The embedded shim binary (shim_blob.h, generated by xxd -i from shim.bin) - * is now included by src/core/launch.c, the single site that hands the blob to - * guest_bootstrap_prepare. main() no longer references shim_bin/shim_bin_len - * directly, so the static blob has one object definition site. - */ - /* The infra-reserve layout invariants documented in guest.h are derived from * raw offset constants, so a future edit that grows the pool by shifting one * offset without the others would silently overlap two regions. Enforce them at @@ -224,6 +340,17 @@ static int host_dc_zva_assert(void) return 0; } +/* One-line usage synopsis shared by the argument-error paths; --help prints + * the long multi-line form. A single definition keeps the copies from + * drifting (one copy had already lost the --gdb flags). + */ +#define ELFUSE_USAGE \ + "usage: elfuse [--verbose] [--timeout N] " \ + "[--sysroot PATH] [--create-sysroot PATH] [--no-rosetta] " \ + "[--fakeroot] [--gdb PORT] [--gdb-stop-on-entry] " \ + "[--user UID[:GID]] [--workdir DIR] [--env KEY=VAL] " \ + "[--clear-env] [args...]" + int main(int argc, char **argv) { log_init(); @@ -247,7 +374,24 @@ int main(int argc, char **argv) int gdb_port = 0; bool gdb_stop_on_entry = false; bool fakeroot = false; + /* Launch flags driven by `elfuse-oci run` (and usable directly). They + * map onto launch_args_t fields; --user overrides the guest identity, + * --workdir sets the guest's initial cwd, --env/--clear-env build the + * guest environment. All are additive: existing flags are unchanged. + */ + bool has_creds = false; + uint32_t uid = 0, gid = 0; + char *workdir = NULL; + char **env_overrides = NULL; + int n_env_overrides = 0, env_cap = 0; + bool clear_env = false; int arg_start = 1; + /* The heap copies of the ELF and sysroot paths are declared here, ahead of + * every `goto fail_parse` below, so that unwind can free them: a goto that + * skipped their initializers would leave it freeing indeterminate pointers. + */ + char *elf_path = NULL; + char *sysroot_path = NULL; /* 'elfuse rosettad translate ' runs the real Apple rosettad * binary inside an elfuse guest to materialise an AOT translation. The @@ -280,6 +424,8 @@ int main(int argc, char **argv) " [--create-sysroot PATH]\n" " [--no-rosetta] [--fakeroot]\n" " [--gdb PORT] [--gdb-stop-on-entry]\n" + " [--user UID[:GID]] [--workdir DIR]\n" + " [--env KEY=VAL] [--clear-env]\n" " [args...]\n" "\n" "Options:\n" @@ -301,6 +447,16 @@ int main(int argc, char **argv) "Protocol on PORT\n" " --gdb-stop-on-entry Halt before the first guest " "instruction\n" + " --user UID[:GID] Run the guest as UID (and GID; " + "defaults to UID). Numeric; elfuse-oci resolves symbolic " + "names\n" + " --workdir DIR Guest-absolute initial working " + "directory (resolved under --sysroot)\n" + " --env KEY=VAL Set a guest environment variable; " + "repeatable. 'KEY' (no '=') inherits from the host environ\n" + " --clear-env Start the guest environment empty " + "(only --env entries apply); default inherits the host " + "environ\n" "\n" "Environment:\n" " ELFUSE_NO_ROSETTA=1 Same as --no-rosetta\n" @@ -373,24 +529,97 @@ int main(int argc, char **argv) } else if (!strcmp(argv[arg_start], "--gdb-stop-on-entry")) { gdb_stop_on_entry = true; arg_start++; + } else if (!strcmp(argv[arg_start], "--user") && arg_start + 1 < argc) { + /* Numeric UID[:GID]; elfuse-oci resolves symbolic User + * against the image /etc/passwd+group and passes numbers. A bare + * UID sets gid=uid (typical single-user image). + */ + const char *spec = argv[arg_start + 1]; + char *end; + errno = 0; + unsigned long u = strtoul(spec, &end, 10); + if (errno || end == spec || u > UINT32_MAX) { + log_error("invalid --user UID: %s", spec); + goto fail_parse; + } + unsigned long gg = u; + if (*end == ':') { + errno = 0; + char *end2; + gg = strtoul(end + 1, &end2, 10); + if (errno || end2 == end + 1 || *end2 != '\0' || + gg > UINT32_MAX) { + log_error("invalid --user UID:GID: %s", spec); + goto fail_parse; + } + } else if (*end != '\0') { + log_error("invalid --user spec: %s", spec); + goto fail_parse; + } + uid = (uint32_t) u; + gid = (uint32_t) gg; + has_creds = true; + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--workdir") && + arg_start + 1 < argc) { + /* Guest-absolute working directory; elfuse_launch translates it + * against the sysroot and chdirs there. Reject relative paths up + * front: translation would resolve them against the host cwd, + * silently starting the guest outside the intended tree. strdup + * now because runtime_set_process_title clobbers the original + * argv block. + */ + if (argv[arg_start + 1][0] != '/') { + log_error("--workdir requires a guest-absolute path, got %s", + argv[arg_start + 1]); + goto fail_parse; + } + free(workdir); + workdir = strdup(argv[arg_start + 1]); + if (!workdir) { + log_error("out of memory"); + goto fail_parse; + } + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--env") && arg_start + 1 < argc) { + /* "KEY=VAL" sets; "KEY" inherits from the host environ (resolved + * in build_guest_env). strdup now; argv is clobbered later. + */ + if (n_env_overrides == env_cap) { + int ncap = env_cap ? env_cap * 2 : 8; + char **grown = (char **) realloc( + (void *) env_overrides, (size_t) ncap * sizeof(char *)); + if (!grown) { + log_error("out of memory"); + goto fail_parse; + } + env_overrides = grown; + env_cap = ncap; + } + env_overrides[n_env_overrides] = strdup(argv[arg_start + 1]); + if (!env_overrides[n_env_overrides]) { + log_error("out of memory"); + goto fail_parse; + } + n_env_overrides++; + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--clear-env")) { + clear_env = true; + arg_start++; } else if (!strcmp(argv[arg_start], "--")) { arg_start++; break; } else { log_error("unknown option: %s", argv[arg_start]); - log_error( - "usage: elfuse [--verbose] [--timeout N] " - "[--sysroot PATH] [--create-sysroot PATH] [--no-rosetta] " - "[--fakeroot] [--gdb PORT] " - "[--gdb-stop-on-entry] [args...]"); - return 1; + log_error(ELFUSE_USAGE); + goto fail_parse; } } if (sysroot && create_sysroot) { log_error( "use either --sysroot PATH or --create-sysroot PATH, not both"); - return 1; + goto fail_parse; } /* ELFUSE_NO_ROSETTA=1 mirrors --no-rosetta for environments where passing @@ -411,6 +640,20 @@ int main(int argc, char **argv) if (fakeroot_env && strcmp(fakeroot_env, "1") == 0) fakeroot = true; } + /* Fakeroot means the guest starts as uid/gid 0: proc_identity_init's + * defaults and the ELFUSE_FAKEROOT_EXEC transition both set root together + * with the flag, and uid_is_permitted() grants every setuid under fakeroot + * on that basis. A non-root --user would keep that grant while reporting an + * unprivileged identity, so the guest could call setuid(0) at will. Refuse + * the contradiction rather than silently handing out the privilege. + */ + if (fakeroot && has_creds && (uid != 0 || gid != 0)) { + log_error( + "--fakeroot runs the guest as uid/gid 0 and cannot be combined " + "with --user %u:%u", + uid, gid); + goto fail_parse; + } proc_set_fakeroot_enabled(fakeroot); /* Opt-in sudo-style transition: name one executable whose exec enters @@ -440,14 +683,14 @@ int main(int argc, char **argv) * internal host reserve. */ if (host_nofile_ensure_capacity() < 0) - return 1; + goto fail_parse; /* Block the vCPU-preemption signals and start the sigwait thread before any * vCPU thread exists, so both the normal path and the fork-child path below * inherit the block on every thread they spawn. */ if (proc_preempt_init() < 0) - return 1; + goto fail_parse; /* Fork-child mode: receive VM state over IPC and run */ if (fork_child_fd >= 0) @@ -455,10 +698,22 @@ int main(int argc, char **argv) timeout_sec); if (arg_start >= argc) { - log_error( - "usage: elfuse [--verbose] [--timeout N] " - "[--sysroot PATH] [--create-sysroot PATH] [--no-rosetta] " - "[--fakeroot] [args...]"); + log_error(ELFUSE_USAGE); + goto fail_parse; + } + + /* Shared unwind for argument-parsing errors: frees the heap state owned + * before the guest_argv copy below and exits 1. The if (0) wrapper makes + * the label reachable only by goto, and placing it before the later + * declarations keeps any goto from crossing into their scope; paths past + * that copy use the cleanup: label instead. + */ + if (0) { + fail_parse: + free(elf_path); + free(sysroot_path); + free_env_overrides(env_overrides, n_env_overrides); + free(workdir); return 1; } @@ -466,10 +721,9 @@ int main(int argc, char **argv) * data lives in a contiguous stack region that elfuse clobbers below for * the process title (PostgreSQL/nginx argv-clobber technique). */ - char *elf_path = strdup(argv[arg_start]); + elf_path = strdup(argv[arg_start]); bool have_sysroot = (sysroot != NULL || create_sysroot != NULL); const char *sysroot_src = create_sysroot ? create_sysroot : sysroot; - char *sysroot_path = NULL; if (have_sysroot) { sysroot_path = (char *) calloc(LINUX_PATH_MAX, 1); if (sysroot_path) { @@ -478,9 +732,7 @@ int main(int argc, char **argv) if (src_len >= LINUX_PATH_MAX) { log_error("sysroot path too long (%zu bytes, max %d): %s", src_len, LINUX_PATH_MAX - 1, sysroot_src); - free(elf_path); - free(sysroot_path); - return 1; + goto fail_parse; } } } @@ -493,6 +745,10 @@ int main(int argc, char **argv) char elf_host_path[LINUX_PATH_MAX]; bool elf_host_temp = false; bool have_host_cwd = (getcwd(host_cwd, sizeof(host_cwd)) != NULL); + /* Declared (and NULL-initialized) before the first `goto fail` so the + * shared cleanup below never frees an uninitialized pointer. + */ + char **envp = NULL; int exit_code; memset(&sysroot_mount, 0, sizeof(sysroot_mount)); if (!elf_path || (have_sysroot && !sysroot_path) || !guest_argv) { @@ -630,22 +886,37 @@ int main(int argc, char **argv) } } + /* Build the guest environment vector late (after the shebang loop and the + * --gdb guard) so only this block's OOM path and the post-launch cleanup + * must free it. With neither --env nor --clear-env given, build_guest_env + * leaves envp NULL and elfuse_launch uses the host environ (pre-flag + * behavior, preserved); it owns that condition, not this call site. + */ + if (build_guest_env(env_overrides, n_env_overrides, clear_env, &envp) < 0) { + /* build_guest_env has already logged the specific reason (OOM or a + * malformed --env entry). + */ + goto fail; + } + /* build_guest_env strdups its own copies, so the raw override array and + * its strings are no longer needed. + */ + free_env_overrides(env_overrides, n_env_overrides); + env_overrides = NULL; + n_env_overrides = 0; + /* Rewrite the host-visible process title from the guest entrypoint. This * clobbers the original argv block (already snapshotted into the heap * elf_path / guest_argv above), so it must run before elfuse_launch hands - * control to the guest but after the shebang loop has fixed elf_path. The - * call only touches the original argv and the kernel procname; it does not - * depend on guest_bootstrap_prepare having run, so hoisting it out of the - * bring-up (where it previously sat between prepare and create_vcpu) is - * behavior-preserving. + * control to the guest but after the shebang loop has fixed elf_path. */ runtime_set_process_title(argc, argv, elf_path); - /* Hand the bring-up, run loop, and guest teardown to elfuse_launch. main() - * retains ownership of the original argv (proctitle above), the sysroot - * mount (detached in cleanup_main_resources after the guest exits so the - * mount stays live for the whole run), host cwd, and the heap elf_path / - * sysroot_path / guest_argv copies. + /* Hand bring-up, run loop, and guest teardown to elfuse_launch. main() + * keeps ownership of the original argv (proctitle above), the sysroot mount + * (detached in cleanup_main_resources after the guest exits so it stays + * live for the whole run), host cwd, and the heap elf_path / sysroot_path / + * guest_argv / envp / workdir copies. */ launch_args_t largs = { .elf_path = elf_host_path, @@ -653,6 +924,11 @@ int main(int argc, char **argv) .sysroot = sysroot, .guest_argc = guest_argc, .guest_argv = guest_argv, + .envp = envp, + .has_creds = has_creds, + .uid = uid, + .gid = gid, + .cwd_guest = workdir, .gdb_port = gdb_port, .gdb_stop_on_entry = gdb_stop_on_entry, .timeout_sec = timeout_sec, @@ -671,8 +947,13 @@ int main(int argc, char **argv) cleanup: /* Single unwind for every exit past the heap-copy allocations: frees the * caller-owned heap copies, detaches the sysroot mount, restores the host - * cwd, and drops a still-owned FUSE-materialized temp ELF. + * cwd, and drops a still-owned FUSE-materialized temp ELF. A successful + * build_guest_env already freed and reset the override array, so freeing it + * here is then a no-op. */ + free_env_overrides(env_overrides, n_env_overrides); + free_envp(envp); + free(workdir); cleanup_main_resources(&sysroot_mount, have_host_cwd ? host_cwd : NULL, guest_argv, guest_argc, elf_path, sysroot_path); if (elf_host_temp) diff --git a/src/syscall/proc-identity.c b/src/syscall/proc-identity.c index c2f94339..412c242a 100644 --- a/src/syscall/proc-identity.c +++ b/src/syscall/proc-identity.c @@ -33,6 +33,10 @@ static _Atomic int32_t guest_has_ctty = 1; static _Atomic bool fakeroot_enabled = false; +static _Atomic bool initial_ids_staged = false; +static _Atomic uint32_t initial_uid = GUEST_UID; +static _Atomic uint32_t initial_gid = GUEST_GID; + void proc_set_fakeroot_enabled(bool enabled) { atomic_store(&fakeroot_enabled, enabled); @@ -64,6 +68,13 @@ const char *proc_fakeroot_exec_path(void) return fakeroot_exec_path[0] ? fakeroot_exec_path : NULL; } +void proc_set_initial_ids(uint32_t uid, uint32_t gid) +{ + atomic_store(&initial_uid, uid); + atomic_store(&initial_gid, gid); + atomic_store(&initial_ids_staged, true); +} + void proc_identity_init(void) { guest_pid = 1; @@ -78,6 +89,19 @@ void proc_identity_init(void) gid = 0; } + /* An explicit --user request wins over the defaults and over fakeroot. + * It is staged before init rather than applied afterwards because + * build_linux_stack snapshots these values into auxv AT_UID/AT_GID; a + * post-init override would leave getauxval() disagreeing with getuid(). + * Consume the staged value so it applies only to the bring-up it was + * staged for; a later launch in the same host process without credentials + * falls back to the defaults instead of inheriting the prior identity. + */ + if (atomic_exchange(&initial_ids_staged, false)) { + uid = atomic_load(&initial_uid); + gid = atomic_load(&initial_gid); + } + emu_uid = uid; emu_euid = uid; emu_suid = uid; diff --git a/src/syscall/proc.h b/src/syscall/proc.h index 6096f98b..4fa9c014 100644 --- a/src/syscall/proc.h +++ b/src/syscall/proc.h @@ -140,6 +140,14 @@ bool proc_set_fakeroot_exec_path(const char *path); */ const char *proc_fakeroot_exec_path(void); +/* Stage the initial guest credentials (--user) before proc_init. + * proc_identity_init applies them in place of the GUEST_UID/GUEST_GID + * defaults, so the auxv AT_UID/AT_GID snapshot taken by build_linux_stack + * matches what getuid()/getgid() later report. The staged value is consumed + * by the next proc_identity_init, so it applies to a single bring-up only. + */ +void proc_set_initial_ids(uint32_t uid, uint32_t gid); + /* Store the guest command line for /proc/self/cmdline emulation. argv is a * NULL-terminated array of strings. */ diff --git a/tests/test-launch-flags.sh b/tests/test-launch-flags.sh new file mode 100755 index 00000000..2a8f7253 --- /dev/null +++ b/tests/test-launch-flags.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# test-launch-flags.sh -- Pin the rejection rules of the guest launch flags +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Usage: tests/test-launch-flags.sh +# +# --user, --workdir, and --env select the guest identity, cwd, and +# environment. Each rejects a contradictory request during option parsing, +# before any VM is created, so a launcher (`elfuse-oci run` above all) gets a +# diagnostic instead of a guest that silently runs as something other than +# what was asked for. +# +# The --fakeroot lane covers a privilege-model rule rather than a typo: +# fakeroot means the guest starts as uid/gid 0, and the setuid permission +# check grants every id switch on that basis. Pairing it with a non-root +# --user left that grant in place while the guest reported an unprivileged +# uid, so the guest could call setuid(0) at will. The other lanes are +# regression guards for parse rules that already held. + +set -euo pipefail + +ELFUSE="${1:?Usage: $0 }" +GUEST="${2:?Usage: $0 }" + +fail=0 + +# Asserts elfuse exits nonzero and says why, without starting a guest. +reject() +{ + local desc="$1" want="$2" + shift 2 + local out status=0 + out="$("$ELFUSE" "$@" "$GUEST" 2>&1)" || status=$? + if [ "$status" -eq 0 ]; then + printf '[ FAIL ] %s: accepted (exit 0), want rejection\n' "$desc" + fail=1 + return + fi + if ! printf '%s' "$out" | grep -qF "$want"; then + printf '[ FAIL ] %s: exit %d but message lacks %q:\n%s\n' \ + "$desc" "$status" "$want" "$out" + fail=1 + return + fi + printf '[ OK ] %s\n' "$desc" +} + +accept() +{ + local desc="$1" + shift + local out status=0 + out="$("$ELFUSE" "$@" "$GUEST" 2>&1)" || status=$? + if [ "$status" -ne 0 ]; then + printf '[ FAIL ] %s: exit %d, want success:\n%s\n' "$desc" "$status" "$out" + fail=1 + return + fi + printf '[ OK ] %s\n' "$desc" +} + +reject "--fakeroot with a non-root --user" "cannot be combined" \ + --fakeroot --user 1000:1000 +reject "--fakeroot with a root uid but non-root gid" "cannot be combined" \ + --fakeroot --user 0:1000 +reject "--workdir relative path" "absolute" --workdir rel/path +reject "--user non-numeric" "invalid --user" --user alice + +# --fakeroot and --user agree here, so the pair must still launch: the check +# refuses a contradiction, not the combination itself. +accept "--fakeroot with an explicit root --user" --fakeroot --user 0:0 + +if [ "$fail" -ne 0 ]; then + printf 'test-launch-flags: FAILED\n' + exit 1 +fi +printf 'test-launch-flags: all checks passed\n' From c88b4b54c104b58026a07a166d5fa5be233e5300 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:22:51 +0200 Subject: [PATCH 03/26] Add elfuse-oci store with pull and inspect elfuse-oci is a standalone Go binary that owns the OCI image pipeline; elfuse itself stays a pure Linux syscall-to-Darwin runtime with no OCI commands. This first slice is the acquisition half: an OCI image-layout store plus the pull and inspect commands, built on go-containerregistry ($ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci by default). The store is a spec-shape image layout other tools can read, with a refs.json pin table mapping references to manifest digests. An exclusive flock serializes refs.json/index.json updates so concurrent pulls cannot lose pins (the cold-store bootstrap of oci-layout and index.json runs under the same lock, double-checked so a warm store skips it), pin persistence syncs the temp file and directory around the rename, and a nil-object refs.json is rejected as corrupt instead of treated as empty. addImage distinguishes genuinely-absent from unreadable descriptors by scanning index membership, so store corruption surfaces rather than duplicating entries. digestFor returns a distinct errNotPulled for a missing ref so later callers can tell "not pulled" from "store broken". Two helpers keep the plumbing in one place: every subcommand opens the store through commonFlags.openResolvedStore (resolve the store path, then open the layout), and store.withLock scopes lock-held sections; pin and addImage wrap their load-modify-save cycles in it so a critical section cannot leak its lock on an error path. pull resolves the requested platform (default linux/arm64) and validates --platform shape up front; inspect prints the manifest and config summary (or --json) and propagates digest/size and writer errors instead of reporting partial output as success. Credentials come from the ambient default keychain, but its resolution is time-bounded: it shells out to whatever helper the Docker config names, and go-containerregistry drops the context around that exec, so a wedged helper would otherwise hang the pull with no output. A wrapper keychain caps the wait and fails with an explanation and the DOCKER_CONFIG escape hatch instead; a progress line is printed before the pull so it is never silent. --platform is registered per command rather than in the shared flag set, so a subcommand that cannot honor it rejects the flag instead of silently discarding target selection. The test output-capture helper closes its pipe ends in a defer, so a callee that panics or Fatals cannot park the reader goroutines on open write ends. Tests cover the pin table, store locking, error kinds, flag parsing, and the command dispatch, with cranePull as a swappable seam so no test touches the network. `make all` builds elfuse-oci when a Go toolchain is on PATH and skips it with a notice otherwise, so a Go host gets both binaries by default while a C-only host still builds. The Go rule depends on the same version metadata as the C build, so an incremental rebuild restamps --version after a checkout or commit. --- Makefile | 29 +++ cmd/elfuse-oci/commands.go | 60 ++++++ cmd/elfuse-oci/common.go | 192 ++++++++++++++++++ cmd/elfuse-oci/common_test.go | 97 +++++++++ cmd/elfuse-oci/inspect.go | 84 ++++++++ cmd/elfuse-oci/inspect_test.go | 155 +++++++++++++++ cmd/elfuse-oci/keychain.go | 56 ++++++ cmd/elfuse-oci/keychain_test.go | 74 +++++++ cmd/elfuse-oci/main.go | 85 ++++++++ cmd/elfuse-oci/main_command_test.go | 84 ++++++++ cmd/elfuse-oci/pull.go | 50 +++++ cmd/elfuse-oci/store.go | 294 ++++++++++++++++++++++++++++ cmd/elfuse-oci/store_test.go | 247 +++++++++++++++++++++++ cmd/elfuse-oci/test_helpers_test.go | 165 ++++++++++++++++ go.mod | 17 ++ go.sum | 30 +++ mk/toolchain.mk | 5 + 17 files changed, 1724 insertions(+) create mode 100644 cmd/elfuse-oci/commands.go create mode 100644 cmd/elfuse-oci/common.go create mode 100644 cmd/elfuse-oci/common_test.go create mode 100644 cmd/elfuse-oci/inspect.go create mode 100644 cmd/elfuse-oci/inspect_test.go create mode 100644 cmd/elfuse-oci/keychain.go create mode 100644 cmd/elfuse-oci/keychain_test.go create mode 100644 cmd/elfuse-oci/main.go create mode 100644 cmd/elfuse-oci/main_command_test.go create mode 100644 cmd/elfuse-oci/pull.go create mode 100644 cmd/elfuse-oci/store.go create mode 100644 cmd/elfuse-oci/store_test.go create mode 100644 cmd/elfuse-oci/test_helpers_test.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/Makefile b/Makefile index 049863bd..cbbfe4c9 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,19 @@ endef .PHONY: all elfuse .PHONY: gen-syscall-dispatch check-syscall-dispatch +# `make all` builds everything the host toolchain supports: the C runtime +# always, and elfuse-oci when a Go toolchain is on PATH. A host without Go +# still builds elfuse, which is why this is a detection rather than a plain +# dependency; `make elfuse-oci` names the target directly and fails loudly +# when Go is missing. +HAVE_GO := $(shell command -v $(GO) > /dev/null 2>&1 && echo yes) + +ifeq ($(HAVE_GO),yes) +all: elfuse elfuse-oci +else all: elfuse + @echo " NOTE elfuse-oci skipped: no '$(GO)' toolchain on PATH" +endif ## Regenerate build/dispatch.h from src/syscall/dispatch.tbl gen-syscall-dispatch: @@ -127,6 +139,23 @@ elfuse: $(ELFUSE_BIN) $(ELFUSE_BIN): $(OBJS) | $(BUILD_DIR) $(call link-and-sign,$@,$(OBJS)) +# OCI image CLI (Go). Pure Go, no HVF entitlement or codesigning required, +# so it also builds under Linux for spec-conformance / interop CI. The version +# is stamped from the same VERSION string the C binary uses. +OCI_BIN := $(BUILD_DIR)/elfuse-oci +OCI_SRCS := $(shell find cmd/elfuse-oci -type f -name '*.go' 2>/dev/null) + +.PHONY: elfuse-oci +elfuse-oci: $(OCI_BIN) + +# rm -f first: `go build -o` follows an existing symlink at the output path, +# so a stale build/elfuse-oci symlink would clobber build/elfuse. +$(OCI_BIN): go.mod $(OCI_SRCS) $(VERSION_DEPS) | $(BUILD_DIR) + @echo " GO $@" + $(Q)rm -f $@ + $(Q)cd $(CURDIR) && $(GO) build -ldflags "-X main.version=$(VERSION)" \ + -o $@ ./cmd/elfuse-oci + # Native test binaries (macOS, Hypervisor.framework) ## Build the multi-vCPU HVF validation test (native macOS binary) diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go new file mode 100644 index 00000000..ddd8b04e --- /dev/null +++ b/cmd/elfuse-oci/commands.go @@ -0,0 +1,60 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" +) + +// The subcommands share common-flag parsing (common.go) and the OCI +// image-layout store (store.go). pull and inspect are pure store ops. + +// cmdPull implements `elfuse-oci pull [--store] [--platform] `. +func cmdPull(args []string) error { + cf, ref, err := parsePullArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return pullImage(cf, s, ref) +} + +func parsePullArgs(args []string) (commonFlags, string, error) { + var cf commonFlags + fs := newCommandFlagSet("pull", &cf) + addPlatformFlag(fs, &cf) + if err := fs.Parse(args); err != nil { + return cf, "", err + } + ref, err := oneArg("pull", fs.Args(), "") + return cf, ref, err +} + +// cmdInspect implements `elfuse-oci inspect [--store] [--json] `. +func cmdInspect(args []string) error { + cf, asJSON, ref, err := parseInspectArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return inspect(os.Stdout, s, ref, asJSON) +} + +func parseInspectArgs(args []string) (commonFlags, bool, string, error) { + var cf commonFlags + var asJSON bool + fs := newCommandFlagSet("inspect", &cf) + fs.BoolVar(&asJSON, "json", false, "") + if err := fs.Parse(args); err != nil { + return cf, false, "", err + } + ref, err := oneArg("inspect", fs.Args(), "") + return cf, asJSON, ref, err +} diff --git a/cmd/elfuse-oci/common.go b/cmd/elfuse-oci/common.go new file mode 100644 index 00000000..11e6dd99 --- /dev/null +++ b/cmd/elfuse-oci/common.go @@ -0,0 +1,192 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// version is stamped at build time via -ldflags "-X main.version=...". The +// default "dev" is what `go build` without the stamp produces. +var version = "dev" + +// Platform is an OCI platform triple. Variant is optional (e.g. "v8" for +// arm64); empty means "the default variant for this arch". +type Platform struct { + OS string + Arch string + Variant string +} + +func (p Platform) String() string { + if p.Variant != "" { + return p.OS + "/" + p.Arch + "/" + p.Variant + } + return p.OS + "/" + p.Arch +} + +// Set implements flag.Value for --platform. +func (p *Platform) Set(s string) error { + parsed, err := parsePlatform(s) + if err != nil { + return err + } + *p = parsed + return nil +} + +// defaultPlatform is linux/arm64: elfuse runs aarch64-linux guests natively via +// HVF, and x86_64 guests via Rosetta. elfuse-oci targets arm64 by default; +// --platform selects another (e.g. linux/amd64 for an x86_64 image run under +// Rosetta). +var defaultPlatform = Platform{OS: "linux", Arch: "arm64"} + +// formatCreated renders a config's creation time in the CLI's fixed +// second-precision UTC form, shared so inspect and list print one format. +func formatCreated(cfg *v1.ConfigFile) string { + return cfg.Created.UTC().Format("2006-01-02T15:04:05Z") +} + +// parsePlatform parses "os/arch" or "os/arch/variant". The value must have +// exactly two or three slash-separated components, each non-empty: "linux//", +// "/arm64", "linux/arm64/", and "linux/arm64/v8/extra" are all rejected rather +// than riding through to the registry client as a nonsense platform. +func parsePlatform(s string) (Platform, error) { + parts := strings.Split(s, "/") + if len(parts) < 2 || len(parts) > 3 || slices.Contains(parts, "") { + return Platform{}, fmt.Errorf("invalid --platform %q (want os/arch[/variant])", s) + } + if len(parts) == 3 { + return Platform{OS: parts[0], Arch: parts[1], Variant: parts[2]}, nil + } + return Platform{OS: parts[0], Arch: parts[1]}, nil +} + +// defaultStore returns the OCI store directory: $ELFUSE_OCI_STORE if set, +// otherwise ~/.local/share/elfuse/oci. The store is an OCI image-layout +// (blobs/, index.json) plus a ref->digest pin table (see store.go). +func defaultStore() (string, error) { + if s := os.Getenv("ELFUSE_OCI_STORE"); s != "" { + return s, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("no --store given and $HOME unset: %w", err) + } + return filepath.Join(home, ".local", "share", "elfuse", "oci"), nil +} + +// commonFlags holds the flags shared by every subcommand. +type commonFlags struct { + store string + platform Platform + // platformSet records an explicit --platform: run uses it to validate the + // pinned image, while the default platform stays advisory so a ref pulled + // for another platform still runs without re-specifying --platform. + platformSet bool +} + +// platformFlag adapts commonFlags.platform to flag.Value while recording +// that the flag was set explicitly. +type platformFlag struct{ cf *commonFlags } + +func (pf platformFlag) String() string { + if pf.cf == nil { + return "" + } + return pf.cf.platform.String() +} + +func (pf platformFlag) Set(s string) error { + if err := pf.cf.platform.Set(s); err != nil { + return err + } + pf.cf.platformSet = true + return nil +} + +// resolveStore fills cf.store with the default when unset and ensures the +// directory exists. +func (cf *commonFlags) resolveStore() error { + if cf.store == "" { + s, err := defaultStore() + if err != nil { + return err + } + cf.store = s + } + return os.MkdirAll(cf.store, 0o755) +} + +// openResolvedStore is every subcommand's store preamble: resolve the store +// path (defaulting and creating it) and open the layout. +func (cf *commonFlags) openResolvedStore() (*store, error) { + if err := cf.resolveStore(); err != nil { + return nil, err + } + return openStore(cf.store) +} + +// newCommandFlagSet creates a FlagSet whose parse errors are returned (not +// exited on) so main reports them uniformly, while ` -h` and a bad flag +// still print that subcommand's own flag list. The FlagSet's own error line is +// discarded (main prints the returned error); the Usage closure writes the flag +// list straight to stderr so it survives regardless. +func newCommandFlagSet(name string, cf *commonFlags) *flag.FlagSet { + *cf = commonFlags{platform: defaultPlatform} + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: elfuse-oci %s [flags]\n", name) + fs.SetOutput(os.Stderr) + fs.PrintDefaults() + fs.SetOutput(io.Discard) + } + fs.StringVar(&cf.store, "store", "", "OCI store directory (default $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci)") + return fs +} + +// addPlatformFlag registers --platform on the commands that resolve a +// platform (pull, and run's pinned-image agreement check). The other +// subcommands must not accept it: a successful parse that discards the flag +// reads as target selection that never happened. +func addPlatformFlag(fs *flag.FlagSet, cf *commonFlags) { + fs.Var(platformFlag{cf}, "platform", "target platform os/arch[/variant]") +} + +type repeatedStringFlag []string + +func (f *repeatedStringFlag) String() string { + if f == nil { + return "" + } + return strings.Join(*f, ",") +} + +func (f *repeatedStringFlag) Set(s string) error { + *f = append(*f, s) + return nil +} + +func oneArg(cmd string, args []string, what string) (string, error) { + if len(args) != 1 { + return "", fmt.Errorf("%s: expected one %s, got %d", cmd, what, len(args)) + } + return args[0], nil +} + +func noArgs(cmd string, args []string) error { + if len(args) != 0 { + return fmt.Errorf("%s: takes no argument", cmd) + } + return nil +} diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go new file mode 100644 index 00000000..e981938a --- /dev/null +++ b/cmd/elfuse-oci/common_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "reflect" + "testing" +) + +func TestParsePlatform(t *testing.T) { + cases := []struct { + in string + want Platform + wantErr bool + }{ + {"linux/arm64", Platform{OS: "linux", Arch: "arm64"}, false}, + {"linux/amd64/v8", Platform{OS: "linux", Arch: "amd64", Variant: "v8"}, false}, + {"darwin/arm64", Platform{OS: "darwin", Arch: "arm64"}, false}, + {"linux", Platform{}, true}, + {"", Platform{}, true}, + {"linux//", Platform{}, true}, + {"/arm64", Platform{}, true}, + {"linux//v8", Platform{}, true}, + {"linux/arm64/", Platform{}, true}, + {"linux/arm64/v8/extra", Platform{}, true}, + {"//", Platform{}, true}, + } + for _, c := range cases { + got, err := parsePlatform(c.in) + if (err != nil) != c.wantErr { + t.Errorf("parsePlatform(%q): err=%v, wantErr=%v", c.in, err, c.wantErr) + continue + } + if c.wantErr { + continue + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("parsePlatform(%q): got %+v, want %+v", c.in, got, c.want) + } + if got.String() != c.in { + t.Errorf("Platform(%q).String() = %q, want %q", c.in, got.String(), c.in) + } + } +} + +func TestParsePullArgs(t *testing.T) { + cf, ref, err := parsePullArgs([]string{"--store", "/s", "--platform", "linux/amd64", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if ref != "alpine:3" { + t.Fatalf("ref = %q, want alpine:3", ref) + } + if cf.store != "/s" { + t.Errorf("store = %q, want /s", cf.store) + } + if !reflect.DeepEqual(cf.platform, Platform{OS: "linux", Arch: "amd64"}) { + t.Errorf("platform = %+v, want linux/amd64", cf.platform) + } +} + +func TestParseInspectArgs(t *testing.T) { + _, asJSON, ref, err := parseInspectArgs([]string{"--json", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if !asJSON { + t.Error("asJSON = false, want true") + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } +} + +func TestParseCommandFlagErrors(t *testing.T) { + cases := []struct { + name string + err error + }{ + {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, + {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, + {"missing ref", func() error { _, _, err := parsePullArgs(nil); return err }()}, + // --platform belongs to the commands that resolve a platform; + // inspect discarding a successfully parsed flag would mislead the + // caller. + {"inspect rejects platform", func() error { + _, _, _, err := parseInspectArgs([]string{"--platform", "linux/amd64", "alpine:3"}) + return err + }()}, + } + for _, tc := range cases { + if tc.err == nil { + t.Errorf("%s: got nil error", tc.name) + } + } +} diff --git a/cmd/elfuse-oci/inspect.go b/cmd/elfuse-oci/inspect.go new file mode 100644 index 00000000..0a33591e --- /dev/null +++ b/cmd/elfuse-oci/inspect.go @@ -0,0 +1,84 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "fmt" + "io" +) + +// inspect prints a stored image's manifest + config. With --json the raw config +// JSON is emitted (one object); otherwise a human-readable summary. +func inspect(w io.Writer, s *store, ref string, asJSON bool) error { + img, err := s.image(ref) + if err != nil { + return err + } + d, err := img.Digest() + if err != nil { + return err + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + cf := cfg.Config + + if asJSON { + // The raw config blob, not a re-marshal of the parsed struct: fields + // ggcr does not model (vendor extensions) must survive, and the bytes + // should diff cleanly against `skopeo inspect --config`. + b, err := img.RawConfigFile() + if err != nil { + return err + } + // A failed write (closed pipe, full disk on a redirect) must not + // exit 0: callers would consume truncated JSON as success. + if _, err := fmt.Fprintf(w, "%s\n", b); err != nil { + return fmt.Errorf("inspect: write: %w", err) + } + return nil + } + + // bufio latches the first write error and reports it at Flush, so the + // summary can print unconditionally without an if around every line. + bw := bufio.NewWriter(w) + w = bw + + fmt.Fprintf(w, "%-12s %s\n", "Ref:", ref) + fmt.Fprintf(w, "%-12s %s\n", "Digest:", d) + fmt.Fprintf(w, "%-12s %s/%s\n", "Platform:", cfg.OS, cfg.Architecture) + if !cfg.Created.IsZero() { + fmt.Fprintf(w, "%-12s %s\n", "Created:", formatCreated(cfg)) + } + fmt.Fprintf(w, "%-12s %v\n", "Entrypoint:", cf.Entrypoint) + fmt.Fprintf(w, "%-12s %v\n", "Cmd:", cf.Cmd) + fmt.Fprintf(w, "%-12s %s\n", "WorkingDir:", cf.WorkingDir) + fmt.Fprintf(w, "%-12s %s\n", "User:", cf.User) + fmt.Fprintf(w, "Env (%d):\n", len(cf.Env)) + for _, e := range cf.Env { + fmt.Fprintf(w, " %s\n", e) + } + layers, err := img.Layers() + if err != nil { + return err + } + fmt.Fprintf(w, "Layers (%d):\n", len(layers)) + for i, l := range layers { + ld, err := l.Digest() + if err != nil { + return fmt.Errorf("inspect: layer %d digest: %w", i, err) + } + ls, err := l.Size() + if err != nil { + return fmt.Errorf("inspect: layer %d size: %w", i, err) + } + fmt.Fprintf(w, " %2d %s %d bytes\n", i, ld, ls) + } + if err := bw.Flush(); err != nil { + return fmt.Errorf("inspect: write: %w", err) + } + return nil +} diff --git a/cmd/elfuse-oci/inspect_test.go b/cmd/elfuse-oci/inspect_test.go new file mode 100644 index 00000000..0a2b240b --- /dev/null +++ b/cmd/elfuse-oci/inspect_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +// TestInspectHuman asserts the human-readable summary surfaces the ref, +// platform, and command. Substring checks (not column spacing) keep it robust +// to formatting tweaks. +func TestInspectHuman(t *testing.T) { + s := openTestStore(t) + ref := "local:tiny" + if _, err := s.addImage(ref, tinyImage(t)); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := inspect(&buf, s, ref, false); err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{"local:tiny", "Platform:", "linux/arm64", "Cmd:", "/hello"} { + if !strings.Contains(out, want) { + t.Errorf("inspect output missing %q:\n%s", want, out) + } + } +} + +// TestInspectJSON asserts --json emits a valid v1.ConfigFile with the tiny +// image's architecture/os/cmd. +func TestInspectJSON(t *testing.T) { + s := openTestStore(t) + ref := "local:tiny" + if _, err := s.addImage(ref, tinyImage(t)); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := inspect(&buf, s, ref, true); err != nil { + t.Fatal(err) + } + var cf v1.ConfigFile + if err := json.Unmarshal(buf.Bytes(), &cf); err != nil { + t.Fatalf("json unmarshal: %v (raw %q)", err, buf.String()) + } + if cf.Architecture != "arm64" { + t.Errorf("Architecture: got %q, want arm64", cf.Architecture) + } + if cf.OS != "linux" { + t.Errorf("OS: got %q, want linux", cf.OS) + } + if len(cf.Config.Cmd) != 1 || cf.Config.Cmd[0] != "/hello" { + t.Errorf("Cmd: got %v, want [/hello]", cf.Config.Cmd) + } +} + +func imageWithRichConfig(t *testing.T) v1.Image { + t.Helper() + img := tinyImage(t) + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cfg.Created = v1.Time{Time: time.Date(2026, 7, 9, 12, 34, 56, 0, time.FixedZone("test", 2*60*60))} + cfg.Config.Entrypoint = []string{"/entry"} + cfg.Config.Cmd = []string{"arg"} + cfg.Config.Env = []string{"A=1", "B=2"} + cfg.Config.WorkingDir = "/work" + cfg.Config.User = "1000:1000" + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatal(err) + } + return img +} + +func TestInspectHumanIncludesCreatedEnvAndLayers(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:rich", imageWithRichConfig(t)); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := inspect(&buf, s, "local:rich", false); err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{ + "Ref: local:rich", + "Created: 2026-07-09T10:34:56Z", + "Entrypoint: [/entry]", + "Cmd: [arg]", + "WorkingDir: /work", + "User: 1000:1000", + "Env (2):", + "A=1", + "Layers (1):", + } { + if !strings.Contains(out, want) { + t.Fatalf("inspect output missing %q:\n%s", want, out) + } + } +} + +func TestInspectMissingRefAndConfigErrors(t *testing.T) { + s := openTestStore(t) + if err := inspect(&bytes.Buffer{}, s, "local:missing", false); err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Fatalf("inspect missing ref err = %v, want not pulled", err) + } + + img := tinyImage(t) + config, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + if err := os.Remove(blobPath(s.root, config.String())); err != nil { + t.Fatal(err) + } + if err := inspect(&bytes.Buffer{}, s, "local:tiny", false); err == nil { + t.Fatal("inspect with missing config succeeded, want error") + } +} + +// failingWriter fails every write, standing in for a closed pipe or a full +// filesystem behind a redirect. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("sink failed") } + +func TestInspectPropagatesWriteErrors(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:tiny", tinyImage(t)); err != nil { + t.Fatal(err) + } + if err := inspect(failingWriter{}, s, "local:tiny", true); err == nil { + t.Fatal("inspect --json exited clean on a failed write, want error") + } + if err := inspect(failingWriter{}, s, "local:tiny", false); err == nil { + t.Fatal("inspect summary exited clean on a failed write, want error") + } +} diff --git a/cmd/elfuse-oci/keychain.go b/cmd/elfuse-oci/keychain.go new file mode 100644 index 00000000..40eea883 --- /dev/null +++ b/cmd/elfuse-oci/keychain.go @@ -0,0 +1,56 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "time" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// credResolveTimeout bounds how long credential resolution may run before the +// pull gives up. Resolution shells out to whatever helper the ambient Docker +// config names (credsStore / credHelpers), and go-containerregistry's default +// keychain discards the context around that exec, so a wedged helper (for +// example docker-credential-desktop with Docker Desktop not running) otherwise +// hangs the pull with no output and no way out but a signal. +const credResolveTimeout = 10 * time.Second + +// timedKeychain runs an inner keychain's Resolve under a deadline. On timeout it +// returns an actionable error instead of blocking forever; the inner goroutine +// is abandoned and reaped when the process exits. It preserves the inner +// keychain's result in the normal case, so registry auth is unchanged when the +// helper answers promptly. +type timedKeychain struct { + inner authn.Keychain + timeout time.Duration +} + +func (t timedKeychain) Resolve(r authn.Resource) (authn.Authenticator, error) { + type resolved struct { + auth authn.Authenticator + err error + } + // Buffered so the goroutine's send never blocks after a timeout, leaving + // nothing to leak once the helper finally returns. + ch := make(chan resolved, 1) + go func() { + auth, err := t.inner.Resolve(r) + ch <- resolved{auth, err} + }() + + select { + case out := <-ch: + return out.auth, out.err + case <-time.After(t.timeout): + return nil, fmt.Errorf( + "resolving credentials for %s timed out after %s; a docker "+ + "credential helper (credsStore or credHelpers in "+ + "~/.docker/config.json) may be blocked. Retry with "+ + "DOCKER_CONFIG pointing at a directory whose config.json "+ + "omits it for an anonymous pull", + r.String(), t.timeout) + } +} diff --git a/cmd/elfuse-oci/keychain_test.go b/cmd/elfuse-oci/keychain_test.go new file mode 100644 index 00000000..6e40e462 --- /dev/null +++ b/cmd/elfuse-oci/keychain_test.go @@ -0,0 +1,74 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// fakeResource is a minimal authn.Resource for driving timedKeychain.Resolve. +type fakeResource struct{ s string } + +func (f fakeResource) String() string { return f.s } +func (f fakeResource) RegistryStr() string { return f.s } + +// blockingKeychain models a wedged credential helper: Resolve never returns +// until the test releases it. +type blockingKeychain struct{ release chan struct{} } + +func (b blockingKeychain) Resolve(authn.Resource) (authn.Authenticator, error) { + <-b.release + return authn.Anonymous, nil +} + +// A wedged helper must surface a bounded, actionable error rather than hang. +// The old failure was an indefinite block with no output; a regression here +// reads as Resolve never returning within the timeout. +func TestTimedKeychainReportsTimeout(t *testing.T) { + inner := blockingKeychain{release: make(chan struct{})} + defer close(inner.release) // let the parked goroutine exit after the test + + tk := timedKeychain{inner: inner, timeout: 50 * time.Millisecond} + + done := make(chan error, 1) + go func() { + _, err := tk.Resolve(fakeResource{s: "registry.example/img"}) + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected a timeout error, got nil") + } + if !strings.Contains(err.Error(), "DOCKER_CONFIG") { + t.Fatalf("error does not name the workaround: %v", err) + } + if !strings.Contains(err.Error(), "registry.example/img") { + t.Fatalf("error does not name the resource: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Resolve did not return within the timeout; it hung") + } +} + +// A helper that answers promptly must pass its result through untouched, so +// registry auth is unchanged when nothing is wedged. +func TestTimedKeychainPassesThroughFastResult(t *testing.T) { + ready := blockingKeychain{release: make(chan struct{})} + close(ready.release) // already released: Resolve returns immediately + + tk := timedKeychain{inner: ready, timeout: 2 * time.Second} + auth, err := tk.Resolve(fakeResource{s: "registry.example/img"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if auth != authn.Anonymous { + t.Fatalf("authenticator = %v, want the inner keychain's result", auth) + } +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go new file mode 100644 index 00000000..d2941991 --- /dev/null +++ b/cmd/elfuse-oci/main.go @@ -0,0 +1,85 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +// elfuse-oci is the OCI image CLI for elfuse. +// +// It owns the OCI image pipeline (pull, store, and inspect for now) +// using go-containerregistry. elfuse itself stays a pure Linux +// syscall-to-Darwin runtime with no OCI awareness. +// +// Usage: +// +// elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +// elfuse-oci inspect [--store DIR] [--json] +// +// is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., +// localhost:5000/foo:tag, or name@sha256:...). The default store is +// $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci. +package main + +import ( + "errors" + "flag" + "fmt" + "os" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "elfuse-oci: %s\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `usage: elfuse-oci [flags] [args...] + +commands: + pull Pull an image reference into the local OCI store + inspect Print a stored image's manifest + config + help Show this help + version Print the elfuse-oci version + +common flags: + --store DIR OCI store directory (default $ELFUSE_OCI_STORE or + ~/.local/share/elfuse/oci) + +pull flags: + --platform os/arch[/variant] Target platform (default linux/arm64) +`) +} + +func run(args []string) error { + if len(args) == 0 { + usage() + return fmt.Errorf("no command given") + } + cmd, rest := args[0], args[1:] + // A ` -h`/`--help` makes the subcommand FlagSet print its own flag list + // and return flag.ErrHelp; treat that as success rather than an error. + if err := dispatch(cmd, rest); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + return nil +} + +func dispatch(cmd string, rest []string) error { + switch cmd { + case "help", "-h", "--help": + usage() + return nil + case "version", "-V", "--version": + fmt.Println("elfuse-oci " + version) + return nil + case "pull": + return cmdPull(rest) + case "inspect": + return cmdInspect(rest) + default: + usage() + return fmt.Errorf("unknown command: %s", cmd) + } +} diff --git a/cmd/elfuse-oci/main_command_test.go b/cmd/elfuse-oci/main_command_test.go new file mode 100644 index 00000000..9a8636da --- /dev/null +++ b/cmd/elfuse-oci/main_command_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestRunDispatchHelpVersionAndErrors(t *testing.T) { + stdout, stderr, err := captureOutput(t, func() error { return run([]string{"help"}) }) + if err != nil { + t.Fatalf("run help: %v", err) + } + if stdout != "" { + t.Fatalf("help stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "usage: elfuse-oci") || !strings.Contains(stderr, "commands:") { + t.Fatalf("help stderr missing usage:\n%s", stderr) + } + + stdout, stderr, err = captureOutput(t, func() error { return run([]string{"--version"}) }) + if err != nil { + t.Fatalf("run --version: %v", err) + } + if strings.TrimSpace(stdout) != "elfuse-oci "+version { + t.Fatalf("version stdout = %q, want elfuse-oci %s", stdout, version) + } + if stderr != "" { + t.Fatalf("version stderr = %q, want empty", stderr) + } + + _, stderr, err = captureOutput(t, func() error { return run(nil) }) + if err == nil || !strings.Contains(err.Error(), "no command") { + t.Fatalf("run nil err = %v, want no command", err) + } + if !strings.Contains(stderr, "usage: elfuse-oci") { + t.Fatalf("no-arg stderr missing usage:\n%s", stderr) + } + + _, stderr, err = captureOutput(t, func() error { return run([]string{"bogus"}) }) + if err == nil || !strings.Contains(err.Error(), "unknown command: bogus") { + t.Fatalf("run bogus err = %v, want unknown command", err) + } + if !strings.Contains(stderr, "usage: elfuse-oci") { + t.Fatalf("unknown-command stderr missing usage:\n%s", stderr) + } +} + +func TestMainSubprocessExitBehavior(t *testing.T) { + stdout, stderr, err := runMainSubprocess(t, "version") + if err != nil { + t.Fatalf("main version err = %v, stdout=%q stderr=%q", err, stdout, stderr) + } + if strings.TrimSpace(stdout) != "elfuse-oci "+version { + t.Fatalf("main version stdout = %q", stdout) + } + if stderr != "" { + t.Fatalf("main version stderr = %q, want empty", stderr) + } + + stdout, stderr, err = runMainSubprocess(t, "bogus") + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("main bogus err = %T %v, want exit 1", err, err) + } + if stdout != "" { + t.Fatalf("main bogus stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "elfuse-oci: unknown command: bogus") { + t.Fatalf("main bogus stderr = %q, want formatted error", stderr) + } + + _, stderr, err = runMainSubprocess(t) + exit, ok = err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("main no-arg err = %T %v, want exit 1", err, err) + } + if !strings.Contains(stderr, "elfuse-oci: no command given") { + t.Fatalf("main no-arg stderr = %q, want formatted error", stderr) + } +} diff --git a/cmd/elfuse-oci/pull.go b/cmd/elfuse-oci/pull.go new file mode 100644 index 00000000..ff9c63a2 --- /dev/null +++ b/cmd/elfuse-oci/pull.go @@ -0,0 +1,50 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" +) + +var cranePull = crane.Pull + +// platformOption converts the parsed --platform into a crane option. +func platformOption(cf commonFlags) crane.Option { + p := v1.Platform{ + OS: cf.platform.OS, + Architecture: cf.platform.Arch, + Variant: cf.platform.Variant, + } + return crane.WithPlatform(&p) +} + +// pullImage fetches ref from a registry into the store, pinning ref to the +// image's manifest digest. Re-pulling the same digest is a no-op on the +// layout index (dedup by digest); only the pin table is refreshed. +func pullImage(cf commonFlags, s *store, ref string) error { + // Progress goes to stderr: pullImage also runs on the `run` path, where + // stdout belongs to the guest and callers capture it ($(run ...)). Printed + // before the pull so a slow or credential-blocked registry is not silent. + fmt.Fprintf(os.Stderr, "Pulling %s...\n", ref) + + // Wrap the ambient keychain so a wedged credential helper fails fast with + // an explanation instead of hanging the pull; see timedKeychain. + keychain := crane.WithAuthFromKeychain( + timedKeychain{authn.DefaultKeychain, credResolveTimeout}) + img, err := cranePull(ref, platformOption(cf), keychain) + if err != nil { + return fmt.Errorf("pull %s: %w", ref, err) + } + digest, err := s.addImage(ref, img) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "Pulled %s -> %s\n", ref, digest) + return nil +} diff --git a/cmd/elfuse-oci/store.go b/cmd/elfuse-oci/store.go new file mode 100644 index 00000000..ec7972af --- /dev/null +++ b/cmd/elfuse-oci/store.go @@ -0,0 +1,294 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" +) + +// The store is a real OCI image-layout on disk: an `oci-layout` version +// file, a `blobs/sha256/` tree, and an `index.json` image index (managed by +// go-containerregistry's layout package). Multiple pulled images coexist as +// separate manifest descriptors in the one index, distinguished by digest. +// +// On top of the spec layout we keep a ref->manifest-digest pin table +// (refs.json) so `unpack`/`inspect`/`run` can resolve an image by its +// original reference. This is elfuse-specific lookup metadata; OCI readers can +// still parse the layout through index.json and the content-addressed blobs. +// Keeping it separate lets us preserve the exact pull reference, including +// `docker.io/library/alpine:3` or `name@sha256:...`. + +const ( + ociLayoutFile = `{"imageLayoutVersion":"1.0.0"}` + emptyIndex = `{"schemaVersion":2,"manifests":[]}` +) + +type store struct { + path layout.Path + root string +} + +// openStore ensures the layout scaffolding exists and returns a handle. +// Creating an empty layout (oci-layout + empty index.json + blobs/sha256/) +// here, rather than via layout.Write, lets the first pull go through the same +// Append path as every subsequent one. The bootstrap runs under the store +// lock: writeIfAbsent's stat-then-write is check-then-act, and without the +// lock a parallel first-use pull could rename an empty index.json over one +// that another process had just populated, leaving that process's pin +// pointing at a manifest the index no longer lists. +func openStore(root string) (*store, error) { + for _, d := range []string{root, filepath.Join(root, "blobs"), filepath.Join(root, "blobs", "sha256")} { + if err := os.MkdirAll(d, 0o755); err != nil { + return nil, err + } + } + s := &store{path: layout.Path(root), root: root} + layoutFile := filepath.Join(root, "oci-layout") + indexFile := filepath.Join(root, "index.json") + // Fast path: a warm store has both files, and they are never removed once + // created, so no lock is needed. A run's startup must not block behind a + // concurrent pull's store lock just to bootstrap no-ops; writeIfAbsent + // re-checks under the lock, making this a double-checked bootstrap. + if fileMissing(layoutFile) || fileMissing(indexFile) { + err := s.withLock(func() error { + if err := writeIfAbsent(layoutFile, []byte(ociLayoutFile)); err != nil { + return err + } + return writeIfAbsent(indexFile, []byte(emptyIndex)) + }) + if err != nil { + return nil, err + } + } + return s, nil +} + +func fileMissing(path string) bool { + _, err := os.Stat(path) + return err != nil +} + +// writeIfAbsent writes data to path unless the file already exists. The +// stat-then-write pair is check-then-act; the caller must hold the store +// lock so no metadata writer can slip between the two steps. +func writeIfAbsent(path string, data []byte) error { + if _, err := os.Stat(path); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + return os.WriteFile(path, data, 0o644) +} + +// refPins maps an image reference to its manifest digest ("sha256:..."). +type refPins map[string]string + +func (s *store) loadPins() (refPins, error) { + b, err := os.ReadFile(filepath.Join(s.root, "refs.json")) + if os.IsNotExist(err) { + return refPins{}, nil + } else if err != nil { + return nil, err + } + var p refPins + if err := json.Unmarshal(b, &p); err != nil { + return nil, fmt.Errorf("store: corrupt refs.json: %w", err) + } + if p == nil { + return nil, fmt.Errorf("store: corrupt refs.json: expected object") + } + return p, nil +} + +func (s *store) savePins(p refPins) error { + b, err := json.MarshalIndent(p, "", " ") + if err != nil { + return err + } + // Unique temp name: a fixed name would let two writers clobber each + // other's half-written temp even before the rename race. + tmp, err := os.CreateTemp(s.root, ".refs.json.*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // no-op once the rename succeeds + if _, err := tmp.Write(b); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(0o644); err != nil { + tmp.Close() + return err + } + // Durability, not just atomicity: rmi's crash-ordering argument (drop the + // pin before dropping the descriptor) only holds if the new refs.json + // cannot revert to an old pin after a crash. Sync the temp file before the + // rename and the directory after it, so the rename is durable once this + // returns. + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), filepath.Join(s.root, "refs.json")); err != nil { + return err + } + return fsyncDir(s.root) +} + +// fsyncDir flushes a directory's entries (renames, unlinks) to stable +// storage. +func fsyncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +// lock takes an exclusive advisory flock on /.lock and returns the +// unlock func. It serializes read-modify-write cycles on refs.json and +// index.json across concurrent elfuse-oci processes (parallel pulls, or +// a pull racing an rmi); without it, last-writer-wins on refs.json can drop a +// just-recorded pin. +func (s *store) lock() (func(), error) { + f, err := os.OpenFile(filepath.Join(s.root, ".lock"), os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, fmt.Errorf("store: open lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return nil, fmt.Errorf("store: lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// withLock runs fn while holding the store lock. Results cross the closure +// boundary by capture; the lock is released before withLock returns, so +// callers can keep post-lock work (reporting, other stores) outside the +// critical section. +func (s *store) withLock(fn func() error) error { + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + return fn() +} + +// pin records ref->digest in the pin table. +func (s *store) pin(ref, digest string) error { + return s.withLock(func() error { return s.pinLocked(ref, digest) }) +} + +// pinLocked is pin's load-modify-save cycle; the caller holds the store lock. +func (s *store) pinLocked(ref, digest string) error { + p, err := s.loadPins() + if err != nil { + return err + } + p[ref] = digest + return s.savePins(p) +} + +// errNotPulled marks the ref-simply-missing case, distinguishing it from +// store corruption or IO failures: `run` auto-pulls only on this error. +var errNotPulled = fmt.Errorf("not pulled") + +// digestFor returns the manifest digest pinned for ref, or an error wrapping +// errNotPulled if the ref has not been pulled into this store. +func (s *store) digestFor(ref string) (string, error) { + p, err := s.loadPins() + if err != nil { + return "", err + } + d, ok := p[ref] + if !ok { + return "", fmt.Errorf("store: %q %w (run `elfuse-oci pull %s` first)", ref, errNotPulled, ref) + } + return d, nil +} + +// addImage appends img to the layout index if its manifest is not already +// present (dedup by digest), and pins ref to that digest. Returns the digest. +// The store lock covers the whole check-append-pin sequence: index.json is +// itself updated by read-modify-write inside the layout package, so two +// concurrent pulls could otherwise duplicate or drop descriptors. +func (s *store) addImage(ref string, img v1.Image) (string, error) { + d, err := img.Digest() + if err != nil { + return "", fmt.Errorf("store: compute manifest digest: %w", err) + } + h, err := v1.NewHash(d.String()) + if err != nil { + return "", err + } + err = s.withLock(func() error { + // Re-pulling the same digest must not append a duplicate descriptor + // to the index. + present, err := s.hasImageLocked(h) + if err != nil { + return fmt.Errorf("store: read layout index: %w", err) + } + if !present { + if err := s.path.AppendImage(img); err != nil { + return fmt.Errorf("store: append image: %w", err) + } + } + return s.pinLocked(ref, d.String()) + }) + if err != nil { + return "", err + } + return d.String(), nil +} + +// hasImageLocked reports whether the layout index already carries a manifest +// descriptor for h. The caller holds the store lock. This is a positive +// membership scan rather than a probe via s.path.Image(h): the layout package +// returns an untyped error for both "not found" and a corrupt or unreadable +// index.json, and treating the latter as "absent" would silently append into +// a broken store, masking the corruption. +func (s *store) hasImageLocked(h v1.Hash) (bool, error) { + ii, err := s.path.ImageIndex() + if err != nil { + return false, err + } + im, err := ii.IndexManifest() + if err != nil { + return false, err + } + for _, desc := range im.Manifests { + if desc.Digest == h { + return true, nil + } + } + return false, nil +} + +// image returns the v1.Image pinned for ref. +func (s *store) image(ref string) (v1.Image, error) { + d, err := s.digestFor(ref) + if err != nil { + return nil, err + } + h, err := v1.NewHash(d) + if err != nil { + return nil, err + } + return s.path.Image(h) +} diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go new file mode 100644 index 00000000..d8ce31f1 --- /dev/null +++ b/cmd/elfuse-oci/store_test.go @@ -0,0 +1,247 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// TestDigestForErrorKinds pins the distinction cmdRun's auto-pull relies on: +// a merely-absent ref is errNotPulled (triggers the pull), while a corrupt +// refs.json is a different error that must surface instead of being masked by +// a network pull. +func TestDigestForErrorKinds(t *testing.T) { + s := openTestStore(t) + if _, err := s.digestFor("local:absent"); !errors.Is(err, errNotPulled) { + t.Fatalf("missing ref err = %v, want errNotPulled", err) + } + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte("{corrupt"), 0o644); err != nil { + t.Fatal(err) + } + _, err := s.digestFor("local:absent") + if err == nil || errors.Is(err, errNotPulled) { + t.Fatalf("corrupt refs.json err = %v, must not be errNotPulled", err) + } +} + +// TestAddImageCorruptIndexSurfaces pins that addImage distinguishes "image +// not in the layout" from "layout index unreadable": appending into a corrupt +// store would mask the corruption behind a fresh descriptor. +func TestAddImageCorruptIndexSurfaces(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "index.json"), []byte("{corrupt"), 0o644); err != nil { + t.Fatal(err) + } + _, err := s.addImage("local:corrupt", tinyImage(t)) + if err == nil || !strings.Contains(err.Error(), "read layout index") { + t.Fatalf("addImage with corrupt index.json err = %v, want read-layout-index error", err) + } + // The corrupt index must be left as-is for diagnosis, not clobbered by an + // append. + b, rerr := os.ReadFile(filepath.Join(s.root, "index.json")) + if rerr != nil || string(b) != "{corrupt" { + t.Fatalf("index.json after failed addImage = %q, err=%v; want untouched", b, rerr) + } +} + +// TestPinConcurrentWritersKeepAllEntries pins the store-lock behavior: N +// concurrent pin calls (as parallel `pull` processes would issue) must all +// survive into refs.json. Without the flock around the load-modify-save +// cycle, last-writer-wins drops entries. +func TestPinConcurrentWritersKeepAllEntries(t *testing.T) { + s := openTestStore(t) + const n = 16 + var wg sync.WaitGroup + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs <- s.pin(fmt.Sprintf("local:ref%d", i), fmt.Sprintf("sha256:%064d", i)) + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + pins, err := s.loadPins() + if err != nil { + t.Fatal(err) + } + if len(pins) != n { + t.Fatalf("refs.json has %d pins after %d concurrent writers, want %d", len(pins), n, n) + } +} + +// TestOpenStoreBootstrapWaitsForStoreLock pins that openStore's bootstrap +// runs under the store lock. writeIfAbsent's stat-then-write is +// check-then-act: without the lock, a parallel first-use pull could rename an +// empty index.json over one the lock holder just populated, so openStore must +// block until the holder releases and then leave the populated index alone. +func TestOpenStoreBootstrapWaitsForStoreLock(t *testing.T) { + root := filepath.Join(t.TempDir(), "store") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + // Hold the store lock as a concurrent pull's metadata write would. + lockFile, err := os.OpenFile(filepath.Join(root, ".lock"), os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + t.Fatal(err) + } + defer lockFile.Close() + if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := openStore(root) + done <- err + }() + select { + case <-done: + t.Fatal("openStore finished while the store lock was held; bootstrap must serialize with metadata writers") + case <-time.After(100 * time.Millisecond): + } + // The lock holder commits a populated index, then releases. Bootstrap must + // observe it and must not replace it with the empty scaffold. + populated := `{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}]}` + if err := os.WriteFile(filepath.Join(root, "index.json"), []byte(populated), 0o644); err != nil { + t.Fatal(err) + } + if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(filepath.Join(root, "index.json")) + if err != nil { + t.Fatal(err) + } + if string(b) != populated { + t.Fatalf("index.json after bootstrap = %q, want the populated index left untouched", b) + } +} + +func TestDefaultStoreFromEnvAndResolveStore(t *testing.T) { + want := filepath.Join(t.TempDir(), "store") + t.Setenv("ELFUSE_OCI_STORE", want) + got, err := defaultStore() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("defaultStore = %q, want %q", got, want) + } + + var cf commonFlags + if err := cf.resolveStore(); err != nil { + t.Fatalf("resolveStore: %v", err) + } + if cf.store != want { + t.Fatalf("resolved store = %q, want %q", cf.store, want) + } + if fi, err := os.Stat(want); err != nil || !fi.IsDir() { + t.Fatalf("resolved store dir = %v, err=%v; want directory", fi, err) + } + + fileStore := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(fileStore, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + cf = commonFlags{store: fileStore} + if err := cf.resolveStore(); err == nil { + t.Fatal("resolveStore on file path succeeded, want error") + } + + home := t.TempDir() + t.Setenv("ELFUSE_OCI_STORE", "") + t.Setenv("HOME", home) + got, err = defaultStore() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(home, ".local", "share", "elfuse", "oci") + if got != want { + t.Fatalf("defaultStore without env = %q, want %q", got, want) + } +} + +func TestRepeatedStringFlag(t *testing.T) { + var nilFlag *repeatedStringFlag + if got := nilFlag.String(); got != "" { + t.Fatalf("nil repeatedStringFlag String = %q, want empty", got) + } + + var f repeatedStringFlag + if err := f.Set("A=1"); err != nil { + t.Fatal(err) + } + if err := f.Set("B=2"); err != nil { + t.Fatal(err) + } + if got := f.String(); got != "A=1,B=2" { + t.Fatalf("repeatedStringFlag String = %q, want A=1,B=2", got) + } +} + +func TestOpenStoreAndWriteIfAbsentErrorCases(t *testing.T) { + rootFile := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(rootFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := openStore(rootFile); err == nil { + t.Fatal("openStore on file path succeeded, want error") + } + + p := filepath.Join(t.TempDir(), "existing") + if err := os.WriteFile(p, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeIfAbsent(p, []byte("new")); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(b) != "old" { + t.Fatalf("writeIfAbsent overwrote existing file with %q, want old", b) + } +} + +func TestLoadPinsCorruptNullAndPinError(t *testing.T) { + s := openTestStore(t) + for _, tc := range []struct { + name string + data string + want string + }{ + {"malformed", "{", "corrupt refs.json"}, + {"null", "null", "expected object"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(tc.data), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.loadPins(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("loadPins err = %v, want %q", err, tc.want) + } + if err := s.pin("local:a", "sha256:"+strings.Repeat("1", 64)); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("pin err = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go new file mode 100644 index 00000000..bbbc643b --- /dev/null +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func TestMain(m *testing.M) { + if raw, ok := os.LookupEnv("ELFUSE_OCI_MAIN_TEST_ARGS"); ok { + var args []string + if raw != "" { + args = strings.Split(raw, "\x00") + } + os.Args = append([]string{os.Args[0]}, args...) + main() + return + } + os.Exit(m.Run()) +} + +func sliceContains(xs []string, want string) bool { + return slices.Contains(xs, want) +} + +func captureOutput(t *testing.T, fn func() error) (string, string, error) { + t.Helper() + + oldStdout, oldStderr := os.Stdout, os.Stderr + stdoutR, stdoutW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + stderrR, stderrW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + stdoutCh := make(chan string, 1) + stderrCh := make(chan string, 1) + go func() { + b, _ := io.ReadAll(stdoutR) + stdoutCh <- string(b) + }() + go func() { + b, _ := io.ReadAll(stderrR) + stderrCh <- string(b) + }() + + os.Stdout, os.Stderr = stdoutW, stderrW + // Close every pipe end in the defer, not only inline: a fn that panics + // or runs t.Fatal (Goexit) would otherwise leave the write ends open, + // parking both ReadAll goroutines forever and leaking four fds per + // aborted test. The inline closes below stay because they are what + // terminates the readers on the success path; the double Close is a + // harmless ErrClosed. + defer func() { + os.Stdout, os.Stderr = oldStdout, oldStderr + _ = stdoutW.Close() + _ = stderrW.Close() + _ = stdoutR.Close() + _ = stderrR.Close() + }() + + fnErr := fn() + _ = stdoutW.Close() + _ = stderrW.Close() + stdout := <-stdoutCh + stderr := <-stderrCh + return stdout, stderr, fnErr +} + +func openTestStore(t *testing.T) *store { + t.Helper() + s, err := openStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return s +} + +// buildImage builds a one-layer in-memory image whose layer content is fixed +// ("hello"="world") but whose Cmd is cmd, so two calls with different cmds +// produce the same layer blob but distinct config/manifest digests. This is +// what TestRmiKeepsSharedBlobs needs to exercise reachability GC: two pinned +// refs sharing one layer, with distinct manifests. +func buildImage(t *testing.T, cmd []string) v1.Image { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + if err := tw.WriteHeader(&tar.Header{Name: "hello", Mode: 0o644, Size: 5, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("world")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + // LayerFromOpener calls the opener per read so digest/diffid can be queried + // repeatedly (LayerFromReader is deprecated and single-shot). + opener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil + } + layer, err := tarball.LayerFromOpener(opener) + if err != nil { + t.Fatal(err) + } + img, err := mutate.AppendLayers(empty.Image, layer) + if err != nil { + t.Fatal(err) + } + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + img, err = mutate.ConfigFile(img, &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Config: v1.Config{Cmd: cmd}, + RootFS: v1.RootFS{Type: "layers", DiffIDs: []v1.Hash{diffID}}, + }) + if err != nil { + t.Fatal(err) + } + return img +} + +// tinyImage is the canonical single-layer offline test image. +func tinyImage(t *testing.T) v1.Image { + t.Helper() + return buildImage(t, []string{"/hello"}) +} + +func blobPath(root, digest string) string { + return filepath.Join(root, "blobs", "sha256", strings.TrimPrefix(digest, "sha256:")) +} + +func runMainSubprocess(t *testing.T, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), "ELFUSE_OCI_MAIN_TEST_ARGS="+strings.Join(args, "\x00")) + // Buffers instead of pipes: exec.Cmd drains both concurrently, so a child + // filling one stream can't deadlock against a sequential reader of the + // other (the pattern the os/exec docs warn about). + var outB, errB bytes.Buffer + cmd.Stdout = &outB + cmd.Stderr = &errB + waitErr := cmd.Run() + return outB.String(), errB.String(), waitErr +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..d004d0b6 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/sysprog21/elfuse + +go 1.26.4 + +require github.com/google/go-containerregistry v0.21.7 + +require ( + github.com/docker/cli v29.5.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/klauspost/compress v1.18.7 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + gotest.tools/v3 v3.5.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..627c493a --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= +github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnOhzei4X2DMW9IU= +github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/mk/toolchain.mk b/mk/toolchain.mk index e0f6be4d..b701e5e4 100644 --- a/mk/toolchain.mk +++ b/mk/toolchain.mk @@ -42,3 +42,8 @@ SHIM_ASFLAGS ?= -arch arm64 # clang-format CLANG_FORMAT ?= clang-format + +# Go toolchain for the OCI image CLI (build/elfuse-oci). It is a +# pure Go program with no HVF dependency, so it builds and runs on Linux CI +# too (for spec-conformance / interop tests). `go` from PATH by default. +GO ?= go From f8885c2ce20abe576ba5db86e04a117e700201f7 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:23:35 +0200 Subject: [PATCH 04/26] Add elfuse-oci unpack of OCI layers Apply a stored image's layers into a rootfs directory, whiteouts and all, so `unpack` (and later `run`) can materialize a filesystem tree from the store without any external tool. Layer application is hardened against hostile archives: extraction is bounded by os.Root so no entry, symlink, or hardlink escapes the destination; member names and hard-link targets archived absolute (GNU tar -P builders) are applied root-relative, as other OCI consumers do; parent components are Lstat'd and a symlinked or non-dir intermediate is replaced with a real directory (containerd/Docker behavior); opaque whiteouts clear through a real directory only, are order-independent within a layer, and an invalid bare .wh. entry fails extraction instead of deleting its parent; a plain whiteout whose parent chain is not all real directories is a no-op, so it cannot remove through a lower layer's symlink; a directory entry replaces a lower-layer non-directory. File bodies are bounded by the tar header size and short bodies are an error, and permissions are finalized with an explicit chmod so the host umask cannot skew modes. The decompressor is drained past tar's end-of-archive marker, so a corrupted gzip trailer fails the unpack instead of vanishing with the discarded Close error. setuid/setgid/sticky bits are re-applied where the host allows it, and degrade gracefully where it does not. An unprivileged chmod that sets setuid/setgid is rejected with EPERM on macOS when the unpacked file's inherited group is one the invoking user is not in (a new file takes its parent directory's group under BSD semantics, e.g. wheel under /tmp, not the tar's root/shadow owner). Since the rootfs is owned by the invoking user those bits could not be honored at runtime there anyway, so they are dropped with a warning naming the lost bit rather than aborting the whole unpack, which is what lets Debian-family images and their shadow suite (chage, passwd, ...) unpack at all. Reported at https://github.com/sysprog21/elfuse/pull/191#issuecomment-5011520776 Unpack stages into a temp sibling directory and renames into the final cache path (keyed by manifest digest under /rootfs/), so a concurrent reader never observes a partial tree and the loser of a rename race adopts the winner's complete one. A failed unpack removes only what it created. Tests drive whiteouts, opaque ordering, parent-symlink replacement, the whiteout-through-symlink no-op, trailer corruption, hardlink identity, exact-size reads, mode preservation, the special-bit degrade decision, and the staged-rename semantics over synthetic layer tarballs. Two malformed-input rules the layer format needs. A whiteout suffix that collapses to a dot name (".wh..", ".wh...") is rejected: Join folds it into the containing directory or its parent, turning the removal of one named entry into a subtree wipe. And a store-managed rootfs cache path that is not a real directory is refused, because os.OpenRoot follows a symlink in the directory name it opens, so a symlink planted at the digest path would redirect the whole extraction out of the store; an explicit --rootfs still follows links, as its merge-in-place contract requires. --- cmd/elfuse-oci/cache_key.go | 58 +++ cmd/elfuse-oci/commands.go | 46 +- cmd/elfuse-oci/common_test.go | 26 +- cmd/elfuse-oci/main.go | 8 +- cmd/elfuse-oci/store_test.go | 13 + cmd/elfuse-oci/unpack.go | 523 +++++++++++++++++++ cmd/elfuse-oci/unpack_image_test.go | 755 ++++++++++++++++++++++++++++ cmd/elfuse-oci/unpack_test.go | 472 +++++++++++++++++ 8 files changed, 1895 insertions(+), 6 deletions(-) create mode 100644 cmd/elfuse-oci/cache_key.go create mode 100644 cmd/elfuse-oci/unpack.go create mode 100644 cmd/elfuse-oci/unpack_image_test.go create mode 100644 cmd/elfuse-oci/unpack_test.go diff --git a/cmd/elfuse-oci/cache_key.go b/cmd/elfuse-oci/cache_key.go new file mode 100644 index 00000000..6f9e8715 --- /dev/null +++ b/cmd/elfuse-oci/cache_key.go @@ -0,0 +1,58 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// Store subdirectories holding unpacked caches, keyed by cacheKeyForDigest: +// plain rootfs trees and (darwin) case-sensitive sparsebundle bundles. +const ( + rootfsCacheDirName = "rootfs" + csCacheDirName = "cs" +) + +// legacyCacheNameForRef reproduces the pre-digest cache naming scheme, which +// flattened the ref itself into a single (intentionally lossy) path +// component. New caches are keyed by digest (cacheKeyForDigest); this helper +// survives to document the old layout. +func legacyCacheNameForRef(ref string) string { + return strings.NewReplacer("/", "_", ":", "_", "@", "_").Replace(ref) +} + +// cacheKeyForDigest returns the relative cache key used under rootfs/ and cs/. +// The current store writes sha256 blobs only; keep the algorithm component in +// the path so the layout remains explicit and non-lossy. +func cacheKeyForDigest(digest string) (string, error) { + h, err := v1.NewHash(digest) + if err != nil { + return "", err + } + if h.Algorithm != "sha256" || h.Hex == "" { + // Unreachable with the pinned go-containerregistry, whose NewHash + // accepts sha256 only: a version-independence guard, not a reachable + // validation. The store layout and prune's rootfs/sha256 sweep assume + // sha256 keys, so an upstream that starts accepting more algorithms + // must fail here rather than mint keys the sweeps misread. + return "", fmt.Errorf("unsupported cache digest %q", digest) + } + return filepath.Join(h.Algorithm, h.Hex), nil +} + +func defaultRootfsForDigest(store, digest string) (string, error) { + key, err := cacheKeyForDigest(digest) + if err != nil { + return "", err + } + return filepath.Join(store, rootfsCacheDirName, key), nil +} + +func legacyRootfsForRef(store, ref string) string { + return filepath.Join(store, rootfsCacheDirName, legacyCacheNameForRef(ref)) +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index ddd8b04e..4a056e2e 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -4,11 +4,12 @@ package main import ( + "fmt" "os" ) // The subcommands share common-flag parsing (common.go) and the OCI -// image-layout store (store.go). pull and inspect are pure store ops. +// image-layout store (store.go). pull/unpack/inspect are pure store ops. // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -34,6 +35,49 @@ func parsePullArgs(args []string) (commonFlags, string, error) { return cf, ref, err } +// cmdUnpack implements `elfuse-oci unpack [--store] [--rootfs DIR] `. +func cmdUnpack(args []string) error { + cf, rootfs, ref, err := parseUnpackArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + digest, err := s.digestFor(ref) + if err != nil { + return err + } + if rootfs == "" { + rootfs, err = defaultRootfsForDigest(cf.store, digest) + if err != nil { + return err + } + if _, err := storeRootfsPublished(rootfs); err != nil { + return err + } + } + fmt.Printf("Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(s, ref, rootfs); err != nil { + return err + } + fmt.Printf("Unpacked %s\n", ref) + return nil +} + +func parseUnpackArgs(args []string) (commonFlags, string, string, error) { + var cf commonFlags + var rootfs string + fs := newCommandFlagSet("unpack", &cf) + fs.StringVar(&rootfs, "rootfs", "", "") + if err := fs.Parse(args); err != nil { + return cf, "", "", err + } + ref, err := oneArg("unpack", fs.Args(), "") + return cf, rootfs, ref, err +} + // cmdInspect implements `elfuse-oci inspect [--store] [--json] `. func cmdInspect(args []string) error { cf, asJSON, ref, err := parseInspectArgs(args) diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index e981938a..702d8999 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -60,6 +60,22 @@ func TestParsePullArgs(t *testing.T) { } } +func TestParseUnpackArgs(t *testing.T) { + cf, rootfs, ref, err := parseUnpackArgs([]string{"--rootfs=/tmp/rootfs", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if cf.platform != defaultPlatform { + t.Errorf("platform = %+v, want default %+v", cf.platform, defaultPlatform) + } + if rootfs != "/tmp/rootfs" { + t.Errorf("rootfs = %q, want /tmp/rootfs", rootfs) + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } +} + func TestParseInspectArgs(t *testing.T) { _, asJSON, ref, err := parseInspectArgs([]string{"--json", "alpine:3"}) if err != nil { @@ -80,10 +96,14 @@ func TestParseCommandFlagErrors(t *testing.T) { }{ {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, - {"missing ref", func() error { _, _, err := parsePullArgs(nil); return err }()}, + {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, // --platform belongs to the commands that resolve a platform; - // inspect discarding a successfully parsed flag would mislead the - // caller. + // unpack and inspect discarding a successfully parsed flag would + // mislead the caller. + {"unpack rejects platform", func() error { + _, _, _, err := parseUnpackArgs([]string{"--platform", "linux/amd64", "alpine:3"}) + return err + }()}, {"inspect rejects platform", func() error { _, _, _, err := parseInspectArgs([]string{"--platform", "linux/amd64", "alpine:3"}) return err diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index d2941991..f8bcd502 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -3,13 +3,14 @@ // elfuse-oci is the OCI image CLI for elfuse. // -// It owns the OCI image pipeline (pull, store, and inspect for now) -// using go-containerregistry. elfuse itself stays a pure Linux +// It owns the OCI image pipeline (pull, store, inspect, and unpack for +// now) using go-containerregistry. elfuse itself stays a pure Linux // syscall-to-Darwin runtime with no OCI awareness. // // Usage: // // elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +// elfuse-oci unpack [--store DIR] [--rootfs DIR] // elfuse-oci inspect [--store DIR] [--json] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., @@ -36,6 +37,7 @@ func usage() { commands: pull Pull an image reference into the local OCI store + unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config help Show this help version Print the elfuse-oci version @@ -76,6 +78,8 @@ func dispatch(cmd string, rest []string) error { return nil case "pull": return cmdPull(rest) + case "unpack": + return cmdUnpack(rest) case "inspect": return cmdInspect(rest) default: diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go index d8ce31f1..4f507ff2 100644 --- a/cmd/elfuse-oci/store_test.go +++ b/cmd/elfuse-oci/store_test.go @@ -245,3 +245,16 @@ func TestLoadPinsCorruptNullAndPinError(t *testing.T) { }) } } + +func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { + if _, err := cacheKeyForDigest("not-a-digest"); err == nil { + t.Fatal("cacheKeyForDigest accepted malformed digest") + } + if _, err := defaultRootfsForDigest(t.TempDir(), "not-a-digest"); err == nil { + t.Fatal("defaultRootfsForDigest accepted malformed digest") + } + unsupported := "sha512:" + strings.Repeat("1", 128) + if _, err := cacheKeyForDigest(unsupported); err == nil { + t.Fatalf("cacheKeyForDigest(%q) succeeded, want rejection", unsupported) + } +} diff --git a/cmd/elfuse-oci/unpack.go b/cmd/elfuse-oci/unpack.go new file mode 100644 index 00000000..67221d2e --- /dev/null +++ b/cmd/elfuse-oci/unpack.go @@ -0,0 +1,523 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// unpackImage extracts every layer of the stored image (base first) into a +// plain directory rootfs. Each layer is a tar stream (crane decompresses gzip +// and zstd transparently via layer.Uncompressed). +// +// Layer application implements the OCI whiteout conventions: +// - `.wh.` in a directory removes `` (from this and lower layers). +// - `.wh..wh..opq` in a directory clears that directory's existing contents +// before the layer's own additions are applied. +// +// Containment uses os.OpenRoot (Go 1.24+): every write is resolved relative +// to the rootfs and may not escape via ".." or a symlink. os.Root forbids +// absolute symlinks, so absolute symlink targets are rewritten to their +// equivalent relative form, which is behavior-preserving because under +// elfuse's --sysroot both forms resolve to the same guest path. +// +// Ownership is not applied here: elfuse runs as the host user and overrides +// identity at runtime via --user, so the rootfs carries only mode bits. +// storeRootfsPublished reports whether a store-managed rootfs cache path is +// already published, and refuses anything there that is not a real directory. +// +// The store names these paths by digest and publishes them only by renaming a +// staging directory into place, so a symlink or file found there did not come +// from this package. It must be refused rather than used: os.OpenRoot follows +// symlinks in the directory name it opens, and an existence probe that follows +// them reports "already unpacked", so a symlink at the digest path would +// redirect both the extraction and the guest that runs against the result out +// of the store. An explicit --rootfs is the user's own directory and keeps +// following links, which is why this guards the store-managed callers instead +// of unpackImage itself. +func storeRootfsPublished(path string) (bool, error) { + fi, err := os.Lstat(path) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + if fi.Mode()&os.ModeSymlink != 0 { + return false, fmt.Errorf("rootfs cache %s is a symlink, refusing to use it", path) + } + if !fi.IsDir() { + return false, fmt.Errorf("rootfs cache %s is not a directory", path) + } + return true, nil +} + +func unpackImage(s *store, ref, dest string) error { + img, err := s.image(ref) + if err != nil { + return err + } + if _, statErr := os.Lstat(dest); statErr == nil { + // An explicit pre-existing --rootfs directory: merge in place and + // never remove it, failed or not; it is not ours to delete. + return unpackInto(img, dest) + } else if !os.IsNotExist(statErr) { + return statErr + } + // The run paths treat the rootfs path's existence as "fully unpacked", so + // a partial tree must never be visible under dest, not even while this + // unpack is still running: a concurrent run probing dest with os.Stat + // would execute against the half-written tree. Unpack into a temp sibling + // (same volume, so the rename cannot degrade to a copy) and atomically + // rename into place on success. + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+".tmp-") + if err != nil { + return err + } + // MkdirTemp creates 0o700; the rootfs root must be traversable once + // published. + if err := os.Chmod(tmp, 0o755); err != nil { + os.RemoveAll(tmp) + return err + } + if err := unpackInto(img, tmp); err != nil { + os.RemoveAll(tmp) + return err + } + if err := os.Rename(tmp, dest); err != nil { + os.RemoveAll(tmp) + if _, statErr := os.Lstat(dest); statErr == nil { + // A concurrent unpack of the same image won the rename; its tree + // is complete, so use it. + return nil + } + return err + } + return nil +} + +func unpackInto(img v1.Image, dest string) error { + root, err := os.OpenRoot(dest) + if err != nil { + return fmt.Errorf("unpack: open rootfs %s: %w", dest, err) + } + defer root.Close() + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("unpack: list layers: %w", err) + } + for i, layer := range layers { + if err := applyLayer(root, layer); err != nil { + return fmt.Errorf("unpack: layer %d: %w", i, err) + } + } + return nil +} + +func applyLayer(root *os.Root, layer v1.Layer) error { + r, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("open layer: %w", err) + } + defer r.Close() + // Paths this layer has already created. Whiteouts hide lower-layer + // content only, but tar entry order within a layer is not guaranteed: an + // opaque marker may arrive after the directory's own same-layer children, + // which must survive the clear (Docker's unpacker keeps the same set). + applied := make(map[string]bool) + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("read tar entry: %w", err) + } + if err := applyEntry(root, hdr, tr, applied); err != nil { + return fmt.Errorf("entry %q: %w", hdr.Name, err) + } + } + // Tar's end-of-archive marker (two zero blocks) arrives before the + // compressed stream's end, so the decompressor never reads the gzip + // trailer during the entry loop: drain it so CRC/ISIZE corruption fails + // the unpack instead of vanishing in the deferred Close, whose error is + // discarded. + if _, err := io.Copy(io.Discard, r); err != nil { + return fmt.Errorf("layer trailer: %w", err) + } + return nil +} + +// whiteoutPrefix is the OCI/Docker whiteout marker prefix. +const whiteoutPrefix = ".wh." + +// opaqueMarker is the opaque-directory whiteout marker: a directory's +// existing children are hidden before the layer's own additions apply. +const opaqueMarker = whiteoutPrefix + ".wh..opq" + +// rootRelative strips the leading slash from a cleaned tar path. Some +// builders (GNU tar -P) archive member names and hard-link targets absolute; +// OCI consumers apply both root-relative. Clean has already collapsed any +// ".." in an absolute path against the root, so dropping the slash cannot +// introduce an escape. +func rootRelative(cleaned string) string { + return strings.TrimPrefix(cleaned, "/") +} + +// applyEntry applies one tar header to the rootfs. applied tracks the paths +// the current layer has created so far, so an opaque whiteout arriving after +// its directory's same-layer children does not delete them. +func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string]bool) error { + name := filepath.Clean(hdr.Name) + if strings.HasPrefix(name, "../") || name == ".." { + return fmt.Errorf("unsafe entry path %q", hdr.Name) + } + name = rootRelative(name) + if name == "" || name == "." { + // The layer root itself; nothing to create. + return nil + } + + base := filepath.Base(name) + if base == opaqueMarker { + return clearDirectory(root, filepath.Dir(name), applied) + } + if trimmed, ok := strings.CutPrefix(base, whiteoutPrefix); ok { + // A whiteout names one sibling to remove, so the suffix must be a + // plain file name. An empty (".wh."), dot (".wh.."), or dot-dot + // (".wh...") suffix makes Join resolve to the containing directory or + // above it, turning a one-entry removal into a subtree wipe; a + // separator would escape the marker's own directory. A malformed + // layer must fail rather than delete more than it named. + if trimmed == "" || trimmed == "." || trimmed == ".." || + strings.ContainsRune(trimmed, filepath.Separator) { + return fmt.Errorf("invalid whiteout entry %q", hdr.Name) + } + target := filepath.Join(filepath.Dir(name), trimmed) + // A lower layer may have planted a symlink at a parent component; + // RemoveAll resolves in-root links, so removing through one deletes + // under the link target, files the layer never named (ensureParent's + // rule applied to removal, except a whiteout must not materialize + // directories either). A parent that is absent or not a real + // directory means the literal path does not exist: the whiteout is a + // no-op, and any replacement of the parent is the job of the layer's + // own directory entry. + if dir := filepath.Dir(name); dir != "." { + ok, err := realDirChain(root, dir) + if err != nil || !ok { + return err + } + } + return root.RemoveAll(target) + } + applied[name] = true + + // hdr.FileInfo().Mode() maps the tar header's unix mode bits to os.FileMode + // with the special bits (ModeSetuid/Setgid/Sticky) at os.FileMode's high + // positions, not the raw unix positions. os.Root.MkdirAll/OpenFile reject + // any non-permission bits, so split perm (0o777) from special bits and + // re-apply special bits via Chmod (which syscallMode maps to the syscall). + mode := hdr.FileInfo().Mode() + perm := mode.Perm() + special := mode & (os.ModeSetuid | os.ModeSetgid | os.ModeSticky) + switch hdr.Typeflag { + case tar.TypeDir: + return mkdirAll(root, name, perm, special) + case tar.TypeReg, tar.TypeRegA: + if err := ensureParent(root, name); err != nil { + return err + } + // Remove any prior entry (file, symlink, dir remnant) so we never + // write through a symlink planted by a lower layer. + _ = root.RemoveAll(name) + f, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + // Bound the copy to the header's declared size and require exactly + // that many bytes. The tar reader normally guarantees both; enforcing + // them here keeps the entry-size contract out of the caller's hands. + n, err := io.Copy(f, io.LimitReader(r, hdr.Size)) + if err != nil { + f.Close() + return err + } + if n != hdr.Size { + f.Close() + return fmt.Errorf("entry body: read %d bytes, want %d", n, hdr.Size) + } + if err := f.Close(); err != nil { + return err + } + return applyMode(root, name, perm, special) + case tar.TypeSymlink: + // makeSymlink runs ensureParent itself; no second walk here. + return makeSymlink(root, name, hdr.Linkname, mode) + case tar.TypeLink: + if err := ensureParent(root, name); err != nil { + return err + } + _ = root.RemoveAll(name) + // A hard-link target names another member of the same rootfs; an + // absolute target is applied root-relative like the member names + // (the symlink path instead rewrites absolute targets, which + // os.Root would reject here). + return root.Link(rootRelative(filepath.Clean(hdr.Linkname)), name) + case tar.TypeChar, tar.TypeBlock, tar.TypeFifo: + return fmt.Errorf("unsupported special file type %s", tarTypeName(hdr.Typeflag)) + default: + return fmt.Errorf("unsupported tar type %d", hdr.Typeflag) + } +} + +func tarTypeName(t byte) string { + switch t { + case tar.TypeChar: + return "char" + case tar.TypeBlock: + return "block" + case tar.TypeFifo: + return "fifo" + default: + return fmt.Sprintf("%d", t) + } +} + +// makeSymlink creates a symlink at name pointing to target. Absolute targets +// are rewritten to their equivalent relative form so os.Root accepts them +// (it rejects absolute symlinks); the relative form resolves to the same +// guest path under --sysroot, so this is behavior-preserving. +// +// Both name and target are guest paths. Clean an absolute target as a guest path +// first, then strip the leading "/" to make it rootfs-relative and compute the +// link-relative form with filepath.Rel (which needs both sides in the same +// form). +func makeSymlink(root *os.Root, name, target string, mode os.FileMode) error { + if err := ensureParent(root, name); err != nil { + return err + } + _ = root.RemoveAll(name) + if filepath.IsAbs(target) { + tgt := strings.TrimPrefix(filepath.Clean(target), string(filepath.Separator)) + if tgt == "" { + tgt = "." + } + rel, err := filepath.Rel(filepath.Dir(name), tgt) + if err != nil { + return fmt.Errorf("rewrite absolute symlink %q: %w", target, err) + } + target = rel + } + if err := root.Symlink(target, name); err != nil { + return err + } + // Symlink mode is not portable to set across platforms; ignore mode. + _ = mode + return nil +} + +// ensureParent creates name's missing parent directories with the 0o755 +// default. It never chmods: a parent that already exists may carry an exact +// mode from its own tar entry, which must not be reset to the default here. +// +// The walk Lstats every intermediate component instead of calling MkdirAll: +// a lower layer may have planted a symlink (or plain file) where this entry +// needs a directory, and MkdirAll would resolve through it, silently landing +// the entry in whatever the link points at; containment via os.Root still +// holds, but the file ends up in the wrong directory while the entry's own +// path stays unresolved. Replace any such component with a real directory, +// matching containerd's and Docker's unpackers. +// realDirChain reports whether every component of dir exists as a real +// directory (no symlink, no plain file) inside root. Callers that must not +// create or replace components (the whiteout path) use this where entry +// creation would use ensureParent. +func realDirChain(root *os.Root, dir string) (bool, error) { + cur := "" + for part := range strings.SplitSeq(dir, string(filepath.Separator)) { + cur = filepath.Join(cur, part) + fi, err := root.Lstat(cur) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + if !fi.IsDir() { + return false, nil + } + } + return true, nil +} + +func ensureParent(root *os.Root, name string) error { + dir := filepath.Dir(name) + if dir == "." { + return nil + } + cur := "" + for part := range strings.SplitSeq(dir, string(filepath.Separator)) { + cur = filepath.Join(cur, part) + fi, err := root.Lstat(cur) + if err == nil { + if fi.IsDir() { + continue + } + if err := root.RemoveAll(cur); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + if err := root.Mkdir(cur, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} + +// mkdirAll creates a directory entry and any missing parents, then finalizes +// the entry's own mode. os.Root.Mkdir rejects modes that carry +// setuid/setgid/sticky bits ("unsupported file mode"), so create with the +// permission bits only and finalize via applyMode. Parents go through +// ensureParent so lower-layer symlinks on the path are replaced, not +// traversed. +func mkdirAll(root *os.Root, name string, perm, special os.FileMode) error { + if err := ensureParent(root, name); err != nil { + return err + } + // A lower layer may have left a non-directory here, typically a symlink, + // which Mkdir would otherwise resolve through, silently handing this + // layer's children to whatever the link points at. Replace it with a real + // directory, mirroring the RemoveAll the regular-file path does. + if fi, err := root.Lstat(name); err == nil && !fi.IsDir() { + if err := root.RemoveAll(name); err != nil { + return err + } + } + if err := root.Mkdir(name, perm); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + return applyMode(root, name, perm, special) +} + +// applyMode finalizes a just-created entry's mode to exactly perm|special. +// The chmod is unconditional: creation modes passed to os.Root.OpenFile and +// MkdirAll are masked by the process umask, so a restrictive host umask +// (e.g. 0077) would otherwise silently corrupt layer permissions. It also +// re-applies setuid/setgid/sticky, which os.Root creation methods reject at +// create time; Chmod -> syscallMode maps os.FileMode's high special-bit flags +// to the corresponding syscall bits. +// +// When the host cannot set the special bits, degrade to the plain permission +// bits rather than aborting the whole unpack. An unprivileged chmod that sets +// setuid/setgid is rejected with EPERM on macOS when the unpacked file's group +// is one the invoking user is not a member of: a new file inherits its parent +// directory's group (BSD semantics), e.g. wheel under /tmp, not the tar's +// root/shadow owner. The rootfs is owned by the invoking user, so these bits +// could not be honored at runtime on such a host regardless. Debian-family +// images (their shadow suite: chage, passwd, ...) would otherwise fail to +// unpack entirely. +func applyMode(root *os.Root, name string, perm, special os.FileMode) error { + err := root.Chmod(name, perm|special) + if shouldDropSpecial(err, special) { + fmt.Fprintf(os.Stderr, + "elfuse-oci: unpack: dropped %s on %q (unprivileged host)\n", + specialBitNames(special), name) + return root.Chmod(name, perm) + } + return err +} + +// specialBitNames returns a "/"-joined list of the special mode bits present in +// mode (setuid, setgid, sticky), so the drop diagnostic names the bit actually +// lost instead of assuming setuid/setgid. +func specialBitNames(mode os.FileMode) string { + var names []string + if mode&os.ModeSetuid != 0 { + names = append(names, "setuid") + } + if mode&os.ModeSetgid != 0 { + names = append(names, "setgid") + } + if mode&os.ModeSticky != 0 { + names = append(names, "sticky") + } + return strings.Join(names, "/") +} + +// shouldDropSpecial reports whether a failed mode-finalizing chmod should be +// retried without the special bits. Only a permission error qualifies, and +// only when special bits were actually requested, so a genuine chmod failure +// (a permission error with no special bits, or any non-permission error) still +// surfaces to the caller unchanged. +func shouldDropSpecial(err error, special os.FileMode) bool { + return err != nil && special != 0 && errors.Is(err, os.ErrPermission) +} + +// clearDirectory removes the existing children of dir (opaque whiteout), +// keeping entries the current layer itself created: opaque markers hide +// lower-layer content, and a marker ordered after its directory's same-layer +// additions must not wipe them. +func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { + fi, err := root.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + // The marker's directory may be a symlink (or other non-dir) planted by a + // lower layer. Reading through it would clear the link target's contents + // (files the image author never whited out). Replace it with a real + // empty directory instead: the opaque marker hides all lower content + // under this name anyway, mirroring the non-dir replacement mkdirAll and + // the regular-file path perform. + if !fi.IsDir() { + if applied[dir] { + // This layer created the non-dir itself; there is no lower + // content beneath it for the marker to hide. + return nil + } + if err := root.RemoveAll(dir); err != nil { + return err + } + return root.Mkdir(dir, 0o755) + } + d, err := root.Open(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + entries, err := d.ReadDir(-1) + d.Close() + if err != nil { + return err + } + for _, e := range entries { + child := filepath.Join(dir, e.Name()) + if applied[child] { + continue + } + if err := root.RemoveAll(child); err != nil { + return err + } + } + return nil +} diff --git a/cmd/elfuse-oci/unpack_image_test.go b/cmd/elfuse-oci/unpack_image_test.go new file mode 100644 index 00000000..11bf85bd --- /dev/null +++ b/cmd/elfuse-oci/unpack_image_test.go @@ -0,0 +1,755 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type tarEntry struct { + header tar.Header + body string +} + +func testTarLayer(t *testing.T, entries ...tarEntry) v1.Layer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, e := range entries { + h := e.header + if h.Typeflag == tar.TypeReg || h.Typeflag == tar.TypeRegA { + h.Size = int64(len(e.body)) + } + if err := tw.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if h.Size > 0 { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + opener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil + } + layer, err := tarball.LayerFromOpener(opener) + if err != nil { + t.Fatal(err) + } + return layer +} + +func testImageWithLayers(t *testing.T, layers ...v1.Layer) v1.Image { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, layers...) + if err != nil { + t.Fatal(err) + } + diffIDs := make([]v1.Hash, 0, len(layers)) + for _, layer := range layers { + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + diffIDs = append(diffIDs, diffID) + } + img, err = mutate.ConfigFile(img, &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Config: v1.Config{Cmd: []string{"/bin/sh"}}, + RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, + }) + if err != nil { + t.Fatal(err) + } + return img +} + +func TestUnpackImageAppliesLayersInOrder(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("etc", 0o755)}, + tarEntry{header: regHeader("etc/keep", 0o644, 0), body: "lower"}, + tarEntry{header: regHeader("etc/gone", 0o644, 0), body: "gone"}, + tarEntry{header: dirHeader("opt", 0o755)}, + tarEntry{header: regHeader("opt/lower", 0o644, 0), body: "hidden"}, + tarEntry{header: dirHeader("bin", 0o755)}, + tarEntry{header: regHeader("bin/busybox", 0o755, 0), body: "busy"}, + tarEntry{header: symHeader("bin/sh", "/bin/busybox")}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("etc/keep", 0o600, 0), body: "upper"}, + tarEntry{header: tar.Header{Name: "etc/.wh.gone", Typeflag: tar.TypeReg, Mode: 0o644}}, + tarEntry{header: tar.Header{Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + tarEntry{header: regHeader("opt/new", 0o644, 0), body: "new"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:layered", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:layered", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if b, err := os.ReadFile(filepath.Join(dest, "etc", "keep")); err != nil || string(b) != "upper" { + t.Fatalf("etc/keep = %q, err=%v; want upper", b, err) + } + if fi, err := os.Stat(filepath.Join(dest, "etc", "keep")); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("etc/keep mode = %v, err=%v; want 0600", fi, err) + } + if _, err := os.Stat(filepath.Join(dest, "etc", "gone")); !os.IsNotExist(err) { + t.Fatalf("whiteout target etc/gone = %v, want IsNotExist", err) + } + if _, err := os.Stat(filepath.Join(dest, "opt", "lower")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden opt/lower = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "opt", "new")); err != nil || string(b) != "new" { + t.Fatalf("opt/new = %q, err=%v; want new", b, err) + } + target, err := os.Readlink(filepath.Join(dest, "bin", "sh")) + if err != nil { + t.Fatal(err) + } + if target != "busybox" { + t.Fatalf("bin/sh target = %q, want busybox", target) + } +} + +// TestUnpackImageCleansUpPartialRootfs pins that a failed unpack never leaves +// anything at dest: the run paths infer "unpacked" from the path's existence, +// so a partial tree must not survive (or even be transiently visible under +// dest) to be executed by a later or concurrent run. The temp staging +// directory must not leak either. A pre-existing dest (explicit --rootfs) must +// be preserved. +func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { + good := testTarLayer(t, tarEntry{header: regHeader("ok", 0o644, 0), body: "x"}) + bad := testTarLayer(t, + tarEntry{header: tar.Header{Name: "dev/fifo", Typeflag: tar.TypeFifo, Mode: 0o644}}) + + s := openTestStore(t) + if _, err := s.addImage("local:partial", testImageWithLayers(t, good, bad)); err != nil { + t.Fatal(err) + } + + parent := t.TempDir() + dest := filepath.Join(parent, "rootfs") + if err := unpackImage(s, "local:partial", dest); err == nil { + t.Fatal("unpackImage succeeded, want failure on fifo entry") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("partial rootfs still present after failed unpack: err=%v", err) + } + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed unpack left litter next to dest: %v", entries) + } + + pre := t.TempDir() + sentinel := filepath.Join(pre, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + if err := unpackImage(s, "local:partial", pre); err == nil { + t.Fatal("unpackImage succeeded, want failure on fifo entry") + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("pre-existing rootfs dir was deleted on failed unpack: %v", err) + } +} + +// TestUnpackImagePreexistingDestUnpacksInPlace pins the explicit --rootfs +// contract: a destination that already exists is merged into in place rather +// than staged-and-renamed, so files the caller already put there survive a +// successful unpack. +func TestUnpackImagePreexistingDestUnpacksInPlace(t *testing.T) { + layer := testTarLayer(t, tarEntry{header: regHeader("ok", 0o644, 0), body: "x"}) + s := openTestStore(t) + if _, err := s.addImage("local:merge", testImageWithLayers(t, layer)); err != nil { + t.Fatal(err) + } + + dest := t.TempDir() + sentinel := filepath.Join(dest, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + if err := unpackImage(s, "local:merge", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + if b, err := os.ReadFile(sentinel); err != nil || string(b) != "keep" { + t.Fatalf("sentinel = %q, err=%v; want preserved by in-place unpack", b, err) + } + if b, err := os.ReadFile(filepath.Join(dest, "ok")); err != nil || string(b) != "x" { + t.Fatalf("ok = %q, err=%v; want unpacked", b, err) + } +} + +func TestApplyLayerErrors(t *testing.T) { + root, _ := newRoot(t) + if err := applyLayer(root, fakeLayer{uncompressedErr: errors.New("open failed")}); err == nil || + !strings.Contains(err.Error(), "open layer") { + t.Fatalf("applyLayer open err = %v, want open layer error", err) + } + if err := applyLayer(root, fakeLayer{uncompressed: "not a tar archive"}); err == nil || + !strings.Contains(err.Error(), "read tar entry") { + t.Fatalf("applyLayer corrupt tar err = %v, want read tar entry error", err) + } +} + +func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { + root, dir := newRoot(t) + for _, name := range []string{".", "/"} { + h := regHeader(name, 0o644, 0) + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry root no-op %q: %v", name, err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("root no-op entries = %v, want empty root", entries) + } + + hardlink := linkHeader("escape-link", "../outside") + if err := applyEntry(root, &hardlink, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatal("applyEntry accepted hardlink target escaping root") + } + + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte("outside"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(dir, "replace-me")); err != nil { + t.Fatal(err) + } + file := regHeader("replace-me", 0o644, int64(len("inside"))) + if err := applyEntry(root, &file, strings.NewReader("inside"), map[string]bool{}); err != nil { + t.Fatalf("applyEntry replacing symlink: %v", err) + } + if b, err := os.ReadFile(outside); err != nil || string(b) != "outside" { + t.Fatalf("outside target = %q, err=%v; want unchanged", b, err) + } + if li, err := os.Lstat(filepath.Join(dir, "replace-me")); err != nil || li.Mode()&os.ModeSymlink != 0 { + t.Fatalf("replace-me mode = %v, err=%v; want regular file", li, err) + } + if b, err := os.ReadFile(filepath.Join(dir, "replace-me")); err != nil || string(b) != "inside" { + t.Fatalf("replace-me content = %q, err=%v; want inside", b, err) + } +} + +func TestApplyEntryAdditionalErrorBranches(t *testing.T) { + t.Run("unsupported tar type", func(t *testing.T) { + root, _ := newRoot(t) + h := tar.Header{Name: "weird", Typeflag: 'x'} + err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "unsupported tar type") || !strings.Contains(err.Error(), tarTypeName('x')) { + t.Fatalf("unsupported type err = %v, want tar type error", err) + } + }) + + t.Run("absolute symlink to root", func(t *testing.T) { + root, dir := newRoot(t) + h := symHeader("usr/root-link", "/") + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry symlink to root: %v", err) + } + target, err := os.Readlink(filepath.Join(dir, "usr", "root-link")) + if err != nil { + t.Fatal(err) + } + if target != ".." { + t.Fatalf("root symlink target = %q, want ..", target) + } + }) + + t.Run("parent path is regular file", func(t *testing.T) { + root, dir := newRoot(t) + parent := regHeader("parent", 0o644, 0) + if err := applyEntry(root, &parent, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatal(err) + } + // A non-directory on the parent path is replaced with a real + // directory (containerd/Docker behavior), not an error: layers may + // legitimately turn a lower layer's file into a directory. + child := regHeader("parent/child", 0o644, 0) + if err := applyEntry(root, &child, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry child under regular-file parent: %v, want file replaced by directory", err) + } + fi, err := os.Lstat(filepath.Join(dir, "parent")) + if err != nil || !fi.IsDir() { + t.Fatalf("parent = %v, err=%v; want a real directory", fi, err) + } + if _, err := os.Stat(filepath.Join(dir, "parent", "child")); err != nil { + t.Fatalf("parent/child: %v, want created", err) + } + }) + + t.Run("opaque missing directory is no-op", func(t *testing.T) { + root, _ := newRoot(t) + h := tar.Header{Name: "missing/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("opaque missing dir: %v", err) + } + }) + + t.Run("opaque marker under lower-layer regular file replaces it", func(t *testing.T) { + root, dir := newRoot(t) + file := regHeader("notdir", 0o644, 0) + if err := applyEntry(root, &file, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatal(err) + } + // The opaque marker hides all lower content under the name, so a + // lower layer's non-directory there is replaced with an empty real + // directory rather than cleared through or rejected. + h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("opaque marker under regular file: %v, want file replaced by empty directory", err) + } + fi, err := os.Lstat(filepath.Join(dir, "notdir")) + if err != nil || !fi.IsDir() { + t.Fatalf("notdir = %v, err=%v; want a real directory", fi, err) + } + }) + + t.Run("opaque marker under same-layer regular file is a no-op", func(t *testing.T) { + root, dir := newRoot(t) + file := regHeader("notdir", 0o644, 0) + applied := map[string]bool{} + if err := applyEntry(root, &file, strings.NewReader(""), applied); err != nil { + t.Fatal(err) + } + h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), applied); err != nil { + t.Fatalf("opaque marker under same-layer file: %v, want no-op", err) + } + fi, err := os.Lstat(filepath.Join(dir, "notdir")) + if err != nil || !fi.Mode().IsRegular() { + t.Fatalf("notdir = %v, err=%v; want the same-layer file kept", fi, err) + } + }) +} + +type fakeLayer struct { + uncompressed string + uncompressedErr error +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("1", 64)}, nil +} + +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("2", 64)}, nil +} + +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil +} + +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + if f.uncompressedErr != nil { + return nil, f.uncompressedErr + } + return io.NopCloser(strings.NewReader(f.uncompressed)), nil +} + +func (f fakeLayer) Size() (int64, error) { + return int64(len(f.uncompressed)), nil +} + +func (f fakeLayer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + +// TestUnpackOpaqueAfterSameLayerChildren pins the whiteout scoping rule: +// opaque markers hide lower-layer content only. Tar entry order within a +// layer is not guaranteed, so a marker ordered after its directory's own +// same-layer additions must clear the lower content yet keep the additions. +func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("opt", 0o755)}, + tarEntry{header: regHeader("opt/lower", 0o644, 0), body: "hidden"}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("opt/new", 0o644, 0), body: "new"}, + tarEntry{header: tar.Header{Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opq-late", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:opq-late", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if _, err := os.Stat(filepath.Join(dest, "opt", "lower")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden opt/lower = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "opt", "new")); err != nil || string(b) != "new" { + t.Fatalf("opt/new = %q, err=%v; want same-layer addition to survive the late marker", b, err) + } +} + +// TestUnpackRejectsBareWhiteout pins that a malformed ".wh." entry with no +// target suffix fails extraction instead of resolving to Join(dir, "") and +// deleting the containing directory. +func TestUnpackRejectsBareWhiteout(t *testing.T) { + dest := t.TempDir() + if err := os.MkdirAll(filepath.Join(dest, "opt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "opt", "keep"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dest) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + hdr := tar.Header{Name: "opt/.wh.", Typeflag: tar.TypeReg, Mode: 0o644} + if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatal("bare .wh. entry applied, want invalid-whiteout error") + } + if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { + t.Fatalf("opt/keep after rejected whiteout: %v, want untouched", err) + } +} + +// TestUnpackRejectsDotWhiteout pins that a whiteout whose target collapses to +// a dot name fails extraction. A whiteout removes exactly one named sibling, +// but ".wh.." leaves "." and ".wh..." leaves "..", which Join folds into the +// containing directory and its parent. +// +// The ".wh.." lane covers a real defect: the entry was applied and +// RemoveAll(Join("opt", ".")) deleted the whole directory the marker sits in, +// silently discarding every lower layer's content under it. The ".wh..." lane +// is a regression guard: Join yields the rootfs root, which os.Root already +// refuses to remove, so it fails closed today: the subtest pins that the +// rejection stays explicit rather than resting on that accident. Layer +// tarballs are untrusted input, so both must be refused by name. +func TestUnpackRejectsDotWhiteout(t *testing.T) { + for _, name := range []string{"opt/.wh..", "opt/.wh..."} { + t.Run(name, func(t *testing.T) { + dest := t.TempDir() + if err := os.MkdirAll(filepath.Join(dest, "opt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "opt", "keep"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dest) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + hdr := tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o644} + if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + t.Errorf("%s entry applied, want invalid-whiteout error", name) + } + if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { + t.Errorf("opt/keep after rejected whiteout: %v, want untouched", err) + } + }) + } +} + +// TestUnpackDirReplacesLowerSymlink pins that a directory entry replaces a +// lower layer's symlink at the same path. Without the replacement, MkdirAll +// resolves through the link and the layer's children land in the link target, +// materializing a different filesystem tree. +func TestUnpackDirReplacesLowerSymlink(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("dir", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: dirHeader("dir", 0o755)}, + tarEntry{header: regHeader("dir/f", 0o644, 0), body: "payload"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:dir-over-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:dir-over-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "dir")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("dir mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + if b, err := os.ReadFile(filepath.Join(dest, "dir", "f")); err != nil || string(b) != "payload" { + t.Fatalf("dir/f = %q, err=%v; want payload", b, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "f")); !os.IsNotExist(err) { + t.Fatalf("real/f = %v, want IsNotExist (children must not leak through the lower symlink)", err) + } +} + +// TestUnpackParentSymlinkReplaced pins that an entry whose parent path +// component is a lower layer's symlink lands under a real directory at that +// name, not inside the link's target: MkdirAll-style parent creation would +// resolve through the link and write real/sub/f while leaving link/sub/f +// unresolved. +func TestUnpackParentSymlinkReplaced(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("link", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("link/sub/f", 0o644, 0), body: "payload"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:parent-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:parent-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "link")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("link mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + if b, err := os.ReadFile(filepath.Join(dest, "link", "sub", "f")); err != nil || string(b) != "payload" { + t.Fatalf("link/sub/f = %q, err=%v; want payload", b, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "sub")); !os.IsNotExist(err) { + t.Fatalf("real/sub = %v, want IsNotExist (entry must not land through the lower symlink)", err) + } +} + +// TestUnpackDirEntryParentSymlinkReplaced is the directory-entry variant of +// TestUnpackParentSymlinkReplaced: a dir entry beneath a lower-layer symlink +// parent must materialize under a real directory, not inside the link target. +func TestUnpackDirEntryParentSymlinkReplaced(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("link", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: dirHeader("link/sub", 0o750)}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:parent-link-dir", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:parent-link-dir", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "link")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("link mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + sub, err := os.Lstat(filepath.Join(dest, "link", "sub")) + if err != nil || !sub.IsDir() || sub.Mode().Perm() != 0o750 { + t.Fatalf("link/sub = %v, err=%v; want a 0750 directory", sub, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "sub")); !os.IsNotExist(err) { + t.Fatalf("real/sub = %v, want IsNotExist (dir must not land through the lower symlink)", err) + } +} + +// TestUnpackOpaqueThroughSymlinkKeepsTarget pins that an opaque whiteout whose +// directory is a lower layer's symlink does not clear the link target's +// contents: the marker hides lower content under its own name, so the link is +// replaced with an empty real directory and the target's files survive. +func TestUnpackOpaqueThroughSymlinkKeepsTarget(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("target", 0o755)}, + tarEntry{header: regHeader("target/keep", 0o644, 0), body: "keep"}, + tarEntry{header: symHeader("d", "/target")}, + ) + upper := testTarLayer(t, + tarEntry{header: tar.Header{Name: "d/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opaque-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:opaque-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if b, err := os.ReadFile(filepath.Join(dest, "target", "keep")); err != nil || string(b) != "keep" { + t.Fatalf("target/keep = %q, err=%v; want untouched by opaque-through-symlink", b, err) + } + fi, err := os.Lstat(filepath.Join(dest, "d")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("d mode = %v, want a real empty directory replacing the symlink", fi.Mode()) + } + entries, err := os.ReadDir(filepath.Join(dest, "d")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("d entries = %v, want empty", entries) + } +} + +// TestUnpackWhiteoutThroughSymlinkKeepsTarget pins that a plain whiteout whose +// parent is a lower layer's symlink does not delete through the link: the +// whiteout hides lower content at its literal path, and a parent that is not a +// real directory means that path does not exist, so the whiteout is a no-op. +// The regression this guards: root.RemoveAll resolves in-root symlink parents, +// so d/.wh.keep over a lower `d -> /target` deleted /target/keep, a file the +// layer never named. +func TestUnpackWhiteoutThroughSymlinkKeepsTarget(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("target", 0o755)}, + tarEntry{header: regHeader("target/keep", 0o644, 0), body: "keep"}, + tarEntry{header: symHeader("d", "/target")}, + ) + upper := testTarLayer(t, + tarEntry{header: tar.Header{Name: "d/.wh.keep", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:whiteout-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:whiteout-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if b, err := os.ReadFile(filepath.Join(dest, "target", "keep")); err != nil || string(b) != "keep" { + t.Fatalf("target/keep = %q, err=%v; want untouched by whiteout-through-symlink", b, err) + } + fi, err := os.Lstat(filepath.Join(dest, "d")) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("d mode = %v, want the lower symlink left in place (the layer ships no d entry)", fi.Mode()) + } +} + +// TestUnpackFailsOnCorruptGzipTrailer pins that a layer blob whose gzip +// trailer (CRC/ISIZE) is corrupted fails the unpack. Tar's end-of-archive +// marker arrives before the compressed stream's end, so without an explicit +// drain the decompressor never reads the trailer and the corruption vanished +// in the deferred Close, whose error is discarded: the old behavior unpacked +// such a layer successfully. +func TestUnpackFailsOnCorruptGzipTrailer(t *testing.T) { + layer := testTarLayer(t, + tarEntry{header: dirHeader("etc", 0o755)}, + tarEntry{header: regHeader("etc/keep", 0o644, 0), body: "keep"}, + ) + s := openTestStore(t) + if _, err := s.addImage("local:trailer", testImageWithLayers(t, layer)); err != nil { + t.Fatal(err) + } + img, err := s.image("local:trailer") + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil || len(layers) != 1 { + t.Fatalf("layers = %d, err=%v; want 1", len(layers), err) + } + digest, err := layers[0].Digest() + if err != nil { + t.Fatal(err) + } + blob := filepath.Join(s.root, "blobs", digest.Algorithm, digest.Hex) + raw, err := os.ReadFile(blob) + if err != nil { + t.Fatal(err) + } + // The gzip member ends with CRC32 (4 bytes) then ISIZE (4 bytes); flip a + // CRC bit so the deflate payload (and the tar inside it) stays intact. + raw[len(raw)-8] ^= 0x01 + if err := os.WriteFile(blob, raw, 0o644); err != nil { + t.Fatal(err) + } + + if err := unpackImage(s, "local:trailer", t.TempDir()); err == nil { + t.Fatal("unpackImage succeeded on a corrupted gzip trailer, want error") + } +} + +// TestUnpackRefusesSymlinkedStoreRootfs pins that a store-managed rootfs cache +// path which is not a real directory is refused. The store names that path by +// digest and only ever creates it by renaming a staging directory into place, +// so a symlink found there did not come from this package. os.OpenRoot follows +// symlinks in the directory name it is given, so a symlink planted at the +// digest path redirected the whole extraction (and any guest later run against +// it) outside the store. An explicit --rootfs keeps following links, which is +// why the check guards the store-managed path only. +func TestUnpackRefusesSymlinkedStoreRootfs(t *testing.T) { + s := openTestStore(t) + digest, err := s.addImage("local:tiny", tinyImage(t)) + if err != nil { + t.Fatal(err) + } + cache, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.MkdirAll(filepath.Dir(cache), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, cache); err != nil { + t.Fatal(err) + } + + _, _, err = captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "local:tiny"}) + }) + if err == nil { + t.Error("cmdUnpack into a symlinked store rootfs succeeded, want refusal") + } + if entries, rerr := os.ReadDir(outside); rerr != nil || len(entries) != 0 { + t.Errorf("symlink target holds %d entries (err=%v), want the unpack to never reach it", len(entries), rerr) + } +} diff --git a/cmd/elfuse-oci/unpack_test.go b/cmd/elfuse-oci/unpack_test.go new file mode 100644 index 00000000..85630c33 --- /dev/null +++ b/cmd/elfuse-oci/unpack_test.go @@ -0,0 +1,472 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func newRoot(t *testing.T) (*os.Root, string) { + t.Helper() + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("OpenRoot: %v", err) + } + t.Cleanup(func() { root.Close() }) + return root, dir +} + +func applyEntries(t *testing.T, root *os.Root, entries []tar.Header) { + t.Helper() + // One shared applied-set across all entries, matching applyLayer's one + // per layer: the helper models a single layer, so same-layer tracking + // must span the entries. Callers model a layer boundary as a separate + // call. + applied := map[string]bool{} + for _, h := range entries { + var content []byte + if (h.Typeflag == tar.TypeReg || h.Typeflag == tar.TypeRegA) && h.Size > 0 { + content = []byte(strings.Repeat("x", int(h.Size))) + } + hdr := h + if err := applyEntry(root, &hdr, bytes.NewReader(content), applied); err != nil { + t.Fatalf("applyEntry %q: %v", h.Name, err) + } + } +} + +func applyEntryWithContent(t *testing.T, root *os.Root, h tar.Header, content string) { + t.Helper() + hdr := h + hdr.Size = int64(len(content)) + if err := applyEntry(root, &hdr, strings.NewReader(content), map[string]bool{}); err != nil { + t.Fatalf("applyEntry %q: %v", h.Name, err) + } +} + +func regHeader(name string, mode int64, size int64) tar.Header { + return tar.Header{Name: name, Mode: mode, Typeflag: tar.TypeReg, Size: size} +} +func dirHeader(name string, mode int64) tar.Header { + return tar.Header{Name: name, Mode: mode, Typeflag: tar.TypeDir} +} +func symHeader(name, target string) tar.Header { + return tar.Header{Name: name, Typeflag: tar.TypeSymlink, Linkname: target} +} +func linkHeader(name, target string) tar.Header { + return tar.Header{Name: name, Typeflag: tar.TypeLink, Linkname: target} +} + +func TestUnpackRegularFilePerm(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("bin/prog", 0o755, 0), "") + applyEntryWithContent(t, root, regHeader("etc/secret", 0o600, 0), "") + + fi, err := os.Stat(filepath.Join(dir, "bin", "prog")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("perm: got %o, want 755", fi.Mode().Perm()) + } + fi, _ = os.Stat(filepath.Join(dir, "etc", "secret")) + if fi.Mode().Perm() != 0o600 { + t.Errorf("perm: got %o, want 600", fi.Mode().Perm()) + } +} + +func TestUnpackOldStyleRegularFile(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, tar.Header{ + Name: "old-style", + Mode: 0o644, + Typeflag: tar.TypeRegA, + }, "hello") + + got, err := os.ReadFile(filepath.Join(dir, "old-style")) + if err != nil { + t.Fatal(err) + } + if string(got) != "hello" { + t.Errorf("old-style file content = %q, want hello", got) + } +} + +func TestUnpackStickyAndSetuid(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("tmp", 0o1777), + regHeader("bin/su", 0o4755, 0), + }) + + fi, _ := os.Stat(filepath.Join(dir, "tmp")) + if fi.Mode()&os.ModeSticky == 0 { + t.Errorf("tmp missing sticky bit: %o", fi.Mode()) + } + fi, _ = os.Stat(filepath.Join(dir, "bin", "su")) + if fi.Mode()&os.ModeSetuid == 0 { + t.Errorf("su missing setuid bit: %o", fi.Mode()) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("su perm: got %o, want 755", fi.Mode().Perm()) + } +} + +// TestShouldDropSpecial pins the degrade decision applyMode makes when a +// mode-finalizing chmod fails: retry without the special bits only for a +// permission error that actually carried special bits, so a genuine chmod +// failure still surfaces. This is the portable stand-in for the live EPERM, +// which cannot be forced deterministically (t.TempDir() sits under a +// staff-group path where an unprivileged setgid chmod succeeds). +func TestShouldDropSpecial(t *testing.T) { + for _, tc := range []struct { + name string + err error + special os.FileMode + want bool + }{ + {"eperm with setgid degrades", syscall.EPERM, os.ModeSetgid, true}, + {"wrapped eperm with setuid degrades", + &os.PathError{Op: "chmodat", Path: "usr/bin/su", Err: syscall.EPERM}, + os.ModeSetuid, true}, + {"eperm without special bits surfaces", syscall.EPERM, 0, false}, + {"non-permission error surfaces", syscall.EINVAL, os.ModeSetgid, false}, + {"success is not a degrade", nil, os.ModeSetgid, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := shouldDropSpecial(tc.err, tc.special); got != tc.want { + t.Errorf("shouldDropSpecial(%v, %o) = %v, want %v", + tc.err, tc.special, got, tc.want) + } + }) + } +} + +// TestUnpackSetgidForeignGroupDoesNotAbort pins that a setgid entry never +// aborts the unpack, even on a host that rejects the special-bit chmod (macOS, +// when the file's inherited group is one the invoking user is not in; see +// applyMode). The permission bits must always land; the setgid bit may or may +// not survive depending on the host group, so it is deliberately not asserted. +func TestUnpackSetgidForeignGroupDoesNotAbort(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("usr", 0o755), + dirHeader("usr/bin", 0o755), + // chage in Debian: setgid group shadow, mode 02755. + regHeader("usr/bin/chage", 0o2755, 0), + }) + fi, err := os.Stat(filepath.Join(dir, "usr", "bin", "chage")) + if err != nil { + t.Fatalf("stat chage: %v", err) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("chage perm: got %o, want 755", fi.Mode().Perm()) + } +} + +// TestUnpackModesSurviveUmask pins that layer permissions are finalized with +// an explicit chmod: creation modes are masked by the process umask, so an +// image mode like 0755 or 0644 must survive a restrictive host umask even +// when no setuid/setgid/sticky bit is present. It also pins that a parent +// directory's exact mode from its own tar entry is not reset to the 0755 +// default by the ensure-parent pass of a later child entry. +// +// syscall.Umask is process-global: neither this test nor any sibling in the +// package may adopt t.Parallel while the mutation exists. +func TestUnpackModesSurviveUmask(t *testing.T) { + old := syscall.Umask(0o077) + defer syscall.Umask(old) + + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("opt", 0o755), + regHeader("opt/tool", 0o755, 0), + regHeader("opt/data", 0o644, 0), + dirHeader("secret", 0o700), + regHeader("secret/key", 0o600, 0), + }) + + for _, c := range []struct { + name string + want os.FileMode + }{ + {"opt", 0o755}, + {"opt/tool", 0o755}, + {"opt/data", 0o644}, + {"secret", 0o700}, + {"secret/key", 0o600}, + } { + fi, err := os.Stat(filepath.Join(dir, c.name)) + if err != nil { + t.Fatalf("stat %s: %v", c.name, err) + } + if fi.Mode().Perm() != c.want { + t.Errorf("%s perm: got %o, want %o", c.name, fi.Mode().Perm(), c.want) + } + } +} + +func TestUnpackAbsoluteSymlinkRewritten(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("bin", 0o755), + regHeader("bin/busybox", 0o755, 0), + // /bin/sh -> /bin/busybox (absolute). Should be rewritten to "busybox" + // (relative), resolving to /bin/busybox under the sysroot. + symHeader("bin/sh", "/bin/busybox"), + // /lib/ld -> /lib/ld-musl.so.1 + dirHeader("lib", 0o755), + regHeader("lib/ld-musl.so.1", 0o644, 0), + symHeader("lib/ld", "/lib/ld-musl.so.1"), + }) + + got, err := os.Readlink(filepath.Join(dir, "bin", "sh")) + if err != nil { + t.Fatal(err) + } + if filepath.IsAbs(got) { + t.Errorf("absolute symlink not rewritten: %q", got) + } + if got != "busybox" { + t.Errorf("rewritten target: got %q, want busybox", got) + } + // Resolving the rewritten link must reach the real file. + target := filepath.Join(dir, "bin", got) + if _, err := os.Stat(target); err != nil { + t.Errorf("rewritten link does not resolve: %v", err) + } + got2, _ := os.Readlink(filepath.Join(dir, "lib", "ld")) + if filepath.IsAbs(got2) { + t.Errorf("deep absolute symlink not rewritten: %q", got2) + } +} + +func TestUnpackAbsoluteSymlinkCleansRootTraversal(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("escape", 0o644, 0), "") + applyEntries(t, root, []tar.Header{ + dirHeader("usr", 0o755), + dirHeader("usr/bin", 0o755), + symHeader("usr/bin/link", "/../escape"), + }) + + link := filepath.Join(dir, "usr", "bin", "link") + got, err := os.Readlink(link) + if err != nil { + t.Fatal(err) + } + resolved := filepath.Clean(filepath.Join(filepath.Dir(link), got)) + want := filepath.Join(dir, "escape") + if resolved != want { + t.Errorf("rewritten target resolves to %s, want %s", resolved, want) + } +} + +func TestUnpackRelativeSymlinkPreserved(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("lib", 0o755), + regHeader("lib/real.so", 0o644, 0), + symHeader("lib/link.so", "real.so"), + }) + got, _ := os.Readlink(filepath.Join(dir, "lib", "link.so")) + if got != "real.so" { + t.Errorf("relative symlink changed: got %q, want real.so", got) + } +} + +func TestUnpackWhiteoutRemovesFile(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("etc", 0o755), + regHeader("etc/keep", 0o644, 0), + regHeader("etc/gone", 0o644, 0), + {Name: "etc/.wh.gone", Typeflag: tar.TypeReg}, + }) + if _, err := os.Stat(filepath.Join(dir, "etc", "gone")); !os.IsNotExist(err) { + t.Errorf("whiteout did not remove etc/gone: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "etc", "keep")); err != nil { + t.Errorf("whiteout removed etc/keep: %v", err) + } +} + +func TestUnpackOpaqueClearsDirectory(t *testing.T) { + root, dir := newRoot(t) + // Lower layer ships opt with two files. + applyEntries(t, root, []tar.Header{ + dirHeader("opt", 0o755), + regHeader("opt/lower-a", 0o644, 0), + regHeader("opt/lower-b", 0o644, 0), + }) + // Upper layer: the opaque marker clears the lower content, then this + // layer re-adds only lower-a. In one layer the marker would keep both + // files as same-layer children; the boundary is what makes lower-b go. + applyEntries(t, root, []tar.Header{ + {Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg}, + regHeader("opt/lower-a", 0o644, 0), + }) + if _, err := os.Stat(filepath.Join(dir, "opt", "lower-b")); !os.IsNotExist(err) { + t.Errorf("opaque did not clear opt/lower-b: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "opt", "lower-a")); err != nil { + t.Errorf("opaque removed re-added opt/lower-a: %v", err) + } +} + +func TestUnpackHardlink(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("etc/passwd", 0o644, 5), "hello") + applyEntries(t, root, []tar.Header{linkHeader("etc/passwd-link", "etc/passwd")}) + // Both must refer to the same inode (hardlink), same content. + orig, err := os.Stat(filepath.Join(dir, "etc", "passwd")) + if err != nil { + t.Fatalf("hardlink source missing: %v", err) + } + link, err := os.Stat(filepath.Join(dir, "etc", "passwd-link")) + if err != nil { + t.Fatalf("hardlink target missing: %v", err) + } + if !os.SameFile(orig, link) { + t.Fatal("passwd and passwd-link are distinct inodes, want a hardlink") + } + if b, err := os.ReadFile(filepath.Join(dir, "etc", "passwd-link")); err != nil || string(b) != "hello" { + t.Fatalf("hardlink content = %q, err=%v; want hello", b, err) + } +} + +func TestUnpackSpecialFilesRejected(t *testing.T) { + cases := []struct { + name string + typeflag byte + want string + }{ + {"dev/ttyS0", tar.TypeChar, "char"}, + {"dev/sda", tar.TypeBlock, "block"}, + {"run/pipe", tar.TypeFifo, "fifo"}, + } + for _, tc := range cases { + t.Run(tc.want, func(t *testing.T) { + root, dir := newRoot(t) + hdr := tar.Header{Name: tc.name, Mode: 0o644, Typeflag: tc.typeflag} + err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "unsupported special file type "+tc.want) { + t.Fatalf("applyEntry special %s err = %v, want unsupported error", tc.want, err) + } + if _, err := os.Lstat(filepath.Join(dir, tc.name)); !os.IsNotExist(err) { + t.Fatalf("special entry %s on disk: %v, want IsNotExist", tc.name, err) + } + }) + } +} + +func TestUnpackPathEscapeRejected(t *testing.T) { + root, _ := newRoot(t) + hdr := regHeader("../escape", 0o644, 0) + if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatalf("applyEntry accepted ../escape path") + } +} + +func TestUnpackWritesFileContent(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("msg.txt", 0o644, 5), "hello") + b, err := os.ReadFile(filepath.Join(dir, "msg.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "hello" { + t.Errorf("content: got %q, want hello", b) + } +} + +// Ensure applyEntry reads exactly the header's Size bytes from the reader +// (no over-read, no short read). The reader carries trailing bytes beyond +// Size so an over-read would be visible in both the count and the content, +// and a too-small reader must fail rather than write a truncated file. +func TestUnpackReadsExactSize(t *testing.T) { + root, dir := newRoot(t) + content := "abc123" + r := &countingReader{b: []byte(content + "TRAILING-JUNK")} + hdr := regHeader("f", 0o644, int64(len(content))) + if err := applyEntry(root, &hdr, r, map[string]bool{}); err != nil { + t.Fatalf("applyEntry: %v", err) + } + if r.n != len(content) { + t.Errorf("bytes read: got %d, want %d", r.n, len(content)) + } + if b, _ := os.ReadFile(filepath.Join(dir, "f")); string(b) != content { + t.Errorf("content mismatch: got %q", b) + } + + short := &countingReader{b: []byte("abc")} + hdr = regHeader("g", 0o644, int64(len(content))) + if err := applyEntry(root, &hdr, short, map[string]bool{}); err == nil { + t.Fatal("applyEntry accepted a short body, want size-mismatch error") + } +} + +type countingReader struct { + b []byte + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + if c.n >= len(c.b) { + return 0, io.EOF + } + n := copy(p, c.b[c.n:]) + c.n += n + return n, nil +} + +// TestUnpackAbsoluteMemberNames pins the root-relative application of member +// names archived absolute (GNU tar -P builders): "/etc/foo" must land at +// /etc/foo instead of every os.Root operation rejecting the name +// with "path escapes from parent". +func TestUnpackAbsoluteMemberNames(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("/etc", 0o755), + regHeader("/etc/foo", 0o644, 5), + }) + b, err := os.ReadFile(filepath.Join(dir, "etc", "foo")) + if err != nil { + t.Fatal(err) + } + if string(b) != "xxxxx" { + t.Errorf("content = %q, want xxxxx", b) + } +} + +// TestUnpackAbsoluteHardlinkTarget pins root-relative hard-link targets: a +// layer entry "bin/sh" hardlinked to "/bin/busybox" (absolute Linkname, legal +// in OCI layers) must link to the rootfs's own busybox, not abort the unpack +// with os.Root's "path escapes from parent". +func TestUnpackAbsoluteHardlinkTarget(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("bin/busybox", 0o755, 0), "hello") + applyEntries(t, root, []tar.Header{ + linkHeader("bin/sh", "/bin/busybox"), + }) + a, err := os.Stat(filepath.Join(dir, "bin", "busybox")) + if err != nil { + t.Fatal(err) + } + b, err := os.Stat(filepath.Join(dir, "bin", "sh")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(a, b) { + t.Error("bin/sh is not a hard link to bin/busybox") + } +} From 76f436f4519e41ac6a1c29b439a483eeb4222baa Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:24:06 +0200 Subject: [PATCH 05/26] Add elfuse-oci run command run is the last pipeline stage: resolve the image config into a concrete runspec, materialize the rootfs, and exec the existing `elfuse --sysroot ...` positional launch path, reusing elfuse's HVF bring-up, shebang, and dynamic-linker plumbing rather than reinventing guest launch. elfuse is located as a sibling binary ($ELFUSE_BIN overrides for tests) and replaced via exec so the shell reaps the same pid and terminal signals reach the guest directly. The runspec resolves Entrypoint/Cmd/Env/User/WorkingDir with the usual precedence (--entrypoint drops image Cmd; --env and --clear-env follow env(1) semantics; --workdir must be guest-absolute). A PATH is guaranteed: when neither the image config nor --env supplies one, Docker's conventional default is appended, so a guest whose image omits PATH still has a search path after the --clear-env launch. A relative path command resolves against the working directory and a bare name against the merged PATH inside the image rootfs, following Docker's exec-form rules (elfuse resolves the initial ELF before applying --workdir and does no PATH lookup, so the launcher must); a config-only image WorkingDir no layer ships is created at run time, as runc does. Non-absolute PATH elements follow the POSIX rule runc inherits: an empty element names the working directory and a relative one joins it, both resolved inside the rootfs like every other candidate. The workdir is cleaned before use: the guest path resolver folds doubled slashes and clamps /.. at /, so a config WorkingDir the runtime accepts cannot fail the pre-launch mkdir. A user --env with an empty variable name is rejected up front, where elfuse itself would reject it only after the rootfs work. A symbolic --user or image User is resolved against the image's own /etc/passwd and /etc/group through os.Root-bounded, no-follow opens, so a crafted rootfs cannot redirect resolution to host account files. run auto-pulls only when the ref is genuinely absent (errNotPulled) and surfaces store corruption instead of masking it behind a network pull; an explicit --platform must match the pinned image so a ref pulled for another architecture is not silently launched. Before exec, host-truth /etc/{resolv.conf,hosts,hostname} are injected into the rootfs through the same os.Root bounds. Tests cover runspec resolution and precedence, workdir normalization, the empty-env rejection, symbolic user lookup, runtime-file injection including staging cleanup when the final rename fails, and the exec argv shape via the execElfuseForRun seam and a subprocess exec probe. The run path applies the same store-managed rootfs rule as unpack: the digest-keyed cache must be a real directory, so a planted symlink cannot redirect the "already unpacked" probe and hand the guest a tree outside the store. --- cmd/elfuse-oci/commands.go | 126 ++++++- cmd/elfuse-oci/common.go | 7 + cmd/elfuse-oci/common_test.go | 37 ++ cmd/elfuse-oci/etc.go | 154 ++++++++ cmd/elfuse-oci/etc_test.go | 322 ++++++++++++++++ cmd/elfuse-oci/main.go | 25 +- cmd/elfuse-oci/run.go | 64 ++++ cmd/elfuse-oci/run_test.go | 97 +++++ cmd/elfuse-oci/runspec.go | 375 +++++++++++++++++++ cmd/elfuse-oci/runspec_test.go | 545 ++++++++++++++++++++++++++++ cmd/elfuse-oci/test_helpers_test.go | 15 + 11 files changed, 1762 insertions(+), 5 deletions(-) create mode 100644 cmd/elfuse-oci/etc.go create mode 100644 cmd/elfuse-oci/etc_test.go create mode 100644 cmd/elfuse-oci/run.go create mode 100644 cmd/elfuse-oci/run_test.go create mode 100644 cmd/elfuse-oci/runspec.go create mode 100644 cmd/elfuse-oci/runspec_test.go diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index 4a056e2e..ae4ab616 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -4,12 +4,14 @@ package main import ( + "errors" "fmt" "os" ) -// The subcommands share common-flag parsing (common.go) and the OCI -// image-layout store (store.go). pull/unpack/inspect are pure store ops. +// The four subcommands share common-flag parsing (common.go) and the OCI +// image-layout store (store.go). pull/unpack/inspect are pure store ops; run +// additionally resolves the runspec and execs elfuse (run.go). // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -102,3 +104,123 @@ func parseInspectArgs(args []string) (commonFlags, bool, string, error) { ref, err := oneArg("inspect", fs.Args(), "") return cf, asJSON, ref, err } + +// cmdRun runs an image: `elfuse-oci run [flags] [args...]`. +// +// Flags are parsed only up to the first positional (the reference); everything +// after the reference is the guest argv tail and is passed verbatim (no flag +// parsing), matching Docker's `run IMAGE ARGS` convention. +func cmdRun(args []string) error { + cf, rf, ref, tail, err := parseRunArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + img, err := s.image(ref) + if err != nil { + // Auto-pull only when the ref is simply absent, so `run` is + // self-sufficient on first use. Any other failure (corrupt refs.json, + // unreadable layout) must surface rather than mask itself behind a + // fresh network pull. + if !errors.Is(err, errNotPulled) { + return err + } + if err := pullImage(cf, s, ref); err != nil { + return err + } + img, err = s.image(ref) + if err != nil { + return err + } + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + // An explicit --platform must match the pinned image: the store pins one + // digest per ref, so a ref pulled for another platform would otherwise + // launch silently under the wrong architecture. + if cf.platformSet { + got := platformOf(cfg) + want := cf.platform + if got.OS != want.OS || got.Arch != want.Arch || + (want.Variant != "" && got.Variant != want.Variant) { + return fmt.Errorf( + "run: %s is pinned for %s, not %s; `pull --platform %s %s` (after rmi) to switch", + ref, got, want, want, ref) + } + } + digest, err := img.Digest() + if err != nil { + return err + } + storeManaged := rf.rootfs == "" + if storeManaged { + rf.rootfs, err = defaultRootfsForDigest(cf.store, digest.String()) + if err != nil { + return err + } + } + + // Ensure the rootfs is unpacked before computing the spec, because + // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if + // absent; a stale rootfs is the user's concern (run `unpack` to refresh). + // A store-managed path must also be a real directory, or the run would + // execute against whatever a planted symlink points at. + var published bool + if storeManaged { + if published, err = storeRootfsPublished(rf.rootfs); err != nil { + return err + } + } else if _, serr := os.Stat(rf.rootfs); serr == nil { + published = true + } else if !os.IsNotExist(serr) { + return serr + } + if !published { + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rf.rootfs) + if err := unpackImage(s, ref, rf.rootfs); err != nil { + return err + } + } + + spec, err := computeRunSpec(cfg, rf, rf.rootfs, tail) + if err != nil { + return err + } + // Inject host-truth /etc/{resolv.conf,hosts,hostname} into the rootfs so + // the guest's resolver/hostname work. On the plain path this mutates the + // unpacked rootfs directory (acceptable: --plain-rootfs is the v1/debug + // path; re-runs overwrite the same small files). + if err := injectRuntimeFiles(rf.rootfs); err != nil { + return err + } + + return execElfuseForRun(rf.rootfs, spec) +} + +func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error) { + var cf commonFlags + var rf runFlags + var env repeatedStringFlag + fs := newCommandFlagSet("run", &cf) + addPlatformFlag(fs, &cf) + fs.StringVar(&rf.entrypoint, "entrypoint", "", "") + fs.Var(&env, "env", "") + fs.BoolVar(&rf.clearEnv, "clear-env", false, "") + fs.StringVar(&rf.user, "user", "", "") + fs.StringVar(&rf.workdir, "workdir", "", "") + fs.StringVar(&rf.rootfs, "rootfs", "", "") + if err := fs.Parse(args); err != nil { + return cf, rf, "", nil, err + } + rf.env = []string(env) + rest := fs.Args() + if len(rest) == 0 { + return cf, rf, "", nil, fmt.Errorf("run: expected [args...]") + } + return cf, rf, rest[0], rest[1:], nil +} diff --git a/cmd/elfuse-oci/common.go b/cmd/elfuse-oci/common.go index 11e6dd99..9b2c026c 100644 --- a/cmd/elfuse-oci/common.go +++ b/cmd/elfuse-oci/common.go @@ -50,6 +50,13 @@ func (p *Platform) Set(s string) error { // Rosetta). var defaultPlatform = Platform{OS: "linux", Arch: "arm64"} +// platformOf reads the platform an image config declares. inspect, list, and +// run's --platform agreement check all derive it the same way, so they cannot +// disagree about what an image's platform is. +func platformOf(cfg *v1.ConfigFile) Platform { + return Platform{OS: cfg.OS, Arch: cfg.Architecture, Variant: cfg.Variant} +} + // formatCreated renders a config's creation time in the CLI's fixed // second-precision UTC form, shared so inspect and list print one format. func formatCreated(cfg *v1.ConfigFile) string { diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index 702d8999..95976cdd 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -89,6 +89,42 @@ func TestParseInspectArgs(t *testing.T) { } } +func TestParseRunArgs(t *testing.T) { + cf, rf, ref, tail, err := parseRunArgs([]string{ + "--store", "/s", + "--entrypoint", "/bin/sh", + "--env", "A=1", + "--env=B=2", + "--clear-env", + "--user", "1000:1000", + "--workdir", "/work", + "--rootfs", "/tmp/rootfs", + "alpine:3", + "-c", "echo hi", + }) + if err != nil { + t.Fatal(err) + } + if cf.store != "/s" { + t.Errorf("store = %q, want /s", cf.store) + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } + if !reflect.DeepEqual(tail, []string{"-c", "echo hi"}) { + t.Errorf("tail = %v, want [-c echo hi]", tail) + } + if rf.entrypoint != "/bin/sh" || rf.user != "1000:1000" || rf.workdir != "/work" || rf.rootfs != "/tmp/rootfs" { + t.Errorf("run flags = %+v", rf) + } + if !rf.clearEnv { + t.Error("clearEnv = false, want true") + } + if !reflect.DeepEqual(rf.env, []string{"A=1", "B=2"}) { + t.Errorf("env = %v, want [A=1 B=2]", rf.env) + } +} + func TestParseCommandFlagErrors(t *testing.T) { cases := []struct { name string @@ -97,6 +133,7 @@ func TestParseCommandFlagErrors(t *testing.T) { {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, + {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }()}, // --platform belongs to the commands that resolve a platform; // unpack and inspect discarding a successfully parsed flag would // mislead the caller. diff --git a/cmd/elfuse-oci/etc.go b/cmd/elfuse-oci/etc.go new file mode 100644 index 00000000..a6c88302 --- /dev/null +++ b/cmd/elfuse-oci/etc.go @@ -0,0 +1,154 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "io/fs" + "os" + "strings" + "time" +) + +var ( + hostnameForRuntime = os.Hostname + readHostResolvConfig = func() ([]byte, error) { return os.ReadFile("/etc/resolv.conf") } +) + +// injectRuntimeFiles writes host-truth /etc/{resolv.conf,hosts,hostname} into +// sysroot before elfuse launches the guest. Runtimes that consume OCI images +// synthesize these per-run rather than handing the guest the image's (often +// stub or empty) copies: the guest's resolver reads /etc/resolv.conf to find its +// nameserver, and because --sysroot redirects guest absolute paths into the +// rootfs, the guest would otherwise read the image's file, not the host's. +// elfuse does not do network namespacing (the guest uses the host network +// directly), so the host's resolver config is the correct one to hand it. +// +// Overwrite is intentional: these files are runtime-controlled, not image +// content. The caller passes the final sysroot elfuse will receive as +// --sysroot (the per-run COW clone on the case-sensitive path, or the plain +// rootfs directory on the --plain-rootfs path), so writes are isolated to +// this run except when the caller opted into mutating the base tree +// (--no-clone / --plain-rootfs). +func injectRuntimeFiles(sysroot string) error { + // All access goes through os.Root so image-controlled symlinks (a + // symlinked /etc directory or a symlinked target file such as + // etc/resolv.conf -> /etc/resolv.conf) cannot redirect the writes + // outside the rootfs. + root, err := os.OpenRoot(sysroot) + if err != nil { + return err + } + defer root.Close() + + // Guard against a stray non-directory at /etc (e.g. a malformed image): + // replace a symlink rather than chasing it, and reject any other + // non-directory up front; letting it slide would only surface later as + // an opaque "not a directory" from the first runtime-file write. + if li, err := root.Lstat("etc"); err == nil { + switch { + case li.Mode()&os.ModeSymlink != 0: + if err := root.Remove("etc"); err != nil { + return err + } + case !li.IsDir(): + return fmt.Errorf("rootfs /etc is a %s, want a directory", li.Mode().Type()) + } + } else if !os.IsNotExist(err) { + return err + } + if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + + host, err := hostnameForRuntime() + if err != nil || host == "" { + host = "localhost" + } + + if err := writeRuntimeFile(root, "etc/hostname", []byte(host+"\n")); err != nil { + return err + } + + // Minimal hosts map: localhost + the guest's own hostname, mirroring what + // image runtimes conventionally write. + hosts := "127.0.0.1\tlocalhost " + host + "\n::1\tlocalhost ip6-localhost\n" + if err := writeRuntimeFile(root, "etc/hosts", []byte(hosts)); err != nil { + return err + } + + // resolv.conf: copy the host's verbatim (host-truth) so the guest's DNS + // lookups hit the same nameservers the host uses. Fall back to a minimal + // default if the host file is absent or empty. + resolv, err := readHostResolvConfig() + if err != nil || len(resolv) == 0 { + resolv = []byte("nameserver 8.8.8.8\n") + } + return writeRuntimeFile(root, "etc/resolv.conf", resolv) +} + +// writeRuntimeFile replaces the rootfs-relative name with content. The +// content is written to a unique temp file beside name and renamed into +// place: rename replaces the existing directory entry without following it, +// so a symlink shipped by the image at that name is unlinked rather than +// chased, and a concurrent writer (two --no-clone / --plain-rootfs runs of +// the same digest share the base tree) never observes a missing or +// half-written file the way a remove-then-create sequence would expose. +func writeRuntimeFile(root *os.Root, name string, content []byte) error { + tmp := fmt.Sprintf("%s.tmp.%d.%d", name, os.Getpid(), time.Now().UnixNano()) + f, err := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := f.Write(content); err != nil { + f.Close() + _ = root.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = root.Remove(tmp) + return err + } + if err := root.Rename(tmp, name); err != nil { + _ = root.Remove(tmp) + return err + } + return nil +} + +// prepareRootfsForRun performs the per-run rootfs mutations both launch +// paths need, in one place so a new preparation step cannot land in one path +// and silently miss the other: inject the host-truth /etc files, then make +// sure the spec's working directory exists. +func prepareRootfsForRun(sysroot string, spec *runSpec) error { + if err := injectRuntimeFiles(sysroot); err != nil { + return err + } + if err := ensureWorkdir(sysroot, spec.Workdir); err != nil { + return fmt.Errorf("create workdir %s: %w", spec.Workdir, err) + } + return nil +} + +// ensureWorkdir creates the working directory inside the rootfs when it is +// absent. An image config may name a WorkingDir no layer ships (a config-only +// WORKDIR); Docker's runtime creates the directory at container start, so a +// run here must too rather than failing elfuse's chdir. An existing path, +// including one reached through image symlinks, is left untouched. +func ensureWorkdir(sysroot, workdir string) error { + rel := strings.TrimPrefix(workdir, "/") + if rel == "" { + return nil + } + root, err := os.OpenRoot(sysroot) + if err != nil { + return err + } + defer root.Close() + if _, err := root.Stat(rel); err == nil { + return nil + } + return root.MkdirAll(rel, 0o755) +} diff --git a/cmd/elfuse-oci/etc_test.go b/cmd/elfuse-oci/etc_test.go new file mode 100644 index 00000000..ec746832 --- /dev/null +++ b/cmd/elfuse-oci/etc_test.go @@ -0,0 +1,322 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestInjectRuntimeFiles asserts the three runtime files are written with the +// expected shape, and that a second call overwrites (not appends). +func TestInjectRuntimeFiles(t *testing.T) { + root := t.TempDir() + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + host, err := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if err != nil { + t.Fatalf("hostname: %v", err) + } + hostname := strings.TrimSpace(string(host)) + if hostname == "" { + t.Error("hostname is empty") + } + + hosts, err := os.ReadFile(filepath.Join(root, "etc", "hosts")) + if err != nil { + t.Fatalf("hosts: %v", err) + } + hs := string(hosts) + if !strings.Contains(hs, "127.0.0.1\tlocalhost") { + t.Errorf("hosts missing 127.0.0.1 localhost: %q", hs) + } + if !strings.Contains(hs, "::1\tlocalhost") { + t.Errorf("hosts missing ::1 localhost: %q", hs) + } + if !strings.Contains(hs, hostname) { + t.Errorf("hosts missing hostname %q: %q", hostname, hs) + } + + resolv, err := os.ReadFile(filepath.Join(root, "etc", "resolv.conf")) + if err != nil { + t.Fatalf("resolv.conf: %v", err) + } + // Substring only: the host's nameserver varies across macOS/Linux CI, and + // the fallback is "nameserver 8.8.8.8"; either way a nameserver line is + // present. + if !strings.Contains(string(resolv), "nameserver") { + t.Errorf("resolv.conf missing nameserver: %q", resolv) + } + + // Second call overwrites in place, never appends: hostname stays the same. + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + host2, _ := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if strings.TrimSpace(string(host2)) != hostname { + t.Errorf("hostname changed on re-inject: got %q want %q", host2, host) + } + + // Exactly the three runtime files: the temp-and-rename writes must not + // leave *.tmp.* staging litter behind. + entries, err := os.ReadDir(filepath.Join(root, "etc")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("etc entries = %v, want exactly hostname, hosts, resolv.conf", names) + } +} + +// TestWriteRuntimeFileLeavesNoTempOnFailure pins that a failed write does not +// leave a staging temp file behind. +func TestWriteRuntimeFileLeavesNoTempOnFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("directory write permissions do not bind as root") + } + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + + if err := writeRuntimeFile(root, "resolv.conf", []byte("nameserver 8.8.8.8\n")); err == nil { + t.Fatal("writeRuntimeFile into read-only dir succeeded, want error") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed write left litter: %v", entries) + } +} + +// TestWriteRuntimeFileLeavesNoTempOnRenameFailure drives the staging-cleanup +// branch the read-only-dir case cannot reach (there the create fails before +// any temp exists): a non-empty directory at the destination name makes the +// final rename fail after the temp file was written, so the cleanup must +// remove it. Regression guard: the cleanup is already correct; deleting the +// rename branch's root.Remove(tmp) turns this red. +func TestWriteRuntimeFileLeavesNoTempOnRenameFailure(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if err := os.MkdirAll(filepath.Join(dir, "resolv.conf", "x"), 0o755); err != nil { + t.Fatal(err) + } + + if err := writeRuntimeFile(root, "resolv.conf", []byte("nameserver 8.8.8.8\n")); err == nil { + t.Fatal("writeRuntimeFile onto a non-empty directory succeeded, want error") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != "resolv.conf" { + t.Fatalf("rename failure left litter: %v", entries) + } +} + +// TestInjectRuntimeFilesReplacesSymlinkEtc pins the symlink guard: a stray +// /etc symlink (e.g. from a malformed image) is replaced with a real directory +// so the writes cannot escape the rootfs. +func TestInjectRuntimeFilesReplacesSymlinkEtc(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "elsewhere") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatal(err) + } + etcLink := filepath.Join(root, "etc") + if err := os.Symlink(target, etcLink); err != nil { + t.Fatal(err) + } + + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + li, err := os.Lstat(etcLink) + if err != nil { + t.Fatalf("Lstat etc: %v", err) + } + if li.Mode()&os.ModeSymlink != 0 { + t.Fatalf("etc is still a symlink: mode %o", li.Mode()) + } + if !li.IsDir() { + t.Fatalf("etc is not a directory: mode %o", li.Mode()) + } + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + if _, err := os.Stat(filepath.Join(etcLink, name)); err != nil { + t.Errorf("etc/%s missing after symlink replacement: %v", name, err) + } + } + // The symlink target directory must not have received the files. + if _, err := os.Stat(filepath.Join(target, "hostname")); err == nil { + t.Error("hostname leaked into the symlink target directory") + } +} + +// TestInjectRuntimeFilesReplacesSymlinkTargets asserts that a symlink shipped +// by the image AT a runtime file's own name (etc/resolv.conf -> host path) is +// replaced with a regular file rather than followed: the write must not land +// in the symlink's target outside the rootfs. +func TestInjectRuntimeFilesReplacesSymlinkTargets(t *testing.T) { + outside := t.TempDir() + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + + const sentinel = "host-owned\n" + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + hostFile := filepath.Join(outside, name) + if err := os.WriteFile(hostFile, []byte(sentinel), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(hostFile, filepath.Join(root, "etc", name)); err != nil { + t.Fatal(err) + } + } + + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + li, err := os.Lstat(filepath.Join(root, "etc", name)) + if err != nil { + t.Fatalf("Lstat etc/%s: %v", name, err) + } + if li.Mode()&os.ModeSymlink != 0 { + t.Errorf("etc/%s is still a symlink after inject", name) + } + got, err := os.ReadFile(filepath.Join(outside, name)) + if err != nil { + t.Fatalf("read outside %s: %v", name, err) + } + if string(got) != sentinel { + t.Errorf("outside %s was overwritten through the symlink: %q", name, got) + } + } +} + +func TestInjectRuntimeFilesFallbacks(t *testing.T) { + oldHostname := hostnameForRuntime + oldReadResolv := readHostResolvConfig + hostnameForRuntime = func() (string, error) { return "", errors.New("hostname unavailable") } + readHostResolvConfig = func() ([]byte, error) { return nil, errors.New("resolv unavailable") } + t.Cleanup(func() { + hostnameForRuntime = oldHostname + readHostResolvConfig = oldReadResolv + }) + + root := t.TempDir() + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + hostname, err := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if err != nil { + t.Fatal(err) + } + if string(hostname) != "localhost\n" { + t.Fatalf("fallback hostname = %q, want localhost", hostname) + } + hosts, err := os.ReadFile(filepath.Join(root, "etc", "hosts")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(hosts), "localhost") { + t.Fatalf("fallback hosts = %q, want localhost mapping", hosts) + } + resolv, err := os.ReadFile(filepath.Join(root, "etc", "resolv.conf")) + if err != nil { + t.Fatal(err) + } + if string(resolv) != "nameserver 8.8.8.8\n" { + t.Fatalf("fallback resolv.conf = %q, want Google DNS fallback", resolv) + } +} + +func TestInjectRuntimeFilesFilesystemErrors(t *testing.T) { + rootFile := filepath.Join(t.TempDir(), "sysroot-file") + if err := os.WriteFile(rootFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if err := injectRuntimeFiles(rootFile); err == nil { + t.Fatal("injectRuntimeFiles with file sysroot succeeded, want error") + } +} + +// TestInjectRuntimeFilesRejectsRegularFileEtc pins the up-front check: an +// image shipping /etc as a regular file must fail with a clear error, not a +// confusing "not a directory" from the first runtime-file write. +func TestInjectRuntimeFilesRejectsRegularFileEtc(t *testing.T) { + sysroot := t.TempDir() + if err := os.WriteFile(filepath.Join(sysroot, "etc"), []byte("not a dir"), 0o644); err != nil { + t.Fatal(err) + } + err := injectRuntimeFiles(sysroot) + if err == nil || !strings.Contains(err.Error(), "want a directory") { + t.Fatalf("injectRuntimeFiles err = %v, want explicit non-directory /etc error", err) + } + if b, rerr := os.ReadFile(filepath.Join(sysroot, "etc")); rerr != nil || string(b) != "not a dir" { + t.Fatalf("etc file after rejection = %q, err=%v; want untouched", b, rerr) + } +} + +// TestEnsureWorkdir pins the config-only WORKDIR behavior: a working +// directory no layer ships is created at run time (as runc does), while an +// existing path, including one reached through an image symlink, is left +// untouched. +func TestEnsureWorkdir(t *testing.T) { + t.Run("creates missing", func(t *testing.T) { + rootfs := t.TempDir() + if err := ensureWorkdir(rootfs, "/app/nested"); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(filepath.Join(rootfs, "app", "nested")) + if err != nil || !fi.IsDir() { + t.Fatalf("workdir not created: fi=%v err=%v", fi, err) + } + }) + t.Run("root is a no-op", func(t *testing.T) { + if err := ensureWorkdir(t.TempDir(), "/"); err != nil { + t.Fatal(err) + } + }) + t.Run("existing symlink kept", func(t *testing.T) { + rootfs := t.TempDir() + if err := os.Mkdir(filepath.Join(rootfs, "real"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real", filepath.Join(rootfs, "app")); err != nil { + t.Fatal(err) + } + if err := ensureWorkdir(rootfs, "/app"); err != nil { + t.Fatal(err) + } + fi, err := os.Lstat(filepath.Join(rootfs, "app")) + if err != nil || fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("existing symlink replaced: fi=%v err=%v", fi, err) + } + }) +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index f8bcd502..d6de2ab6 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -3,8 +3,11 @@ // elfuse-oci is the OCI image CLI for elfuse. // -// It owns the OCI image pipeline (pull, store, inspect, and unpack for -// now) using go-containerregistry. elfuse itself stays a pure Linux +// It owns the OCI image pipeline (pull, store, inspect, unpack, and run +// orchestration) using go-containerregistry. For `run` it execs the existing +// `elfuse --sysroot ` positional launch path, +// reusing elfuse's HVF bring-up / shebang / dynamic-linker plumbing rather +// than reinventing guest launch. elfuse itself stays a pure Linux // syscall-to-Darwin runtime with no OCI awareness. // // Usage: @@ -12,6 +15,9 @@ // elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] // elfuse-oci unpack [--store DIR] [--rootfs DIR] // elfuse-oci inspect [--store DIR] [--json] +// elfuse-oci run [--store DIR] [--entrypoint E] [--env K=V]... +// [--user UID[:GID]] [--workdir DIR] [--platform ...] +// [args...] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., // localhost:5000/foo:tag, or name@sha256:...). The default store is @@ -39,6 +45,7 @@ commands: pull Pull an image reference into the local OCI store unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config + run Pull + unpack + exec the image's entrypoint under elfuse help Show this help version Print the elfuse-oci version @@ -46,8 +53,18 @@ common flags: --store DIR OCI store directory (default $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci) -pull flags: +pull and run flags: --platform os/arch[/variant] Target platform (default linux/arm64) + +run flags: + --entrypoint PATH Override the image Entrypoint (drops image Cmd) + --env KEY=VAL Set a guest env var (repeatable; bare KEY inherits + from the host environ) + --clear-env Start the guest env empty (only --env apply) + --user UID[:GID] Run as UID (and GID; defaults to UID). Symbolic names + are resolved against the image /etc/passwd and + /etc/group before exec. + --workdir DIR Guest-absolute initial working directory `) } @@ -82,6 +99,8 @@ func dispatch(cmd string, rest []string) error { return cmdUnpack(rest) case "inspect": return cmdInspect(rest) + case "run": + return cmdRun(rest) default: usage() return fmt.Errorf("unknown command: %s", cmd) diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go new file mode 100644 index 00000000..653f5f00 --- /dev/null +++ b/cmd/elfuse-oci/run.go @@ -0,0 +1,64 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +var execElfuseForRun = execElfuse + +// elfuseBin locates the elfuse binary to exec for `run`. Precedence: +// - $ELFUSE_BIN (an override hook for tests and wrapper scripts); +// - the sibling of this executable (build/elfuse-oci -> build/elfuse). +func elfuseBin() (string, error) { + if p := os.Getenv("ELFUSE_BIN"); p != "" { + return p, nil + } + exe, err := os.Executable() + if err != nil { + return "", fmt.Errorf("locate elfuse: %w", err) + } + return filepath.Join(filepath.Dir(exe), "elfuse"), nil +} + +// execElfuse replaces this process with `elfuse --sysroot --user U:G +// --workdir D --clear-env --env K=V ... `. +// +// --clear-env plus every final env var as an explicit --env makes the guest +// see exactly the runspec env (image Env merged with --env overrides, per the +// precedence matrix) rather than the host environ. +// +// syscall.Exec replaces elfuse-oci in place: the invoking shell reaps +// the same pid, and signals such as Ctrl-C go straight to elfuse rather than +// through a Go middleman. +func execElfuse(rootfs string, spec *runSpec) error { + bin, err := elfuseBin() + if err != nil { + return err + } + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + + argv := []string{ + "elfuse", + "--sysroot", rootfs, + "--user", fmt.Sprintf("%d:%d", spec.UID, spec.GID), + "--workdir", spec.Workdir, + "--clear-env", + } + for _, e := range spec.Env { + argv = append(argv, "--env", e) + } + argv = append(argv, spec.Args...) + + if err := syscall.Exec(bin, argv, os.Environ()); err != nil { + return fmt.Errorf("exec %s: %w", bin, err) + } + return nil // unreachable +} diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go new file mode 100644 index 00000000..f254182b --- /dev/null +++ b/cmd/elfuse-oci/run_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// writeElfuseStub writes a #!/bin/sh stub script that stands in for the +// elfuse binary, and points elfuseBin() at it via $ELFUSE_BIN. spawnElfuseWait +// exec.Command's whatever $ELFUSE_BIN names, so no real elfuse (and no HVF) is +// needed. t.Setenv restores the env on cleanup. +func writeElfuseStub(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "elfuse-stub.sh") + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("ELFUSE_BIN", p) + return p +} + +func TestElfuseBinEnvAndFallback(t *testing.T) { + want := filepath.Join(t.TempDir(), "elfuse-custom") + t.Setenv("ELFUSE_BIN", want) + got, err := elfuseBin() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("elfuseBin with env = %q, want %q", got, want) + } + + t.Setenv("ELFUSE_BIN", "") + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(filepath.Dir(exe), "elfuse") + got, err = elfuseBin() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("elfuseBin fallback = %q, want %q", got, want) + } +} + +func TestExecElfuseMissingBinary(t *testing.T) { + t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) + err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}) + if err == nil || !strings.Contains(err.Error(), "elfuse binary not found") { + t.Fatalf("execElfuse missing binary err = %v, want not found", err) + } +} + +func TestExecElfuseSuccessSubprocess(t *testing.T) { + dir := t.TempDir() + outPath := filepath.Join(dir, "argv.txt") + stub := filepath.Join(dir, "elfuse-stub.sh") + body := "printf '%s\\n' \"$@\" > \"$ELFUSE_EXEC_ARGV_OUT\"\nexit 17\n" + if err := os.WriteFile(stub, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(dir, "rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), + "ELFUSE_EXEC_ELFUSE_TEST=1", + "ELFUSE_BIN="+stub, + "ELFUSE_EXEC_ROOTFS="+rootfs, + "ELFUSE_EXEC_ARGV_OUT="+outPath, + ) + err := cmd.Run() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 17 { + t.Fatalf("execElfuse subprocess err = %T %v, want stub exit 17", err, err) + } + b, err := os.ReadFile(outPath) + if err != nil { + t.Fatal(err) + } + out := string(b) + for _, want := range []string{"--sysroot", rootfs, "--user", "1:2", "--workdir", "/", "--clear-env", "--env", "A=1", "/bin/echo", "hi"} { + if !strings.Contains(out, want) { + t.Fatalf("exec argv missing %q in:\n%s", want, out) + } + } +} diff --git a/cmd/elfuse-oci/runspec.go b/cmd/elfuse-oci/runspec.go new file mode 100644 index 00000000..9831de77 --- /dev/null +++ b/cmd/elfuse-oci/runspec.go @@ -0,0 +1,375 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// runSpec is the fully-resolved launch specification handed to elfuse. +type runSpec struct { + // Args is the final command vector: resolved Entrypoint followed by the + // resolved Cmd (image Cmd, the CLI tail, or nothing per the precedence + // matrix below). + Args []string + // Env is the final environment (image Env, overridden/appended by --env; + // --clear-env starts from empty). Bare KEY entries are expanded against + // the host environ here so elfuse receives only KEY=VAL. + Env []string + // Workdir is the guest-absolute initial working directory. + Workdir string + // UID/GID are the resolved numeric identity. + UID uint32 + GID uint32 +} + +// defaultGuestPath is Docker's conventional default PATH. computeRunSpec +// appends it when neither the image config nor --env supplies a PATH: run +// launches elfuse with --clear-env, so a guest whose image config omits PATH +// would otherwise start with no search path at all. +const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +// runFlags are the run-specific flags parsed between the common flags and the +// image reference. Everything after the reference is the guest argv tail and +// is not flag-parsed. +type runFlags struct { + entrypoint string + env []string + clearEnv bool + user string + workdir string + rootfs string +} + +// computeRunSpec applies the Entrypoint/Cmd/Env/WorkingDir/User precedence. +// +// Command (Docker/OCI semantics, matching the branch's runspec.c): +// - --entrypoint overrides the image Entrypoint AND discards the image Cmd. +// The CLI tail then becomes the new Cmd; with no tail the command is just +// the --entrypoint. +// - Without --entrypoint, a non-empty CLI tail replaces the image Cmd while +// the image Entrypoint is kept. +// - With neither --entrypoint nor a tail, the command is image Entrypoint + +// image Cmd. +// +// Env: +// - --clear-env starts from empty; otherwise the base is the image Env. +// - --env KEY=VAL overrides any existing KEY and appends if new. +// - --env KEY (bare) inherits KEY from the host environ (resolved here). +// - a PATH is guaranteed: when the merged result carries none, Docker's +// conventional default is appended (defaultGuestPath). +// +// WorkingDir: --workdir, else image WorkingDir, else "/". +// User: --user, else image User, resolved to numeric uid:gid against the +// rootfs /etc/passwd and /etc/group. +func computeRunSpec(cfg *v1.ConfigFile, rf runFlags, rootfs string, tail []string) (*runSpec, error) { + args := resolveArgs(cfg.Config.Entrypoint, cfg.Config.Cmd, rf.entrypoint, tail) + if len(args) == 0 { + return nil, fmt.Errorf("no command: image has no Entrypoint/Cmd and none given") + } + + // --env overrides are user input: reject an empty variable name ("=VAL" + // or "") before any rootfs work. elfuse applies the same rule + // (src/main.c), but only after the rootfs has been unpacked and + // prepared. Image-carried entries stay droppable in resolveEnv: an image + // shipping one still starts under Docker. + for _, e := range rf.env { + if k, _, _ := strings.Cut(e, "="); k == "" { + return nil, fmt.Errorf("invalid --env %q: empty variable name", e) + } + } + env := resolveEnv(cfg.Config.Env, rf.env, rf.clearEnv) + if !slices.ContainsFunc(env, func(kv string) bool { + return strings.HasPrefix(kv, "PATH=") + }) { + env = append(env, "PATH="+defaultGuestPath) + } + + workdir := rf.workdir + if workdir == "" { + workdir = cfg.Config.WorkingDir + } + if workdir == "" { + workdir = "/" + } + if !filepath.IsAbs(workdir) { + return nil, fmt.Errorf("workdir %q is not guest-absolute", workdir) + } + // Clean folds "//" and clamps "/.." at "/", matching the guest path + // resolver's clamp (src/syscall/path.c), so a WorkingDir the runtime + // would accept cannot fail the pre-launch ensureWorkdir: os.Root rejects + // a raw leading separator or ".." as an escape. + workdir = filepath.Clean(workdir) + + // Docker resolves a relative path command (one containing a slash, like + // "./server") against the working directory and a bare name against the + // merged PATH. elfuse resolves the initial ELF before elfuse_launch + // chdirs to --workdir and performs no PATH lookup, so both happen here: + // WorkingDir /app with Entrypoint ./server must load /app/server (not + // ./server relative to wherever the user invoked elfuse-oci), and an + // image Cmd of ["node"] must resolve inside the rootfs via PATH. + switch { + case filepath.IsAbs(args[0]): + case strings.Contains(args[0], "/"): + args[0] = filepath.Join(workdir, args[0]) + default: + resolved, err := lookPathInRootfs(rootfs, envValue(env, "PATH"), workdir, args[0]) + if err != nil { + return nil, err + } + args[0] = resolved + } + + user := rf.user + if user == "" { + user = cfg.Config.User + } + uid, gid, err := resolveUser(rootfs, user) + if err != nil { + return nil, err + } + + return &runSpec{ + Args: args, + Env: env, + Workdir: workdir, + UID: uid, + GID: gid, + }, nil +} + +// resolveArgs implements the Entrypoint/Cmd precedence described above. +func resolveArgs(imgEntry, imgCmd []string, cliEntry string, tail []string) []string { + if cliEntry != "" { + // --entrypoint clobbers image Entrypoint and image Cmd. The CLI tail, + // if any, is the new Cmd. + return append([]string{cliEntry}, tail...) + } + if len(tail) > 0 { + // CLI args replace image Cmd, keep image Entrypoint. + return slices.Concat(imgEntry, tail) + } + // No --entrypoint, no tail: image Entrypoint + image Cmd. + return slices.Concat(imgEntry, imgCmd) +} + +// resolveEnv builds the final environment list. +func resolveEnv(imgEnv []string, overrides []string, clearEnv bool) []string { + var out []string + seen := map[string]int{} + set := func(k, v string) { + if idx, ok := seen[k]; ok { + out[idx] = k + "=" + v + return + } + seen[k] = len(out) + out = append(out, k+"="+v) + } + if !clearEnv { + for _, kv := range imgEnv { + // Drop empty-key entries ("=VAL") instead of forwarding them: + // elfuse rejects --env with an empty variable name, and an image + // carrying such an entry still starts under Docker. + if k, v, ok := strings.Cut(kv, "="); ok && k != "" { + set(k, v) + } + } + } + for _, e := range overrides { + if k, v, ok := strings.Cut(e, "="); ok { + set(k, v) + continue + } + // Bare KEY: inherit from the host environ, or skip if unset. + if v, ok := os.LookupEnv(e); ok { + set(e, v) + } + } + return out +} + +// resolveUser resolves a user spec ("uid", "uid:gid", "name", "name:group") +// to numeric uid:gid against the rootfs /etc/passwd and /etc/group. A bare +// numeric uid defaults gid to uid (matching elfuse's --user convention). +// "root" resolves through /etc/passwd like any other name (its gid can +// differ from 0), but falls back to 0:0 when the rootfs has no usable +// passwd entry for it: root's uid is 0 by definition, so FROM scratch-style +// images with USER root keep working. An explicit ":group" part is still +// resolved normally. +func resolveUser(rootfs, spec string) (uint32, uint32, error) { + if spec == "" { + return 0, 0, nil + } + userPart, groupPart, _ := strings.Cut(spec, ":") + + uid, puidGid, err := resolveUserPart(rootfs, userPart) + if err != nil { + if userPart != "root" { + return 0, 0, err + } + // No readable /etc/passwd or no root entry: uid 0 by definition, + // gid 0 as the only sane default. + uid, puidGid = 0, 0 + } + var gid uint32 + switch { + case groupPart == "": + gid = puidGid // passwd gid, or == uid for bare numeric + case isAllDigits(groupPart): + g, err := strconv.ParseUint(groupPart, 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("invalid gid %q: %w", groupPart, err) + } + gid = uint32(g) + default: + g, err := lookupGroup(rootfs, groupPart) + if err != nil { + return 0, 0, err + } + gid = g + } + return uid, gid, nil +} + +// resolveUserPart resolves the user component to (uid, defaultGid). For a +// numeric uid the default gid is the uid itself; for a name it is the gid +// field of the matching /etc/passwd entry. +func resolveUserPart(rootfs, part string) (uint32, uint32, error) { + if isAllDigits(part) { + u, err := strconv.ParseUint(part, 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("invalid uid %q: %w", part, err) + } + uid := uint32(u) + return uid, uid, nil + } + return lookupPasswd(rootfs, part) +} + +func isAllDigits(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return true +} + +// envValue returns the value of key in a resolved KEY=VALUE environment +// slice, or "" when absent. resolveEnv dedups keys, so the first match is +// the only one. +func envValue(env []string, key string) string { + for _, kv := range env { + if v, ok := strings.CutPrefix(kv, key+"="); ok { + return v + } + } + return "" +} + +// lookPathInRootfs resolves a bare command name against the merged guest PATH +// inside the image rootfs, as Docker does for exec-form commands. Candidates +// resolve through os.Root so image symlinks stay confined to the rootfs; a +// match must be a regular file with an execute bit. Non-absolute PATH +// elements follow the POSIX rule runc inherits from exec.LookPath: an empty +// element names the working directory and a relative one resolves against +// it, so the search still happens entirely inside the rootfs. +func lookPathInRootfs(rootfs, pathList, workdir, name string) (string, error) { + root, err := os.OpenRoot(rootfs) + if err != nil { + return "", err + } + defer root.Close() + for dir := range strings.SplitSeq(pathList, ":") { + if dir == "" { + dir = workdir + } else if !filepath.IsAbs(dir) { + dir = filepath.Join(workdir, dir) + } + guest := filepath.Join(dir, name) + st, err := root.Stat(strings.TrimPrefix(guest, "/")) + if err != nil || !st.Mode().IsRegular() || st.Mode()&0o111 == 0 { + continue + } + return guest, nil + } + return "", fmt.Errorf("%q: executable file not found in image PATH", name) +} + +// openInRootfs opens a rootfs-relative path via os.Root so an +// image-controlled symlink (e.g. etc/passwd -> /etc/passwd) cannot redirect +// the read to host files outside the rootfs. The returned file stays valid +// after the root handle is closed. +func openInRootfs(rootfs, name string) (*os.File, error) { + root, err := os.OpenRoot(rootfs) + if err != nil { + return nil, err + } + defer root.Close() + return root.Open(name) +} + +// findColonEntry scans the colon-separated database / (e.g. +// etc/passwd) for the line whose first field is name and has at least +// minFields fields, returning the fields. Errors name the file guest-absolute +// so callers only add their own context prefix. +func findColonEntry(rootfs, file, name string, minFields int) ([]string, error) { + f, err := openInRootfs(rootfs, file) + if err != nil { + return nil, fmt.Errorf("open /%s: %w", file, err) + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Split(sc.Text(), ":") + if len(fields) >= minFields && fields[0] == name { + return fields, nil + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("scan /%s: %w", file, err) + } + return nil, fmt.Errorf("not found in /%s", file) +} + +// lookupPasswd finds name in /etc/passwd, returning (uid, gid). +func lookupPasswd(rootfs, name string) (uint32, uint32, error) { + fields, err := findColonEntry(rootfs, "etc/passwd", name, 4) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: %w", name, err) + } + uid, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: bad uid in /etc/passwd: %w", name, err) + } + gid, err := strconv.ParseUint(fields[3], 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: bad gid in /etc/passwd: %w", name, err) + } + return uint32(uid), uint32(gid), nil +} + +// lookupGroup finds name in /etc/group, returning gid. +func lookupGroup(rootfs, name string) (uint32, error) { + fields, err := findColonEntry(rootfs, "etc/group", name, 3) + if err != nil { + return 0, fmt.Errorf("resolve group %q: %w", name, err) + } + gid, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + return 0, fmt.Errorf("resolve group %q: bad gid in /etc/group: %w", name, err) + } + return uint32(gid), nil +} diff --git a/cmd/elfuse-oci/runspec_test.go b/cmd/elfuse-oci/runspec_test.go new file mode 100644 index 00000000..42872577 --- /dev/null +++ b/cmd/elfuse-oci/runspec_test.go @@ -0,0 +1,545 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" +) + +func TestResolveArgs(t *testing.T) { + cases := []struct { + name string + imgEntry, imgCmd []string + cliEntry string + tail []string + want []string + }{ + {"image entry+cmd, no overrides", []string{"/ep"}, []string{"-c"}, "", nil, []string{"/ep", "-c"}}, + {"tail replaces cmd, keeps entry", []string{"/ep"}, []string{"-c"}, "", []string{"-x"}, []string{"/ep", "-x"}}, + {"--entrypoint clobbers entry+cmd, no tail", []string{"/ep"}, []string{"-c"}, "/new", nil, []string{"/new"}}, + {"--entrypoint + tail", []string{"/ep"}, []string{"-c"}, "/new", []string{"-x"}, []string{"/new", "-x"}}, + {"no entrypoint, image cmd", nil, []string{"/bin/sh"}, "", nil, []string{"/bin/sh"}}, + {"no entrypoint, tail replaces cmd", nil, []string{"/bin/sh"}, "", []string{"/bin/echo", "hi"}, []string{"/bin/echo", "hi"}}, + {"nothing at all", nil, nil, "", nil, nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := resolveArgs(c.imgEntry, c.imgCmd, c.cliEntry, c.tail) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("resolveArgs: got %v, want %v", got, c.want) + } + }) + } +} + +func TestResolveEnv(t *testing.T) { + t.Setenv("ELFUSE_TEST_HOST", "from-host") + // The unset-KEY case must not depend on the ambient environment: + // t.Setenv records the prior state for restore (and bars t.Parallel), + // then the immediate Unsetenv guarantees the name is absent. + t.Setenv("ELFUSE_TEST_UNSET", "") + os.Unsetenv("ELFUSE_TEST_UNSET") + cases := []struct { + name string + imgEnv []string + overrides []string + clearEnv bool + want []string + }{ + {"image env only", []string{"A=1", "B=2"}, nil, false, []string{"A=1", "B=2"}}, + {"override existing", []string{"A=1"}, []string{"A=9"}, false, []string{"A=9"}}, + {"append new", []string{"A=1"}, []string{"B=2"}, false, []string{"A=1", "B=2"}}, + {"clear-env drops image env", []string{"A=1"}, []string{"B=2"}, true, []string{"B=2"}}, + {"bare KEY inherits host", []string{"A=1"}, []string{"ELFUSE_TEST_HOST"}, false, []string{"A=1", "ELFUSE_TEST_HOST=from-host"}}, + {"bare KEY unset on host is skipped", []string{"A=1"}, []string{"ELFUSE_TEST_UNSET"}, false, []string{"A=1"}}, + {"empty-key image entry dropped", []string{"=1", "A=2"}, nil, false, []string{"A=2"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := resolveEnv(c.imgEnv, c.overrides, c.clearEnv) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("resolveEnv: got %v, want %v", got, c.want) + } + }) + } +} + +func TestResolveUser(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte( + "root:x:0:0:root:/root:/bin/sh\nbin:x:1:1:bin:/bin:/sbin/nologin\nnobody:x:65534:65534:nobody:/:/sbin/nologin\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte( + "root:x:0:\nbin:x:1:\nstaff:x:20:\n"), 0o644); err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + spec string + wantUID uint32 + wantGID uint32 + wantErr bool + }{ + {"empty is root", "", 0, 0, false}, + {"root name", "root", 0, 0, false}, + {"bare numeric uid defaults gid=uid", "1000", 1000, 1000, false}, + {"numeric uid:gid", "1000:20", 1000, 20, false}, + {"name from passwd", "bin", 1, 1, false}, + {"name:group", "bin:staff", 1, 20, false}, + {"name:numeric gid", "bin:99", 1, 99, false}, + {"unknown user errors", "ghost", 0, 0, true}, + {"unknown group errors", "bin:ghost", 0, 0, true}, + {"root:group resolves the group part", "root:staff", 0, 20, false}, + {"root with unknown group errors", "root:ghost", 0, 0, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + uid, gid, err := resolveUser(root, c.spec) + if (err != nil) != c.wantErr { + t.Fatalf("resolveUser(%q): err=%v, wantErr=%v", c.spec, err, c.wantErr) + } + if c.wantErr { + return + } + if uid != c.wantUID || gid != c.wantGID { + t.Errorf("resolveUser(%q): uid=%d gid=%d, want %d:%d", c.spec, uid, gid, c.wantUID, c.wantGID) + } + }) + } +} + +// TestResolveUserRootGidFromPasswd pins that "root" resolves through +// /etc/passwd like any other name: a root entry with a non-zero gid wins +// over the 0:0 fallback. +func TestResolveUserRootGidFromPasswd(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte( + "root:x:0:50:root:/root:/bin/sh\n"), 0o644); err != nil { + t.Fatal(err) + } + uid, gid, err := resolveUser(root, "root") + if err != nil { + t.Fatalf("resolveUser(root): %v", err) + } + if uid != 0 || gid != 50 { + t.Errorf("resolveUser(root): uid=%d gid=%d, want 0:50", uid, gid) + } +} + +// TestResolveUserRootWithoutPasswd pins the FROM scratch fallback: with no +// readable /etc/passwd (or one lacking a root entry), "root" must resolve +// to 0:0 instead of erroring. +func TestResolveUserRootWithoutPasswd(t *testing.T) { + cases := []struct { + name string + passwd string // written to etc/passwd when non-empty + }{ + {"no passwd", ""}, + {"no root entry", "bin:x:1:1:bin:/bin:/sbin/nologin\n"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + root := t.TempDir() + if c.passwd != "" { + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(c.passwd), 0o644); err != nil { + t.Fatal(err) + } + } + uid, gid, err := resolveUser(root, "root") + if err != nil { + t.Fatalf("resolveUser(root): %v", err) + } + if uid != 0 || gid != 0 { + t.Errorf("resolveUser(root): uid=%d gid=%d, want 0:0", uid, gid) + } + }) + } +} + +func TestLookupPasswdScannerError(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + longLine := strings.Repeat("x", bufio.MaxScanTokenSize+1) + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(longLine), 0o644); err != nil { + t.Fatal(err) + } + _, _, err := lookupPasswd(root, "root") + if err == nil || !strings.Contains(err.Error(), "scan /etc/passwd") { + t.Fatalf("lookupPasswd err = %v, want scan /etc/passwd error", err) + } +} + +// TestLookupPasswdRejectsSymlinkEscape pins the rootfs-bounded open: an image +// whose etc/passwd is a symlink to a file outside the rootfs must not have +// user resolution read that host file. +func TestLookupPasswdRejectsSymlinkEscape(t *testing.T) { + outside := t.TempDir() + hostPasswd := filepath.Join(outside, "passwd") + if err := os.WriteFile(hostPasswd, []byte("evil:x:0:0::/:/bin/sh\n"), 0o644); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(hostPasswd, filepath.Join(root, "etc", "passwd")); err != nil { + t.Fatal(err) + } + if _, _, err := lookupPasswd(root, "evil"); err == nil { + t.Fatal("lookupPasswd resolved a user through a symlink escaping the rootfs") + } + + if err := os.Symlink(hostPasswd, filepath.Join(root, "etc", "group")); err != nil { + t.Fatal(err) + } + if _, err := lookupGroup(root, "evil"); err == nil { + t.Fatal("lookupGroup resolved a group through a symlink escaping the rootfs") + } +} + +func TestLookupGroupScannerError(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + longLine := strings.Repeat("x", bufio.MaxScanTokenSize+1) + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte(longLine), 0o644); err != nil { + t.Fatal(err) + } + _, err := lookupGroup(root, "root") + if err == nil || !strings.Contains(err.Error(), "scan /etc/group") { + t.Fatalf("lookupGroup err = %v, want scan /etc/group error", err) + } +} + +// TestComputeRunSpecNoCommand exercises the empty-command error branch: no +// image Entrypoint/Cmd and no --entrypoint/tail yields an error. computeRunSpec +// takes a *v1.ConfigFile directly, so no real image is needed; with User empty, +// resolveUser returns 0:0 without touching /etc/passwd. +func TestComputeRunSpecNoCommand(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{}} // no Entrypoint, no Cmd + if _, err := computeRunSpec(cfg, runFlags{}, t.TempDir(), nil); err == nil || + !strings.Contains(err.Error(), "no command") { + t.Fatalf("err=%v, want an error containing %q", err, "no command") + } +} + +// TestComputeRunSpecWorkdirNotAbsolute covers the non-absolute workdir error +// branch. The command check (runspec.go:73) runs before the workdir check +// (:89), so the config must carry a valid Cmd to reach it. A subtest covers +// the image-config WorkingDir path too. +func TestComputeRunSpecWorkdirNotAbsolute(t *testing.T) { + t.Run("flag workdir", func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}}} + rf := runFlags{workdir: "relative/path"} + _, err := computeRunSpec(cfg, rf, t.TempDir(), nil) + if err == nil || !strings.Contains(err.Error(), "not guest-absolute") { + t.Fatalf("err=%v, want an error containing %q", err, "not guest-absolute") + } + }) + t.Run("image workdir", func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}, WorkingDir: "rel"}} + _, err := computeRunSpec(cfg, runFlags{}, t.TempDir(), nil) + if err == nil || !strings.Contains(err.Error(), "not guest-absolute") { + t.Fatalf("err=%v, want an error containing %q", err, "not guest-absolute") + } + }) +} + +// TestComputeRunSpecWorkdirNormalized pins that an absolute WorkingDir is +// cleaned before use: the guest path resolver folds "//" and clamps "/.." at +// the root (src/syscall/path.c), so a config the runtime would accept must +// not fail the pre-launch ensureWorkdir, whose os.Root rejects a raw leading +// separator or "..". The old behavior forwarded the raw string and aborted +// the run before launch. +func TestComputeRunSpecWorkdirNormalized(t *testing.T) { + for _, c := range []struct{ in, want string }{ + {"//opt", "/opt"}, + {"/../opt", "/opt"}, + {"/app//nested/", "/app/nested"}, + } { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}}} + spec, err := computeRunSpec(cfg, runFlags{workdir: c.in}, t.TempDir(), nil) + if err != nil { + t.Fatalf("workdir %q: %v", c.in, err) + } + if spec.Workdir != c.want { + t.Errorf("workdir %q: got %q, want %q", c.in, spec.Workdir, c.want) + } + } +} + +// TestComputeRunSpecRejectsEmptyEnvKey pins that a user --env with an empty +// variable name fails before any rootfs work: elfuse rejects the same input +// (src/main.c), but only after unpack, /etc injection, and workdir creation. +// The old behavior forwarded "=VAL" into that late failure and silently +// dropped "". Image-carried empty keys stay droppable (resolveEnv): such an +// image still starts under Docker. +func TestComputeRunSpecRejectsEmptyEnvKey(t *testing.T) { + for _, bad := range []string{"=VAL", ""} { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}}} + _, err := computeRunSpec(cfg, runFlags{env: []string{bad}}, t.TempDir(), nil) + if err == nil || !strings.Contains(err.Error(), "empty variable name") { + t.Errorf("env %q: err=%v, want an empty-variable-name error", bad, err) + } + } +} + +func writeUserFiles(t *testing.T, root, passwd, group string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(passwd), 0o644); err != nil { + t.Fatal(err) + } + if group != "" { + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte(group), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// TestComputeRunSpecRelativeEntrypoint pins the Docker rules for exec-form +// commands: a relative path entrypoint (contains a slash) resolves against +// the working directory, and a bare name resolves via the merged PATH inside +// the image rootfs. elfuse resolves the initial ELF before chdiring to +// --workdir and does no PATH lookup, so both must happen in the spec. +func TestComputeRunSpecRelativeEntrypoint(t *testing.T) { + rootfs := t.TempDir() + if err := os.MkdirAll(filepath.Join(rootfs, "usr", "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfs, "usr", "bin", "node"), []byte("#!"), 0o755); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + args []string + want string + wantErr string + }{ + {"dot-relative", []string{"./server"}, "/app/server", ""}, + {"subdir-relative", []string{"bin/tool"}, "/app/bin/tool", ""}, + {"bare name via image PATH", []string{"node"}, "/usr/bin/node", ""}, + {"bare name absent", []string{"missing"}, "", "not found in image PATH"}, + {"absolute untouched", []string{"/entry"}, "/entry", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{ + Entrypoint: tc.args, + WorkingDir: "/app", + }} + spec, err := computeRunSpec(cfg, runFlags{}, rootfs, nil) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want an error containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if spec.Args[0] != tc.want { + t.Errorf("Args[0] = %q, want %q", spec.Args[0], tc.want) + } + }) + } +} + +// TestComputeRunSpecRelativePathEntries pins the POSIX treatment of +// non-absolute PATH elements runc inherits from exec.LookPath: an empty +// element names the working directory and a relative one resolves against +// it, both confined to the rootfs. The old behavior skipped such entries +// unconditionally, failing a bare command Docker starts. +func TestComputeRunSpecRelativePathEntries(t *testing.T) { + rootfs := t.TempDir() + if err := os.MkdirAll(filepath.Join(rootfs, "app", "tools"), 0o755); err != nil { + t.Fatal(err) + } + for _, p := range []string{ + filepath.Join(rootfs, "app", "server"), + filepath.Join(rootfs, "app", "tools", "helper"), + } { + if err := os.WriteFile(p, []byte("#!"), 0o755); err != nil { + t.Fatal(err) + } + } + cases := []struct { + name string + path string + cmd string + want string + }{ + {"empty entry means workdir", "PATH=:/bin", "server", "/app/server"}, + {"relative entry joins workdir", "PATH=tools:/bin", "helper", "/app/tools/helper"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{ + Entrypoint: []string{tc.cmd}, + Env: []string{tc.path}, + WorkingDir: "/app", + }} + spec, err := computeRunSpec(cfg, runFlags{}, rootfs, nil) + if err != nil { + t.Fatal(err) + } + if spec.Args[0] != tc.want { + t.Errorf("Args[0] = %q, want %q", spec.Args[0], tc.want) + } + }) + } +} + +func TestComputeRunSpecSuccessFullPrecedence(t *testing.T) { + root := t.TempDir() + writeUserFiles(t, root, + "root:x:0:0:root:/root:/bin/sh\nbin:x:1:1:bin:/bin:/sbin/nologin\n", + "root:x:0:\nstaff:x:20:\n", + ) + cfg := &v1.ConfigFile{Config: v1.Config{ + Entrypoint: []string{"/entry"}, + Cmd: []string{"image-cmd"}, + Env: []string{"A=1", "B=2"}, + WorkingDir: "/image-workdir", + User: "root", + }} + rf := runFlags{ + env: []string{"B=9", "C=3"}, + workdir: "/flag-workdir", + user: "bin:staff", + } + spec, err := computeRunSpec(cfg, rf, root, []string{"tail-cmd", "arg"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Args, []string{"/entry", "tail-cmd", "arg"}) { + t.Fatalf("Args = %v, want entrypoint plus CLI tail", spec.Args) + } + if !reflect.DeepEqual(spec.Env, []string{"A=1", "B=9", "C=3", "PATH=" + defaultGuestPath}) { + t.Fatalf("Env = %v, want ordered override plus default PATH", spec.Env) + } + if spec.Workdir != "/flag-workdir" { + t.Fatalf("Workdir = %q, want flag workdir", spec.Workdir) + } + if spec.UID != 1 || spec.GID != 20 { + t.Fatalf("UID:GID = %d:%d, want 1:20", spec.UID, spec.GID) + } +} + +// TestComputeRunSpecDefaultPath pins the PATH guarantee: the guest always +// receives a PATH, the image's own PATH is never rewritten, and an --env +// override wins over both. +func TestComputeRunSpecDefaultPath(t *testing.T) { + cases := []struct { + name string + imgEnv []string + env []string + clearEnv bool + want []string + }{ + {"no PATH anywhere gets the default", []string{"A=1"}, nil, false, + []string{"A=1", "PATH=" + defaultGuestPath}}, + {"image PATH is preserved", []string{"PATH=/opt/bin"}, nil, false, + []string{"PATH=/opt/bin"}}, + {"--env PATH wins", []string{"PATH=/opt/bin"}, []string{"PATH=/bin"}, false, + []string{"PATH=/bin"}}, + {"--clear-env still yields a PATH", []string{"PATH=/opt/bin"}, nil, true, + []string{"PATH=" + defaultGuestPath}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{ + Cmd: []string{"/bin/true"}, + Env: c.imgEnv, + }} + rf := runFlags{env: c.env, clearEnv: c.clearEnv} + spec, err := computeRunSpec(cfg, rf, t.TempDir(), nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Env, c.want) { + t.Errorf("Env = %v, want %v", spec.Env, c.want) + } + }) + } +} + +func TestResolveArgsDoesNotMutateInputs(t *testing.T) { + entry := []string{"/entry"} + cmd := []string{"image-cmd"} + tail := []string{"tail"} + got := resolveArgs(entry, cmd, "", tail) + got[0] = "/changed" + if !reflect.DeepEqual(entry, []string{"/entry"}) { + t.Fatalf("entry mutated to %v", entry) + } + if !reflect.DeepEqual(cmd, []string{"image-cmd"}) { + t.Fatalf("cmd mutated to %v", cmd) + } + if !reflect.DeepEqual(tail, []string{"tail"}) { + t.Fatalf("tail mutated to %v", tail) + } +} + +func TestResolveEnvDuplicateOrdering(t *testing.T) { + got := resolveEnv([]string{"A=1", "B=2"}, []string{"A=3", "C=4", "B=5"}, false) + want := []string{"A=3", "B=5", "C=4"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveEnv = %v, want %v", got, want) + } +} + +func TestResolveUserErrorBranches(t *testing.T) { + cases := []struct { + name string + passwd string // when empty, /etc/passwd is not written + group string // when empty, /etc/group is not written + user string + wantErr string + }{ + {"missing passwd", "", "", "bin", "open /etc/passwd"}, + {"bad passwd uid", "bin:x:not-a-uid:1:bin:/bin:/bin/sh\n", "", "bin", "bad uid"}, + {"bad passwd gid", "bin:x:1:not-a-gid:bin:/bin:/bin/sh\n", "", "bin", "bad gid"}, + {"missing group", "bin:x:1:1:bin:/bin:/bin/sh\n", "", "bin:staff", "open /etc/group"}, + {"bad group gid", "bin:x:1:1:bin:/bin:/bin/sh\n", "staff:x:not-a-gid:\n", "bin:staff", "bad gid"}, + {"numeric uid overflow", "", "", strings.Repeat("9", 20), "invalid uid"}, + {"numeric gid overflow", "", "", "1:" + strings.Repeat("9", 20), "invalid gid"}, + {"empty user part", "bin:x:1:1:bin:/bin:/bin/sh\n", "staff:x:20:\n", ":staff", `resolve user ""`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + if tc.passwd != "" { + writeUserFiles(t, root, tc.passwd, tc.group) + } + _, _, err := resolveUser(root, tc.user) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("resolveUser(%q) err = %v, want %q", tc.user, err, tc.wantErr) + } + }) + } +} diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go index bbbc643b..6c9e8acc 100644 --- a/cmd/elfuse-oci/test_helpers_test.go +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -6,6 +6,7 @@ package main import ( "archive/tar" "bytes" + "fmt" "io" "os" "os/exec" @@ -30,6 +31,20 @@ func TestMain(m *testing.M) { main() return } + if os.Getenv("ELFUSE_EXEC_ELFUSE_TEST") == "1" { + spec := &runSpec{ + Args: []string{"/bin/echo", "hi"}, + Env: []string{"A=1"}, + Workdir: "/", + UID: 1, + GID: 2, + } + if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(98) + } + os.Exit(99) + } os.Exit(m.Run()) } From 790bd9c6093dac35cd66bb4097e8d58b25a4ca72 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:24:37 +0200 Subject: [PATCH 06/26] Add macOS sparsebundle rootfs with COW clones A plain directory rootfs on the default case-insensitive APFS volume folds Linux filenames that differ only by case. Default runs now use a case-sensitive APFS sparsebundle per pinned manifest digest, with a per-run clonefile COW rootfs so guest writes never mutate the warm base tree and repeated runs skip the unpack. Liveness and lifecycle transitions are decided by per-digest advisory flocks in the bundle directory (bundlelock.go), not by pids or directory scans: every live run holds run.lock shared from before the volume is attached until guest exit, so a killed run cannot leak liveness, and attach.lock serializes provisioning (always acquired before run.lock, making the exclusive-to-shared downgrade in provision race-free). Provisioning reuses a mount a live run still holds and only force-detaches one proven stale by an exclusive run.lock probe, so a second run of the same digest can never rip the rootfs out from under a live guest. The spawned elfuse child inherits the run.lock descriptor (cmd.ExtraFiles re-opens it without close-on-exec), so a wrapper killed with an uncatchable signal leaves the flock with the still-running guest: the sweeps keep seeing the bundle busy for exactly as long as elfuse executes out of it, and the next provision cannot mistake the mount for stale. A fake-elfuse spawn test pins the inheritance across the wrapper's fd close. The plain-rootfs path remains available with --plain-rootfs, and the non-Darwin stub keeps elfuse-oci buildable for pull/inspect/unpack tests on Linux. Darwin tests drive runCaseSensitive through seams for provisioning, clonefile, spawn, cleanup, and exit, and cover sparsebundle provisioning, mount/detach, attach-failure teardown, and the lock protocol with a fake hdiutil; the mount-probe and force-detach hooks are function variables so tests need no real disk images. The spawn path intercepts SIGHUP alongside INT/TERM/QUIT and installs the handler before the child starts, so a hangup or an early signal still flows through the forward/reap/teardown path. elfuse is invoked with a "--" separator before the guest command, so an image Entrypoint beginning with "-" cannot steer the host launcher. hdiutil attach failures keep stderr in the error message (stdout stays clean for the plist parse), and the parsed mount path is XML-entity-decoded so a store path containing "&" or quotes still round-trips. The bundle directory's layout is named once beside its lock paths (mount point, sparsebundle image) rather than rebuilt from string literals at each use, and the two plain hdiutil invocations share one wrapper that folds the tool's output into the error. --- cmd/elfuse-oci/bundlelock.go | 157 +++++++++++ cmd/elfuse-oci/bundlelock_test.go | 215 +++++++++++++++ cmd/elfuse-oci/cache_key.go | 10 + cmd/elfuse-oci/commands.go | 41 ++- cmd/elfuse-oci/common_test.go | 7 + cmd/elfuse-oci/csrun.go | 181 ++++++++++++ cmd/elfuse-oci/csrun_darwin_test.go | 364 ++++++++++++++++++++++++ cmd/elfuse-oci/csrun_other.go | 22 ++ cmd/elfuse-oci/main.go | 9 + cmd/elfuse-oci/run.go | 132 +++++++-- cmd/elfuse-oci/run_test.go | 110 ++++++++ cmd/elfuse-oci/runspec.go | 6 + cmd/elfuse-oci/sparsebundle.go | 322 ++++++++++++++++++++++ cmd/elfuse-oci/sparsebundle_test.go | 412 ++++++++++++++++++++++++++++ cmd/elfuse-oci/store.go | 14 +- go.mod | 6 +- 16 files changed, 1967 insertions(+), 41 deletions(-) create mode 100644 cmd/elfuse-oci/bundlelock.go create mode 100644 cmd/elfuse-oci/bundlelock_test.go create mode 100644 cmd/elfuse-oci/csrun.go create mode 100644 cmd/elfuse-oci/csrun_darwin_test.go create mode 100644 cmd/elfuse-oci/csrun_other.go create mode 100644 cmd/elfuse-oci/sparsebundle.go create mode 100644 cmd/elfuse-oci/sparsebundle_test.go diff --git a/cmd/elfuse-oci/bundlelock.go b/cmd/elfuse-oci/bundlelock.go new file mode 100644 index 00000000..c311b265 --- /dev/null +++ b/cmd/elfuse-oci/bundlelock.go @@ -0,0 +1,157 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// Per-digest bundle locks. +// +// A case-sensitive sparsebundle bundle (/cs///) is shared +// mutable state: concurrent `run`s of one digest share its attached volume, +// while prune --cache and rmi --force want to detach and remove it. Liveness +// is decided by advisory flocks, not by pids or directory scans: a held +// lock proves a live holder regardless of pid reuse, and a free lock proves +// the holder is gone regardless of what the directory contains. +// +// Two lock files live in the bundle directory, deliberately OUTSIDE the +// mounted volume: hdiutil detach -force revokes descriptors inside the +// volume, which would silently drop a lock held there, and the locks must be +// probeable while the volume is not attached at all. +// +// - attach.lock: exclusive, serializes bundle lifecycle transitions. A run +// holds it (blocking) across provisioning (stale-mount recovery, +// hdiutil create/attach) and the last-one-out detach; a sweep holds it +// (non-blocking) for its whole reap-detach-remove sequence. +// - run.lock: every live run holds it shared from before the volume is +// attached until the guest exits (the process exiting releases it, so a +// killed run cannot leak liveness). Anyone holding it exclusively has +// proven there are zero live runs: sweeps take it non-blocking (busy => +// skip the bundle), and provision probes it to tell a stale leftover +// mount from one that is live. +// +// Lock ordering: attach.lock is always acquired before run.lock is taken +// exclusively. That makes the EX->SH downgrade in provision safe (flock +// downgrades by release-and-reacquire, but no EX taker can slip in without +// attach.lock, which the downgrader holds) and rules out lock-order cycles +// with the store-level .lock, which prune/rmi already hold around the sweep +// while runs never take bundle locks under the store lock. + +// errCacheBusy reports that a bundle lock is held by a live run (or an +// in-flight provision), so the caller must not detach or remove the bundle. +var errCacheBusy = errors.New("in use by a live run") + +func attachLockPath(bundle string) string { return filepath.Join(bundle, "attach.lock") } +func runLockPath(bundle string) string { return filepath.Join(bundle, "run.lock") } + +// The rest of a bundle directory's layout, named here beside the lock paths so +// the whole shape is defined in one place: hdiutil attaches the volume at the +// mount point, and the sparsebundle is the image file it attaches. Both are +// read by provisioning, the run path, and the prune sweeps. +func mountPointPath(bundle string) string { return filepath.Join(bundle, "mnt") } +func imagePath(bundle string) string { return filepath.Join(bundle, "rootfs.sparsebundle") } + +// flockFile is an open file holding (or having held) an advisory flock. +type flockFile struct { + f *os.File +} + +// acquireFlock opens path (creating it if absent) and takes the flock mode +// `how` (syscall.LOCK_SH or LOCK_EX, optionally |LOCK_NB). A non-blocking +// request that loses returns errCacheBusy (wrapped with the path). +// +// A sweeper removes the whole bundle directory, lock files included, +// while holding both locks. A racing acquirer may then have opened the path +// just before the unlink and be holding a lock on an orphaned inode no later +// process can observe. Guard against that: after locking, verify the path +// still resolves to the locked inode; otherwise retry against the recreated +// file. The retry count is a defense bound, not a correctness knob; one +// retry per concurrent unlink is the worst case. +func acquireFlock(path string, how int) (*flockFile, error) { + for range 16 { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if err := flockRetryIntr(int(f.Fd()), how); err != nil { + f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, fmt.Errorf("%s: %w", path, errCacheBusy) + } + return nil, fmt.Errorf("lock %s: %w", path, err) + } + var pathSt, fdSt syscall.Stat_t + if err := syscall.Stat(path, &pathSt); err != nil { + f.Close() + if errors.Is(err, syscall.ENOENT) { + continue // unlinked under us; retry on the recreated file + } + return nil, fmt.Errorf("lock %s: %w", path, err) + } + if err := syscall.Fstat(int(f.Fd()), &fdSt); err != nil { + f.Close() + return nil, fmt.Errorf("lock %s: %w", path, err) + } + if pathSt.Dev == fdSt.Dev && pathSt.Ino == fdSt.Ino { + return &flockFile{f: f}, nil + } + f.Close() // path now names a different file; lock that one instead + } + return nil, fmt.Errorf("lock %s: persistent unlink race", path) +} + +// flockRetryIntr issues flock, retrying on EINTR (a blocking acquisition may +// be interrupted by the signal forwarding the run wrapper installs). +func flockRetryIntr(fd, how int) error { + for { + err := syscall.Flock(fd, how) + if !errors.Is(err, syscall.EINTR) { + return err + } + } +} + +// Downgrade converts a held exclusive lock to shared. flock implements this +// as release-then-reacquire, so it is race-free only while the caller holds +// attach.lock: every exclusive taker of run.lock acquires attach.lock first, +// so none can slip into the gap. +func (l *flockFile) Downgrade() error { + return flockRetryIntr(int(l.f.Fd()), syscall.LOCK_SH) +} + +// Close releases the lock and closes the file. Safe on nil and after a prior +// Close. +func (l *flockFile) Close() error { + if l == nil || l.f == nil { + return nil + } + _ = syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) + err := l.f.Close() + l.f = nil + return err +} + +// acquireAttachLock takes the exclusive attach.lock for bundleDir, recreating +// the directory when a concurrent sweep removed it. A prune --cache --all can +// win both bundle locks and RemoveAll the whole bundle dir (lock files +// included) while a provisioning run is blocked on attach.lock; that run then +// wakes on the orphaned lock inode, retries, and finds the open of the lock +// path failing with ENOENT on the vanished parent. Recreating the dir and +// retrying lets the run re-provision the swept bundle from scratch instead of +// failing; a few tries bound the pathological case of back-to-back sweeps. +func acquireAttachLock(bundleDir string) (*flockFile, error) { + lock, err := acquireFlock(attachLockPath(bundleDir), syscall.LOCK_EX) + for tries := 0; errors.Is(err, syscall.ENOENT) && tries < 4; tries++ { + if mkErr := os.MkdirAll(bundleDir, 0o755); mkErr != nil { + return nil, mkErr + } + lock, err = acquireFlock(attachLockPath(bundleDir), syscall.LOCK_EX) + } + return lock, err +} diff --git a/cmd/elfuse-oci/bundlelock_test.go b/cmd/elfuse-oci/bundlelock_test.go new file mode 100644 index 00000000..4a01b67c --- /dev/null +++ b/cmd/elfuse-oci/bundlelock_test.go @@ -0,0 +1,215 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestAcquireFlockSharedCoexists(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB) + if err != nil { + t.Fatalf("second shared lock: %v, want success", err) + } + defer b.Close() +} + +func TestAcquireFlockExclusiveBlockedIsCacheBusy(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer a.Close() + _, err = acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if !errors.Is(err, errCacheBusy) { + t.Fatalf("exclusive over shared err = %v, want errCacheBusy", err) + } + + // Releasing the shared lock frees the exclusive probe. + if err := a.Close(); err != nil { + t.Fatal(err) + } + b, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("exclusive after release: %v, want success", err) + } + defer b.Close() +} + +func TestFlockDowngradeAdmitsSharedBlocksExclusive(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + defer a.Close() + if _, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("shared over exclusive err = %v, want errCacheBusy", err) + } + if err := a.Downgrade(); err != nil { + t.Fatalf("Downgrade: %v", err) + } + b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB) + if err != nil { + t.Fatalf("shared after downgrade: %v, want success", err) + } + defer b.Close() + if _, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("exclusive after downgrade err = %v, want errCacheBusy", err) + } +} + +// TestAcquireFlockUnlinkRace pins the verify-retry: when a sweeper unlinks +// the lock file while another process still holds a lock on the orphaned +// inode, a fresh acquire must land on the recreated file, not block on or +// share fate with the orphan. +func TestAcquireFlockUnlinkRace(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + orphan, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + defer orphan.Close() + // Simulate the sweeper's RemoveAll of the bundle: the path is gone while + // the orphan's lock is still held on the old inode. + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + fresh, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("acquire after unlink: %v, want success on recreated file", err) + } + defer fresh.Close() +} + +func TestFlockCloseIdempotentAndNilSafe(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatalf("second Close: %v, want nil", err) + } + var nilLock *flockFile + if err := nilLock.Close(); err != nil { + t.Fatalf("nil Close: %v, want nil", err) + } +} + +func TestBundleLockPaths(t *testing.T) { + if got := attachLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/attach.lock" { + t.Fatalf("attachLockPath = %q", got) + } + if got := runLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/run.lock" { + t.Fatalf("runLockPath = %q", got) + } +} + +// TestSpawnElfuseWaitPassesLocksToChild pins the SIGKILL half of the +// case-sensitive path's liveness: the spawned guest must inherit the run +// lock's descriptor (ExtraFiles shares the open file description, exactly +// like exec inheritance), so a wrapper killed with an uncatchable signal +// leaves the flock held by the still-running guest and the sweeps keep +// seeing the bundle busy. The old behavior spawned the child without the +// descriptor: the lock died with the wrapper while the guest kept running, +// letting prune/rmi reclaim the tree under it. +func TestSpawnElfuseWaitPassesLocksToChild(t *testing.T) { + dir := t.TempDir() + lockPath := filepath.Join(dir, "run.lock") + hold, err := acquireFlock(lockPath, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + lockHeld := func() bool { + probe, err := acquireFlock(lockPath, syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + return true + } + t.Fatal(err) + } + probe.Close() + return false + } + + // A fake elfuse that outlives the parent's fd close long enough for the + // probe below. + script := filepath.Join(dir, "fake-elfuse") + if err := os.WriteFile(script, + []byte("#!/bin/sh\nexec /bin/sleep 3\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("ELFUSE_BIN", script) + + // The parent-side fd close below must happen after cmd.Start's reads of + // the ExtraFiles descriptor; the child's own progress cannot order that + // (a file touch is no happens-before edge), so take the edge from the + // afterSpawnStart hook through a channel. + started := make(chan struct{}) + oldHook := afterSpawnStart + afterSpawnStart = func() { close(started) } + t.Cleanup(func() { afterSpawnStart = oldHook }) + + rootfs := t.TempDir() + done := make(chan error, 1) + go func() { + _, err := spawnElfuseWait(rootfs, &runSpec{Args: []string{"/bin/true"}, Workdir: "/"}, hold) + done <- err + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("spawnElfuseWait never started the child") + } + + // Close the wrapper's fd only (no LOCK_UN), modeling its uncatchable + // death. The child's inherited descriptor must keep the flock held. + if err := hold.f.Close(); err != nil { + t.Fatal(err) + } + hold.f = nil + if !lockHeld() { + t.Error("lock not held by the child; a killed wrapper would free the bundle under the guest") + } + if err := <-done; err != nil { + t.Fatal(err) + } + if lockHeld() { + t.Error("lock still held after the child exited; the kernel should have released it") + } +} + +// TestAcquireAttachLockRecreatesSweptBundleDir pins the provision-vs-sweep +// race recovery: when a prune --cache --all removed the whole bundle dir +// after provision's MkdirAll, the attach.lock open fails with ENOENT on the +// vanished parent. acquireAttachLock must recreate the dir and take the lock +// so the run re-provisions instead of failing. +func TestAcquireAttachLockRecreatesSweptBundleDir(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "sha256-deadbeef") + // The dir is deliberately never created: this is the post-sweep state. + lock, err := acquireAttachLock(bundle) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if _, err := os.Stat(attachLockPath(bundle)); err != nil { + t.Fatalf("attach.lock not recreated in swept bundle dir: %v", err) + } +} diff --git a/cmd/elfuse-oci/cache_key.go b/cmd/elfuse-oci/cache_key.go index 6f9e8715..4133a3fa 100644 --- a/cmd/elfuse-oci/cache_key.go +++ b/cmd/elfuse-oci/cache_key.go @@ -56,3 +56,13 @@ func defaultRootfsForDigest(store, digest string) (string, error) { func legacyRootfsForRef(store, ref string) string { return filepath.Join(store, rootfsCacheDirName, legacyCacheNameForRef(ref)) } + +// csBundleDirForDigest is /cs//: it holds the case-sensitive +// sparsebundle image and the attach mount point for one pinned manifest digest. +func csBundleDirForDigest(store, digest string) (string, error) { + key, err := cacheKeyForDigest(digest) + if err != nil { + return "", err + } + return filepath.Join(store, csCacheDirName, key), nil +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index ae4ab616..04c54c56 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -11,7 +11,7 @@ import ( // The four subcommands share common-flag parsing (common.go) and the OCI // image-layout store (store.go). pull/unpack/inspect are pure store ops; run -// additionally resolves the runspec and execs elfuse (run.go). +// additionally resolves the runspec and execs elfuse (run.go, csrun.go). // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -72,7 +72,7 @@ func parseUnpackArgs(args []string) (commonFlags, string, string, error) { var cf commonFlags var rootfs string fs := newCommandFlagSet("unpack", &cf) - fs.StringVar(&rootfs, "rootfs", "", "") + fs.StringVar(&rootfs, "rootfs", "", "unpack into DIR (default: the store's digest-keyed rootfs cache)") if err := fs.Parse(args); err != nil { return cf, "", "", err } @@ -97,7 +97,7 @@ func parseInspectArgs(args []string) (commonFlags, bool, string, error) { var cf commonFlags var asJSON bool fs := newCommandFlagSet("inspect", &cf) - fs.BoolVar(&asJSON, "json", false, "") + fs.BoolVar(&asJSON, "json", false, "print the raw image config JSON") if err := fs.Parse(args); err != nil { return cf, false, "", err } @@ -157,14 +157,27 @@ func cmdRun(args []string) error { if err != nil { return err } + digestStr := digest.String() + + // Choose the rootfs path. Default: a case-sensitive APFS sparsebundle so + // the guest's case-sensitive filenames don't collide on the host's + // case-insensitive volume, with a per-run COW clone for isolation and + // warm-run speed. --plain-rootfs (or an explicit --rootfs) opts out to the + // plain-directory path (syscall.Exec, no mount lifecycle). + useCS := rf.rootfs == "" && !rf.plainRootfs + + if useCS { + return runCaseSensitive(cf, s, ref, digestStr, cfg, rf, tail) + } + + // Plain-directory path. storeManaged := rf.rootfs == "" if storeManaged { - rf.rootfs, err = defaultRootfsForDigest(cf.store, digest.String()) + rf.rootfs, err = defaultRootfsForDigest(cf.store, digestStr) if err != nil { return err } } - // Ensure the rootfs is unpacked before computing the spec, because // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if // absent; a stale rootfs is the user's concern (run `unpack` to refresh). @@ -186,7 +199,6 @@ func cmdRun(args []string) error { return err } } - spec, err := computeRunSpec(cfg, rf, rf.rootfs, tail) if err != nil { return err @@ -198,7 +210,6 @@ func cmdRun(args []string) error { if err := injectRuntimeFiles(rf.rootfs); err != nil { return err } - return execElfuseForRun(rf.rootfs, spec) } @@ -208,12 +219,16 @@ func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error var env repeatedStringFlag fs := newCommandFlagSet("run", &cf) addPlatformFlag(fs, &cf) - fs.StringVar(&rf.entrypoint, "entrypoint", "", "") - fs.Var(&env, "env", "") - fs.BoolVar(&rf.clearEnv, "clear-env", false, "") - fs.StringVar(&rf.user, "user", "", "") - fs.StringVar(&rf.workdir, "workdir", "", "") - fs.StringVar(&rf.rootfs, "rootfs", "", "") + fs.StringVar(&rf.entrypoint, "entrypoint", "", "override the image Entrypoint (drops the image Cmd)") + fs.Var(&env, "env", "set a guest env var KEY=VAL (repeatable; bare KEY inherits from the host)") + fs.BoolVar(&rf.clearEnv, "clear-env", false, "start the guest env empty (only --env applies)") + fs.StringVar(&rf.user, "user", "", "run as UID[:GID] or name[:group] resolved via the image /etc/passwd,group") + fs.StringVar(&rf.workdir, "workdir", "", "guest-absolute initial working directory") + fs.StringVar(&rf.rootfs, "rootfs", "", "use an explicit rootfs directory (plain dir, no sparsebundle)") + fs.BoolVar(&rf.plainRootfs, "plain-rootfs", false, "use a plain directory rootfs instead of the macOS sparsebundle") + fs.StringVar(&rf.sparseSize, "sparse-size", "", "sparsebundle virtual size (default 16g; macOS only)") + fs.BoolVar(&rf.noClone, "no-clone", false, "run against the base tree without a per-run COW clone (macOS only)") + fs.BoolVar(&rf.keepRootfs, "keep", false, "keep the per-run COW clone and mount for inspection (macOS only)") if err := fs.Parse(args); err != nil { return cf, rf, "", nil, err } diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index 95976cdd..c53979c3 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -99,6 +99,10 @@ func TestParseRunArgs(t *testing.T) { "--user", "1000:1000", "--workdir", "/work", "--rootfs", "/tmp/rootfs", + "--plain-rootfs", + "--sparse-size", "32g", + "--no-clone", + "--keep", "alpine:3", "-c", "echo hi", }) @@ -117,6 +121,9 @@ func TestParseRunArgs(t *testing.T) { if rf.entrypoint != "/bin/sh" || rf.user != "1000:1000" || rf.workdir != "/work" || rf.rootfs != "/tmp/rootfs" { t.Errorf("run flags = %+v", rf) } + if !rf.plainRootfs || rf.sparseSize != "32g" || !rf.noClone || !rf.keepRootfs { + t.Errorf("sparse run flags = %+v", rf) + } if !rf.clearEnv { t.Error("clearEnv = false, want true") } diff --git a/cmd/elfuse-oci/csrun.go b/cmd/elfuse-oci/csrun.go new file mode 100644 index 00000000..39341808 --- /dev/null +++ b/cmd/elfuse-oci/csrun.go @@ -0,0 +1,181 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/google/go-containerregistry/pkg/v1" + "golang.org/x/sys/unix" +) + +var ( + ensureCaseSensitiveRootfsForRun = ensureCaseSensitiveRootfs + clonefileForRun = unix.Clonefile + spawnElfuseWaitForRun = spawnElfuseWait + cleanupCloneAndMountForRun = cleanupCloneAndMount + osExitForRun = os.Exit + runNowUnixNano = func() int64 { return time.Now().UnixNano() } +) + +// keptSidecarName marks a whole sparsebundle as holding run --keep retained +// output. It lives beside the bundle's sparsebundle image and flocks, outside +// the mounted volume, so a cold rmi can detect a deliberate keep without +// attaching a detached bundle to look for .elfuse-keep clones inside it. The +// only thing that removes a kept clone is whole-bundle removal (rmi --force, +// prune --cache --all), which deletes this sidecar along with it, so the marker +// never goes stale. +const keptSidecarName = "kept" + +func keptSidecarPath(bundle string) string { + return filepath.Join(bundle, keptSidecarName) +} + +// writeKeptSidecar records that m's bundle holds run --keep retained output. +// m.mountPath is /mnt, so the sidecar lands beside the bundle image. +func writeKeptSidecar(m *csMount) error { + return touchFile(keptSidecarPath(filepath.Dir(m.mountPath))) +} + +// ensureCaseSensitiveRootfs provisions (creating if absent) and attaches a +// case-sensitive APFS sparsebundle for ref, unpacking the image's layers into +// /rootfs when that base tree is absent. It returns the attached mount +// (the caller must Close it to detach) and the rootfs path to use as --sysroot. +// +// The unpacked base tree persists in the sparsebundle image file across +// attach/detach cycles, so warm re-runs skip the (slow) unpack and pay only the +// attach. +func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + bundle, err := csBundleDirForDigest(cf.store, digest) + if err != nil { + return nil, "", err + } + mountPath := mountPointPath(bundle) + m, err := provisionCaseSensitive(bundle, mountPath, size) + if err != nil { + return nil, "", err + } + rootfs := m.rootfsDir() + published, err := storeRootfsPublished(rootfs) + if err != nil { + return nil, "", errors.Join(err, closeMount(m)) + } + if !published { + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(s, ref, rootfs); err != nil { + return nil, "", errors.Join(err, closeMount(m)) + } + } + return m, rootfs, nil +} + +// runCaseSensitive is the default `run` path: attach the digest-keyed +// case-sensitive sparsebundle, make a per-run COW clone of the warm base tree, +// exec elfuse against the clone, then tear the clone down and detach. On the +// happy path it does not return: it os.Exits with elfuse's status. It returns +// an error on setup failure, and on a post-run cleanup failure after a guest +// exit of zero; when the guest exits nonzero, the cleanup error is only +// printed and the guest status still wins (os.Exit), so a guest failure code +// is never masked by teardown. +// +// The clone lives in the same APFS volume as the base tree (clonefile is +// intra-volume only), so it is instant and free until the guest writes (COW). +// It isolates each run's mutations from the warm base, so re-runs stay clean. +func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { + m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, digest, rf.sparseSize) + if err != nil { + return err + } + // NOTE: we cannot defer m.Close() because os.Exit below skips defers, + // which would leak the attached sparsebundle. Close explicitly on every + // exit path. + + // Per-run COW clone. --no-clone runs against the base tree directly (mutations + // then persist into the warm tree; useful for debugging or when clonefile is + // unavailable). + sysroot := baseRootfs + var cloneDir string + if !rf.noClone { + cloneDir = filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) + if err := os.RemoveAll(cloneDir); err != nil { + err = fmt.Errorf("remove stale COW clone %s: %w", cloneDir, err) + return errors.Join(err, closeMount(m)) + } + if err := clonefileForRun(baseRootfs, cloneDir, unix.CLONE_NOFOLLOW); err != nil { + err = fmt.Errorf("COW clone %s -> %s: %w", baseRootfs, cloneDir, err) + return errors.Join(err, closeMount(m)) + } + sysroot = cloneDir + } + + // Any failure past this point must tear down the clone and the mount. + fail := func(err error) error { + return errors.Join(err, cleanupCloneAndMountForRun(cloneDir, rf.keepRootfs, m)) + } + spec, err := computeRunSpec(cfg, rf, sysroot, tail) + if err != nil { + return fail(err) + } + // On the clone path sysroot is the ephemeral COW clone, so the warm base + // tree stays clean; under --no-clone sysroot is the base tree and the + // injected /etc files are overwritten in place (the user opted into + // mutating the base). + if err := prepareRootfsForRun(sysroot, spec); err != nil { + return fail(err) + } + + code, err := spawnElfuseWaitForRun(sysroot, spec, m.runLock) + var cleanupErr error + if rf.keepRootfs { + // --keep leaves the clone and the mount in place for inspection. The + // clone lives in the sparsebundle volume, so the mount must stay + // attached for it to be reachable on the host; a later run reattaches + // (detaching this stale mount first) and the kept clone reappears. + if cloneDir != "" { + fmt.Fprintf(os.Stderr, "kept clone: %s\n", cloneDir) + } + fmt.Fprintf(os.Stderr, "mount stays attached: %s\n", m.mountPath) + } else { + cleanupErr = cleanupCloneAndMountForRun(cloneDir, false, m) + } + if err != nil { + return errors.Join(err, cleanupErr) + } + if cleanupErr != nil { + if code != 0 { + fmt.Fprintf(os.Stderr, "elfuse-oci: cleanup after exit %d: %v\n", code, cleanupErr) + osExitForRun(code) + return nil // unreachable + } + return cleanupErr + } + osExitForRun(code) + return nil // unreachable +} + +func cleanupCloneAndMount(cloneDir string, keep bool, m *csMount) error { + return errors.Join(removeClone(cloneDir, keep), closeMount(m)) +} + +func closeMount(m *csMount) error { + if err := m.Close(); err != nil { + return fmt.Errorf("detach %s: %w", m.mountPath, err) + } + return nil +} + +// removeClone deletes the ephemeral COW clone unless --keep was requested or +// there is none (the --no-clone path). +func removeClone(cloneDir string, keep bool) error { + if cloneDir == "" || keep { + return nil + } + return os.RemoveAll(cloneDir) +} diff --git a/cmd/elfuse-oci/csrun_darwin_test.go b/cmd/elfuse-oci/csrun_darwin_test.go new file mode 100644 index 00000000..7a8b9763 --- /dev/null +++ b/cmd/elfuse-oci/csrun_darwin_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "golang.org/x/sys/unix" +) + +type runExitCode int + +func withCSRunSeams(t *testing.T) { + t.Helper() + oldEnsure := ensureCaseSensitiveRootfsForRun + oldClone := clonefileForRun + oldSpawn := spawnElfuseWaitForRun + oldCleanup := cleanupCloneAndMountForRun + oldExit := osExitForRun + oldNow := runNowUnixNano + t.Cleanup(func() { + ensureCaseSensitiveRootfsForRun = oldEnsure + clonefileForRun = oldClone + spawnElfuseWaitForRun = oldSpawn + cleanupCloneAndMountForRun = oldCleanup + osExitForRun = oldExit + runNowUnixNano = oldNow + }) +} + +func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 123 } + expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-123", os.Getpid())) + var clonedSrc, clonedDst string + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + if cf.store != "store" || ref != "local:a" || digest != "sha256:"+strings.Repeat("7", 64) || size != "64m" { + t.Fatalf("ensure args = store=%q ref=%q digest=%q size=%q", cf.store, ref, digest, size) + } + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + clonedSrc, clonedDst = src, dst + if flags != unix.CLONE_NOFOLLOW { + t.Fatalf("clone flags = %d, want CLONE_NOFOLLOW", flags) + } + return os.MkdirAll(dst, 0o755) + } + var spawnRootfs string + var spawnSpec *runSpec + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec, _ ...*flockFile) (int, error) { + spawnRootfs = rootfs + spawnSpec = spec + return 7, nil + } + var cleanupClone string + var cleanupKeep bool + cleanupCloneAndMountForRun = func(cloneDir string, keep bool, got *csMount) error { + cleanupClone, cleanupKeep = cloneDir, keep + if got != m { + t.Fatalf("cleanup mount = %+v, want fake mount", got) + } + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 7 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 7", r, r) + } + if clonedSrc != base || clonedDst != expectedClone { + t.Fatalf("clone = %q -> %q, want %q -> %q", clonedSrc, clonedDst, base, expectedClone) + } + if spawnRootfs != expectedClone { + t.Fatalf("spawn rootfs = %q, want clone %q", spawnRootfs, expectedClone) + } + if spawnSpec == nil || !reflect.DeepEqual(spawnSpec.Args, []string{"/cmd"}) { + t.Fatalf("spawn spec = %+v, want /cmd", spawnSpec) + } + if cleanupClone != expectedClone || cleanupKeep { + t.Fatalf("cleanup clone=%q keep=%v, want clone and keep=false", cleanupClone, cleanupKeep) + } + }() + + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}} + err := runCaseSensitive( + commonFlags{store: "store"}, + &store{}, + "local:a", + "sha256:"+strings.Repeat("7", 64), + cfg, + runFlags{sparseSize: "64m"}, + nil, + ) + t.Fatalf("runCaseSensitive returned %v, want osExitForRun panic", err) +} + +func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + return &csMount{mountPath: mount}, base, nil + } + clonefileForRun = func(string, string, int) error { + t.Fatal("clonefile called with --no-clone") + return nil + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec, _ ...*flockFile) (int, error) { + if rootfs != base { + t.Fatalf("spawn rootfs = %q, want base rootfs", rootfs) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { + t.Fatal("cleanup called with --keep") + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("8", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{noClone: true, keepRootfs: true}, + nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} + +func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 456 } + expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-456", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(string, *runSpec, ...*flockFile) (int, error) { + t.Fatal("spawn called after spec error") + return 0, nil + } + var cleanupClone string + cleanupCloneAndMountForRun = func(cloneDir string, keep bool, got *csMount) error { + cleanupClone = cloneDir + if keep { + t.Fatal("cleanup keep = true, want false") + } + if got != m { + t.Fatalf("cleanup mount = %+v, want fake mount", got) + } + return nil + } + osExitForRun = func(code int) { t.Fatalf("exit called after spec error with code %d", code) } + + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("9", 64), + &v1.ConfigFile{Config: v1.Config{}}, + runFlags{}, + nil) + if err == nil || !strings.Contains(err.Error(), "no command") { + t.Fatalf("runCaseSensitive spec err = %v, want no command", err) + } + if cleanupClone != expectedClone { + t.Fatalf("cleanup clone = %q, want %q", cleanupClone, expectedClone) + } +} + +func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T) { + t.Run("unpacks missing rootfs", func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + s := openTestStore(t) + digest, err := s.addImage("local:tiny", tinyImage(t)) + if err != nil { + t.Fatal(err) + } + + m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + if err != nil { + t.Fatalf("ensureCaseSensitiveRootfs: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + if rootfs != filepath.Join(actualMount, "rootfs") { + t.Fatalf("rootfs = %q, want actual mount rootfs", rootfs) + } + if b, err := os.ReadFile(filepath.Join(rootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("rootfs hello = %q, err=%v; want world", b, err) + } + }) + + t.Run("keeps existing rootfs", func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + rootfs := filepath.Join(actualMount, "rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfs, "marker"), []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + s := openTestStore(t) + digest, err := s.addImage("local:tiny", tinyImage(t)) + if err != nil { + t.Fatal(err) + } + + m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + if err != nil { + t.Fatalf("ensureCaseSensitiveRootfs existing: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + if gotRootfs != rootfs { + t.Fatalf("rootfs = %q, want %q", gotRootfs, rootfs) + } + if b, err := os.ReadFile(filepath.Join(rootfs, "marker")); err != nil || string(b) != "keep" { + t.Fatalf("marker = %q, err=%v; want keep", b, err) + } + if _, err := os.Stat(filepath.Join(rootfs, "hello")); !os.IsNotExist(err) { + t.Fatalf("existing rootfs was unpacked over: %v", err) + } + }) +} + +func TestEnsureCaseSensitiveRootfsClosesMountOnUnpackError(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + detachLog := filepath.Join(t.TempDir(), "detach.log") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + s := openTestStore(t) + + _, _, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:missing", "sha256:"+strings.Repeat("a", 64), "32m") + if err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack not-pulled error", err) + } + b, readErr := os.ReadFile(detachLog) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(b), actualMount) { + t.Fatalf("detach log = %q, want actual mount %s", b, actualMount) + } +} + +func TestCleanupCloneAndMountAndCloseMount(t *testing.T) { + oldDetach := detachForce + var detached string + detachForce = func(path string) error { + detached = path + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + clone := filepath.Join(t.TempDir(), "clone") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: "/tmp/cleanup-mount", owned: true} + if err := cleanupCloneAndMount(clone, false, m); err != nil { + t.Fatalf("cleanupCloneAndMount: %v", err) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("clone after cleanup: %v, want IsNotExist", err) + } + if detached != "/tmp/cleanup-mount" || m.owned { + t.Fatalf("detached=%q owned=%v, want detached mount and owned=false", detached, m.owned) + } + + detachForce = func(path string) error { return fmt.Errorf("detach boom") } + err := closeMount(&csMount{mountPath: "/tmp/bad-mount", owned: true}) + if err == nil || !strings.Contains(err.Error(), "detach /tmp/bad-mount") || !strings.Contains(err.Error(), "detach boom") { + t.Fatalf("closeMount err = %v, want wrapped detach error", err) + } +} + +func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { + withCSRunSeams(t) + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + runNowUnixNano = func() int64 { return 789 } + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref, digest, size string) (*csMount, string, error) { + if cf.store != s.root || ref != "local:a" || size != "" { + t.Fatalf("ensure from cmdRun got store=%q ref=%q size=%q", cf.store, ref, size) + } + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("ensure digest = %q, want sha256 digest", digest) + } + return &csMount{mountPath: mount}, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec, _ ...*flockFile) (int, error) { + if !strings.Contains(rootfs, fmt.Sprintf("run-%d-789", os.Getpid())) { + t.Fatalf("spawn rootfs = %q, want generated clone", rootfs) + } + if !reflect.DeepEqual(spec.Args, []string{"/image-cmd"}) { + t.Fatalf("spec args = %v, want image cmd", spec.Args) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { return nil } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("cmdRun default sparse path panic = %T %v, want exit 0", r, r) + } + }() + err := cmdRun([]string{"--store", s.root, "local:a"}) + t.Fatalf("cmdRun returned %v, want exit panic", err) +} diff --git a/cmd/elfuse-oci/csrun_other.go b/cmd/elfuse-oci/csrun_other.go new file mode 100644 index 00000000..b16d9b68 --- /dev/null +++ b/cmd/elfuse-oci/csrun_other.go @@ -0,0 +1,22 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !darwin + +package main + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// runCaseSensitive is the macOS case-sensitive sparsebundle + COW clone path. +// On non-Darwin there is no APFS/hdiutil/clonefile, and `run` is unusable +// anyway without Hypervisor.framework, so the default (case-sensitive) path +// reports a clear error and directs the user at --plain-rootfs. This stub +// exists so elfuse-oci compiles on Linux, where pull/inspect/unpack and +// conformance/interop tests run. +func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { + return fmt.Errorf("case-sensitive sparsebundle rootfs requires macOS; pass --plain-rootfs for a plain directory") +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index d6de2ab6..1e655607 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -65,6 +65,15 @@ run flags: are resolved against the image /etc/passwd and /etc/group before exec. --workdir DIR Guest-absolute initial working directory + --rootfs DIR Use an explicit rootfs directory (implies a plain dir, + no sparsebundle/clone lifecycle) + --plain-rootfs Use a plain directory rootfs instead of the default + macOS case-sensitive sparsebundle + --sparse-size SIZE Sparsebundle virtual size (default 16g; macOS only) + --no-clone Run against the base tree, skipping the per-run COW + clone (mutations persist; macOS only) + --keep Keep the per-run COW clone and mount for inspection + (macOS only) `) } diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go index 653f5f00..6290b811 100644 --- a/cmd/elfuse-oci/run.go +++ b/cmd/elfuse-oci/run.go @@ -6,12 +6,23 @@ package main import ( "fmt" "os" + "os/exec" + "os/signal" "path/filepath" + "runtime" "syscall" ) var execElfuseForRun = execElfuse +// afterSpawnStart fires once cmd.Start has returned, at which point the +// child owns dups of every ExtraFiles descriptor. Production no-op; the +// lock-inheritance test injects a channel close here so its parent-side fd +// close is ordered after Start's descriptor reads: the child's own progress +// cannot provide that edge (the race detector cannot see through the +// filesystem). +var afterSpawnStart = func() {} + // elfuseBin locates the elfuse binary to exec for `run`. Precedence: // - $ELFUSE_BIN (an override hook for tests and wrapper scripts); // - the sibling of this executable (build/elfuse-oci -> build/elfuse). @@ -26,25 +37,17 @@ func elfuseBin() (string, error) { return filepath.Join(filepath.Dir(exe), "elfuse"), nil } -// execElfuse replaces this process with `elfuse --sysroot --user U:G -// --workdir D --clear-env --env K=V ... `. +// elfuseArgv builds the argv for `elfuse --sysroot --user U:G +// --workdir D --clear-env --env K=V ... -- `. // -// --clear-env plus every final env var as an explicit --env makes the guest -// see exactly the runspec env (image Env merged with --env overrides, per the +// --clear-env plus every final env var as an explicit --env makes the guest see +// exactly the runspec env (image Env merged with --env overrides, per the // precedence matrix) rather than the host environ. // -// syscall.Exec replaces elfuse-oci in place: the invoking shell reaps -// the same pid, and signals such as Ctrl-C go straight to elfuse rather than -// through a Go middleman. -func execElfuse(rootfs string, spec *runSpec) error { - bin, err := elfuseBin() - if err != nil { - return err - } - if _, err := os.Stat(bin); err != nil { - return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) - } - +// "--" ends elfuse's own option parsing: spec.Args comes from untrusted image +// config, so an Entrypoint beginning with "-" must reach the guest as its +// argv, not steer the host launcher (e.g. an image config carrying "--gdb"). +func elfuseArgv(rootfs string, spec *runSpec) []string { argv := []string{ "elfuse", "--sysroot", rootfs, @@ -55,10 +58,105 @@ func execElfuse(rootfs string, spec *runSpec) error { for _, e := range spec.Env { argv = append(argv, "--env", e) } + argv = append(argv, "--") argv = append(argv, spec.Args...) + return argv +} - if err := syscall.Exec(bin, argv, os.Environ()); err != nil { +// execElfuse replaces this process with elfuse (syscall.Exec). Used for the +// plain-rootfs path, which owns no mount to tear down: elfuse-oci +// becomes elfuse in place, so the invoking shell reaps the same pid and +// terminal signals such as Ctrl-C go straight to elfuse rather than through a +// Go middleman. +func execElfuse(rootfs string, spec *runSpec) error { + bin, err := elfuseBin() + if err != nil { + return err + } + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + if err := syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()); err != nil { return fmt.Errorf("exec %s: %w", bin, err) } return nil // unreachable } + +// spawnElfuseWait runs elfuse as a child and waits for it, returning the exit +// status the way a shell would (exit code, or 128+signal for signal death). +// Unlike execElfuse, elfuse-oci stays alive to reap the child, letting the +// case-sensitive path tear down its mount and COW clone after elfuse exits. +// +// The child shares this process's process group, so terminal signals (Ctrl-C) +// reach it directly; we additionally forward any such signal we receive to the +// child so a signal targeted at elfuse-oci alone still propagates, and we +// survive to reap and report the child's status rather than dying first. +func spawnElfuseWait(rootfs string, spec *runSpec, locks ...*flockFile) (int, error) { + bin, err := elfuseBin() + if err != nil { + return 0, err + } + if _, err := os.Stat(bin); err != nil { + return 0, fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + // exec.Command uses `bin` as argv[0], so drop the leading "elfuse" + // program-name that elfuseArgv includes for syscall.Exec's sake; otherwise + // elfuse would see "elfuse" as its first positional and try to boot a + // guest path named "elfuse". + cmd := exec.Command(bin, elfuseArgv(rootfs, spec)[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // The child inherits each lock's descriptor (ExtraFiles re-opens them + // without close-on-exec), so the flock lives exactly as long as the + // guest: a wrapper killed with an uncatchable signal (the signal.Notify + // set below cannot include SIGKILL) must not free the bundle while + // elfuse still executes out of it, or the sweeps reclaim a live tree. + for _, l := range locks { + if l != nil { + cmd.ExtraFiles = append(cmd.ExtraFiles, l.f) + } + } + + // Intercept before Start so no window exists where a signal takes the + // default action and kills this wrapper between launching the child and + // entering the forward/reap loop; the channel buffers until then. SIGHUP + // is included so a terminal hangup also flows through the forward/reap + // path and the caller's mount/clone teardown still runs. + sigCh := make(chan os.Signal, 4) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, + syscall.SIGHUP) + defer signal.Stop(sigCh) + + if err := cmd.Start(); err != nil { + return 0, fmt.Errorf("spawn %s: %w", bin, err) + } + // Keep the lock files live past Start so a finalizer cannot close an fd + // (dropping the flock) before the child has duplicated it. + runtime.KeepAlive(locks) + afterSpawnStart() + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + for { + select { + case err := <-done: + state := cmd.ProcessState + if state == nil { + return 0, err + } + if ws, ok := state.Sys().(syscall.WaitStatus); ok { + if ws.Signaled() { + return 128 + int(ws.Signal()), nil + } + return ws.ExitStatus(), nil + } + return state.ExitCode(), nil + case sig := <-sigCh: + if cmd.Process != nil { + _ = cmd.Process.Signal(sig) + } + } + } +} diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go index f254182b..b8b9a2a7 100644 --- a/cmd/elfuse-oci/run_test.go +++ b/cmd/elfuse-oci/run_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strings" "testing" ) @@ -25,6 +26,72 @@ func writeElfuseStub(t *testing.T, body string) string { return p } +// TestSpawnElfuseWaitExitCode verifies the child's exit code is returned as-is. +func TestSpawnElfuseWaitExitCode(t *testing.T) { + writeElfuseStub(t, "exit 42") + spec := &runSpec{Args: []string{"/hello"}, Workdir: "/", UID: 0, GID: 0} + code, err := spawnElfuseWait(t.TempDir(), spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 42 { + t.Errorf("exit code: got %d, want 42", code) + } +} + +// TestSpawnElfuseWaitSignalDeath verifies signal death is reported the +// shell-style way: 128 + signal. The child kills itself with SIGTERM (15), so +// cmd.Wait() observes WaitStatus.Signaled() independently of elfuse-oci's own +// signal forwarding. +func TestSpawnElfuseWaitSignalDeath(t *testing.T) { + writeElfuseStub(t, "kill -TERM $$") + spec := &runSpec{Args: []string{"/hello"}, Workdir: "/", UID: 0, GID: 0} + code, err := spawnElfuseWait(t.TempDir(), spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 143 { // 128 + SIGTERM(15) + t.Errorf("signal death: got %d, want 143 (128+15)", code) + } +} + +// TestElfuseArgvShape verifies the argv handed to elfuse is exactly +// elfuseArgv(rootfs, spec) minus the leading "elfuse" program-name (exec.Command +// prepends the binary path as argv[0], so spawnElfuseWait drops elfuseArgv[0]). +// The stub records its own argv ($@, which excludes $0) one per line. +func TestElfuseArgvShape(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "argv.txt") + t.Setenv("ELFUSE_ARGV_OUT", outPath) + writeElfuseStub(t, `printf '%s\n' "$@" > "$ELFUSE_ARGV_OUT"`) + + rootfs := t.TempDir() + spec := &runSpec{ + Args: []string{"/bin/echo", "hi"}, + Env: []string{"A=1", "B=2"}, + Workdir: "/work", + UID: 1000, + GID: 1000, + } + wantArgv := elfuseArgv(rootfs, spec)[1:] + + code, err := spawnElfuseWait(rootfs, spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 0 { + t.Fatalf("stub exited %d, want 0", code) + } + + data, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("read argv out: %v", err) + } + gotLines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if !reflect.DeepEqual(gotLines, wantArgv) { + t.Errorf("argv:\n got %v\nwant %v", gotLines, wantArgv) + } +} + func TestElfuseBinEnvAndFallback(t *testing.T) { want := filepath.Join(t.TempDir(), "elfuse-custom") t.Setenv("ELFUSE_BIN", want) @@ -95,3 +162,46 @@ func TestExecElfuseSuccessSubprocess(t *testing.T) { } } } + +func TestSpawnElfuseWaitMissingAndStartErrors(t *testing.T) { + t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) + if _, err := spawnElfuseWait(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}); err == nil || + !strings.Contains(err.Error(), "elfuse binary not found") { + t.Fatalf("spawn missing binary err = %v, want not found", err) + } + + nonExecutable := filepath.Join(t.TempDir(), "elfuse-not-executable") + if err := os.WriteFile(nonExecutable, []byte("#!/bin/sh\nexit 0\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("ELFUSE_BIN", nonExecutable) + if _, err := spawnElfuseWait(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}); err == nil || + !strings.Contains(err.Error(), "spawn") { + t.Fatalf("spawn start err = %v, want spawn error", err) + } +} + +// TestElfuseArgvSeparatesGuestArgs pins the "--" end-of-options marker: the +// guest command comes from untrusted image config, so an Entrypoint that +// begins with "-" must arrive as guest argv, not be parsed as an elfuse +// option by the host launcher. +func TestElfuseArgvSeparatesGuestArgs(t *testing.T) { + spec := &runSpec{ + Args: []string{"--gdb", "1234"}, + Workdir: "/", + } + argv := elfuseArgv("/rootfs", spec) + sep := -1 + for i, a := range argv { + if a == "--" { + sep = i + break + } + } + if sep < 0 { + t.Fatalf("argv %v carries no \"--\" separator before guest args", argv) + } + if !reflect.DeepEqual(argv[sep+1:], spec.Args) { + t.Fatalf("argv after -- = %v, want %v", argv[sep+1:], spec.Args) + } +} diff --git a/cmd/elfuse-oci/runspec.go b/cmd/elfuse-oci/runspec.go index 9831de77..e5ae37a2 100644 --- a/cmd/elfuse-oci/runspec.go +++ b/cmd/elfuse-oci/runspec.go @@ -48,6 +48,12 @@ type runFlags struct { user string workdir string rootfs string + + // Case-sensitive sparsebundle and COW clone controls. + plainRootfs bool // --plain-rootfs: skip the sparsebundle, use a plain dir + sparseSize string // --sparse-size SIZE: sparsebundle virtual size (default 16g) + noClone bool // --no-clone: run against the base tree, no COW clone + keepRootfs bool // --keep: do not remove the COW clone on exit } // computeRunSpec applies the Entrypoint/Cmd/Env/WorkingDir/User precedence. diff --git a/cmd/elfuse-oci/sparsebundle.go b/cmd/elfuse-oci/sparsebundle.go new file mode 100644 index 00000000..3d3acad1 --- /dev/null +++ b/cmd/elfuse-oci/sparsebundle.go @@ -0,0 +1,322 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "bytes" + "errors" + "fmt" + "html" + "os" + "os/exec" + "path/filepath" + "regexp" + "syscall" +) + +var isMountPointFn = isMountPoint + +// csMount is a case-sensitive APFS sparsebundle attached at a mount point. +// It mirrors the C sysroot_create_mount machinery in src/core/sysroot.c so +// the guest rootfs is case-sensitive (the host volume is not), fixing +// the case-collision limitation of a plain-directory rootfs. +type csMount struct { + mountPath string // where the volume is attached + owned bool // we attached (or share) it; tear down on Close + bundleDir string // bundle directory holding the lock files; "" = no locking (unit tests) + runLock *flockFile // shared liveness lock held for this run's lifetime +} + +// defaultSparseSize is the sparsebundle's virtual size. APFS sparsebundles are +// sparse, so this is a ceiling, not preallocation; 16g matches the C side and +// comfortably covers base images (the actual disk use is the unpacked size). +const defaultSparseSize = "16g" + +// provisionCaseSensitive creates (if absent) and attaches a case-sensitive +// APFS sparsebundle. The unpacked base tree lives at /rootfs and +// persists in the sparsebundle image file across attach/detach cycles, so warm +// re-runs skip the unpack. The caller must Close the returned mount (which +// detaches it when this is the last live run) when done. +// +// Locking (see bundlelock.go): the whole provision runs under an exclusive +// attach.lock, and the returned mount holds run.lock shared until Close, so +// this run is visible to prune/rmi sweeps from BEFORE the volume is +// attached: there is no window in which the mount exists but no liveness +// marker does. An attached leftover mount is detached only after winning the +// run.lock exclusive probe, which proves no live run is executing out of it; +// when the probe reports busy the mount belongs to live runs of the same +// digest and is shared instead of ripped out from under them. +func provisionCaseSensitive(bundleDir, mountPath, size string) (*csMount, error) { + if size == "" { + size = defaultSparseSize + } + if err := os.MkdirAll(bundleDir, 0o755); err != nil { + return nil, err + } + image := imagePath(bundleDir) + + attachLock, err := acquireAttachLock(bundleDir) + if err != nil { + return nil, err + } + defer attachLock.Close() + + // Probe run.lock. Winning it exclusively proves zero live runs: any + // attached mount is stale (crash, kill, --keep) and safe to detach; hold + // the lock and downgrade to shared once provisioned (safe under + // attach.lock, see Downgrade). Losing the probe proves live runs exist: + // take it shared (which cannot block, since exclusive takers must hold + // attach.lock) and never detach. + staleDetachOK := false + runLock, err := acquireFlock(runLockPath(bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + switch { + case err == nil: + staleDetachOK = true + case errors.Is(err, errCacheBusy): + runLock, err = acquireFlock(runLockPath(bundleDir), syscall.LOCK_SH) + if err != nil { + return nil, err + } + default: + return nil, err + } + fail := func(err error) (*csMount, error) { + runLock.Close() + return nil, err + } + + if _, err := os.Stat(image); os.IsNotExist(err) { + if err := runHdiutil("create", image, + "-fs", "Case-sensitive APFS", + "-size", size, + "-type", "SPARSEBUNDLE", + "-volname", "elfuse_sysroot"); err != nil { + return fail(err) + } + } else if err != nil { + return fail(err) + } + + // Reject a symlinked mount path before any mount-status probe or detach: + // isMountPoint/detachForce follow the link (os.Stat) and could force-detach + // an unrelated volume. clearDir has the same guard, but only runs after the + // detach below. + if li, err := os.Lstat(mountPath); err == nil && li.Mode()&os.ModeSymlink != 0 { + return fail(fmt.Errorf("mount path %s is a symlink; refusing to detach/clear", mountPath)) + } + + if isMountPointFn(mountPath) { + if !staleDetachOK { + // Live runs of this digest own the attach; share it. + return &csMount{mountPath: mountPath, owned: true, bundleDir: bundleDir, runLock: runLock}, nil + } + // A prior run left the volume attached (crash, kill, --keep) and the + // won run.lock probe proves nothing is executing out of it: detach so + // we own a clean attach. + if err := detachForce(mountPath); err != nil { + return fail(fmt.Errorf("detach stale %s: %w", mountPath, err)) + } + } + // Ensure the mount point is an empty directory so hdiutil will mount onto + // it. + if err := clearDir(mountPath); err != nil { + return fail(err) + } + + // Keep stdout (the plist) separate from stderr: the failure message must + // carry hdiutil's diagnostic, which Output() alone would discard, while + // CombinedOutput() would corrupt the plist parse. + attach := exec.Command("hdiutil", "attach", + "-mountpoint", mountPath, "-plist", image) + var attachStderr bytes.Buffer + attach.Stderr = &attachStderr + out, err := attach.Output() + if err != nil { + return fail(fmt.Errorf("hdiutil attach %s: %w: %s%s", image, err, out, + attachStderr.Bytes())) + } + actual, err := parseMountpoint(string(out)) + if err != nil { + err = fmt.Errorf("parse attach plist for %s: %w", image, err) + return fail(detachAfterAttachError(mountPath, err)) + } + + if err := writeSpotlightMarker(actual); err != nil { + err = fmt.Errorf("spotlight marker: %w", err) + return fail(detachAfterAttachError(actual, err)) + } + if staleDetachOK { + if err := runLock.Downgrade(); err != nil { + return fail(detachAfterAttachError(actual, err)) + } + } + return &csMount{mountPath: actual, owned: true, bundleDir: bundleDir, runLock: runLock}, nil +} + +// rootfsDir is the base unpacked tree inside the volume. +func (m *csMount) rootfsDir() string { return filepath.Join(m.mountPath, "rootfs") } + +// Close releases this run's liveness lock and detaches the volume when this +// was the last live run of the digest (last-one-out): with concurrent runs +// sharing one attach, an unconditional detach here would rip the rootfs out +// from under the survivors. A csMount without a bundleDir (unit tests, +// hand-built mounts) has no locks to consult and detaches unconditionally. +func (m *csMount) Close() error { + if !m.owned { + return nil + } + if m.bundleDir == "" { + if err := detachForce(m.mountPath); err != nil { + return err + } + m.owned = false + return nil + } + // Take attach.lock BEFORE releasing our shared run.lock: it fences out a + // concurrent sweep, which could otherwise win both locks between our + // release and our probe and remove the bundle we are about to detach. If + // the lifecycle lock is busy (another provision or a sweep), its holder + // owns the mount's fate; just drop our liveness and go. + attachLock, err := acquireFlock(attachLockPath(m.bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + m.runLock.Close() + m.runLock = nil + m.owned = false + if errors.Is(err, errCacheBusy) { + return nil + } + return err + } + defer attachLock.Close() + if err := m.runLock.Close(); err != nil { + m.owned = false + return err + } + m.runLock = nil + m.owned = false + runLock, err := acquireFlock(runLockPath(m.bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + // Other live runs share the attach; leave the volume to them. + return nil + } + return err + } + defer runLock.Close() + return detachForce(m.mountPath) +} + +func detachAfterAttachError(mountPath string, cause error) error { + if err := detachForce(mountPath); err != nil { + return errors.Join(cause, fmt.Errorf("detach %s: %w", mountPath, err)) + } + return cause +} + +// runHdiutil runs `hdiutil ` and folds the output into +// the error, since hdiutil reports the actual cause (a busy volume, a bad +// size) only in its output. path is both the operand and the subject of the +// error, so the two cannot disagree. The attach path does not use this: it +// parses a plist from stdout and so must keep stderr separate. +func runHdiutil(verb, path string, flags ...string) error { + argv := append([]string{verb}, flags...) + argv = append(argv, path) + out, err := exec.Command("hdiutil", argv...).CombinedOutput() + if err != nil { + return fmt.Errorf("hdiutil %s %s: %w: %s", verb, path, err, out) + } + return nil +} + +var detachForce = func(mountPath string) error { + return runHdiutil("detach", mountPath, "-force") +} + +// writeSpotlightMarker drops .metadata_never_index so Spotlight does not index +// the (potentially large) rootfs volume. +func writeSpotlightMarker(mountPath string) error { + return touchFile(filepath.Join(mountPath, ".metadata_never_index")) +} + +// touchFile creates (or truncates) an empty marker file. The markers carry +// no content; only their existence matters. +func touchFile(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + return f.Close() +} + +// isMountPoint reports whether path is currently a mount point by comparing its +// device id against its parent's. +func isMountPoint(path string) bool { + if fi, err := os.Stat(path); err != nil || !fi.IsDir() { + return false + } + dev, ok := devOf(path) + if !ok { + return false + } + parent, ok := devOf(filepath.Dir(path)) + if !ok { + return false + } + return dev != parent +} + +func devOf(path string) (int64, bool) { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return 0, false + } + return int64(st.Dev), true +} + +// clearDir removes all children of dir (creating it if absent) without removing +// dir itself, so hdiutil can mount onto it. A symlink at dir is rejected: +// ReadDir/RemoveAll would follow it and empty the link's target instead of the +// mount point, so a corrupt or tampered store must fail here rather than +// delete files elsewhere. +func clearDir(dir string) error { + if li, err := os.Lstat(dir); err == nil { + if li.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("mount point %s is a symlink; refusing to clear it", dir) + } + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + +var mountpointRe = regexp.MustCompile(`mount-point\s*([^<]+)`) + +// parseMountpoint extracts the mount-point from an hdiutil attach -plist output +// (mirroring the C parse_attach_mountpoint string scan). The scan reads raw +// XML text content, so entity references must be decoded: a store path +// containing "&" or "'" is otherwise returned in its escaped form and every +// later use (markers, rootfs, detach) targets a nonexistent path. +// html.UnescapeString covers the XML predefined entities plus numeric +// references. +func parseMountpoint(plist string) (string, error) { + m := mountpointRe.FindStringSubmatch(plist) + if m == nil { + return "", fmt.Errorf("mount-point key not found in plist") + } + return html.UnescapeString(m[1]), nil +} diff --git a/cmd/elfuse-oci/sparsebundle_test.go b/cmd/elfuse-oci/sparsebundle_test.go new file mode 100644 index 00000000..01b027c8 --- /dev/null +++ b/cmd/elfuse-oci/sparsebundle_test.go @@ -0,0 +1,412 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// withDarwinCacheSeams swaps the darwin mount-probe and force-detach seams +// for a test and restores them on cleanup. +func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { + t.Helper() + oldIsMount := isMountPointFn + oldDetach := detachForce + if isMount != nil { + isMountPointFn = isMount + } + if detach != nil { + detachForce = detach + } + t.Cleanup(func() { + isMountPointFn = oldIsMount + detachForce = oldDetach + }) +} + +// hdiutil attach -plist output is an array of system entity dictionaries. We +// only care about the mount-point. This fixture is a trimmed-down capture of the +// real shape (system-entities -> dict -> mount-point key/string). +const attachPlistFixture = ` + + + + + system-entities + + + content-hint + Apple_APFS + dev-entry + /dev/disk3s1 + mount-point + /Volumes/elfuse_sysroot + potentially-mountable + 1 + + + + +` + +func TestParseMountpoint(t *testing.T) { + got, err := parseMountpoint(attachPlistFixture) + if err != nil { + t.Fatal(err) + } + if got != "/Volumes/elfuse_sysroot" { + t.Errorf("got %q, want /Volumes/elfuse_sysroot", got) + } +} + +func TestParseMountpointMissing(t *testing.T) { + if _, err := parseMountpoint(""); err == nil { + t.Fatal("expected error when mount-point absent") + } +} + +func TestParseMountpointWhitespaceBetweenKeyAndString(t *testing.T) { + in := `mount-point + /Volumes/x` + got, err := parseMountpoint(in) + if err != nil { + t.Fatal(err) + } + if got != "/Volumes/x" { + t.Errorf("got %q, want /Volumes/x", got) + } +} + +func TestIsMountPointPlainDir(t *testing.T) { + dir := t.TempDir() + if isMountPoint(dir) { + t.Errorf("fresh temp dir reported as mount point") + } +} + +func TestRemoveCloneRemovesDir(t *testing.T) { + clone := filepath.Join(t.TempDir(), "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + if err := removeClone(clone, false); err != nil { + t.Fatalf("removeClone: %v", err) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("clone after removeClone: %v, want IsNotExist", err) + } +} + +func TestRemoveCloneKeepLeavesDir(t *testing.T) { + clone := filepath.Join(t.TempDir(), "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + if err := removeClone(clone, true); err != nil { + t.Fatalf("removeClone keep: %v", err) + } + if _, err := os.Stat(clone); err != nil { + t.Fatalf("clone after keep: %v, want present", err) + } +} + +func TestCSMountCloseReportsDetachError(t *testing.T) { + oldDetach := detachForce + t.Cleanup(func() { detachForce = oldDetach }) + detachForce = func(path string) error { + return fmt.Errorf("detach failed for %s", path) + } + + m := &csMount{mountPath: "/tmp/elfuse-test-mount", owned: true} + err := m.Close() + if err == nil || !strings.Contains(err.Error(), "detach failed") { + t.Fatalf("Close err = %v, want detach failure", err) + } + if !m.owned { + t.Fatal("Close cleared ownership after failed detach") + } +} + +func TestCSMount(t *testing.T) { + t.Run("rootfsDir", func(t *testing.T) { + m := &csMount{mountPath: "/tmp/elfuse-mounted", owned: true} + if got := m.rootfsDir(); got != "/tmp/elfuse-mounted/rootfs" { + t.Fatalf("rootfsDir = %q, want /tmp/elfuse-mounted/rootfs", got) + } + }) + + t.Run("close detaches once", func(t *testing.T) { + oldDetach := detachForce + var detached []string + detachForce = func(path string) error { + detached = append(detached, path) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + m := &csMount{mountPath: "/tmp/elfuse-mounted", owned: true} + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if m.owned { + t.Fatal("Close left mount owned after successful detach") + } + if len(detached) != 1 || detached[0] != "/tmp/elfuse-mounted" { + t.Fatalf("detached = %v, want [/tmp/elfuse-mounted]", detached) + } + if err := m.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + if len(detached) != 1 { + t.Fatalf("second Close detached again: %v", detached) + } + }) + + t.Run("detachAfterAttachError joins both errors", func(t *testing.T) { + oldDetach := detachForce + detachForce = func(path string) error { return errors.New("detach failed") } + t.Cleanup(func() { detachForce = oldDetach }) + + err := detachAfterAttachError("/tmp/mnt", errors.New("attach parse failed")) + if err == nil || !strings.Contains(err.Error(), "attach parse failed") || !strings.Contains(err.Error(), "detach failed") { + t.Fatalf("detachAfterAttachError = %v, want joined cause and detach error", err) + } + }) +} + +func TestSparsebundleFilesystemHelpers(t *testing.T) { + dir := t.TempDir() + if err := writeSpotlightMarker(dir); err != nil { + t.Fatalf("writeSpotlightMarker: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, ".metadata_never_index")); err != nil { + t.Fatalf("spotlight marker missing: %v", err) + } + + childFile := filepath.Join(dir, "file") + childDir := filepath.Join(dir, "subdir") + if err := os.WriteFile(childFile, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(childDir, 0o755); err != nil { + t.Fatal(err) + } + if err := clearDir(dir); err != nil { + t.Fatalf("clearDir existing: %v", err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("clearDir left entries %v, want empty dir", entries) + } + + missing := filepath.Join(t.TempDir(), "created") + if err := clearDir(missing); err != nil { + t.Fatalf("clearDir missing: %v", err) + } + if fi, err := os.Stat(missing); err != nil || !fi.IsDir() { + t.Fatalf("clearDir missing produced fi=%v err=%v, want dir", fi, err) + } + + // A symlinked mount dir must be rejected, not followed: clearing through + // it would empty the link's target directory outside the OCI cache. + target := t.TempDir() + if err := os.WriteFile(filepath.Join(target, "precious"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "mnt") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := clearDir(link); err == nil { + t.Fatal("clearDir followed a symlinked mount dir, want error") + } + if _, err := os.Stat(filepath.Join(target, "precious")); err != nil { + t.Fatalf("symlink target contents were removed: %v", err) + } + + if _, ok := devOf(dir); !ok { + t.Fatal("devOf temp dir returned ok=false") + } + if _, ok := devOf(filepath.Join(dir, "does-not-exist")); ok { + t.Fatal("devOf missing path returned ok=true") + } +} + +func installFakeHdiutil(t *testing.T) { + t.Helper() + dir := t.TempDir() + script := filepath.Join(dir, "hdiutil") + body := `#!/bin/sh +case "$1" in +create) + if [ "${HDIUTIL_FAIL_CREATE:-}" = "1" ]; then + echo "create failed" + exit 7 + fi + for last do :; done + mkdir -p "$last" + exit 0 + ;; +attach) + if [ "${HDIUTIL_FAIL_ATTACH:-}" = "1" ]; then + echo "attach diagnostic on stderr" >&2 + exit 5 + fi + if [ "${HDIUTIL_BAD_PLIST:-}" = "1" ]; then + printf '' + exit 0 + fi + printf 'mount-point%s' "$HDIUTIL_MOUNT" + exit 0 + ;; +detach) + if [ -n "${HDIUTIL_DETACH_LOG:-}" ]; then + echo "$3" >> "$HDIUTIL_DETACH_LOG" + fi + exit 0 + ;; +*) + echo "unexpected hdiutil command $1" + exit 9 + ;; +esac +` + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestProvisionCaseSensitiveWithFakeHdiutilSuccess(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + bundle := filepath.Join(t.TempDir(), "bundle") + requestedMount := filepath.Join(t.TempDir(), "requested-mount") + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + + m, err := provisionCaseSensitive(bundle, requestedMount, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive: %v", err) + } + if _, err := os.Stat(filepath.Join(bundle, "rootfs.sparsebundle")); err != nil { + t.Fatalf("sparsebundle image not created: %v", err) + } + if m.mountPath != actualMount || !m.owned { + t.Fatalf("mount = %+v, want actual mount and owned", m) + } + if _, err := os.Stat(filepath.Join(actualMount, ".metadata_never_index")); err != nil { + t.Fatalf("spotlight marker missing: %v", err) + } + if err := m.Close(); err != nil { + t.Fatalf("Close fake mount: %v", err) + } +} + +func TestProvisionCaseSensitiveWithFakeHdiutilFailures(t *testing.T) { + cases := []struct { + name string + // setup configures the fake hdiutil for this failure mode and returns the + // requested mount path plus the mount path the detach log must record + // ("" to skip the detach-log check). + setup func(t *testing.T, detachLog string) (requestedMount, wantDetach string) + wantErr []string + }{ + { + name: "create failure", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_FAIL_CREATE", "1") + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + return filepath.Join(t.TempDir(), "mnt"), "" + }, + wantErr: []string{"hdiutil create", "create failed"}, + }, + { + // hdiutil writes its diagnostics to stderr, which must reach the + // error message: with a bare Output() the operator only sees + // "exit status N". + name: "attach failure surfaces hdiutil stderr", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_FAIL_ATTACH", "1") + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + return filepath.Join(t.TempDir(), "mnt"), "" + }, + wantErr: []string{"hdiutil attach", "attach diagnostic on stderr"}, + }, + { + name: "bad attach plist detaches requested mount", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_BAD_PLIST", "1") + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + requestedMount := filepath.Join(t.TempDir(), "requested") + return requestedMount, requestedMount + }, + wantErr: []string{"parse attach plist"}, + }, + { + name: "marker failure detaches actual mount", + setup: func(t *testing.T, detachLog string) (string, string) { + actualMount := filepath.Join(t.TempDir(), "missing-parent", "actual") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + return filepath.Join(t.TempDir(), "requested"), actualMount + }, + wantErr: []string{"spotlight marker"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + detachLog := filepath.Join(t.TempDir(), "detach.log") + requestedMount, wantDetach := tc.setup(t, detachLog) + _, err := provisionCaseSensitive(filepath.Join(t.TempDir(), "bundle"), requestedMount, "32m") + if err == nil { + t.Fatalf("provisionCaseSensitive succeeded, want error containing %v", tc.wantErr) + } + for _, want := range tc.wantErr { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err = %v, want substring %q", err, want) + } + } + if wantDetach != "" { + b, readErr := os.ReadFile(detachLog) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(b), wantDetach) { + t.Fatalf("detach log = %q, want mount %s", b, wantDetach) + } + } + }) + } +} + +// TestParseMountpointDecodesXMLEntities pins entity decoding: hdiutil's plist +// escapes XML-special characters in the mount path (a store path may carry +// "&" or "'"), and the raw escaped form would make every later use of the +// path (markers, rootfs, detach) target a nonexistent location. +func TestParseMountpointDecodesXMLEntities(t *testing.T) { + in := `mount-point/tmp/a & b's store/mnt` + got, err := parseMountpoint(in) + if err != nil { + t.Fatal(err) + } + if want := "/tmp/a & b's store/mnt"; got != want { + t.Fatalf("parseMountpoint = %q, want %q", got, want) + } +} diff --git a/cmd/elfuse-oci/store.go b/cmd/elfuse-oci/store.go index ec7972af..657f7561 100644 --- a/cmd/elfuse-oci/store.go +++ b/cmd/elfuse-oci/store.go @@ -163,18 +163,14 @@ func fsyncDir(dir string) error { // a pull racing an rmi); without it, last-writer-wins on refs.json can drop a // just-recorded pin. func (s *store) lock() (func(), error) { - f, err := os.OpenFile(filepath.Join(s.root, ".lock"), os.O_CREATE|os.O_RDWR, 0o644) + // acquireFlock rather than a bare Flock: it retries EINTR (run's signal + // forwarding can interrupt a blocking wait) and re-checks the lock file's + // identity, one lock discipline for the whole package. + l, err := acquireFlock(filepath.Join(s.root, ".lock"), syscall.LOCK_EX) if err != nil { - return nil, fmt.Errorf("store: open lock: %w", err) - } - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { - f.Close() return nil, fmt.Errorf("store: lock: %w", err) } - return func() { - _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) - _ = f.Close() - }, nil + return func() { _ = l.Close() }, nil } // withLock runs fn while holding the store lock. Results cross the closure diff --git a/go.mod b/go.mod index d004d0b6..69efb907 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/sysprog21/elfuse go 1.26.4 -require github.com/google/go-containerregistry v0.21.7 +require ( + github.com/google/go-containerregistry v0.21.7 + golang.org/x/sys v0.46.0 +) require ( github.com/docker/cli v29.5.3+incompatible // indirect @@ -12,6 +15,5 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect gotest.tools/v3 v3.5.2 // indirect ) From faa6fe68f806c604c143b025d1fa4541d191c1a8 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:33:29 +0200 Subject: [PATCH 07/26] Add OCI lifecycle commands and reachability GC Add list/images, rmi, and prune on top of the OCI image-layout store. rmi and prune use reachability GC: shared manifests, configs, and layers stay on disk while any remaining ref reaches them. Cache cleanup handles plain rootfs caches and macOS sparsebundle caches through platform-specific seams: a prune without --all never touches a still-pinned digest's cache, and one a live run uses is never reclaimed. Liveness is the same flock discipline on both cache kinds, the bundle's run.lock and a sibling .lock for the plain rootfs, held shared from before the existence probe until guest exit. The plain path execs elfuse in place, so that descriptor is made exec-survivable and the kernel releases it exactly when elfuse exits, SIGKILL included. prune skips busy caches (dry runs never advertise them); rmi refuses one even with --force, before any pin or descriptor is touched. The lock is a sibling of the cache dir, not inside it, because the dir is the guest's / and its existence is unpackImage's publication signal. An explicit --rootfs naming the store's own managed trees is rejected up front: the explicit path runs without the per-digest lock, so the sweeps would see the digest idle and could reclaim the tree under the live guest. The bundle sweep reaps staging directories only; a plain file wearing a clone or unpack-temp name is left alone. Liveness also roots blob reachability, not just cache sweeps. resolveImageForUse takes a digest's run lock inside the store lock and then releases the store lock, so a run reads that manifest's config and layers unlocked; if a concurrent pull repins the tag in that window, rooting only at pins would let the next prune or rmi reclaim the blobs that run is still reading. A busy digest therefore keeps its descriptor and blobs. The dry-run bundle sweep holds run.lock across its clone listing rather than probing and releasing first, because a clone without a keep marker is only proven abandoned while no run can create one. gc resolves every liveness root before reconciling index.json, so a stale or malformed pin fails the pass with the descriptors intact rather than after the reconcile dropped entries it could no longer justify (and then wedged every later prune and rmi on the same error). rmi's cache-existence probe uses Lstat and propagates errors, so a dangling symlink or unreadable cache aborts the removal instead of silently leaving the cache behind. The rootfs sweep also reclaims orphaned sibling .lock files whose cache dir never appeared (every run creates the lock without creating the plain dir), and the bundle busy probe opens run.lock without O_CREATE, so a dry run mutates nothing. The reference lock rides into the spawned guest beside run.lock, so rmi keeps refusing while an orphaned guest still reads the image. prune's GC sweep and list's snapshot run under the store lock, closing the windows where a concurrent pull's fresh blobs could be reclaimed before their descriptor lands or a listing could observe a half-removed ref, and gc reads each liveness root once per pass. list reports the full os/arch/variant platform and the creation time in both output modes, and propagates write errors the way inspect does, so truncated output cannot exit 0. Store writes become crash-durable, because reclamation may now act on what a crash left behind: the pin table, index.json, and a pull's appended blobs are synced before the pin that makes them reachable is committed, so a surviving pin can never name a manifest or layer still sitting in the page cache. Two corrections to the unpack path come with the locking, because both are what the lock discipline needs to be true. unpackImage now takes the caller's already-resolved image rather than re-resolving a ref: the caller keys the cache by the digest it resolved, so a re-resolution could observe a different pin from a concurrent repull and fill digest A's cache with image B's content. Layer application also tracks the ancestors of each entry, so an opaque whiteout arriving after an implicit parent directory no longer clears content the same layer added. Tests cover reachability GC (shared blobs, stale temp blobs, digest prefixes, and blobs held live by a busy unpinned digest), the prune and rmi busy semantics for both cache kinds, the store-path --rootfs rejection, the fail-closed gc ordering and cache probe, orphan-lock sweeping, dry-run probe purity, lock survival across the exec boundary, and end-to-end command wrappers. The darwin sweep runs through the isMountPointFn and detachForce seams, so it needs no disk images. --- cmd/elfuse-oci/bundlelock_test.go | 68 + cmd/elfuse-oci/cache_darwin.go | 351 +++++ cmd/elfuse-oci/cache_darwin_test.go | 644 +++++++++ cmd/elfuse-oci/cache_key.go | 34 +- cmd/elfuse-oci/cache_other.go | 71 + cmd/elfuse-oci/clone_sweep.go | 158 +++ cmd/elfuse-oci/commands.go | 133 +- cmd/elfuse-oci/commands_integration_test.go | 612 ++++++++ cmd/elfuse-oci/common_test.go | 69 +- cmd/elfuse-oci/csrun.go | 58 +- cmd/elfuse-oci/csrun_darwin_test.go | 176 ++- cmd/elfuse-oci/csrun_other.go | 2 +- cmd/elfuse-oci/gc.go | 478 +++++++ cmd/elfuse-oci/inspect.go | 13 +- cmd/elfuse-oci/inspect_test.go | 90 ++ cmd/elfuse-oci/lifecycle_darwin_test.go | 119 ++ cmd/elfuse-oci/lifecycle_test.go | 1392 +++++++++++++++++++ cmd/elfuse-oci/list.go | 157 +++ cmd/elfuse-oci/main.go | 30 +- cmd/elfuse-oci/main_command_test.go | 69 + cmd/elfuse-oci/prune.go | 239 ++++ cmd/elfuse-oci/rmi.go | 59 + cmd/elfuse-oci/rootfslock.go | 193 +++ cmd/elfuse-oci/run.go | 52 +- cmd/elfuse-oci/run_test.go | 30 +- cmd/elfuse-oci/sparsebundle.go | 10 - cmd/elfuse-oci/sparsebundle_test.go | 151 +- cmd/elfuse-oci/store.go | 191 ++- cmd/elfuse-oci/store_test.go | 303 ++++ cmd/elfuse-oci/test_helpers_test.go | 22 +- cmd/elfuse-oci/unpack.go | 80 +- cmd/elfuse-oci/unpack_image_test.go | 100 +- cmd/elfuse-oci/unpack_test.go | 21 +- 33 files changed, 5943 insertions(+), 232 deletions(-) create mode 100644 cmd/elfuse-oci/cache_darwin.go create mode 100644 cmd/elfuse-oci/cache_darwin_test.go create mode 100644 cmd/elfuse-oci/cache_other.go create mode 100644 cmd/elfuse-oci/clone_sweep.go create mode 100644 cmd/elfuse-oci/commands_integration_test.go create mode 100644 cmd/elfuse-oci/gc.go create mode 100644 cmd/elfuse-oci/lifecycle_darwin_test.go create mode 100644 cmd/elfuse-oci/lifecycle_test.go create mode 100644 cmd/elfuse-oci/list.go create mode 100644 cmd/elfuse-oci/prune.go create mode 100644 cmd/elfuse-oci/rmi.go create mode 100644 cmd/elfuse-oci/rootfslock.go diff --git a/cmd/elfuse-oci/bundlelock_test.go b/cmd/elfuse-oci/bundlelock_test.go index 4a01b67c..b8bda4e3 100644 --- a/cmd/elfuse-oci/bundlelock_test.go +++ b/cmd/elfuse-oci/bundlelock_test.go @@ -6,10 +6,13 @@ package main import ( "errors" "os" + "os/exec" "path/filepath" "syscall" "testing" "time" + + "golang.org/x/sys/unix" ) func TestAcquireFlockSharedCoexists(t *testing.T) { @@ -122,6 +125,71 @@ func TestBundleLockPaths(t *testing.T) { } } +// TestPreserveAcrossExecClearsCloexec pins the exec-survival half of the +// plain-rootfs run lock: Go opens files close-on-exec, so without the +// FD_SETFD clear the flock would silently drop at syscall.Exec. +func TestPreserveAcrossExecClearsCloexec(t *testing.T) { + l, err := acquireFlock(filepath.Join(t.TempDir(), "x.lock"), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer l.Close() + flags, err := unix.FcntlInt(l.f.Fd(), unix.F_GETFD, 0) + if err != nil { + t.Fatal(err) + } + if flags&unix.FD_CLOEXEC == 0 { + t.Fatal("lock fd unexpectedly not close-on-exec before PreserveAcrossExec") + } + if err := l.PreserveAcrossExec(); err != nil { + t.Fatal(err) + } + flags, err = unix.FcntlInt(l.f.Fd(), unix.F_GETFD, 0) + if err != nil { + t.Fatal(err) + } + if flags&unix.FD_CLOEXEC != 0 { + t.Fatal("FD_CLOEXEC still set after PreserveAcrossExec") + } +} + +// TestRootfsLockSurvivesIntoChildProcess models the exec handoff with the +// closest testable analog: hand the lock's descriptor to a child via +// ExtraFiles (a dup shares the open file description, exactly like exec +// inheritance), drop the parent's fd WITHOUT unlocking, and require the +// flock to live precisely as long as the child: released by the kernel at +// kill, no unlock code run. An in-process syscall.Exec is untestable by +// construction; this pins the same kernel behavior the run path relies on. +func TestRootfsLockSurvivesIntoChildProcess(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + hold, err := acquireRootfsRunLock(dir) + if err != nil { + t.Fatal(err) + } + child := exec.Command("/bin/sleep", "30") + child.ExtraFiles = []*os.File{hold.f} + if err := child.Start(); err != nil { + t.Fatal(err) + } + // Close the parent's fd only (no LOCK_UN), mirroring the process image + // being replaced. The child's dup keeps the description locked. + if err := hold.f.Close(); err != nil { + t.Fatal(err) + } + hold.f = nil + + if !rootfsCacheBusy(dir) { + t.Error("lock not held while child lives; exec inheritance would drop it") + } + if err := child.Process.Kill(); err != nil { + t.Fatal(err) + } + _ = child.Wait() + if rootfsCacheBusy(dir) { + t.Error("lock still held after child death; kernel should have released it") + } +} + // TestSpawnElfuseWaitPassesLocksToChild pins the SIGKILL half of the // case-sensitive path's liveness: the spawned guest must inherit the run // lock's descriptor (ExtraFiles shares the open file description, exactly diff --git a/cmd/elfuse-oci/cache_darwin.go b/cmd/elfuse-oci/cache_darwin.go new file mode 100644 index 00000000..30287bb4 --- /dev/null +++ b/cmd/elfuse-oci/cache_darwin.go @@ -0,0 +1,351 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// On Darwin an unpacked cache can be either (or both) of: +// - a case-sensitive APFS sparsebundle bundle at cs/// holding the +// warm unpacked base tree (image file rootfs.sparsebundle + mount point mnt), +// - a plain rootfs// directory (the --plain-rootfs path). +// +// cacheExists / removeRefCaches / pruneCaches are the lifecycle seam the +// cross-platform gc.go/rmi.go/prune.go code calls; the darwin versions add the +// sparsebundle lifecycle (detach a still-mounted volume before removing its +// bundle directory) on top of the shared rootfs sweep. + +// cacheHasKeptData reports whether digest's cache holds run --keep retained +// output. A --keep run drops the `kept` sidecar beside the sparsebundle (outside +// the mounted volume, like the bundle flocks), so this is a cheap stat that does +// not need to attach a cold, detached bundle to inspect its clones. rmi refuses +// to reclaim such a cache without force. +func cacheHasKeptData(root, digest string) (bool, error) { + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + // An unparseable digest key has no bundle and so no kept data; a real + // rmi target is always a valid digest. + return false, nil + } + if _, err := os.Stat(keptSidecarPath(bundle)); err == nil { + return true, nil + } else if !os.IsNotExist(err) { + return false, err + } + return false, nil +} + +// cacheExists reports whether digest has any unpacked cache under the store: +// the case-sensitive sparsebundle bundle and/or the plain rootfs directory. +// Lstat, not Stat: unpack and run refuse a symlink at the cache path, so a +// dangling link planted there must read as present (and thus removable), not +// vanish behind the follow. A probe that cannot reach a verdict returns its +// error so rmi fails closed instead of dropping the image while silently +// leaving the cache behind. +func cacheExists(root, digest string) (bool, error) { + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + return false, err + } + if _, err := os.Lstat(bundle); err == nil { + return true, nil + } else if !os.IsNotExist(err) { + return false, err + } + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false, err + } + if _, err := os.Lstat(rootfs); err == nil { + return true, nil + } else if !os.IsNotExist(err) { + return false, err + } + return false, nil +} + +// cacheBusyForDigest reports whether a live run holds either of digest's +// unpacked caches: the sparsebundle bundle (run.lock, held from before the +// volume is attached until guest exit) or the plain rootfs directory (its +// sibling per-digest lock). Either one proves the digest is in use. A digest +// with no bundle probes as idle (csBundleBusy treats the missing lock file +// as no-holder), so no existence pre-check is needed. +func cacheBusyForDigest(root, digest string) bool { + if bundle, err := csBundleDirForDigest(root, digest); err == nil && csBundleBusy(bundle) { + return true + } + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false + } + return rootfsCacheBusy(rootfs) +} + +// removeRefCaches deletes digest's unpacked caches: the case-sensitive +// sparsebundle and the plain rootfs, when a digest has both. A crash leftover +// mount is recovered (orphan clones reaped, stale volume detached) so the +// bundle can be removed, but a volume that still hosts a live run's clone, or a +// plain rootfs a live --plain-rootfs run holds, refuses: dropping the cache +// (via rmi) means "reclaim derived state", not "rip the rootfs out from under a +// running guest". +// +// Both cache locks are preflighted before either form is deleted, so a busy +// side leaves BOTH caches intact. Deleting the sparsebundle first and only then +// discovering the plain rootfs is busy (or vice versa) would strand a +// half-deleted cache under a still-live pin. The plain reference lock is taken +// first, matching a run's acquisition order (reference lock, then bundle +// locks); both acquisitions are non-blocking, so the order cannot deadlock. +func removeRefCaches(s *store, digest string) error { + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + return err + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + busyErr := fmt.Errorf("cache for %s is in use by a live run; stop it before removing the image", digest) + + // Preflight the plain rootfs lock. An ENOENT (no rootfs/ scaffolding) means + // no plain cache and no plain run can exist, so there is nothing to hold. + rootfsUnlock, rootfsBusy, err := lockRootfsCacheForRemoval(rootfs) + haveRootfsLock := err == nil + if err != nil && !os.IsNotExist(err) { + return err + } + if rootfsBusy { + return busyErr + } + if haveRootfsLock { + defer rootfsUnlock() + } + + if _, err := os.Stat(bundle); err == nil { + _, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + return err + } + if busy { + // The plain lock (if held) is released by the deferred unlock; no + // cache was deleted, so the refusal leaves both forms intact. + return busyErr + } + // Hold the bundle locks across the removal: a concurrent run's + // provision would otherwise race in between the sweep and the + // RemoveAll and lose its freshly attached volume. + err = os.RemoveAll(bundle) + unlock() + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + + // Plain rootfs, removed under the lock already held from the preflight. + if haveRootfsLock { + if err := os.RemoveAll(rootfs); err != nil { + return err + } + if err := os.Remove(rootfsCacheLockPath(rootfs)); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} + +// pruneCaches drops elfuse's unpacked caches. Without opts.all, only caches for +// refs no longer pinned (orphan caches) are dropped; with opts.all, every +// cache. The plain rootfs sweep is shared (pruneRootfsCaches); the darwin-only +// sparsebundle sweep walks cs/// plus legacy cs// directories, +// detaching a still-mounted volume before removing its bundle. The bytes +// reported for a sparsebundle are the on-disk allocation of its image file +// (dirSize of rootfs.sparsebundle), not the 16g virtual ceiling and not a live +// mount's contents. +func (s *store) pruneCaches(opts pruneOpts) (pruneReport, error) { + live, err := s.liveCacheKeys() + if err != nil { + return pruneReport{}, err + } + rep, err := pruneRootfsCaches(s, live, opts) + if err != nil { + return rep, err + } + + csBase := filepath.Join(s.root, csCacheDirName) + entries, err := os.ReadDir(csBase) + if err != nil { + if os.IsNotExist(err) { + return rep, nil + } + return rep, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + top := filepath.Join(csBase, e.Name()) + if e.Name() == "sha256" { + children, err := os.ReadDir(top) + if err != nil { + return rep, err + } + for _, child := range children { + if !child.IsDir() { + continue + } + key := filepath.Join("sha256", child.Name()) + bundle := filepath.Join(top, child.Name()) + var err error + rep, err = pruneCSBundle(rep, bundle, key, live, opts) + if err != nil { + return rep, err + } + } + continue + } + + // Legacy ref-named sparsebundle caches are no longer live under the + // digest-keyed scheme; prune --cache reclaims them as orphan caches. + var err error + rep, err = pruneCSBundle(rep, top, "", live, opts) + if err != nil { + return rep, err + } + } + return rep, nil +} + +func pruneCSBundle(rep pruneReport, bundle, key string, live map[string]bool, opts pruneOpts) (pruneReport, error) { + // A still-pinned digest is off-limits to a non---all prune BEFORE any + // sweep: its attached volume may belong to an active run, and + // sweepCSBundle cannot tell a crashed leftover mount from a live one by + // mount state alone. A crashed pinned bundle's stale mount is recovered + // by the next run's provision (which re-attaches cleanly) or by an + // explicit prune --cache --all. + if key != "" && !opts.all && live[key] { + return rep, nil + } + + // Crash recovery: if a killed run left this bundle's volume attached, + // reap orphan COW clones inside it and detach the stale mount. If a live + // run still owns a clone in the volume, leave the whole bundle alone; + // force-detaching would rip the rootfs out from under that guest + // (reachable with --all, or via a legacy/unpinned bundle). A dry-run + // makes the same decisions read-only: it reports the orphan clones a + // real prune would reap and skips a busy bundle a real prune would + // leave, but never detaches or removes anything. + if opts.dryRun { + // A busy bundle (a live run holds run.lock) is left alone, exactly as + // a real prune would; otherwise report the clones a real sweep would + // reap. Read-only: hold the lock and list, never detach or remove. + // + // The lock is held across the listing, not merely probed and released, + // because listSweepableClones treats a clone without a keep marker as + // abandoned and that is only true while no run can be creating one: a + // run makes its clone before writing the marker, so a probe-then-list + // would report a starting run's clone as reapable. + runLock, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + return rep, nil + } + defer runLock.Close() + mnt := mountPointPath(bundle) + if isMountPointFn(mnt) { + rep.CacheDirs = append(rep.CacheDirs, listSweepableClones(mnt)...) + } + image := imagePath(bundle) + rep.Bytes += dirSize(image) + rep.CacheDirs = append(rep.CacheDirs, bundle) + return rep, nil + } + + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + return rep, err + } + if len(reaped) > 0 { + rep.CacheDirs = append(rep.CacheDirs, reaped...) + } + if busy { + return rep, nil + } + + image := imagePath(bundle) + rep.Bytes += dirSize(image) + // sweepCSBundle already detached a stale mount if there was one, and holds + // the bundle locks so a concurrent provision cannot re-populate the bundle + // between here and the removal. + err = os.RemoveAll(bundle) + unlock() + if err != nil { + return rep, err + } + rep.CacheDirs = append(rep.CacheDirs, bundle) + return rep, nil +} + +// sweepCSBundle performs crash recovery for one sparsebundle bundle under the +// per-bundle advisory flocks. It acquires attach.lock and run.lock +// exclusively (non-blocking): if either is held, a live run (or an in-flight +// provision) owns the bundle, so it returns busy=true without touching +// anything. Holding run.lock exclusively proves no run is executing out of the +// volume, so every clone left inside is abandoned by construction: reap the +// sweepable ones (all but --keep-marked clones, plus unpack leftovers) and +// detach a still-attached stale mount. +// +// On success it returns the reaped clone directories and an unlock func that +// releases both locks. The caller must invoke unlock AFTER it finishes with +// the bundle (typically after os.RemoveAll), so a concurrent provision cannot +// re-attach and re-populate the bundle in the gap. On busy or error the +// returned unlock is a non-nil no-op, so callers may always defer it. +func sweepCSBundle(bundle string) (reaped []string, busy bool, unlock func(), err error) { + noop := func() {} + attachLock, err := acquireFlock(attachLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + return nil, true, noop, nil + } + return nil, false, noop, err + } + runLock, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + attachLock.Close() + if errors.Is(err, errCacheBusy) { + return nil, true, noop, nil + } + return nil, false, noop, err + } + release := func() { + runLock.Close() + attachLock.Close() + } + + mnt := mountPointPath(bundle) + // Reject a symlinked mount path before any mount-status probe, clone reap, + // or detach: isMountPoint (os.Stat) and detachForce follow the link, so a + // tampered store with mnt symlinked at an unrelated attached volume would + // otherwise get that volume's contents reaped and the volume force-detached. + // provisionCaseSensitive guards its own attach path the same way; the + // destructive sweep (prune --cache, rmi) needs the guard too. + if li, err := os.Lstat(mnt); err == nil && li.Mode()&os.ModeSymlink != 0 { + release() + return nil, false, noop, fmt.Errorf("mount path %s is a symlink; refusing to detach/reap", mnt) + } + if isMountPointFn(mnt) { + reaped = reapSweepableClones(mnt) + if err := detachForce(mnt); err != nil { + release() + return reaped, false, noop, fmt.Errorf("detach %s: %w", mnt, err) + } + } + return reaped, false, release, nil +} diff --git a/cmd/elfuse-oci/cache_darwin_test.go b/cmd/elfuse-oci/cache_darwin_test.go new file mode 100644 index 00000000..1af314a5 --- /dev/null +++ b/cmd/elfuse-oci/cache_darwin_test.go @@ -0,0 +1,644 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" +) + +func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { + t.Helper() + oldIsMount := isMountPointFn + oldDetach := detachForce + if isMount != nil { + isMountPointFn = isMount + } + if detach != nil { + detachForce = detach + } + t.Cleanup(func() { + isMountPointFn = oldIsMount + detachForce = oldDetach + }) +} + +// holdRunLock takes a shared run.lock on the bundle for the test's lifetime, +// simulating a live run so a sweep sees the bundle busy. +func holdRunLock(t *testing.T, bundle string) { + t.Helper() + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + l, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) +} + +func writeSparseBundleMarker(t *testing.T, bundle string) { + t.Helper() + image := filepath.Join(bundle, "rootfs.sparsebundle") + if err := os.MkdirAll(image, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(image, "band"), []byte("data"), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestDarwinCacheExistsBundleAndPlainRootfs(t *testing.T) { + root := t.TempDir() + digest := "sha256:" + strings.Repeat("1", 64) + mustExists := func(want bool, context string) { + t.Helper() + got, err := cacheExists(root, digest) + if err != nil { + t.Fatalf("cacheExists (%s): %v", context, err) + } + if got != want { + t.Fatalf("cacheExists = %v for %s, want %v", got, context, want) + } + } + mustExists(false, "absent cache") + + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + mustExists(true, "sparsebundle cache") + if err := os.RemoveAll(bundle); err != nil { + t.Fatal(err) + } + + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + mustExists(true, "plain rootfs cache") + if _, err := cacheExists(root, "not-a-digest"); err == nil { + t.Fatal("cacheExists accepted an invalid digest, want error") + } +} + +func TestDarwinCacheHasKeptData(t *testing.T) { + root := t.TempDir() + digest := "sha256:" + strings.Repeat("6", 64) + + if kept, err := cacheHasKeptData(root, digest); err != nil || kept { + t.Fatalf("cacheHasKeptData absent = (%v, %v), want (false, nil)", kept, err) + } + + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + // A bundle from a normal (non-keep) run has no sidecar: not kept. + if kept, err := cacheHasKeptData(root, digest); err != nil || kept { + t.Fatalf("cacheHasKeptData no-sidecar = (%v, %v), want (false, nil)", kept, err) + } + + if err := os.WriteFile(keptSidecarPath(bundle), nil, 0o644); err != nil { + t.Fatal(err) + } + if kept, err := cacheHasKeptData(root, digest); err != nil || !kept { + t.Fatalf("cacheHasKeptData with sidecar = (%v, %v), want (true, nil)", kept, err) + } +} + +// TestDarwinRmiRefusesKeptCacheWithoutForce pins the one case rmi still refuses: +// a bundle holding run --keep retained output is not discarded without --force, +// and --force then drops the whole bundle and removes the image. +func TestDarwinRmiRefusesKeptCacheWithoutForce(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keptSidecarPath(bundle), nil, 0o644); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err == nil || !strings.Contains(err.Error(), "keep") { + t.Fatalf("rmi kept cache without --force err = %v, want --keep refusal", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Fatalf("pin lost after refused rmi: %v", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("bundle removed after refused rmi: %v, want present", err) + } + + rep, err := s.rmi("local:a", true) + if err != nil { + t.Fatalf("rmi --force kept cache: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi --force did not report dropping the cache") + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Fatalf("bundle after rmi --force: %v, want removed", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin present after rmi --force, want gone") + } +} + +func TestDarwinRemoveRefCachesDropsBundleAndRootfs(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("2", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + for _, p := range []string{bundle, rootfs} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("%s after removeRefCaches: %v, want IsNotExist", p, err) + } + } +} + +func TestDarwinRemoveRefCachesDetachesMountedBundle(t *testing.T) { + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("3", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + var detached string + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(path string) error { + detached = path + return nil + }, + ) + + // No run holds run.lock, so the still-attached mount is stale: the sweep + // detaches it and the bundle is removed. + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + if detached != mnt { + t.Fatalf("detached = %q, want %q", detached, mnt) + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Fatalf("bundle after removeRefCaches: %v, want IsNotExist", err) + } +} + +func TestDarwinPruneCachesDropsOrphanAndLegacyCSBundles(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := openTestStore(t) + liveDigest := "sha256:" + strings.Repeat("4", 64) + orphanDigest := "sha256:" + strings.Repeat("5", 64) + if err := s.savePins(refPins{"live": liveDigest}); err != nil { + t.Fatal(err) + } + liveBundle, _ := csBundleDirForDigest(s.root, liveDigest) + orphanBundle, _ := csBundleDirForDigest(s.root, orphanDigest) + legacyBundle := filepath.Join(s.root, "cs", "legacy_ref") + for _, bundle := range []string{liveBundle, orphanBundle, legacyBundle} { + writeSparseBundleMarker(t, bundle) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + if !slices.Contains(rep.CacheDirs, orphanBundle) || !slices.Contains(rep.CacheDirs, legacyBundle) { + t.Fatalf("pruneCaches dirs = %v, want orphan and legacy bundles", rep.CacheDirs) + } + if slices.Contains(rep.CacheDirs, liveBundle) { + t.Fatalf("pruneCaches dropped live bundle %s: %v", liveBundle, rep.CacheDirs) + } + if _, err := os.Stat(orphanBundle); !os.IsNotExist(err) { + t.Fatalf("orphan bundle after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(legacyBundle); !os.IsNotExist(err) { + t.Fatalf("legacy bundle after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(liveBundle); err != nil { + t.Fatalf("live bundle after prune: %v, want present", err) + } +} + +// TestDarwinPruneCSBundleDryRunDoesNotSweepOrDelete pins that a dry-run makes +// the same decisions as a real prune (report the clones a real sweep would +// reap, skip a busy bundle) while never detaching or removing anything. The +// busy check is a real (non-blocking) probe of the bundle locks. +func TestDarwinPruneCSBundleDryRunDoesNotSweepOrDelete(t *testing.T) { + key := filepath.Join("sha256", strings.Repeat("6", 64)) + dryRun := func(t *testing.T, bundle string) pruneReport { + t.Helper() + rep, err := pruneCSBundle(pruneReport{}, bundle, key, nil, pruneOpts{cache: true, dryRun: true}) + if err != nil { + t.Fatalf("pruneCSBundle dry-run: %v", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("dry-run removed bundle: %v", err) + } + return rep + } + noDetach := func(t *testing.T, isMount func(string) bool) { + t.Helper() + withDarwinCacheSeams(t, isMount, func(string) error { + t.Fatal("detachForce called during dry-run") + return nil + }) + } + + t.Run("unmounted bundle reported", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + noDetach(t, func(string) bool { return false }) + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != bundle { + t.Fatalf("dry-run dirs = %v, want [%s]", rep.CacheDirs, bundle) + } + }) + + t.Run("sweepable clone reported", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + mnt := filepath.Join(bundle, "mnt") + orphan := filepath.Join(mnt, "run-42-1") + if err := os.MkdirAll(orphan, 0o755); err != nil { + t.Fatal(err) + } + noDetach(t, func(path string) bool { return path == mnt }) + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 2 || rep.CacheDirs[0] != orphan || rep.CacheDirs[1] != bundle { + t.Fatalf("dry-run dirs = %v, want [%s %s]", rep.CacheDirs, orphan, bundle) + } + if rep.Bytes == 0 { + t.Fatal("dry-run Bytes = 0, want the bundle's on-disk size counted") + } + if _, err := os.Stat(orphan); err != nil { + t.Fatalf("dry-run reaped %s, want read-only: %v", orphan, err) + } + }) + + t.Run("busy bundle skipped", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + mnt := filepath.Join(bundle, "mnt") + noDetach(t, func(path string) bool { return path == mnt }) + holdRunLock(t, bundle) // a live run holds run.lock + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 0 { + t.Fatalf("dry-run dirs = %v, want empty for a busy bundle", rep.CacheDirs) + } + if rep.Bytes != 0 { + t.Fatalf("dry-run Bytes = %d, want 0 for a busy bundle", rep.Bytes) + } + }) + + // listSweepableClones is only sound while run.lock is held: a clone without + // a keep marker is abandoned only if no run can be creating one, and a run + // creates its clone before writing the marker. Probing the lock and + // releasing it before the listing left a window where a starting run's + // clone would be reported as reapable. flock conflicts between separate + // opens of the same file even within one process, so probing from inside + // the listing window observes whether the dry run really holds it. + t.Run("run lock held across the listing", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + heldDuringListing := false + noDetach(t, func(path string) bool { + if path == mnt { + heldDuringListing = csBundleBusy(bundle) + } + return path == mnt + }) + + dryRun(t, bundle) + if !heldDuringListing { + t.Error("run.lock was free while the dry run listed clones, want it held across the listing") + } + }) +} + +// TestDarwinSweepCSBundleIdleReapsAndHoldsLocks pins that an idle bundle +// (no run holds run.lock) is swept: a mounted volume's sweepable clones are +// reaped, the stale mount detached, and the returned unlock releases the +// bundle locks the sweep held for the caller's removal. +func TestDarwinSweepCSBundleIdleReapsAndHoldsLocks(t *testing.T) { + unmounted := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(unmounted, 0o755); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, func(string) bool { return false }, func(string) error { + t.Fatal("detachForce called for non-mount") + return nil + }) + reaped, busy, unlock, err := sweepCSBundle(unmounted) + if err != nil { + t.Fatalf("sweepCSBundle no mount: %v", err) + } + if busy || len(reaped) != 0 { + t.Fatalf("sweepCSBundle no mount = (reaped %v, busy %v), want idle empty", reaped, busy) + } + // While the sweep holds the locks, a would-be run's exclusive probe fails. + if _, err := acquireFlock(runLockPath(unmounted), syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("run.lock during sweep err = %v, want held", err) + } + unlock() + if free, err := acquireFlock(runLockPath(unmounted), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatalf("run.lock after unlock err = %v, want free", err) + } else { + free.Close() + } + + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + clone := filepath.Join(mnt, "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + var detached string + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(path string) error { detached = path; return nil }, + ) + reaped, busy, unlock, err = sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle mounted: %v", err) + } + defer unlock() + if busy { + t.Fatal("sweepCSBundle mounted-but-idle reported busy") + } + if len(reaped) != 1 || reaped[0] != clone { + t.Fatalf("mounted reaped = %v, want [%s]", reaped, clone) + } + if detached != mnt { + t.Fatalf("detached = %q, want %q", detached, mnt) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("sweepable clone not reaped: %v", err) + } +} + +// TestDarwinSweepCSBundleBusySkips pins that a bundle whose run.lock a live +// run holds is reported busy and NOT force-detached. +func TestDarwinSweepCSBundleBusySkips(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + t.Fatal("detachForce called although a live run holds the volume") + return nil + }, + ) + holdRunLock(t, bundle) + + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle: %v", err) + } + defer unlock() + if !busy { + t.Fatal("sweepCSBundle did not report busy for a live run") + } + if len(reaped) != 0 { + t.Fatalf("reaped = %v, want empty", reaped) + } +} + +// TestDarwinPruneCSBundleSkipsLivePinnedBeforeSweep pins the guard order: a +// still-pinned digest's bundle is skipped by a non---all prune before any +// sweep runs, so an active run's mount is never probed or detached. +func TestDarwinPruneCSBundleSkipsLivePinnedBeforeSweep(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + key := filepath.Join("sha256", strings.Repeat("7", 64)) + withDarwinCacheSeams(t, + func(string) bool { + t.Fatal("isMountPoint probed for a live pinned bundle") + return false + }, + func(string) error { + t.Fatal("detachForce called for a live pinned bundle") + return nil + }, + ) + + rep, err := pruneCSBundle(pruneReport{}, bundle, key, map[string]bool{key: true}, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCSBundle: %v", err) + } + if len(rep.CacheDirs) != 0 || rep.Bytes != 0 { + t.Fatalf("live pinned bundle was touched: %+v", rep) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("live pinned bundle missing after prune: %v", err) + } +} + +// TestDarwinPruneCSBundleLeavesBusyBundle pins that even when the sweep runs +// (e.g. --all), a bundle whose volume hosts a live run is left in place. +func TestDarwinPruneCSBundleLeavesBusyBundle(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + writeSparseBundleMarker(t, bundle) + key := filepath.Join("sha256", strings.Repeat("8", 64)) + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + t.Fatal("detachForce called although a live run holds the volume") + return nil + }, + ) + holdRunLock(t, bundle) + + rep, err := pruneCSBundle(pruneReport{}, bundle, key, map[string]bool{key: true}, pruneOpts{cache: true, all: true}) + if err != nil { + t.Fatalf("pruneCSBundle: %v", err) + } + if len(rep.CacheDirs) != 0 { + t.Fatalf("busy bundle reported as pruned: %v", rep.CacheDirs) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("busy bundle missing after prune: %v", err) + } +} + +// TestDarwinRemoveRefCachesRefusesLiveRun pins the rmi --force guard: a +// volume whose run.lock a live run holds must refuse cache removal instead of +// force-detaching the guest's rootfs. +func TestDarwinRemoveRefCachesRefusesLiveRun(t *testing.T) { + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("7", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + detached := false + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + detached = true + return nil + }, + ) + holdRunLock(t, bundle) + + err = removeRefCaches(s, digest) + if err == nil || !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("removeRefCaches err = %v, want live-run refusal", err) + } + if detached { + t.Fatal("removeRefCaches force-detached a live run's volume") + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("bundle after refusal: %v, want untouched", err) + } +} + +// TestDarwinSweepCSBundleRejectsSymlinkedMnt pins S8: a tampered store whose +// mnt is a symlink at an unrelated volume must be refused before any mount +// probe, clone reap, or detach, so the sweep cannot force-detach or wipe that +// volume. provisionCaseSensitive guards its attach path the same way. +func TestDarwinSweepCSBundleRejectsSymlinkedMnt(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(t.TempDir(), "unrelated-volume") + if err := os.MkdirAll(victim, 0o755); err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.Symlink(victim, mnt); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, + func(string) bool { + t.Fatal("isMountPoint probed a symlinked mnt") + return false + }, + func(string) error { + t.Fatal("detachForce followed a symlinked mnt") + return nil + }, + ) + + _, _, unlock, err := sweepCSBundle(bundle) + if unlock != nil { + unlock() + } + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("sweepCSBundle symlinked mnt err = %v, want symlink refusal", err) + } + if _, err := os.Lstat(mnt); err != nil { + t.Fatalf("symlink mnt disturbed: %v", err) + } +} + +// TestDarwinRemoveRefCachesBusyPlainRootfsLeavesBundle: when a digest +// has both cache forms and a live --plain-rootfs run holds the plain rootfs +// lock, removeRefCaches must refuse before deleting either form, so the +// sparsebundle is not left half-removed under a still-live pin. The bundle +// survives and the plain rootfs is untouched. +func TestDarwinRemoveRefCachesBusyPlainRootfsLeavesBundle(t *testing.T) { + withDarwinCacheSeams(t, + func(string) bool { return false }, + func(string) error { + t.Fatal("detachForce called although the plain rootfs run is live") + return nil + }, + ) + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("9", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + writeSparseBundleMarker(t, bundle) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + // A live --plain-rootfs run holds the plain rootfs lock shared. + hold, err := acquireRootfsRunLock(rootfs) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + + err = removeRefCaches(s, digest) + if err == nil || !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("removeRefCaches err = %v, want live-run refusal", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("sparsebundle removed despite refusal (half-deleted cache): %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Fatalf("plain rootfs removed despite refusal: %v", err) + } +} diff --git a/cmd/elfuse-oci/cache_key.go b/cmd/elfuse-oci/cache_key.go index 4133a3fa..a6441ada 100644 --- a/cmd/elfuse-oci/cache_key.go +++ b/cmd/elfuse-oci/cache_key.go @@ -18,14 +18,6 @@ const ( csCacheDirName = "cs" ) -// legacyCacheNameForRef reproduces the pre-digest cache naming scheme, which -// flattened the ref itself into a single (intentionally lossy) path -// component. New caches are keyed by digest (cacheKeyForDigest); this helper -// survives to document the old layout. -func legacyCacheNameForRef(ref string) string { - return strings.NewReplacer("/", "_", ":", "_", "@", "_").Replace(ref) -} - // cacheKeyForDigest returns the relative cache key used under rootfs/ and cs/. // The current store writes sha256 blobs only; keep the algorithm component in // the path so the layout remains explicit and non-lossy. @@ -45,6 +37,28 @@ func cacheKeyForDigest(digest string) (string, error) { return filepath.Join(h.Algorithm, h.Hex), nil } +// insideStore reports whether path lies under one of the store's managed +// trees (the rootfs/ digest caches or the cs/ bundle dirs). Lexical only: +// the caller guards a user-supplied --rootfs against naming a tree the +// store's sweeps may reclaim, and a symlinked alias of the store is that +// user's own arrangement. +func insideStore(store, path string) bool { + for _, base := range []string{ + filepath.Join(store, rootfsCacheDirName), + filepath.Join(store, csCacheDirName), + } { + rel, err := filepath.Rel(base, filepath.Clean(path)) + if err != nil { + continue + } + if rel == "." || (rel != ".." && + !strings.HasPrefix(rel, ".."+string(filepath.Separator))) { + return true + } + } + return false +} + func defaultRootfsForDigest(store, digest string) (string, error) { key, err := cacheKeyForDigest(digest) if err != nil { @@ -53,10 +67,6 @@ func defaultRootfsForDigest(store, digest string) (string, error) { return filepath.Join(store, rootfsCacheDirName, key), nil } -func legacyRootfsForRef(store, ref string) string { - return filepath.Join(store, rootfsCacheDirName, legacyCacheNameForRef(ref)) -} - // csBundleDirForDigest is /cs//: it holds the case-sensitive // sparsebundle image and the attach mount point for one pinned manifest digest. func csBundleDirForDigest(store, digest string) (string, error) { diff --git a/cmd/elfuse-oci/cache_other.go b/cmd/elfuse-oci/cache_other.go new file mode 100644 index 00000000..86cf6a8c --- /dev/null +++ b/cmd/elfuse-oci/cache_other.go @@ -0,0 +1,71 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !darwin + +package main + +import "os" + +// On non-Darwin the case-sensitive sparsebundle path is unavailable (no APFS, +// no hdiutil, no clonefile), so an unpacked cache is only ever the plain +// rootfs// directory. The lifecycle primitives (rmi, prune) touch +// caches through cacheExists / removeRefCaches / pruneCaches so the pure-Go +// blob GC and the rootfs cache sweep build and test on Linux CI; the darwin +// sparsebundle sweep lives in cache_darwin.go. + +// cacheHasKeptData reports whether digest's cache holds run --keep retained +// output. On non-Darwin the only cache is a plain rootfs directory with no +// per-run COW clones, so there is never retained output to protect: rmi always +// reclaims it. +func cacheHasKeptData(root, digest string) (bool, error) { + return false, nil +} + +// cacheExists reports whether digest has an unpacked cache under the store. On +// non-Darwin this is just the plain rootfs directory. Lstat, not Stat: unpack +// and run refuse a symlink at the cache path, so a dangling link planted there +// must read as present (and thus removable), not vanish behind the follow. A +// probe that cannot reach a verdict returns its error so rmi fails closed +// instead of dropping the image while silently leaving the cache behind. +func cacheExists(root, digest string) (bool, error) { + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false, err + } + if _, err := os.Lstat(rootfs); err == nil { + return true, nil + } else if !os.IsNotExist(err) { + return false, err + } + return false, nil +} + +// cacheBusyForDigest reports whether a live run holds digest's unpacked cache. +// On non-Darwin the only cache is the plain rootfs directory, guarded by its +// sibling per-digest lock. +func cacheBusyForDigest(root, digest string) bool { + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false + } + return rootfsCacheBusy(rootfs) +} + +// removeRefCaches deletes digest's unpacked cache(s). On non-Darwin, the plain +// rootfs directory only, refusing while a live run holds its per-digest lock. +func removeRefCaches(s *store, digest string) error { + return removeRootfsCacheForDigest(s, digest) +} + +// pruneCaches drops elfuse's unpacked caches. Without opts.all, only caches for +// refs no longer pinned (orphan caches) are dropped; with opts.all, every +// cache. On non-Darwin only plain rootfs// directories exist; the +// rootfs sweep is shared via pruneRootfsCaches. +func (s *store) pruneCaches(opts pruneOpts) (pruneReport, error) { + live, err := s.liveCacheKeys() + if err != nil { + return pruneReport{}, err + } + return pruneRootfsCaches(s, live, opts) +} diff --git a/cmd/elfuse-oci/clone_sweep.go b/cmd/elfuse-oci/clone_sweep.go new file mode 100644 index 00000000..7bbd6065 --- /dev/null +++ b/cmd/elfuse-oci/clone_sweep.go @@ -0,0 +1,158 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "syscall" +) + +// Per-run COW clones of the warm base tree live inside an attached sparsebundle +// volume as run-- directories (see csrun.go). A crashed unpack +// can also leave a rootfs.tmp- staging directory behind (see +// unpackImage). Liveness is decided by the per-bundle advisory flocks +// (bundlelock.go), never by pids: a sweep only reaps once it holds run.lock +// exclusively, which proves no run is executing out of the volume, so every +// clone it finds is abandoned by construction. Pid parsing is gone; pid +// reuse can no longer make a still-wanted clone look dead. +// +// A run started with --keep records a keep sidecar for its clone so the sweep +// preserves it; without the sidecar every run-* clone is reapable. +// +// These helpers are pure path/lock logic with no darwin-specific calls, so +// they build and unit-test on Linux; sweepCSBundle (which needs isMountPoint + +// detachForce) is darwin-only and lives in cache_darwin.go. + +// keepDirName is a mount-root directory holding one marker file per kept clone, +// named by clone directory name. It lives in the mount root, BESIDE the clones, +// never inside any clone: a clone directory is a guest's /, so a marker there +// would collide with an image that ships its own /.elfuse-keep and could be +// forged by the guest at runtime, either of which would make an ordinary run's +// clone masquerade as kept and leak past every sweep. The mount root is outside +// every guest's --sysroot view, so markers here are writable only by +// elfuse-oci. A sweep skips a clone whose marker is present; only +// whole-bundle removal (prune of an unpinned/--all bundle, rmi --force) +// reclaims a kept clone, and that deletes this directory with it. +const keepDirName = ".elfuse-keep" + +// keepDirPath returns the mount-root keep-marker directory. +func keepDirPath(mountPath string) string { + return filepath.Join(mountPath, keepDirName) +} + +// cloneKeepMarkerPath returns the keep marker path for the clone named +// cloneName under mountPath. +func cloneKeepMarkerPath(mountPath, cloneName string) string { + return filepath.Join(keepDirPath(mountPath), cloneName) +} + +// writeKeepMarker records that the COW clone at cloneDir should survive sweeps, +// via a marker in the mount-root keep directory, so a later sweep preserves it +// after the creating run has exited and released run.lock. +func writeKeepMarker(cloneDir string) error { + mountPath := filepath.Dir(cloneDir) + if err := os.MkdirAll(keepDirPath(mountPath), 0o755); err != nil { + return err + } + return touchFile(cloneKeepMarkerPath(mountPath, filepath.Base(cloneDir))) +} + +// touchFile creates (or truncates) an empty marker file. The markers carry +// no content; only their existence matters. +func touchFile(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + return f.Close() +} + +// isRunCloneDir reports whether a directory entry name is a per-run COW clone +// (run--). The pid is not parsed; only the naming matters. +func isRunCloneDir(name string) bool { + return strings.HasPrefix(name, "run-") +} + +// isUnpackTempDir reports whether name is a leftover unpack staging directory +// (rootfs.tmp-) from a crashed unpackImage into the volume. +func isUnpackTempDir(name string) bool { + return strings.HasPrefix(name, "rootfs.tmp-") +} + +// listSweepableClones returns the reapable directories inside mountPath: every +// run-- clone WITHOUT a keep marker, plus any rootfs.tmp-* unpack +// leftover. The caller must hold run.lock exclusively (no live run), so a +// clone lacking a keep marker is guaranteed abandoned. Removes nothing; it is +// the read-only detection half shared with prune --dry-run. +func listSweepableClones(mountPath string) []string { + entries, err := os.ReadDir(mountPath) + if err != nil { + return nil + } + var sweepable []string + for _, e := range entries { + name := e.Name() + // Both branches reap staging DIRECTORIES only; a plain file wearing + // either name did not come from this package and is not swept. + if !e.IsDir() { + continue + } + switch { + case isRunCloneDir(name): + // Preserve the clone unless its keep marker is DEFINITIVELY + // absent: err == nil means present, and a transient non-ENOENT + // error (e.g. EIO on the mounted volume) must not cause a + // --keep clone to be reaped; fail safe toward preservation. + if _, err := os.Stat(cloneKeepMarkerPath(mountPath, name)); !os.IsNotExist(err) { + continue + } + sweepable = append(sweepable, filepath.Join(mountPath, name)) + case isUnpackTempDir(name): + sweepable = append(sweepable, filepath.Join(mountPath, name)) + } + } + return sweepable +} + +// reapSweepableClones removes the directories listSweepableClones names. +// Removal is best-effort: a busy entry (e.g. still unmounting) is skipped and +// not reported. Returns the directories that were removed. The caller must +// hold run.lock exclusively. +func reapSweepableClones(mountPath string) []string { + var reaped []string + for _, dir := range listSweepableClones(mountPath) { + if err := os.RemoveAll(dir); err != nil { + continue // busy or evaporating; leave it for next time + } + reaped = append(reaped, dir) + } + return reaped +} + +// csBundleBusy reports whether a live run holds the bundle via a non-blocking +// exclusive probe of run.lock. It is the read-only busy check used by prune +// --dry-run and diagnostics; it acquires and immediately releases, mutating +// nothing: run.lock is opened without O_CREATE (the bundle twin of +// rootfsCacheBusy), so a missing file means no holder rather than a file +// conjured into every bundle a dry run merely looks at. It deliberately does +// NOT touch attach.lock: attach.lock is a lifecycle lock whose holders +// (provision, sweep, a run's last-one-out Close) each own the mount's detach +// fate, and a read-only prober transiently holding it would fool a +// concurrent Close into skipping its detach (leaving the volume attached +// with no owner). Any failure other than the missing lock file fails closed +// (busy) so a dry-run never advertises a reap it could not safely perform. +func csBundleBusy(bundle string) bool { + f, err := os.OpenFile(runLockPath(bundle), os.O_RDWR, 0) + if err != nil { + return !os.IsNotExist(err) + } + defer f.Close() + if err := flockRetryIntr(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return true + } + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return false +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index 04c54c56..4d02e1c3 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -7,12 +7,65 @@ import ( "errors" "fmt" "os" + + "github.com/google/go-containerregistry/pkg/v1" ) // The four subcommands share common-flag parsing (common.go) and the OCI // image-layout store (store.go). pull/unpack/inspect are pure store ops; run // additionally resolves the runspec and execs elfuse (run.go, csrun.go). +// afterImageResolve fires just after resolveImageForUse returns, while the +// caller holds the per-digest reference lock but no store lock. Production +// no-op; tests inject a concurrent repull here to prove the caller keeps using +// the image it resolved (keyed by the locked digest) and not a repinned one. +var afterImageResolve = func(digest string) {} + +// resolveImageForUse resolves ref to its pinned image and returns it with the +// manifest digest and a held per-digest reference lock (the digest's plain +// rootfs run lock, taken shared). Resolution and lock acquisition happen in +// one store-locked critical section, so the digest returned is exactly the +// digest locked: a concurrent repull (which needs the store lock to move the +// pin) cannot slip between the two, and the caller can key digest-scoped +// caches without a later re-resolution poisoning them. The reference lock +// then marks the image in use for the caller's lifetime: rmi probes it (and +// refuses) before dropping the last pin's descriptor and blobs, even before +// any cache dir exists, so a cold run that has resolved but not yet unpacked +// is never GC'd out from under. +// +// The SH acquisition cannot block: the only EX takers (rmi, prune) hold the +// store lock, which this holds, so no EX holder can exist meanwhile. +func resolveImageForUse(s *store, ref string) (v1.Image, string, *flockFile, error) { + var img v1.Image + var digest string + var lock *flockFile + err := s.withLock(func() error { + var err error + img, err = s.image(ref) + if err != nil { + return err + } + // Digest reads the manifest blob; do it under the lock so the blob + // cannot vanish mid-read. + d, err := img.Digest() + if err != nil { + return err + } + digest = d.String() + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + lock, err = acquireRootfsRunLock(rootfs) + return err + }) + if err != nil { + return nil, "", nil, err + } + afterImageResolve(digest) + return img, digest, lock, nil +} + // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { cf, ref, err := parsePullArgs(args) @@ -47,10 +100,15 @@ func cmdUnpack(args []string) error { if err != nil { return err } - digest, err := s.digestFor(ref) + // resolveImageForUse pairs the image with the digest lock, so the cache + // keyed below by this digest is filled from this image even if the ref is + // repulled mid-unpack, and a concurrent rmi/prune cannot reclaim the tree + // (or the blobs the unpack still reads) mid-merge. + img, digest, lock, err := resolveImageForUse(s, ref) if err != nil { return err } + defer lock.Close() if rootfs == "" { rootfs, err = defaultRootfsForDigest(cf.store, digest) if err != nil { @@ -60,11 +118,13 @@ func cmdUnpack(args []string) error { return err } } - fmt.Printf("Unpacking %s -> %s\n", ref, rootfs) - if err := unpackImage(s, ref, rootfs); err != nil { + // Progress goes to stderr, like every other report in this CLI; stdout + // is reserved for command output. + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(img, rootfs); err != nil { return err } - fmt.Printf("Unpacked %s\n", ref) + fmt.Fprintf(os.Stderr, "Unpacked %s\n", ref) return nil } @@ -119,23 +179,24 @@ func cmdRun(args []string) error { if err != nil { return err } - img, err := s.image(ref) - if err != nil { + // Resolve under the store lock and come back holding the per-digest run + // lock: from here to guest exit, rmi cannot reclaim this image's + // descriptor, blobs, or caches out from under the setup. + img, digestStr, refLock, err := resolveImageForUse(s, ref) + if errors.Is(err, errNotPulled) { // Auto-pull only when the ref is simply absent, so `run` is // self-sufficient on first use. Any other failure (corrupt refs.json, // unreadable layout) must surface rather than mask itself behind a // fresh network pull. - if !errors.Is(err, errNotPulled) { - return err - } if err := pullImage(cf, s, ref); err != nil { return err } - img, err = s.image(ref) - if err != nil { - return err - } + img, digestStr, refLock, err = resolveImageForUse(s, ref) } + if err != nil { + return err + } + defer refLock.Close() cfg, err := img.ConfigFile() if err != nil { return err @@ -153,12 +214,6 @@ func cmdRun(args []string) error { ref, got, want, want, ref) } } - digest, err := img.Digest() - if err != nil { - return err - } - digestStr := digest.String() - // Choose the rootfs path. Default: a case-sensitive APFS sparsebundle so // the guest's case-sensitive filenames don't collide on the host's // case-insensitive volume, with a per-run COW clone for isolation and @@ -167,16 +222,41 @@ func cmdRun(args []string) error { useCS := rf.rootfs == "" && !rf.plainRootfs if useCS { - return runCaseSensitive(cf, s, ref, digestStr, cfg, rf, tail) + // refLock stays held by this wrapper AND is handed to the guest child + // (spawnElfuseWait's ExtraFiles), so the reference lock lives for the + // whole run even when the wrapper dies uncatchably: rmi sees the + // digest in use before the bundle exists and for as long as elfuse + // still executes. + return runCaseSensitive(cf, s, ref, img, digestStr, cfg, rf, tail, refLock) } - // Plain-directory path. + // Plain-directory path. The reference lock resolveImageForUse holds IS the + // per-digest run lock the store-managed cache is swept under, so a store + // cache run reuses it: it was taken before the existence probe, so the + // cache cannot be reclaimed between the probe and the guest's first read, + // and execElfuse threads its descriptor through the exec so the lock lives + // exactly as long as the guest. An explicit --rootfs is user-managed and + // never store-swept, so the run itself needs no lock; but a cold unpack + // below still reads layer blobs lazily from the store, so the reference + // lock is held until the unpack is done rather than released here (an + // rmi/prune racing the unpack must see the digest busy, not GC the blobs + // mid-read). + var rootfsLock *flockFile storeManaged := rf.rootfs == "" if storeManaged { rf.rootfs, err = defaultRootfsForDigest(cf.store, digestStr) if err != nil { return err } + rootfsLock = refLock + } else if insideStore(cf.store, rf.rootfs) { + // An explicit --rootfs naming the store's own managed trees would + // run without the per-digest flock, and rmi/prune decide liveness by + // that flock alone: the sweeps could reclaim the tree under the live + // guest. Reject it, keeping the user-managed premise above true. + return fmt.Errorf( + "--rootfs %s is inside the store; drop --rootfs to run the managed cache", + rf.rootfs) } // Ensure the rootfs is unpacked before computing the spec, because // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if @@ -195,10 +275,15 @@ func cmdRun(args []string) error { } if !published { fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rf.rootfs) - if err := unpackImage(s, ref, rf.rootfs); err != nil { + if err := unpackImage(img, rf.rootfs); err != nil { return err } } + if rootfsLock == nil { + // Explicit --rootfs: the store blobs are no longer needed and the + // rootfs is not digest-keyed, so the reference lock ends here. + refLock.Close() + } spec, err := computeRunSpec(cfg, rf, rf.rootfs, tail) if err != nil { return err @@ -207,10 +292,10 @@ func cmdRun(args []string) error { // the guest's resolver/hostname work. On the plain path this mutates the // unpacked rootfs directory (acceptable: --plain-rootfs is the v1/debug // path; re-runs overwrite the same small files). - if err := injectRuntimeFiles(rf.rootfs); err != nil { + if err := prepareRootfsForRun(rf.rootfs, spec); err != nil { return err } - return execElfuseForRun(rf.rootfs, spec) + return execElfuseForRun(rf.rootfs, spec, rootfsLock) } func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error) { diff --git a/cmd/elfuse-oci/commands_integration_test.go b/cmd/elfuse-oci/commands_integration_test.go new file mode 100644 index 00000000..06aed4c5 --- /dev/null +++ b/cmd/elfuse-oci/commands_integration_test.go @@ -0,0 +1,612 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" +) + +func withFakeCranePull(t *testing.T, fn func(string, ...crane.Option) (v1.Image, error)) { + t.Helper() + old := cranePull + cranePull = fn + t.Cleanup(func() { cranePull = old }) +} + +func withFakeExecElfuse(t *testing.T, fn func(string, *runSpec, *flockFile) error) { + t.Helper() + old := execElfuseForRun + execElfuseForRun = fn + t.Cleanup(func() { execElfuseForRun = old }) +} + +func TestCmdPullPinsImageOffline(t *testing.T) { + root := t.TempDir() + img := tinyImage(t) + wantDigest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + var gotRef string + var gotOptions int + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + gotRef = ref + gotOptions = len(opts) + return img, nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return cmdPull([]string{"--store", root, "--platform", "linux/amd64", "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdPull: %v", err) + } + if stdout != "" { + t.Fatalf("cmdPull stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "Pulled local:tiny -> "+wantDigest.String()) { + t.Fatalf("cmdPull stderr = %q, want pull summary", stderr) + } + if gotRef != "local:tiny" || gotOptions != 2 { + t.Fatalf("fake crane.Pull got ref=%q options=%d, want local:tiny with the platform and keychain options", gotRef, gotOptions) + } + + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + gotDigest, err := s.digestFor("local:tiny") + if err != nil { + t.Fatal(err) + } + if gotDigest != wantDigest.String() { + t.Fatalf("pin digest = %s, want %s", gotDigest, wantDigest) + } +} + +func TestCmdPullWrapsPullError(t *testing.T) { + root := t.TempDir() + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + return nil, errors.New("registry unavailable") + }) + + _, _, err := captureOutput(t, func() error { + return cmdPull([]string{"--store", root, "local:missing"}) + }) + if err == nil || !strings.Contains(err.Error(), "pull local:missing") || !strings.Contains(err.Error(), "registry unavailable") { + t.Fatalf("cmdPull error = %v, want wrapped pull error", err) + } +} + +func TestCmdListInspectRmiAndPruneWrappers(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + stdout, stderr, err := captureOutput(t, func() error { + return cmdList([]string{"--store", s.root}) + }) + if err != nil { + t.Fatalf("cmdList: %v", err) + } + if stderr != "" || !strings.Contains(stdout, "local:a") || !strings.Contains(stdout, "linux/arm64") { + t.Fatalf("cmdList stdout=%q stderr=%q, want list row", stdout, stderr) + } + listedDigest := shortDigest(manifest.String()) + if !strings.Contains(stdout, listedDigest) { + t.Fatalf("cmdList stdout=%q, want digest %s", stdout, listedDigest) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdInspect([]string{"--store", s.root, "--json", "local:a"}) + }) + if err != nil { + t.Fatalf("cmdInspect --json: %v", err) + } + // The raw config blob (compact, as stored), not a re-marshal: vendor + // extension fields must survive inspect --json. + if stderr != "" || !strings.Contains(stdout, `"architecture":"arm64"`) || !strings.HasSuffix(stdout, "\n") { + t.Fatalf("cmdInspect stdout=%q stderr=%q, want raw config JSON with trailing newline", stdout, stderr) + } + + orphan := writeOrphanBlob(t, s.root, "command-prune-orphan") + stdout, stderr, err = captureOutput(t, func() error { + return cmdPrune([]string{"--store", s.root, "--dry-run"}) + }) + if err != nil { + t.Fatalf("cmdPrune --dry-run: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Would reclaim: 1 blob(s)") || !strings.Contains(stderr, orphan) { + t.Fatalf("cmdPrune dry-run stdout=%q stderr=%q, want dry-run summary", stdout, stderr) + } + if _, err := os.Stat(blobPath(s.root, orphan)); err != nil { + t.Fatalf("dry-run removed orphan blob: %v", err) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdPrune([]string{"--store", s.root}) + }) + if err != nil { + t.Fatalf("cmdPrune: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Reclaimed: 1 blob(s)") { + t.Fatalf("cmdPrune stdout=%q stderr=%q, want reclaim summary", stdout, stderr) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Fatalf("orphan blob after prune: %v, want IsNotExist", err) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdRmi([]string{"--store", s.root, listedDigest}) + }) + if err != nil { + t.Fatalf("cmdRmi: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Removed local:a:") { + t.Fatalf("cmdRmi stdout=%q stderr=%q, want removal summary", stdout, stderr) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after cmdRmi") + } +} + +func TestCmdUnpackWrapperExplicitAndDefaultRootfs(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + digest, err := s.addImage("local:tiny", img) + if err != nil { + t.Fatal(err) + } + + explicit := filepath.Join(t.TempDir(), "explicit-rootfs") + stdout, stderr, err := captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "--rootfs", explicit, "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdUnpack explicit: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Unpacking local:tiny -> "+explicit) || !strings.Contains(stderr, "Unpacked local:tiny") { + t.Fatalf("cmdUnpack explicit stdout=%q stderr=%q", stdout, stderr) + } + if b, err := os.ReadFile(filepath.Join(explicit, "hello")); err != nil || string(b) != "world" { + t.Fatalf("explicit rootfs hello = %q, err=%v; want world", b, err) + } + + defaultRootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + stdout, stderr, err = captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdUnpack default: %v", err) + } + if stdout != "" || !strings.Contains(stderr, defaultRootfs) { + t.Fatalf("cmdUnpack default stdout=%q stderr=%q, want default rootfs path", stdout, stderr) + } + if b, err := os.ReadFile(filepath.Join(defaultRootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("default rootfs hello = %q, err=%v; want world", b, err) + } +} + +func TestCmdRunPlainRootfsUnpacksInjectsAndExecs(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "rootfs") + var gotRootfs string + var gotSpec *runSpec + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + gotRootfs = rootfs + gotSpec = spec + if b, err := os.ReadFile(filepath.Join(rootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("rootfs hello = %q, err=%v; want world before exec", b, err) + } + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + if _, err := os.Stat(filepath.Join(rootfs, "etc", name)); err != nil { + t.Fatalf("runtime file %s missing before exec: %v", name, err) + } + } + return nil + }) + + stderrExpected := "Unpacking local:a -> " + rootfs + stdout, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, + "--plain-rootfs", + "--rootfs", rootfs, + "--env", "A=2", + "local:a", + "/cli-cmd", "arg", + }) + }) + if err != nil { + t.Fatalf("cmdRun --plain-rootfs: %v", err) + } + if stdout != "" || !strings.Contains(stderr, stderrExpected) { + t.Fatalf("cmdRun stdout=%q stderr=%q, want unpack message %q", stdout, stderr, stderrExpected) + } + if gotRootfs != rootfs { + t.Fatalf("exec rootfs = %q, want %q", gotRootfs, rootfs) + } + if gotSpec == nil { + t.Fatal("exec spec was nil") + } + if !reflect.DeepEqual(gotSpec.Args, []string{"/cli-cmd", "arg"}) { + t.Fatalf("spec args = %v, want CLI tail", gotSpec.Args) + } + if !reflect.DeepEqual(gotSpec.Env, []string{"A=2", "PATH=" + defaultGuestPath}) { + t.Fatalf("spec env = %v, want [A=2] plus default PATH", gotSpec.Env) + } +} + +func TestCmdRunPlainRootfsEntrypointOverride(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "rootfs") + var gotSpec *runSpec + withFakeExecElfuse(t, func(_ string, spec *runSpec, _ *flockFile) error { + gotSpec = spec + return nil + }) + + _, _, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, "--plain-rootfs", "--rootfs", rootfs, + "--entrypoint", "/override", "local:a", "x", "y", + }) + }) + if err != nil { + t.Fatalf("cmdRun --entrypoint: %v", err) + } + if gotSpec == nil { + t.Fatal("exec spec was nil") + } + // --entrypoint replaces the image Entrypoint AND drops the image Cmd; the + // CLI tail becomes the new Cmd. + if want := []string{"/override", "x", "y"}; !reflect.DeepEqual(gotSpec.Args, want) { + t.Fatalf("spec args = %v, want %v", gotSpec.Args, want) + } +} + +func TestCmdRunPlainRootfsExistingSkipsUnpack(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "existing-rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + if _, err := os.Stat(filepath.Join(rootfs, "hello")); !os.IsNotExist(err) { + t.Fatalf("existing rootfs was unpacked over: stat hello = %v, want IsNotExist", err) + } + if !reflect.DeepEqual(spec.Args, []string{"/image-cmd"}) { + t.Fatalf("spec args = %v, want image cmd", spec.Args) + } + return nil + }) + + _, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}) + }) + if err != nil { + t.Fatalf("cmdRun existing rootfs: %v", err) + } + if strings.Contains(stderr, "Unpacking") { + t.Fatalf("existing rootfs stderr = %q, want no unpack message", stderr) + } +} + +// TestCmdRunRejectsExplicitRootfsInsideStore pins that --rootfs naming the +// store's own managed trees (the rootfs/ digest caches, the cs/ bundle dirs) +// is refused. The explicit-rootfs path runs without the per-digest flock and +// prune/rmi decide liveness by that flock alone, so the old behavior let the +// sweeps reclaim the cache under the live guest. +func TestCmdRunRejectsExplicitRootfsInsideStore(t *testing.T) { + s := openTestStore(t) + digest, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})) + if err != nil { + t.Fatal(err) + } + cache, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + execCalled := false + withFakeExecElfuse(t, func(_ string, _ *runSpec, _ *flockFile) error { + execCalled = true + return nil + }) + + _, _, err = captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "--rootfs", cache, "local:a"}) + }) + if err == nil || !strings.Contains(err.Error(), "inside the store") { + t.Fatalf("err=%v, want an inside-the-store rejection", err) + } + if execCalled { + t.Fatal("guest exec reached despite the rejected --rootfs") + } +} + +func TestCmdRunPlainRootfsAutoPullsMissingImage(t *testing.T) { + root := t.TempDir() + rootfs := filepath.Join(t.TempDir(), "rootfs") + pullCalls := 0 + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + pullCalls++ + if ref != "local:pulled" { + t.Fatalf("pull ref = %q, want local:pulled", ref) + } + return buildImage(t, []string{"/pulled-cmd"}), nil + }) + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + if !reflect.DeepEqual(spec.Args, []string{"/pulled-cmd"}) { + t.Fatalf("spec args = %v, want pulled image cmd", spec.Args) + } + return nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", root, "--plain-rootfs", "--rootfs", rootfs, "local:pulled"}) + }) + if err != nil { + t.Fatalf("cmdRun auto-pull: %v", err) + } + if pullCalls != 1 { + t.Fatalf("pullCalls = %d, want 1", pullCalls) + } + // run's stdout belongs to the guest (callers capture it), so the pull + // summary must land on stderr with the unpack progress. + if strings.Contains(stdout, "Pulled local:pulled") { + t.Fatalf("cmdRun auto-pull stdout = %q, pull summary leaked into guest stdout", stdout) + } + if !strings.Contains(stderr, "Pulled local:pulled") || !strings.Contains(stderr, "Unpacking local:pulled") { + t.Fatalf("cmdRun auto-pull stderr = %q, want pull and unpack summaries", stderr) + } + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + if _, err := s.digestFor("local:pulled"); err != nil { + t.Fatalf("auto-pulled ref was not pinned: %v", err) + } +} + +func TestCommandWrappersReturnParseAndStoreErrors(t *testing.T) { + parseCases := []struct { + name string + fn func() error + }{ + {"pull", func() error { return cmdPull(nil) }}, + {"unpack", func() error { return cmdUnpack(nil) }}, + {"inspect", func() error { return cmdInspect(nil) }}, + {"run", func() error { return cmdRun(nil) }}, + {"list", func() error { return cmdList([]string{"extra"}) }}, + {"rmi", func() error { return cmdRmi(nil) }}, + {"prune", func() error { return cmdPrune([]string{"--all"}) }}, + } + for _, tc := range parseCases { + t.Run("parse "+tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Fatalf("%s parse error case succeeded, want error", tc.name) + } + }) + } + + storeFile := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(storeFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + storeCases := []struct { + name string + fn func() error + }{ + {"pull", func() error { return cmdPull([]string{"--store", storeFile, "local:a"}) }}, + {"unpack", func() error { return cmdUnpack([]string{"--store", storeFile, "local:a"}) }}, + {"inspect", func() error { return cmdInspect([]string{"--store", storeFile, "local:a"}) }}, + {"run", func() error { return cmdRun([]string{"--store", storeFile, "local:a"}) }}, + {"list", func() error { return cmdList([]string{"--store", storeFile}) }}, + {"rmi", func() error { return cmdRmi([]string{"--store", storeFile, "local:a"}) }}, + {"prune", func() error { return cmdPrune([]string{"--store", storeFile}) }}, + } + for _, tc := range storeCases { + t.Run("store "+tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Fatalf("%s store error case succeeded, want error", tc.name) + } + }) + } +} + +// TestCmdRunPlatformMismatchOnPinnedRef pins the --platform check: the store +// pins one digest per ref, so an explicit --platform that disagrees with the +// pinned image must fail instead of silently launching the wrong +// architecture. Without --platform the pinned image runs as-is. +func TestCmdRunPlatformMismatchOnPinnedRef(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + withFakeExecElfuse(t, func(string, *runSpec, *flockFile) error { return nil }) + rootfs := filepath.Join(t.TempDir(), "rootfs") + + _, _, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, "--platform", "linux/amd64", + "--plain-rootfs", "--rootfs", rootfs, "local:a", + }) + }) + if err == nil || !strings.Contains(err.Error(), "pinned for linux/arm64") { + t.Fatalf("cmdRun --platform mismatch err = %v, want pinned-platform error", err) + } + + for _, args := range [][]string{ + {"--store", s.root, "--platform", "linux/arm64", "--plain-rootfs", "--rootfs", rootfs, "local:a"}, + {"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}, + } { + if _, _, err := captureOutput(t, func() error { return cmdRun(args) }); err != nil { + t.Fatalf("cmdRun %v: %v", args, err) + } + } +} + +// TestCmdRunPlainRootfsLockDiscipline pins which runs hold the per-digest +// cache lock at exec time: a store-default rootfs arrives with the run lock +// held (a concurrent prune would see busy), while an explicit --rootfs is +// user-managed and locks nothing. +func TestCmdRunPlainRootfsLockDiscipline(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/image-cmd"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + var gotLock *flockFile + var busyAtExec bool + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + gotLock = lock + busyAtExec = rootfsCacheBusy(rootfs) + return nil + }) + if _, _, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "local:a"}) + }); err != nil { + t.Fatalf("cmdRun store-default: %v", err) + } + if gotLock == nil { + t.Error("store-default rootfs exec got nil lock, want held run lock") + } + if !busyAtExec { + t.Error("store-default cache not busy at exec time, want lock held") + } + + gotLock = nil + rootfs := filepath.Join(t.TempDir(), "explicit") + if _, _, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}) + }); err != nil { + t.Fatalf("cmdRun explicit rootfs: %v", err) + } + if gotLock != nil { + t.Error("explicit --rootfs exec got a lock, want nil") + } +} + +// markedImage builds a single-layer image whose /marker file holds content, so +// a test can tell which image populated a digest-keyed cache. +func markedImage(t *testing.T, content string) v1.Image { + t.Helper() + return testImageWithLayers(t, testTarLayer(t, + tarEntry{header: regHeader("marker", 0o644, 0), body: content})) +} + +// TestCmdUnpackUsesResolvedImageDespiteRepull pins that unpack fills the +// digest-keyed cache from the image it resolved, not a re-resolution of the +// mutable ref: a repull that moves the tag to a different image mid-setup must +// not poison digest A's cache with image B's content. The afterImageResolve +// seam fires in the exact window between resolution and unpack. +func TestCmdUnpackUsesResolvedImageDespiteRepull(t *testing.T) { + s := openTestStore(t) + imgA := markedImage(t, "image-A") + imgB := markedImage(t, "image-B") + digestA, err := s.addImage("local:tag", imgA) + if err != nil { + t.Fatal(err) + } + + old := afterImageResolve + t.Cleanup(func() { afterImageResolve = old }) + repinned := false + afterImageResolve = func(digest string) { + if repinned || digest != digestA { + return + } + repinned = true + // Move the tag to image B while the unpack still holds digest A's + // reference lock. addImage takes the store lock, which resolveImageForUse + // has already released by now. + if _, err := s.addImage("local:tag", imgB); err != nil { + t.Errorf("repull to image B: %v", err) + } + } + + if _, _, err := captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "local:tag"}) + }); err != nil { + t.Fatalf("cmdUnpack: %v", err) + } + if !repinned { + t.Fatal("afterImageResolve never fired for digest A") + } + rootfsA, err := defaultRootfsForDigest(s.root, digestA) + if err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(rootfsA, "marker")); err != nil || string(b) != "image-A" { + t.Fatalf("digest-A cache marker = %q, err=%v; want image-A (not poisoned by repull)", b, err) + } +} + +// TestRmiRefusesWhileReferenceLockHeld: a starting run holds the +// per-digest reference lock before any cache dir or bundle exists, so rmi of +// the last pin must refuse (even with --force) rather than GC blobs out from +// under it, and the pin plus its blobs survive the refusal. Once the lock is +// released the same rmi succeeds. +func TestRmiRefusesWhileReferenceLockHeld(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + // A cold run's reference lock: taken before the rootfs cache dir exists. + hold, err := acquireRootfsRunLock(rootfs) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Fatalf("cache dir exists before run unpacked it: %v", err) + } + + if _, err := s.rmi("local:a", true); err == nil { + t.Fatal("rmi --force succeeded while reference lock held, want refusal") + } + if _, err := s.image("local:a"); err != nil { + t.Fatalf("pin/blobs gone after refused rmi: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi after lock released: %v", err) + } +} diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index c53979c3..357ec7e5 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -135,27 +135,70 @@ func TestParseRunArgs(t *testing.T) { func TestParseCommandFlagErrors(t *testing.T) { cases := []struct { name string - err error + fn func() error }{ - {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, - {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, - {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, - {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }()}, - // --platform belongs to the commands that resolve a platform; - // unpack and inspect discarding a successfully parsed flag would - // mislead the caller. + {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }}, + {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }}, + {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }}, + {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }}, + {"list extra arg", func() error { _, _, err := parseListArgs([]string{"alpine:3"}); return err }}, + {"rmi missing ref", func() error { _, _, _, err := parseRmiArgs([]string{"--force"}); return err }}, + {"prune all without cache", func() error { _, _, err := parsePruneArgs([]string{"--all"}); return err }}, + // --platform is accepted only by the commands that resolve a + // platform (pull, run); everywhere else a successful parse would + // silently discard it and mislead the caller. {"unpack rejects platform", func() error { _, _, _, err := parseUnpackArgs([]string{"--platform", "linux/amd64", "alpine:3"}) return err - }()}, + }}, {"inspect rejects platform", func() error { _, _, _, err := parseInspectArgs([]string{"--platform", "linux/amd64", "alpine:3"}) return err - }()}, + }}, + {"list rejects platform", func() error { + _, _, err := parseListArgs([]string{"--platform", "linux/amd64"}) + return err + }}, + {"rmi rejects platform", func() error { + _, _, _, err := parseRmiArgs([]string{"--platform", "linux/amd64", "alpine:3"}) + return err + }}, + {"prune rejects platform", func() error { + _, _, err := parsePruneArgs([]string{"--platform", "linux/amd64"}) + return err + }}, } for _, tc := range cases { - if tc.err == nil { - t.Errorf("%s: got nil error", tc.name) - } + t.Run(tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Errorf("%s: got nil error, want parse failure", tc.name) + } + }) + } +} + +func TestParseLifecycleArgs(t *testing.T) { + cf, asJSON, err := parseListArgs([]string{"--store", "/s", "--json"}) + if err != nil { + t.Fatal(err) + } + if cf.store != "/s" || !asJSON { + t.Fatalf("list parse = store %q json %v, want /s true", cf.store, asJSON) + } + + cf, force, ref, err := parseRmiArgs([]string{"--force", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if force != true || ref != "alpine:3" || cf.platform != defaultPlatform { + t.Fatalf("rmi parse = force %v ref %q platform %+v", force, ref, cf.platform) + } + + cf, opts, err := parsePruneArgs([]string{"--cache", "--all", "--dry-run"}) + if err != nil { + t.Fatal(err) + } + if !opts.cache || !opts.all || !opts.dryRun || cf.platform != defaultPlatform { + t.Fatalf("prune parse = opts %+v platform %+v", opts, cf.platform) } } diff --git a/cmd/elfuse-oci/csrun.go b/cmd/elfuse-oci/csrun.go index 39341808..b32ed56d 100644 --- a/cmd/elfuse-oci/csrun.go +++ b/cmd/elfuse-oci/csrun.go @@ -21,6 +21,7 @@ var ( clonefileForRun = unix.Clonefile spawnElfuseWaitForRun = spawnElfuseWait cleanupCloneAndMountForRun = cleanupCloneAndMount + writeKeptSidecarForRun = writeKeptSidecar osExitForRun = os.Exit runNowUnixNano = func() int64 { return time.Now().UnixNano() } ) @@ -52,7 +53,7 @@ func writeKeptSidecar(m *csMount) error { // The unpacked base tree persists in the sparsebundle image file across // attach/detach cycles, so warm re-runs skip the (slow) unpack and pay only the // attach. -func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { +func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { bundle, err := csBundleDirForDigest(cf.store, digest) if err != nil { return nil, "", err @@ -69,7 +70,11 @@ func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size strin } if !published { fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) - if err := unpackImage(s, ref, rootfs); err != nil { + // Unpack the caller's already-resolved image, not a re-resolution of + // ref: the bundle is keyed by digest, so a repull moving the tag + // mid-setup must not fill this digest's sparsebundle with another + // image. + if err := unpackImage(img, rootfs); err != nil { return nil, "", errors.Join(err, closeMount(m)) } } @@ -88,8 +93,8 @@ func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size strin // The clone lives in the same APFS volume as the base tree (clonefile is // intra-volume only), so it is instant and free until the guest writes (COW). // It isolates each run's mutations from the warm base, so re-runs stay clean. -func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { - m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, digest, rf.sparseSize) +func runCaseSensitive(cf commonFlags, s *store, ref string, img v1.Image, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string, refLock *flockFile) error { + m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, img, digest, rf.sparseSize) if err != nil { return err } @@ -99,11 +104,26 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf // Per-run COW clone. --no-clone runs against the base tree directly (mutations // then persist into the warm tree; useful for debugging or when clonefile is - // unavailable). + // unavailable). Liveness no longer depends on a clone directory existing: + // this run holds the bundle's run.lock (via the csMount) for its whole + // lifetime, so prune --cache/rmi --force see the volume busy and leave it + // attached regardless of whether a clone was made, so --no-clone needs + // no placeholder directory. sysroot := baseRootfs - var cloneDir string + cloneDir := filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) + if rf.keepRootfs { + // Record the bundle-level keep FIRST, before the per-clone marker, so + // the two keep records never disagree: if this succeeds but the + // per-clone marker below fails, a cold rmi still refuses to discard the + // bundle without --force. In the reverse order a failed sidecar write + // would leave a sweep-preserved clone that rmi silently discards. It + // also covers the --no-clone --keep case (mutations land in the base + // tree, no clone marker is written at all). + if err := writeKeptSidecarForRun(m); err != nil { + return errors.Join(err, closeMount(m)) + } + } if !rf.noClone { - cloneDir = filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) if err := os.RemoveAll(cloneDir); err != nil { err = fmt.Errorf("remove stale COW clone %s: %w", cloneDir, err) return errors.Join(err, closeMount(m)) @@ -113,6 +133,21 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf return errors.Join(err, closeMount(m)) } sysroot = cloneDir + if rf.keepRootfs { + // Mark the clone so a later prune/rmi sweep preserves it even + // after this run exits and releases run.lock: without the marker + // the sweep, which only runs when no run is live, would reap it. + // The clone is sweep-visible for a moment before the marker + // lands, but the guest has not started (the spawn is below), so + // a crash in that window forfeits only an unused byte-identical + // COW copy; the user-facing keep record is the sidecar above, + // already durable. Marker-first would instead leave a permanent + // orphan marker shielding an unrelated future clone of the same + // name whenever the clone step fails. + if err := writeKeepMarker(cloneDir); err != nil { + return errors.Join(err, cleanupCloneAndMountForRun(cloneDir, rf.keepRootfs, m)) + } + } } // Any failure past this point must tear down the clone and the mount. @@ -131,14 +166,17 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf return fail(err) } - code, err := spawnElfuseWaitForRun(sysroot, spec, m.runLock) + code, err := spawnElfuseWaitForRun(sysroot, spec, m.runLock, refLock) var cleanupErr error if rf.keepRootfs { // --keep leaves the clone and the mount in place for inspection. The // clone lives in the sparsebundle volume, so the mount must stay // attached for it to be reachable on the host; a later run reattaches - // (detaching this stale mount first) and the kept clone reappears. - if cloneDir != "" { + // (detaching this stale mount first) and the kept clone, protected + // by its keep marker from any intervening sweep, reappears. Under + // --no-clone there is no clone to keep (mutations landed in the base + // tree), only the still-attached mount. + if !rf.noClone { fmt.Fprintf(os.Stderr, "kept clone: %s\n", cloneDir) } fmt.Fprintf(os.Stderr, "mount stays attached: %s\n", m.mountPath) diff --git a/cmd/elfuse-oci/csrun_darwin_test.go b/cmd/elfuse-oci/csrun_darwin_test.go index 7a8b9763..488f9fd5 100644 --- a/cmd/elfuse-oci/csrun_darwin_test.go +++ b/cmd/elfuse-oci/csrun_darwin_test.go @@ -6,6 +6,7 @@ package main import ( + "archive/tar" "fmt" "os" "path/filepath" @@ -25,6 +26,7 @@ func withCSRunSeams(t *testing.T) { oldClone := clonefileForRun oldSpawn := spawnElfuseWaitForRun oldCleanup := cleanupCloneAndMountForRun + oldSidecar := writeKeptSidecarForRun oldExit := osExitForRun oldNow := runNowUnixNano t.Cleanup(func() { @@ -32,6 +34,7 @@ func withCSRunSeams(t *testing.T) { clonefileForRun = oldClone spawnElfuseWaitForRun = oldSpawn cleanupCloneAndMountForRun = oldCleanup + writeKeptSidecarForRun = oldSidecar osExitForRun = oldExit runNowUnixNano = oldNow }) @@ -48,7 +51,7 @@ func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { runNowUnixNano = func() int64 { return 123 } expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-123", os.Getpid())) var clonedSrc, clonedDst string - ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { if cf.store != "store" || ref != "local:a" || digest != "sha256:"+strings.Repeat("7", 64) || size != "64m" { t.Fatalf("ensure args = store=%q ref=%q digest=%q size=%q", cf.store, ref, digest, size) } @@ -104,10 +107,12 @@ func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { commonFlags{store: "store"}, &store{}, "local:a", + nil, "sha256:"+strings.Repeat("7", 64), cfg, runFlags{sparseSize: "64m"}, nil, + nil, ) t.Fatalf("runCaseSensitive returned %v, want osExitForRun panic", err) } @@ -119,7 +124,7 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { if err := os.MkdirAll(base, 0o755); err != nil { t.Fatal(err) } - ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { return &csMount{mountPath: mount}, base, nil } clonefileForRun = func(string, string, int) error { @@ -136,6 +141,13 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { t.Fatal("cleanup called with --keep") return nil } + // --no-clone --keep still records the keep beside the bundle so a cold rmi + // refuses to discard the mutated base tree without --force. + sidecarWritten := false + writeKeptSidecarForRun = func(*csMount) error { + sidecarWritten = true + return nil + } osExitForRun = func(code int) { panic(runExitCode(code)) } defer func() { @@ -144,11 +156,14 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { if !ok || code != 0 { t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) } + if !sidecarWritten { + t.Error("--no-clone --keep did not write the kept sidecar") + } }() - err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("8", 64), + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("8", 64), &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, runFlags{noClone: true, keepRootfs: true}, - nil) + nil, nil) t.Fatalf("runCaseSensitive returned %v, want exit panic", err) } @@ -162,7 +177,7 @@ func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { m := &csMount{mountPath: mount} runNowUnixNano = func() int64 { return 456 } expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-456", os.Getpid())) - ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { return m, base, nil } clonefileForRun = func(src, dst string, flags int) error { @@ -185,10 +200,10 @@ func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { } osExitForRun = func(code int) { t.Fatalf("exit called after spec error with code %d", code) } - err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("9", 64), + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("9", 64), &v1.ConfigFile{Config: v1.Config{}}, runFlags{}, - nil) + nil, nil) if err == nil || !strings.Contains(err.Error(), "no command") { t.Fatalf("runCaseSensitive spec err = %v, want no command", err) } @@ -211,8 +226,12 @@ func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T if err != nil { t.Fatal(err) } + img, err := s.image("local:tiny") + if err != nil { + t.Fatal(err) + } - m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", img, digest, "32m") if err != nil { t.Fatalf("ensureCaseSensitiveRootfs: %v", err) } @@ -242,8 +261,12 @@ func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T if err != nil { t.Fatal(err) } + img, err := s.image("local:tiny") + if err != nil { + t.Fatal(err) + } - m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", img, digest, "32m") if err != nil { t.Fatalf("ensureCaseSensitiveRootfs existing: %v", err) } @@ -272,9 +295,21 @@ func TestEnsureCaseSensitiveRootfsClosesMountOnUnpackError(t *testing.T) { t.Setenv("HDIUTIL_DETACH_LOG", detachLog) s := openTestStore(t) - _, _, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:missing", "sha256:"+strings.Repeat("a", 64), "32m") - if err == nil || !strings.Contains(err.Error(), "not pulled") { - t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack not-pulled error", err) + // The image is resolved by the caller now, so an unpack failure has to come + // from the image content, not a missing ref (that is caught upstream in + // resolveImageForUse). A fifo entry is an unsupported special file, so + // unpackImage fails and ensureCaseSensitiveRootfs must still detach the + // mount it attached. + bad := testImageWithLayers(t, testTarLayer(t, + tarEntry{header: tar.Header{Name: "dev/fifo", Typeflag: tar.TypeFifo, Mode: 0o644}})) + digest, err := s.addImage("local:bad", bad) + if err != nil { + t.Fatal(err) + } + + _, _, err = ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:bad", bad, digest, "32m") + if err == nil || !strings.Contains(err.Error(), "fifo") { + t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack fifo error", err) } b, readErr := os.ReadFile(detachLog) if readErr != nil { @@ -328,7 +363,7 @@ func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { t.Fatal(err) } runNowUnixNano = func() int64 { return 789 } - ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref, digest, size string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { if cf.store != s.root || ref != "local:a" || size != "" { t.Fatalf("ensure from cmdRun got store=%q ref=%q size=%q", cf.store, ref, size) } @@ -362,3 +397,118 @@ func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { err := cmdRun([]string{"--store", s.root, "local:a"}) t.Fatalf("cmdRun returned %v, want exit panic", err) } + +// TestRunCaseSensitiveNoCloneCreatesNoMarkerDir pins that a --no-clone run +// leaves no run-- placeholder in the volume: liveness now rides on +// the bundle's run.lock (held via the csMount), not on a marker directory, so +// none is created and none is cleaned up. +func TestRunCaseSensitiveNoCloneCreatesNoMarkerDir(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 789 } + cloneName := filepath.Join(mount, fmt.Sprintf("run-%d-789", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(string, string, int) error { + t.Fatal("clonefile called with --no-clone") + return nil + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec, _ ...*flockFile) (int, error) { + if rootfs != base { + t.Fatalf("spawn rootfs = %q, want base rootfs", rootfs) + } + if _, err := os.Lstat(cloneName); !os.IsNotExist(err) { + t.Fatalf("--no-clone created a placeholder %q: %v, want none", cloneName, err) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(clone string, keep bool, cm *csMount) error { + return removeClone(clone, keep) + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("6", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{noClone: true}, + nil, nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} + +// TestRunCaseSensitiveKeepWritesKeepMarker pins that a --keep run records a +// keep sidecar beside (not inside) its COW clone so a later sweep preserves the +// clone after this run exits and releases run.lock, and so image content or a +// guest cannot forge the keep by writing /.elfuse-keep in the clone. +func TestRunCaseSensitiveKeepWritesKeepMarker(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 321 } + cloneDir := filepath.Join(mount, fmt.Sprintf("run-%d-321", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec, _ ...*flockFile) (int, error) { + if rootfs != cloneDir { + t.Fatalf("spawn rootfs = %q, want clone %q", rootfs, cloneDir) + } + if _, err := os.Stat(cloneKeepMarkerPath(mount, filepath.Base(cloneDir))); err != nil { + t.Fatalf("keep marker during run: %v, want present", err) + } + // The keep record must live OUTSIDE the clone (the guest's /), so + // image content or the guest cannot forge it. + if _, err := os.Stat(filepath.Join(cloneDir, ".elfuse-keep")); !os.IsNotExist(err) { + t.Fatalf("keep record found inside the clone: %v, want only the sidecar", err) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { + t.Fatal("cleanup called with --keep") + return nil + } + sidecarWritten := false + writeKeptSidecarForRun = func(*csMount) error { + sidecarWritten = true + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + // The kept clone with its marker survives: a later sweep skips it. + if listed := listSweepableClones(mount); len(listed) != 0 { + t.Fatalf("listSweepableClones = %v, want the kept clone skipped", listed) + } + if !sidecarWritten { + t.Error("--keep did not write the kept sidecar") + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("6", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{keepRootfs: true}, + nil, nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} diff --git a/cmd/elfuse-oci/csrun_other.go b/cmd/elfuse-oci/csrun_other.go index b16d9b68..d44eb7f1 100644 --- a/cmd/elfuse-oci/csrun_other.go +++ b/cmd/elfuse-oci/csrun_other.go @@ -17,6 +17,6 @@ import ( // reports a clear error and directs the user at --plain-rootfs. This stub // exists so elfuse-oci compiles on Linux, where pull/inspect/unpack and // conformance/interop tests run. -func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { +func runCaseSensitive(cf commonFlags, s *store, ref string, img v1.Image, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string, refLock *flockFile) error { return fmt.Errorf("case-sensitive sparsebundle rootfs requires macOS; pass --plain-rootfs for a plain directory") } diff --git a/cmd/elfuse-oci/gc.go b/cmd/elfuse-oci/gc.go new file mode 100644 index 00000000..9f0adbd7 --- /dev/null +++ b/cmd/elfuse-oci/gc.go @@ -0,0 +1,478 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// rmReport summarizes a reachability-GC pass: the blob digests removed (or that +// would be removed, under --dry-run) and the bytes reclaimed. CacheDropped +// records whether rmi also reclaimed the image's unpacked cache, so the command +// can report it rather than deleting a large warm tree silently. +type rmReport struct { + Ref string + Blobs []string + Bytes int64 + CacheDropped bool +} + +// gc runs a reachability pass over the OCI image-layout store and removes any +// sha256 blob that is not reachable from an index.json manifest descriptor. +// Reachability follows manifest/index descriptors recursively, then marks image +// manifests, configs, and layers live. When dryRun is set, gc reports what it +// would reclaim and deletes nothing. +// +// gc is the shared engine behind `rmi` (called after a manifest descriptor is +// removed from index.json, so the dropped image's config/layers surface as +// unreachable) and `prune` (a standalone sweep that reclaims retag/partial-pull +// orphans). It also reclaims stale temporary blob files left by interrupted +// writes under blobs/sha256/; those filenames are not valid sha256 digests and +// make layout.Path.GarbageCollect abort before it can sweep anything. +// +// It first reconciles index.json against the pin set (pruneUnpinnedDescriptors) +// so a re-pulled mutable tag's orphaned prior descriptor stops keeping its +// blobs live; the blob sweep then reclaims them. A dry run reconciles nothing +// and reports against the current index. +func (s *store) gc(dryRun bool) (rmReport, error) { + // One read of each liveness root serves the whole pass: the caller holds + // the store lock, so neither can change under us. The busy set is + // collected before any descriptor is dropped, because a busy digest's + // descriptor is exactly what must survive the reconcile. + pins, err := s.loadPins() + if err != nil { + return rmReport{}, err + } + busy, err := s.busyDigests() + if err != nil { + return rmReport{}, err + } + // Resolve every liveness root before reconciling index.json: a stale or + // malformed pin must fail the pass closed, not after descriptors it can + // no longer justify were already dropped. The old order committed the + // reconcile's rewrite first, so one bad pin (rmi's documented crash + // window is a reachable producer) shrank the index and then blocked + // every later prune and rmi at this same error. Reachability depends + // only on pins and busy, so computing it first changes no verdict. + live, err := s.liveBlobDigests(pins, busy) + if err != nil { + return rmReport{}, fmt.Errorf("gc: compute reachability: %w", err) + } + if !dryRun { + if err := s.pruneUnpinnedDescriptors(pins, busy); err != nil { + return rmReport{}, fmt.Errorf("gc: reconcile descriptors: %w", err) + } + } + var rep rmReport + blobs, err := s.localBlobFiles() + if err != nil { + return rep, err + } + for _, b := range blobs { + if !b.malformed && live[b.digest] { + continue + } + fi, err := os.Stat(b.path) + if os.IsNotExist(err) { + continue // raced or already removed + } + if err != nil { + return rep, err + } + rep.Blobs = append(rep.Blobs, b.digest) + rep.Bytes += fi.Size() + if !dryRun { + err := b.remove(s) + if err != nil && !os.IsNotExist(err) { + return rep, err + } + } + } + return rep, nil +} + +type localBlob struct { + path string + digest string + hash v1.Hash + malformed bool +} + +func (b localBlob) remove(s *store) error { + if b.malformed { + return os.Remove(b.path) + } + return s.path.RemoveBlob(b.hash) +} + +// liveBlobDigests roots reachability at the refs.json pin set, not at every +// index.json descriptor: an unpinned descriptor (a re-pulled tag's orphaned +// prior manifest, or a pull that appended a manifest but crashed before +// pinning) must not keep its blobs live. Each pinned digest's manifest, +// config, and layers are marked live. A real gc calls pruneUnpinnedDescriptors +// first so index.json never keeps referencing a manifest whose blobs this +// sweep reclaims; a dry run skips that but, rooting here at pins too, still +// reports exactly the blobs a real run would reclaim. +// Digests whose cache a live run holds (busy) are the second root: see +// busyDigests for why a pin set alone is not enough. +func (s *store) liveBlobDigests(pins refPins, busy map[string]bool) (map[string]bool, error) { + live := map[string]bool{} + seen := map[string]bool{} + mark := func(digest string) error { + if seen[digest] { + return nil // shared manifest already marked + } + seen[digest] = true + h, err := v1.NewHash(digest) + if err != nil { + return err + } + img, err := s.path.Image(h) + if err != nil { + return fmt.Errorf("gc: read live manifest %s: %w", digest, err) + } + return markLiveImage(img, live) + } + for _, digest := range pins { + if err := mark(digest); err != nil { + return nil, err + } + } + for digest := range busy { + if err := mark(digest); err != nil { + return nil, err + } + } + return live, nil +} + +func markLiveImage(image v1.Image, live map[string]bool) error { + h, err := image.Digest() + if err != nil { + return err + } + live[h.String()] = true + + h, err = image.ConfigName() + if err != nil { + return err + } + live[h.String()] = true + + layers, err := image.Layers() + if err != nil { + return err + } + for _, layer := range layers { + h, err := layer.Digest() + if err != nil { + return err + } + live[h.String()] = true + } + return nil +} + +func (s *store) localBlobFiles() ([]localBlob, error) { + base := filepath.Join(s.root, "blobs", "sha256") + entries, err := os.ReadDir(base) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + blobs := make([]localBlob, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + continue + } + info, err := e.Info() + if err != nil { + // The entry vanished between ReadDir and Info (a concurrent + // rmi/prune already reclaimed it); skip it rather than aborting + // the whole GC pass, matching the IsNotExist tolerance elsewhere + // in this file. + if os.IsNotExist(err) { + continue + } + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + + name := e.Name() + path := filepath.Join(base, name) + if validSHA256Hex(name) { + h := v1.Hash{Algorithm: "sha256", Hex: name} + blobs = append(blobs, localBlob{path: path, digest: h.String(), hash: h}) + continue + } + blobs = append(blobs, localBlob{path: path, digest: "sha256:" + name, malformed: true}) + } + return blobs, nil +} + +func validSHA256Hex(s string) bool { + return len(s) == 64 && isLowerHex(s) +} + +// isLowerHex reports whether s contains only lowercase hex digits. Callers +// bound the length themselves (the empty string passes). +func isLowerHex(s string) bool { + for _, c := range s { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return false + } + } + return true +} + +// rewriteIndexManifests rewrites index.json to the descriptors keep returns +// true for, writing atomically and durably (writeFileDurable), not through the +// layout package's in-place os.WriteFile: rmi commits an index.json change +// before dropping the pin from refs.json, and that ordering only survives a +// crash if each write is individually durable. A plain truncate-in-place write +// could leave a torn index.json (store unreadable) or a non-durable change a +// crash reverts after the fsynced pin drop, stranding a descriptor and every +// blob it keeps live. It reports whether anything changed so callers can skip +// the write (and its fsync) when the index already matches. +func (s *store) rewriteIndexManifests(keep func(v1.Descriptor) bool) (bool, error) { + idx, err := s.path.ImageIndex() + if err != nil { + return false, fmt.Errorf("rewrite index: read index: %w", err) + } + im, err := idx.IndexManifest() + if err != nil { + return false, fmt.Errorf("rewrite index: parse index: %w", err) + } + kept := make([]v1.Descriptor, 0, len(im.Manifests)) + for _, desc := range im.Manifests { + if keep(desc) { + kept = append(kept, desc) + } + } + if len(kept) == len(im.Manifests) { + return false, nil + } + next := *im + next.Manifests = kept + b, err := json.Marshal(&next) + if err != nil { + return false, fmt.Errorf("rewrite index: marshal index: %w", err) + } + if err := writeFileDurable(filepath.Join(s.root, "index.json"), b, 0o644); err != nil { + return false, err + } + return true, nil +} + +// removeManifestDescriptor removes the manifest descriptor with the given digest +// from index.json. It does not touch the manifest's config or layer blobs; a +// subsequent gc pass reclaims them once they are unreachable from every +// remaining descriptor, so a blob shared with another still-pinned image is +// kept. +func (s *store) removeManifestDescriptor(digest string) error { + h, err := v1.NewHash(digest) + if err != nil { + return fmt.Errorf("remove manifest descriptor: %w", err) + } + _, err = s.rewriteIndexManifests(func(desc v1.Descriptor) bool { + return desc.Digest != h + }) + return err +} + +// pruneUnpinnedDescriptors removes index.json manifest descriptors that no +// refs.json pin references. Reachability GC roots liveness at index.json +// descriptors, so a re-pull that moves a mutable tag to a new digest would +// otherwise leak the prior image forever: its descriptor stays in the index +// (keeping every blob live) even though no ref pins it, and rmi cannot target +// a digest it can no longer resolve through a pin. Reconciling the index to +// the pin set before the blob sweep makes that orphan reclaimable. The store +// only ever appends single-platform image manifests (addImage), and pins point +// at them directly, so a top-level descriptor absent from the pin set is +// genuinely unreferenced. +// A busy digest is kept as well: an unpinned digest a live run still uses is +// reachable in practice even though no ref names it, and dropping its +// descriptor would make the following blob sweep reclaim the blobs that run is +// reading. +func (s *store) pruneUnpinnedDescriptors(pins refPins, busy map[string]bool) error { + pinned := make(map[string]bool, len(pins)) + for _, digest := range pins { + pinned[digest] = true + } + _, err := s.rewriteIndexManifests(func(desc v1.Descriptor) bool { + d := desc.Digest.String() + return pinned[d] || busy[d] + }) + return err +} + +// busyDigests returns the index.json manifest digests whose unpacked cache a +// live run currently holds, keyed by digest string. +// +// This is the second liveness root, alongside the pin set. resolveImageForUse +// takes a digest's per-digest run lock inside the store lock and then releases +// the store lock, so a run goes on reading that manifest's config and layer +// blobs with no store lock held. A concurrent pull may repin the tag in that +// window, leaving the running digest unpinned: rooting only at pins would let +// the next prune or rmi reclaim the descriptor and blobs out from under the +// live guest. The lock is what proves the digest is in use, the same evidence +// the cache sweeps already act on. +func (s *store) busyDigests() (map[string]bool, error) { + idx, err := s.path.ImageIndex() + if err != nil { + return nil, fmt.Errorf("gc: read index: %w", err) + } + im, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("gc: parse index: %w", err) + } + busy := map[string]bool{} + for _, desc := range im.Manifests { + d := desc.Digest.String() + if busy[d] { + continue + } + if cacheBusyForDigest(s.root, d) { + busy[d] = true + } + } + return busy, nil +} + +// liveCacheKeys returns the set of digest cache keys for every currently pinned +// ref. pruneCaches uses it to keep digest-keyed rootfs/sparsebundle caches that +// are still reachable through refs.json. +func (s *store) liveCacheKeys() (map[string]bool, error) { + pins, err := s.loadPins() + if err != nil { + return nil, err + } + m := make(map[string]bool, len(pins)) + for _, digest := range pins { + key, err := cacheKeyForDigest(digest) + if err != nil { + return nil, err + } + m[key] = true + } + return m, nil +} + +// rmi removes one selected ref from the store. The target may be an exact ref +// or a unique sha256 digest prefix. If other refs still pin the same manifest +// digest, only the resolved pin is dropped: the shared descriptor, blobs, and +// digest-keyed caches stay live through the remaining refs. If this was the last +// pin for the digest, rmi removes the manifest descriptor, GCs the +// now-unreachable blobs, and reclaims the image's unpacked cache: the cache is +// derived state subordinate to the image, so it goes with it instead of being +// left as an orphan only prune --cache could reap. Two safety rules survive +// force: a cache whose volume a live run still uses is never dropped (even with +// force), and a cache holding run --keep retained output refuses without force +// so a deliberate keep is not discarded silently. +func (s *store) rmi(target string, force bool) (rmReport, error) { + // The store lock spans the whole resolve-modify-GC sequence so an rmi + // cannot interleave with a concurrent pull's check-append-pin (or another + // rmi) and lose one side's refs.json/index.json update. + unlock, err := s.lock() + if err != nil { + return rmReport{}, err + } + defer unlock() + pins, err := s.loadPins() + if err != nil { + return rmReport{}, err + } + ref, digest, err := resolvePinnedTarget(pins, target) + if err != nil { + return rmReport{}, err + } + + lastPin := true + for otherRef, otherDigest := range pins { + if otherRef != ref && otherDigest == digest { + lastPin = false + break + } + } + + var cacheDropped bool + if lastPin { + // A live or starting run holds the digest's reference lock (the plain + // rootfs run lock, taken by resolveImageForUse for every run path + // before any cache dir or bundle exists). Probe it directly rather + // than inferring liveness from cacheExists: a cold run that has + // resolved the image but not yet unpacked has no cache to detect, yet + // its descriptor and blobs must survive. rmi holds the store lock + // throughout, and a run claims the reference lock only while holding + // that same store lock (resolveImageForUse), so the set of holders is + // frozen here: a busy probe cannot be a run that is about to appear. + // Refuse regardless of force; force discards derived state, it does + // not evict a running guest. + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return rmReport{}, err + } + if rootfsCacheBusy(rootfs) { + return rmReport{}, fmt.Errorf("rmi: %q is in use by a live run; stop it before removing the image", ref) + } + // A run --keep clone is deliberately retained output living in the + // cache; refuse to discard it without force. Everything else in the + // cache is derived state reclaimed with the image below. + kept, err := cacheHasKeptData(s.root, digest) + if err != nil { + return rmReport{}, fmt.Errorf("rmi: inspect cache for %q: %w", ref, err) + } + if kept && !force { + return rmReport{}, fmt.Errorf("rmi: %q has retained run --keep output; pass --force to discard it, or inspect it with a fresh run then 'elfuse-oci prune --cache'", ref) + } + // Reclaim the unpacked cache as part of removing the image. removeRefCaches + // fails closed if a live run still holds the volume (even with force), so + // this never yanks a rootfs out from under a running guest. + exists, err := cacheExists(s.root, digest) + if err != nil { + return rmReport{}, fmt.Errorf("rmi: probe cache for %q: %w", ref, err) + } + if exists { + if err := removeRefCaches(s, digest); err != nil { + return rmReport{}, fmt.Errorf("rmi: drop cache for %q: %w", ref, err) + } + cacheDropped = true + } + } + + // On a last-pin removal, update index.json BEFORE committing the pin + // removal to refs.json. In the opposite order, a failure between the two + // writes strands the manifest: the ref is gone from refs.json (so rmi can + // no longer resolve it) while the descriptor keeps every blob live, and + // prune never removes descriptors; the image becomes unreclaimable. In + // this order the same crash window leaves a stale pin over a removed + // descriptor, which a retried rmi resolves and finishes + // (RemoveDescriptors is a filter, so re-removing is a no-op). + if lastPin { + if err := s.removeManifestDescriptor(digest); err != nil { + return rmReport{}, fmt.Errorf("rmi: remove manifest descriptor for %q: %w", ref, err) + } + } + delete(pins, ref) + if err := s.savePins(pins); err != nil { + return rmReport{}, err + } + if !lastPin { + return rmReport{Ref: ref}, nil + } + rep, err := s.gc(false) + rep.Ref = ref + rep.CacheDropped = cacheDropped + return rep, err +} diff --git a/cmd/elfuse-oci/inspect.go b/cmd/elfuse-oci/inspect.go index 0a33591e..80becc4c 100644 --- a/cmd/elfuse-oci/inspect.go +++ b/cmd/elfuse-oci/inspect.go @@ -12,6 +12,16 @@ import ( // inspect prints a stored image's manifest + config. With --json the raw config // JSON is emitted (one object); otherwise a human-readable summary. func inspect(w io.Writer, s *store, ref string, asJSON bool) error { + // Hold the store lock across resolution and every metadata read, as list + // does: ConfigFile/Layers are lazy blob reads, so a concurrent rmi on the + // last ref could delete the config/layer blobs between resolving the pin + // and reading them, failing inspect partway through. + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + img, err := s.image(ref) if err != nil { return err @@ -47,9 +57,10 @@ func inspect(w io.Writer, s *store, ref string, asJSON bool) error { bw := bufio.NewWriter(w) w = bw + platform := Platform{OS: cfg.OS, Arch: cfg.Architecture, Variant: cfg.Variant}.String() fmt.Fprintf(w, "%-12s %s\n", "Ref:", ref) fmt.Fprintf(w, "%-12s %s\n", "Digest:", d) - fmt.Fprintf(w, "%-12s %s/%s\n", "Platform:", cfg.OS, cfg.Architecture) + fmt.Fprintf(w, "%-12s %s\n", "Platform:", platform) if !cfg.Created.IsZero() { fmt.Fprintf(w, "%-12s %s\n", "Created:", formatCreated(cfg)) } diff --git a/cmd/elfuse-oci/inspect_test.go b/cmd/elfuse-oci/inspect_test.go index 0a2b240b..c2196069 100644 --- a/cmd/elfuse-oci/inspect_test.go +++ b/cmd/elfuse-oci/inspect_test.go @@ -113,6 +113,80 @@ func TestInspectHumanIncludesCreatedEnvAndLayers(t *testing.T) { } } +// TestInspectHoldsStoreLock: inspect resolves and reads image metadata +// under the store lock, so a concurrent rmi cannot delete blobs mid-inspect. +// The proof is that inspect blocks while another holder keeps the lock and +// completes once it is released. +func TestInspectHoldsStoreLock(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:tiny", tinyImage(t)); err != nil { + t.Fatal(err) + } + unlock, err := s.lock() + if err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + var buf bytes.Buffer + done <- inspect(&buf, s, "local:tiny", false) + }() + + select { + case <-done: + unlock() + t.Fatal("inspect completed while the store lock was held; it does not take the lock") + case <-time.After(150 * time.Millisecond): + } + + unlock() + select { + case err := <-done: + if err != nil { + t.Fatalf("inspect after unlock: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("inspect did not complete after the store lock was released") + } +} + +// TestInspectAndListAgreeOnVariant: an image with a platform variant +// (linux/arm/v7) must show the full os/arch/variant in inspect, matching list, +// not the truncated os/arch. +func TestInspectAndListAgreeOnVariant(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cfg.Architecture = "arm" + cfg.Variant = "v7" + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:v7", img); err != nil { + t.Fatal(err) + } + + var ins bytes.Buffer + if err := inspect(&ins, s, "local:v7", false); err != nil { + t.Fatal(err) + } + if !strings.Contains(ins.String(), "linux/arm/v7") { + t.Fatalf("inspect platform missing variant:\n%s", ins.String()) + } + var lst bytes.Buffer + if err := list(&lst, s, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(lst.String(), "linux/arm/v7") { + t.Fatalf("list platform missing variant:\n%s", lst.String()) + } +} + func TestInspectMissingRefAndConfigErrors(t *testing.T) { s := openTestStore(t) if err := inspect(&bytes.Buffer{}, s, "local:missing", false); err == nil || !strings.Contains(err.Error(), "not pulled") { @@ -153,3 +227,19 @@ func TestInspectPropagatesWriteErrors(t *testing.T) { t.Fatal("inspect summary exited clean on a failed write, want error") } } + +// TestListPropagatesWriteErrors pins the same contract for list: a closed +// pipe or full disk behind a redirect must not exit 0 with truncated +// output a script would consume as complete. +func TestListPropagatesWriteErrors(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:tiny", tinyImage(t)); err != nil { + t.Fatal(err) + } + if err := list(failingWriter{}, s, true); err == nil { + t.Fatal("list --json exited clean on a failed write, want error") + } + if err := list(failingWriter{}, s, false); err == nil { + t.Fatal("list table exited clean on a failed write, want error") + } +} diff --git a/cmd/elfuse-oci/lifecycle_darwin_test.go b/cmd/elfuse-oci/lifecycle_darwin_test.go new file mode 100644 index 00000000..70110c56 --- /dev/null +++ b/cmd/elfuse-oci/lifecycle_darwin_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The darwin-only sparsebundle lifecycle (prune --cache detaching a stale mount +// and reaping abandoned COW clones, rmi --force dropping a mounted bundle) +// cannot run in hosted CI: it needs hdiutil + APFS + a real attach. The +// cross-platform pieces it rests on (listSweepableClones, csBundleBusy, the +// bundle flocks, pruneCaches, pruneRootfsCaches) are covered in +// lifecycle_test.go and bundlelock_test.go. This file adds the darwin-only +// round-trip behind ELFUSE_OCI_DARWIN_CS=1 so a Mac operator can opt in; +// without the flag it skips, so `go test ./cmd/elfuse-oci/` stays green +// everywhere by default. + +// TestDarwinCSSweep exercises the crash-recovery path prune --cache runs per +// sparsebundle bundle: while a live run holds the bundle's run.lock the volume +// is reported busy and stays attached; once that run is gone the next sweep +// reaps the abandoned (unmarked) clone, preserves a --keep-marked clone, and +// detaches, after which removeRefCaches drops the whole bundle. Gated because +// it provisions a real APFS sparsebundle via hdiutil. +func TestDarwinCSSweep(t *testing.T) { + if os.Getenv("ELFUSE_OCI_DARWIN_CS") == "" { + t.Skip("set ELFUSE_OCI_DARWIN_CS=1 to exercise the darwin sparsebundle sweep (needs hdiutil + APFS)") + } + + s := openTestStore(t) + digest := "sha256:" + strings.Repeat("1", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + // provision returns a mount holding run.lock shared; this stands in for + // the live run. Drop owned so Close does not detach; we release the + // liveness lock explicitly to simulate the run exiting. + m, err := provisionCaseSensitive(bundle, mnt, "32m") + if err != nil { + t.Fatalf("provision: %v", err) + } + m.owned = false + t.Cleanup(func() { + if isMountPoint(mnt) { + _ = detachForce(mnt) + } + }) + + // Plant an abandoned clone (reapable) and a --keep-marked clone (preserved). + reapClone := filepath.Join(mnt, "run-1-1") + keepClone := filepath.Join(mnt, "run-2-2") + for _, p := range []string{reapClone, keepClone} { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + } + if err := writeKeepMarker(keepClone); err != nil { + t.Fatal(err) + } + if !isMountPoint(mnt) { + t.Fatalf("mnt %s not attached after provision", mnt) + } + + // First sweep: the live run still holds run.lock, so the bundle is busy; + // nothing is reaped and the volume stays attached. + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle: %v", err) + } + unlock() + if !busy { + t.Fatal("sweepCSBundle busy = false while a live run holds run.lock") + } + if len(reaped) != 0 { + t.Fatalf("sweepCSBundle reaped = %v while busy, want none", reaped) + } + if !isMountPoint(mnt) { + t.Fatal("volume detached although a live run remains") + } + + // The live run exits: release its run.lock. + if err := m.runLock.Close(); err != nil { + t.Fatal(err) + } + + // Second sweep, idle now: the unmarked clone is reaped, the kept clone + // preserved, and the stale mount detached. + reaped, busy, unlock, err = sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle after run exit: %v", err) + } + unlock() + if busy { + t.Fatalf("second sweep reported busy, want idle") + } + if len(reaped) != 1 || reaped[0] != reapClone { + t.Fatalf("second sweep reaped = %v, want [%s]", reaped, reapClone) + } + if isMountPoint(mnt) { + t.Errorf("mnt still attached after idle sweepCSBundle, want detached") + } + + // removeRefCaches should now delete the whole bundle directory, kept clone + // and all. + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Errorf("bundle dir after removeRefCaches: %v, want IsNotExist", err) + } +} diff --git a/cmd/elfuse-oci/lifecycle_test.go b/cmd/elfuse-oci/lifecycle_test.go new file mode 100644 index 00000000..f17a7770 --- /dev/null +++ b/cmd/elfuse-oci/lifecycle_test.go @@ -0,0 +1,1392 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +func firstLayerDigest(t *testing.T, img v1.Image) v1.Hash { + t.Helper() + ls, err := img.Layers() + if err != nil || len(ls) == 0 { + t.Fatalf("layers: %v (n=%d)", err, len(ls)) + } + d, err := ls[0].Digest() + if err != nil { + t.Fatal(err) + } + return d +} + +func indexManifestCount(t *testing.T, root string) int { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Manifests []json.RawMessage `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json unmarshal: %v", err) + } + return len(idx.Manifests) +} + +func rootfsForDigest(t *testing.T, s *store, digest string) string { + t.Helper() + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + return rootfs +} + +func rootfsForImage(t *testing.T, s *store, img v1.Image) string { + t.Helper() + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + return rootfsForDigest(t, s, d.String()) +} + +// TestPruneReclaimsRepulledTagPriorImage: re-pulling a mutable tag to +// a new digest must not leak the prior image. Reachability GC roots at the pin +// set, and gc reconciles index.json to it, so after the tag moves A->B a prune +// TestCSBundleBusyProbeCreatesNoState pins the probe's read-only contract: +// run.lock is opened without O_CREATE, a missing file means no holder +// (holders create it), and any other failure still fails closed to busy. +// The old probe created run.lock in every bundle a prune --dry-run merely +// looked at, mutating the store the docstring promised to leave alone. +func TestCSBundleBusyProbeCreatesNoState(t *testing.T) { + bundle := t.TempDir() + if csBundleBusy(bundle) { + t.Error("bundle without run.lock reads busy, want idle") + } + if _, err := os.Lstat(runLockPath(bundle)); !os.IsNotExist(err) { + t.Errorf("busy probe created run.lock: %v", err) + } +} + +// TestPruneCacheSweepsOrphanLockFiles pins that prune --cache reclaims a +// sibling .lock whose cache dir does not exist. Every case-sensitive +// run takes the per-digest reference lock and so creates the lock file +// without ever creating the plain cache dir, and a failed unpack leaves one +// too; the old sweep visited directories only, so these zero-byte orphans +// accumulated without bound. A lock whose holder is live stays. +func TestPruneCacheSweepsOrphanLockFiles(t *testing.T) { + s := openTestStore(t) + base := filepath.Join(s.root, rootfsCacheDirName, "sha256") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + orphan := filepath.Join(base, strings.Repeat("aa", 32)+".lock") + if err := os.WriteFile(orphan, nil, 0o644); err != nil { + t.Fatal(err) + } + heldDir := filepath.Join(base, strings.Repeat("bb", 32)) + hold, err := acquireRootfsRunLock(heldDir) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + + if err := cmdPrune([]string{"--store", s.root, "--cache"}); err != nil { + t.Fatalf("prune --cache: %v", err) + } + if _, err := os.Lstat(orphan); !os.IsNotExist(err) { + t.Errorf("orphan lock survived prune --cache: %v", err) + } + if _, err := os.Lstat(rootfsRunLockPath(heldDir)); err != nil { + t.Errorf("held lock was swept: %v", err) + } +} + +// TestCacheExistsDanglingSymlink pins the Lstat probe: unpack and run refuse +// a symlink at the cache path, so a dangling link planted there must read as +// present (and thus removable by rmi), not vanish behind Stat's follow. The +// old probe followed the link, answered absent forever, and rmi left the +// entry wedged in the store. +func TestCacheExistsDanglingSymlink(t *testing.T) { + root := t.TempDir() + digest := "sha256:" + strings.Repeat("ab", 32) + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(rootfs), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(t.TempDir(), "gone"), rootfs); err != nil { + t.Fatal(err) + } + exists, err := cacheExists(root, digest) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Error("dangling symlink at the cache path reads as absent") + } +} + +// TestPruneFailsClosedOnUnresolvablePin pins that gc resolves every liveness +// root before reconciling index.json: a pin whose digest cannot be resolved +// must fail the pass with the descriptors intact. The old order committed +// the descriptor rewrite first, so a malformed pin dropped descriptors it +// could no longer justify, and the pass then failed anyway, permanently +// blocking every later prune and rmi at the same error (the crash window +// rmi documents is a reachable producer of stale pins). +func TestPruneFailsClosedOnUnresolvablePin(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + // Repoint the only pin at a digest NewHash rejects: the reconcile then + // sees no valid pin covering A's descriptor while reachability errors. + if err := s.pin("local:a", "sha256:not-hex"); err != nil { + t.Fatal(err) + } + if err := cmdPrune([]string{"--store", s.root}); err == nil { + t.Fatal("prune with an unresolvable pin succeeded, want error") + } + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index has %d descriptors after the failed prune, want 1 (drop must not precede validation)", n) + } +} + +// removes A's orphaned descriptor and its unique blobs (manifest, config) while +// keeping the layer blob B still shares, and B stays intact. +func TestPruneReclaimsRepulledTagPriorImage(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + + manifestA, err := imgA.Digest() + if err != nil { + t.Fatal(err) + } + configA, err := imgA.ConfigName() + if err != nil { + t.Fatal(err) + } + sharedLayer := firstLayerDigest(t, imgA) + if got := firstLayerDigest(t, imgB); got != sharedLayer { + t.Fatalf("test images do not share a layer: A=%s B=%s", sharedLayer, got) + } + + if _, err := s.addImage("local:tag", imgA); err != nil { + t.Fatal(err) + } + digestB, err := s.addImage("local:tag", imgB) // repin tag A->B; A now orphaned + if err != nil { + t.Fatal(err) + } + if n := indexManifestCount(t, s.root); n != 2 { + t.Fatalf("index has %d descriptors after repull, want 2 (A leaked, B live)", n) + } + + if err := cmdPrune([]string{"--store", s.root}); err != nil { + t.Fatalf("prune: %v", err) + } + + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index has %d descriptors after prune, want 1 (A reclaimed)", n) + } + for _, gone := range []v1.Hash{manifestA, configA} { + if _, err := os.Stat(blobPath(s.root, gone.String())); !os.IsNotExist(err) { + t.Errorf("orphaned blob %s still present after prune: %v", gone, err) + } + } + if _, err := os.Stat(blobPath(s.root, sharedLayer.String())); err != nil { + t.Errorf("shared layer %s reclaimed although B still references it: %v", sharedLayer, err) + } + got, err := s.digestFor("local:tag") + if err != nil || got != digestB { + t.Fatalf("tag resolves to %q err=%v; want B %s", got, err, digestB) + } + if _, err := s.image("local:tag"); err != nil { + t.Fatalf("image B unreadable after prune: %v", err) + } +} + +// TestPruneKeepsBlobsOfBusyUnpinnedDigest: blob reachability must root at the +// per-digest run locks as well as the pin set. resolveImageForUse takes a +// digest's run lock inside the store lock and then releases the store lock, +// so a run keeps reading that manifest's config and layers with no store lock +// held. If a concurrent pull repins the tag meanwhile, the running digest is +// no longer pinned, and a prune rooting only at pins reclaims the very blobs +// that run is still reading: the guest then fails mid-setup with a missing +// blob. The cache sweep already skips busy caches (TestPruneCacheHonorsRoot +// fsRunLock); this pins the same rule for blobs. +func TestPruneKeepsBlobsOfBusyUnpinnedDigest(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + + manifestA, err := imgA.Digest() + if err != nil { + t.Fatal(err) + } + configA, err := imgA.ConfigName() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tag", imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tag", imgB); err != nil { // tag moves A->B + t.Fatal(err) + } + + // Stand in for the live run: its cache dir exists and it holds the + // digest's run lock, exactly as resolveImageForUse leaves things while + // the run reads A's blobs. + cache := rootfsForImage(t, s, imgA) + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + lock, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + + if err := cmdPrune([]string{"--store", s.root}); err != nil { + t.Fatalf("prune: %v", err) + } + + for _, keep := range []v1.Hash{manifestA, configA} { + if _, err := os.Stat(blobPath(s.root, keep.String())); err != nil { + t.Errorf("blob %s of the busy digest reclaimed: %v", keep, err) + } + } + if _, err := os.Stat(blobPath(s.root, firstLayerDigest(t, imgA).String())); err != nil { + t.Errorf("layer blob of the busy digest reclaimed: %v", err) + } +} + +// --- list ------------------------------------------------------------------- + +func TestListEmpty(t *testing.T) { + s := openTestStore(t) + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + if buf.Len() != 0 { + t.Errorf("human list of empty store produced output %q, want none", buf.String()) + } + var jbuf bytes.Buffer + if err := list(&jbuf, s, true); err != nil { + t.Fatal(err) + } + if strings.TrimSpace(jbuf.String()) != "[]" { + t.Errorf("json list of empty store = %q, want []", jbuf.String()) + } +} + +func TestListShape(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + out := buf.String() + // CREATED belongs to the human table too: JSON and inspect expose the + // creation time, and the table dropping a computed column made the two + // output modes disagree. + for _, want := range []string{"local:a", "linux/arm64", "CREATED"} { + if !strings.Contains(out, want) { + t.Errorf("human list missing %q in:\n%s", want, out) + } + } + + var jbuf bytes.Buffer + if err := list(&jbuf, s, true); err != nil { + t.Fatal(err) + } + var entries []listEntry + if err := json.Unmarshal(jbuf.Bytes(), &entries); err != nil { + t.Fatalf("json list unmarshal: %v (raw %q)", err, jbuf.String()) + } + if len(entries) != 1 || entries[0].Ref != "local:a" { + t.Fatalf("json list entries = %+v, want one local:a", entries) + } + e := entries[0] + if !strings.HasPrefix(e.Digest, "sha256:") { + t.Errorf("json digest = %q, want sha256: prefix", e.Digest) + } + if e.Platform != "linux/arm64" { + t.Errorf("json platform = %q, want linux/arm64", e.Platform) + } + if e.Layers != 1 { + t.Errorf("json layers = %d, want 1", e.Layers) + } + if e.Size <= 0 { + t.Errorf("json size = %d, want > 0 (compressed layer size)", e.Size) + } +} + +func TestListCorruptConfigErrors(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + config, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if err := os.Remove(blobPath(s.root, config.String())); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = list(&buf, s, false) + if err == nil || !strings.Contains(err.Error(), "list: local:a: config") { + t.Fatalf("list err = %v, want local:a config error", err) + } +} + +func TestListMultipleRefsSorted(t *testing.T) { + s := openTestStore(t) + for _, ref := range []string{"local:b", "local:a"} { + if _, err := s.addImage(ref, buildImage(t, []string{ref})); err != nil { + t.Fatal(err) + } + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + out := buf.String() + if i, j := strings.Index(out, "local:a"), strings.Index(out, "local:b"); i < 0 || j < 0 || i > j { + t.Errorf("list not sorted by ref:\n%s", out) + } +} + +// --- rmi -------------------------------------------------------------------- + +func TestRmiDropsPinAndBlobs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi: %v", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("digestFor after rmi succeeded, want not-pulled error") + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); !os.IsNotExist(err) { + t.Errorf("blob %s after rmi: %v, want IsNotExist", d, err) + } + } + if n := indexManifestCount(t, s.root); n != 0 { + t.Errorf("index.json after rmi has %d manifest descriptors, want 0", n) + } +} + +func TestRmiByRefDropsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi by ref with stale temp blob: %v", err) + } + if !slices.Contains(rep.Blobs, stale) { + t.Fatalf("rmi report blobs = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); !os.IsNotExist(err) { + t.Fatalf("stale temp blob after rmi by ref: %v, want IsNotExist", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after rmi by ref") + } +} + +func TestRmiKeepsSharedBlobs(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + sharedLayer := firstLayerDigest(t, imgA) // same content -> same digest as imgB's layer + if _, err := s.addImage("local:a", imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", imgB); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a: %v", err) + } + if _, err := os.Stat(blobPath(s.root, sharedLayer.String())); err != nil { + t.Errorf("shared layer blob after rmi local:a: %v, want present (still reachable via local:b)", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } + if img, err := s.image("local:b"); err != nil { + t.Errorf("s.image(local:b) after rmi local:a: %v", err) + } else if ls, _ := img.Layers(); len(ls) != 1 { + t.Errorf("local:b layers after rmi local:a = %d, want 1", len(ls)) + } +} + +func TestRmiKeepsSameDigestPinnedByOtherRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index manifest count before rmi = %d, want 1", n) + } + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi local:a: %v", err) + } + if len(rep.Blobs) != 0 || rep.Bytes != 0 { + t.Fatalf("rmi local:a report = %+v, want no blobs removed", rep) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("local:a pin still present after rmi, want gone") + } + if got, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } else if got != manifest.String() { + t.Fatalf("local:b digest = %s, want %s", got, manifest) + } + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index manifest count after rmi local:a = %d, want 1", n) + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); err != nil { + t.Errorf("blob %s after rmi local:a: %v, want present", d, err) + } + } + if _, err := s.image("local:b"); err != nil { + t.Fatalf("s.image(local:b) after rmi local:a: %v", err) + } +} + +func TestRmiAbsentRefErrors(t *testing.T) { + s := openTestStore(t) + _, err := s.rmi("local:never", false) + if err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Errorf("rmi absent ref err = %v, want an error mentioning \"not pulled\"", err) + } +} + +func TestRmiByDigestDropsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.rmi(shortDigest(manifest.String()), false) + if err != nil { + t.Fatalf("rmi by digest with stale temp blob: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if !slices.Contains(rep.Blobs, stale) { + t.Fatalf("rmi report blobs = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); !os.IsNotExist(err) { + t.Fatalf("stale temp blob after rmi by digest: %v, want IsNotExist", err) + } +} + +func TestRmiAcceptsDigestFromList(t *testing.T) { + for _, tc := range []struct { + name string + target func(string) string + }{ + {"short hex", shortDigest}, + {"full digest", func(d string) string { return d }}, + {"qualified prefix", func(d string) string { return "sha256:" + shortDigest(d) }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + rep, err := s.rmi(tc.target(manifest.String()), false) + if err != nil { + t.Fatalf("rmi by digest: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after digest rmi") + } + if _, err := os.Stat(blobPath(s.root, manifest.String())); !os.IsNotExist(err) { + t.Fatalf("manifest blob after digest rmi: %v, want IsNotExist", err) + } + }) + } +} + +func TestCmdRmiAcceptsDigestPrintedByList(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + fields := strings.Fields(buf.String()) + if len(fields) < 7 { + t.Fatalf("list output has too few fields:\n%s", buf.String()) + } + listedDigest := fields[6] + + if err := cmdRmi([]string{"--store", s.root, listedDigest}); err != nil { + t.Fatalf("cmdRmi by listed digest %q: %v", listedDigest, err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after cmdRmi by listed digest") + } +} + +func TestRmiDigestPrefixAmbiguous(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + + _, err := s.rmi(shortDigest(manifest.String()), false) + if err == nil || !strings.Contains(err.Error(), "ambiguous") || !strings.Contains(err.Error(), "local:a, local:b") { + t.Fatalf("rmi ambiguous digest err = %v, want both refs listed", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Fatalf("local:a pin lost after ambiguous rmi: %v", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after ambiguous rmi: %v", err) + } +} + +func TestRmiKeepsCacheForSameDigestPinnedByOtherRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a with shared cache: %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Fatalf("shared digest rootfs after rmi local:a: %v, want present", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } + + // Removing the last ref reclaims the now-unshared cache with the image. + rep, err := s.rmi("local:b", false) + if err != nil { + t.Fatalf("rmi final shared-cache ref: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi last shared-cache ref did not report dropping the cache") + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Fatalf("shared digest rootfs after final rmi: %v, want removed", err) + } +} + +func TestRmiForceDropsCache(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", true); err != nil { + t.Fatalf("rmi --force: %v", err) + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Errorf("rootfs cache after rmi --force: %v, want IsNotExist", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin still present after rmi --force, want gone") + } +} + +// TestRmiDropsColdCacheWithoutForce pins the fixed behavior: a plain rmi +// reclaims a cold unpacked cache as part of removing the image, so the natural +// run -> rmi lifecycle needs no --force and leaves no orphan cache. +func TestRmiDropsColdCacheWithoutForce(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi cold cache without --force: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi did not report dropping the cold cache") + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Errorf("cold cache after rmi: %v, want removed", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin present after rmi, want gone") + } +} + +func TestDigestCachePathChangesWhenRefDigestChanges(t *testing.T) { + s := openTestStore(t) + ref := "local:tag" + + imgA := buildImage(t, []string{"/a"}) + if _, err := s.addImage(ref, imgA); err != nil { + t.Fatal(err) + } + rootfsA := rootfsForImage(t, s, imgA) + + imgB := buildImage(t, []string{"/b"}) + if _, err := s.addImage(ref, imgB); err != nil { + t.Fatal(err) + } + rootfsB := rootfsForImage(t, s, imgB) + + if rootfsA == rootfsB { + t.Fatalf("digest-keyed rootfs path did not change across repull: %s", rootfsA) + } + if got := filepath.Dir(rootfsB); filepath.Base(got) != "sha256" { + t.Fatalf("rootfs path %s not under rootfs/sha256/", rootfsB) + } +} + +func TestDigestCachePathsAvoidRefEncodingCollisions(t *testing.T) { + s := openTestStore(t) + refA := "local/a:b" + refB := "local:a/b" + if legacyCacheNameForRef(refA) != legacyCacheNameForRef(refB) { + t.Fatalf("test refs no longer collide under legacy encoding: %q vs %q", refA, refB) + } + + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + if _, err := s.addImage(refA, imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage(refB, imgB); err != nil { + t.Fatal(err) + } + + rootfsA := rootfsForImage(t, s, imgA) + rootfsB := rootfsForImage(t, s, imgB) + if rootfsA == rootfsB { + t.Fatalf("digest-keyed refs collided at %s", rootfsA) + } +} + +// --- prune ------------------------------------------------------------------ + +// writeOrphanBlob writes an unreferenced file under blobs/sha256/ named with a +// valid sha256 digest so the local GC treats it as an unreachable blob. +func writeOrphanBlob(t *testing.T, root, content string) string { + t.Helper() + sum := sha256.Sum256([]byte(content)) + hex := hex.EncodeToString(sum[:]) + p := blobPath(root, hex) + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return "sha256:" + hex +} + +func writeStaleTempBlob(t *testing.T, root, baseDigest string) string { + t.Helper() + name := strings.TrimPrefix(baseDigest, "sha256:") + "1072211852" + p := blobPath(root, name) + if err := os.WriteFile(p, []byte("stale temp blob"), 0o644); err != nil { + t.Fatal(err) + } + return "sha256:" + name +} + +func TestPruneSweepsOrphanBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc: %v", err) + } + if len(rep.Blobs) != 1 || rep.Blobs[0] != orphan { + t.Errorf("gc removed = %v, want [%s]", rep.Blobs, orphan) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Errorf("orphan blob after prune: %v, want gone", err) + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); err != nil { + t.Errorf("referenced blob %s lost after prune: %v", d, err) + } + } +} + +func TestPruneSweepsOrphanAndStaleTempBlobs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc with stale temp blob: %v", err) + } + for _, want := range []string{orphan, stale} { + if !slices.Contains(rep.Blobs, want) { + t.Fatalf("gc removed = %v, want %s", rep.Blobs, want) + } + if _, err := os.Stat(blobPath(s.root, want)); !os.IsNotExist(err) { + t.Fatalf("blob %s after prune: %v, want gone", want, err) + } + } + if _, err := os.Stat(blobPath(s.root, layer.String())); err != nil { + t.Fatalf("live layer blob after prune: %v, want present", err) + } +} + +func TestPruneDryRunDeletesNothing(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + + rep, err := s.gc(true) + if err != nil { + t.Fatalf("gc --dry-run: %v", err) + } + if len(rep.Blobs) != 1 || rep.Blobs[0] != orphan { + t.Errorf("dry-run gc reported = %v, want [%s]", rep.Blobs, orphan) + } + if _, err := os.Stat(blobPath(s.root, orphan)); err != nil { + t.Errorf("orphan blob deleted under --dry-run: %v, want present", err) + } +} + +func TestPruneDryRunKeepsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.gc(true) + if err != nil { + t.Fatalf("gc --dry-run with stale temp blob: %v", err) + } + if !slices.Contains(rep.Blobs, stale) { + t.Fatalf("dry-run gc reported = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); err != nil { + t.Fatalf("stale temp blob deleted under --dry-run: %v, want present", err) + } +} + +func TestPruneCacheDropsUnpulledRootfs(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + // Legacy cache for an unpulled ref "local:b", an orphan cache under the + // digest-keyed layout. + orphanCache := legacyRootfsForRef(s.root, "local:b") + if err := os.MkdirAll(orphanCache, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanCache { + t.Errorf("pruneCaches dropped = %v, want [%s]", rep.CacheDirs, orphanCache) + } + if _, err := os.Stat(orphanCache); !os.IsNotExist(err) { + t.Errorf("orphan rootfs cache after prune --cache: %v, want gone", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("local:a pin lost after prune --cache: %v", err) + } +} + +func TestPruneCacheAllDropsPulledRootfs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + liveCache := rootfsForImage(t, s, img) + if err := os.MkdirAll(liveCache, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true}) + if err != nil { + t.Fatalf("pruneCaches --all: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != liveCache { + t.Errorf("pruneCaches --all dropped = %v, want [%s]", rep.CacheDirs, liveCache) + } + if _, err := os.Stat(liveCache); !os.IsNotExist(err) { + t.Errorf("live rootfs cache after prune --cache --all: %v, want gone", err) + } + // --all drops the cache only; the store (pin + blobs) is untouched. + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("local:a pin lost after prune --cache --all: %v", err) + } +} + +func TestRmiThenPruneIdempotent(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi: %v", err) + } + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc after rmi: %v", err) + } + if len(rep.Blobs) != 0 { + t.Errorf("gc after rmi removed %v, want nothing (already reclaimed)", rep.Blobs) + } +} + +// --- sweepable clone detection ---------------------------------------------- + +// TestListSweepableClonesSkipsKeepMarkerAndNonClones pins the reap set the +// bundle sweep uses once it holds run.lock exclusively: every run-- +// clone WITHOUT a keep marker plus any rootfs.tmp-* unpack leftover, and +// nothing else. Pids are irrelevant now; the exclusive lock already proves +// no run is live. Non-directory entries carrying either name shape stay +// untouched: both branches sweep staging DIRECTORIES only (the old behavior +// swept a plain rootfs.tmp-* file). +func TestListSweepableClonesSkipsKeepMarkerAndNonClones(t *testing.T) { + dir := t.TempDir() + reapClone := filepath.Join(dir, "run-1234-1") + keepClone := filepath.Join(dir, "run-5678-2") + tmpUnpack := filepath.Join(dir, "rootfs.tmp-abcd") + rootfs := filepath.Join(dir, "rootfs") + for _, p := range []string{reapClone, keepClone, tmpUnpack, rootfs} { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + } + tmpFile := filepath.Join(dir, "rootfs.tmp-file") + runFile := filepath.Join(dir, "run-9999-9") + for _, p := range []string{tmpFile, runFile} { + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := writeKeepMarker(keepClone); err != nil { + t.Fatal(err) + } + + got := listSweepableClones(dir) + want := map[string]bool{reapClone: true, tmpUnpack: true} + if len(got) != len(want) { + t.Fatalf("listSweepableClones = %v, want %v", got, want) + } + for _, g := range got { + if !want[g] { + t.Fatalf("listSweepableClones included %q, want only %v", g, want) + } + } + + // The read-only lister removes nothing. + for _, p := range []string{reapClone, keepClone, tmpUnpack, rootfs} { + if _, err := os.Stat(p); err != nil { + t.Errorf("listSweepableClones removed %s, want read-only: %v", p, err) + } + } + + // reapSweepableClones removes exactly the sweepable set, keeping the + // marked clone, the base rootfs, and the keep marker's clone dir. + reaped := reapSweepableClones(dir) + if len(reaped) != 2 { + t.Fatalf("reapSweepableClones = %v, want 2 entries", reaped) + } + if _, err := os.Stat(reapClone); !os.IsNotExist(err) { + t.Errorf("unmarked clone not reaped: %v", err) + } + if _, err := os.Stat(tmpUnpack); !os.IsNotExist(err) { + t.Errorf("unpack leftover not reaped: %v", err) + } + if _, err := os.Stat(keepClone); err != nil { + t.Errorf("kept clone was reaped, want preserved: %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Errorf("base rootfs was reaped, want left alone: %v", err) + } + for _, p := range []string{tmpFile, runFile} { + if _, err := os.Stat(p); err != nil { + t.Errorf("non-directory %s was reaped, want left alone: %v", p, err) + } + } +} + +// TestListSweepableClonesPreservesOnMarkerStatError pins the fail-safe: if the +// keep-marker stat returns a transient non-ENOENT error (here EACCES from an +// unreadable keep directory), the clone is preserved rather than reaped; +// reaping a --keep clone on a flaky read would be data loss. +func TestListSweepableClonesPreservesOnMarkerStatError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission bits do not bind as root") + } + dir := t.TempDir() + clone := filepath.Join(dir, "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + // 0o000 on the keep dir makes os.Stat(.elfuse-keep/run-1-1) fail with + // EACCES (cannot traverse .elfuse-keep), not ENOENT, while dir itself stays + // readable so the clone is still discovered. + if err := os.MkdirAll(keepDirPath(dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(keepDirPath(dir), 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(keepDirPath(dir), 0o755) }) + + if got := listSweepableClones(dir); len(got) != 0 { + t.Fatalf("listSweepableClones = %v, want empty (fail safe on marker stat error)", got) + } +} + +// TestListSweepableClonesReapsImageShippedKeepFile: a clone whose +// own contents include /.elfuse-keep (an image that ships that path, or a guest +// that wrote it) is NOT preserved. The keep record lives in the mount-root keep +// directory, outside every guest's view, so only elfuse-oci's --keep can +// set it; forged in-clone files are ignored and the clone is reaped. +func TestListSweepableClonesReapsImageShippedKeepFile(t *testing.T) { + dir := t.TempDir() + clone := filepath.Join(dir, "run-7-7") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + // The image (or guest) planted /.elfuse-keep inside the clone. + if err := os.WriteFile(filepath.Join(clone, ".elfuse-keep"), nil, 0o644); err != nil { + t.Fatal(err) + } + + got := listSweepableClones(dir) + if len(got) != 1 || got[0] != clone { + t.Fatalf("listSweepableClones = %v, want the clone reapable despite in-clone /.elfuse-keep", got) + } + + // A genuine --keep record in the mount-root keep dir does preserve it. + if err := writeKeepMarker(clone); err != nil { + t.Fatal(err) + } + if got := listSweepableClones(dir); len(got) != 0 { + t.Fatalf("listSweepableClones = %v, want kept clone skipped after writeKeepMarker", got) + } +} + +// TestCSBundleBusy pins the read-only busy probe used by prune --dry-run: a +// bundle whose run.lock is held (a live run) reports busy; an idle one does +// not. +func TestCSBundleBusy(t *testing.T) { + bundle := t.TempDir() + if csBundleBusy(bundle) { + t.Fatal("csBundleBusy(idle) = true, want false") + } + hold, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + if !csBundleBusy(bundle) { + t.Fatal("csBundleBusy(live run) = false, want true") + } + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if csBundleBusy(bundle) { + t.Fatal("csBundleBusy after release = true, want false") + } +} + +func TestListMissingImage(t *testing.T) { + s := openTestStore(t) + missingDigest := "sha256:" + strings.Repeat("9", 64) + if err := s.savePins(refPins{"missing": missingDigest}); err != nil { + t.Fatal(err) + } + if err := list(&bytes.Buffer{}, s, false); err == nil || !strings.Contains(err.Error(), "list: missing: image") { + t.Fatalf("list missing image err = %v, want image error", err) + } +} + +func TestShortDigest(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"short hex passthrough", "abcdef", "abcdef"}, + {"exactly twelve", "123456789012", "123456789012"}, + {"truncated to twelve", "123456789012345", "123456789012"}, + {"sha256 prefix stripped and truncated", "sha256:abcdef1234567890abcdef00", "abcdef123456"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := shortDigest(tc.in); got != tc.want { + t.Fatalf("shortDigest(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestListShowsPlatformVariant pins that a variant-qualified platform is not +// truncated to os/arch in list output. +func TestListShowsPlatformVariant(t *testing.T) { + s := openTestStore(t) + img, err := mutate.ConfigFile(buildImage(t, []string{"/hello"}), &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Variant: "v8", + }) + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:variant", img); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "linux/arm64/v8") { + t.Fatalf("list output %q missing linux/arm64/v8", buf.String()) + } +} + +// --- plain-rootfs run lock --------------------------------------------------- + +func TestRootfsCacheBusy(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + if rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy(no lock file) = true, want false") + } + hold, err := acquireRootfsRunLock(dir) + if err != nil { + t.Fatal(err) + } + if !rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy(live run) = false, want true") + } + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy after release = true, want false") + } +} + +// TestPruneCacheHonorsRootfsRunLock pins the busy semantics of the plain +// cache sweep: a digest cache whose run lock is held by a live run survives +// prune --cache --all and is never advertised by a dry run; an idle cache +// (including one with a stale lock file left by a dead run) is reclaimed, +// lock file included. +func TestPruneCacheHonorsRootfsRunLock(t *testing.T) { + cases := []struct { + name string + holdLock bool + staleLock bool + dryRun bool + wantGone bool + wantInRep bool + }{ + {name: "live run skipped", holdLock: true}, + {name: "live run hidden from dry-run", holdLock: true, dryRun: true}, + {name: "idle reclaimed", wantGone: true, wantInRep: true}, + {name: "stale lock file reclaimed", staleLock: true, wantGone: true, wantInRep: true}, + {name: "idle dry-run reported only", dryRun: true, wantInRep: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + if tc.holdLock { + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + } + if tc.staleLock { + if err := os.WriteFile(rootfsRunLockPath(cache), nil, 0o644); err != nil { + t.Fatal(err) + } + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true, dryRun: tc.dryRun}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + + inRep := slices.Contains(rep.CacheDirs, cache) + if inRep != tc.wantInRep { + t.Errorf("reported = %v, want %v (dirs %v)", inRep, tc.wantInRep, rep.CacheDirs) + } + _, statErr := os.Stat(cache) + gone := os.IsNotExist(statErr) + wantGone := tc.wantGone && !tc.dryRun + if gone != wantGone { + t.Errorf("cache gone = %v, want %v (stat err %v)", gone, wantGone, statErr) + } + if wantGone { + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("lock file after reclaim: %v, want gone", err) + } + } + }) + } +} + +// TestPruneCacheHonorsLockForStagingDir pins that the sweep guards a +// staging dir (.tmp-, unpackImage's pre-rename workspace) by +// the DIGEST's run lock, which the unpacker holds for the whole unpack: a +// mid-flight unpack survives prune --cache --all and never appears in a dry +// run, while a crashed unpack's leftover (digest lock free) is reclaimed. +func TestPruneCacheHonorsLockForStagingDir(t *testing.T) { + cases := []struct { + name string + holdLock bool + dryRun bool + wantGone bool + wantInRep bool + }{ + {name: "live unpack skipped", holdLock: true}, + {name: "live unpack hidden from dry-run", holdLock: true, dryRun: true}, + {name: "crashed leftover reclaimed", wantGone: true, wantInRep: true}, + {name: "crashed leftover dry-run reported only", dryRun: true, wantInRep: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + staging := cache + rootfsStagingSuffix + "123456" + if err := os.MkdirAll(staging, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, "f"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if tc.holdLock { + // The digest's lock, exactly what a live unpacker holds; a + // lock named after the staging dir itself never exists. + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true, dryRun: tc.dryRun}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + + inRep := slices.Contains(rep.CacheDirs, staging) + if inRep != tc.wantInRep { + t.Errorf("reported = %v, want %v (dirs %v)", inRep, tc.wantInRep, rep.CacheDirs) + } + _, statErr := os.Stat(staging) + gone := os.IsNotExist(statErr) + if gone != tc.wantGone { + t.Errorf("staging gone = %v, want %v (stat err %v)", gone, tc.wantGone, statErr) + } + }) + } +} + +// TestRemoveRootfsCacheMissingDirRespectsLock pins the lock-first rule for +// a cache dir that is already gone: a staging dir vanishes exactly when its +// unpack renames it into place, and that holder still owns the digest lock, +// so removal must refuse rather than unlink the lock out from under it. +// With no holder the stale lock file is swept, and a path with no store +// structure at all is a plain no-op. +func TestRemoveRootfsCacheMissingDirRespectsLock(t *testing.T) { + cache := filepath.Join(t.TempDir(), "sha256", strings.Repeat("ab", 32)) + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + staging := cache + rootfsStagingSuffix + "123456" + + if err := removeRootfsCache(staging); !errors.Is(err, errCacheBusy) { + t.Fatalf("removeRootfsCache(vanished staging, lock held) = %v, want errCacheBusy", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); err != nil { + t.Errorf("digest lock file unlinked by refused removal: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if err := removeRootfsCache(staging); err != nil { + t.Fatalf("removeRootfsCache(vanished staging, idle) = %v", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("stale digest lock survived idle removal: %v", err) + } + + if err := removeRootfsCache(filepath.Join(t.TempDir(), "nope", "sha256", "feed")); err != nil { + t.Fatalf("removeRootfsCache(no store structure) = %v, want nil", err) + } +} + +// TestRmiRefusesLiveRootfsRun pins rmi's fail-closed rule for the plain +// cache: even --force refuses while a live run holds the per-digest lock, +// and the pin survives the refusal; once the run exits the same rmi +// succeeds and reclaims cache and lock file. +func TestRmiRefusesLiveRootfsRun(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", true); err == nil || + !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("rmi under live run err = %v, want live-run refusal", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("pin lost after refused rmi: %v", err) + } + if _, err := os.Stat(cache); err != nil { + t.Errorf("cache damaged after refused rmi: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", true); err != nil { + t.Fatalf("rmi after run exit: %v", err) + } + if _, err := os.Stat(cache); !os.IsNotExist(err) { + t.Errorf("cache after rmi: %v, want gone", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("lock file after rmi: %v, want gone", err) + } +} diff --git a/cmd/elfuse-oci/list.go b/cmd/elfuse-oci/list.go new file mode 100644 index 00000000..cda6938e --- /dev/null +++ b/cmd/elfuse-oci/list.go @@ -0,0 +1,157 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// listEntry is one row of `elfuse-oci list --json`. +type listEntry struct { + Ref string `json:"ref"` + Digest string `json:"digest"` + Platform string `json:"platform"` + Created string `json:"created,omitempty"` + Size int64 `json:"size"` + Layers int `json:"layers"` +} + +// list prints every ref pinned in the store with its manifest digest, platform, +// creation time, total compressed layer size, and layer count. With asJSON it +// emits a JSON array of listEntry; otherwise a human-readable table sorted by +// ref. The size is the sum of the layers' compressed descriptor sizes +// (layer.Size()), matching `docker images` and the blob bytes rmi or a plain +// prune would reclaim, not the uncompressed size, which would require +// streaming every layer and defeat a fast listing. (prune --cache reports a +// different pool entirely: the on-disk allocation of the unpacked caches.) +func list(w io.Writer, s *store, asJSON bool) error { + // Snapshot pins and manifests under the store lock: a concurrent rmi + // deletes both under the same lock, and reading unlocked could observe a + // pin whose descriptor or blobs are already gone and fail mid-listing. + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + + pins, err := s.loadPins() + if err != nil { + return err + } + refs := make([]string, 0, len(pins)) + for ref := range pins { + refs = append(refs, ref) + } + sort.Strings(refs) + + entries := make([]listEntry, 0, len(refs)) + for _, ref := range refs { + e := listEntry{Ref: ref, Digest: pins[ref]} + h, err := v1.NewHash(pins[ref]) + if err != nil { + return fmt.Errorf("list: %s: digest %q: %w", ref, pins[ref], err) + } + img, err := s.path.Image(h) + if err != nil { + return fmt.Errorf("list: %s: image: %w", ref, err) + } + cfg, err := img.ConfigFile() + if err != nil { + return fmt.Errorf("list: %s: config: %w", ref, err) + } + e.Platform = platformOf(cfg).String() + if !cfg.Created.IsZero() { + e.Created = formatCreated(cfg) + } + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("list: %s: layers: %w", ref, err) + } + e.Layers = len(layers) + for i, l := range layers { + sz, err := l.Size() + if err != nil { + return fmt.Errorf("list: %s: layer %d size: %w", ref, i, err) + } + e.Size += sz + } + entries = append(entries, e) + } + + // A failed write (closed pipe, full disk behind a redirect) must not + // exit 0 with truncated output a script would consume as complete; + // same contract as inspect. + if asJSON { + b, err := json.MarshalIndent(entries, "", " ") + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "%s\n", b); err != nil { + return fmt.Errorf("list: write: %w", err) + } + return nil + } + if len(entries) == 0 { + return nil + } + bw := bufio.NewWriter(w) + fmt.Fprintf(bw, "%-40s %-12s %-14s %-20s %12s %s\n", "REF", "DIGEST", "PLATFORM", "CREATED", "SIZE", "LAYERS") + for _, e := range entries { + created := e.Created + if created == "" { + created = "-" + } + fmt.Fprintf(bw, "%-40s %s %-14s %-20s %12d %d\n", e.Ref, shortDigest(e.Digest), e.Platform, created, e.Size, e.Layers) + } + if err := bw.Flush(); err != nil { + return fmt.Errorf("list: write: %w", err) + } + return nil +} + +// shortDigest returns the first 12 hex characters of a "sha256:..." digest. +func shortDigest(d string) string { + if i := strings.Index(d, ":"); i >= 0 { + d = d[i+1:] + } + if len(d) > 12 { + return d[:12] + } + return d +} + +// cmdList implements `elfuse-oci list [--store] [--json]` (alias: images). +func cmdList(args []string) error { + cf, asJSON, err := parseListArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return list(os.Stdout, s, asJSON) +} + +func parseListArgs(args []string) (commonFlags, bool, error) { + var cf commonFlags + var asJSON bool + fs := newCommandFlagSet("list", &cf) + fs.BoolVar(&asJSON, "json", false, "emit a JSON array of {ref,digest,platform,created,size,layers}") + if err := fs.Parse(args); err != nil { + return cf, false, err + } + if err := noArgs("list", fs.Args()); err != nil { + return cf, false, err + } + return cf, asJSON, nil +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index 1e655607..11867edf 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -18,10 +18,14 @@ // elfuse-oci run [--store DIR] [--entrypoint E] [--env K=V]... // [--user UID[:GID]] [--workdir DIR] [--platform ...] // [args...] +// elfuse-oci list [--store DIR] [--json] (alias: images) +// elfuse-oci rmi [--store DIR] [--force] +// elfuse-oci prune [--store DIR] [--cache] [--all] [--dry-run] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., -// localhost:5000/foo:tag, or name@sha256:...). The default store is -// $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci. +// localhost:5000/foo:tag, or name@sha256:...). `rmi` also accepts a unique +// sha256 digest prefix from `list`. The default store is $ELFUSE_OCI_STORE or +// ~/.local/share/elfuse/oci. package main import ( @@ -46,6 +50,9 @@ commands: unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config run Pull + unpack + exec the image's entrypoint under elfuse + list List refs pinned in the store with digest/platform/created/size (alias: images) + rmi Remove a ref or unique digest and garbage-collect unreachable blobs + prune Garbage-collect unreachable blobs; --cache also drops rootfs/sparsebundle caches help Show this help version Print the elfuse-oci version @@ -74,6 +81,19 @@ run flags: clone (mutations persist; macOS only) --keep Keep the per-run COW clone and mount for inspection (macOS only) + +list flags: + --json Emit a JSON array of {ref,digest,platform,created,size,layers} + +rmi flags: + --force Also discard run --keep retained output held in the cache; + without it rmi refuses. A cold cache is always reclaimed + with its image; a cache a live run uses is never removed + +prune flags: + --cache Also drop elfuse unpacked caches (rootfs/ and, on macOS, cs/) + --all With --cache, drop caches even for still-pulled refs + --dry-run Report what would be reclaimed without deleting `) } @@ -110,6 +130,12 @@ func dispatch(cmd string, rest []string) error { return cmdInspect(rest) case "run": return cmdRun(rest) + case "list", "images": + return cmdList(rest) + case "rmi": + return cmdRmi(rest) + case "prune": + return cmdPrune(rest) default: usage() return fmt.Errorf("unknown command: %s", cmd) diff --git a/cmd/elfuse-oci/main_command_test.go b/cmd/elfuse-oci/main_command_test.go index 9a8636da..7a761c07 100644 --- a/cmd/elfuse-oci/main_command_test.go +++ b/cmd/elfuse-oci/main_command_test.go @@ -5,8 +5,12 @@ package main import ( "os/exec" + "path/filepath" "strings" "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" ) func TestRunDispatchHelpVersionAndErrors(t *testing.T) { @@ -82,3 +86,68 @@ func TestMainSubprocessExitBehavior(t *testing.T) { t.Fatalf("main no-arg stderr = %q, want formatted error", stderr) } } + +func TestRunDispatchesAllSubcommands(t *testing.T) { + root := t.TempDir() + img := tinyImage(t) + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + if ref != "local:tiny" { + t.Fatalf("pull ref = %q, want local:tiny", ref) + } + return img, nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return run([]string{"pull", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Pulled local:tiny") { + t.Fatalf("run pull stderr=%q err=%v, want pull summary", stderr, err) + } + + for _, cmd := range []string{"list", "images"} { + stdout, _, err = captureOutput(t, func() error { + return run([]string{cmd, "--store", root}) + }) + if err != nil || !strings.Contains(stdout, "local:tiny") { + t.Fatalf("run %s stdout=%q err=%v, want list row", cmd, stdout, err) + } + } + + stdout, _, err = captureOutput(t, func() error { + return run([]string{"inspect", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stdout, "Ref: local:tiny") { + t.Fatalf("run inspect stdout=%q err=%v, want inspect output", stdout, err) + } + + unpackRoot := filepath.Join(t.TempDir(), "unpack-rootfs") + _, stderr, err = captureOutput(t, func() error { + return run([]string{"unpack", "--store", root, "--rootfs", unpackRoot, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Unpacked local:tiny") { + t.Fatalf("run unpack stderr=%q err=%v, want unpack summary", stderr, err) + } + + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, _ *flockFile) error { return nil }) + runRoot := filepath.Join(t.TempDir(), "run-rootfs") + _, _, err = captureOutput(t, func() error { + return run([]string{"run", "--store", root, "--plain-rootfs", "--rootfs", runRoot, "local:tiny"}) + }) + if err != nil { + t.Fatalf("run run --plain-rootfs: %v", err) + } + + _, stderr, err = captureOutput(t, func() error { + return run([]string{"prune", "--store", root, "--dry-run"}) + }) + if err != nil || !strings.Contains(stderr, "Would reclaim") { + t.Fatalf("run prune stderr=%q err=%v, want dry-run summary", stderr, err) + } + + _, stderr, err = captureOutput(t, func() error { + return run([]string{"rmi", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Removed local:tiny") { + t.Fatalf("run rmi stderr=%q err=%v, want rmi summary", stderr, err) + } +} diff --git a/cmd/elfuse-oci/prune.go b/cmd/elfuse-oci/prune.go new file mode 100644 index 00000000..7b31d692 --- /dev/null +++ b/cmd/elfuse-oci/prune.go @@ -0,0 +1,239 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" +) + +// pruneOpts controls a prune pass. +type pruneOpts struct { + cache bool // also drop elfuse rootfs/sparsebundle caches + all bool // with cache: drop caches even for still-pulled refs + dryRun bool // report only; delete nothing +} + +// pruneReport summarizes a prune pass: blobs reclaimed, cache dirs dropped, +// and the approximate bytes freed. The total mixes two accountings: blob +// bytes are logical file sizes, cache-dir bytes are on-disk allocation +// (st_blocks, the honest figure for sparse files and APFS clones), so it is +// an estimate, not one uniform metric. +type pruneReport struct { + Blobs []string + CacheDirs []string + Bytes int64 +} + +// cmdPrune implements `elfuse-oci prune [--store] [--cache] [--all] [--dry-run]`. +// +// Without --cache, prune runs a reachability GC over blobs/sha256/ and reclaims +// any blob not reachable from an index.json manifest descriptor (retag or +// partial-pull orphans). With --cache it additionally drops elfuse's unpacked +// caches: the plain rootfs// directories and, on darwin, the +// case-sensitive sparsebundle bundles (cs//). By default only +// digest-keyed caches no longer reachable from refs.json are dropped, plus any +// legacy ref-named caches; --all drops every cache, including for still-pulled +// refs. --dry-run reports what would be removed without deleting. --all +// requires --cache (it has no meaning for the blob GC, which is already +// unconditional). +func cmdPrune(args []string) error { + cf, opts, err := parsePruneArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + + // The whole sweep runs under the store lock: the GC's reachability scan + // must not race a concurrent pull, whose fresh blobs land before the + // index descriptor that makes them reachable and would otherwise be + // reclaimed in the window between the two writes. rmi already runs its + // own gc under this lock. Reporting below happens after the lock drops. + var rep pruneReport + err = s.withLock(func() error { + gcr, err := s.gc(opts.dryRun) + if err != nil { + return err + } + rep.Blobs = gcr.Blobs + rep.Bytes += gcr.Bytes + + if opts.cache { + cr, err := s.pruneCaches(opts) + if err != nil { + return err + } + rep.CacheDirs = cr.CacheDirs + rep.Bytes += cr.Bytes + } + return nil + }) + if err != nil { + return err + } + + verb := "Reclaimed" + if opts.dryRun { + verb = "Would reclaim" + } + fmt.Fprintf(os.Stderr, "%s: %d blob(s), %d cache dir(s), ~%d bytes\n", + verb, len(rep.Blobs), len(rep.CacheDirs), rep.Bytes) + for _, b := range rep.Blobs { + fmt.Fprintf(os.Stderr, " blob %s\n", b) + } + for _, d := range rep.CacheDirs { + fmt.Fprintf(os.Stderr, " cache %s\n", d) + } + return nil +} + +func parsePruneArgs(args []string) (commonFlags, pruneOpts, error) { + var cf commonFlags + var opts pruneOpts + fs := newCommandFlagSet("prune", &cf) + fs.BoolVar(&opts.cache, "cache", false, "also drop unpacked caches (rootfs/ and, on macOS, cs/ sparsebundles)") + fs.BoolVar(&opts.all, "all", false, "with --cache, drop caches even for still-pulled refs") + fs.BoolVar(&opts.dryRun, "dry-run", false, "report what would be reclaimed without deleting") + if err := fs.Parse(args); err != nil { + return cf, opts, err + } + if err := noArgs("prune", fs.Args()); err != nil { + return cf, opts, err + } + if opts.all && !opts.cache { + return cf, opts, fmt.Errorf("prune: --all requires --cache") + } + return cf, opts, nil +} + +// pruneRootfsCaches drops plain rootfs// cache directories. Without +// opts.all, only dirs whose digest key is not live are dropped; with opts.all, +// every digest cache is dropped. Pre-digest rootfs/ caches are +// always treated as orphaned legacy caches because they are no longer used by +// run/unpack. Shared across platforms; the darwin-only sparsebundle sweep lives +// in cache_darwin.go. +func pruneRootfsCaches(s *store, live map[string]bool, opts pruneOpts) (pruneReport, error) { + var rep pruneReport + base := filepath.Join(s.root, rootfsCacheDirName) + entries, err := os.ReadDir(base) + if err != nil { + if os.IsNotExist(err) { + return rep, nil + } + return rep, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + top := filepath.Join(base, e.Name()) + if e.Name() == "sha256" { + children, err := os.ReadDir(top) + if err != nil { + return rep, err + } + for _, child := range children { + if !child.IsDir() { + // A sibling .lock whose cache dir is absent is an + // orphan (every run creates the lock without ever + // creating the plain dir; a failed unpack leaves one + // too); sweep it, held locks excepted. A lock whose dir + // exists belongs to that dir's own sweep below. + hex, isLock := strings.CutSuffix(child.Name(), ".lock") + if !isLock || opts.dryRun { + continue + } + if _, err := os.Lstat(filepath.Join(top, hex)); !os.IsNotExist(err) { + continue + } + if err := sweepOrphanRootfsLock(filepath.Join(top, hex)); err != nil { + return rep, err + } + continue + } + key := filepath.Join("sha256", child.Name()) + if !opts.all && live[key] { + continue + } + dir := filepath.Join(top, child.Name()) + // A cache whose per-digest run lock is held belongs to a live + // --plain-rootfs guest: skip it, and never advertise it in a + // dry run, the way the bundle sweep skips a busy sparsebundle. + // This also covers the non---all case where a re-pull moved + // the pin off a digest a guest is still running from. + if err := sweepPlainDir(&rep, dir, opts.dryRun); err != nil { + return rep, err + } + } + continue + } + + // Legacy ref-named rootfs caches are no longer live under the + // digest-keyed scheme; prune --cache reclaims them as orphan caches. + // No run ever locks a legacy path, but the same locked removal keeps + // one code path. + if err := sweepPlainDir(&rep, top, opts.dryRun); err != nil { + return rep, err + } + } + return rep, nil +} + +// sweepPlainDir reclaims (or, in a dry run, reports) one plain rootfs cache +// dir, skipping it when a live --plain-rootfs guest holds its run lock, the +// way the bundle sweep skips a busy sparsebundle. +func sweepPlainDir(rep *pruneReport, dir string, dryRun bool) error { + if dryRun { + if rootfsCacheBusy(dir) { + return nil + } + rep.Bytes += dirSize(dir) + rep.CacheDirs = append(rep.CacheDirs, dir) + return nil + } + size := dirSize(dir) + if err := removeRootfsCache(dir); err != nil { + if errors.Is(err, errCacheBusy) { + return nil + } + return err + } + rep.Bytes += size + rep.CacheDirs = append(rep.CacheDirs, dir) + return nil +} + +// dirSize reports the on-disk allocation (stat block count, not logical file +// size) of a tree, so a sparse APFS sparsebundle image is measured by the bytes +// it actually occupies rather than its 16g virtual ceiling. Best-effort: walk +// errors are ignored so a busy/evaporating entry does not abort the sweep. +func dirSize(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if info, err := d.Info(); err == nil { + total += diskUsage(info) + } + return nil + }) + return total +} + +// diskUsage returns the bytes actually allocated to a file (Blocks * 512) when +// the platform exposes stat blocks, falling back to logical size otherwise. +func diskUsage(fi os.FileInfo) int64 { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + return int64(st.Blocks) * 512 + } + return fi.Size() +} diff --git a/cmd/elfuse-oci/rmi.go b/cmd/elfuse-oci/rmi.go new file mode 100644 index 00000000..171ef6fe --- /dev/null +++ b/cmd/elfuse-oci/rmi.go @@ -0,0 +1,59 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" +) + +// cmdRmi implements `elfuse-oci rmi [--store] [--force] `. +// +// rmi drops the selected ref's pin. The target may be an exact ref or a unique +// sha256 digest prefix from `list`. If no remaining ref pins the same manifest +// digest, it removes that descriptor from index.json, garbage-collects the +// now-unreachable blobs, and reclaims the image's unpacked cache (rootfs/ or cs/ +// sparsebundle); the cache is derived state that goes with the image. A blob, +// descriptor, or cache still reachable through another pinned ref is kept. rmi +// refuses when a live run still uses the cache (never overridable) or when the +// cache holds run --keep retained output (then --force discards it). An absent +// ref/digest is an error. +func cmdRmi(args []string) error { + cf, force, ref, err := parseRmiArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + rep, err := s.rmi(ref, force) + if err != nil { + return err + } + removed := ref + if rep.Ref != "" { + removed = rep.Ref + } + fmt.Fprintf(os.Stderr, "Removed %s: %d blob(s), %d bytes\n", removed, len(rep.Blobs), rep.Bytes) + for _, b := range rep.Blobs { + fmt.Fprintf(os.Stderr, " blob %s\n", b) + } + if rep.CacheDropped { + fmt.Fprintf(os.Stderr, " dropped unpacked cache\n") + } + return nil +} + +func parseRmiArgs(args []string) (commonFlags, bool, string, error) { + var cf commonFlags + var force bool + fs := newCommandFlagSet("rmi", &cf) + fs.BoolVar(&force, "force", false, "discard run --keep retained output when removing an image that has it") + if err := fs.Parse(args); err != nil { + return cf, false, "", err + } + ref, err := oneArg("rmi", fs.Args(), "") + return cf, force, ref, err +} diff --git a/cmd/elfuse-oci/rootfslock.go b/cmd/elfuse-oci/rootfslock.go new file mode 100644 index 00000000..2bc18dea --- /dev/null +++ b/cmd/elfuse-oci/rootfslock.go @@ -0,0 +1,193 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" +) + +// Per-digest plain-rootfs cache locks. +// +// The plain digest-keyed rootfs cache (/rootfs///) is the +// same kind of shared mutable state as a sparsebundle bundle: a +// --plain-rootfs run executes a guest out of it while prune --cache and rmi +// want to RemoveAll it. Liveness follows the bundlelock.go discipline (a +// held flock proves a live holder regardless of pids) with one lock +// instead of two: there is no mount lifecycle to serialize (no attach.lock +// analog), and concurrent cold unpacks already reconcile through +// unpackImage's stage-and-rename. +// +// The lock is a SIBLING file (.lock next to the cache dir), not a file +// inside it: the cache dir is the guest's /, so a lock inside would appear +// at the guest's / in every run, and the dir's very existence is the +// "fully unpacked" signal that unpackImage publishes by atomic rename. +// Prune's sweep enumerations skip non-directories, so the lock file is +// invisible to them. +// +// - A run (and a store-cache unpack) holds the lock SHARED from before the +// cache-existence probe until the guest exits. The plain run path execs +// elfuse in place, so PreserveAcrossExec threads the descriptor through +// the exec: the kernel releases the flock exactly when the elfuse +// process exits, SIGKILL included, the same no-leaked-liveness property +// run.lock gives the sparsebundle path. +// - prune --cache takes it EXCLUSIVE non-blocking and skips a busy cache; +// rmi refuses a busy cache outright. The remover deletes the lock file +// while still holding the lock; acquireFlock's stat-after-lock guard +// already handles a racing acquirer. +// +// Ordering stays acyclic: runs take the rootfs lock while holding no store +// lock, and prune/rmi (which do hold the store lock) only ever probe the +// rootfs lock non-blocking. + +func rootfsRunLockPath(dir string) string { return dir + ".lock" } + +// rootfsCacheLockPath returns the lock file guarding a cache dir in the +// prune sweep. A published cache is guarded by its sibling .lock. +// A staging dir .tmp- (unpackImage's pre-rename workspace, a +// temp sibling in the same parent) is guarded by the SAME .lock: the +// unpacker holds that lock from before the cache-existence probe until the +// guest exits, so a probe on a lock named after the staging dir itself +// proves nothing and would let the sweep reclaim a tree a live unpack is +// still writing. A staging dir whose digest lock is free is a crashed +// unpack's leftover and remains reclaimable. +func rootfsCacheLockPath(dir string) string { + if hex, _, isStaging := strings.Cut(filepath.Base(dir), rootfsStagingSuffix); isStaging { + return rootfsRunLockPath(filepath.Join(filepath.Dir(dir), hex)) + } + return rootfsRunLockPath(dir) +} + +// acquireRootfsRunLock takes dir's run lock shared, blocking: if a prune is +// mid-removal the run waits it out and then re-unpacks the reclaimed cache. +// The parent directory is created first so a cold store can take the lock +// before its first unpack. +func acquireRootfsRunLock(dir string) (*flockFile, error) { + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return nil, err + } + return acquireFlock(rootfsRunLockPath(dir), syscall.LOCK_SH) +} + +// rootfsCacheBusy reports whether a live run holds dir's run lock. Used by +// prune's dry-run so it never advertises a reap the real pass would refuse. +// A missing lock file means no holder (holders create it), so the probe +// creates no state; any other failure fails closed to busy. +func rootfsCacheBusy(dir string) bool { + f, err := os.OpenFile(rootfsCacheLockPath(dir), os.O_RDWR, 0) + if err != nil { + return !os.IsNotExist(err) + } + defer f.Close() + if err := flockRetryIntr(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return true + } + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return false +} + +// lockRootfsCacheForRemoval takes dir's run lock exclusively, non-blocking. +// busy=true means a live run holds it and the cache must be left alone. On +// success the caller removes the cache dir and the lock file, then unlocks, +// the same unlock-after-removal rule sweepCSBundle follows, so a blocked +// acquirer never sees a half-removed tree. +func lockRootfsCacheForRemoval(dir string) (unlock func(), busy bool, err error) { + l, err := acquireFlock(rootfsCacheLockPath(dir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + return func() {}, true, nil + } + return func() {}, false, err + } + return func() { l.Close() }, false, nil +} + +// removeRootfsCache removes dir and its guarding lock file under an +// exclusive run lock, returning errCacheBusy (wrapped) when a live holder +// pins the cache: a running guest for a published dir, a mid-flight unpack +// for a staging dir (both hold the digest's lock, see rootfsCacheLockPath). +// The lock comes first even when dir is already gone: a vanished staging +// dir usually means its unpack just renamed it into place, and the holder +// still owns the digest lock, which an unlocked cleanup here would unlink +// out from under it. A missing lock parent means nothing was ever unpacked +// or locked, so an rmi of a run-less image stays a no-op that conjures no +// store structure. +func removeRootfsCache(dir string) error { + unlock, busy, err := lockRootfsCacheForRemoval(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer unlock() + if busy { + return fmt.Errorf("%s: %w", dir, errCacheBusy) + } + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.Remove(rootfsCacheLockPath(dir)); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// sweepOrphanRootfsLock removes dir's sibling lock file when the cache dir +// itself is absent: every run takes the per-digest reference lock and so +// creates the file without ever creating the plain dir, and a failed unpack +// leaves one too. The unlink happens while holding the exclusive lock +// (removeRootfsCache's order), so a live holder keeps its lock and a racing +// acquirer is caught by acquireFlock's stat-after-lock retry. +func sweepOrphanRootfsLock(dir string) error { + unlock, busy, err := lockRootfsCacheForRemoval(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer unlock() + if busy { + return nil + } + if err := os.Remove(rootfsCacheLockPath(dir)); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// removeRootfsCacheForDigest is the non-darwin removeRefCaches' plain-cache +// half: delete digest's unpacked plain rootfs, refusing while a live run uses +// it. The darwin removeRefCaches inlines the same removal because it deletes +// under a lock already held from its two-cache preflight. +func removeRootfsCacheForDigest(s *store, digest string) error { + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + if err := removeRootfsCache(rootfs); err != nil { + if errors.Is(err, errCacheBusy) { + return fmt.Errorf( + "cache for %s is in use by a live run; stop it before removing the image", + digest) + } + return err + } + return nil +} + +// PreserveAcrossExec clears FD_CLOEXEC on the lock's descriptor so the flock +// rides through syscall.Exec into the replacement process and is released by +// the kernel exactly when that process exits, SIGKILL included. +func (l *flockFile) PreserveAcrossExec() error { + _, err := unix.FcntlInt(l.f.Fd(), unix.F_SETFD, 0) + return err +} diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go index 6290b811..581497ba 100644 --- a/cmd/elfuse-oci/run.go +++ b/cmd/elfuse-oci/run.go @@ -23,18 +23,23 @@ var execElfuseForRun = execElfuse // filesystem). var afterSpawnStart = func() {} -// elfuseBin locates the elfuse binary to exec for `run`. Precedence: +// resolveElfuseBin locates the elfuse binary to exec for `run` and verifies +// it exists. Precedence: // - $ELFUSE_BIN (an override hook for tests and wrapper scripts); // - the sibling of this executable (build/elfuse-oci -> build/elfuse). -func elfuseBin() (string, error) { - if p := os.Getenv("ELFUSE_BIN"); p != "" { - return p, nil +func resolveElfuseBin() (string, error) { + bin := os.Getenv("ELFUSE_BIN") + if bin == "" { + exe, err := os.Executable() + if err != nil { + return "", fmt.Errorf("locate elfuse: %w", err) + } + bin = filepath.Join(filepath.Dir(exe), "elfuse") } - exe, err := os.Executable() - if err != nil { - return "", fmt.Errorf("locate elfuse: %w", err) + if _, err := os.Stat(bin); err != nil { + return "", fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) } - return filepath.Join(filepath.Dir(exe), "elfuse"), nil + return bin, nil } // elfuseArgv builds the argv for `elfuse --sysroot --user U:G @@ -68,18 +73,28 @@ func elfuseArgv(rootfs string, spec *runSpec) []string { // becomes elfuse in place, so the invoking shell reaps the same pid and // terminal signals such as Ctrl-C go straight to elfuse rather than through a // Go middleman. -func execElfuse(rootfs string, spec *runSpec) error { - bin, err := elfuseBin() +// +// A non-nil lock is the store cache's per-digest run lock. Its descriptor is +// made exec-survivable so the kernel holds the flock for exactly the elfuse +// process's lifetime (SIGKILL included) and releases it at guest exit; the +// one inherited fd is the price of not leaving a Go wrapper alive to babysit +// the lock. +func execElfuse(rootfs string, spec *runSpec, lock *flockFile) error { + bin, err := resolveElfuseBin() if err != nil { return err } - if _, err := os.Stat(bin); err != nil { - return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) - } - if err := syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()); err != nil { - return fmt.Errorf("exec %s: %w", bin, err) + if lock != nil { + if err := lock.PreserveAcrossExec(); err != nil { + return fmt.Errorf("preserve rootfs lock across exec: %w", err) + } } - return nil // unreachable + err = syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()) + // Unreachable on success. Keep the lock's *os.File live until the exec + // verdict so its finalizer cannot close the fd (dropping the flock) in + // the window before the process image is replaced. + runtime.KeepAlive(lock) + return fmt.Errorf("exec %s: %w", bin, err) } // spawnElfuseWait runs elfuse as a child and waits for it, returning the exit @@ -92,13 +107,10 @@ func execElfuse(rootfs string, spec *runSpec) error { // child so a signal targeted at elfuse-oci alone still propagates, and we // survive to reap and report the child's status rather than dying first. func spawnElfuseWait(rootfs string, spec *runSpec, locks ...*flockFile) (int, error) { - bin, err := elfuseBin() + bin, err := resolveElfuseBin() if err != nil { return 0, err } - if _, err := os.Stat(bin); err != nil { - return 0, fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) - } // exec.Command uses `bin` as argv[0], so drop the leading "elfuse" // program-name that elfuseArgv includes for syscall.Exec's sake; otherwise // elfuse would see "elfuse" as its first positional and try to boot a diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go index b8b9a2a7..de9710d9 100644 --- a/cmd/elfuse-oci/run_test.go +++ b/cmd/elfuse-oci/run_test.go @@ -13,7 +13,7 @@ import ( ) // writeElfuseStub writes a #!/bin/sh stub script that stands in for the -// elfuse binary, and points elfuseBin() at it via $ELFUSE_BIN. spawnElfuseWait +// elfuse binary, and points resolveElfuseBin() at it via $ELFUSE_BIN. spawnElfuseWait // exec.Command's whatever $ELFUSE_BIN names, so no real elfuse (and no HVF) is // needed. t.Setenv restores the env on cleanup. func writeElfuseStub(t *testing.T, body string) string { @@ -92,35 +92,31 @@ func TestElfuseArgvShape(t *testing.T) { } } -func TestElfuseBinEnvAndFallback(t *testing.T) { +func TestResolveElfuseBinEnvAndMissing(t *testing.T) { want := filepath.Join(t.TempDir(), "elfuse-custom") + if err := os.WriteFile(want, []byte("#!"), 0o755); err != nil { + t.Fatal(err) + } t.Setenv("ELFUSE_BIN", want) - got, err := elfuseBin() + got, err := resolveElfuseBin() if err != nil { t.Fatal(err) } if got != want { - t.Fatalf("elfuseBin with env = %q, want %q", got, want) + t.Fatalf("resolveElfuseBin with env = %q, want %q", got, want) } - t.Setenv("ELFUSE_BIN", "") - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - want = filepath.Join(filepath.Dir(exe), "elfuse") - got, err = elfuseBin() - if err != nil { - t.Fatal(err) - } - if got != want { - t.Fatalf("elfuseBin fallback = %q, want %q", got, want) + // A missing binary must fail up front with the $ELFUSE_BIN hint instead + // of surfacing later as an opaque exec error. + t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "absent")) + if _, err := resolveElfuseBin(); err == nil || !strings.Contains(err.Error(), "ELFUSE_BIN") { + t.Fatalf("resolveElfuseBin missing binary err = %v, want not-found with hint", err) } } func TestExecElfuseMissingBinary(t *testing.T) { t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) - err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}) + err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}, nil) if err == nil || !strings.Contains(err.Error(), "elfuse binary not found") { t.Fatalf("execElfuse missing binary err = %v, want not found", err) } diff --git a/cmd/elfuse-oci/sparsebundle.go b/cmd/elfuse-oci/sparsebundle.go index 3d3acad1..56daa942 100644 --- a/cmd/elfuse-oci/sparsebundle.go +++ b/cmd/elfuse-oci/sparsebundle.go @@ -241,16 +241,6 @@ func writeSpotlightMarker(mountPath string) error { return touchFile(filepath.Join(mountPath, ".metadata_never_index")) } -// touchFile creates (or truncates) an empty marker file. The markers carry -// no content; only their existence matters. -func touchFile(path string) error { - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - if err != nil { - return err - } - return f.Close() -} - // isMountPoint reports whether path is currently a mount point by comparing its // device id against its parent's. func isMountPoint(path string) bool { diff --git a/cmd/elfuse-oci/sparsebundle_test.go b/cmd/elfuse-oci/sparsebundle_test.go index 01b027c8..8d106232 100644 --- a/cmd/elfuse-oci/sparsebundle_test.go +++ b/cmd/elfuse-oci/sparsebundle_test.go @@ -11,27 +11,10 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" ) -// withDarwinCacheSeams swaps the darwin mount-probe and force-detach seams -// for a test and restores them on cleanup. -func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { - t.Helper() - oldIsMount := isMountPointFn - oldDetach := detachForce - if isMount != nil { - isMountPointFn = isMount - } - if detach != nil { - detachForce = detach - } - t.Cleanup(func() { - isMountPointFn = oldIsMount - detachForce = oldDetach - }) -} - // hdiutil attach -plist output is an array of system entity dictionaries. We // only care about the mount-point. This fixture is a trimmed-down capture of the // real shape (system-entities -> dict -> mount-point key/string). @@ -396,6 +379,138 @@ func TestProvisionCaseSensitiveWithFakeHdiutilFailures(t *testing.T) { } } +// withMountSeam overrides the mount-point probe directly (not via any shared +// seam helper) so these lock-behavior tests are self-contained. +func withMountSeam(t *testing.T, fn func(string) bool) { + t.Helper() + old := isMountPointFn + isMountPointFn = fn + t.Cleanup(func() { isMountPointFn = old }) +} + +// TestProvisionSharesLiveMount pins the F1 fix: when live runs of the digest +// hold run.lock, a new provision must share the already-attached volume, not +// force-detach it out from under the running guests. +func TestProvisionSharesLiveMount(t *testing.T) { + installFakeHdiutil(t) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "mnt") + withMountSeam(t, func(p string) bool { return p == requested }) + oldDetach := detachForce + detachForce = func(p string) error { + t.Errorf("detachForce(%s) called although a live run holds the volume", p) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + holder, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + + m, err := provisionCaseSensitive(bundle, requested, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive with live run: %v", err) + } + if m.mountPath != requested || !m.owned { + t.Fatalf("mount = %+v, want shared attach at requested mount", m) + } + + // The new run holds run.lock shared: an exclusive probe must report busy. + if _, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("run.lock probe err = %v, want errCacheBusy while run is live", err) + } + + // Close with the other holder still live: last-one-out must NOT detach + // (the detachForce seam above fails the test if it does). + if err := m.Close(); err != nil { + t.Fatalf("Close with surviving run: %v", err) + } +} + +// TestProvisionRejectsSymlinkedMountPath pins the G4 fix: a symlinked mount +// path must be refused before any mount-status probe or force-detach, so a +// tampered cache cannot trick provision into detaching an unrelated volume. +func TestProvisionRejectsSymlinkedMountPath(t *testing.T) { + installFakeHdiutil(t) + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "mnt") + if err := os.Symlink(t.TempDir(), requested); err != nil { + t.Fatal(err) + } + // Even if the path reads as a mount point, the symlink guard must win and + // no detach may run. + withMountSeam(t, func(string) bool { return true }) + oldDetach := detachForce + detachForce = func(p string) error { + t.Errorf("detachForce(%s) called on a symlinked mount path", p) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + _, err := provisionCaseSensitive(bundle, requested, "32m") + if err == nil || !strings.Contains(err.Error(), "is a symlink") { + t.Fatalf("provisionCaseSensitive(symlink) err = %v, want 'is a symlink'", err) + } +} + +// TestProvisionDetachesStaleMountAndHoldsRunLock pins the crash-recovery +// path: with no live run holding run.lock, a leftover attached mount is +// provably stale: provision detaches it, re-attaches cleanly, and the +// returned mount holds run.lock shared until Close, whose last-one-out probe +// then detaches. +func TestProvisionDetachesStaleMountAndHoldsRunLock(t *testing.T) { + installFakeHdiutil(t) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "requested-mnt") + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + detachLog := filepath.Join(t.TempDir(), "detach.log") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + // The stale leftover: the requested mount point reads as attached. + withMountSeam(t, func(p string) bool { return p == requested }) + + m, err := provisionCaseSensitive(bundle, requested, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive: %v", err) + } + b, err := os.ReadFile(detachLog) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), requested) { + t.Fatalf("detach log = %q, want stale mount %s detached", b, requested) + } + if m.mountPath != actualMount { + t.Fatalf("mountPath = %q, want re-attached %q", m.mountPath, actualMount) + } + + // Liveness is held from provision until Close. + if _, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("run.lock probe err = %v, want errCacheBusy while mount is open", err) + } + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + b, err = os.ReadFile(detachLog) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), actualMount) { + t.Fatalf("detach log = %q, want last-one-out detach of %s", b, actualMount) + } + free, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("run.lock probe after Close err = %v, want free", err) + } + free.Close() +} + // TestParseMountpointDecodesXMLEntities pins entity decoding: hdiutil's plist // escapes XML-special characters in the mount path (a store path may carry // "&" or "'"), and the raw escaped form would make every later use of the diff --git a/cmd/elfuse-oci/store.go b/cmd/elfuse-oci/store.go index 657f7561..71edcd86 100644 --- a/cmd/elfuse-oci/store.go +++ b/cmd/elfuse-oci/store.go @@ -8,6 +8,8 @@ import ( "fmt" "os" "path/filepath" + "sort" + "strings" "syscall" "github.com/google/go-containerregistry/pkg/v1" @@ -85,7 +87,47 @@ func writeIfAbsent(path string, data []byte) error { } else if !os.IsNotExist(err) { return err } - return os.WriteFile(path, data, 0o644) + // Durable even on cold-store bootstrap: a crash mid-write must not leave a + // truncated oci-layout or index.json that later opens fail to parse. + return writeFileDurable(path, data, 0o644) +} + +// writeFileDurable writes data to path atomically and durably: a uniquely +// named temp sibling is written, fsynced, and renamed into place, then the +// parent directory is fsynced so the rename itself survives a crash. A crash +// leaves either the prior file or the complete new one, never a truncated +// mix. This is the store's shared durability primitive for refs.json and +// index.json; rmi's crash-ordering rule (index.json committed before +// refs.json) holds only if each write is individually durable, which the +// layout package's plain os.WriteFile is not. +func writeFileDurable(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + // A unique temp name: a fixed name would let two writers clobber each + // other's half-written temp even before the rename race. + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // no-op once the rename succeeds + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return err + } + return fsyncDir(dir) } // refPins maps an image reference to its manifest digest ("sha256:..."). @@ -113,48 +155,85 @@ func (s *store) savePins(p refPins) error { if err != nil { return err } - // Unique temp name: a fixed name would let two writers clobber each - // other's half-written temp even before the rename race. - tmp, err := os.CreateTemp(s.root, ".refs.json.*") + // Durability, not just atomicity: rmi's crash-ordering argument (commit the + // index.json descriptor removal before dropping the pin) only holds if the + // new refs.json cannot revert to an old pin after a crash. + return writeFileDurable(filepath.Join(s.root, "refs.json"), b, 0o644) +} + +// fsyncDir flushes a directory's entries (renames, unlinks) to stable +// storage. +func fsyncDir(dir string) error { + f, err := os.Open(dir) if err != nil { return err } - defer os.Remove(tmp.Name()) // no-op once the rename succeeds - if _, err := tmp.Write(b); err != nil { - tmp.Close() - return err - } - if err := tmp.Chmod(0o644); err != nil { - tmp.Close() + defer f.Close() + return f.Sync() +} + +// fsyncFile flushes an existing file's data to stable storage. Used to make a +// blob durable before the pin that references it is committed. +func fsyncFile(path string) error { + f, err := os.Open(path) + if err != nil { return err } - // Durability, not just atomicity: rmi's crash-ordering argument (drop the - // pin before dropping the descriptor) only holds if the new refs.json - // cannot revert to an old pin after a crash. Sync the temp file before the - // rename and the directory after it, so the rename is durable once this - // returns. - if err := tmp.Sync(); err != nil { - tmp.Close() + defer f.Close() + return f.Sync() +} + +// syncAppendedImage makes img's on-disk state durable after AppendImage but +// before the pin is committed: the config and layer blobs, then index.json, +// then the store directory. Without this the pin (refs.json) is fsynced while +// the blobs and index it references may still sit in the page cache, so a +// crash could leave a durable pin over content that never reached disk, which +// every later resolve of the ref then fails on. The layout package writes +// blobs and index.json with plain os.WriteFile, so the durability is ours to +// add here. +func (s *store) syncAppendedImage(img v1.Image) error { + digests, err := imageBlobDigests(img) + if err != nil { return err } - if err := tmp.Close(); err != nil { - return err + for _, h := range digests { + p := filepath.Join(s.root, "blobs", h.Algorithm, h.Hex) + if err := fsyncFile(p); err != nil { + return err + } } - if err := os.Rename(tmp.Name(), filepath.Join(s.root, "refs.json")); err != nil { + if err := fsyncFile(filepath.Join(s.root, "index.json")); err != nil { return err } return fsyncDir(s.root) } -// fsyncDir flushes a directory's entries (renames, unlinks) to stable -// storage. -func fsyncDir(dir string) error { - f, err := os.Open(dir) +// imageBlobDigests returns the hashes of every blob img introduces: its +// manifest, config, and layers. +func imageBlobDigests(img v1.Image) ([]v1.Hash, error) { + var hs []v1.Hash + mh, err := img.Digest() if err != nil { - return err + return nil, err } - defer f.Close() - return f.Sync() + hs = append(hs, mh) + ch, err := img.ConfigName() + if err != nil { + return nil, err + } + hs = append(hs, ch) + layers, err := img.Layers() + if err != nil { + return nil, err + } + for _, l := range layers { + lh, err := l.Digest() + if err != nil { + return nil, err + } + hs = append(hs, lh) + } + return hs, nil } // lock takes an exclusive advisory flock on /.lock and returns the @@ -219,6 +298,56 @@ func (s *store) digestFor(ref string) (string, error) { return d, nil } +// resolvePinnedTarget resolves an exact pulled ref, or a unique sha256 digest +// prefix such as the 12-character digest printed by `list`. +func resolvePinnedTarget(pins refPins, target string) (string, string, error) { + if d, ok := pins[target]; ok { + return target, d, nil + } + + prefix, ok := digestPrefix(target) + if !ok { + return "", "", fmt.Errorf("store: %q %w (run `elfuse-oci pull %s` first)", target, errNotPulled, target) + } + + var matches []string + matchDigest := "" + for ref, digest := range pins { + h, err := v1.NewHash(digest) + if err != nil { + return "", "", fmt.Errorf("store: pinned digest for %q: %w", ref, err) + } + if h.Algorithm == "sha256" && strings.HasPrefix(h.Hex, prefix) { + matches = append(matches, ref) + matchDigest = digest + } + } + sort.Strings(matches) + + switch len(matches) { + case 0: + return "", "", fmt.Errorf("store: digest %q %w", target, errNotPulled) + case 1: + return matches[0], matchDigest, nil + default: + return "", "", fmt.Errorf("store: digest %q is ambiguous; matches refs: %s", target, strings.Join(matches, ", ")) + } +} + +func digestPrefix(target string) (string, bool) { + if i := strings.IndexByte(target, ':'); i >= 0 { + if target[:i] != "sha256" { + return "", false + } + target = target[i+1:] + } + target = strings.ToLower(target) + if len(target) < 12 || len(target) > 64 || !isLowerHex(target) { + return "", false + } + return target, true +} + // addImage appends img to the layout index if its manifest is not already // present (dedup by digest), and pins ref to that digest. Returns the digest. // The store lock covers the whole check-append-pin sequence: index.json is @@ -244,6 +373,12 @@ func (s *store) addImage(ref string, img v1.Image) (string, error) { if err := s.path.AppendImage(img); err != nil { return fmt.Errorf("store: append image: %w", err) } + // Make the blobs and index.json durable before the pin that will + // point at them, so a crash never strands a fsynced pin over + // content still in the page cache. + if err := s.syncAppendedImage(img); err != nil { + return fmt.Errorf("store: sync appended image: %w", err) + } } return s.pinLocked(ref, d.String()) }) diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go index 4f507ff2..b43deaf4 100644 --- a/cmd/elfuse-oci/store_test.go +++ b/cmd/elfuse-oci/store_test.go @@ -4,10 +4,12 @@ package main import ( + "bytes" "errors" "fmt" "os" "path/filepath" + "slices" "strings" "sync" "syscall" @@ -135,6 +137,80 @@ func TestOpenStoreBootstrapWaitsForStoreLock(t *testing.T) { } } +// TestRmiKeepsPinWhenDescriptorRemovalFails pins the rmi write ordering: +// index.json must be updated before the pin is dropped from refs.json. In the +// reverse order a failure between the writes strands the manifest: the ref +// no longer resolves while the descriptor keeps all blobs live, and prune +// never removes descriptors. With the correct order the pin survives the +// failure and a retried rmi completes. +func TestRmiKeepsPinWhenDescriptorRemovalFails(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("running as root: a read-only store dir cannot induce the write failure") + } + s := openTestStore(t) + if _, err := s.addImage("local:stuck", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + // index.json is now written atomically (temp + rename), so a read-only + // index.json no longer blocks the write: a rename replaces the file + // regardless of its mode. Make the store directory read-only instead, so + // the descriptor removal's temp create fails before refs.json is touched. + if err := os.Chmod(s.root, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(s.root, 0o755) }) + + if _, err := s.rmi("local:stuck", false); err == nil { + t.Fatal("rmi succeeded although the store directory is read-only") + } + pins, err := s.loadPins() + if err != nil { + t.Fatal(err) + } + if _, ok := pins["local:stuck"]; !ok { + t.Fatal("pin dropped although descriptor removal failed; image is stranded") + } + + if err := os.Chmod(s.root, 0o755); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:stuck", false); err != nil { + t.Fatalf("retried rmi after transient failure: %v", err) + } + pins, err = s.loadPins() + if err != nil { + t.Fatal(err) + } + if _, ok := pins["local:stuck"]; ok { + t.Fatal("pin still present after successful retried rmi") + } +} + +// TestGCReclaimsOrphanKeepsLive exercises store.gc directly (the lifecycle tests +// otherwise only reach it through rmi/prune): an unreferenced blob is reclaimed +// while a still-pinned image's own manifest/config/layers survive. +func TestGCReclaimsOrphanKeepsLive(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "gc-direct-orphan") + + rep, err := s.gc(false) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(rep.Blobs, orphan) { + t.Fatalf("gc did not report orphan %s; blobs=%v", orphan, rep.Blobs) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Fatalf("orphan blob still present after gc: %v", err) + } + if _, err := s.image("local:a"); err != nil { + t.Fatalf("live image unreadable after gc (live blobs reclaimed): %v", err) + } +} + func TestDefaultStoreFromEnvAndResolveStore(t *testing.T) { want := filepath.Join(t.TempDir(), "store") t.Setenv("ELFUSE_OCI_STORE", want) @@ -246,6 +322,43 @@ func TestLoadPinsCorruptNullAndPinError(t *testing.T) { } } +func TestInvalidPinnedDigestErrors(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(`{"bad":"not-a-digest"}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.image("bad"); err == nil { + t.Fatal("image with invalid pinned digest succeeded, want error") + } + var buf bytes.Buffer + if err := list(&buf, s, false); err == nil || !strings.Contains(err.Error(), `digest "not-a-digest"`) { + t.Fatalf("list err = %v, want invalid digest error", err) + } + if _, err := s.liveCacheKeys(); err == nil { + t.Fatal("liveCacheKeys with invalid pinned digest succeeded, want error") + } +} + +func TestRemoveManifestDescriptorAndGCErrors(t *testing.T) { + s := openTestStore(t) + if err := s.removeManifestDescriptor("not-a-digest"); err == nil { + t.Fatal("removeManifestDescriptor invalid digest succeeded, want error") + } + + // Pin a ref so gc's descriptor reconciliation and reachability both have to + // read the (now corrupt) index.json rather than short-circuiting on an + // empty pin set. + if _, err := s.addImage("local:a", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(s.root, "index.json"), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.gc(false); err == nil || !strings.Contains(err.Error(), "index") { + t.Fatalf("gc corrupt index err = %v, want an index parse error", err) + } +} + func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { if _, err := cacheKeyForDigest("not-a-digest"); err == nil { t.Fatal("cacheKeyForDigest accepted malformed digest") @@ -258,3 +371,193 @@ func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { t.Fatalf("cacheKeyForDigest(%q) succeeded, want rejection", unsupported) } } + +func TestPruneRootfsCachesKeepsLiveDropsOrphanAndDryRun(t *testing.T) { + s := &store{root: t.TempDir()} + liveHex := strings.Repeat("a", 64) + orphanHex := strings.Repeat("b", 64) + liveDir := filepath.Join(s.root, "rootfs", "sha256", liveHex) + orphanDir := filepath.Join(s.root, "rootfs", "sha256", orphanHex) + for _, dir := range []string{liveDir, orphanDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(s.root, "rootfs", "sha256", "not-a-dir"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + live := map[string]bool{filepath.Join("sha256", liveHex): true} + rep, err := pruneRootfsCaches(s, live, pruneOpts{cache: true, dryRun: true}) + if err != nil { + t.Fatalf("dry-run pruneRootfsCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanDir { + t.Fatalf("dry-run cache dirs = %v, want [%s]", rep.CacheDirs, orphanDir) + } + if _, err := os.Stat(orphanDir); err != nil { + t.Fatalf("dry-run removed orphan cache: %v", err) + } + + rep, err = pruneRootfsCaches(s, live, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneRootfsCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanDir { + t.Fatalf("cache dirs = %v, want [%s]", rep.CacheDirs, orphanDir) + } + if _, err := os.Stat(orphanDir); !os.IsNotExist(err) { + t.Fatalf("orphan cache after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(liveDir); err != nil { + t.Fatalf("live cache removed: %v", err) + } +} + +func TestPruneRootfsCachesMissingRootAndDiskUsageFallback(t *testing.T) { + s := &store{root: t.TempDir()} + rep, err := pruneRootfsCaches(s, nil, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("missing rootfs prune: %v", err) + } + if len(rep.CacheDirs) != 0 || rep.Bytes != 0 { + t.Fatalf("missing rootfs report = %+v, want empty", rep) + } + + if got := diskUsage(fakeFileInfo{size: 123}); got != 123 { + t.Fatalf("diskUsage fallback = %d, want logical size 123", got) + } +} + +func TestPruneCachesErrorsOnInvalidLivePin(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(`{"bad":"not-a-digest"}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.pruneCaches(pruneOpts{cache: true}); err == nil { + t.Fatal("pruneCaches with invalid live pin succeeded, want error") + } +} + +func TestDigestPrefix(t *testing.T) { + cases := []struct { + in string + want string + wantOK bool + }{ + {"abcdef123456", "abcdef123456", true}, + {"sha256:ABCDEF123456", "abcdef123456", true}, + {"abcdef12345", "", false}, + {strings.Repeat("a", 65), "", false}, + {"sha512:" + strings.Repeat("a", 64), "", false}, + {"not-hex-12345", "", false}, + } + for _, tc := range cases { + got, ok := digestPrefix(tc.in) + if ok != tc.wantOK || got != tc.want { + t.Errorf("digestPrefix(%q) = (%q, %v), want (%q, %v)", tc.in, got, ok, tc.want, tc.wantOK) + } + } +} + +func TestResolvePinnedTarget(t *testing.T) { + digestA := "sha256:" + strings.Repeat("a", 64) + digestB := "sha256:" + strings.Repeat("b", 64) + digestAmbiguous := "sha256:" + strings.Repeat("a", 12) + strings.Repeat("c", 52) + base := refPins{ + "local:a": digestA, + "local:b": digestB, + } + ambiguous := refPins{ + "local:a": digestA, + "local:b": digestB, + "local:ambiguous": digestAmbiguous, + } + + cases := []struct { + name string + pins refPins + target string + wantRef string + wantDigest string + wantErr string // when set, expect an error containing this and ignore wantRef/wantDigest + }{ + {name: "exact ref", pins: base, target: "local:a", wantRef: "local:a", wantDigest: digestA}, + {name: "unique prefix", pins: base, target: strings.Repeat("b", 12), wantRef: "local:b", wantDigest: digestB}, + {name: "uppercase prefix", pins: base, target: "sha256:" + strings.Repeat("B", 12), wantRef: "local:b", wantDigest: digestB}, + {name: "missing digest", pins: base, target: strings.Repeat("d", 12), wantErr: "not pulled"}, + {name: "invalid target", pins: base, target: "not-a-ref", wantErr: "not pulled"}, + {name: "ambiguous prefix", pins: ambiguous, target: strings.Repeat("a", 12), wantErr: "ambiguous"}, + {name: "invalid pinned digest", pins: refPins{"bad": "not-a-digest"}, target: strings.Repeat("e", 12), wantErr: "pinned digest"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ref, digest, err := resolvePinnedTarget(tc.pins, tc.target) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want substring %q", err, tc.wantErr) + } + return + } + if err != nil || ref != tc.wantRef || digest != tc.wantDigest { + t.Fatalf("resolve = ref=%q digest=%q err=%v, want %s %s", ref, digest, err, tc.wantRef, tc.wantDigest) + } + }) + } + + // The ambiguous case must list the colliding refs in sorted order. + if _, _, err := resolvePinnedTarget(ambiguous, strings.Repeat("a", 12)); err == nil || + !strings.Contains(err.Error(), "local:a, local:ambiguous") { + t.Fatalf("ambiguous err = %v, want sorted ambiguous refs", err) + } +} + +func TestRmiByDigestPrefixReportsResolvedRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + prefix := strings.TrimPrefix(digest, "sha256:")[:12] + rep, err := s.rmi(prefix, false) + if err != nil { + t.Fatalf("rmi by prefix: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after rmi by digest prefix") + } + + s = openTestStore(t) + digest, err = s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + prefix = strings.TrimPrefix(digest, "sha256:")[:12] + _, stderr, err := captureOutput(t, func() error { + return cmdRmi([]string{"--store", s.root, prefix}) + }) + if err != nil { + t.Fatalf("cmdRmi by prefix: %v", err) + } + if !strings.Contains(stderr, "Removed local:a:") || strings.Contains(stderr, "Removed "+prefix+":") { + t.Fatalf("cmdRmi stderr = %q, want resolved ref in summary", stderr) + } +} + +type fakeFileInfo struct { + size int64 +} + +func (f fakeFileInfo) Name() string { return "fake" } +func (f fakeFileInfo) Size() int64 { return f.size } +func (f fakeFileInfo) Mode() os.FileMode { return 0o644 } +func (f fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (f fakeFileInfo) IsDir() bool { return false } +func (f fakeFileInfo) Sys() any { return nil } diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go index 6c9e8acc..1d4b4d90 100644 --- a/cmd/elfuse-oci/test_helpers_test.go +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -11,7 +11,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "strings" "testing" @@ -39,7 +38,7 @@ func TestMain(m *testing.M) { UID: 1, GID: 2, } - if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec); err != nil { + if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec, nil); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(98) } @@ -48,10 +47,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func sliceContains(xs []string, want string) bool { - return slices.Contains(xs, want) -} - func captureOutput(t *testing.T, fn func() error) (string, string, error) { t.Helper() @@ -178,3 +173,18 @@ func runMainSubprocess(t *testing.T, args ...string) (string, string, error) { waitErr := cmd.Run() return outB.String(), errB.String(), waitErr } + +// legacyCacheNameForRef reproduces the pre-digest cache naming scheme, which +// flattened the ref itself into a single (intentionally lossy) path +// component. New caches are keyed by digest (cacheKeyForDigest). prune +// --cache recognizes legacy caches purely by their top-level directory name +// (anything under rootfs/ or cs/ that is not "sha256"), never through this +// helper; it lives with the tests that fabricate legacy caches, documenting +// the old layout. +func legacyCacheNameForRef(ref string) string { + return strings.NewReplacer("/", "_", ":", "_", "@", "_").Replace(ref) +} + +func legacyRootfsForRef(store, ref string) string { + return filepath.Join(store, rootfsCacheDirName, legacyCacheNameForRef(ref)) +} diff --git a/cmd/elfuse-oci/unpack.go b/cmd/elfuse-oci/unpack.go index 67221d2e..7e4c9261 100644 --- a/cmd/elfuse-oci/unpack.go +++ b/cmd/elfuse-oci/unpack.go @@ -16,9 +16,19 @@ import ( "github.com/google/go-containerregistry/pkg/v1" ) -// unpackImage extracts every layer of the stored image (base first) into a -// plain directory rootfs. Each layer is a tar stream (crane decompresses gzip -// and zstd transparently via layer.Uncompressed). +// rootfsStagingSuffix marks unpackImage's pre-rename staging siblings, +// . rootfsCacheLockPath parses it to map a staging +// dir back to its digest's lock, so the two must never drift apart. +const rootfsStagingSuffix = ".tmp-" + +// unpackImage extracts every layer of img (base first) into a plain directory +// rootfs. Each layer is a tar stream (crane decompresses gzip and zstd +// transparently via layer.Uncompressed). +// +// img is the caller's already-resolved image, not a ref re-resolved here: the +// caller keys caches by the digest it resolved, and a re-resolution could +// observe a different pin (a concurrent repull) and fill a digest-keyed cache +// with another image's content. // // Layer application implements the OCI whiteout conventions: // - `.wh.` in a directory removes `` (from this and lower layers). @@ -62,11 +72,7 @@ func storeRootfsPublished(path string) (bool, error) { return true, nil } -func unpackImage(s *store, ref, dest string) error { - img, err := s.image(ref) - if err != nil { - return err - } +func unpackImage(img v1.Image, dest string) error { if _, statErr := os.Lstat(dest); statErr == nil { // An explicit pre-existing --rootfs directory: merge in place and // never remove it, failed or not; it is not ours to delete. @@ -83,7 +89,7 @@ func unpackImage(s *store, ref, dest string) error { if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return err } - tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+".tmp-") + tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+rootfsStagingSuffix) if err != nil { return err } @@ -128,6 +134,35 @@ func unpackInto(img v1.Image, dest string) error { return nil } +// layerPaths tracks what the current layer has created, so a late opaque +// whiteout does not wipe the layer's own additions. entries holds the exact tar +// entry paths; subtree additionally holds every ancestor directory of those +// entries, so an opaque clear can preserve an implicit parent directory the +// layer never gave its own tar entry (e.g. a layer with dir/sub/file but no +// dir/sub entry). +type layerPaths struct { + entries map[string]bool + subtree map[string]bool +} + +func newLayerPaths() layerPaths { + return layerPaths{entries: map[string]bool{}, subtree: map[string]bool{}} +} + +// add records name as a current-layer entry and marks name and all its ancestor +// directories as carrying current-layer content. +func (lp layerPaths) add(name string) { + lp.entries[name] = true + for p := name; p != "." && p != string(filepath.Separator); { + lp.subtree[p] = true + parent := filepath.Dir(p) + if parent == p { + break + } + p = parent + } +} + func applyLayer(root *os.Root, layer v1.Layer) error { r, err := layer.Uncompressed() if err != nil { @@ -138,7 +173,7 @@ func applyLayer(root *os.Root, layer v1.Layer) error { // content only, but tar entry order within a layer is not guaranteed: an // opaque marker may arrive after the directory's own same-layer children, // which must survive the clear (Docker's unpacker keeps the same set). - applied := make(map[string]bool) + lp := newLayerPaths() tr := tar.NewReader(r) for { hdr, err := tr.Next() @@ -148,7 +183,7 @@ func applyLayer(root *os.Root, layer v1.Layer) error { if err != nil { return fmt.Errorf("read tar entry: %w", err) } - if err := applyEntry(root, hdr, tr, applied); err != nil { + if err := applyEntry(root, hdr, tr, lp); err != nil { return fmt.Errorf("entry %q: %w", hdr.Name, err) } } @@ -179,10 +214,11 @@ func rootRelative(cleaned string) string { return strings.TrimPrefix(cleaned, "/") } -// applyEntry applies one tar header to the rootfs. applied tracks the paths -// the current layer has created so far, so an opaque whiteout arriving after -// its directory's same-layer children does not delete them. -func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string]bool) error { +// applyEntry applies one tar header to the rootfs. lp tracks the paths the +// current layer has created so far, so an opaque whiteout arriving after its +// directory's same-layer children (or their implicit parents) does not delete +// them. +func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, lp layerPaths) error { name := filepath.Clean(hdr.Name) if strings.HasPrefix(name, "../") || name == ".." { return fmt.Errorf("unsafe entry path %q", hdr.Name) @@ -195,7 +231,7 @@ func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string] base := filepath.Base(name) if base == opaqueMarker { - return clearDirectory(root, filepath.Dir(name), applied) + return clearDirectory(root, filepath.Dir(name), lp) } if trimmed, ok := strings.CutPrefix(base, whiteoutPrefix); ok { // A whiteout names one sibling to remove, so the suffix must be a @@ -225,7 +261,7 @@ func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string] } return root.RemoveAll(target) } - applied[name] = true + lp.add(name) // hdr.FileInfo().Mode() maps the tar header's unix mode bits to os.FileMode // with the special bits (ModeSetuid/Setgid/Sticky) at os.FileMode's high @@ -473,7 +509,7 @@ func shouldDropSpecial(err error, special os.FileMode) bool { // keeping entries the current layer itself created: opaque markers hide // lower-layer content, and a marker ordered after its directory's same-layer // additions must not wipe them. -func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { +func clearDirectory(root *os.Root, dir string, lp layerPaths) error { fi, err := root.Lstat(dir) if err != nil { if os.IsNotExist(err) { @@ -488,7 +524,7 @@ func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { // under this name anyway, mirroring the non-dir replacement mkdirAll and // the regular-file path perform. if !fi.IsDir() { - if applied[dir] { + if lp.entries[dir] { // This layer created the non-dir itself; there is no lower // content beneath it for the marker to hide. return nil @@ -512,7 +548,11 @@ func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { } for _, e := range entries { child := filepath.Join(dir, e.Name()) - if applied[child] { + // Keep a child the current layer created OR that has current-layer + // content beneath it: an implicit parent directory (dir/sub with no + // own tar entry, created for dir/sub/file) has no entries[child] but is + // in subtree, and deleting it would take the layer's own file with it. + if lp.subtree[child] { continue } if err := root.RemoveAll(child); err != nil { diff --git a/cmd/elfuse-oci/unpack_image_test.go b/cmd/elfuse-oci/unpack_image_test.go index 11bf85bd..f4640227 100644 --- a/cmd/elfuse-oci/unpack_image_test.go +++ b/cmd/elfuse-oci/unpack_image_test.go @@ -25,6 +25,20 @@ type tarEntry struct { body string } +// unpackRef resolves ref to its pinned image and unpacks it into dest, the +// two-step the production callers do (resolve under the store lock, then +// unpack the resolved image). It fails the test if the ref cannot be +// resolved; the returned error is unpackImage's, so callers testing unpack +// failures still see them. +func unpackRef(t *testing.T, s *store, ref, dest string) error { + t.Helper() + img, err := s.image(ref) + if err != nil { + t.Fatalf("resolve %s: %v", ref, err) + } + return unpackImage(img, dest) +} + func testTarLayer(t *testing.T, entries ...tarEntry) v1.Layer { t.Helper() var buf bytes.Buffer @@ -105,7 +119,7 @@ func TestUnpackImageAppliesLayersInOrder(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:layered", dest); err != nil { + if err := unpackRef(t, s, "local:layered", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -151,7 +165,7 @@ func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { parent := t.TempDir() dest := filepath.Join(parent, "rootfs") - if err := unpackImage(s, "local:partial", dest); err == nil { + if err := unpackRef(t, s, "local:partial", dest); err == nil { t.Fatal("unpackImage succeeded, want failure on fifo entry") } if _, err := os.Lstat(dest); !os.IsNotExist(err) { @@ -170,7 +184,7 @@ func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { t.Fatal(err) } - if err := unpackImage(s, "local:partial", pre); err == nil { + if err := unpackRef(t, s, "local:partial", pre); err == nil { t.Fatal("unpackImage succeeded, want failure on fifo entry") } if _, err := os.Stat(sentinel); err != nil { @@ -194,7 +208,7 @@ func TestUnpackImagePreexistingDestUnpacksInPlace(t *testing.T) { if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { t.Fatal(err) } - if err := unpackImage(s, "local:merge", dest); err != nil { + if err := unpackRef(t, s, "local:merge", dest); err != nil { t.Fatalf("unpackImage: %v", err) } if b, err := os.ReadFile(sentinel); err != nil || string(b) != "keep" { @@ -221,7 +235,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { root, dir := newRoot(t) for _, name := range []string{".", "/"} { h := regHeader(name, 0o644, 0) - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry root no-op %q: %v", name, err) } } @@ -234,7 +248,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { } hardlink := linkHeader("escape-link", "../outside") - if err := applyEntry(root, &hardlink, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hardlink, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatal("applyEntry accepted hardlink target escaping root") } @@ -246,7 +260,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { t.Fatal(err) } file := regHeader("replace-me", 0o644, int64(len("inside"))) - if err := applyEntry(root, &file, strings.NewReader("inside"), map[string]bool{}); err != nil { + if err := applyEntry(root, &file, strings.NewReader("inside"), newLayerPaths()); err != nil { t.Fatalf("applyEntry replacing symlink: %v", err) } if b, err := os.ReadFile(outside); err != nil || string(b) != "outside" { @@ -264,7 +278,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("unsupported tar type", func(t *testing.T) { root, _ := newRoot(t) h := tar.Header{Name: "weird", Typeflag: 'x'} - err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}) + err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()) if err == nil || !strings.Contains(err.Error(), "unsupported tar type") || !strings.Contains(err.Error(), tarTypeName('x')) { t.Fatalf("unsupported type err = %v, want tar type error", err) } @@ -273,7 +287,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("absolute symlink to root", func(t *testing.T) { root, dir := newRoot(t) h := symHeader("usr/root-link", "/") - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry symlink to root: %v", err) } target, err := os.Readlink(filepath.Join(dir, "usr", "root-link")) @@ -288,14 +302,14 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("parent path is regular file", func(t *testing.T) { root, dir := newRoot(t) parent := regHeader("parent", 0o644, 0) - if err := applyEntry(root, &parent, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &parent, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatal(err) } // A non-directory on the parent path is replaced with a real // directory (containerd/Docker behavior), not an error: layers may // legitimately turn a lower layer's file into a directory. child := regHeader("parent/child", 0o644, 0) - if err := applyEntry(root, &child, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &child, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry child under regular-file parent: %v, want file replaced by directory", err) } fi, err := os.Lstat(filepath.Join(dir, "parent")) @@ -310,7 +324,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque missing directory is no-op", func(t *testing.T) { root, _ := newRoot(t) h := tar.Header{Name: "missing/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("opaque missing dir: %v", err) } }) @@ -318,14 +332,14 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque marker under lower-layer regular file replaces it", func(t *testing.T) { root, dir := newRoot(t) file := regHeader("notdir", 0o644, 0) - if err := applyEntry(root, &file, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &file, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatal(err) } // The opaque marker hides all lower content under the name, so a // lower layer's non-directory there is replaced with an empty real // directory rather than cleared through or rejected. h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("opaque marker under regular file: %v, want file replaced by empty directory", err) } fi, err := os.Lstat(filepath.Join(dir, "notdir")) @@ -337,12 +351,12 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque marker under same-layer regular file is a no-op", func(t *testing.T) { root, dir := newRoot(t) file := regHeader("notdir", 0o644, 0) - applied := map[string]bool{} - if err := applyEntry(root, &file, strings.NewReader(""), applied); err != nil { + lp := newLayerPaths() + if err := applyEntry(root, &file, strings.NewReader(""), lp); err != nil { t.Fatal(err) } h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), applied); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), lp); err != nil { t.Fatalf("opaque marker under same-layer file: %v, want no-op", err) } fi, err := os.Lstat(filepath.Join(dir, "notdir")) @@ -403,7 +417,7 @@ func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:opq-late", dest); err != nil { + if err := unpackRef(t, s, "local:opq-late", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -415,6 +429,40 @@ func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { } } +// TestUnpackOpaqueAfterImplicitParentChildren: a late opaque marker +// must preserve the current layer's descendants even when their immediate +// parent directory has no tar entry of its own (an implicit parent, created +// only because a deeper file needed it). Here the upper layer writes +// dir/sub/file with NO dir/sub entry, then an opaque marker on dir: dir/sub and +// its file must survive while the lower layer's dir/old is cleared. +func TestUnpackOpaqueAfterImplicitParentChildren(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("dir", 0o755)}, + tarEntry{header: regHeader("dir/old", 0o644, 0), body: "hidden"}, + ) + upper := testTarLayer(t, + // No dir/sub entry: the parent is implicit, created for dir/sub/file. + tarEntry{header: regHeader("dir/sub/file", 0o644, 0), body: "kept"}, + tarEntry{header: tar.Header{Name: "dir/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opq-implicit", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackRef(t, s, "local:opq-implicit", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if _, err := os.Stat(filepath.Join(dest, "dir", "old")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden dir/old = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "dir", "sub", "file")); err != nil || string(b) != "kept" { + t.Fatalf("dir/sub/file = %q, err=%v; want the implicit-parent child to survive the late marker", b, err) + } +} + // TestUnpackRejectsBareWhiteout pins that a malformed ".wh." entry with no // target suffix fails extraction instead of resolving to Join(dir, "") and // deleting the containing directory. @@ -433,7 +481,7 @@ func TestUnpackRejectsBareWhiteout(t *testing.T) { defer root.Close() hdr := tar.Header{Name: "opt/.wh.", Typeflag: tar.TypeReg, Mode: 0o644} - if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatal("bare .wh. entry applied, want invalid-whiteout error") } if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { @@ -470,7 +518,7 @@ func TestUnpackRejectsDotWhiteout(t *testing.T) { defer root.Close() hdr := tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o644} - if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()); err == nil { t.Errorf("%s entry applied, want invalid-whiteout error", name) } if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { @@ -499,7 +547,7 @@ func TestUnpackDirReplacesLowerSymlink(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:dir-over-link", dest); err != nil { + if err := unpackRef(t, s, "local:dir-over-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -537,7 +585,7 @@ func TestUnpackParentSymlinkReplaced(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:parent-link", dest); err != nil { + if err := unpackRef(t, s, "local:parent-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -573,7 +621,7 @@ func TestUnpackDirEntryParentSymlinkReplaced(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:parent-link-dir", dest); err != nil { + if err := unpackRef(t, s, "local:parent-link-dir", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -612,7 +660,7 @@ func TestUnpackOpaqueThroughSymlinkKeepsTarget(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:opaque-link", dest); err != nil { + if err := unpackRef(t, s, "local:opaque-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -657,7 +705,7 @@ func TestUnpackWhiteoutThroughSymlinkKeepsTarget(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:whiteout-link", dest); err != nil { + if err := unpackRef(t, s, "local:whiteout-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -712,7 +760,7 @@ func TestUnpackFailsOnCorruptGzipTrailer(t *testing.T) { t.Fatal(err) } - if err := unpackImage(s, "local:trailer", t.TempDir()); err == nil { + if err := unpackRef(t, s, "local:trailer", t.TempDir()); err == nil { t.Fatal("unpackImage succeeded on a corrupted gzip trailer, want error") } } diff --git a/cmd/elfuse-oci/unpack_test.go b/cmd/elfuse-oci/unpack_test.go index 85630c33..bd6419fb 100644 --- a/cmd/elfuse-oci/unpack_test.go +++ b/cmd/elfuse-oci/unpack_test.go @@ -27,18 +27,17 @@ func newRoot(t *testing.T) (*os.Root, string) { func applyEntries(t *testing.T, root *os.Root, entries []tar.Header) { t.Helper() - // One shared applied-set across all entries, matching applyLayer's one - // per layer: the helper models a single layer, so same-layer tracking - // must span the entries. Callers model a layer boundary as a separate - // call. - applied := map[string]bool{} + // One layerPaths across all entries, matching applyLayer's one per + // layer: the helper models a single layer, so same-layer tracking must + // span the entries. Callers model a layer boundary as a separate call. + lp := newLayerPaths() for _, h := range entries { var content []byte if (h.Typeflag == tar.TypeReg || h.Typeflag == tar.TypeRegA) && h.Size > 0 { content = []byte(strings.Repeat("x", int(h.Size))) } hdr := h - if err := applyEntry(root, &hdr, bytes.NewReader(content), applied); err != nil { + if err := applyEntry(root, &hdr, bytes.NewReader(content), lp); err != nil { t.Fatalf("applyEntry %q: %v", h.Name, err) } } @@ -48,7 +47,7 @@ func applyEntryWithContent(t *testing.T, root *os.Root, h tar.Header, content st t.Helper() hdr := h hdr.Size = int64(len(content)) - if err := applyEntry(root, &hdr, strings.NewReader(content), map[string]bool{}); err != nil { + if err := applyEntry(root, &hdr, strings.NewReader(content), newLayerPaths()); err != nil { t.Fatalf("applyEntry %q: %v", h.Name, err) } } @@ -358,7 +357,7 @@ func TestUnpackSpecialFilesRejected(t *testing.T) { t.Run(tc.want, func(t *testing.T) { root, dir := newRoot(t) hdr := tar.Header{Name: tc.name, Mode: 0o644, Typeflag: tc.typeflag} - err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}) + err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()) if err == nil || !strings.Contains(err.Error(), "unsupported special file type "+tc.want) { t.Fatalf("applyEntry special %s err = %v, want unsupported error", tc.want, err) } @@ -372,7 +371,7 @@ func TestUnpackSpecialFilesRejected(t *testing.T) { func TestUnpackPathEscapeRejected(t *testing.T) { root, _ := newRoot(t) hdr := regHeader("../escape", 0o644, 0) - if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatalf("applyEntry accepted ../escape path") } } @@ -398,7 +397,7 @@ func TestUnpackReadsExactSize(t *testing.T) { content := "abc123" r := &countingReader{b: []byte(content + "TRAILING-JUNK")} hdr := regHeader("f", 0o644, int64(len(content))) - if err := applyEntry(root, &hdr, r, map[string]bool{}); err != nil { + if err := applyEntry(root, &hdr, r, newLayerPaths()); err != nil { t.Fatalf("applyEntry: %v", err) } if r.n != len(content) { @@ -410,7 +409,7 @@ func TestUnpackReadsExactSize(t *testing.T) { short := &countingReader{b: []byte("abc")} hdr = regHeader("g", 0o644, int64(len(content))) - if err := applyEntry(root, &hdr, short, map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, short, newLayerPaths()); err == nil { t.Fatal("applyEntry accepted a short body, want size-mismatch error") } } From 07379fd4104d287261fc2cc37d2de6ea130b99ea Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:38:02 +0200 Subject: [PATCH 08/26] Add OCI conformance checks and lifecycle CI The on-disk store is the contract: pulls must produce a valid OCI image-layout that other tools can read and that agrees with registry truth on the manifest digest. Conformance tests pin the layout shape, and scripts/oci-interop.sh cross-checks the store with crane, skopeo, and umoci. CI splits by runner capability: a Linux job runs the pure-Go store paths (pull, unpack, inspect, lifecycle) without Hypervisor.framework, a hosted macOS job builds and tests the darwin sparsebundle code and drives a run-less pull/inspect/list/rmi/prune lifecycle smoke, and the self-hosted release leg boots guests end to end under HVF: the alpine:3 default-entrypoint smoke plus a full pull -> inspect -> list -> run -> rmi -> prune lifecycle that runs python:3.12-slim with an --entrypoint override and asserts the teardown half of the lifecycle. The lifecycle teardown covers all three reclamation guardrails: a plain rmi reclaims the cold cache with the image, run --keep output refuses rmi without --force, and a live --plain-rootfs guest parked in sleep pins its cache; prune --cache --all must skip it and rmi must refuse until the guest exits, the only end-to-end exercise of the run-lock descriptor riding through the exec into elfuse. The guest dies by SIGKILL, so the reclamation that follows proves the kernel dropped the flock, not that a graceful teardown ran. The run smoke ends with prune --cache, keeping the persistent store's stranded caches, and not only its blobs, bounded across tag moves. The store durability test drives the writer's rename-failure branch (a non-empty directory at the destination), pinning the staging cleanup the read-only-dir injection cannot reach. --- .github/workflows/main.yml | 192 +++++++++++- Makefile | 35 ++- cmd/elfuse-oci/store_conformance_test.go | 371 +++++++++++++++++++++++ scripts/ci/oci-cli-smoke.sh | 111 +++++++ scripts/ci/oci-lib.sh | 154 ++++++++++ scripts/ci/oci-lifecycle.sh | 157 ++++++++++ scripts/ci/oci-python-workload.py | 117 +++++++ scripts/ci/oci-run-smoke.sh | 57 ++++ scripts/oci-interop.sh | 205 +++++++++++++ 9 files changed, 1393 insertions(+), 6 deletions(-) create mode 100644 cmd/elfuse-oci/store_conformance_test.go create mode 100755 scripts/ci/oci-cli-smoke.sh create mode 100755 scripts/ci/oci-lib.sh create mode 100755 scripts/ci/oci-lifecycle.sh create mode 100644 scripts/ci/oci-python-workload.py create mode 100755 scripts/ci/oci-run-smoke.sh create mode 100755 scripts/oci-interop.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6276c986..4e9a5903 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,12 @@ # scan-macos : LLVM scan-build via `make analyze` # infer-macos : Facebook Infer capture + analyze over the full build # runtime-macos : HVF runtime tests on self-hosted Apple Silicon, -# including release, ASAN, UBSAN, and TSAN variants +# including release, ASAN, UBSAN, and TSAN variants, +# plus the end-to-end OCI run and image-lifecycle checks +# oci-conformance : OCI image-layout conformance + cross-tool interop +# (crane/skopeo/umoci) on Linux +# oci-image-macos : elfuse-oci darwin build, unit tests, sparsebundle +# round-trip, and a run-less image-lifecycle smoke # # Runtime and sanitizer tests require Hypervisor.framework, which # GitHub-hosted macOS runners do not expose. Those tests run on self-hosted @@ -94,12 +99,13 @@ jobs: run: .ci/check-security.sh - name: shellcheck - # Scoped to .ci/ -- tests/ has pre-existing warnings that the - # repository's own check-format target already surfaces. + # Scoped to .ci/ and scripts/; tests/ has pre-existing warnings that + # the repository's own check-format target already surfaces. if: ${{ !cancelled() }} run: | set -euo pipefail - mapfile -d '' files < <(git ls-files -z -- '.ci/*.sh') + # git pathspec globs cross '/', so 'scripts/*.sh' covers scripts/ci/ too. + mapfile -d '' files < <(git ls-files -z -- '.ci/*.sh' 'scripts/*.sh') shellcheck --severity=warning "${files[@]}" - name: cppcheck @@ -460,7 +466,7 @@ jobs: echo "Run targets : $RUN_SHA" echo "PR HEAD now : ${latest:-}" if [ -n "$latest" ] && [ "$latest" != "$RUN_SHA" ]; then - echo "::error::This run targets $RUN_SHA, but PR #$PR_NUMBER HEAD is now $latest -- the commit is no longer the latest. Failing instead of re-testing stale code on the self-hosted runner; re-run CI on the current commit." + echo "::error::This run targets $RUN_SHA, but PR #$PR_NUMBER HEAD is now $latest, so the commit is no longer the latest. Failing instead of re-testing stale code on the self-hosted runner; re-run CI on the current commit." exit 1 fi @@ -551,6 +557,15 @@ jobs: ls -l "$ROSETTA" + - name: Set up Go + # Only the release leg runs the OCI run smoke below, which needs the Go + # toolchain to build build/elfuse-oci. + if: ${{ matrix.run_matrix }} + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + - name: Build elfuse # make does not track EXTRA_CFLAGS changes, so an object built for one # sanitizer must not be reused for another. Checkout already wipes @@ -580,6 +595,44 @@ jobs: run: | make EXTRA_CFLAGS="$EXTRA_CFLAGS" ${{ matrix.check_target }} + - name: OCI run smoke (pull -> sparsebundle -> COW clone -> HVF boot) + # The only leg that exercises the full default `run` path end to end: + # pull an image, provision the case-sensitive sparsebundle, COW-clone it, + # boot the guest under HVF, and propagate its exit status. Release leg + # only (sanitizer legs skip the fixture/qemu-heavy paths). + if: ${{ matrix.run_matrix }} + run: | + set -euo pipefail + # Same persistent-disk convention as the fixture cache above: + # checkout wipes the workspace but this self-hosted runner's disk + # survives. pull is idempotent per digest, so a warm store skips + # the image blob downloads (only the manifest HEAD/GET goes + # out) and the warm sparsebundle cache skips the unpack. env: + # values don't expand $HOME, so export here instead. + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-store" + make build/elfuse-oci + scripts/ci/oci-run-smoke.sh + + - name: OCI image lifecycle (pull -> inspect -> list -> run -> rmi -> prune) + # Walks the whole user-facing image lifecycle on the only leg with + # HVF. python:3.12-slim adds what the alpine smoke above does not: + # an --entrypoint override, a glibc dynamically-linked guest, and + # the teardown half of the lifecycle (see the phase functions in + # scripts/ci/oci-lifecycle.sh). Separate store from the smoke step + # so the empty-store assertions are meaningful. Release leg only. + if: ${{ matrix.run_matrix }} + env: + ELFUSE_OCI_STORE: ${{ runner.temp }}/oci-lifecycle-store + IMG: python:3.12-slim + run: | + set -euo pipefail + # Built by the smoke step above; the make target is idempotent. + make build/elfuse-oci + # The seed store lives on the runner's persistent disk (env: does + # not expand $HOME, so export here instead). + export ELFUSE_OCI_SEED_STORE="$HOME/.cache/elfuse-ci/oci-seed-store" + scripts/ci/oci-lifecycle.sh + - name: Test matrix if: ${{ matrix.run_matrix }} run: | @@ -611,3 +664,132 @@ jobs: else echo "No externals/test-fixtures to save" fi + + # OCI image-layout conformance + cross-tool interop on Linux. elfuse-oci + # is pure Go (no Hypervisor.framework), so pull/inspect/unpack and the + # conformance tests run in hosted CI; only `run` needs HVF and is excluded. + # The on-disk store is the contract: it must be a valid OCI image-layout that + # crane/skopeo/umoci can read and that agrees with registry truth. + oci-conformance: + name: OCI conformance + interop (Linux) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + # Pinned to elfuse-oci's go-containerregistry version so the crane + # CLI reads layouts with the same schema handling it writes with. + GGCR_VERSION: v0.21.7 + # umoci release tag for the interop gate; built from a checkout below. + UMOCI_VERSION: v0.6.0 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Install jq + skopeo + # skopeo reads our layout via the oci: transport. CI treats it as part + # of the conformance gate; local runs may omit it and get a skipped + # interop section from scripts/oci-interop.sh. + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y jq skopeo + + - name: Install crane + umoci from source + # crane (registry-truth comparison) and umoci (layout parse) are Go + # tools; install crane at elfuse-oci's ggcr version where applicable. + run: | + set -euo pipefail + go install github.com/google/go-containerregistry/cmd/crane@${GGCR_VERSION} + # `go install pkg@version` refuses umoci: its go.mod carries replace + # directives. Build from a pinned checkout instead, where replace + # directives apply; a read-only `umoci list --layout` conformance + # check needs nothing newer. + git clone --quiet --depth 1 --branch "$UMOCI_VERSION" \ + https://github.com/opencontainers/umoci.git "$RUNNER_TEMP/umoci" + (cd "$RUNNER_TEMP/umoci" && \ + go build -o "$(go env GOPATH)/bin/umoci" ./cmd/umoci) + echo "$(go env GOPATH)/bin" >>"$GITHUB_PATH" + + - name: Build elfuse-oci + # Pure Go target; does not require the C toolchain or HVF. + run: make build/elfuse-oci + + - name: Go fmt + vet (Linux and darwin cross-check) + # The Makefile gate, so CI and local runs cannot drift. oci-lint vets + # native, darwin/arm64, and linux; the darwin pass compile-checks the + # sparsebundle files (csrun.go, sparsebundle.go, cache_darwin.go) + # that never build on this Linux runner. + run: make oci-lint + + - name: CLI lifecycle smoke (pull/list/inspect/unpack/rmi/prune) + # Exercises the built binary through the same user-facing flow that the + # Go unit tests model in-process. `run` itself remains covered by Go + # orchestration tests here and by macOS/HVF runtime jobs. --unpack adds + # the unpack + cache-reclaiming rmi phase to the shared smoke. + run: scripts/ci/oci-cli-smoke.sh --unpack + + - name: Go unit + conformance tests (with network pull round-trip) + # ELFUSE_OCI_NETTEST enables the pull round-trip that re-opens the store + # with crane's independent layout reader and asserts digest agreement. + env: + ELFUSE_OCI_NETTEST: "1" + run: go test -race ./cmd/elfuse-oci/ + + - name: Cross-tool interop (crane + skopeo + umoci) + # Pulls fixtures, then asserts the on-disk layout is spec-shaped and + # that available tools read it and agree with registry truth. + run: scripts/oci-interop.sh + + # Darwin elfuse-oci build + tests on a hosted macOS runner. The default `run` + # path (csrun.go, sparsebundle.go, cache_darwin.go) only compiles on darwin, so + # the Linux job above can only cross-vet it; this job actually builds and runs + # it, and drives the run-less image lifecycle (pull/inspect/list/rmi/prune) + # through the darwin binary. Hosted runners provide hdiutil + case-sensitive + # APFS (so the real sparsebundle round-trip runs) even though they lack + # Hypervisor.framework; the HVF-backed guest boot is covered by the + # self-hosted runtime-macos job. + oci-image-macos: + name: OCI image CLI (macOS Apple Silicon) + runs-on: macos-15 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Build elfuse-oci + run: make build/elfuse-oci + + - name: Go unit tests (darwin native) + # Runs the whole suite on darwin, exercising the sparsebundle/clone seams + # in csrun/sparsebundle/cache_darwin that the Linux job cannot compile. + # -race: the darwin concurrency-sensitive code (sweep/provision/clone, + # cache-removal lock discipline) compiles only here, so this is the only + # place the race detector ever sees it. + run: go test -race ./cmd/elfuse-oci/ + + - name: Sparsebundle round-trip (hdiutil + case-sensitive APFS) + # ELFUSE_OCI_DARWIN_CS un-skips the real hdiutil create/attach/detach + + # case-sensitive APFS sweep; hosted runners have hdiutil and APFS. + env: + ELFUSE_OCI_DARWIN_CS: "1" + run: go test -run TestDarwinCSSweep ./cmd/elfuse-oci/ + + - name: CLI lifecycle smoke (pull/inspect/list/rmi/prune, no HVF) + # The same shared smoke the Linux job drives, but through the darwin + # binary: everything short of `run` (which needs HVF) works end to + # end on a hosted runner. Without --unpack every rmi takes the + # cache-free path (nothing was ever unpacked, no --force involved) + # that the Linux job's cache-reclaiming flow does not cover. jq + # ships on the macos-15 image. + run: scripts/ci/oci-cli-smoke.sh diff --git a/Makefile b/Makefile index cbbfe4c9..b037e7d6 100644 --- a/Makefile +++ b/Makefile @@ -148,12 +148,45 @@ OCI_SRCS := $(shell find cmd/elfuse-oci -type f -name '*.go' 2>/dev/null) .PHONY: elfuse-oci elfuse-oci: $(OCI_BIN) +# OCI image-layout conformance + cross-tool interop. Pulls fixtures into a +# throwaway store and asserts the on-disk layout is spec-shaped and readable by +# crane/skopeo/umoci (whichever are installed locally; all are required in CI). +# Pure Go + jq; no HVF, runs on Linux. Requires network to pull fixtures. +.PHONY: oci-interop +oci-interop: $(OCI_BIN) + $(Q)scripts/oci-interop.sh + +# Go unit tests for the OCI image CLI (offline). Set ELFUSE_OCI_NETTEST=1 +# to also exercise the network pull round-trip conformance test. +.PHONY: oci-test +oci-test: + $(Q)$(GO) test ./cmd/elfuse-oci/ + +# gofmt + go vet gate for the OCI image CLI. `go vet` is run for both GOOS +# values so the darwin-only sparsebundle files are checked from Linux CI and the +# non-darwin stubs are checked from a macOS host. The darwin pass pins +# GOARCH=arm64 (the only supported darwin target, and what CI checks) so an +# amd64 Linux host does not silently vet darwin/amd64 instead. oci-lint +# bundles both so a local run matches the CI gate. +.PHONY: oci-vet oci-fmt-check oci-lint +oci-vet: + $(Q)GOOS=darwin GOARCH=arm64 $(GO) vet ./cmd/elfuse-oci/ + $(Q)GOOS=linux $(GO) vet ./cmd/elfuse-oci/ + +oci-fmt-check: + $(Q)out="$$(gofmt -l cmd/elfuse-oci)"; \ + if [ -n "$$out" ]; then \ + echo "gofmt needs to run on:"; echo "$$out"; exit 1; \ + fi + +oci-lint: oci-fmt-check oci-vet + # rm -f first: `go build -o` follows an existing symlink at the output path, # so a stale build/elfuse-oci symlink would clobber build/elfuse. $(OCI_BIN): go.mod $(OCI_SRCS) $(VERSION_DEPS) | $(BUILD_DIR) @echo " GO $@" $(Q)rm -f $@ - $(Q)cd $(CURDIR) && $(GO) build -ldflags "-X main.version=$(VERSION)" \ + $(Q)$(GO) build -ldflags "-X main.version=$(VERSION)" \ -o $@ ./cmd/elfuse-oci # Native test binaries (macOS, Hypervisor.framework) diff --git a/cmd/elfuse-oci/store_conformance_test.go b/cmd/elfuse-oci/store_conformance_test.go new file mode 100644 index 00000000..229ea227 --- /dev/null +++ b/cmd/elfuse-oci/store_conformance_test.go @@ -0,0 +1,371 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" +) + +// TestStoreLayoutFiles asserts openStore creates the OCI image-layout +// scaffolding exactly: an oci-layout file with imageLayoutVersion 1.0.0, an +// index.json with an empty manifests array, and a blobs/sha256/ tree. This is +// the part of the OCI image-layout spec every conforming reader expects. +func TestStoreLayoutFiles(t *testing.T) { + s := openTestStore(t) + + b, err := os.ReadFile(filepath.Join(s.root, "oci-layout")) + if err != nil { + t.Fatal(err) + } + var lay struct { + Version string `json:"imageLayoutVersion"` + } + if err := json.Unmarshal(b, &lay); err != nil { + t.Fatalf("oci-layout is not JSON: %v (raw %q)", err, b) + } + if lay.Version != "1.0.0" { + t.Errorf("imageLayoutVersion: got %q, want 1.0.0", lay.Version) + } + + b, err = os.ReadFile(filepath.Join(s.root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Schema int `json:"schemaVersion"` + Manifests []json.RawMessage `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json is not JSON: %v", err) + } + if idx.Schema != 2 { + t.Errorf("index schemaVersion: got %d, want 2", idx.Schema) + } + if len(idx.Manifests) != 0 { + t.Errorf("fresh index has %d manifests, want 0", len(idx.Manifests)) + } + + if fi, err := os.Stat(filepath.Join(s.root, "blobs", "sha256")); err != nil || !fi.IsDir() { + t.Errorf("blobs/sha256/ missing or not a directory: %v", err) + } +} + +// TestStoreLayoutRoundTrip stores a tiny image, then re-opens the layout with +// crane's own layout.FromPath reader (independent of our store.go write path) +// and asserts the manifest, config, and layer digests all round-trip. This +// is the offline OCI image-layout conformance signal: the on-disk bytes are +// parseable by the canonical go-containerregistry reader. +func TestStoreLayoutRoundTrip(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + wantManifest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + wantConfig, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + wantLayerDigests := make([]v1.Hash, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + wantLayerDigests[i] = d + } + + digest, err := s.addImage("local:tiny", img) + if err != nil { + t.Fatal(err) + } + if digest != wantManifest.String() { + t.Errorf("addImage digest: got %s, want %s", digest, wantManifest) + } + + // Re-open the layout from disk with crane's reader, not our handle. + p, err := layout.FromPath(s.root) + if err != nil { + t.Fatalf("layout.FromPath: %v (store is not a readable OCI layout)", err) + } + got, err := p.Image(wantManifest) + if err != nil { + t.Fatalf("re-opened layout cannot find manifest %s: %v", wantManifest, err) + } + gotManifest, err := got.Digest() + if err != nil { + t.Fatal(err) + } + if gotManifest != wantManifest { + t.Errorf("manifest digest: got %s, want %s", gotManifest, wantManifest) + } + gotConfig, err := got.ConfigName() + if err != nil { + t.Fatal(err) + } + if gotConfig != wantConfig { + t.Errorf("config digest: got %s, want %s", gotConfig, wantConfig) + } + gotLayers, err := got.Layers() + if err != nil { + t.Fatal(err) + } + if len(gotLayers) != len(wantLayerDigests) { + t.Fatalf("layer count: got %d, want %d", len(gotLayers), len(wantLayerDigests)) + } + for i, l := range gotLayers { + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + if d != wantLayerDigests[i] { + t.Errorf("layer %d digest: got %s, want %s", i, d, wantLayerDigests[i]) + } + } + + for _, h := range append([]v1.Hash{wantManifest, wantConfig}, wantLayerDigests...) { + p := filepath.Join(s.root, "blobs", h.Algorithm, h.Hex) + if _, err := os.Stat(p); err != nil { + t.Errorf("blob %s missing on disk: %v", h, err) + } + } + + gotDigest, err := s.digestFor("local:tiny") + if err != nil { + t.Fatal(err) + } + if gotDigest != wantManifest.String() { + t.Errorf("pin: got %s, want %s", gotDigest, wantManifest) + } +} + +// TestStoreInteropPullRoundTrip pulls a real image and asserts the store +// round-trips through crane's independent reader, exactly like the offline +// round-trip but with a registry-fetched image. Gated behind ELFUSE_OCI_NETTEST +// so `go test` stays green offline; CI enables it on the Linux conformance job. +func TestStoreInteropPullRoundTrip(t *testing.T) { + if os.Getenv("ELFUSE_OCI_NETTEST") != "1" { + t.Skip("set ELFUSE_OCI_NETTEST=1 to pull real images") + } + ref := "alpine:3" + cf := commonFlags{platform: defaultPlatform} + + s := openTestStore(t) + if err := pullImage(cf, s, ref); err != nil { + t.Fatalf("pull %s failed with ELFUSE_OCI_NETTEST=1: %v", ref, err) + } + + digestStr, err := s.digestFor(ref) + if err != nil { + t.Fatal(err) + } + wantManifest, err := v1.NewHash(digestStr) + if err != nil { + t.Fatal(err) + } + + p, err := layout.FromPath(s.root) + if err != nil { + t.Fatalf("layout.FromPath: %v", err) + } + got, err := p.Image(wantManifest) + if err != nil { + t.Fatalf("crane reader cannot find manifest %s: %v", wantManifest, err) + } + gotManifest, err := got.Digest() + if err != nil { + t.Fatal(err) + } + if gotManifest != wantManifest { + t.Errorf("manifest digest: got %s, want %s", gotManifest, wantManifest) + } + if _, err := got.ConfigFile(); err != nil { + t.Errorf("ConfigFile: %v", err) + } + layers, err := got.Layers() + if err != nil || len(layers) == 0 { + t.Fatalf("layers: %v (got %d)", err, len(layers)) + } + for _, l := range layers { + if _, err := l.Digest(); err != nil { + t.Errorf("layer digest: %v", err) + } + } +} + +// TestStoreDedupOnRePull asserts that adding the same image twice does not +// accumulate a second manifest descriptor in the layout index (addImage dedups +// by digest), while the ref pin still resolves to that digest. +func TestStoreDedupOnRePull(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + want, err := img.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile(filepath.Join(s.root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Manifests []struct { + Digest string `json:"digest"` + } `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json unmarshal: %v", err) + } + count := 0 + for _, m := range idx.Manifests { + if m.Digest == want.String() { + count++ + } + } + if count != 1 { + t.Errorf("manifest descriptor for %s appears %d times, want 1 (dedup)", want, count) + } + + got, err := s.digestFor("local:tiny") + if err != nil { + t.Fatalf("digestFor: %v", err) + } + if got != want.String() { + t.Errorf("pin: got %s, want %s", got, want) + } +} + +// TestStoreLayoutRoundTripAfterDescriptorRemoval asserts the index.json our +// removeManifestDescriptor hand-marshals stays parseable by the canonical +// go-containerregistry reader: after rmi drops one of two images, the reader +// finds the survivor and no longer finds the removed manifest. This guards the +// switch from the layout package's RemoveDescriptors to our own durable +// atomic write. +func TestStoreLayoutRoundTripAfterDescriptorRemoval(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + digestA, err := imgA.Digest() + if err != nil { + t.Fatal(err) + } + digestB, err := imgB.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", imgB); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a: %v", err) + } + + // Re-open with crane's reader, independent of our write path. + p, err := layout.FromPath(s.root) + if err != nil { + t.Fatalf("layout.FromPath after descriptor removal: %v (index.json not canonical)", err) + } + ii, err := p.ImageIndex() + if err != nil { + t.Fatal(err) + } + im, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + for _, d := range im.Manifests { + if d.Digest == digestA { + t.Fatalf("removed manifest %s still present in index.json", digestA) + } + } + if _, err := p.Image(digestB); err != nil { + t.Fatalf("surviving manifest %s not readable after removal: %v", digestB, err) + } +} + +// TestWriteFileDurableAtomicAndLeavesPriorOnFailure pins writeFileDurable's two +// guarantees: a successful write replaces the file with exactly the new bytes, +// and a failed write (a read-only directory blocks the temp create) leaves any +// prior file untouched rather than truncating it. +func TestWriteFileDurableAtomicAndLeavesPriorOnFailure(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f") + if err := writeFileDurable(path, []byte("first"), 0o644); err != nil { + t.Fatalf("writeFileDurable first: %v", err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "first" { + t.Fatalf("after first write = %q, err=%v; want first", b, err) + } + if err := writeFileDurable(path, []byte("second"), 0o644); err != nil { + t.Fatalf("writeFileDurable second: %v", err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "second" { + t.Fatalf("after second write = %q, err=%v; want second", b, err) + } + + if os.Getuid() == 0 { + t.Skip("running as root: a read-only dir does not block the temp create") + } + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if err := writeFileDurable(path, []byte("third"), 0o644); err == nil { + t.Fatal("writeFileDurable into read-only dir succeeded, want failure") + } + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "second" { + t.Fatalf("after failed write = %q, err=%v; want prior content second", b, err) + } +} + +// TestWriteFileDurableLeavesNoTempOnRenameFailure drives the staging-cleanup +// branch the read-only-dir case cannot reach (there CreateTemp fails before +// any temp exists): a non-empty directory at the destination name makes the +// final rename fail after the temp was fully written, so the deferred +// os.Remove must reclaim it. Regression guard for that cleanup: deleting the +// defer in writeFileDurable turns this red. +func TestWriteFileDurableLeavesNoTempOnRenameFailure(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f") + if err := os.MkdirAll(filepath.Join(path, "x"), 0o755); err != nil { + t.Fatal(err) + } + + if err := writeFileDurable(path, []byte("data"), 0o644); err == nil { + t.Fatal("writeFileDurable onto a non-empty directory succeeded, want error") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != "f" { + t.Fatalf("rename failure left litter: %v", entries) + } + if _, err := os.Stat(filepath.Join(path, "x")); err != nil { + t.Fatalf("destination directory disturbed by the failed write: %v", err) + } +} diff --git a/scripts/ci/oci-cli-smoke.sh b/scripts/ci/oci-cli-smoke.sh new file mode 100755 index 00000000..efb528a8 --- /dev/null +++ b/scripts/ci/oci-cli-smoke.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Store-level CLI lifecycle smoke: everything short of `run` (which needs +# HVF), driving the built binary through the same user-facing flow the Go +# unit tests model in-process: pull, inspect, list, rmi by ref and by +# unique digest prefix, stale-temp-blob GC, and orphan-blob prune. +# +# With --unpack it also unpacks a rootfs and checks that a plain rmi +# reclaims the cold cache (the Linux CI job). Without it, every rmi runs +# the deliberate cache-free path where nothing was ever unpacked and no +# --force is involved (the hosted-macOS CI job, whose runners lack HVF but +# exercise the darwin binary). Needs jq and network. +# +# Usage: scripts/ci/oci-cli-smoke.sh [--unpack] +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; } + +# The interface is a single optional flag; a stray second argument is a +# mistyped invocation, not something to silently ignore. +[ "$#" -le 1 ] || { echo "usage: $0 [--unpack]" >&2; exit 2; } +UNPACK=0 +case "${1:-}" in +--unpack) UNPACK=1 ;; +'') ;; +*) + echo "usage: $0 [--unpack]" >&2 + exit 2 + ;; +esac + +STORE="$(mktemp -d)" +# Clean up on every exit path; a failing phase must not leak a populated +# blob store into the runner's temp dir. +trap 'rm -rf "$STORE"' EXIT +REF=alpine:3 + +phase_pull_inspect_list() { + "$BIN" version + "$BIN" pull --store "$STORE" "$REF" + "$BIN" inspect --store "$STORE" --json "$REF" \ + | jq -e '(.os == "linux") and (.architecture == "arm64")' >/dev/null + "$BIN" list --store "$STORE" | expect_grep "$REF" +} + +# A cold unpacked cache is derived state: a plain `rmi` reclaims it as part +# of removing the image, no --force needed. --force is only for a +# `run --keep` cache (retained output) or a live run's volume, neither of +# which a bare `unpack` produces. See TestRmiDropsColdCacheWithoutForce in +# cmd/elfuse-oci/lifecycle_test.go. +phase_unpack_rmi() { + local digest cache refs + "$BIN" unpack --store "$STORE" "$REF" + digest="$("$BIN" images --store "$STORE" --json | jq -er '.[0].digest')" + cache="$STORE/rootfs/sha256/${digest#sha256:}" + test -e "$cache/bin/sh" || fail "unpacked rootfs has no /bin/sh" + + must_report 'dropped unpacked cache' 'plain rmi of an unpacked cache' \ + "$BIN" rmi --store "$STORE" "$REF" + test ! -e "$cache" || fail "unpacked cache survived a plain rmi" + refs="$("$BIN" list --store "$STORE")" + [ -z "$refs" ] || fail "list not empty after rmi" + + # Restore the pulled image for the digest-rmi phase below. + "$BIN" pull --store "$STORE" "$REF" +} + +# rmi resolves a unique digest prefix from the list table, and its GC also +# sweeps an aborted download's temp blob (digest name plus random suffix), +# which is unreachable by digest. +phase_digest_rmi_stale_blob() { + local digest layer_hex stale table short refs + digest="$("$BIN" images --store "$STORE" --json | jq -er '.[0].digest')" + layer_hex="$(jq -er '.layers[0].digest' \ + "$STORE/blobs/sha256/${digest#sha256:}" | sed 's/^sha256://')" + stale="$STORE/blobs/sha256/${layer_hex}1072211852" + printf 'stale temp blob' >"$stale" + + table="$("$BIN" list --store "$STORE")" + printf '%s\n' "$table" + short="$(printf '%s\n' "$table" | awk 'NR == 2 {print $2}')" + test -n "$short" || fail "list table has no digest column to resolve" + "$BIN" rmi --store "$STORE" "$short" + test ! -e "$stale" || fail "stale temp blob survived the rmi GC" + refs="$("$BIN" list --store "$STORE")" + [ -z "$refs" ] || fail "list not empty after digest rmi" +} + +# prune's reachability GC reclaims orphan blobs whether or not their name +# parses as a digest. +phase_orphan_prune() { + local valid malformed + valid="$(printf 'ci-prune-orphan' | sha256_hex)" + malformed="$STORE/blobs/sha256/${valid}9999" + printf 'orphan blob' >"$STORE/blobs/sha256/$valid" + printf 'malformed orphan blob' >"$malformed" + "$BIN" prune --store "$STORE" + test ! -e "$STORE/blobs/sha256/$valid" || fail "orphan blob survived prune" + test ! -e "$malformed" || fail "malformed orphan blob survived prune" + "$BIN" prune --store "$STORE" --cache +} + +phase_pull_inspect_list +if [ "$UNPACK" = 1 ]; then + phase_unpack_rmi +fi +phase_digest_rmi_stale_blob +phase_orphan_prune +assert_store_empty "$STORE" + +echo "cli smoke OK (unpack=$UNPACK)" diff --git a/scripts/ci/oci-lib.sh b/scripts/ci/oci-lib.sh new file mode 100755 index 00000000..654040d5 --- /dev/null +++ b/scripts/ci/oci-lib.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Shared helpers for the OCI CI test scripts (oci-run-smoke.sh, +# oci-lifecycle.sh, oci-cli-smoke.sh). Source this first; it enables strict +# mode and an ERR trap so any unguarded failure reports its file, line, and +# command. Without the trap, a bare `test`/`grep -q` failing under plain +# `set -e` kills the script with no output at all, leaving a CI failure with +# nothing to diagnose it from. +# +# Bash 3.2 compatible: macOS ships /bin/bash 3.2, so no mapfile, wait -n, +# or ${var,,} here or in the scripts that source this. + +# -E so the ERR trap fires inside functions too. +set -Eeuo pipefail +on_err() { + local s=$? where cmd=$BASH_COMMAND + where="${BASH_SOURCE[1]:-$0}:${BASH_LINENO[0]:-?}" + # ::error:: mirrors what the pre-extraction inline steps emitted, so + # failures still surface as annotations on the PR checks page. + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::error::$where: $cmd (exit $s)" + fi + echo "FAIL $where: $cmd (exit $s)" >&2 +} +trap on_err ERR + +OCI_CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$OCI_CI_DIR/../.." && pwd)" + +fail() { + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::error::$*" + fi + echo "FAIL: $*" >&2 + exit 1 +} + +# require_bin resolves the elfuse-oci binary into BIN, the same +# resolution scripts/oci-interop.sh uses. +require_bin() { + BIN="${ELFUSE_OCI_BIN:-$ROOT/build/elfuse-oci}" + if [ ! -x "$BIN" ]; then + echo "elfuse-oci not found at $BIN (set ELFUSE_OCI_BIN or run 'make build/elfuse-oci')" >&2 + exit 2 + fi +} + +# wait_for TIMEOUT_SEC DESC CMD... polls CMD twice a second until it +# succeeds, and fails loudly with DESC on timeout. Poll loops must live +# here: an inline loop under set -e dies silently on the first transient +# failure of a command substitution. +wait_for() { + local timeout="$1" desc="$2" tries i=0 + shift 2 + tries=$((timeout * 2)) + while [ "$i" -lt "$tries" ]; do + if "$@" >/dev/null 2>&1; then + return 0 + fi + sleep 0.5 + i=$((i + 1)) + done + fail "timed out after ${timeout}s waiting for: $desc" +} + +# reap_guest kills and waits the backgrounded guest recorded in $guest, +# if one is still alive. The scripts that background a guest call this +# from their EXIT trap: a failed phase must not leak the guest, which +# would keep holding its per-digest flock or a bound host loopback port +# and poison the next run on a persistent self-hosted runner. +reap_guest() { + if [ -n "${guest:-}" ] && kill -0 "$guest" 2>/dev/null; then + kill "$guest" 2>/dev/null || true + wait "$guest" 2>/dev/null || true + fi +} + +# expect_grep PATTERN asserts stdin contains the fixed string PATTERN. +# No -q: grep must drain the pipe, or the producer dies of SIGPIPE +# (exit 141) under pipefail when grep exits at the first match. +expect_grep() { + grep -F -- "$1" >/dev/null +} + +# must_report PATTERN DESC CMD... runs a command that must SUCCEED and +# report the fixed string PATTERN on stderr. The stderr is echoed through +# either way so the actual report is visible in the log. +must_report() { + local pattern="$1" desc="$2" errfile + shift 2 + errfile="$(mktemp)" + if ! "$@" 2>"$errfile"; then + cat "$errfile" >&2 + rm -f "$errfile" + fail "$desc: command failed" + fi + cat "$errfile" >&2 + if ! grep -F -- "$pattern" "$errfile" >/dev/null; then + rm -f "$errfile" + fail "$desc: stderr does not mention '$pattern'" + fi + rm -f "$errfile" +} + +# must_refuse PATTERN DESC CMD... runs a command that must FAIL with a +# stderr containing the fixed string PATTERN. The stderr is echoed either +# way so a wrong refusal message is visible in the log. +must_refuse() { + local pattern="$1" desc="$2" errfile + shift 2 + errfile="$(mktemp)" + if "$@" 2>"$errfile"; then + cat "$errfile" >&2 + rm -f "$errfile" + fail "$desc: command succeeded, expected a refusal" + fi + cat "$errfile" >&2 + if ! grep -F -- "$pattern" "$errfile" >/dev/null; then + rm -f "$errfile" + fail "$desc: refusal does not mention '$pattern'" + fi + rm -f "$errfile" +} + +# assert_store_empty STORE asserts a store retains no image state: no +# pinned refs, no blobs, no cs/ sparsebundle bundles, no plain rootfs +# caches, and no volume still mounted beneath it. On Linux the cs/ and +# mount checks pass vacuously. +assert_store_empty() { + local store="$1" refs + # Capture first: a failing `list` inside `[ -z "$(...)" ]` would be + # swallowed (the [ builtin's status is all set -e sees), letting a + # broken list pass the gate; a plain assignment propagates the status. + refs="$("$BIN" list --store "$store")" + [ -z "$refs" ] || fail "store not empty: list still shows pinned refs" + [ -z "$(ls "$store/blobs/sha256" 2>/dev/null || true)" ] \ + || fail "store not empty: blobs remain" + [ -z "$(find "$store/cs" -mindepth 2 -maxdepth 2 -type d 2>/dev/null || true)" ] \ + || fail "store not empty: cs/ bundle dirs remain" + [ -z "$(find "$store/rootfs/sha256" -mindepth 1 -maxdepth 1 2>/dev/null || true)" ] \ + || fail "store not empty: plain rootfs caches remain" + if mount | grep -F "$store" >/dev/null; then + fail "store not empty: a volume is still mounted under $store" + fi +} + +# sha256_hex prints the hex digest of stdin. Hosted macOS runners ship +# shasum but not coreutils sha256sum. +sha256_hex() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} diff --git a/scripts/ci/oci-lifecycle.sh b/scripts/ci/oci-lifecycle.sh new file mode 100755 index 00000000..52e713f0 --- /dev/null +++ b/scripts/ci/oci-lifecycle.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Whole user-facing image lifecycle against a real HVF-booted guest, then +# the teardown guardrails, one function per phase: +# run_workloads pull/inspect/list, an --entrypoint override, +# and a glibc dynamically-linked python workload +# teardown_plain_rmi a plain rmi reclaims the cold unpacked cache +# with the image +# teardown_keep_guardrails rmi refuses to discard run --keep output +# without --force; --force detaches the still- +# attached volume and drops the bundle +# teardown_live_run_lock a live --plain-rootfs guest pins its cache +# against prune --cache --all and rmi until the +# guest exits; only this exercises the lock +# descriptor's ride through the exec into elfuse +# Needs macOS with Hypervisor.framework and network for the seed pull. +# +# Usage: ELFUSE_OCI_STORE= \ +# ELFUSE_OCI_SEED_STORE= \ +# [IMG=] scripts/ci/oci-lifecycle.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +STORE="${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to an ephemeral store directory}" +SEED="${ELFUSE_OCI_SEED_STORE:?set ELFUSE_OCI_SEED_STORE to the warm seed store directory}" +IMG="${IMG:-python:3.12-slim}" +export ELFUSE_OCI_STORE + +guest="" +on_exit() { + rc=$? + # A failed phase must not leak the backgrounded guest: it would keep + # holding the per-digest flock (and its sleep) and poison the next + # prune/rmi on a persistent self-hosted runner. + if [ -n "$guest" ] && kill -0 "$guest" 2>/dev/null; then + kill "$guest" 2>/dev/null || true + wait "$guest" 2>/dev/null || true + fi + exit "$rc" +} +trap on_exit EXIT + +# The teardowns leave the ephemeral store EMPTY (asserted), so the +# lifecycle store itself cannot persist between CI runs. Keep a warm seed +# store on the persistent disk instead and clone it in per phase (cp -Rc, +# APFS clonefile, the same trick as the fixture-cache restore): the pull +# in run_workloads then dedups every blob by digest and only the manifest +# HEAD/GET goes out, while the empty-store assertions stay meaningful. +seed_warm_store() { + ELFUSE_OCI_STORE="$SEED" "$BIN" pull "$IMG" + # GC blobs stranded in the seed when the tag moves to a new digest. + ELFUSE_OCI_STORE="$SEED" "$BIN" prune >/dev/null +} + +reseed() { + rm -rf "$STORE" + cp -Rc "$SEED" "$STORE" +} + +run_workloads() { + "$BIN" version + "$BIN" pull "$IMG" + "$BIN" inspect "$IMG" | expect_grep 'linux/arm64' + "$BIN" list | expect_grep "$IMG" + + local out + out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ + -c 'import json,math; print(json.dumps({"pi":round(math.pi,5),"ok":True}))')" + printf 'guest said: %s\n' "$out" + [ "$out" = '{"pi": 3.14159, "ok": true}' ] || fail "python one-liner said '$out'" + + # A non-trivial application: concurrent SQLite writers (fcntl locking, + # fsync, WAL where the guest FS supports it) plus a 64-file + # write/read/checksum fan-out. Exercises far more of the dynamically- + # linked glibc guest than the one-liner above; prints a single sentinel + # token only on full success. + out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ + -c "$(cat "$OCI_CI_DIR/oci-python-workload.py")")" + printf 'python workload said: %s\n' "$out" + [ "$out" = elfuse-oci-python-workload-ok ] || fail "python workload said '$out'" +} + +# The runs above left the cache warm and then detached on exit, so a plain +# rmi (no --force, no separate prune) must reclaim it with the image. +teardown_plain_rmi() { + must_report 'dropped unpacked cache' 'plain rmi of a cold cache' \ + "$BIN" rmi "$IMG" + assert_store_empty "$STORE" +} + +# run --keep retains the per-run clone with the volume still attached: rmi +# must refuse without --force, and rmi --force must detach the volume, +# drop the bundle, and GC the blobs. +teardown_keep_guardrails() { + reseed + "$BIN" run --keep --entrypoint /usr/local/bin/python3 "$IMG" -c 'pass' + must_refuse 'retained run --keep output; pass --force' \ + 'rmi of run --keep output without --force' \ + "$BIN" rmi "$IMG" + "$BIN" rmi --force "$IMG" + assert_store_empty "$STORE" +} + +# The published cache is rootfs/sha256/, renamed into place when the +# unpack completes. Never match the unpacker's .tmp- staging +# sibling: it is mid-write, and prune skips it via the digest lock. +published_plain_cache() { + # sed, not `head -n1`: head exits at the first line, and under pipefail + # a find killed by the resulting SIGPIPE would fail the whole pipeline + # even though a cache was found. sed -n 1p drains its input. + find "$STORE/rootfs/sha256" -mindepth 1 -maxdepth 1 -type d \ + ! -name '*.tmp-*' 2>/dev/null | sed -n 1p | grep . +} + +# A live --plain-rootfs guest must pin its cache against prune --cache +# --all and rmi, and its exit alone (no cleanup code, SIGKILL included) +# must free it: the kernel drops the flock with the process. +teardown_live_run_lock() { + reseed + "$BIN" run --plain-rootfs --entrypoint /bin/sleep "$IMG" 60 & + guest=$! + # The run takes its per-digest lock before unpacking, so once the + # cache dir has been published the guest provably holds the lock. + wait_for 120 'published plain rootfs cache' published_plain_cache + local plain_cache + plain_cache="$(published_plain_cache)" + + "$BIN" prune --cache --all + test -d "$plain_cache" || fail 'prune --cache --all reclaimed a live plain rootfs' + must_refuse 'in use by a live run' \ + 'rmi of an image whose plain rootfs hosts a live run' \ + "$BIN" rmi "$IMG" + + # SIGKILL, deliberately: the claim above is that the kernel (not any + # elfuse cleanup code) drops the flock, and a plain SIGTERM would let a + # graceful teardown pass this test while the no-cleanup path regressed. + kill -9 "$guest" 2>/dev/null || true + wait "$guest" || true + guest="" + # Guest gone means the kernel released the flock; prune reclaims the + # cache (lock file included) and the image can finally be removed. + "$BIN" prune --cache --all + test ! -e "$plain_cache" || fail 'plain rootfs cache survived prune after guest exit' + "$BIN" rmi "$IMG" + assert_store_empty "$STORE" + + # Nothing left to reclaim. + "$BIN" prune --cache +} + +seed_warm_store +reseed +run_workloads +teardown_plain_rmi +teardown_keep_guardrails +teardown_live_run_lock + +echo "lifecycle OK" diff --git a/scripts/ci/oci-python-workload.py b/scripts/ci/oci-python-workload.py new file mode 100644 index 00000000..6c1c044d --- /dev/null +++ b/scripts/ci/oci-python-workload.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Non-trivial guest workload for the elfuse-oci OCI lifecycle CI. + +Exercises more of a real dynamically-linked glibc guest than a one-line print: +concurrent SQLite writers (fcntl locking, fsync, and, when the guest FS +supports it, WAL mmap), plus a filesystem fan-out whose written content is +read back and checksummed. On full success it prints a single sentinel token +the workflow asserts on; any failure exits non-zero with a diagnostic. + +Self-contained (stdlib only) so it runs under the image's bundled Python with +no network or extra packages, and is passed to the guest via `python3 -c`. +""" + +import hashlib +import os +import sqlite3 +import sys +import threading + +DB = "/tmp/elfuse-workload.db" +TREE = "/tmp/elfuse-workload-tree" +THREADS = 8 +PER_THREAD = 1000 +FANOUT = 8 + + +def setup_db(): + con = sqlite3.connect(DB) + try: + # WAL exercises the guest FS's shared-memory index (mmap) and is the + # more demanding path; if the FS cannot back it, SQLite reports a + # different mode and the concurrent-writer count check below still + # validates fcntl locking and durable commits under rollback journal. + con.execute("PRAGMA journal_mode=WAL") + con.execute( + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "tid INTEGER NOT NULL, n INTEGER NOT NULL)" + ) + con.commit() + finally: + con.close() + + +def worker(tid): + con = sqlite3.connect(DB, timeout=60) + try: + con.execute("PRAGMA busy_timeout=60000") + for n in range(PER_THREAD): + con.execute("INSERT INTO t (tid, n) VALUES (?, ?)", (tid, n)) + con.commit() + finally: + con.close() + + +def db_count(): + con = sqlite3.connect(DB, timeout=60) + try: + (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() + return count + finally: + con.close() + + +def content(i, j): + return "elfuse-workload-%d-%d\n" % (i, j) + + +def fs_fanout_ok(): + for i in range(FANOUT): + for j in range(FANOUT): + d = os.path.join(TREE, str(i), str(j)) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "f"), "w") as fh: + fh.write(content(i, j)) + fh.flush() + os.fsync(fh.fileno()) + read_back = [] + for i in range(FANOUT): + for j in range(FANOUT): + with open(os.path.join(TREE, str(i), str(j), "f")) as fh: + read_back.append(fh.read()) + expected = [content(i, j) for i in range(FANOUT) for j in range(FANOUT)] + got = hashlib.sha256() + for p in sorted(read_back): + got.update(p.encode()) + want = hashlib.sha256() + for p in sorted(expected): + want.update(p.encode()) + return got.hexdigest() == want.hexdigest() + + +def main(): + setup_db() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + count = db_count() + if count != THREADS * PER_THREAD: + print( + "sqlite row count %d != %d (concurrent writers lost rows)" + % (count, THREADS * PER_THREAD), + file=sys.stderr, + ) + sys.exit(1) + + if not fs_fanout_ok(): + print("filesystem fan-out checksum mismatch", file=sys.stderr) + sys.exit(1) + + print("elfuse-oci-python-workload-ok") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/oci-run-smoke.sh b/scripts/ci/oci-run-smoke.sh new file mode 100755 index 00000000..f7b4bb8a --- /dev/null +++ b/scripts/ci/oci-run-smoke.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# End-to-end smoke for the default `run` path: pull an image, provision the +# case-sensitive sparsebundle, COW-clone it, boot the guest under HVF, and +# check output, exit status, and per-run isolation. Needs macOS with +# Hypervisor.framework; network only when the store is cold (pull is +# idempotent per digest, so a warm store skips the blob downloads). +# +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-run-smoke.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +: "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" + +out="$("$BIN" run alpine:3 /bin/echo elfuse-oci-ci-ok)" +printf 'guest said: %s\n' "$out" +printf '%s\n' "$out" | expect_grep elfuse-oci-ci-ok + +# The guest's exit status must propagate through the runner untouched; +# cleanup errors must never win over it. +code=0 +"$BIN" run alpine:3 /bin/sh -c 'exit 7' || code=$? +[ "$code" -eq 7 ] || fail "guest exit status: got $code, want 7" + +# Non-trivial multi-stage shell pipeline: generate a 200k-line file, +# gzip it, decompress and byte-compare the round-trip, then checksum +# the original against a constant precomputed from the exact same +# deterministic input. Exercises fork/exec pipelines, pipes, and +# coreutils gzip/sha256sum in the guest. debian:stable-slim rather +# than alpine because the bare-name PATH search must resolve every +# candidate inside the rootfs: the sysroot resolver falls back to +# the host for absent absolute paths, and alpine ships gzip only at +# /bin/gzip while its PATH tries /usr/bin first, which on a macOS +# host holds an incompatible Mach-O gzip. Debian is usr-merged, so +# each searched binary exists at /usr/bin inside the image. +want=5af7b95208fdcff454bab3f5eddf567a688a3796c703d4fef91072e38645c062 +got="$("$BIN" run debian:stable-slim /bin/sh -c 'set -e + seq 1 200000 > /tmp/data.txt + gzip -c /tmp/data.txt > /tmp/data.gz + gunzip -c /tmp/data.gz | cmp - /tmp/data.txt + sha256sum /tmp/data.txt | cut -d" " -f1')" +printf 'debian pipeline sha256: %s\n' "$got" +[ "$got" = "$want" ] || fail "debian pipeline sha256: got $got, want $want" + +# Per-run COW clone isolation: the previous run's /tmp writes must not be +# visible to a fresh run of the same digest. Exact match, not a substring: +# a diagnostic quoting the failed command would also contain the token. +out="$("$BIN" run debian:stable-slim /bin/sh -c 'test ! -e /tmp/data.txt && echo isolated-ok')" +[ "$out" = isolated-ok ] || fail "isolation check said '$out'" + +# When a pinned tag moves, the re-pin strands the old digest's blobs AND its +# unpacked caches; --cache reclaims both so a persistent store stays bounded +# (the stranded sparsebundles dwarf the blobs). Safe beside concurrent legs: +# the sweep runs under the store lock, skips still-pinned digests, and skips +# busy caches via their non-blocking flocks. +"$BIN" prune --cache >/dev/null + +echo "run smoke OK" diff --git a/scripts/oci-interop.sh b/scripts/oci-interop.sh new file mode 100755 index 00000000..82e353d9 --- /dev/null +++ b/scripts/oci-interop.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# OCI image-layout conformance + cross-tool interop for the elfuse-oci store. +# +# Treats the on-disk store as the contract: after `elfuse-oci pull`, the store +# must be a valid OCI image-layout that other tools can read and that agrees +# with registry truth on the manifest digest. +# +# Hard assertions (always available, gate the script): +# - oci-layout has imageLayoutVersion 1.0.0; index.json is schemaVersion 2 +# with a manifest descriptor matching the pinned digest. +# - the manifest blob parses and references a config blob + >=1 layer blob, +# all present under blobs/sha256/. +# - if `crane` is installed, the store's pinned manifest digest matches +# registry truth for the selected platform. +# +# Best-effort third-party reads (run when present; fatal on failure so CI can +# promote them once the invocation is confirmed): +# - skopeo inspect --raw oci::@ reads our layout +# - umoci list --layout parses our layout +# +# Usage: scripts/oci-interop.sh [STORE_DIR] +# Env: ELFUSE_OCI_BIN (path to elfuse-oci; default build/elfuse-oci) +# FIXTURES (space-separated refs; default "alpine:3 busybox") +# PLAT_OS / PLAT_ARCH (platform to pull and compare; default linux/arm64) +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# fail and require_bin (which resolves BIN) are shared with the CI scripts so +# one wording and one resolution rule cover every entry point. +# shellcheck source=scripts/ci/oci-lib.sh +. "$HERE/ci/oci-lib.sh" + +# A store passed as $1 belongs to the caller; an auto-created one is ours to +# reclaim, including on a mid-run failure. +if [ -n "${1:-}" ]; then + STORE="$1" + STORE_OWNED=0 +else + STORE="$(mktemp -d -t elfuse-interop.XXXXXX)" + STORE_OWNED=1 +fi +# Absolutize the store path: the skopeo view symlinks blobs -> "$STORE/blobs", +# a target resolved relative to the (separate) view dir, so a relative store arg +# would dangle. skopeo/umoci oci: transports also want an absolute layout path. +case "$STORE" in + /*) ;; + *) STORE="$(pwd)/$STORE" ;; +esac + +# Temp files/dirs created per fixture. An EXIT trap reclaims them plus the +# auto-created store, so a failure (set -e) or fail() no longer orphans the +# store's pulled blobs or the per-iteration scratch. Bash 3.2 (macOS) errors on +# "${arr[@]}" for an empty array under set -u, so guard on length. +TMPFILES=() +cleanup() { + if [ "${#TMPFILES[@]}" -gt 0 ]; then + rm -rf "${TMPFILES[@]}" + fi + if [ "$STORE_OWNED" = 1 ]; then + rm -rf "$STORE" + fi + return 0 +} +trap cleanup EXIT + +FIXTURES="${FIXTURES:-alpine:3 busybox}" +# One platform drives BOTH the pull and the registry-truth comparison; setting +# it only on the crane side would fail the comparison against a correctly +# pulled default-platform image. +PLAT_OS="${PLAT_OS:-linux}" +PLAT_ARCH="${PLAT_ARCH:-arm64}" + +have() { command -v "$1" >/dev/null 2>&1; } + +require_bin +have jq || { echo "jq is required" >&2; exit 2; } + +echo "store: $STORE" +echo "bin: $BIN" +mkdir -p "$STORE" + +for ref in $FIXTURES; do + echo + echo "=== $ref ===" + + # Pull into the store. + "$BIN" pull --store "$STORE" --platform "$PLAT_OS/$PLAT_ARCH" "$ref" >/dev/null + digest="$(jq -er --arg ref "$ref" '.[$ref]' "$STORE/refs.json" \ + || fail "refs.json has no pin for $ref")" + echo "pinned manifest digest: $digest" + + # --- Conformance: oci-layout --- + [ "$(jq -r .imageLayoutVersion "$STORE/oci-layout")" = "1.0.0" ] \ + || fail "oci-layout imageLayoutVersion != 1.0.0" + + # --- Conformance: index.json has our manifest descriptor --- + [ "$(jq -r .schemaVersion "$STORE/index.json")" = "2" ] \ + || fail "index.json schemaVersion != 2" + jq -e --arg d "$digest" 'any(.manifests[]; .digest == $d)' \ + "$STORE/index.json" >/dev/null \ + || fail "index.json has no manifest descriptor with digest $digest" + + # --- Conformance: manifest blob parses and references config + layers --- + hex="${digest#sha256:}" + manifest_path="$STORE/blobs/sha256/$hex" + [ -f "$manifest_path" ] || fail "manifest blob missing at $manifest_path" + config_digest="$(jq -er .config.digest "$manifest_path" \ + || fail "manifest blob is not a valid image manifest (no .config.digest)")" + config_hex="${config_digest#sha256:}" + [ -f "$STORE/blobs/sha256/$config_hex" ] \ + || fail "config blob missing for $config_digest" + layer_count="$(jq '.layers | length' "$manifest_path")" + [ "$layer_count" -ge 1 ] || fail "manifest has no layers" + # Every layer blob must exist on disk. + while IFS= read -r ld; do + lx="${ld#sha256:}" + [ -f "$STORE/blobs/sha256/$lx" ] || fail "layer blob missing for $ld" + done < <(jq -r '.layers[].digest' "$manifest_path") + echo "ok: layout valid, $layer_count layer(s), config + manifest + layers present" + + # --- Interop: registry truth via crane (if installed) --- + # `elfuse-oci pull` uses crane.Pull(WithPlatform), which resolves a manifest + # list to the per-arch child manifest and pins THAT digest. So for a + # multi-arch ref, `crane digest` (the list digest) legitimately differs; we + # resolve the platform child from `crane manifest` and compare that. + if have crane; then + plat_os="$PLAT_OS"; plat_arch="$PLAT_ARCH" + top="$(crane manifest "$ref")" + if [ "$(printf '%s' "$top" | jq '.manifests // [] | any(.platform != null)')" = "true" ]; then + # first(...) keeps the comparison single-valued if several entries + # match the platform (e.g. multiple variants), and the annotation + # filter drops BuildKit attestation manifests, which are not + # runnable images. crane.Pull resolves the same first match. + reg_digest="$(printf '%s' "$top" | jq -er --arg os "$plat_os" --arg ar "$plat_arch" \ + 'first(.manifests[] + | select(.platform.os==$os and .platform.architecture==$ar + and (.annotations["vnd.docker.reference.type"] != "attestation-manifest")) + | .digest)' \ + || fail "crane manifest list for $ref has no $plat_os/$plat_arch entry")" + else + reg_digest="$(crane digest "$ref")" + fi + [ "$reg_digest" = "$digest" ] \ + || fail "crane ($reg_digest) != store pin ($digest) for $ref [$plat_os/$plat_arch]" + echo "ok: crane agrees on manifest digest ($plat_os/$plat_arch)" + else + echo "info: crane not installed; skipping registry-truth comparison" + fi + + # --- Interop: skopeo reads our layout (if installed) --- + if have skopeo; then + # skopeo's oci: transport addresses an image by ref-name annotation or + # (since skopeo 1.14) by @source-index. Our layout intentionally + # carries no ref-name annotations, and distro skopeo is often older + # than 1.14, so neither form is portable. Present a single-manifest + # view instead (the same oci-layout and blobs, index.json filtered + # to the pinned descriptor), which every skopeo version resolves + # with a bare oci: reference. + skopeo_view="$(mktemp -d -t elfuse-skopeo-view.XXXXXX)" + TMPFILES+=("$skopeo_view") + cp "$STORE/oci-layout" "$skopeo_view/oci-layout" + jq --arg d "$digest" \ + '.manifests = [.manifests[] | select(.digest == $d)]' \ + "$STORE/index.json" >"$skopeo_view/index.json" + ln -s "$STORE/blobs" "$skopeo_view/blobs" + skopeo_ref="oci:$skopeo_view" + skopeo_raw="$(mktemp -t elfuse-skopeo-raw.XXXXXX)" + skopeo_err="$(mktemp -t elfuse-skopeo-err.XXXXXX)" + TMPFILES+=("$skopeo_raw" "$skopeo_err") + if skopeo inspect --raw "$skopeo_ref" >"$skopeo_raw" 2>"$skopeo_err"; then + jq -e '(.schemaVersion == 2) and (.config.digest | type == "string") and + (.layers | type == "array") and (.layers | length >= 1)' \ + "$skopeo_raw" >/dev/null \ + || fail "skopeo read $skopeo_ref but did not return an image manifest" + echo "ok: skopeo inspect --raw $skopeo_ref read the pinned manifest" + else + echo "skopeo stderr: $(cat "$skopeo_err")" >&2 + fail "skopeo could not read $skopeo_ref" + fi + else + echo "info: skopeo not installed; skipping skopeo interop" + fi + + # --- Interop: umoci parses our layout (if installed) --- + # The layout is valid even when no descriptors carry ref-name annotations. + # elfuse keeps refs.json as its own lookup table for full pull references; + # umoci list may therefore show zero tags, but it must still parse. + if have umoci; then + umoci_out="$(mktemp -t elfuse-umoci-list.XXXXXX)" + umoci_err="$(mktemp -t elfuse-umoci-err.XXXXXX)" + TMPFILES+=("$umoci_out" "$umoci_err") + if umoci list --layout "$STORE" >"$umoci_out" 2>"$umoci_err"; then + echo "ok: umoci list --layout parsed our layout (tags: $(wc -l <"$umoci_out" | tr -d ' '))" + else + echo "umoci stderr: $(cat "$umoci_err")" >&2 + fail "umoci could not parse layout $STORE" + fi + else + echo "info: umoci not installed; skipping umoci interop" + fi +done + +echo +echo "ALL OK: store is a valid OCI image-layout and interops with available tools" +# The EXIT trap reclaims the auto-created store and any per-iteration scratch. From a21f3b03a1e3f65b9a6f1412beade00d4e30ace0 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:38:59 +0200 Subject: [PATCH 09/26] Document OCI image support Cover the two-binary model in README and docs: usage.md documents the elfuse-oci commands and flags, testing.md the offline/CI validation split, internals.md the host-literal path fallback semantics, and oci-design.md records the design rationale (the C/Go boundary, the image-layout store with its refs.json pin table, layer application, run paths, and lifecycle GC), plus an explicit scope-and-limitations accounting of which OCI features are and are not implemented. oci-design.md also records the concurrency model: store metadata is lock-serialized, per-digest sparsebundle state is coordinated by the attach.lock/run.lock pair, and the plain digest-keyed rootfs cache is guarded by a sibling per-digest lock a run holds across the exec into elfuse, so prune skips and rmi refuses a cache a live guest still uses. usage.md notes the resolv.conf fallback nameserver and its split-DNS implication. README states the positioning up front: OCI images are consumed as a distribution vehicle for Linux root filesystems, replacing hand-built --sysroot trees, and the non-isolation limitations (host-path fallback, shared network identity, PID space, and clock) are called out explicitly rather than implied. --- README.md | 43 ++++++- docs/internals.md | 7 + docs/oci-design.md | 310 +++++++++++++++++++++++++++++++++++++++++++++ docs/testing.md | 106 ++++++++++++++++ docs/usage.md | 186 +++++++++++++++++++++++++++ 5 files changed, 650 insertions(+), 2 deletions(-) create mode 100644 docs/oci-design.md diff --git a/README.md b/README.md index 6b7f7148..cd3ee18e 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ boot-time overhead those tools impose. - Xcode Command Line Tools, `clang`, `codesign`, and GNU `make` - GNU `objcopy` or `llvm-objcopy` - Hypervisor entitlement: `com.apple.security.hypervisor` +- Go, for building the `elfuse-oci` OCI CLI To build only (`make elfuse`) without running tests, just the Xcode Command Line Tools and `objcopy` (`brew install binutils`) suffice. @@ -104,11 +105,43 @@ state. The build signs `build/elfuse` before use. Override the signing identity with `SIGN_IDENTITY="Developer ID ..."` when needed. +## OCI Images + +OCI images are handled by `elfuse-oci`, a Go companion binary that owns +the whole image lifecycle and invokes `elfuse` purely as the runtime. The point +is distribution, not containment: an image is consumed as a packaged Linux root +filesystem, so a dynamically-linked guest needs no hand-built `--sysroot` tree. +The execution model is the one every elfuse run uses (Linux ELF, elfuse syscall +translation, macOS kernel), with no VM and no Linux kernel in the loop. + +This is not a container runtime. The image rootfs is the guest's root, not an +isolation boundary: an absolute guest path absent from the rootfs falls back to +the literal host path, and the guest shares the host's network identity, PID +space, and clock. There are no namespaces, cgroups, port mapping, or daemon. +The target is CLI tooling, compilers, and scripting userspace; +[docs/oci-design.md](docs/oci-design.md#scope-and-limitations) lists exactly +which OCI features are implemented. + +```sh +make elfuse elfuse-oci + +build/elfuse-oci pull alpine:3 +build/elfuse-oci run alpine:3 /bin/sh -c 'echo hello from elfuse' +``` + +Images are stored under `$ELFUSE_OCI_STORE`, or `~/.local/share/elfuse/oci` +by default. On macOS, `run` uses a case-sensitive APFS sparsebundle and a +per-run copy-on-write rootfs clone so normal APFS case folding does not +corrupt Linux filenames. + +See [docs/usage.md](docs/usage.md#oci-images) for commands and flags, and +[docs/oci-design.md](docs/oci-design.md) for the implementation model. + ## Documentation - [docs/usage.md](docs/usage.md): command-line options, x86_64 via - Rosetta, dynamic linking via `--sysroot`, and attaching `gdb` / - `lldb` to the built-in stub. + Rosetta, dynamic linking via `--sysroot`, OCI images, and attaching + `gdb` / `lldb` to the built-in stub. - [docs/testing.md](docs/testing.md): build prerequisites, the `make check` flow, the QEMU and Rosetta cross-check matrices, and fixture handling. @@ -119,6 +152,9 @@ The build signs `build/elfuse` before use. Override the signing identity with reference -- runtime lifecycle, HVF constraints, EL1 shim and HVC protocol, page-table splitting, syscall translation tables, threads / futex, fork / clone IPC, signals, ptrace, and the GDB stub. +- [docs/oci-design.md](docs/oci-design.md): how elfuse-oci, the image + store, layer unpacker, sparsebundle run path, and lifecycle commands + fit into elfuse. ## Build And Validation @@ -129,6 +165,7 @@ make elfuse # build and codesign build/elfuse make check # quick unit suite + BusyBox applet smoke make test-gdbstub # debugger integration make test-matrix # cross-check elfuse against QEMU on the same corpus +make oci-test # elfuse-oci unit and conformance tests make lint # clang-tidy ``` @@ -147,6 +184,8 @@ do. - Linux kernel features that have no user-space-syscall analog: namespaces, cgroups, kernel modules, eBPF, `io_uring`, KVM, perf events. +- Docker-compatible container runtime features such as port mapping, + detached containers, `docker exec`, image build/push, and daemon APIs. - Intel Macs. Apple Silicon only (M1 and later). - Hosting a VM from inside a guest. The guest cannot use HVF or KVM. - One guest process tree per `elfuse` host process. HVF allows one VM diff --git a/docs/internals.md b/docs/internals.md index 60f62231..5927518a 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -993,6 +993,13 @@ How it works: disagree about where a path lives. [filenames.md](filenames.md) covers how a name is spelled once it lands on the sysroot volume. +That dispatch applies to image-launched guests as well: the sysroot is a root, +not a boundary, and `elfuse-oci run` inherits it unchanged. A guest `PATH` +search whose candidate is absent under the sysroot can therefore resolve to a +host binary; `run` compensates by guaranteeing the guest a `PATH` (Docker's +conventional default when the image env provides none), and callers can +reorder the search with `--env PATH=...`. + The sysroot is inherited by fork children via IPC state transfer. `sys_execve` also loads the interpreter for dynamically linked targets, so tools that `execve` dynamic children (`env`, `nice`, `nohup`) work diff --git a/docs/oci-design.md b/docs/oci-design.md new file mode 100644 index 00000000..a5c4b1a6 --- /dev/null +++ b/docs/oci-design.md @@ -0,0 +1,310 @@ +# OCI Image Support Design + +This is the reference for how elfuse consumes OCI images without becoming a +container runtime. It is the single source of truth for what is and is not +implemented. For day-to-day commands and flags, see +[usage.md](usage.md#oci-images); for validation targets, see +[testing.md](testing.md). + +## Model + +elfuse uses the OCI image format for distribution and filesystem packaging +only. It does not implement the OCI runtime spec. The goal is narrow: pull an +image, unpack its layers into a Linux rootfs, resolve the image runtime +configuration, and launch the configured program through the existing +`elfuse --sysroot` path. The guest is a single elfuse process translating +Linux syscalls to Darwin, not an isolated container. + +## Scope And Limitations + +The OCI ecosystem is three specifications plus a set of conventions. elfuse +implements the consumer side of the image format and nothing else. + +Implemented: + +- the on-disk OCI image-layout store (`oci-layout`, `index.json`, + content-addressed blobs) plus an elfuse-specific `refs.json` pin file; +- pulling with `go-containerregistry` through the ambient default keychain, + with `--platform` selection resolved against manifest lists; +- layer application: whiteouts and opaque directories, hardlinks and symlinks + (absolute targets rewritten rootfs-relative), setuid/setgid/sticky bits, and + gzip or zstd compression, all applied under `os.OpenRoot`; +- image-config resolution of `Entrypoint`, `Cmd`, `Env`, `User`, and + `WorkingDir` with Docker-style precedence; +- local lifecycle management (`list`, `rmi`, `prune`) with reachability-based + garbage collection and macOS sparsebundle cache handling; +- runtime injection of `/etc/resolv.conf`, `/etc/hosts`, and `/etc/hostname` + into the run rootfs. + +Out of scope: + +- **OCI runtime spec.** No runtime bundle or `config.json`, and no namespaces, + cgroups, seccomp, capabilities, hooks, or mounts/volumes. The image rootfs is + the guest's root but not a boundary: an absolute guest path absent from the + rootfs falls back to the host filesystem (see [Run Paths](#run-paths)), and + the guest shares the host network identity, PID space, and clock. +- **Distribution write side.** Pull only: no `push`, no image building, and no + `login`. Credentials come from the ambient default keychain (for example an + existing Docker credential store); elfuse adds none of its own. Credential + resolution is time-bounded, so a wedged helper (such as a `credsStore` whose + backing app is not running) fails the pull with an explanation rather than + hanging it; `DOCKER_CONFIG` pointing at a config without that helper forces an + anonymous pull. +- **Network and port isolation.** The guest shares the host network. There is + no port mapping, and `ExposedPorts` has no effect. +- **Ignored image-config fields.** `Volumes`, `ExposedPorts`, `Healthcheck`, + `StopSignal`, and `Labels` are accepted but have no runtime effect. +- **Daemon conveniences.** No daemon, no `exec` or `attach`, no detached + containers; each `run` is one foreground guest process. +- **Non-Linux images.** Only `linux` images, on the platforms the runtime + executes: `arm64` natively, `amd64` via Rosetta. +- **Device nodes in layers.** Character, block, and FIFO entries are rejected + rather than materialized; the C runtime synthesizes the supported `/dev` and + `/proc` entries at run time. +- **Supply-chain verification.** Content is verified against manifest digests, + but there is no signature or attestation checking (cosign, notation) and no + policy engine. + +A workload that needs any of the above needs a container runtime, not an ELF +runner that consumes OCI images. + +## Library Boundary + +`elfuse-oci` imports `go-containerregistry` for registry transport and +image-layout blob access, and owns everything above that layer: durable store +writes, layer application, runtime-configuration resolution, and lifecycle +management. + +The alternative boundary is to shell out to `skopeo` for pull and delete and +`umoci` for unpack, both of which install cleanly on macOS. That boundary +costs more than it saves, because each behavior this design guarantees would +have to be rebuilt on top of those tools anyway: + +- `umoci gc` aborts on a stale temporary blob whose name is not a valid + digest, which is exactly the debris [prune](#lifecycle) must sweep; +- umoci stores absolute symlink targets verbatim, so the rootfs-relative + rewrite of [Layer Application](#layer-application) would still be needed; +- no copy tool tolerates the setgid `chmod` `EPERM` that group inheritance + produces on macOS volumes, so the special-bit degrade would still be + needed; +- no tool models unpacked caches, sparsebundles, or per-run clones, so the + whole [Lifecycle](#lifecycle) layer would still be needed. + +What those tools do cover well, registry transport and whiteout-correct layer +application, is the part already delegated to `go-containerregistry` or that +changes rarely. Shelling out would additionally give up crash-durable store +writes (both tools write `index.json` with plain writes), the flock-based +store and live-run locking, the device-node rejection policy (umoci +materializes empty placeholder files instead), and the single-binary install. +The boundary is therefore a library import: own every behavior the CI +asserts, and keep skopeo and umoci as independent readers that cross-check +the store (see [Validation](#validation)). + +## Boundary Between C And Go + +There are two binaries with a one-way dependency: `elfuse-oci` calls +`elfuse`, never the reverse, so the runtime stays useful on its own as a plain +ELF runner. + +`build/elfuse` (C) is purely the Linux syscall-to-Darwin runtime, with no OCI +awareness. It provides the positional ELF launcher, the `--user`/`--workdir`/ +`--env`/`--clear-env` launch flags, and the synthetic `/proc` and `/dev` +entries served to every guest. + +`build/elfuse-oci` (Go) is the only OCI entry point. It pulls images, +maintains the image-layout store, inspects and unpacks stored images, resolves +the runtime configuration, prepares the runtime `/etc` files, and invokes +`elfuse` with the resolved launch flags. It locates `elfuse` as a sibling of +its own executable; `$ELFUSE_BIN` overrides the location for tests and wrapper +scripts. + +## Store + +The store is an OCI image-layout directory plus one pin file: + +```text +/ + oci-layout + index.json + blobs/sha256/ + refs.json +``` + +`oci-layout`, `index.json`, and `blobs/` are the standard layout. `refs.json` +maps each original image reference to the manifest digest elfuse pinned at pull +time. It is elfuse-specific lookup metadata; OCI readers parse the layout +through `index.json` and the content-addressed blobs without it. Keeping it +separate preserves the exact pull reference (`docker.io/library/alpine:3`, +`name@sha256:...`). + +The default store is `$ELFUSE_OCI_STORE` when set, otherwise +`~/.local/share/elfuse/oci`. + +## Pull And Platform Selection + +`pull` defaults to `linux/arm64`, matching the native Apple Silicon guest path. +`--platform os/arch[/variant]` selects another image, such as `linux/amd64` for +a Rosetta-backed guest. When a reference is a manifest list, `pull` fetches and +pins the selected platform's child manifest, so the pinned digest can differ +from the top-level manifest-list digest that registry tools report. + +## Layer Application + +Extraction runs under `os.OpenRoot`, so every layer path resolves relative to +the target rootfs and cannot escape via `..` or a symlink. The unpacker +implements the layer behavior common images need: + +- regular files, directories, symlinks, and hardlinks; +- Docker/OCI whiteouts and opaque directory markers; +- permission bits plus setuid, setgid, and sticky, finalized with an explicit + chmod so layer modes survive a restrictive host umask. Where an unprivileged + host rejects the setuid/setgid chmod, the unpacker drops the special bits + (with a warning) and continues rather than aborting: the rootfs is owned by + the invoking user, so those bits could not be honored there anyway. This + fires on macOS when the unpacked file's inherited group is one the invoking + user is not in (a new file takes its parent directory's group, e.g. wheel + under `/tmp`), which is what lets Debian-family images and their shadow suite + (`chage`, `passwd`, ...) unpack at all. Linux keeps the bits (an owned-file + chmod succeeds), so the C runtime's setuid-exec support still applies where + they survive; +- absolute symlink targets rewritten to rootfs-relative links; +- rejection of special files elfuse does not materialize from layers. + +Runtime `/dev` and `/proc` entries are not unpacked from layers; the C runtime +synthesizes the supported ones when the guest opens them. + +## Run Paths + +On macOS the default `run` path uses a case-sensitive APFS sparsebundle per +manifest digest, with the unpacked base rootfs inside it. Each run makes an +APFS copy-on-write clone of the base tree, launches elfuse against the clone, +removes the clone, and detaches the sparsebundle once the last concurrent run +of that digest exits. This protects case-sensitive Linux filenames on normal +(case-insensitive) macOS volumes and keeps repeated runs isolated from +image-layer mutations. + +The plain-rootfs path stays available with `--plain-rootfs` or an explicit +`--rootfs`. It uses a regular directory and execs `elfuse` directly, which is +useful for debugging and for non-Darwin operation. The default sparsebundle +sits on a volume the invoking user created, so unpacked files inherit that +user's own group and the setuid/setgid chmod above succeeds; the special-bit +degrade is therefore mostly a plain-rootfs concern, where `--rootfs` can point +at a directory whose group the user is not in. + +Before launch, `run` writes runtime `/etc/resolv.conf`, `/etc/hosts`, and +`/etc/hostname` into the run rootfs so DNS, localhost, and hostname lookups +work without network namespacing. The writes go through `os.OpenRoot` and +replace any existing entry, so an image that ships one of these names as a +symlink (including a symlinked `/etc`) cannot redirect the write outside the +rootfs. + +`run` launches `elfuse --sysroot ` with the host-literal fallback in +place: an absolute guest path absent under the rootfs resolves to the literal +host path. The rootfs is a root, not a boundary; elfuse favors transparency +over isolation, so an image-launched guest keeps the same filesystem view a +plain positional `elfuse ` run has (a development workflow that +deliberately reaches host resources such as `/etc/resolv.conf` or files under +the user's home). The guest-private prefixes are the exception: `/tmp`, +`/var/tmp`, and `.ccache` paths always resolve inside the rootfs and never +fall back to the host (see usage.md, "Dynamic Linking And +Sysroots"). The practical caveat is +the guest `PATH` search: with an image `Env` `PATH` putting `/usr/bin` before +`/bin`, a bare `gzip` in an image that ships it only at `/bin/gzip` resolves to +the host's incompatible `/usr/bin/gzip` (a macOS Mach-O) first. Prefer +usr-merged images, invoke by absolute path, or reorder the search with +`--env PATH=...`. When the merged environment carries no `PATH`, `run` appends +Docker's conventional default so the guest always has a search path. + +## Runtime Configuration + +`elfuse-oci` resolves the image configuration before calling `elfuse`. + +Command resolution follows Docker-style rules: + +- `--entrypoint` replaces the image Entrypoint and drops the image Cmd; +- without `--entrypoint`, CLI arguments after the reference replace the image + Cmd while preserving the image Entrypoint; +- with neither override, image Entrypoint and Cmd are concatenated; +- an empty final command is an error; +- a relative path command (one containing a slash, like `./server`) resolves + against the working directory; a bare name resolves against the merged + `PATH` inside the image rootfs, and not finding it there is an error. + Unlike Docker, the resolved absolute path is also what the guest sees as + `argv[0]` (elfuse's positional argument names both the binary to load and + the guest argv head). + +Environment resolution starts from image `Env`, or from an empty environment +under `--clear-env`. Each `--env KEY=VALUE` sets or replaces a value; a bare +`--env KEY` imports `KEY` from the host when present. + +User resolution accepts numeric `UID[:GID]` and symbolic `name[:group]`. +Symbolic names resolve against the unpacked rootfs `/etc/passwd` and +`/etc/group`, opened through `os.OpenRoot` so a symlinked account file cannot +redirect resolution to host data. Working directories must be guest-absolute. + +## Lifecycle + +The lifecycle commands operate on the local store: + +- `list` reads `refs.json` and the pinned image metadata; +- `rmi` removes a ref (or a unique SHA-256 digest prefix from `list`), + garbage-collects blobs no remaining manifest reaches, and reclaims the + image's unpacked cache when its last ref goes away; +- `prune` garbage-collects unreachable blobs without removing a named ref; + `prune --cache` also removes unpacked rootfs caches. + +Garbage collection is reachability-based: shared manifests, configs, and layers +stay on disk while any remaining ref reaches them. + +On macOS, cache cleanup also handles sparsebundle state, detaching stale mounts +and reaping per-run clone directories left by killed runs, with two safety +rules. A still-pinned digest's bundle is untouched by a plain `prune --cache` +(recovery of a crashed pinned bundle happens on the next `run`, or via `--all`), +and a volume a live run still uses is never force-detached. Liveness is decided +by a per-digest advisory lock (`run.lock`, held shared by every live run for +its whole lifetime), not by inspecting process ids, which avoids the pid-reuse +hazard. A sweep acquires that lock exclusively; while it cannot, the bundle is +left in place, and once it can, every clone is abandoned by construction and is +reaped except those a `run --keep` retained (a `.elfuse-keep` file) and any +leftover unpack staging directory. `rmi` reclaims a cache the same way: a plain +`rmi` drops the removed image's cache (derived state that goes with the image), +but refuses while a live run uses the volume (not even `--force` overrides) and +refuses without `--force` when the cache holds `run --keep` output. + +### Concurrency + +elfuse-oci is a single-user CLI, not a daemon. Store metadata stays +consistent under concurrency: pin and index updates, `rmi`, `prune`'s GC, and +`list`'s snapshot all serialize on an exclusive store file lock, so parallel +pulls and lifecycle commands cannot corrupt `refs.json`, `index.json`, or +reachability accounting. + +Per-digest sparsebundle state uses two advisory locks kept beside the bundle +(outside the mounted volume, so a detach cannot revoke them): an `attach.lock` +serializing lifecycle transitions, and the shared `run.lock` above. Concurrent +runs of the same image share one attach: the first provisions and attaches, +later runs join under a shared `run.lock`, and the volume detaches only when the +last exits. A `prune`/`rmi` sweep takes both locks non-blocking, so it either +wins (no run is live) or skips the busy bundle. A cold run unpacks into a +temporary directory and atomically renames it into place, so a concurrent probe +never sees a half-written rootfs and an interleaved sweep cannot delete one +mid-unpack. + +The plain digest-keyed rootfs cache (`rootfs//`, the default on +non-Darwin and on the `--plain-rootfs` path) follows the same liveness rule +with a single sibling `.lock`, held shared by a run from before the +cache-existence probe until the guest exits. The lock sits beside the directory +because the directory is the guest's `/` and its existence is the +unpack-complete signal. The plain run path execs `elfuse` in place, so the lock +descriptor is made exec-survivable and rides into the elfuse process: the kernel +releases the flock exactly when the guest exits, `SIGKILL` included. An explicit +`--rootfs` directory is user-managed and outside this scheme. + +## Validation + +The on-disk store is the contract: its layout is checked for spec-conformance +and cross-tool readability (crane/skopeo/umoci), and the pipeline is exercised +offline on Linux and end-to-end on macOS/HVF. `elfuse-oci` builds and +unit-tests without Hypervisor.framework; only an actual `run` guest boot needs +it. The concrete targets, env-var gates, and the Linux/macOS CI split live in +[testing.md](testing.md#oci-image-cli). diff --git a/docs/testing.md b/docs/testing.md index d5bbd978..368ab807 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -15,6 +15,7 @@ Host build requirements: - GNU `make` - GNU `objcopy` or `llvm-objcopy` - GNU coreutils +- Go, for the elfuse-oci OCI CLI - `bash` 3.2+ (the version Apple ships as `/bin/bash`) is sufficient for the test harness; no Homebrew `bash` is required. See `tests/lib/bash-compat.sh` for the cross-version shims (a portable @@ -32,6 +33,33 @@ Guest test builds additionally require: - An AArch64 Linux cross-compiler for C test programs - An AArch64 bare-metal toolchain for the assembly smoke test +OCI interop checks additionally require `jq`. `crane`, `skopeo`, and `umoci` +are used when available by `make oci-interop`; CI installs them so cross-tool +layout parsing and registry-truth checks are hard gates there. + +`elfuse-oci` uses the Go `go-containerregistry` library directly for pulling +images, selecting platforms, and maintaining the OCI image-layout store. The +`crane` CLI is only used here as an independent registry/layout checker, while +`umoci` is used to verify that another OCI implementation can parse the store; +neither tool is on the normal `elfuse-oci run` execution path. + +For a full local `make oci-interop` run on macOS: + +```sh +brew install jq skopeo umoci +go install github.com/google/go-containerregistry/cmd/crane@v0.21.7 +export PATH="$(go env GOPATH)/bin:$PATH" +``` + +For a full local run on Ubuntu: + +```sh +sudo apt-get install -y jq skopeo +go install github.com/google/go-containerregistry/cmd/crane@v0.21.7 +go install github.com/opencontainers/umoci/cmd/umoci@latest +export PATH="$(go env GOPATH)/bin:$PATH" +``` + The toolchain defaults are defined in `mk/toolchain.mk`. These variables are intended to be overridden when needed: @@ -81,6 +109,10 @@ make check make test-rosetta-all make test-gdbstub make test-matrix +make elfuse-oci +make oci-test +make oci-lint +make oci-interop make lint make clean ``` @@ -146,6 +178,13 @@ What they do: - `make test-gdbstub`: debugger integration checks against the built-in GDB stub - `make test-matrix`: cross-check `elfuse` (aarch64), QEMU (aarch64), and `elfuse` (x86_64-via-Rosetta) on overlapping corpora +- `make elfuse-oci`: build the Go OCI image CLI +- `make oci-test`: run offline elfuse-oci tests +- `make oci-lint`: `gofmt` check plus `go vet` for elfuse-oci (vet runs for both + `GOOS` values, so the darwin sparsebundle files are checked from Linux and the + non-darwin stubs from macOS) +- `make oci-interop`: pull OCI fixtures, check the raw image-layout files with + `jq`, and ask available external tools to read the same store - `make lint`: static analysis through `clang-tidy` ## Quick Iteration @@ -176,6 +215,72 @@ or run all matrix modes back-to-back with `make test-matrix`. green `make check` covers BusyBox validation. Use `make test-busybox` to iterate on a single applet failure without rerunning the unit suite. +### OCI Image CLI + +The OCI image CLI (`build/elfuse-oci`) is pure Go, so most of it builds and tests +without Hypervisor.framework; only an actual `elfuse-oci run` guest boot needs +HVF. For elfuse-oci changes: + +```sh +make elfuse-oci # build +make oci-test # offline unit + conformance tests +make oci-lint # gofmt check + go vet (both GOOS values) +ELFUSE_OCI_NETTEST=1 make oci-test # add the registry pull round-trip +make oci-interop # image-layout + cross-tool interop (needs jq) +``` + +`make oci-interop` always requires `jq`; install `crane`, `skopeo`, and `umoci` +too when you want the local run to match the CI cross-tool gate. The Darwin +sparsebundle round-trip (real `hdiutil` + case-sensitive APFS) is gated behind an +env var and runs only on macOS: + +```sh +ELFUSE_OCI_DARWIN_CS=1 go test -run TestDarwinCSSweep ./cmd/elfuse-oci/ +``` + +CI splits this coverage by what each runner can do: + +- **Linux (hosted)**: build, `gofmt`/`go vet` (including a `GOOS=darwin` + cross-vet of the sparsebundle files), the pull/inspect/unpack/list/rmi/prune + lifecycle, the `ELFUSE_OCI_NETTEST` round-trip, and crane/skopeo/umoci interop. + The unit suite runs with `-race` here. +- **macOS (hosted)**: build plus the full `go test -race` on darwin (exercising + the sparsebundle/clone code the Linux job can only cross-vet). The darwin + concurrency-sensitive code (cache sweep, provision, clone, cache-removal lock + discipline) compiles only on darwin, so this is the only place the race + detector ever sees it; keep `-race` on this step. Plus the real + `ELFUSE_OCI_DARWIN_CS` sparsebundle round-trip, and a run-less + pull/inspect/list/rmi/prune lifecycle smoke through the darwin binary. + No HVF needed. + +### Writing OCI unit-test fixtures + +An image is untrusted input: its layers control file *names and shapes*, not +just contents. When testing unpack, cache, or sweep code, build fixtures that +adversarially exercise those, not only escape-by-content: + +- **Reserved marker names.** An image can ship a path that collides with an + elfuse-managed marker (e.g. a layer with `/.elfuse-keep`). Assert the marker + is ignored when it comes from image content and honored only when + elfuse-oci wrote it out of band (see + `TestListSweepableClonesReapsImageShippedKeepFile`). +- **Implicit directories.** A layer may create `dir/sub/file` with no `dir/sub` + tar entry. Whiteout/opaque logic must treat that implicit parent as + current-layer content (see `TestUnpackOpaqueAfterImplicitParentChildren`). +- **Platform variants.** Set `cfg.Variant` (e.g. `linux/arm/v7`) so `inspect` + and `list` are checked to agree, not just os/arch. +- **Concurrency.** Use the package-level function-var seams (`afterImageResolve`, + `cranePull`, `isMountPointFn`, ...) to stage a racing pull/rmi in the exact + TOCTOU window rather than only single-command paths, and hold the store or + cache flocks directly to assert a busy path refuses. +- **macOS + HVF (self-hosted)**: end-to-end `elfuse-oci run` guest boots: + the alpine:3 default-entrypoint smoke that proves pull → sparsebundle → + COW clone → HVF launch → exit-code propagation, and a full + pull → inspect → list → run → rmi → prune lifecycle that runs + python:3.12-slim with an `--entrypoint` override and asserts both teardowns + (a plain rmi reclaiming the cold cache with the image, then a `run --keep` + cache that rmi refuses without `--force` and `--force` detaches and drops). + ## Test Matrix The matrix driver lives in `tests/test-matrix.sh`. It currently covers three @@ -406,6 +511,7 @@ Suggested minimum validation: |-------------|------------------------| | CLI, logging, docs-only build rules | `make elfuse` | | Filename codec, case-exact walk, sysroot resolvers | `make check` (runs the codec unit tests, name lanes, and byte-exact oracle lane), plus `make test-sysroot-name-soak` for resolver concurrency. A red golden vector in `test-casefold-host` means the on-disk format moved: see `docs/filenames.md` before touching `tests/casefold-vectors.h` | +| OCI lifecycle or store behavior | `make oci-lint && make oci-test && make oci-interop` | | General syscall or runtime logic | `make elfuse && make check && make test-matrix-elfuse-aarch64` | | `/proc`, `/dev`, path, or BusyBox-sensitive behavior | `make elfuse && make check && make test-matrix-elfuse-aarch64` | | Rosetta hosting, x86_64 dispatch, VZ ioctls, AOT cache | `make elfuse && make test-rosetta-all` | diff --git a/docs/usage.md b/docs/usage.md index 50d09133..951f4150 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -248,3 +248,189 @@ That has a few direct implications: work entirely inside the VM. Programs that link against `libfuse` (sshfs, ntfs-3g, AppImage runtimes) run without macFUSE, FUSE-T, or FSKit on the host. + +## OCI Images + +elfuse can run programs from OCI images. All image work is handled by +`build/elfuse-oci` (Go); execution still goes through the normal +`build/elfuse --sysroot` runtime, so `elfuse` itself has no OCI commands. + +```sh +build/elfuse-oci [flags] +``` + +This consumes the OCI image format for distribution; it is not a container +runtime. There are no namespaces, cgroups, port mapping, daemon, `docker exec`, +or image build/push, and the rootfs is the guest's root, not an isolation +boundary. See +[oci-design.md](oci-design.md#scope-and-limitations) for the exact list of what +is and is not implemented, and for the design model. + +### Store And Platform + +`elfuse-oci` stores images in an OCI image-layout directory. The default +store is `$ELFUSE_OCI_STORE` when set, otherwise `~/.local/share/elfuse/oci`. +Use `--store DIR` on any subcommand to override it. + +Pulls default to `linux/arm64`. Use `--platform os/arch[/variant]` to select +another platform, such as `linux/amd64` for a Rosetta-backed guest: + +```sh +build/elfuse-oci pull --platform linux/amd64 alpine:3 +``` + +### Commands + +```sh +build/elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +``` + +Pull `` into the local store and pin it by its original reference. + +```sh +build/elfuse-oci inspect [--store DIR] [--json] +``` + +Print the stored image's manifest and config summary. `--json` prints the raw +config JSON. + +```sh +build/elfuse-oci unpack [--store DIR] [--rootfs DIR] +``` + +Unpack the stored image's layers into a rootfs directory. Without `--rootfs`, +the store's digest-keyed rootfs cache is used. + +```sh +build/elfuse-oci run [flags] [args...] +``` + +Run an image. If `` is not present, `run` pulls it first; if the rootfs +cache is missing, it unpacks it first. + +```sh +build/elfuse-oci list [--store DIR] [--json] # alias: images +``` + +List pinned refs, their manifest digests, platform, creation time, compressed +layer size, and layer count. + +```sh +build/elfuse-oci rmi [--store DIR] [--force] +``` + +Remove a ref, or a unique SHA-256 digest prefix from `list`, and +garbage-collect blobs no remaining ref reaches. Removing the last ref for an +image also reclaims its unpacked cache. `rmi` refuses while a live run still +uses the cache (never overridable), and refuses without `--force` when the +cache holds `run --keep` output. + +```sh +build/elfuse-oci prune [--store DIR] [--cache] [--all] [--dry-run] +``` + +Garbage-collect unreachable blobs. `--cache` also removes unpacked rootfs +caches; with `--cache`, `--all` removes them even for still-pulled refs. +`--dry-run` reports what would be removed. + +### Running Images + +```sh +make elfuse elfuse-oci + +build/elfuse-oci run alpine:3 /bin/sh -c 'echo hello' +``` + +`run` accepts the common flags plus these run-specific flags: + +| Flag | Meaning | +|------|---------| +| `--entrypoint PATH` | Replace the image Entrypoint. The image Cmd is dropped. | +| `--env KEY=VALUE` | Set or replace a guest environment variable. Repeatable. | +| `--env KEY` | Import `KEY` from the host environment when present. | +| `--clear-env` | Start from an empty environment instead of image `Env`. | +| `--user UID[:GID]` | Run as a numeric user and optional group. | +| `--user name[:group]` | Resolve names through rootfs `/etc/passwd` and `/etc/group`. | +| `--workdir DIR` | Set the initial guest working directory. Must be absolute. | +| `--rootfs DIR` | Use an explicit rootfs directory. | +| `--plain-rootfs` | Use a plain directory cache instead of the macOS sparsebundle. | +| `--sparse-size SIZE` | Set the sparsebundle virtual size. Default is `16g`. | +| `--no-clone` | Run against the cached base rootfs directly. Mutations persist. | +| `--keep` | Keep the per-run clone and sparsebundle mount for inspection. | + +The command vector follows Docker-style rules: with no CLI args, the image +Entrypoint plus Cmd is used; CLI args after `` replace image Cmd but keep +image Entrypoint; `--entrypoint` replaces image Entrypoint and discards Cmd; an +empty final command is an error. A relative path command (`./server`) resolves +against the working directory, and a bare name resolves via the merged `PATH` +inside the image rootfs (see `oci-design.md` for the `argv[0]` caveat). + +Environment resolution starts from image `Env` (or an empty environment under +`--clear-env`); each `--env KEY=VALUE` sets or appends, and a bare `--env KEY` +imports the host value when set. `--user` defaults to image `User`, then root; +symbolic users and groups are resolved after the rootfs exists, and a name that +fails to resolve is an error. + +### macOS Rootfs Behavior + +By default, `run` uses a case-sensitive APFS sparsebundle per image digest so +the guest's case-sensitive filenames do not collide on a case-insensitive host +volume. The unpacked image lives as a cached base tree inside the sparsebundle, +and each run gets an APFS copy-on-write clone of it, so guest writes do not +mutate the cached rootfs. The clone is removed and the sparsebundle detached +when the guest exits; `--keep` leaves both for inspection, and `prune --cache` +reaps stale caches (including clones left by killed runs) later. + +Use `--plain-rootfs` or `--rootfs DIR` when you explicitly want the regular +directory path. + +### Runtime Files + +Before launching the guest, `run` writes these files into the run rootfs: + +| File | Source | +|------|--------| +| `/etc/resolv.conf` | Host resolver config, with a fallback nameserver if needed. | +| `/etc/hosts` | localhost plus the host name. | +| `/etc/hostname` | Host name. | + +The `resolv.conf` fallback (used only when the host's own config is unreadable +or empty) is Google's public `8.8.8.8`; on hosts using private or split-horizon +DNS, fix the host `/etc/resolv.conf` if guest lookups must stay on the local +resolver. The C runtime also serves synthetic `/proc` and selected `/dev` +entries to every guest, image-launched or not. + +### Host Fallback And PATH + +The image rootfs is a root, not a boundary: an absolute guest path absent from +the rootfs falls back to the literal host path, the same mechanism that lets a +plain positional `elfuse ` run reach host resources such as +`/etc/resolv.conf`. The guest-private prefixes (`/tmp`, `/var/tmp`, `.ccache`) +are the exception and always resolve inside the rootfs; see +[Dynamic Linking And Sysroots](#dynamic-linking-and-sysroots) for the +dispatch model. + +The visible consequence is the guest `PATH` search. With an image `PATH` +searching `/usr/bin` before `/bin`, a bare `gzip` in an image that ships it only +at `/bin/gzip` resolves to the host's incompatible `/usr/bin/gzip` (a macOS +Mach-O) first. Prefer usr-merged images, invoke by absolute path, or reorder the +search with `--env PATH=...`. When neither the image config nor `--env` provides +a `PATH`, `run` appends Docker's conventional default +(`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`). See +[oci-design.md](oci-design.md#run-paths) for the rationale. + +### Lifecycle + +The store keeps image blobs separately from unpacked rootfs caches, and garbage +collection is reachability-based: removing a ref never deletes blobs another ref +still reaches, and removing one of several refs to the same digest keeps the +shared cache. A normal cleanup sequence: + +```sh +build/elfuse-oci list +build/elfuse-oci rmi alpine:3 +build/elfuse-oci prune --cache --dry-run +build/elfuse-oci prune --cache +``` + +See [oci-design.md](oci-design.md#lifecycle) for the GC and cache-safety model. From 45873b84a4c1dc9a3aeae453bdfa78e6b8eba28a Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Sat, 18 Jul 2026 20:55:16 +0200 Subject: [PATCH 10/26] Add per-image real-workload CI for elfuse-oci Issue #224 profiled five real images (python:3.12-slim, node:22-alpine, golang:1.23-alpine, eclipse-temurin:21, gcc:14). Add one CI job per image that boots the image under HVF via `elfuse-oci run` and drives that image's operations, so a change that breaks any of them is caught on the PR rather than by hand. A shared driver, scripts/ci/oci-workload.sh , maps each key to its image and guest workload under scripts/ci/workloads/ and asserts a per-image sentinel token: - python: a single-threaded SQLite insert plus an aggregate query, a small file write/read/checksum fan-out, and a JSON round-trip. - node: in-guest compute (fs/crypto/zlib/JSON) plus an HTTP server the job reaches over the host loopback; elfuse maps guest sockets to host sockets and does no network-namespace isolation, so a guest bound to 127.0.0.1 is reachable host-side. - go: gofmt over a tree of misformatted fixtures the workload writes itself, asserting the file set it names, the bytes it produces, and that a second pass names nothing. The toolchain binaries are Go programs, so this drives the runtime's own scheduler and raw syscalls; it compiles nothing, because the guest compiler dies on SIGHUP before finishing a package. - jvm: javac + java exercising collections, file I/O, SHA-256, an 8-thread pool, and a subprocess. - c: a small multi-file make project plus a heavier single translation unit compiled with gcc -O1. Each workload self-check asserts exact known outputs (pinned digests, exact sums, byte-compared read-backs), not just output shape, and the node server request count is validated so a zero cannot pass the request loop vacuously. The go and python lanes stay inside the runtime's current limits; heavier variants that stress those limits, including an in-guest Go build, are submitted separately. The jobs run only on self-hosted Apple Silicon because `run` needs Hypervisor.framework. They share a composite action that fetches the elfuse binary from build-macos and builds elfuse-oci, and each keeps a warm per-key store on the runner's persistent disk so only the first run pulls over the network. gcc:14 and eclipse-temurin:21 ship the shadow suite, so those jobs also exercise the unpack setuid/setgid degrade end to end. Each leg ends with prune --cache, so a moved tag's stranded caches, and not only its blobs, stay bounded on the runner's persistent store. The lanes are legs of one matrixed job, differing only in the workload key and a timeout, so the shared runner, guards, and setup action are stated once. fail-fast is off: each image is an independent signal. The python workload moves under scripts/ci/workloads/ and is rewritten for this lane, dropping the multi-threaded and WAL SQLite stress in favor of a single-threaded insert plus a JSON round-trip; the heavier variant lives on the workload-stress branch. oci-lifecycle.sh follows the new path. --- .github/actions/hvf-elfuse-setup/action.yml | 66 +++++++++ .github/workflows/main.yml | 60 ++++++++ docs/testing.md | 48 ++++-- scripts/ci/oci-lifecycle.sh | 29 +--- scripts/ci/oci-python-workload.py | 117 --------------- scripts/ci/oci-workload.sh | 154 ++++++++++++++++++++ scripts/ci/workloads/c-workload.sh | 73 ++++++++++ scripts/ci/workloads/go-workload.sh | 84 +++++++++++ scripts/ci/workloads/jvm-workload.sh | 93 ++++++++++++ scripts/ci/workloads/node-compute.js | 50 +++++++ scripts/ci/workloads/node-server.js | 34 +++++ scripts/ci/workloads/python-workload.py | 78 ++++++++++ 12 files changed, 734 insertions(+), 152 deletions(-) create mode 100644 .github/actions/hvf-elfuse-setup/action.yml delete mode 100644 scripts/ci/oci-python-workload.py create mode 100755 scripts/ci/oci-workload.sh create mode 100644 scripts/ci/workloads/c-workload.sh create mode 100644 scripts/ci/workloads/go-workload.sh create mode 100644 scripts/ci/workloads/jvm-workload.sh create mode 100644 scripts/ci/workloads/node-compute.js create mode 100644 scripts/ci/workloads/node-server.js create mode 100644 scripts/ci/workloads/python-workload.py diff --git a/.github/actions/hvf-elfuse-setup/action.yml b/.github/actions/hvf-elfuse-setup/action.yml new file mode 100644 index 00000000..629e2ada --- /dev/null +++ b/.github/actions/hvf-elfuse-setup/action.yml @@ -0,0 +1,66 @@ +name: HVF elfuse setup +description: > + Shared setup for the self-hosted HVF workload jobs: fail fast if the run is + superseded by a newer PR commit, then fetch the prebuilt elfuse binary from + the build-macos job and build the pure-Go elfuse-oci CLI. Assumes the repo is + already checked out. + +runs: + using: composite + steps: + # Fail fast if this run targets a commit that is no longer the PR's HEAD. + # cancel-in-progress covers a newer push, but not a manual "Re-run jobs" on + # an old run, which would burn the self-hosted runner re-testing stale code. + # Mirrors the runtime-macos guard; fails (not cancels) because repo policy + # caps the token at actions: read. The lookup fails open. + - name: Fail fast if superseded by a newer PR commit + if: github.event_name == 'pull_request' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -uo pipefail + latest=$(curl -fsSL \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$REPO/pulls/$PR_NUMBER" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["head"]["sha"])') \ + || latest="" + echo "Run targets : $RUN_SHA" + echo "PR HEAD now : ${latest:-}" + if [ -n "$latest" ] && [ "$latest" != "$RUN_SHA" ]; then + echo "::error::This run targets $RUN_SHA, but PR #$PR_NUMBER HEAD is now $latest, so the commit is no longer the latest. Failing instead of re-testing stale code on the self-hosted runner; re-run CI on the current commit." + exit 1 + fi + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + # Reuse the arm64 elfuse binary built + entitlement-checked by build-macos + # instead of rebuilding the C project on the self-hosted runner five times. + # Mach-O code signatures (and their embedded HVF entitlement) travel inside + # the binary, so they survive the artifact zip round-trip; only the execute + # bit is lost and restored here. + - name: Download prebuilt elfuse binary + uses: actions/download-artifact@v7 + with: + name: elfuse-${{ runner.os }}-${{ runner.arch }} + path: build + + - name: Restore execute bit + verify HVF entitlement + shell: bash + run: | + set -euo pipefail + chmod +x build/elfuse + codesign -d --entitlements - build/elfuse 2>&1 \ + | grep -q 'com\.apple\.security\.hypervisor' + + - name: Build elfuse-oci + shell: bash + run: make build/elfuse-oci diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e9a5903..2c95b5bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,10 @@ # (crane/skopeo/umoci) on Linux # oci-image-macos : elfuse-oci darwin build, unit tests, sparsebundle # round-trip, and a run-less image-lifecycle smoke +# workload : per-image real-workload smokes (python/node/go/jvm/c, one +# matrix leg each) that boot the image under HVF on +# self-hosted Apple Silicon and drive its characteristic +# operations # # Runtime and sanitizer tests require Hypervisor.framework, which # GitHub-hosted macOS runners do not expose. Those tests run on self-hosted @@ -793,3 +797,59 @@ jobs: # that the Linux job's cache-reclaiming flow does not cover. jq # ships on the macos-15 image. run: scripts/ci/oci-cli-smoke.sh + + # Per-image real-workload jobs, one matrix leg per profiled image. Each leg + # boots a real image under HVF via `elfuse-oci run` and drives that image's + # characteristic operations, so a code change that breaks any of them is + # caught on the PR. The legs differ only in the workload key and a timeout, + # so a matrix states the shared runner, guards, and setup action once and + # makes adding an image a one-line change. fail-fast is off because each + # image is an independent signal: one image regressing must not hide the + # state of the others. + # + # They share .github/actions/hvf-elfuse-setup, which fetches the elfuse + # binary from build-macos and builds elfuse-oci. `run` needs + # Hypervisor.framework, so these are self-hosted only. Each keeps a warm + # per-key store on the runner's persistent disk so only the first run pulls + # over the network. gcc:14 and eclipse-temurin:21 ship the shadow suite, so + # they also exercise the unpack setuid/setgid degrade end to end. + workload: + name: Workload (${{ matrix.name }}) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + timeout-minutes: ${{ matrix.timeout }} + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-${{ matrix.key }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + strategy: + fail-fast: false + matrix: + include: + # The node lane includes the HTTP-server phase: the guest binds + # 127.0.0.1 (elfuse maps sockets to host sockets, no netns) and the + # workload curls it host-side. + - { key: python, name: Python, timeout: 30 } + - { key: node, name: Node, timeout: 30 } + - { key: go, name: Go, timeout: 30 } + # javac + java startup and the compile step run slower than the + # lighter images, and eclipse-temurin is a large first pull. + - { key: jvm, name: JVM, timeout: 45 } + # gcc:14 is the largest first pull and the compile bursts (make plus + # a heavier single TU) dominate the wall time. + - { key: c, name: C, timeout: 45 } + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run ${{ matrix.key }} workload + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-${{ matrix.key }}" + scripts/ci/oci-workload.sh ${{ matrix.key }} diff --git a/docs/testing.md b/docs/testing.md index 368ab807..e4d52f34 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -229,10 +229,9 @@ ELFUSE_OCI_NETTEST=1 make oci-test # add the registry pull round-trip make oci-interop # image-layout + cross-tool interop (needs jq) ``` -`make oci-interop` always requires `jq`; install `crane`, `skopeo`, and `umoci` -too when you want the local run to match the CI cross-tool gate. The Darwin -sparsebundle round-trip (real `hdiutil` + case-sensitive APFS) is gated behind an -env var and runs only on macOS: +Tool prerequisites for `make oci-interop` are listed under Build Requirements +above. The Darwin sparsebundle round-trip (real `hdiutil` + case-sensitive +APFS) is gated behind an env var and runs only on macOS: ```sh ELFUSE_OCI_DARWIN_CS=1 go test -run TestDarwinCSSweep ./cmd/elfuse-oci/ @@ -252,6 +251,40 @@ CI splits this coverage by what each runner can do: `ELFUSE_OCI_DARWIN_CS` sparsebundle round-trip, and a run-less pull/inspect/list/rmi/prune lifecycle smoke through the darwin binary. No HVF needed. +- **macOS (self-hosted, HVF)**: the only place `elfuse-oci run` actually boots a + guest. An alpine:3 default-entrypoint smoke proves pull → sparsebundle → + COW clone → HVF launch → exit-code propagation; a pull → inspect → list → + run → rmi → prune lifecycle runs python:3.12-slim with an `--entrypoint` + override and asserts the teardown guardrails (a plain rmi reclaiming the + cold cache with the image, then a `run --keep` cache that rmi refuses + without `--force` and `--force` detaches and drops). Besides those, a + per-image workload suite + (the `workload` job, one matrix leg per image: python, node, go, jvm, c) + drives each image through its characteristic operations. The shared driver is + `scripts/ci/oci-workload.sh `; each image's guest workload lives in + `scripts/ci/workloads/`: + + - **python** (`python:3.12-slim`): a single-threaded SQLite insert plus + aggregate query, a 50-file write/read/checksum pass, and a JSON round-trip. + - **node** (`node:22-alpine`): in-guest compute (fs fan-out, crypto, zlib, + JSON) plus an HTTP server the job curls over the host loopback before + `/quit`. + - **go** (`golang:1.23-alpine`): `go version` plus a `gofmt` of a tiny file + (deliberately build-free; a full build overruns elfuse's fixed thread + table). + - **jvm** (`eclipse-temurin:21`): `javac` + `java` exercising collections, + file I/O, SHA-256, an 8-thread pool, and a subprocess. + - **c** (`gcc:14`): a small multi-file `make` project plus a larger single + translation unit compiled with `gcc -O1`. + + `gcc:14` and `eclipse-temurin:21` are Debian/Ubuntu-based and ship the shadow + suite, so these jobs also exercise the unpack setuid/setgid degrade end to + end. Each job keeps a warm per-key store on the runner's persistent disk, so + only the first run pulls over the network. Run one locally with: + + ```sh + ELFUSE_OCI_STORE=/tmp/elfuse-oci-store scripts/ci/oci-workload.sh c + ``` ### Writing OCI unit-test fixtures @@ -273,13 +306,6 @@ adversarially exercise those, not only escape-by-content: `cranePull`, `isMountPointFn`, ...) to stage a racing pull/rmi in the exact TOCTOU window rather than only single-command paths, and hold the store or cache flocks directly to assert a busy path refuses. -- **macOS + HVF (self-hosted)**: end-to-end `elfuse-oci run` guest boots: - the alpine:3 default-entrypoint smoke that proves pull → sparsebundle → - COW clone → HVF launch → exit-code propagation, and a full - pull → inspect → list → run → rmi → prune lifecycle that runs - python:3.12-slim with an `--entrypoint` override and asserts both teardowns - (a plain rmi reclaiming the cold cache with the image, then a `run --keep` - cache that rmi refuses without `--force` and `--force` detaches and drops). ## Test Matrix diff --git a/scripts/ci/oci-lifecycle.sh b/scripts/ci/oci-lifecycle.sh index 52e713f0..f076ad55 100755 --- a/scripts/ci/oci-lifecycle.sh +++ b/scripts/ci/oci-lifecycle.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Whole user-facing image lifecycle against a real HVF-booted guest, then # the teardown guardrails, one function per phase: -# run_workloads pull/inspect/list, an --entrypoint override, -# and a glibc dynamically-linked python workload +# run_workloads pull/inspect/list and a glibc dynamically- +# linked python one-liner via --entrypoint # teardown_plain_rmi a plain rmi reclaims the cold unpacked cache # with the image # teardown_keep_guardrails rmi refuses to discard run --keep output @@ -25,19 +25,10 @@ SEED="${ELFUSE_OCI_SEED_STORE:?set ELFUSE_OCI_SEED_STORE to the warm seed store IMG="${IMG:-python:3.12-slim}" export ELFUSE_OCI_STORE +# reap_guest (in oci-lib.sh) keeps a failed phase from leaking the +# backgrounded guest. guest="" -on_exit() { - rc=$? - # A failed phase must not leak the backgrounded guest: it would keep - # holding the per-digest flock (and its sleep) and poison the next - # prune/rmi on a persistent self-hosted runner. - if [ -n "$guest" ] && kill -0 "$guest" 2>/dev/null; then - kill "$guest" 2>/dev/null || true - wait "$guest" 2>/dev/null || true - fi - exit "$rc" -} -trap on_exit EXIT +trap reap_guest EXIT # The teardowns leave the ephemeral store EMPTY (asserted), so the # lifecycle store itself cannot persist between CI runs. Keep a warm seed @@ -67,16 +58,6 @@ run_workloads() { -c 'import json,math; print(json.dumps({"pi":round(math.pi,5),"ok":True}))')" printf 'guest said: %s\n' "$out" [ "$out" = '{"pi": 3.14159, "ok": true}' ] || fail "python one-liner said '$out'" - - # A non-trivial application: concurrent SQLite writers (fcntl locking, - # fsync, WAL where the guest FS supports it) plus a 64-file - # write/read/checksum fan-out. Exercises far more of the dynamically- - # linked glibc guest than the one-liner above; prints a single sentinel - # token only on full success. - out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ - -c "$(cat "$OCI_CI_DIR/oci-python-workload.py")")" - printf 'python workload said: %s\n' "$out" - [ "$out" = elfuse-oci-python-workload-ok ] || fail "python workload said '$out'" } # The runs above left the cache warm and then detached on exit, so a plain diff --git a/scripts/ci/oci-python-workload.py b/scripts/ci/oci-python-workload.py deleted file mode 100644 index 6c1c044d..00000000 --- a/scripts/ci/oci-python-workload.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -"""Non-trivial guest workload for the elfuse-oci OCI lifecycle CI. - -Exercises more of a real dynamically-linked glibc guest than a one-line print: -concurrent SQLite writers (fcntl locking, fsync, and, when the guest FS -supports it, WAL mmap), plus a filesystem fan-out whose written content is -read back and checksummed. On full success it prints a single sentinel token -the workflow asserts on; any failure exits non-zero with a diagnostic. - -Self-contained (stdlib only) so it runs under the image's bundled Python with -no network or extra packages, and is passed to the guest via `python3 -c`. -""" - -import hashlib -import os -import sqlite3 -import sys -import threading - -DB = "/tmp/elfuse-workload.db" -TREE = "/tmp/elfuse-workload-tree" -THREADS = 8 -PER_THREAD = 1000 -FANOUT = 8 - - -def setup_db(): - con = sqlite3.connect(DB) - try: - # WAL exercises the guest FS's shared-memory index (mmap) and is the - # more demanding path; if the FS cannot back it, SQLite reports a - # different mode and the concurrent-writer count check below still - # validates fcntl locking and durable commits under rollback journal. - con.execute("PRAGMA journal_mode=WAL") - con.execute( - "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " - "tid INTEGER NOT NULL, n INTEGER NOT NULL)" - ) - con.commit() - finally: - con.close() - - -def worker(tid): - con = sqlite3.connect(DB, timeout=60) - try: - con.execute("PRAGMA busy_timeout=60000") - for n in range(PER_THREAD): - con.execute("INSERT INTO t (tid, n) VALUES (?, ?)", (tid, n)) - con.commit() - finally: - con.close() - - -def db_count(): - con = sqlite3.connect(DB, timeout=60) - try: - (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() - return count - finally: - con.close() - - -def content(i, j): - return "elfuse-workload-%d-%d\n" % (i, j) - - -def fs_fanout_ok(): - for i in range(FANOUT): - for j in range(FANOUT): - d = os.path.join(TREE, str(i), str(j)) - os.makedirs(d, exist_ok=True) - with open(os.path.join(d, "f"), "w") as fh: - fh.write(content(i, j)) - fh.flush() - os.fsync(fh.fileno()) - read_back = [] - for i in range(FANOUT): - for j in range(FANOUT): - with open(os.path.join(TREE, str(i), str(j), "f")) as fh: - read_back.append(fh.read()) - expected = [content(i, j) for i in range(FANOUT) for j in range(FANOUT)] - got = hashlib.sha256() - for p in sorted(read_back): - got.update(p.encode()) - want = hashlib.sha256() - for p in sorted(expected): - want.update(p.encode()) - return got.hexdigest() == want.hexdigest() - - -def main(): - setup_db() - threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] - for t in threads: - t.start() - for t in threads: - t.join() - - count = db_count() - if count != THREADS * PER_THREAD: - print( - "sqlite row count %d != %d (concurrent writers lost rows)" - % (count, THREADS * PER_THREAD), - file=sys.stderr, - ) - sys.exit(1) - - if not fs_fanout_ok(): - print("filesystem fan-out checksum mismatch", file=sys.stderr) - sys.exit(1) - - print("elfuse-oci-python-workload-ok") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/oci-workload.sh b/scripts/ci/oci-workload.sh new file mode 100755 index 00000000..b7b7d754 --- /dev/null +++ b/scripts/ci/oci-workload.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Per-image real-workload smoke: drive each image's characteristic operations +# through `elfuse-oci run` under HVF and assert a sentinel token. +# One key per image. `run` pulls on demand, so a warm persistent +# ELFUSE_OCI_STORE keeps reruns network-free. +# +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-workload.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +: "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" +export ELFUSE_OCI_STORE + +key="${1:?usage: oci-workload.sh }" +WL="$OCI_CI_DIR/workloads" + +# assert_sentinel SENTINEL DESC OUTPUT: OUTPUT must contain the fixed SENTINEL. +assert_sentinel() { + printf '%s\n' "$3" | expect_grep "$1" \ + || fail "$2: output missing sentinel '$1'" +} + +# run_capture SENTINEL DESC RUN-ARGS...: run a single-shot guest workload, +# echo its output for the log, and assert the sentinel. Covers every image +# whose workload is one `run` invocation (all but node's two-phase server). +run_capture() { + local sentinel="$1" desc="$2" + shift 2 + local out + out="$("$BIN" run "$@")" + printf '%s\n' "$out" + assert_sentinel "$sentinel" "$desc" "$out" +} + +# Background guest bookkeeping for the node server phase; reap_guest (in +# oci-lib.sh) keeps a failed assertion from leaking the guest. +guest="" +node_outfile="" +on_exit() { + rc=$? + reap_guest + [ -n "$node_outfile" ] && rm -f "$node_outfile" + exit "$rc" +} +trap on_exit EXIT + +# guest_gone succeeds once the backgrounded guest has exited. +guest_gone() { + ! kill -0 "$guest" 2>/dev/null +} + +run_node() { + # Phase A: in-guest compute. + run_capture elfuse-oci-node-compute-ok node-compute \ + --entrypoint /usr/local/bin/node node:22-alpine \ + -e "$(cat "$WL/node-compute.js")" + + # Phase B: HTTP server reached over the host loopback. elfuse forwards + # socket syscalls to host sockets and does no netns isolation, so a guest + # bound to 127.0.0.1 is reachable from the host. The server binds an + # ephemeral port and prints "PORT="; read it back rather than fixing a + # port that could collide with a leaked or concurrent guest. + local reqs="${WL_NODE_REQUESTS:-100}" + # Zero would let the request loop pass vacuously; non-numeric already + # trips the ERR trap at the loop's [ ] comparison. + case "$reqs" in + '' | *[!0-9]* | 0) fail "WL_NODE_REQUESTS must be a positive integer, got '$reqs'" ;; + esac + node_outfile="$(mktemp)" + "$BIN" run --entrypoint /usr/local/bin/node node:22-alpine \ + -e "$(cat "$WL/node-server.js")" >"$node_outfile" 2>&1 & + guest=$! + + # Wait for the server to announce its ephemeral port. Poll rather than + # wait_for so a guest that dies (a bind failure or a runtime crash) surfaces + # its own captured output instead of an opaque timeout. + local waited=0 port="" + while [ "$waited" -lt 120 ]; do + port="$(awk -F= '/^PORT=/{print $2; exit}' "$node_outfile")" + [ -n "$port" ] && break + if ! kill -0 "$guest" 2>/dev/null; then + cat "$node_outfile" >&2 + guest="" + fail "node server exited before announcing a port" + fi + sleep 0.5 + waited=$((waited + 1)) + done + if [ -z "$port" ]; then + cat "$node_outfile" >&2 + fail "node server did not announce a port within 60s" + fi + # The PORT= line is the guest flushing stdout, not proof the socket accepts + # connections yet; probe the port directly before hammering it. + wait_for 30 "node server on 127.0.0.1:$port" \ + curl -fsS -o /dev/null "http://127.0.0.1:$port/" + printf 'node server on 127.0.0.1:%s\n' "$port" + + local i=0 body + while [ "$i" -lt "$reqs" ]; do + # Guard the substitution: a bare body=$(curl ...) would trip the ERR + # trap on any transient failure instead of the specific diagnostic. + if ! body="$(curl -fsS "http://127.0.0.1:$port/")"; then + fail "node server request $i failed (curl)" + fi + [ "$body" = elfuse-node-server-ok ] \ + || fail "node server request $i returned '$body'" + i=$((i + 1)) + done + printf 'node server answered %d requests\n' "$reqs" + + # Clean shutdown: /quit makes the guest exit 0. The connection may reset as + # the guest exits, so tolerate the curl status. If /quit never reaches the + # server the guest would run forever, so bound the wait; on timeout + # wait_for fails and the EXIT trap kills the guest rather than blocking + # to the job's timeout-minutes. + curl -fsS -o /dev/null "http://127.0.0.1:$port/quit" || true + wait_for 10 "node server exit after /quit" guest_gone + if ! wait "$guest"; then + guest="" + fail "node server exited non-zero after /quit" + fi + guest="" +} + +case "$key" in + python) + run_capture elfuse-oci-python-workload-ok python \ + --entrypoint /usr/local/bin/python3 python:3.12-slim \ + -c "$(cat "$WL/python-workload.py")" + ;; + node) run_node ;; + go) + run_capture elfuse-oci-go-workload-ok go \ + golang:1.23-alpine /bin/sh -c "$(cat "$WL/go-workload.sh")" + ;; + jvm) + run_capture elfuse-oci-jvm-workload-ok jvm \ + eclipse-temurin:21 /bin/sh -c "$(cat "$WL/jvm-workload.sh")" + ;; + c) + run_capture elfuse-oci-c-workload-ok c \ + gcc:14 /bin/sh -c "$(cat "$WL/c-workload.sh")" + ;; + *) fail "unknown workload key: $key (want python|node|go|jvm|c)" ;; +esac + +# Keep a persistent store bounded: a moved pin strands the old digest's +# blobs and its unpacked caches; --cache reclaims both (the sparsebundles +# dwarf the blobs). Concurrent legs are safe: the sweep runs under the store +# lock, skips still-pinned digests, and skips busy caches via their flocks. +"$BIN" prune --cache >/dev/null + +echo "workload $key OK" diff --git a/scripts/ci/workloads/c-workload.sh b/scripts/ci/workloads/c-workload.sh new file mode 100644 index 00000000..99bda6a9 --- /dev/null +++ b/scripts/ci/workloads/c-workload.sh @@ -0,0 +1,73 @@ +# shellcheck shell=sh +# C-compile image workload, run in the guest via +# `/bin/sh -c`: a small multi-file project built with make, then a larger single +# translation unit compiled with gcc -O1 (the readlinkat/execve-of-cc1/as/ld +# signature). Prints one sentinel token on success. POSIX sh (dash). gcc:14 is +# Debian-based, so this also proves the setuid/setgid unpack degrade on a +# shadow-suite image. +# +# The Makefile uses .RECIPEPREFIX so its recipes are prefixed with '>' rather +# than a literal tab, which keeps this heredoc robust. Scaled down from the +# profiled 30-file project + 8 MB TU to keep CI minutes sane; the syscall signature +# (per-TU path resolution, compiler/assembler/linker exec) is preserved. +set -e + +# A guest path absent from the rootfs falls back to the literal host path, so a +# generic name like /tmp/cwork would read and write whatever the runner left in +# its own /tmp; an elfuse-owned name is created inside the rootfs instead. +d=/tmp/elfuse-c-work +rm -rf "$d" +mkdir -p "$d" +cd "$d" + +cat > mathx.h <<'EOF' +#ifndef MATHX_H +#define MATHX_H +int tri(int n); +#endif +EOF +cat > mathx.c <<'EOF' +#include "mathx.h" +int tri(int n) { + int s = 0; + for (int i = 1; i <= n; i++) s += i; + return s; +} +EOF +cat > main.c <<'EOF' +#include +#include "mathx.h" +int main(void) { + printf("%d\n", tri(100)); + return 0; +} +EOF +cat > Makefile <<'EOF' +.RECIPEPREFIX = > +CC ?= gcc +app: main.o mathx.o +> $(CC) -O1 -o app main.o mathx.o +%.o: %.c mathx.h +> $(CC) -O1 -c -o $@ $< +EOF + +make -j1 +r=$(./app) +if [ "$r" != "5050" ]; then + echo "make project produced: $r" >&2 + exit 1 +fi + +# A larger single TU: 1000 functions dispatched through a table, summed and +# checked. Exercises a heavier gcc -O1 compile+link than the tiny project above. +awk 'BEGIN { + for (i = 0; i < 1000; i++) print "int f" i "(void){return " i ";}"; + printf "typedef int(*fn)(void);\nstatic fn t[]={"; + for (i = 0; i < 1000; i++) printf "f%d,", i; + print "};"; + print "int main(void){long s=0;for(int i=0;i<1000;i++)s+=t[i]();return s==499500?0:1;}"; +}' > big.c +gcc -O1 -o big big.c +./big + +echo elfuse-oci-c-workload-ok diff --git a/scripts/ci/workloads/go-workload.sh b/scripts/ci/workloads/go-workload.sh new file mode 100644 index 00000000..84f959d0 --- /dev/null +++ b/scripts/ci/workloads/go-workload.sh @@ -0,0 +1,84 @@ +# shellcheck shell=sh +# Go image workload, run in the guest via `/bin/sh -c`. The Go toolchain +# binaries are themselves Go programs, so driving them reaches an elfuse entry +# path no other workload in this suite uses: the Go runtime issues raw Linux +# syscalls instead of routing through libc, and schedules its own goroutines +# over a worker pool that walks and rewrites a directory tree concurrently. +# Prints one sentinel token on success. POSIX sh (busybox ash). +# +# Deliberately does not compile anything in the guest. `go build` spawns +# /usr/local/go/pkg/tool/linux_arm64/compile, which dies on SIGHUP before it +# finishes the first package, so a build step here would test that gap rather +# than the toolchain; the compile-and-run variant lives on the +# oci/workload-stress branch. gofmt needs no compiler, so it exercises the same +# runtime without depending on that gap being closed. +set -e + +# The guest has no HOME, so give the toolchain a writable cache; the go command +# refuses to start without one. GOTOOLCHAIN=local stops it from reaching for a +# toolchain over a network this job does not have. +# +# Both paths carry the elfuse- prefix on purpose. A guest path that is absent +# from the rootfs falls back to the literal host path, so a generic name like +# /tmp/gowork would find, and then write through to, whatever the runner left +# in its own /tmp; an unclaimed name is created inside the rootfs instead. +export GOCACHE=/tmp/elfuse-go-cache GOTOOLCHAIN=local +export GOMAXPROCS=2 + +go version + +d=/tmp/elfuse-go-work +rm -rf "$d" +mkdir -p "$d/src" +cd "$d/src" + +# Every fixture is written here rather than read out of the image, so what the +# assertions below pin belongs to this test and cannot move when the tag is +# republished. Each file is misformatted the same way, so gofmt must rewrite +# all of them and the expected result is one fixed byte sequence. +n=64 +i=1 +while [ "$i" -le "$n" ]; do + printf 'package p\n\nfunc F%s() {\n\tx :=1\n\t_ = x\n}\n' "$i" > "f$i.go" + i=$((i + 1)) +done + +# gofmt walks the tree across goroutines, so this is the concurrent-read half. +# It must name every fixture and nothing else. +listed=$(gofmt -l . | wc -l | tr -d ' ') +if [ "$listed" != "$n" ]; then + echo "gofmt -l named $listed files, expected $n" >&2 + gofmt -l . >&2 + exit 1 +fi + +# The rewrite half: gofmt writes each file through a temporary and renames it +# into place, so this covers create, write and rename as well. +gofmt -w . + +# Byte-exact, because "gofmt changed something" is not the same claim as +# "gofmt produced the right bytes". +expected_file=$(printf 'package p\n\nfunc F7() {\n\tx := 1\n\t_ = x\n}\n') +actual_file=$(cat f7.go) +if [ "$actual_file" != "$expected_file" ]; then + echo "f7.go after gofmt -w:" >&2 + cat f7.go >&2 + exit 1 +fi + +# Nothing may remain unformatted, which is a claim about files this test wrote. +remaining=$(gofmt -l . | wc -l | tr -d ' ') +if [ "$remaining" != "0" ]; then + echo "gofmt -l still names $remaining files after -w" >&2 + gofmt -l . >&2 + exit 1 +fi + +# go env reads the toolchain's own configuration through the same runtime. +root=$(go env GOROOT) +if [ ! -x "$root/bin/go" ]; then + echo "go env GOROOT gave $root, which holds no go binary" >&2 + exit 1 +fi + +echo "elfuse-oci-go-workload-ok files=$n formatted=$n" diff --git a/scripts/ci/workloads/jvm-workload.sh b/scripts/ci/workloads/jvm-workload.sh new file mode 100644 index 00000000..b173d1dc --- /dev/null +++ b/scripts/ci/workloads/jvm-workload.sh @@ -0,0 +1,93 @@ +# shellcheck shell=sh +# JVM image workload, run in the guest via +# `/bin/sh -c`: javac-compile a small program, then run it exercising +# collections, file I/O, a SHA-256 digest, an 8-thread pool, and a subprocess +# (the futex/clock_gettime-heavy signature). The program prints the sentinel +# token itself on success. POSIX sh (dash). eclipse-temurin is Ubuntu-based, so +# this also proves the setuid/setgid unpack degrade on a shadow-suite image. +set -e + +# A guest path absent from the rootfs falls back to the literal host path, so a +# generic name like /tmp/jvmwork would read and write whatever the runner left +# in its own /tmp; an elfuse-owned name is created inside the rootfs instead. +d=/tmp/elfuse-jvm-work +rm -rf "$d" +mkdir -p "$d" +cd "$d" +cat > Main.java <<'EOF' +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.*; + +public class Main { + static String sha256(byte[] b) throws Exception { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] d = md.digest(b); + StringBuilder sb = new StringBuilder(); + for (byte x : d) sb.append(String.format("%02x", x)); + return sb.toString(); + } + + public static void main(String[] args) throws Exception { + // Collections. + Map m = new HashMap<>(); + for (int i = 0; i < 1000; i++) m.put(i, i * i); + long collSum = 0; + for (int v : m.values()) collSum += v; + + // File I/O + digest. Relative to the scratch dir the script cd'd into, + // so it stays inside the rootfs with no second absolute path to drift. + Path p = Paths.get("data.bin"); + byte[] payload = new byte[65536]; + for (int i = 0; i < payload.length; i++) payload[i] = (byte) (i & 0xff); + Files.write(p, payload); + byte[] back = Files.readAllBytes(p); + if (!Arrays.equals(payload, back)) { + System.err.println("file io mismatch"); + System.exit(1); + } + String digest = sha256(back); + + // 8 worker threads. + ExecutorService ex = Executors.newFixedThreadPool(8); + List> fs = new ArrayList<>(); + for (int t = 0; t < 8; t++) { + final int base = t; + fs.add(ex.submit(() -> { + int s = 0; + for (int i = 0; i < 100000; i++) s += (base + i) & 7; + return s; + })); + } + long threadSum = 0; + for (Future f : fs) threadSum += f.get(); + ex.shutdown(); + + // Subprocess. + Process pr = new ProcessBuilder("/bin/echo", "child-ok") + .redirectErrorStream(true).start(); + String childOut = new String(pr.getInputStream().readAllBytes()).trim(); + int rc = pr.waitFor(); + if (rc != 0 || !childOut.equals("child-ok")) { + System.err.println("subprocess failed: " + childOut); + System.exit(1); + } + + // All inputs are fixed, so the results are exact constants: the + // collection sum is sum(i*i) for i in 0..999, each worker's sum is + // 350000 (100000 iterations over a full residue cycle of & 7, so + // base contributes nothing), and the digest is sha256 of the + // 65536-byte i & 0xff ramp. + if (collSum != 332833500L || threadSum != 2800000L + || !digest.equals("7daca2095d0438260fa849183dfc67faa459fdf4936e1bc91eec6b281b27e4c2")) { + System.err.println("sanity failed"); + System.exit(1); + } + System.out.println("elfuse-oci-jvm-workload-ok"); + } +} +EOF + +javac Main.java +java Main diff --git a/scripts/ci/workloads/node-compute.js b/scripts/ci/workloads/node-compute.js new file mode 100644 index 00000000..bee05932 --- /dev/null +++ b/scripts/ci/workloads/node-compute.js @@ -0,0 +1,50 @@ +'use strict'; +// In-guest compute half of the elfuse-oci node image CI (minus the server): core-module loading, a few hundred small-file +// writes read back and hashed, a zlib gzip round-trip, JSON round-trip, and a +// crypto self-check. Prints one sentinel token on full success; any failure +// exits non-zero with a diagnostic. Passed to the guest via `node -e`. + +const crypto = require('crypto'); +const zlib = require('zlib'); +const fs = require('fs'); +const path = require('path'); + +const DIR = '/tmp/elfuse-node'; +fs.mkdirSync(DIR, { recursive: true }); + +// fs write/read fan-out, each file read back and compared so a lost or +// corrupted file is caught. +const N = 400; +for (let i = 0; i < N; i++) { + const p = path.join(DIR, 'f' + i); + const body = 'elfuse-node-' + i + '\n'; + fs.writeFileSync(p, body); + if (fs.readFileSync(p).toString() !== body) { + console.error('file content mismatch: ' + p); + process.exit(1); + } +} + +// zlib gzip round-trip over a non-trivial buffer. +const payload = Buffer.alloc(1 << 16, 0x61); +if (!zlib.gunzipSync(zlib.gzipSync(payload)).equals(payload)) { + console.error('zlib round-trip mismatch'); + process.exit(1); +} + +// JSON round-trip. +const doc = { items: Array.from({ length: 256 }, (_, i) => ({ k: i, v: 'tok-' + i })) }; +const back = JSON.parse(JSON.stringify(doc)); +if (back.items.length !== 256 || back.items[255].v !== 'tok-255') { + console.error('json round-trip mismatch'); + process.exit(1); +} + +// crypto self-check against a fixed vector. +if (crypto.createHash('sha256').update('elfuse').digest('hex') !== + '87097914b16b96d167ef08a895d4d3f81f24267a617faca4b97c30183eb5fe02') { + console.error('sha256 self-check failed'); + process.exit(1); +} + +console.log('elfuse-oci-node-compute-ok'); diff --git a/scripts/ci/workloads/node-server.js b/scripts/ci/workloads/node-server.js new file mode 100644 index 00000000..1437895a --- /dev/null +++ b/scripts/ci/workloads/node-server.js @@ -0,0 +1,34 @@ +'use strict'; +// HTTP-server half of the elfuse-oci node image CI. +// elfuse forwards socket syscalls to host sockets and does no network-namespace +// isolation, so a guest bound to 127.0.0.1 is reachable from the host loopback; +// the driver curls it repeatedly, then GET /quit exits the guest 0. This +// exercises node's accept4/epoll_pwait/writev/shutdown signature. Passed to the +// guest via `node -e`. +// +// The listener binds port 0 (an ephemeral port the kernel picks) and prints +// "PORT=" on stdout so the driver reads the exact port back; a fixed port +// would collide with a leaked or concurrent guest on the shared host loopback. + +const http = require('http'); + +const server = http.createServer((req, res) => { + if (req.url === '/quit') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('bye\n'); + // Close the listener and exit 0 so the run reports a clean shutdown. + server.close(() => process.exit(0)); + return; + } + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('elfuse-node-server-ok\n'); +}); + +server.on('error', (e) => { + console.error('server error: ' + e.message); + process.exit(1); +}); + +server.listen(0, '127.0.0.1', () => { + console.log('PORT=' + server.address().port); +}); diff --git a/scripts/ci/workloads/python-workload.py b/scripts/ci/workloads/python-workload.py new file mode 100644 index 00000000..16d7fab5 --- /dev/null +++ b/scripts/ci/workloads/python-workload.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Guest workload for the elfuse-oci python image CI. + +A deliberately single-threaded exercise of a dynamically-linked glibc python +guest: a small SQLite insert plus an aggregate query, a few dozen files written, +read back, and checksummed, and a JSON round-trip. On full success it prints a +single sentinel token the workflow asserts on; any failure exits non-zero with a +diagnostic. The concurrent/subprocess-heavy stress variant lives on the +oci/workload-stress branch. + +Self-contained (stdlib only) so it runs under the image's bundled Python with no +network or extra packages, and is passed to the guest via `python3 -c`. +""" + +import hashlib +import json +import os +import sqlite3 +import sys + +DB = "/tmp/elfuse-workload.db" +TREE = "/tmp/elfuse-workload-tree" +ROWS = 1000 +FILES = 50 + + +def db_ok(): + con = sqlite3.connect(DB) + try: + con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER NOT NULL)") + for n in range(ROWS): + con.execute("INSERT INTO t (n) VALUES (?)", (n,)) + con.commit() + (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() + (total,) = con.execute("SELECT SUM(n) FROM t").fetchone() + finally: + con.close() + return count == ROWS and total == sum(range(ROWS)) + + +def fs_ok(): + os.makedirs(TREE, exist_ok=True) + for i in range(FILES): + with open(os.path.join(TREE, "f%d" % i), "w") as fh: + fh.write("elfuse-%d\n" % i) + fh.flush() + os.fsync(fh.fileno()) + got = hashlib.sha256() + want = hashlib.sha256() + for i in range(FILES): + with open(os.path.join(TREE, "f%d" % i)) as fh: + got.update(fh.read().encode()) + want.update(("elfuse-%d\n" % i).encode()) + return got.hexdigest() == want.hexdigest() + + +def json_ok(): + doc = {"items": [{"k": i, "v": "tok-%d" % i} for i in range(64)]} + back = json.loads(json.dumps(doc)) + return len(back["items"]) == 64 and back["items"][63]["v"] == "tok-63" + + +def main(): + if not db_ok(): + print("sqlite insert/aggregate check failed", file=sys.stderr) + sys.exit(1) + if not fs_ok(): + print("filesystem write/read checksum mismatch", file=sys.stderr) + sys.exit(1) + if not json_ok(): + print("json round-trip mismatch", file=sys.stderr) + sys.exit(1) + + print("elfuse-oci-python-workload-ok") + + +if __name__ == "__main__": + main() From 4ae554edca9edcf3eb1250179e208368c4fd8f3d Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 22 Jul 2026 15:04:19 +0200 Subject: [PATCH 11/26] Add OCI guest execution checks The run smoke proves an image boots and computes; these checks cross the guest-execution seams it does not. A pathname AF_UNIX socket is bound inside a guest-created directory with a getsockname round-trip: the runtime translates sun_path into the sysroot on the way in and must reverse-map it on the way out, and the sparsebundle clone's deep host path additionally forces the over-length shortening indirection. A cold-provision boot (blobs cloned into an ephemeral store with the cs/ bundles dropped, so no network) must report the unpack, and the following warm re-attach of the same digest must boot without unpacking again. A dynamically linked from-image binary runs under an explicit entrypoint so PT_INTERP and its shared-object closure must resolve inside the rootfs. The socket check and the workload's interpreter children carry explicit deadlines (socket timeouts, a bounded thread join, a per-child subprocess timeout), so an exec regression fails the lane promptly instead of holding the self-hosted runner to the job timeout. The python workload's verdicts compare exact values: the SQLite aggregate pins per-thread MIN/MAX/SUM, the JSON round-trip compares the whole parsed document, and the file fan-out compares contents in path order, so value corruption cannot pass as a matching count or an equal multiset. Wired as a runtime-macos Release-leg step beside the run smoke, sharing its warm store (python:3.12-slim joins alpine and debian there). --- .github/workflows/main.yml | 14 ++ scripts/ci/oci-exec-checks.sh | 81 ++++++++++ scripts/ci/workloads/python-workload.py | 206 +++++++++++++++++++----- 3 files changed, 259 insertions(+), 42 deletions(-) create mode 100755 scripts/ci/oci-exec-checks.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c95b5bb..f9d5d8a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -617,6 +617,20 @@ jobs: make build/elfuse-oci scripts/ci/oci-run-smoke.sh + - name: OCI execution checks (unix sockets, cold/warm boot, dynamic interp) + # Guest-execution seams the smoke above does not cross: a pathname + # AF_UNIX socket bound inside the guest with a getsockname + # round-trip, the cold-provision versus warm re-attach boot path, + # and an explicit dynamically linked from-image binary. Shares the + # smoke step's warm store (alpine/debian, plus python:3.12-slim). + # Release leg only. + if: ${{ matrix.run_matrix }} + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-store" + make build/elfuse-oci + scripts/ci/oci-exec-checks.sh + - name: OCI image lifecycle (pull -> inspect -> list -> run -> rmi -> prune) # Walks the whole user-facing image lifecycle on the only leg with # HVF. python:3.12-slim adds what the alpine smoke above does not: diff --git a/scripts/ci/oci-exec-checks.sh b/scripts/ci/oci-exec-checks.sh new file mode 100755 index 00000000..dae1d4e0 --- /dev/null +++ b/scripts/ci/oci-exec-checks.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Execution checks for the default `run` path beyond oci-run-smoke.sh: +# pathname AF_UNIX sockets inside the guest (bind, getsockname round-trip, +# connect, payload echo), the cold-provision versus warm re-attach boot +# seam, and explicit dynamic-interpreter resolution from the image. Needs +# macOS with Hypervisor.framework; network only when the store is cold. +# +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-exec-checks.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +: "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" + +# Pathname AF_UNIX socket bound inside a guest-created directory. The +# bound name must read back byte-identical through getsockname: under a +# sysroot the runtime translates sun_path on the way in and must +# reverse-map it on the way out, and the sparsebundle clone's deep host +# path forces the over-length shortening indirection as well. +sock_py=' +import socket, threading, os +os.makedirs("/srv-sock", exist_ok=True) +path = "/srv-sock/echo.sock" +srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +srv.settimeout(30) +srv.bind(path) +assert srv.getsockname() == path, srv.getsockname() +srv.listen(1) +def serve(): + conn, _ = srv.accept() + conn.settimeout(30) + conn.sendall(conn.recv(64)) + conn.close() +t = threading.Thread(target=serve) +t.start() +cli = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +cli.settimeout(30) +cli.connect(path) +cli.sendall(b"elfuse-unix-sock-ok") +print(cli.recv(64).decode()) +t.join(30) +assert not t.is_alive(), "echo thread hung" +' +out="$("$BIN" run --entrypoint /usr/local/bin/python3 python:3.12-slim \ + -c "$sock_py")" +printf 'unix socket check: %s\n' "$out" +printf '%s\n' "$out" | expect_grep elfuse-unix-sock-ok + +# Cold-provision versus warm re-attach. Clone the warm store's blobs into +# an ephemeral store but drop the cs/ bundles, so the first run must +# provision the sparsebundle and unpack (network-free cold boot) and the +# second must re-attach the warm base without unpacking again. +coldstore="$(mktemp -d)/store" +errf="$(mktemp)" +trap 'rm -rf "$(dirname "$coldstore")" "$errf"' EXIT +cp -Rc "$ELFUSE_OCI_STORE" "$coldstore" +rm -rf "$coldstore/cs" + +out="$(ELFUSE_OCI_STORE=$coldstore "$BIN" run alpine:3 /bin/echo cold-ok \ + 2>"$errf")" +printf '%s\n' "$out" | expect_grep cold-ok +grep -q 'Unpacking' "$errf" || fail "cold boot did not report an unpack" + +out="$(ELFUSE_OCI_STORE=$coldstore "$BIN" run alpine:3 /bin/echo warm-ok \ + 2>"$errf")" +printf '%s\n' "$out" | expect_grep warm-ok +if grep -q 'Unpacking' "$errf"; then + fail "warm re-attach unpacked again" +fi +echo "cold/warm boot check OK" + +# Dynamic-interpreter resolution: run a glibc dynamically linked binary +# from the image explicitly, so PT_INTERP and its .so closure must resolve +# inside the rootfs through path translation. +out="$("$BIN" run --entrypoint /bin/bash debian:stable-slim -c \ + 'echo elfuse-interp-ok')" +printf 'interp check: %s\n' "$out" +printf '%s\n' "$out" | expect_grep elfuse-interp-ok + +"$BIN" prune >/dev/null + +echo "exec checks OK" diff --git a/scripts/ci/workloads/python-workload.py b/scripts/ci/workloads/python-workload.py index 16d7fab5..04b3ac5c 100644 --- a/scripts/ci/workloads/python-workload.py +++ b/scripts/ci/workloads/python-workload.py @@ -1,74 +1,196 @@ #!/usr/bin/env python3 -"""Guest workload for the elfuse-oci python image CI. +"""Non-trivial guest workload for the elfuse-oci python image CI. -A deliberately single-threaded exercise of a dynamically-linked glibc python -guest: a small SQLite insert plus an aggregate query, a few dozen files written, -read back, and checksummed, and a JSON round-trip. On full success it prints a -single sentinel token the workflow asserts on; any failure exits non-zero with a -diagnostic. The concurrent/subprocess-heavy stress variant lives on the -oci/workload-stress branch. +Mirrors the profiled python workload: JSON/regex churn, a concurrent +SQLite writer set (fcntl locking, fsync, and WAL mmap where the guest FS +supports it), a few hundred small-file writes read back and checksummed, +an os.walk over the bundled standard library, and a batch of interpreter +subprocesses. On full success it prints a single sentinel token the workflow +asserts on; any failure exits non-zero with a diagnostic. -Self-contained (stdlib only) so it runs under the image's bundled Python with no -network or extra packages, and is passed to the guest via `python3 -c`. +Self-contained (stdlib only) so it runs under the image's bundled Python with +no network or extra packages, and is passed to the guest via `python3 -c`. """ -import hashlib import json import os +import re import sqlite3 +import subprocess import sys +import threading DB = "/tmp/elfuse-workload.db" TREE = "/tmp/elfuse-workload-tree" -ROWS = 1000 -FILES = 50 +THREADS = 8 +PER_THREAD = 2500 # 8 * 2500 = 20k inserts, matching the profiled workload +FANOUT = 20 # 20 * 20 = 400 small files, matching the profiled workload +SUBPROCS = 10 -def db_ok(): +def setup_db(): con = sqlite3.connect(DB) try: - con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER NOT NULL)") - for n in range(ROWS): - con.execute("INSERT INTO t (n) VALUES (?)", (n,)) + # WAL exercises the guest FS's shared-memory index (mmap) and is the + # more demanding path; if the FS cannot back it, SQLite reports a + # different mode and the concurrent-writer count check below still + # validates fcntl locking and durable commits under rollback journal. + con.execute("PRAGMA journal_mode=WAL") + con.execute( + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "tid INTEGER NOT NULL, n INTEGER NOT NULL)" + ) con.commit() + finally: + con.close() + + +def worker(tid): + con = sqlite3.connect(DB, timeout=60) + try: + con.execute("PRAGMA busy_timeout=60000") + for n in range(PER_THREAD): + con.execute("INSERT INTO t (tid, n) VALUES (?, ?)", (tid, n)) + con.commit() + finally: + con.close() + + +def db_count(): + con = sqlite3.connect(DB, timeout=60) + try: (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() - (total,) = con.execute("SELECT SUM(n) FROM t").fetchone() + return count + finally: + con.close() + + +def db_query_ok(): + # A GROUP BY aggregate over the 20k rows. Every thread inserts exactly + # range(PER_THREAD), so the aggregates have exact expected values; + # counting rows alone would accept duplicated or corrupted n values. + con = sqlite3.connect(DB, timeout=60) + try: + rows = con.execute( + "SELECT tid, COUNT(*), MIN(n), MAX(n), SUM(n) FROM t" + " GROUP BY tid ORDER BY tid" + ).fetchall() finally: con.close() - return count == ROWS and total == sum(range(ROWS)) + want_sum = PER_THREAD * (PER_THREAD - 1) // 2 + return rows == [ + (tid, PER_THREAD, 0, PER_THREAD - 1, want_sum) for tid in range(THREADS) + ] -def fs_ok(): - os.makedirs(TREE, exist_ok=True) - for i in range(FILES): - with open(os.path.join(TREE, "f%d" % i), "w") as fh: - fh.write("elfuse-%d\n" % i) - fh.flush() - os.fsync(fh.fileno()) - got = hashlib.sha256() - want = hashlib.sha256() - for i in range(FILES): - with open(os.path.join(TREE, "f%d" % i)) as fh: - got.update(fh.read().encode()) - want.update(("elfuse-%d\n" % i).encode()) - return got.hexdigest() == want.hexdigest() +def json_regex_churn(): + # Serialize and reparse a structured document many times, then pull every + # embedded token back out with a regex and confirm the round-trip is exact. + word = re.compile(r"tok-(\d+)") + for r in range(2000): + doc = {"round": r, "items": [{"k": i, "v": "tok-%d" % i} for i in range(16)]} + blob = json.dumps(doc) + back = json.loads(blob) + got = [int(m) for m in word.findall(blob)] + # doc holds only str keys and int/str values, so the parsed value + # must equal it exactly; checking one field would let dropped or + # reordered items pass. + if got != list(range(16)) or back != doc: + return False + return True -def json_ok(): - doc = {"items": [{"k": i, "v": "tok-%d" % i} for i in range(64)]} - back = json.loads(json.dumps(doc)) - return len(back["items"]) == 64 and back["items"][63]["v"] == "tok-63" +def content(i, j): + return "elfuse-workload-%d-%d\n" % (i, j) + + +def fs_fanout_ok(): + for i in range(FANOUT): + for j in range(FANOUT): + d = os.path.join(TREE, str(i), str(j)) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "f"), "w") as fh: + fh.write(content(i, j)) + fh.flush() + os.fsync(fh.fileno()) + read_back = [] + for i in range(FANOUT): + for j in range(FANOUT): + with open(os.path.join(TREE, str(i), str(j), "f")) as fh: + read_back.append(fh.read()) + expected = [content(i, j) for i in range(FANOUT) for j in range(FANOUT)] + # Both lists are built in the same (i, j) path order, so plain equality + # is the whole check; hashing sorted copies would let two files with + # swapped contents pass as an equal multiset. + return read_back == expected + + +def walk_stdlib_ok(): + # os.walk the bundled standard library and count .py modules. This exercises + # getdents/newfstatat over a deep real tree; the exact count varies by + # patch release, so only assert it is unmistakably a full stdlib. + root = os.path.dirname(os.__file__) + n = 0 + for _, _, files in os.walk(root): + n += sum(1 for f in files if f.endswith(".py")) + return n > 200 + + +def subprocesses_ok(): + # Fork/exec the interpreter repeatedly; each child echoes a token this + # parent verifies, exercising execve of a dynamically-linked glibc binary. + for i in range(SUBPROCS): + code = "print('child-%d')" % i + # timeout bounds a hung child the same way the sqlite calls bound + # theirs; TimeoutExpired propagates as a prompt non-zero failure + # naming the child instead of idling to the job's timeout-minutes. + out = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=True, + timeout=60, + ).stdout.strip() + if out != "child-%d" % i: + return False + return True def main(): - if not db_ok(): - print("sqlite insert/aggregate check failed", file=sys.stderr) + setup_db() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + count = db_count() + if count != THREADS * PER_THREAD: + print( + "sqlite row count %d != %d (concurrent writers lost rows)" + % (count, THREADS * PER_THREAD), + file=sys.stderr, + ) sys.exit(1) - if not fs_ok(): - print("filesystem write/read checksum mismatch", file=sys.stderr) + + if not db_query_ok(): + print("sqlite per-thread aggregate mismatch", file=sys.stderr) sys.exit(1) - if not json_ok(): - print("json round-trip mismatch", file=sys.stderr) + + if not json_regex_churn(): + print("json/regex round-trip mismatch", file=sys.stderr) + sys.exit(1) + + if not fs_fanout_ok(): + print("filesystem fan-out checksum mismatch", file=sys.stderr) + sys.exit(1) + + if not walk_stdlib_ok(): + print("stdlib walk found too few modules", file=sys.stderr) + sys.exit(1) + + if not subprocesses_ok(): + print("subprocess output mismatch", file=sys.stderr) sys.exit(1) print("elfuse-oci-python-workload-ok") From a9c81deb8db288ea54c5c167d1a98f6943583887 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sat, 1 Aug 2026 23:12:29 +0800 Subject: [PATCH 12/26] Add Linux-compatible proc smaps emulation Generate Linux-shaped /proc/self/smaps and /proc//smaps snapshots from tracked guest VMAs so Redis can complete its ARM64 COW safety check under elfuse. Add growable VMA/string infrastructure and fork-aware accounting with parser regression coverage. --- Makefile | 19 + docs/internals.md | 70 ++++ mk/config.mk | 2 + mk/tests.mk | 28 +- src/core/guest.c | 4 +- src/dynamic-array.c | 317 ++++++++++++++++ src/dynamic-array.h | 238 ++++++++++++ src/runtime/procemu.c | 459 ++++++++++++++++------- src/runtime/procemu.h | 2 +- src/string-builder.c | 283 +++++++++++++++ src/string-builder.h | 62 ++++ tests/test-dynamic-array-host.c | 109 ++++++ tests/test-matrix.sh | 5 +- tests/test-proc-smap.c | 599 +++++++++++++++++++++++++++++++ tests/test-string-builder-host.c | 182 ++++++++++ tests/test-util.h | 64 ++++ 16 files changed, 2305 insertions(+), 138 deletions(-) create mode 100644 src/dynamic-array.c create mode 100644 src/dynamic-array.h create mode 100644 src/string-builder.c create mode 100644 src/string-builder.h create mode 100644 tests/test-dynamic-array-host.c create mode 100644 tests/test-proc-smap.c create mode 100644 tests/test-string-builder-host.c diff --git a/Makefile b/Makefile index b037e7d6..299762d6 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ include mk/config.mk # Source files. SRCS := \ main.c \ + dynamic-array.c \ + string-builder.c \ core/guest.c \ core/elf.c \ core/stack.c \ @@ -285,6 +287,23 @@ $(BUILD_DIR)/test-absock-names-host: $(BUILD_DIR)/test-absock-names-host.o \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ +## Build the string builder host unit test (native macOS binary) +# This is a pure-C unit test; link the string builder and generic container +# implementations directly and skip the Hypervisor framework and codesign. +$(BUILD_DIR)/test-string-builder-host: \ + $(BUILD_DIR)/test-string-builder-host.o \ + $(BUILD_DIR)/string-builder.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the generic dynamic-array host unit test (native macOS binary) +$(BUILD_DIR)/test-dynamic-array-host: \ + $(BUILD_DIR)/test-dynamic-array-host.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + # Guest test binaries (cross-compiled, aarch64-linux) # Only used when GUEST_TEST_BINARIES is not set. diff --git a/docs/internals.md b/docs/internals.md index 5927518a..abb49fd2 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -115,6 +115,32 @@ Key files: | `src/runtime/proctitle.c` | argv / comm rewriting for `prctl PR_SET_NAME` | | `src/debug/gdbstub.c`, `gdbstub-rsp.c`, `gdbstub-reg.c` | GDB RSP stub | +## Generic Dynamic Containers + +`src/dynamic-array.h` and `src/dynamic-array.c` provide the raw +`dynamic_array_t` used by the procfs VMA snapshot and the string builder. Capacity +is measured in element slots, while `count` is the number of logical elements. +The allocation is one contiguous block of `capacity * element_size` bytes; +both the count addition and the multiplication are checked before a growth. +Arithmetic overflow reports `EOVERFLOW`, invalid arguments report `EINVAL`, +and an allocation failure reports `ENOMEM`. Growth is transactional: on any +failure the old pointer, count, and capacity remain valid. + +The generated typed facades own only their array storage. Elements are copied +as trivially-copyable bytes, so the container does not call destructors and +does not manage pointers or other resources held by an element. `destroy` +frees the contiguous block and restores the zero state. A facade can therefore +be declared as `{0}` and initialized lazily on its first operation. + +`string_builder_t` is a thin facade over a generated `char` container. The +container count is the C-string length and excludes the terminator; its +capacity accessor reports bytes including the trailing NUL slot. Reserve and +append account for that slot, and every successful mutation restores +`data[count] == '\0'`. `string_builder_append` accepts a C string and uses its +first NUL as the end of the input. Formatted appends retain the existing +two-pass `vsnprintf` behavior and commit only the prefix through the first NUL, +matching standard C string semantics. + ## Hypervisor.framework Constraints Apple HVF imposes a handful of constraints that shape the rest of the design: @@ -769,6 +795,50 @@ under `/proc`, `/dev`, and a few Linux-expected compatibility files: - Guest cwd handling preserves a virtual `/proc` working directory even though the host operates on synthetic backing directories. +`/proc/self/smaps` and `/proc//smaps` are generated from the same tracked +VMA list as `/proc/self/maps`. The complete field set currently emitted for +each VMA is the maps header followed by these 25 fields, in this order: + +```text +Size, KernelPageSize, MMUPageSize, Rss, Pss, Pss_Dirty, +Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty, Referenced, +Anonymous, KSM, LazyFree, AnonHugePages, ShmemPmdMapped, FilePmdMapped, +Shared_Hugetlb, Private_Hugetlb, Swap, SwapPss, Locked, THPeligible, +ProtectionKey, VmFlags +``` + +`Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are +reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a +non-zero value: in a fork child, writable private anonymous VMAs report their +full VMA size because they are logically shared with the parent's CoW +snapshot. Every other numeric counter is emitted as zero, including +`THPeligible` and `ProtectionKey`. `VmFlags` is evidence-based and contains +only the permission/sharing flags represented by the tracked VMA (`rd`, `wr`, +`ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. +elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing +against a real Linux kernel should tolerate either conditional field being +absent. +These are coarse VMA-level values, not host page-residency, dirty-bit, or +proportional-sharing accounting, so they are suitable for fork-safety checks +but not precise memory profiling. `smaps_rollup` is not implemented. + +Synthetic proc directories have explicit snapshot boundaries. The backing +trees reached by opening `/proc` or `/proc/self` are materialized once, on the +first access, and their initial `stat`, `status`, `cmdline`, `maps`, and `smaps` +files remain fixed for the process lifetime. An absolute open of one of those +proc paths is intercepted directly and generates fresh content; opening the +same name through an already-open synthetic directory reads that directory's +snapshot. `/proc/self/fd` and `/proc/self/fdinfo` are rebuilt into an +independent scratch directory on every open, so each directory fd sees the +guest-fd table as it existed at that open and concurrent enumerations cannot +mutate one another. `/proc/self/task` is repopulated from the current thread +set whenever the directory is opened. These boundaries are intentional: a +directory stream is stable while it is being read, while dynamic task and fd +listings refresh only when a new directory is opened. The synthetic +`/sys/devices/system/cpu` tree follows the one-shot rule: its CPU count, +cpumask files, and `cpuN` directories are captured on first access and then +remain fixed. + Related implementation: `src/runtime/procemu.c`, `src/syscall/path.c`, `src/syscall/fs.c`, `src/syscall/proc-state.c`. diff --git a/mk/config.mk b/mk/config.mk index af112b76..6f32115e 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -25,6 +25,8 @@ NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c \ tests/test-casefold-host.c \ tests/test-casefold-walk-host.c \ tests/test-absock-names-host.c \ + tests/test-dynamic-array-host.c \ + tests/test-string-builder-host.c \ tests/probe-volume-naming.c SPECIAL_TEST_SRCS := tests/test-lowbase-mem.c SPECIAL_TEST_BINS := $(BUILD_DIR)/test-lowbase-mem-200000 $(BUILD_DIR)/test-lowbase-mem-300000 diff --git a/mk/tests.mk b/mk/tests.mk index 9be5f593..eff296dc 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -14,6 +14,8 @@ test-sysroot-tmp-remove test-sysroot-host-fallback test-sysroot-case-exact \ test-sysroot-create-paths test-fork-ipc-protocol-host \ test-vcpu-run-hooks-host test-identity-override-host \ + test-dynamic-array-host \ + test-string-builder-host \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-launch-flags \ test-fuse-alpine \ @@ -100,7 +102,9 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ $(BUILD_DIR)/test-teardown-live-vcpu-host \ $(BUILD_DIR)/test-casefold-host \ $(BUILD_DIR)/test-casefold-walk-host \ - $(BUILD_DIR)/test-absock-names-host + $(BUILD_DIR)/test-absock-names-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v -s '$(SANITIZER_SECTIONS)' @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -118,6 +122,10 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ @$(BUILD_DIR)/test-casefold-walk-host @printf "\n$(BLUE)━━━ absock derived-name unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-absock-names-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host @printf "\n$(BLUE)━━━ one on-disk name per guest name ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-sysroot-name-unique @printf "\n$(BLUE)━━━ relative and dirfd-relative names ━━━$(RESET)\n" @@ -138,7 +146,9 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ $(BUILD_DIR)/test-teardown-live-vcpu-host \ $(BUILD_DIR)/test-casefold-host \ $(BUILD_DIR)/test-casefold-walk-host \ - $(BUILD_DIR)/test-absock-names-host + $(BUILD_DIR)/test-absock-names-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -156,6 +166,10 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @$(BUILD_DIR)/test-casefold-walk-host @printf "\n$(BLUE)━━━ absock derived-name unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-absock-names-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host @printf "\n$(BLUE)━━━ one on-disk name per guest name ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-sysroot-name-unique @printf "\n$(BLUE)━━━ relative and dirfd-relative names ━━━$(RESET)\n" @@ -1464,6 +1478,16 @@ test-fork-ipc-protocol-host: $(BUILD_DIR)/test-fork-ipc-protocol-host test-vcpu-run-hooks-host: $(BUILD_DIR)/test-vcpu-run-hooks-host $(BUILD_DIR)/test-vcpu-run-hooks-host +# String builder unit test +## Run the growable string builder host unit test +test-string-builder-host: $(BUILD_DIR)/test-string-builder-host + $(BUILD_DIR)/test-string-builder-host + +# Generic dynamic array unit test +## Run the raw/typed dynamic array host unit test +test-dynamic-array-host: $(BUILD_DIR)/test-dynamic-array-host + $(BUILD_DIR)/test-dynamic-array-host + # Proctitle argv-tail regression ## Run the deterministic argv-tail overshoot guard test test-proctitle-host: $(BUILD_DIR)/test-proctitle-host diff --git a/src/core/guest.c b/src/core/guest.c index 2bf395e0..addc79f4 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -2052,8 +2052,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->flags = flags; r->offset = offset; r->backing_fd = owned_backing_fd; - r->shared = (flags & 0x01) != 0; /* LINUX_MAP_SHARED = 0x01 */ - r->noreserve = (flags & 0x4000) != 0; /* LINUX_MAP_NORESERVE = 0x4000 */ + r->shared = (flags & LINUX_MAP_SHARED) != 0; + r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; guest_region_clear_overlay(r); if (name) { diff --git a/src/dynamic-array.c b/src/dynamic-array.c new file mode 100644 index 00000000..a4dd63c8 --- /dev/null +++ b/src/dynamic-array.c @@ -0,0 +1,317 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "dynamic-array.h" + +#include +#include +#include +#include + +#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t)8) + +/* Set errno for a bad argument and return a standard failure code. */ +static int dynamic_array_invalid(void) +{ + errno = EINVAL; + return -1; +} + +/* Check that the array handle is non-null and has a non-zero element size. */ +static int dynamic_array_validate(const dynamic_array_t *array) +{ + if (array == NULL || array->element_size == 0) + return dynamic_array_invalid(); + return 0; +} + +/* Return count + extra after guarding against size_t overflow. */ +static int dynamic_array_count_plus(const dynamic_array_t *array, + size_t extra, size_t *total) +{ + if (extra > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + *total = array->count + extra; + return 0; +} + +/* Compute the byte size for a number of elements with overflow protection. */ +static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, + size_t *bytes) +{ + if (count != 0 && array->element_size > SIZE_MAX / count) { + errno = EOVERFLOW; + return -1; + } + *bytes = count * array->element_size; + return 0; +} + +/* Return the offset of an in-array source span, including capacity bytes. */ +static int dynamic_array_source_offset(const dynamic_array_t *array, + const void *source, size_t bytes, + size_t *offset) +{ + if (array->data == NULL || source == NULL) + return 0; + + uintptr_t base = (uintptr_t) array->data; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t)SIZE_MAX) + return 0; + size_t start = (size_t) delta; + size_t allocation_bytes; + if (dynamic_array_bytes(array, array->capacity, &allocation_bytes) < 0) + return 0; + if (start > allocation_bytes || bytes > allocation_bytes - start) + return 0; + *offset = start; + return 1; +} + +/* Initialize only metadata; allocate storage separately when requested. */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + /* There is no allocation to fail here. Preserve storage only when the + * caller is reinitializing an array with the same element type. */ + if (array->data != NULL && array->element_size != element_size) { + errno = EINVAL; + return -1; + } + if (array->data == NULL) { + array->count = 0; + array->capacity = 0; + } + array->element_size = element_size; + return 0; +} + +/* Initialize and reserve storage for an initial number of elements. */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + size_t bytes; + if (initial_capacity != 0 && element_size > SIZE_MAX / initial_capacity) { + errno = EOVERFLOW; + return -1; + } + bytes = initial_capacity * element_size; + void *storage = NULL; + if (bytes != 0) { + storage = malloc(bytes); + if (storage == NULL) { + errno = ENOMEM; + return -1; + } + } + + free(array->data); + array->data = storage; + array->count = 0; + array->capacity = initial_capacity; + array->element_size = element_size; + return 0; +} + +/* Release backing storage and reset the array object to zero state. */ +void dynamic_array_destroy(dynamic_array_t *array) +{ + if (array == NULL) + return; + free(array->data); + *array = (dynamic_array_t){0}; +} + +/* Ensure the array has capacity for at least extra additional elements. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra) +{ + if (dynamic_array_validate(array) < 0) + return -1; + + size_t needed; + if (dynamic_array_count_plus(array, extra, &needed) < 0) + return -1; + if (needed <= array->capacity) + return 0; + + size_t new_capacity = array->capacity; + if (new_capacity == 0) + new_capacity = DYNAMIC_ARRAY_INITIAL_CAPACITY; + while (new_capacity < needed) { + if (new_capacity > SIZE_MAX / 2) { + new_capacity = needed; + break; + } + new_capacity *= 2; + } + + size_t bytes; + if (dynamic_array_bytes(array, new_capacity, &bytes) < 0) + return -1; + void *grown = realloc(array->data, bytes); + if (grown == NULL && bytes != 0) { + errno = ENOMEM; + return -1; + } + array->data = grown; + array->capacity = new_capacity; + return 0; +} + +/* Resize logical length; zero-initialize any newly visible elements. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count > array->capacity) { + size_t extra = count - array->count; + if (dynamic_array_reserve(array, extra) < 0) + return -1; + } + if (count > array->count) { + size_t old_bytes, new_bytes; + (void) dynamic_array_bytes(array, array->count, &old_bytes); + (void) dynamic_array_bytes(array, count, &new_bytes); + memset((unsigned char *) array->data + old_bytes, 0, + new_bytes - old_bytes); + } + array->count = count; + return 0; +} + +/* Append multiple elements from source memory to the end of the array. */ +int dynamic_array_append_n(dynamic_array_t *array, const void *data, + size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (dynamic_array_count_plus(array, count, &total) < 0) + return -1; + size_t bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0) + return -1; + size_t offset = 0; + int aliases = dynamic_array_source_offset(array, data, bytes, &offset); + + if (dynamic_array_reserve(array, count) < 0) + return -1; + if (aliases) + data = (const unsigned char *) array->data + offset; + size_t old_bytes; + (void) dynamic_array_bytes(array, array->count, &old_bytes); + memmove((unsigned char *) array->data + old_bytes, data, bytes); + array->count = total; + return 0; +} + +/* Insert multiple elements at index while preserving existing elements. */ +int dynamic_array_insert_n(dynamic_array_t *array, size_t index, + const void *data, size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (index > array->count) + return dynamic_array_invalid(); + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (count > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + total = array->count + count; + size_t bytes, index_bytes, tail_bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0 || + dynamic_array_bytes(array, index, &index_bytes) < 0 || + dynamic_array_bytes(array, array->count - index, &tail_bytes) < 0) + return -1; + + size_t source_offset = 0; + int aliases = dynamic_array_source_offset(array, data, bytes, + &source_offset); + void *temporary = NULL; + if (aliases) { + temporary = malloc(bytes); + if (temporary == NULL) { + errno = ENOMEM; + return -1; + } + memcpy(temporary, (const unsigned char *) array->data + source_offset, + bytes); + data = temporary; + } + + if (dynamic_array_reserve(array, count) < 0) { + free(temporary); + return -1; + } + unsigned char *base = array->data; + memmove(base + index_bytes + bytes, base + index_bytes, tail_bytes); + memcpy(base + index_bytes, data, bytes); + array->count = total; + free(temporary); + return 0; +} + +/* Append a single element by forwarding to append_n. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data) +{ + return dynamic_array_append_n(array, data, 1); +} + +/* Insert a single element by forwarding to insert_n. */ +int dynamic_array_insert_one(dynamic_array_t *array, size_t index, + const void *data) +{ + return dynamic_array_insert_n(array, index, data, 1); +} + +/* Return a mutable pointer to the index-th element, or NULL when invalid. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index) +{ + if (dynamic_array_validate(array) < 0 || index >= array->count) { + if (array != NULL && array->element_size != 0) + errno = EINVAL; + return NULL; + } + return (unsigned char *) array->data + index * array->element_size; +} + +/* Return a read-only pointer to the index-th element, or NULL when invalid. */ +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index) +{ + if (array == NULL || array->element_size == 0) { + errno = EINVAL; + return NULL; + } + if (index >= array->count) { + errno = EINVAL; + return NULL; + } + return (const unsigned char *) array->data + index * array->element_size; +} diff --git a/src/dynamic-array.h b/src/dynamic-array.h new file mode 100644 index 00000000..7e95c592 --- /dev/null +++ b/src/dynamic-array.h @@ -0,0 +1,238 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +typedef struct dynamic_array { + void *data; + size_t count; + size_t capacity; + size_t element_size; +} dynamic_array_t; + +#if defined(__GNUC__) || defined(__clang__) +#define DYNAMIC_ARRAY_INLINE static inline __attribute__((unused)) +#else +#define DYNAMIC_ARRAY_INLINE static inline +#endif + +/* Reset type metadata on a failed first-touch typed operation. */ +static inline int dynamic_array_typed_result(dynamic_array_t *array, + int was_uninitialized, + int result) +{ + if (result < 0 && was_uninitialized && array != NULL && + array->data == NULL && array->count == 0 && array->capacity == 0) + array->element_size = 0; + return result; +} + +/* Set the element size on the first typed operation and report that change. */ +static inline int dynamic_array_typed_prepare(dynamic_array_t *array, + size_t element_size) +{ + int was_uninitialized = array != NULL && array->element_size == 0; + if (was_uninitialized) + array->element_size = element_size; + return was_uninitialized; +} + +/* Initialize an array for elements of element_size bytes without allocation. */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size); + +/* Initialize and reserve initial_capacity element slots. */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity); + +/* Release storage and restore the all-zero state. */ +void dynamic_array_destroy(dynamic_array_t *array); + +/* Ensure room for extra elements beyond the current count. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra); + +/* Set the logical element count. Newly exposed elements are zeroed. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count); + +/* Append one element, or count elements when the three-argument form is + * used. The macro keeps both forms available to callers. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data); +int dynamic_array_append_n(dynamic_array_t *array, const void *data, + size_t count); +#define DYNAMIC_ARRAY_APPEND_PICK(_1, _2, _3, NAME, ...) NAME +#define dynamic_array_append(...) \ + DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ + dynamic_array_append_one, dynamic_array_append_dummy) \ + (__VA_ARGS__) + +/* Insert one element, or count elements in the four-argument form. */ +int dynamic_array_insert_one(dynamic_array_t *array, size_t index, + const void *data); +int dynamic_array_insert_n(dynamic_array_t *array, size_t index, + const void *data, size_t count); +#define DYNAMIC_ARRAY_INSERT_PICK(_1, _2, _3, _4, NAME, ...) NAME +#define dynamic_array_insert(...) \ + DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ + dynamic_array_insert_one, dynamic_array_insert_dummy) \ + (__VA_ARGS__) + +/* Return an element pointer, or NULL with errno=EINVAL for a bad index. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index); +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); + +/* Generate a small type-safe facade over the raw container. The facade owns + * no additional state; all growth and copying remains in dynamic-array.c. + */ +#define DYNAMIC_ARRAY_DEFINE(name, type) \ + typedef struct name { \ + dynamic_array_t raw; \ + } name##_t; \ + \ + /* Initialize the typed array with no preallocated storage. */ \ + DYNAMIC_ARRAY_INLINE int name##_init(name##_t *array) \ + { \ + return dynamic_array_init(array != NULL ? &array->raw : NULL, \ + sizeof(type)); \ + } \ + /* Initialize typed array and preallocate initial slots. */ \ + DYNAMIC_ARRAY_INLINE int name##_init_with_capacity(name##_t *array, \ + size_t initial_capacity) \ + { \ + return dynamic_array_init_with_capacity( \ + array != NULL ? &array->raw : NULL, sizeof(type), \ + initial_capacity); \ + } \ + /* Destroy the typed array and release backing storage. */ \ + DYNAMIC_ARRAY_INLINE void name##_destroy(name##_t *array) \ + { \ + if (array != NULL) \ + dynamic_array_destroy(&array->raw); \ + } \ + /* Reserve extra slots in the typed array. */ \ + DYNAMIC_ARRAY_INLINE int name##_reserve(name##_t *array, size_t extra) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_reserve(array != NULL ? &array->raw : NULL, extra)); \ + } \ + /* Resize typed array, zero-filling newly visible elements. */ \ + DYNAMIC_ARRAY_INLINE int name##_resize(name##_t *array, size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_resize(array != NULL ? &array->raw : NULL, count)); \ + } \ + /* Append one value through a typed pointer. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_ptr(name##_t *array, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, value, \ + 1)); \ + } \ + /* Append one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_value(name##_t *array, type value) \ + { \ + return name##_append_ptr(array, &value); \ + } \ + /* Accept either a value or a pointer to a value while retaining compile- \ + * time checking of the element type. */ \ + DYNAMIC_ARRAY_INLINE int name##_append(name##_t *array, const type *value) \ + { \ + return name##_append_ptr(array, value); \ + } \ + /* Append a typed range of values. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_n(name##_t *array, \ + const type *values, \ + size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, values, \ + count)); \ + } \ + /* Insert one typed value through a pointer form. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, \ + size_t index, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + value, 1)); \ + } \ + /* Insert one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_value(name##_t *array, \ + size_t index, \ + type value) \ + { \ + return name##_insert_ptr(array, index, &value); \ + } \ + /* Insert one typed value from a pointer argument. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert(name##_t *array, size_t index, \ + const type *value) \ + { \ + return name##_insert_ptr(array, index, value); \ + } \ + /* Insert a typed range of values at index. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, \ + size_t index, const type *values, \ + size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + values, count)); \ + } \ + /* Return a typed pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE type *name##_at(name##_t *array, size_t index) \ + { \ + return (type *) dynamic_array_at(array != NULL ? &array->raw : NULL, \ + index); \ + } \ + /* Return a typed const pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_at_const(const name##_t *array, \ + size_t index) \ + { \ + return (const type *) dynamic_array_at_const( \ + array != NULL ? &array->raw : NULL, index); \ + } \ + /* Access the underlying typed data pointer. */ \ + DYNAMIC_ARRAY_INLINE type *name##_data(name##_t *array) \ + { \ + return array != NULL ? (type *) array->raw.data : NULL; \ + } \ + /* Access the underlying typed const data pointer. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_data_const(const name##_t *array) \ + { \ + return array != NULL ? (const type *) array->raw.data : NULL; \ + } \ + /* Query current number of elements in the typed array. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_count(const name##_t *array) \ + { \ + return array != NULL ? array->raw.count : 0; \ + } \ + /* Query current allocated capacity. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_capacity(const name##_t *array) \ + { \ + return array != NULL ? array->raw.capacity : 0; \ + } diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index e74801f9..2cb59be1 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -10,10 +10,16 @@ * (caller falls through to real syscall). */ -/* Maximum /proc/self/maps entries. Array is sized to this; loop bounds use - * MAPS_ENTRY_MAX - 1 to leave room for safe increment. +/* Initial capacity for the transient /proc/self/maps and /proc/self/smaps VMA + * snapshot. The region tracker documents that coalescing leaves typical + * workloads at roughly 50 tracked regions (see core/guest.h), so 64 avoids an + * immediate growth in that case. The array grows as needed while mmap_lock is + * held; keep a hard ceiling tied to the guest's region tables so maps/smaps + * can enumerate every tracked VMA while keeping transient snapshot memory + * bounded. */ -#define MAPS_ENTRY_MAX 256 +#define MAPS_ENTRY_INITIAL_CAP 64 +#define MAPS_ENTRY_MAX (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED) /* Bound the transient host-PID snapshot used by /proc/net enumeration. This * is an output-work limit, not the dynamically growing lifecycle-table cap. @@ -51,6 +57,7 @@ #include #include +#include "string-builder.h" #include "utils.h" #include "debug/log.h" @@ -98,55 +105,70 @@ typedef struct { char name[64]; } maps_entry_t; -static void maps_entry_insert(maps_entry_t *entries, - int *nentries, - uint64_t start, - uint64_t end, - int prot, - int flags, - uint64_t offset, - const char *name) +/* A growable VMA snapshot. The generated facade keeps element-size and + * allocation bookkeeping in the generic array implementation. */ +DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) + +static int maps_entries_insert_sorted(maps_entries_t *entries, + uint64_t start, + uint64_t end, + int prot, + int flags, + uint64_t offset, + const char *name) { - if (*nentries >= MAPS_ENTRY_MAX || end <= start) - return; - - int i = *nentries; - while (i > 0 && entries[i - 1].start > start) { - entries[i] = entries[i - 1]; - i--; + if (end <= start) + return 0; + if (maps_entries_count(entries) >= MAPS_ENTRY_MAX) { + errno = ENOMEM; + return -1; } + if (maps_entries_count(entries) == 0 && + maps_entries_reserve(entries, MAPS_ENTRY_INITIAL_CAP) < 0) + return -1; - maps_entry_t *e = &entries[i]; - e->start = start; - e->end = end; - e->prot = prot; - e->flags = flags; - e->offset = offset; + maps_entry_t value = { + .start = start, + .end = end, + .prot = prot, + .flags = flags, + .offset = offset, + }; if (name && name[0]) - str_copy_trunc(e->name, name, sizeof(e->name)); + str_copy_trunc(value.name, name, sizeof(value.name)); else - e->name[0] = '\0'; - (*nentries)++; + value.name[0] = '\0'; + + size_t i = maps_entries_count(entries); + while (i > 0 && maps_entries_at(entries, i - 1)->start > start) { + i--; + } + return maps_entries_insert(entries, i, &value); } -static void maps_entries_merge_adjacent(maps_entry_t *entries, int *nentries) +static void maps_entries_merge_adjacent(maps_entries_t *entries) { - if (*nentries <= 1) + size_t count = maps_entries_count(entries); + if (count <= 1) return; - int out = 0; - for (int i = 1; i < *nentries; i++) { - if (entries[i].start == entries[out].end && - entries[i].prot == entries[out].prot && - entries[i].flags == entries[out].flags && - entries[i].offset == entries[out].offset && - strcmp(entries[i].name, entries[out].name) == 0) { - entries[out].end = entries[i].end; + size_t out = 0; + for (size_t i = 1; i < count; i++) { + maps_entry_t *current = maps_entries_at(entries, i); + maps_entry_t *previous = maps_entries_at(entries, out); + if (current->start == previous->end && + current->prot == previous->prot && + current->flags == previous->flags && + current->offset == previous->offset && + strcmp(current->name, previous->name) == 0) { + previous->end = current->end; continue; } - entries[++out] = entries[i]; + ++out; + if (out != i) + *maps_entries_at(entries, out) = *current; } - *nentries = out + 1; + (void)maps_entries_resize(entries, out + 1); } /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily @@ -446,7 +468,8 @@ static void proc_tmpdir_cleanup(void) /* Remove known files inside // and / */ char path[256]; - const char *files[] = {"stat", "status", "cmdline", "maps", "exe", NULL}; + const char *files[] = {"stat", "status", "cmdline", "maps", "smaps", + "exe", NULL}; char piddir[160]; /* Reconstruct pid subdir by scanning for the first numeric entry */ @@ -1231,6 +1254,7 @@ static const char *ensure_proc_tmpdir(const guest_t *g) populate_proc_snapshot(g, piddir, "status", "/proc/self/status"); populate_proc_snapshot(g, piddir, "cmdline", "/proc/self/cmdline"); populate_proc_snapshot(g, piddir, "maps", "/proc/self/maps"); + populate_proc_snapshot(g, piddir, "smaps", "/proc/self/smaps"); /* Create task subdirectory for /proc/self/task enumeration */ char taskdir[128]; @@ -2376,58 +2400,54 @@ static int pty_open_master(int linux_flags) return master; } -/* Emit /proc/self/maps into a synthetic fd. Merges contiguous regions[] runs - * that came from one mmap, then folds in preannounced[] shadow entries whose - * advertised interval is not yet fully covered by live regions. +/* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges + * contiguous regions[] runs that came from one mmap, then folds in + * preannounced[] shadow entries whose advertised interval is not yet fully + * covered by live regions. * - * Returns a host fd, or -1 on error. Split out of proc_intercept_open to keep - * that dispatcher readable. + * The region and preannounced arrays are mutable from guest mmap/mprotect/ + * munmap operations. Snapshot and merge while mmap_lock is held, then release + * it before formatting output so readers never observe a torn VMA or metadata + * record and output generation does not block page-table mutations. */ -static int proc_open_self_maps(const guest_t *g) +static int proc_build_maps_entries(const guest_t *g, + maps_entries_t *entries_out) { - /* Heap-allocated: guest threads run on host pthreads whose stacks do not - * comfortably hold a 16KiB frame. - */ - const size_t bufsz = 16384; - char *buf = malloc(bufsz); - if (!buf) - return -1; - int off = 0; - - /* Build a flat array of (va_start, va_end, prot, flags, offset, name) from - * regions[] plus /proc/self/maps-only preannounced[] entries. - * preannounced[] is intentionally NOT consulted by mmap conflict detection, - * so advertise-only Rosetta/JIT regions do not trip MAP_FIXED_NOREPLACE - * with -EEXIST. - * - * entries is heap-allocated: MAPS_ENTRY_MAX * sizeof(maps_entry_t) is - * ~24KiB, too large for a guest-thread pthread stack. - */ - maps_entry_t *entries = calloc(MAPS_ENTRY_MAX, sizeof(*entries)); - if (!entries) { - free(buf); + if (!g || !entries_out) { + errno = EINVAL; return -1; } - int nentries = 0; + + maps_entries_t entries = {0}; + int result = -1; + + pthread_mutex_lock(&mmap_lock); /* Convert regions[] to maps entries. regions[] is already sorted by start - * address; merge contiguous runs that came from one mmap. + * address. The MAP_SHARED/MAP_ANONYMOUS/MAP_NORESERVE bits are preserved + * in r->flags, which is the single source of truth for the proc snapshot. */ - for (int i = 0; i < g->nregions && nentries < MAPS_ENTRY_MAX; i++) { + int nregions = g->nregions; + if (nregions < 0) + nregions = 0; + if (nregions > GUEST_MAX_REGIONS) + nregions = GUEST_MAX_REGIONS; + for (int i = 0; i < nregions; i++) { const guest_region_t *r = &g->regions[i]; uint64_t start = r->start & ~0xFFFULL; - uint64_t end = (r->end + 0xFFF) & ~0xFFFULL; - - if (nentries > 0 && entries[nentries - 1].end == start && - entries[nentries - 1].prot == r->prot && - entries[nentries - 1].flags == r->flags && - entries[nentries - 1].offset == r->offset && - !strcmp(entries[nentries - 1].name, r->name)) { - entries[nentries - 1].end = end; + uint64_t end = (r->end + 0xFFFULL) & ~0xFFFULL; + size_t count = maps_entries_count(&entries); + maps_entry_t *last = + count > 0 ? maps_entries_at(&entries, count - 1) : NULL; + if (last != NULL && last->end == start && last->prot == r->prot && + last->flags == r->flags && last->offset == r->offset && + !strcmp(last->name, r->name)) { + last->end = end; continue; } - maps_entry_insert(entries, &nentries, start, end, r->prot, r->flags, - r->offset, r->name); + if (maps_entries_insert_sorted(&entries, start, end, r->prot, r->flags, + r->offset, r->name) < 0) + goto out_unlock; } /* Add preannounced entries only while they still have an uncovered tail. @@ -2436,12 +2456,17 @@ static int proc_open_self_maps(const guest_t *g) * split VMAs. A partial union must stay visible because some * reserved-but-not-realized span remains to advertise. */ - for (int i = 0; i < g->npreannounced && nentries < MAPS_ENTRY_MAX; i++) { + int npreannounced = g->npreannounced; + if (npreannounced < 0) + npreannounced = 0; + if (npreannounced > GUEST_MAX_PREANNOUNCED) + npreannounced = GUEST_MAX_PREANNOUNCED; + for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; bool shadowed = false; uint64_t covered_end = r->start; - for (int j = 0; j < g->nregions; j++) { + for (int j = 0; j < nregions; j++) { const guest_region_t *live = &g->regions[j]; if (live->end <= covered_end) @@ -2459,61 +2484,226 @@ static int proc_open_self_maps(const guest_t *g) if (shadowed) continue; - maps_entry_insert(entries, &nentries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, r->flags, - r->offset, r->name); + if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, + (r->end + 0xFFFULL) & ~0xFFFULL, + r->prot, r->flags, r->offset, + r->name) < 0) + goto out_unlock; + } + maps_entries_merge_adjacent(&entries); + result = (int)maps_entries_count(&entries); + +out_unlock: + pthread_mutex_unlock(&mmap_lock); + if (result < 0) { + maps_entries_destroy(&entries); + errno = ENOMEM; + return -1; + } + *entries_out = entries; + return (int)maps_entries_count(&entries); +} + +/* Format the common VMA header. The maps and smaps header must stay byte-for- + * byte compatible so consumers can use either file interchangeably. + */ +static int proc_format_maps_header(const maps_entry_t *e, + char *header, + size_t headersz) +{ + char perms[5]; + perms[0] = (e->prot & LINUX_PROT_READ) ? 'r' : '-'; + perms[1] = (e->prot & LINUX_PROT_WRITE) ? 'w' : '-'; + perms[2] = (e->prot & LINUX_PROT_EXEC) ? 'x' : '-'; + perms[3] = (e->flags & LINUX_MAP_SHARED) ? 's' : 'p'; + perms[4] = '\0'; + + int header_len = snprintf( + header, headersz, "%llx-%llx %s %08llx 00:00 0", + (unsigned long long) e->start, (unsigned long long) e->end, perms, + (unsigned long long) e->offset); + if (header_len < 0) + return -1; + if ((size_t) header_len >= headersz) + header_len = (int) headersz - 1; + + if (e->name[0]) { + while (header_len < MAPS_NAME_COLUMN && + (size_t) header_len < headersz - 1) + header[header_len++] = ' '; + int n = snprintf(header + header_len, headersz - (size_t) header_len, + "%s", e->name); + if (n > 0) { + if ((size_t) n >= headersz - (size_t) header_len) + n = (int) (headersz - (size_t) header_len - 1); + header_len += n; + } + } else if ((size_t) header_len < headersz - 1) { + header[header_len++] = ' '; } - maps_entries_merge_adjacent(entries, &nentries); + header[header_len] = '\0'; + return header_len; +} + +/* Release the resources shared by maps/smaps output without hiding the errno + * from the operation that produced result. + */ +static int proc_finish_maps_output(int result, + maps_entries_t *entries, + string_builder_t *builder) +{ + int saved_errno = errno; + maps_entries_destroy(entries); + string_builder_destroy(builder); + errno = saved_errno; + return result; +} + +/* Emit /proc/self/maps into a synthetic fd. Addresses are page-aligned and + * output matches the Linux maps header format. + */ +static int proc_open_self_maps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 256; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; /* Emit lines after merging so buffer accounting is centralized. */ - for (int i = 0; i < nentries && off < (int) bufsz - 256; i++) { - const maps_entry_t *e = &entries[i]; - char perms[5]; - perms[0] = (e->prot & 0x1) ? 'r' : '-'; - perms[1] = (e->prot & 0x2) ? 'w' : '-'; - perms[2] = (e->prot & 0x4) ? 'x' : '-'; - perms[3] = (e->flags & 0x01) ? 's' : 'p'; - perms[4] = '\0'; - - /* Format matches real Linux /proc//maps exactly: - * %lx-%lx %s %08lx %02x:%02x %lu %s\n - * Verified against strace in a real Lima VZ VM. - */ + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); char line[256]; - int lineoff = - snprintf(line, sizeof(line), "%llx-%llx %s %08llx 00:00 0", - (unsigned long long) e->start, (unsigned long long) e->end, - perms, (unsigned long long) e->offset); - /* Cap lineoff to buffer size (snprintf may return more than available - * on truncation) - */ - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - if (e->name[0]) { - while (lineoff < MAPS_NAME_COLUMN && - lineoff < (int) sizeof(line) - 1) - line[lineoff++] = ' '; - int n = - snprintf(line + lineoff, sizeof(line) - lineoff, "%s", e->name); - if (n > 0) - lineoff += n; - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - } else if (lineoff < (int) sizeof(line) - 1) { - line[lineoff++] = ' '; - } - int wrote = snprintf(buf + off, bufsz - off, "%.*s\n", lineoff, line); - if (wrote > 0 && off + wrote < (int) bufsz) - off += wrote; - else - break; /* Stop before truncating a maps line. */ + int line_len = proc_format_maps_header(e, line, sizeof(line)); + if (line_len < 0 || + string_builder_appendf(&builder, "%.*s\n", line_len, line) < 0) + goto out; } - log_debug("/proc/self/maps (%d bytes):\n%.*s", off, off, buf); - int fd = proc_synthetic_fd(buf, off); - free(entries); - free(buf); - return fd; + log_debug("/proc/self/maps (%zu bytes):\n%.*s", string_builder_length(&builder), + (int)string_builder_length(&builder), + string_builder_data_const(&builder) ? string_builder_data_const(&builder) + : ""); + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); + +out: + return proc_finish_maps_output(result, &entries, &builder); +} + +/* Emit a Linux-shaped /proc/self/smaps approximation. The runtime tracks + * guest VMAs and logical fork snapshots, but it cannot observe kernel page + * residency or dirty bits on the host. For a fork child, writable private + * anonymous VMAs therefore report their full VMA size as Shared_Dirty; all + * other fields that require kernel page accounting are stable zeroes. + */ +static int proc_open_self_smaps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 768; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; + + bool fork_child = proc_get_ppid() != 0; + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + char header[256]; + int header_len = proc_format_maps_header(e, header, sizeof(header)); + if (header_len < 0) + goto out; + + uint64_t size_kb = (e->end - e->start) / 1024; + bool anonymous = (e->flags & LINUX_MAP_ANONYMOUS) != 0; + bool shared = (e->flags & LINUX_MAP_SHARED) != 0; + bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; + bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && + anonymous && !shared; + bool logical_shared_dirty = fork_child && private_anon && + (e->prot & LINUX_PROT_WRITE); + uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; + + char vmflags[64]; + size_t vmflags_len = 0; +#define APPEND_VMFLAG(flag) \ + do { \ + const char *token = (flag); \ + size_t token_len = strlen(token); \ + if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ + vmflags[vmflags_len++] = ' '; \ + memcpy(vmflags + vmflags_len, token, token_len); \ + vmflags_len += token_len; \ + } \ + } while (0) + /* Only flags directly evidenced by the tracked VMA are reported. In + * particular, do not invent Linux max-permission/accounting flags + * (mr/mw/me/ac/sd/etc.) that the emulator cannot observe. + */ + if (e->prot & LINUX_PROT_READ) + APPEND_VMFLAG("rd"); + if (e->prot & LINUX_PROT_WRITE) + APPEND_VMFLAG("wr"); + if (e->prot & LINUX_PROT_EXEC) + APPEND_VMFLAG("ex"); + if (shared) + APPEND_VMFLAG("sh"); + if (noreserve) + APPEND_VMFLAG("nr"); +#undef APPEND_VMFLAG + vmflags[vmflags_len] = '\0'; + + if (string_builder_appendf( + &builder, + "%.*s\n" + "Size: %llu kB\n" + "KernelPageSize: 4 kB\n" + "MMUPageSize: 4 kB\n" + "Rss: 0 kB\n" + "Pss: 0 kB\n" + "Pss_Dirty: 0 kB\n" + "Shared_Clean: 0 kB\n" + "Shared_Dirty: %llu kB\n" + "Private_Clean: 0 kB\n" + "Private_Dirty: 0 kB\n" + "Referenced: 0 kB\n" + "Anonymous: 0 kB\n" + "KSM: 0 kB\n" + "LazyFree: 0 kB\n" + "AnonHugePages: 0 kB\n" + "ShmemPmdMapped: 0 kB\n" + "FilePmdMapped: 0 kB\n" + "Shared_Hugetlb: 0 kB\n" + "Private_Hugetlb: 0 kB\n" + "Swap: 0 kB\n" + "SwapPss: 0 kB\n" + "Locked: 0 kB\n" + "THPeligible: 0\n" + "ProtectionKey: 0\n" + "VmFlags:%s\n", + header_len, header, (unsigned long long) size_kb, + (unsigned long long) shared_dirty_kb, vmflags) < 0) + goto out; + } + + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); + +out: + return proc_finish_maps_output(result, &entries, &builder); } /* Emit /proc/meminfo from host sysctl (HW_MEMSIZE) plus mach vm_statistics64, @@ -3116,6 +3306,12 @@ int proc_intercept_open(const guest_t *g, if (!strcmp(path, "/proc/self/maps")) return proc_open_self_maps(g); + /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata + * and the coarse fork Shared_Dirty compatibility signal. + */ + if (!strcmp(path, "/proc/self/smaps")) + return proc_open_self_smaps(g); + /* /proc/uptime -> synthetic uptime in seconds. Uses sysctl(KERN_BOOTTIME), * same as sys_sysinfo() in syscall/sys.c. Idle time is 0 (no meaningful * macOS equivalent). @@ -3721,6 +3917,7 @@ int proc_intercept_stat(const char *path, struct stat *st) "/proc/self/status", "/proc/self/cmdline", "/proc/self/maps", + "/proc/self/smaps", "/proc/self/exe", "/proc/self/environ", "/proc/self/auxv", diff --git a/src/runtime/procemu.h b/src/runtime/procemu.h index cd8e75c4..d39083cc 100644 --- a/src/runtime/procemu.h +++ b/src/runtime/procemu.h @@ -25,7 +25,7 @@ #define PROC_NOT_INTERCEPTED (-2) /* Intercept openat for /proc and /dev paths. The guest_t pointer is needed to - * generate /proc/self/maps from region data. + * generate /proc/self/maps and /proc/self/smaps from region data. * Returns a host fd on match (caller should fd_alloc it), -1 on error with * errno set, or PROC_NOT_INTERCEPTED if the path is not intercepted. */ diff --git a/src/string-builder.c b/src/string-builder.c new file mode 100644 index 00000000..5cd994d2 --- /dev/null +++ b/src/string-builder.c @@ -0,0 +1,283 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "string-builder.h" + +#include +#include +#include +#include +#include + +/* Set EILSEQ for invalid input and report failure. */ +static int string_builder_invalid(void) +{ + errno = EILSEQ; + return -1; +} + +/* Initialize storage and establish an empty, NUL-terminated builder. */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity) +{ + if (builder == NULL) + return string_builder_invalid(); + if (initial_capacity == 0) { + string_builder_destroy(builder); + return 0; + } + if (string_builder_storage_init_with_capacity(&builder->storage, + initial_capacity) < 0) + return -1; + string_builder_data(builder)[0] = '\0'; + return 0; +} + +/* Release storage and reset the logical length. */ +void string_builder_destroy(string_builder_t *builder) +{ + if (builder == NULL) + return; + string_builder_storage_destroy(&builder->storage); +} + +/* Reserve enough capacity for extra bytes after the current contents. */ +int string_builder_reserve(string_builder_t *builder, size_t extra) +{ + if (builder == NULL) + return string_builder_invalid(); + if (extra == 0 && string_builder_storage_count(&builder->storage) == 0 && + string_builder_capacity(builder) == 0) + return 0; + + /* The dynamic array counts payload elements. Reserve one additional char + * for the string builder's trailing NUL. */ + if (extra == SIZE_MAX) { + errno = EOVERFLOW; + return -1; + } + if (string_builder_storage_reserve(&builder->storage, extra + 1) < 0) + return -1; + string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = + '\0'; + return 0; +} + +/* Locate a source span that aliases the builder allocation. The offset must be + * captured before reserve because reserve may move the allocation. */ +static int string_builder_source_offset(const string_builder_t *builder, + const void *source, size_t length, + size_t *offset) +{ + const char *base_ptr = string_builder_data_const(builder); + size_t capacity = string_builder_capacity(builder); + if (base_ptr == NULL || source == NULL) + return 0; + uintptr_t base = (uintptr_t) (const void *) base_ptr; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t) SIZE_MAX) + return 0; + size_t start = (size_t) delta; + if (start > capacity || length > capacity - start) + return 0; + *offset = start; + return 1; +} + +/* Commit string bytes through the generic array and restore the terminator. */ +static int string_builder_commit_append(string_builder_t *builder, + const char *data, size_t len) +{ + if (string_builder_storage_append_n(&builder->storage, data, len) < 0) + return -1; + string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = + '\0'; + return 0; +} + +/* Append a C string and keep the builder NUL-terminated. */ +int string_builder_append(string_builder_t *builder, const char *text) +{ + if (builder == NULL || text == NULL) + return string_builder_invalid(); + + size_t len = strlen(text); + if (len == 0) + return 0; + + size_t source_offset = 0; + int aliases = string_builder_source_offset(builder, text, len, + &source_offset); + if (string_builder_reserve(builder, len) < 0) + return -1; + if (aliases) + text = string_builder_data_const(builder) + source_offset; + return string_builder_commit_append(builder, text, len); +} + +/* Convert a formatting failure into the module's documented errno values. */ +static int string_builder_format_failure(void) +{ + if (errno != EOVERFLOW && errno != EILSEQ) + errno = EILSEQ; + return -1; +} + +/* Call vsnprintf with a copied argument list for the current sizing pass. */ +static int string_builder_vsnprintf(char *destination, size_t available, + const char *format, va_list arguments) +{ + va_list pass; + va_copy(pass, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + int result = vsnprintf(destination, available, format, pass); +#pragma clang diagnostic pop + va_end(pass); + return result; +} + +typedef enum string_builder_format_status { + STRING_BUILDER_FORMAT_ERROR = -1, + STRING_BUILDER_FORMAT_FIT = 0, + STRING_BUILDER_FORMAT_TRUNCATED = 1 +} string_builder_format_status_t; + +/* Ensure the builder has a writable tail and capture its current length. */ +static int string_builder_prepare_appendf(string_builder_t *builder, + size_t *old_len) +{ + if (string_builder_capacity(builder) == 0 && + string_builder_reserve(builder, 1) < 0) + return -1; + + char *data = string_builder_data(builder); + size_t capacity = string_builder_capacity(builder); + *old_len = string_builder_storage_count(&builder->storage); + if (data == NULL || *old_len >= capacity) + return string_builder_invalid(); + return 0; +} + +/* Format into the current tail and report whether the result was truncated. */ +static string_builder_format_status_t +string_builder_try_format(string_builder_t *builder, size_t old_len, + const char *format, va_list arguments, + size_t *written) +{ + char *data = string_builder_data(builder); + size_t available = string_builder_capacity(builder) - old_len; + int formatted = string_builder_vsnprintf(data + old_len, available, format, + arguments); + if (formatted < 0) + return STRING_BUILDER_FORMAT_ERROR; + + size_t visible = strlen(data + old_len); + if ((size_t) formatted < available || visible < available - 1) { + *written = visible; + return STRING_BUILDER_FORMAT_FIT; + } + + *written = (size_t) formatted; + return STRING_BUILDER_FORMAT_TRUNCATED; +} + +/* Restore the terminator after a failed formatting attempt. */ +static void string_builder_rollback_appendf(string_builder_t *builder, + size_t old_len) +{ + char *data = string_builder_data(builder); + if (data != NULL && old_len < string_builder_capacity(builder)) + data[old_len] = '\0'; +} + +/* Append printf-formatted text, growing and retrying when the first pass is + * truncated. + */ +int string_builder_appendf(string_builder_t *builder, const char *format, ...) +{ + int result = -1; + int saved_errno; + size_t old_len = 0; + size_t written = 0; + string_builder_format_status_t status; + va_list arguments; + + if (builder == NULL || format == NULL) + return string_builder_invalid(); + + saved_errno = errno; + va_start(arguments, format); + + if (string_builder_prepare_appendf(builder, &old_len) < 0) + goto out; + + status = string_builder_try_format(builder, old_len, format, arguments, + &written); + if (status == STRING_BUILDER_FORMAT_ERROR) { + string_builder_format_failure(); + goto rollback; + } + if (status == STRING_BUILDER_FORMAT_TRUNCATED) { + if (string_builder_reserve(builder, written) < 0) + goto rollback; + + status = string_builder_try_format(builder, old_len, format, arguments, + &written); + if (status == STRING_BUILDER_FORMAT_ERROR) { + string_builder_format_failure(); + goto rollback; + } + if (status == STRING_BUILDER_FORMAT_TRUNCATED) { + errno = EOVERFLOW; + goto rollback; + } + } + + if (string_builder_commit_append( + builder, string_builder_data(builder) + old_len, written) < 0) + goto rollback; + + errno = saved_errno; + result = 0; + goto out; + +rollback: + string_builder_rollback_appendf(builder, old_len); + +out: + va_end(arguments); + return result; +} + +/* Return mutable storage for the builder, if it has been allocated. */ +char *string_builder_data(string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_data(&builder->storage) : NULL; +} + +/* Return const storage for the builder, if it has been allocated. */ +const char *string_builder_data_const(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_data_const(&builder->storage) + : NULL; +} + +/* Return the number of data bytes currently stored. */ +size_t string_builder_length(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_count(&builder->storage) : 0; +} + +/* Return allocated capacity in bytes, including the terminating NUL. */ +size_t string_builder_capacity(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_capacity(&builder->storage) : 0; +} diff --git a/src/string-builder.h b/src/string-builder.h new file mode 100644 index 00000000..8feb6ae3 --- /dev/null +++ b/src/string-builder.h @@ -0,0 +1,62 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "dynamic-array.h" + +DYNAMIC_ARRAY_DEFINE(string_builder_storage, char) + +/* The storage representation is an implementation detail; callers should use + * the accessors below. The generated count is the string length; capacity is + * measured in bytes and includes the trailing NUL slot. + */ +typedef struct string_builder { + string_builder_storage_t storage; +} string_builder_t; + +/* Initialize a builder, reserving initial_capacity bytes including the NUL. + * A zero capacity leaves storage unallocated for lazy growth. Returns 0 on + * success or -1 with errno set on invalid input or allocation failure. + */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity); + +/* Release the builder storage and reset its length. Safe to call with NULL. */ +void string_builder_destroy(string_builder_t *builder); + +/* Ensure room for extra string bytes plus the terminating NUL. */ +int string_builder_reserve(string_builder_t *builder, size_t extra); + +/* Append a C string, accepting aliases into the builder's storage. As with + * standard C string functions, the first NUL terminates the input. A NULL + * pointer is invalid. The resulting C string is always NUL-terminated. + */ +int string_builder_append(string_builder_t *builder, const char *text); + +/* Append formatted text using printf-style arguments. A formatted NUL byte + * terminates the appended C string prefix. + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((format(printf, 2, 3))) +#endif +int string_builder_appendf(string_builder_t *builder, const char *format, ...); + +/* Return mutable builder data, or NULL when builder is NULL or unallocated. + * Callers must preserve the no-embedded-NUL C-string invariant. + */ +char *string_builder_data(string_builder_t *builder); + +/* Return read-only builder data, or NULL when builder is NULL or unallocated. */ +const char *string_builder_data_const(const string_builder_t *builder); + +/* Return the number of data bytes currently stored, excluding the NUL. */ +size_t string_builder_length(const string_builder_t *builder); + +/* Return allocated capacity in bytes, including space for the NUL. */ +size_t string_builder_capacity(const string_builder_t *builder); diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c new file mode 100644 index 00000000..73938842 --- /dev/null +++ b/tests/test-dynamic-array-host.c @@ -0,0 +1,109 @@ +/* + * Native-host unit tests for the generic dynamic array. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#include "dynamic-array.h" + +typedef struct odd { + unsigned char tag; + uint32_t value; +} odd_t; + +DYNAMIC_ARRAY_DEFINE(int_array, int) +DYNAMIC_ARRAY_DEFINE(odd_array, odd_t) + +static void test_zero_and_init(void) +{ + int_array_t values = {0}; + int value = 7; + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + assert(int_array_append(&values, &value) == 0); + assert(int_array_count(&values) == 1); + assert(*int_array_at(&values, 0) == 7); + int_array_destroy(&values); + assert(int_array_data(&values) == NULL); + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_capacity(&values) >= 4); + int_array_destroy(&values); +} + +static void test_growth_insert_and_stride(void) +{ + odd_array_t values = {0}; + odd_t first = {1, 11}; + odd_t second = {2, 22}; + odd_t middle = {3, 33}; + assert(odd_array_append(&values, &first) == 0); + assert(odd_array_append(&values, &second) == 0); + assert(odd_array_insert(&values, 1, &middle) == 0); + assert(odd_array_count(&values) == 3); + assert(odd_array_at(&values, 1)->tag == 3); + assert(odd_array_at(&values, 2)->value == 22); + assert(odd_array_capacity(&values) >= 3); + odd_array_destroy(&values); +} + +static void test_alias_and_resize(void) +{ + int_array_t values = {0}; + int initial[] = {1, 2, 3, 4}; + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_append_n(&values, initial, 4) == 0); + int *old_data = int_array_data(&values); + int *alias = int_array_at(&values, 1); + assert(int_array_append(&values, alias) == 0); + assert(int_array_data(&values) != old_data || int_array_capacity(&values) > 4); + assert(int_array_count(&values) == 5); + assert(*int_array_at(&values, 4) == 2); + assert(int_array_resize(&values, 8) == 0); + for (size_t i = 5; i < 8; i++) + assert(*int_array_at(&values, i) == 0); + assert(int_array_resize(&values, 2) == 0); + int_array_destroy(&values); +} + +static void test_invalid_and_overflow(void) +{ + int_array_t values = {0}; + int value = 9; + errno = 0; + assert(int_array_insert(&values, 1, &value) == -1); + assert(errno == EINVAL); + errno = 0; + assert(int_array_append_n(&values, &value, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + + dynamic_array_t raw = {0}; + errno = 0; + assert(dynamic_array_init(&raw, 0) == -1); + assert(errno == EINVAL); + assert(dynamic_array_init(&raw, sizeof(uint64_t)) == 0); + errno = 0; + assert(dynamic_array_reserve(&raw, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(raw.data == NULL && raw.count == 0 && raw.capacity == 0); + dynamic_array_destroy(&raw); +} + +int main(void) +{ + test_zero_and_init(); + test_growth_insert_and_stride(); + test_alias_and_resize(); + test_invalid_and_overflow(); + puts("test-dynamic-array-host: PASS"); + return 0; +} diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 6f450537..42a6bd27 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -705,6 +705,7 @@ run_unit_tests() test_rc "$runner" "test-procfs-exec" 0 "$bindir/test-procfs-exec" test_rc "$runner" "test-proc-limits" 0 "$bindir/test-proc-limits" test_rc "$runner" "test-proc-fidelity" 0 "$bindir/test-proc-fidelity" + test_rc "$runner" "test-proc-smap" 0 "$bindir/test-proc-smap" printf "\nNetwork\n" test_check "$runner" "test-net" "0 failed" "$bindir/test-net" @@ -1314,8 +1315,8 @@ run_suite() # observed counts diverge. apple-unknown is the fallback row for SoC strings the # detector does not recognize yet. EXPECTED_BASELINES=( - "elfuse-aarch64|241|0" - "qemu-aarch64|224|0" + "elfuse-aarch64|242|0" + "qemu-aarch64|225|0" "elfuse-x86_64:apple-m1-m2|71|0" "elfuse-x86_64:apple-m3-plus|71|0" "elfuse-x86_64:apple-unknown|71|0" diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c new file mode 100644 index 00000000..99786148 --- /dev/null +++ b/tests/test-proc-smap.c @@ -0,0 +1,599 @@ +/* + * Generic /proc//smaps parser and accounting regression test. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" +#include "test-util.h" + +int passes = 0, fails = 0; + +typedef struct { + uintptr_t start; + uintptr_t end; + unsigned long long offset; + char perms[5]; + unsigned long long size_kb; + unsigned long long kernel_page_kb; + unsigned long long mmu_page_kb; + unsigned long long shared_dirty_kb; + bool have_size; + bool have_shared_dirty; + bool have_vmflags; + bool vmflags_wr; +} smaps_vma_t; + +static const char *const smaps_fields[] = { + "Size:", "KernelPageSize:", "MMUPageSize:", + "Rss:", "Pss:", "Pss_Dirty:", + "Shared_Clean:", "Shared_Dirty:", "Private_Clean:", + "Private_Dirty:", "Referenced:", "Anonymous:", + "KSM:", "LazyFree:", "AnonHugePages:", + "ShmemPmdMapped:", + "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", + "Swap:", "SwapPss:", "Locked:", + "THPeligible:", "ProtectionKey:", "VmFlags:", +}; + +#define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) +#define SMAPS_KB_FIELD_COUNT 22 + +typedef struct { + smaps_vma_t *vmas; + size_t count; +} smaps_info_t; + +static const char *skip_space(const char *p) +{ + while (*p && isspace((unsigned char) *p)) + p++; + return p; +} + +static bool parse_token(const char **cursor, char *token, size_t token_size) +{ + const char *p = skip_space(*cursor); + const char *start = p; + while (*p && !isspace((unsigned char) *p)) + p++; + size_t len = (size_t) (p - start); + if (len == 0 || len >= token_size) + return false; + memcpy(token, start, len); + token[len] = '\0'; + *cursor = p; + return true; +} + +static bool parse_unsigned_token(const char *token, int base, + unsigned long long *value) +{ + if (!token[0]) + return false; + for (const unsigned char *p = (const unsigned char *) token; *p; p++) { + if (base == 16 ? !isxdigit(*p) : !isdigit(*p)) + return false; + } + + errno = 0; + char *end = NULL; + unsigned long long n = strtoull(token, &end, base); + if (errno == ERANGE || end == token || *end != '\0') + return false; + *value = n; + return true; +} + +static bool parse_device_token(const char *token) +{ + const char *colon = strchr(token, ':'); + if (!colon || colon == token || colon[1] == '\0' || strchr(colon + 1, ':')) + return false; + + char major[32], minor[32]; + size_t major_len = (size_t) (colon - token); + if (major_len >= sizeof(major) || strlen(colon + 1) >= sizeof(minor)) + return false; + memcpy(major, token, major_len); + major[major_len] = '\0'; + strcpy(minor, colon + 1); + + unsigned long long value; + return parse_unsigned_token(major, 16, &value) && + parse_unsigned_token(minor, 16, &value); +} + +/* Parse the address/perms/offset/dev/inode part of a smaps header. The rest + * of the line is an optional pathname and is intentionally left opaque. */ +static bool parse_header(const char *line, smaps_vma_t *vma) +{ + const char *p = line; + errno = 0; + char *end = NULL; + unsigned long long start = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || *end != '-') + return false; + p = end + 1; + + errno = 0; + unsigned long long finish = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || start >= finish || + !isspace((unsigned char) *end)) + return false; + if (start > UINTPTR_MAX || finish > UINTPTR_MAX) + return false; + p = end; + + char token[128]; + if (!parse_token(&p, token, sizeof(token)) || strlen(token) != 4) + return false; + for (size_t i = 0; i < 3; i++) { + if (token[i] != 'r' && token[i] != 'w' && token[i] != 'x' && + token[i] != '-') + return false; + } + if (token[3] != 'p' && token[3] != 's') + return false; + memcpy(vma->perms, token, sizeof(vma->perms)); + + if (!parse_token(&p, token, sizeof(token)) || + !parse_unsigned_token(token, 16, &vma->offset)) + return false; + if (!parse_token(&p, token, sizeof(token)) || !parse_device_token(token)) + return false; + if (!parse_token(&p, token, sizeof(token))) + return false; + unsigned long long inode; + if (!parse_unsigned_token(token, 10, &inode)) + return false; + + vma->start = (uintptr_t) start; + vma->end = (uintptr_t) finish; + vma->size_kb = 0; + vma->kernel_page_kb = 0; + vma->mmu_page_kb = 0; + vma->shared_dirty_kb = 0; + vma->have_size = false; + vma->have_shared_dirty = false; + vma->have_vmflags = false; + vma->vmflags_wr = false; + return true; +} + +/* Parse a decimal field and optionally require a suffix after its number. + * Both smaps field families share the same label/whitespace/overflow rules; + * only the trailing unit differs ("kB" for the first group, none for the + * remaining numeric fields). */ +static bool parse_numeric_field(const char *line, const char *label, + const char *suffix, + unsigned long long *value) +{ + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + const char *p = skip_space(line + label_len); + const char *start = p; + while (isdigit((unsigned char) *p)) + p++; + if (p == start) + return false; + char number[64]; + size_t number_len = (size_t) (p - start); + if (number_len >= sizeof(number)) + return false; + memcpy(number, start, number_len); + number[number_len] = '\0'; + if (!parse_unsigned_token(number, 10, value)) + return false; + p = skip_space(p); + return suffix ? !strcmp(p, suffix) : *p == '\0'; +} + +static bool parse_vmflags(const char *line, bool *writable) +{ + const char *label = "VmFlags:"; + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + + const char *p = line + label_len; + bool has_wr = false; + while (*(p = skip_space(p))) { + const char *start = p; + while (*p && !isspace((unsigned char) *p)) { + if (!isalpha((unsigned char) *p)) + return false; + p++; + } + if (p == start) + return false; + if ((size_t) (p - start) == 2 && start[0] == 'w' && start[1] == 'r') + has_wr = true; + } + /* A synthetic PROT_NONE VMA has no provable access flags and therefore + * may legitimately emit an empty VmFlags field. */ + *writable = has_wr; + return true; +} + +static bool finish_vma(smaps_vma_t *vma, size_t field_index) +{ + return field_index == SMAPS_FIELD_COUNT && vma->have_size && + vma->have_shared_dirty && vma->have_vmflags; +} + +/* THPeligible and ProtectionKey are conditional in real Linux kernels. The + * elfuse provider always emits both, but qemu may legitimately omit either; + * when the parser reaches one of those labels, advance over any missing + * optional fields before parsing VmFlags. */ +static void skip_optional_fields(const char *line, size_t *field_index) +{ + if (*field_index == SMAPS_FIELD_COUNT - 3 && + (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || + !strncmp(line, "VmFlags:", 8))) + (*field_index)++; + if (*field_index == SMAPS_FIELD_COUNT - 2 && + !strncmp(line, "VmFlags:", 8)) + (*field_index)++; +} + +static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) +{ + smaps_vma_t *vmas = realloc(info->vmas, + (info->count + 1) * sizeof(*info->vmas)); + if (!vmas) + return false; + vmas[info->count++] = *vma; + info->vmas = vmas; + return true; +} + +static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) +{ + memset(info, 0, sizeof(*info)); + if (len == 0 || buf[len - 1] != '\n') + return false; /* catches a truncated final record */ + + smaps_vma_t current; + bool have_current = false; + size_t field_index = 0; + uintptr_t previous_end = 0; + char *line = buf; + while ((size_t) (line - buf) < len) { + char *next = memchr(line, '\n', len - (size_t) (line - buf)); + if (!next) + goto fail; + *next = '\0'; + + /* Linux normally places headers back-to-back; the synthetic proc + * provider separates records with one blank line. Accept that + * separator only after a complete record. */ + if (!*line) { + if (!have_current || field_index != SMAPS_FIELD_COUNT) + goto fail; + line = next + 1; + continue; + } + + smaps_vma_t header; + if (parse_header(line, &header)) { + if (have_current) { + if (!finish_vma(¤t, field_index) || + !append_vma(info, ¤t)) + goto fail; + } + if (info->count > 0 && + (header.start < previous_end || header.start <= + info->vmas[info->count - 1].start)) + goto fail; + current = header; + have_current = true; + field_index = 0; + previous_end = header.end; + } else { + if (!have_current) + goto fail; + skip_optional_fields(line, &field_index); + if (field_index >= SMAPS_FIELD_COUNT) + goto fail; + unsigned long long value; + bool writable; + if (field_index < SMAPS_KB_FIELD_COUNT && + parse_numeric_field(line, smaps_fields[field_index], "kB", + &value)) { + if (field_index == 0) + current.size_kb = value; + if (field_index == 1) + current.kernel_page_kb = value; + if (field_index == 2) + current.mmu_page_kb = value; + if (field_index == 7) + current.shared_dirty_kb = value; + if (field_index == 0) + current.have_size = true; + if (field_index == 7) + current.have_shared_dirty = true; + field_index++; + } else if (field_index >= SMAPS_KB_FIELD_COUNT && + field_index < SMAPS_FIELD_COUNT - 1 && + parse_numeric_field(line, smaps_fields[field_index], + NULL, &value)) { + field_index++; + } else if (field_index == SMAPS_FIELD_COUNT - 1 && + parse_vmflags(line, &writable)) { + current.have_vmflags = true; + current.vmflags_wr = writable; + field_index++; + } else { + goto fail; + } + } + line = next + 1; + } + + if (!have_current || !finish_vma(¤t, field_index) || + !append_vma(info, ¤t) || + info->count == 0) + goto fail; + return true; + +fail: + free(info->vmas); + memset(info, 0, sizeof(*info)); + return false; +} + +static void free_smaps(smaps_info_t *info) +{ + free(info->vmas); + memset(info, 0, sizeof(*info)); +} + +/* Read, parse, and release one smaps snapshot. The parser owns only its VMA + * array, so the transient file buffer can be cleaned up in this single place + * on both success and failure. */ +static bool load_smaps(const char *path, smaps_info_t *info) +{ + char *buf = NULL; + size_t len = 0; + bool ok = read_file_dynamic_nul(path, &buf, &len) >= 0 && + parse_smaps(buf, len, info); + free(buf); + return ok; +} + +static const smaps_vma_t *find_vma(const smaps_info_t *info, uintptr_t address) +{ + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].start <= address && address < info->vmas[i].end) + return &info->vmas[i]; + } + return NULL; +} + +static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, + uintptr_t end) +{ + size_t count = 0; + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].end > start && info->vmas[i].start < end) + count++; + } + return count; +} + +static bool validate_layout(const smaps_info_t *info, uintptr_t target, + uintptr_t stress, size_t page_size, + size_t stress_size) +{ + if (info->count <= 256) + return false; + + const smaps_vma_t *first = find_vma(info, target); + const smaps_vma_t *middle = find_vma(info, target + page_size); + const smaps_vma_t *last = find_vma(info, target + 2 * page_size); + unsigned long long page_kb = page_size / 1024; + if (!first || !middle || !last || page_kb == 0) + return false; + + if (first->start != target || first->end != target + page_size || + middle->start != target + page_size || + middle->end != target + 2 * page_size || + last->start != target + 2 * page_size || + last->end != target + 3 * page_size) + return false; + if (strcmp(first->perms, "rw-p") || strcmp(middle->perms, "r--p") || + strcmp(last->perms, "rw-p")) + return false; + if (first->size_kb != page_kb || middle->size_kb != page_kb || + last->size_kb != page_kb || !first->vmflags_wr || + middle->vmflags_wr || !last->vmflags_wr || + first->kernel_page_kb != 4 || first->mmu_page_kb != 4 || + middle->kernel_page_kb != 4 || middle->mmu_page_kb != 4 || + last->kernel_page_kb != 4 || last->mmu_page_kb != 4) + return false; + + /* Every stress-map page alternates permissions, so each page must remain + * its own VMA. Requiring the full count makes a short read or a producer + * cap observable instead of merely checking that some blocks exceed 256. + */ + size_t expected_stress_vmas = stress_size / page_size; + if (expected_stress_vmas <= 256 || + count_vmas_in_range(info, stress, stress + stress_size) != + expected_stress_vmas) + return false; + for (size_t i = 0; i < expected_stress_vmas; i++) { + const smaps_vma_t *page = find_vma(info, stress + i * page_size); + if (!page || page->start != stress + i * page_size || + page->end != stress + (i + 1) * page_size) + return false; + } + return true; +} + +static bool read_exact(int fd, void *data, size_t len) +{ + char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = read(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + +static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, + size_t stress_size) +{ + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + if (!load_smaps(paths[i], &info)) + return 1; + bool ok = validate_layout(&info, target, stress, page_size, + stress_size); + const smaps_vma_t *first = find_vma(&info, target); + const smaps_vma_t *middle = find_vma(&info, target + page_size); + const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); + const smaps_vma_t *stress_ro = find_vma(&info, stress); + const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || + first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || + last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || + stress_rw->shared_dirty_kb == 0) { + free_smaps(&info); + return 1; + } + free_smaps(&info); + } + return 0; +} + +int main(void) +{ + long page_size_long = sysconf(_SC_PAGESIZE); + TEST("smaps page size and VMA fixture"); + if (page_size_long < 1024 || page_size_long % 1024 != 0) { + FAIL("sysconf(_SC_PAGESIZE)"); + SUMMARY("test-proc-smap"); + return 1; + } + PASS(); + size_t page_size = (size_t) page_size_long; + + size_t target_size = 3 * page_size; + char *target = mmap(NULL, target_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t stress_pages = 320; + size_t stress_size = stress_pages * page_size; + char *stress = MAP_FAILED; + bool fixture_ok = target != MAP_FAILED; + if (fixture_ok) + stress = mmap(NULL, stress_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fixture_ok = fixture_ok && stress != MAP_FAILED; + if (fixture_ok) { + *(volatile unsigned char *) target = 0x5a; /* make one page dirty */ + fixture_ok = mprotect(target + page_size, page_size, PROT_READ) == 0; + } + if (fixture_ok) { + /* Start with a read-only page so an allocator placing this mapping + * immediately after target cannot merge target's final rw page into + * the stress range. Alternating permissions keeps every stress page + * as a distinct VMA while preserving the >256 completeness probe. */ + for (size_t i = 0; i < stress_pages; i += 2) { + if (mprotect(stress + i * page_size, page_size, PROT_READ) < 0) { + fixture_ok = false; + break; + } + } + } + + TEST("smaps headers, order, fields, and completeness"); + if (!fixture_ok) { + FAIL("mmap/mprotect fixture"); + } else { + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + bool ok = true; + size_t expected_count = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + bool parsed = load_smaps(paths[i], &info); + if (!parsed || + !validate_layout(&info, (uintptr_t) target, + (uintptr_t) stress, page_size, stress_size)) { + ok = false; + } else if (i == 0) { + expected_count = info.count; + } else if (info.count != expected_count) { + ok = false; + } + if (parsed) + free_smaps(&info); + } + EXPECT_TRUE(ok, "smaps parser/layout/completeness"); + } + + TEST("fork Shared_Dirty positive and read-only negative"); + if (!fixture_ok) { + FAIL("fixture unavailable"); + } else { + int pipefd[2] = {-1, -1}; + pid_t pid = pipe(pipefd) == 0 ? fork() : -1; + if (pid < 0) { + FAIL("pipe/fork"); + if (pipefd[0] >= 0) + close(pipefd[0]); + if (pipefd[1] >= 0) + close(pipefd[1]); + } else if (pid == 0) { + close(pipefd[0]); + int result = child_probe((uintptr_t) target, (uintptr_t) stress, + page_size, stress_size); + (void) write_fd_all(pipefd[1], &result, sizeof(result)); + close(pipefd[1]); + _exit(result); + } else { + close(pipefd[1]); + int result = 1; + bool received = read_exact(pipefd[0], &result, sizeof(result)); + close(pipefd[0]); + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + EXPECT_TRUE(received && waited && result == 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0, + "fork Shared_Dirty accounting"); + } + } + + if (stress != MAP_FAILED) + munmap(stress, stress_size); + if (target != MAP_FAILED) + munmap(target, target_size); + SUMMARY("test-proc-smap"); + return fails == 0 ? 0 : 1; +} diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c new file mode 100644 index 00000000..e926bb1b --- /dev/null +++ b/tests/test-string-builder-host.c @@ -0,0 +1,182 @@ +/* + * Native-host unit tests for string_builder_t. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "string-builder.h" + +static void expect_string(const string_builder_t *builder, const char *expected) +{ + size_t length = strlen(expected); + assert(string_builder_length(builder) == length); + assert(builder->storage.raw.count == length); + if (string_builder_data_const(builder) != NULL) { + assert(strcmp(string_builder_data_const(builder), expected) == 0); + assert(string_builder_data_const(builder)[length] == '\0'); + } else { + assert(length == 0); + } +} + +static void test_zero_and_initial_capacity(void) +{ + string_builder_t zero = {0}; + assert(string_builder_data(&zero) == NULL); + assert(string_builder_length(&zero) == 0); + assert(string_builder_capacity(&zero) == 0); + /* The public API also accepts a plain {0} value without an init call. */ + assert(string_builder_append(&zero, "zero") == 0); + expect_string(&zero, "zero"); + string_builder_destroy(&zero); + + string_builder_t lazy = {0}; + assert(string_builder_init(&lazy, 0) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_capacity(&lazy) == 0); + assert(string_builder_data(&lazy) == NULL); + assert(string_builder_reserve(&lazy, 4) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_data(&lazy)[0] == '\0'); + assert(string_builder_append(&lazy, "lazy") == 0); + assert(string_builder_capacity(&lazy) >= string_builder_length(&lazy) + 1); + expect_string(&lazy, "lazy"); + string_builder_destroy(&lazy); + + string_builder_t initial = {0}; + /* initial_capacity includes the trailing NUL byte. */ + assert(string_builder_init(&initial, 32) == 0); + assert(string_builder_capacity(&initial) >= 32); + assert(string_builder_data(&initial) != NULL); + assert(string_builder_length(&initial) == 0); + expect_string(&initial, ""); + string_builder_destroy(&initial); +} + +static void test_text_and_formatted_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + assert(string_builder_append(&builder, "prefix") == 0); + + assert(string_builder_appendf(&builder, ":%s:%d", "formatted", 42) == 0); + expect_string(&builder, "prefix:formatted:42"); + + /* An empty C string is a no-op. */ + size_t old_length = string_builder_length(&builder); + assert(string_builder_append(&builder, "") == 0); + assert(string_builder_length(&builder) == old_length); + expect_string(&builder, "prefix:formatted:42"); + string_builder_destroy(&builder); +} + +static void test_c_string_semantics(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "prefix") == 0); + + const char embedded[] = {'a', '\0', 'b', '\0'}; + errno = 0; + assert(string_builder_append(&builder, embedded) == 0); + assert(errno == 0); + expect_string(&builder, "prefixa"); + + errno = 0; + assert(string_builder_appendf(&builder, "x%c y", '\0') == 0); + assert(errno == 0); + expect_string(&builder, "prefixax"); + string_builder_destroy(&builder); + + /* A formatted NUL also terminates the appended C-string prefix when the + * first sizing pass has room. + */ + string_builder_t fit = {0}; + assert(string_builder_init(&fit, 16) == 0); + errno = 0; + assert(string_builder_appendf(&fit, "a%c%d", '\0', 1) == 0); + assert(errno == 0); + expect_string(&fit, "a"); + string_builder_destroy(&fit); +} + +static void test_growth_preserves_content(void) +{ + enum { COUNT = 4096 }; + char expected[COUNT]; + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + for (size_t i = 0; i < COUNT; i++) { + expected[i] = (char)('A' + (i % 26)); + char chunk[2] = {expected[i], '\0'}; + assert(string_builder_append(&builder, chunk) == 0); + } + assert(string_builder_length(&builder) == COUNT); + assert(memcmp(string_builder_data_const(&builder), expected, COUNT) == 0); + assert(string_builder_data_const(&builder)[COUNT] == '\0'); + assert(string_builder_capacity(&builder) >= COUNT + 1); + string_builder_destroy(&builder); +} + +static void test_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 4) == 0); + assert(string_builder_append(&builder, "abc") == 0); + const char *alias = string_builder_data_const(&builder) + 1; + assert(string_builder_append(&builder, alias) == 0); + const char expected[] = "abcbc"; + expect_string(&builder, expected); + string_builder_destroy(&builder); +} + +static void test_overflow_preserves_content(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 0) == 0); + assert(string_builder_appendf(&builder, "prefix:%d", 7) == 0); + + char snapshot[32]; + assert(string_builder_length(&builder) < sizeof(snapshot)); + memcpy(snapshot, string_builder_data_const(&builder), string_builder_length(&builder)); + size_t old_length = string_builder_length(&builder); + size_t old_capacity = string_builder_capacity(&builder); + + errno = 0; + assert(string_builder_reserve(&builder, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + + errno = 0; + assert(string_builder_append(&builder, NULL) == -1); + assert(errno == EILSEQ); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + string_builder_destroy(&builder); +} + +int main(void) +{ + test_zero_and_initial_capacity(); + test_text_and_formatted_append(); + test_c_string_semantics(); + test_growth_preserves_content(); + test_alias_append(); + test_overflow_preserves_content(); + puts("test-string-builder-host: PASS"); + return 0; +} diff --git a/tests/test-util.h b/tests/test-util.h index f37633e8..abe5e433 100644 --- a/tests/test-util.h +++ b/tests/test-util.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,69 @@ static inline ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) return total; } +/* Read a file whose size is not available from st_size (for example a proc + * file) until EOF, growing the buffer and appending a NUL terminator. */ +static inline ssize_t read_file_dynamic_nul(const char *path, char **buf_out, + size_t *len_out) +{ + if (!path || !buf_out || !len_out) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + size_t cap = 64 * 1024; + size_t len = 0; + char *buf = malloc(cap); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + + for (;;) { + if (len + 1 >= cap) { + if (cap > SIZE_MAX / 2) { + free(buf); + close(fd); + errno = EOVERFLOW; + return -1; + } + size_t new_cap = cap * 2; + char *new_buf = realloc(buf, new_cap); + if (!new_buf) { + free(buf); + close(fd); + errno = ENOMEM; + return -1; + } + buf = new_buf; + cap = new_cap; + } + + ssize_t n = read(fd, buf + len, cap - len - 1); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + free(buf); + close(fd); + return -1; + } + if (n == 0) + break; + len += (size_t) n; + } + close(fd); + + buf[len] = '\0'; + *buf_out = buf; + *len_out = len; + return (ssize_t) len; +} + static inline ssize_t raw_read_fd_all_nul(int fd, char *buf, size_t bufsz) { if (bufsz == 0) From cf5c83ba71f2f64e673664bbdd464be88232e130 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:16:30 +0800 Subject: [PATCH 13/26] Fix CI analysis and smaps matrix Keep static-analysis checks and the smaps test matrix green after adding the new procfs emulation coverage. --- src/dynamic-array.c | 40 ++++++++++------- src/dynamic-array.h | 56 ++++++++++++------------ src/runtime/procemu.c | 65 ++++++++++++++-------------- src/string-builder.c | 54 +++++++++++++---------- src/string-builder.h | 3 +- tests/test-dynamic-array-host.c | 27 +++++++++++- tests/test-matrix.sh | 15 ++++--- tests/test-proc-smap.c | 74 +++++++++++++++++--------------- tests/test-string-builder-host.c | 11 +++-- tests/test-util.h | 3 +- 10 files changed, 205 insertions(+), 143 deletions(-) diff --git a/src/dynamic-array.c b/src/dynamic-array.c index a4dd63c8..aad9dda4 100644 --- a/src/dynamic-array.c +++ b/src/dynamic-array.c @@ -12,7 +12,7 @@ #include #include -#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t)8) +#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t) 8) /* Set errno for a bad argument and return a standard failure code. */ static int dynamic_array_invalid(void) @@ -31,7 +31,8 @@ static int dynamic_array_validate(const dynamic_array_t *array) /* Return count + extra after guarding against size_t overflow. */ static int dynamic_array_count_plus(const dynamic_array_t *array, - size_t extra, size_t *total) + size_t extra, + size_t *total) { if (extra > SIZE_MAX - array->count) { errno = EOVERFLOW; @@ -42,7 +43,8 @@ static int dynamic_array_count_plus(const dynamic_array_t *array, } /* Compute the byte size for a number of elements with overflow protection. */ -static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, +static int dynamic_array_bytes(const dynamic_array_t *array, + size_t count, size_t *bytes) { if (count != 0 && array->element_size > SIZE_MAX / count) { @@ -55,7 +57,8 @@ static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, /* Return the offset of an in-array source span, including capacity bytes. */ static int dynamic_array_source_offset(const dynamic_array_t *array, - const void *source, size_t bytes, + const void *source, + size_t bytes, size_t *offset) { if (array->data == NULL || source == NULL) @@ -66,7 +69,7 @@ static int dynamic_array_source_offset(const dynamic_array_t *array, if (address < base) return 0; uintptr_t delta = address - base; - if (delta > (uintptr_t)SIZE_MAX) + if (delta > (uintptr_t) SIZE_MAX) return 0; size_t start = (size_t) delta; size_t allocation_bytes; @@ -135,7 +138,7 @@ void dynamic_array_destroy(dynamic_array_t *array) if (array == NULL) return; free(array->data); - *array = (dynamic_array_t){0}; + *array = (dynamic_array_t) {0}; } /* Ensure the array has capacity for at least extra additional elements. */ @@ -186,8 +189,10 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count) } if (count > array->count) { size_t old_bytes, new_bytes; - (void) dynamic_array_bytes(array, array->count, &old_bytes); - (void) dynamic_array_bytes(array, count, &new_bytes); + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; + if (dynamic_array_bytes(array, count, &new_bytes) < 0) + return -1; memset((unsigned char *) array->data + old_bytes, 0, new_bytes - old_bytes); } @@ -196,7 +201,8 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count) } /* Append multiple elements from source memory to the end of the array. */ -int dynamic_array_append_n(dynamic_array_t *array, const void *data, +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, size_t count) { if (dynamic_array_validate(array) < 0) @@ -220,15 +226,18 @@ int dynamic_array_append_n(dynamic_array_t *array, const void *data, if (aliases) data = (const unsigned char *) array->data + offset; size_t old_bytes; - (void) dynamic_array_bytes(array, array->count, &old_bytes); + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; memmove((unsigned char *) array->data + old_bytes, data, bytes); array->count = total; return 0; } /* Insert multiple elements at index while preserving existing elements. */ -int dynamic_array_insert_n(dynamic_array_t *array, size_t index, - const void *data, size_t count) +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count) { if (dynamic_array_validate(array) < 0) return -1; @@ -252,8 +261,8 @@ int dynamic_array_insert_n(dynamic_array_t *array, size_t index, return -1; size_t source_offset = 0; - int aliases = dynamic_array_source_offset(array, data, bytes, - &source_offset); + int aliases = + dynamic_array_source_offset(array, data, bytes, &source_offset); void *temporary = NULL; if (aliases) { temporary = malloc(bytes); @@ -285,7 +294,8 @@ int dynamic_array_append_one(dynamic_array_t *array, const void *data) } /* Insert a single element by forwarding to insert_n. */ -int dynamic_array_insert_one(dynamic_array_t *array, size_t index, +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, const void *data) { return dynamic_array_insert_n(array, index, data, 1); diff --git a/src/dynamic-array.h b/src/dynamic-array.h index 7e95c592..3899194a 100644 --- a/src/dynamic-array.h +++ b/src/dynamic-array.h @@ -63,24 +63,30 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count); /* Append one element, or count elements when the three-argument form is * used. The macro keeps both forms available to callers. */ int dynamic_array_append_one(dynamic_array_t *array, const void *data); -int dynamic_array_append_n(dynamic_array_t *array, const void *data, +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, size_t count); #define DYNAMIC_ARRAY_APPEND_PICK(_1, _2, _3, NAME, ...) NAME -#define dynamic_array_append(...) \ - DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ - dynamic_array_append_one, dynamic_array_append_dummy) \ - (__VA_ARGS__) +#define dynamic_array_append(...) \ + DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ + dynamic_array_append_one, \ + dynamic_array_append_dummy) \ + (__VA_ARGS__) /* Insert one element, or count elements in the four-argument form. */ -int dynamic_array_insert_one(dynamic_array_t *array, size_t index, +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, const void *data); -int dynamic_array_insert_n(dynamic_array_t *array, size_t index, - const void *data, size_t count); +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count); #define DYNAMIC_ARRAY_INSERT_PICK(_1, _2, _3, _4, NAME, ...) NAME -#define dynamic_array_insert(...) \ - DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ - dynamic_array_insert_one, dynamic_array_insert_dummy) \ - (__VA_ARGS__) +#define dynamic_array_insert(...) \ + DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ + dynamic_array_insert_one, \ + dynamic_array_insert_dummy) \ + (__VA_ARGS__) /* Return an element pointer, or NULL with errno=EINVAL for a bad index. */ void *dynamic_array_at(dynamic_array_t *array, size_t index); @@ -101,8 +107,8 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); sizeof(type)); \ } \ /* Initialize typed array and preallocate initial slots. */ \ - DYNAMIC_ARRAY_INLINE int name##_init_with_capacity(name##_t *array, \ - size_t initial_capacity) \ + DYNAMIC_ARRAY_INLINE int name##_init_with_capacity( \ + name##_t *array, size_t initial_capacity) \ { \ return dynamic_array_init_with_capacity( \ array != NULL ? &array->raw : NULL, sizeof(type), \ @@ -156,8 +162,7 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); } \ /* Append a typed range of values. */ \ DYNAMIC_ARRAY_INLINE int name##_append_n(name##_t *array, \ - const type *values, \ - size_t count) \ + const type *values, size_t count) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -167,9 +172,8 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); count)); \ } \ /* Insert one typed value through a pointer form. */ \ - DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, \ - size_t index, \ - const type *value) \ + DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, size_t index, \ + const type *value) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -180,21 +184,19 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); } \ /* Insert one typed value by value. */ \ DYNAMIC_ARRAY_INLINE int name##_insert_value(name##_t *array, \ - size_t index, \ - type value) \ + size_t index, type value) \ { \ return name##_insert_ptr(array, index, &value); \ } \ /* Insert one typed value from a pointer argument. */ \ DYNAMIC_ARRAY_INLINE int name##_insert(name##_t *array, size_t index, \ - const type *value) \ + const type *value) \ { \ return name##_insert_ptr(array, index, value); \ } \ /* Insert a typed range of values at index. */ \ - DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, \ - size_t index, const type *values, \ - size_t count) \ + DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, size_t index, \ + const type *values, size_t count) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -207,11 +209,11 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); DYNAMIC_ARRAY_INLINE type *name##_at(name##_t *array, size_t index) \ { \ return (type *) dynamic_array_at(array != NULL ? &array->raw : NULL, \ - index); \ + index); \ } \ /* Return a typed const pointer to the element at index. */ \ DYNAMIC_ARRAY_INLINE const type *name##_at_const(const name##_t *array, \ - size_t index) \ + size_t index) \ { \ return (const type *) dynamic_array_at_const( \ array != NULL ? &array->raw : NULL, index); \ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 2cb59be1..ccc7ba5d 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -168,7 +168,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) if (out != i) *maps_entries_at(entries, out) = *current; } - (void)maps_entries_resize(entries, out + 1); + (void) maps_entries_resize(entries, out + 1); } /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily @@ -468,8 +468,8 @@ static void proc_tmpdir_cleanup(void) /* Remove known files inside // and / */ char path[256]; - const char *files[] = {"stat", "status", "cmdline", "maps", "smaps", - "exe", NULL}; + const char *files[] = {"stat", "status", "cmdline", "maps", + "smaps", "exe", NULL}; char piddir[160]; /* Reconstruct pid subdir by scanning for the first numeric entry */ @@ -2411,7 +2411,7 @@ static int pty_open_master(int linux_flags) * record and output generation does not block page-table mutations. */ static int proc_build_maps_entries(const guest_t *g, - maps_entries_t *entries_out) + maps_entries_t *entries_out) { if (!g || !entries_out) { errno = EINVAL; @@ -2485,13 +2485,12 @@ static int proc_build_maps_entries(const guest_t *g, continue; if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, - r->prot, r->flags, r->offset, - r->name) < 0) + (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, + r->flags, r->offset, r->name) < 0) goto out_unlock; } maps_entries_merge_adjacent(&entries); - result = (int)maps_entries_count(&entries); + result = (int) maps_entries_count(&entries); out_unlock: pthread_mutex_unlock(&mmap_lock); @@ -2501,7 +2500,7 @@ static int proc_build_maps_entries(const guest_t *g, return -1; } *entries_out = entries; - return (int)maps_entries_count(&entries); + return (int) maps_entries_count(&entries); } /* Format the common VMA header. The maps and smaps header must stay byte-for- @@ -2518,10 +2517,10 @@ static int proc_format_maps_header(const maps_entry_t *e, perms[3] = (e->flags & LINUX_MAP_SHARED) ? 's' : 'p'; perms[4] = '\0'; - int header_len = snprintf( - header, headersz, "%llx-%llx %s %08llx 00:00 0", - (unsigned long long) e->start, (unsigned long long) e->end, perms, - (unsigned long long) e->offset); + int header_len = + snprintf(header, headersz, "%llx-%llx %s %08llx 00:00 0", + (unsigned long long) e->start, (unsigned long long) e->end, + perms, (unsigned long long) e->offset); if (header_len < 0) return -1; if ((size_t) header_len >= headersz) @@ -2577,7 +2576,7 @@ static int proc_open_self_maps(const guest_t *g) /* Emit lines after merging so buffer accounting is centralized. */ for (int i = 0; i < nentries; i++) { - const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char line[256]; int line_len = proc_format_maps_header(e, line, sizeof(line)); if (line_len < 0 || @@ -2585,10 +2584,12 @@ static int proc_open_self_maps(const guest_t *g) goto out; } - log_debug("/proc/self/maps (%zu bytes):\n%.*s", string_builder_length(&builder), - (int)string_builder_length(&builder), - string_builder_data_const(&builder) ? string_builder_data_const(&builder) - : ""); + log_debug("/proc/self/maps (%zu bytes):\n%.*s", + string_builder_length(&builder), + (int) string_builder_length(&builder), + string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : ""); result = proc_synthetic_fd(string_builder_data_const(&builder) ? string_builder_data_const(&builder) : "", @@ -2619,7 +2620,7 @@ static int proc_open_self_smaps(const guest_t *g) bool fork_child = proc_get_ppid() != 0; for (int i = 0; i < nentries; i++) { - const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char header[256]; int header_len = proc_format_maps_header(e, header, sizeof(header)); if (header_len < 0) @@ -2629,23 +2630,23 @@ static int proc_open_self_smaps(const guest_t *g) bool anonymous = (e->flags & LINUX_MAP_ANONYMOUS) != 0; bool shared = (e->flags & LINUX_MAP_SHARED) != 0; bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; - bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && - anonymous && !shared; - bool logical_shared_dirty = fork_child && private_anon && - (e->prot & LINUX_PROT_WRITE); + bool private_anon = + (e->flags & LINUX_MAP_PRIVATE) && anonymous && !shared; + bool logical_shared_dirty = + fork_child && private_anon && (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; char vmflags[64]; size_t vmflags_len = 0; -#define APPEND_VMFLAG(flag) \ - do { \ - const char *token = (flag); \ - size_t token_len = strlen(token); \ - if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ - vmflags[vmflags_len++] = ' '; \ - memcpy(vmflags + vmflags_len, token, token_len); \ - vmflags_len += token_len; \ - } \ +#define APPEND_VMFLAG(flag) \ + do { \ + const char *token = (flag); \ + size_t token_len = strlen(token); \ + if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ + vmflags[vmflags_len++] = ' '; \ + memcpy(vmflags + vmflags_len, token, token_len); \ + vmflags_len += token_len; \ + } \ } while (0) /* Only flags directly evidenced by the tracked VMA are reported. In * particular, do not invent Linux max-permission/accounting flags diff --git a/src/string-builder.c b/src/string-builder.c index 5cd994d2..fa134065 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -30,7 +30,7 @@ int string_builder_init(string_builder_t *builder, size_t initial_capacity) return 0; } if (string_builder_storage_init_with_capacity(&builder->storage, - initial_capacity) < 0) + initial_capacity) < 0) return -1; string_builder_data(builder)[0] = '\0'; return 0; @@ -61,15 +61,16 @@ int string_builder_reserve(string_builder_t *builder, size_t extra) } if (string_builder_storage_reserve(&builder->storage, extra + 1) < 0) return -1; - string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = - '\0'; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; return 0; } /* Locate a source span that aliases the builder allocation. The offset must be * captured before reserve because reserve may move the allocation. */ static int string_builder_source_offset(const string_builder_t *builder, - const void *source, size_t length, + const void *source, + size_t length, size_t *offset) { const char *base_ptr = string_builder_data_const(builder); @@ -92,12 +93,13 @@ static int string_builder_source_offset(const string_builder_t *builder, /* Commit string bytes through the generic array and restore the terminator. */ static int string_builder_commit_append(string_builder_t *builder, - const char *data, size_t len) + const char *data, + size_t len) { if (string_builder_storage_append_n(&builder->storage, data, len) < 0) return -1; - string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = - '\0'; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; return 0; } @@ -112,8 +114,8 @@ int string_builder_append(string_builder_t *builder, const char *text) return 0; size_t source_offset = 0; - int aliases = string_builder_source_offset(builder, text, len, - &source_offset); + int aliases = + string_builder_source_offset(builder, text, len, &source_offset); if (string_builder_reserve(builder, len) < 0) return -1; if (aliases) @@ -130,8 +132,10 @@ static int string_builder_format_failure(void) } /* Call vsnprintf with a copied argument list for the current sizing pass. */ -static int string_builder_vsnprintf(char *destination, size_t available, - const char *format, va_list arguments) +static int string_builder_vsnprintf(char *destination, + size_t available, + const char *format, + va_list arguments) { va_list pass; va_copy(pass, arguments); @@ -167,15 +171,17 @@ static int string_builder_prepare_appendf(string_builder_t *builder, } /* Format into the current tail and report whether the result was truncated. */ -static string_builder_format_status_t -string_builder_try_format(string_builder_t *builder, size_t old_len, - const char *format, va_list arguments, - size_t *written) +static string_builder_format_status_t string_builder_try_format( + string_builder_t *builder, + size_t old_len, + const char *format, + va_list arguments, + size_t *written) { char *data = string_builder_data(builder); size_t available = string_builder_capacity(builder) - old_len; - int formatted = string_builder_vsnprintf(data + old_len, available, format, - arguments); + int formatted = + string_builder_vsnprintf(data + old_len, available, format, arguments); if (formatted < 0) return STRING_BUILDER_FORMAT_ERROR; @@ -260,24 +266,28 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) /* Return mutable storage for the builder, if it has been allocated. */ char *string_builder_data(string_builder_t *builder) { - return builder != NULL ? string_builder_storage_data(&builder->storage) : NULL; + return builder != NULL ? string_builder_storage_data(&builder->storage) + : NULL; } /* Return const storage for the builder, if it has been allocated. */ const char *string_builder_data_const(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_data_const(&builder->storage) - : NULL; + return builder != NULL + ? string_builder_storage_data_const(&builder->storage) + : NULL; } /* Return the number of data bytes currently stored. */ size_t string_builder_length(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_count(&builder->storage) : 0; + return builder != NULL ? string_builder_storage_count(&builder->storage) + : 0; } /* Return allocated capacity in bytes, including the terminating NUL. */ size_t string_builder_capacity(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_capacity(&builder->storage) : 0; + return builder != NULL ? string_builder_storage_capacity(&builder->storage) + : 0; } diff --git a/src/string-builder.h b/src/string-builder.h index 8feb6ae3..f9fb1c0c 100644 --- a/src/string-builder.h +++ b/src/string-builder.h @@ -52,7 +52,8 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...); */ char *string_builder_data(string_builder_t *builder); -/* Return read-only builder data, or NULL when builder is NULL or unallocated. */ +/* Return read-only builder data, or NULL when builder is NULL or unallocated. + */ const char *string_builder_data_const(const string_builder_t *builder); /* Return the number of data bytes currently stored, excluding the NUL. */ diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c index 73938842..6a3ea3ce 100644 --- a/tests/test-dynamic-array-host.c +++ b/tests/test-dynamic-array-host.c @@ -65,7 +65,8 @@ static void test_alias_and_resize(void) int *old_data = int_array_data(&values); int *alias = int_array_at(&values, 1); assert(int_array_append(&values, alias) == 0); - assert(int_array_data(&values) != old_data || int_array_capacity(&values) > 4); + assert(int_array_data(&values) != old_data || + int_array_capacity(&values) > 4); assert(int_array_count(&values) == 5); assert(*int_array_at(&values, 4) == 2); assert(int_array_resize(&values, 8) == 0); @@ -96,6 +97,30 @@ static void test_invalid_and_overflow(void) assert(errno == EOVERFLOW); assert(raw.data == NULL && raw.count == 0 && raw.capacity == 0); dynamic_array_destroy(&raw); + + /* A malformed metadata state must not let failed byte-size calculations + * feed uninitialized offsets into memory operations. */ + dynamic_array_t resize_overflow = { + .data = NULL, + .count = 0, + .capacity = 2, + .element_size = SIZE_MAX, + }; + errno = 0; + assert(dynamic_array_resize(&resize_overflow, 2) == -1); + assert(errno == EOVERFLOW); + assert(resize_overflow.count == 0); + + dynamic_array_t append_overflow = { + .data = NULL, + .count = SIZE_MAX / 2 + 1, + .capacity = SIZE_MAX, + .element_size = 2, + }; + errno = 0; + assert(dynamic_array_append(&append_overflow, &value) == -1); + assert(errno == EOVERFLOW); + assert(append_overflow.count == SIZE_MAX / 2 + 1); } int main(void) diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 42a6bd27..1c2e16a2 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -243,10 +243,10 @@ unstage_sysroot_fixtures() # Generic test helpers. -# The qemu reference lane now runs every matrix test against the real Alpine -# linux-virt kernel, so QEMU_SKIP is empty. Add a test's name here only if it -# asserts elfuse-specific behavior a real kernel does not honor; it still runs -# in elfuse-aarch64 mode and in 'make check'. +# The qemu reference lane runs the portable matrix tests against the real +# Alpine linux-virt kernel. Add a test's name here only if it asserts +# elfuse-specific behavior a real kernel does not honor; it still runs in +# elfuse-aarch64 mode and in 'make check'. # # The two oom_adj/oom_score_adj sendfile-and-copy_file_range-interception # subtests that used to make test-io-opt diverge here were split out into @@ -286,6 +286,7 @@ QEMU_SKIP=" test-fd-family test-scm-creds test-proc-fidelity + test-proc-smap " # test-session: getpgid/getsid/setsid assume the test is its own session and # process-group leader, true when elfuse launches it directly but not when @@ -362,6 +363,10 @@ QEMU_SKIP=" # under elfuse, while a real kernel allows the open and only rejects the # write -- a genuine behavioral difference worth reviewing on its own, # not just an environment artifact. +# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including a +# coarse Shared_Dirty compatibility signal for untouched writable VMAs. Real +# Linux smaps exposes kernel-owned VMA/page accounting instead, so this is +# intentionally not a reference-kernel invariant. # Tests that only run under qemu. A test belongs here when it needs a writable, # byte-exact root: the elfuse lane runs without a sysroot, and the macOS root is @@ -1316,7 +1321,7 @@ run_suite() # detector does not recognize yet. EXPECTED_BASELINES=( "elfuse-aarch64|242|0" - "qemu-aarch64|225|0" + "qemu-aarch64|224|0" "elfuse-x86_64:apple-m1-m2|71|0" "elfuse-x86_64:apple-m3-plus|71|0" "elfuse-x86_64:apple-unknown|71|0" diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 99786148..97a02e77 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -39,15 +39,13 @@ typedef struct { } smaps_vma_t; static const char *const smaps_fields[] = { - "Size:", "KernelPageSize:", "MMUPageSize:", - "Rss:", "Pss:", "Pss_Dirty:", - "Shared_Clean:", "Shared_Dirty:", "Private_Clean:", - "Private_Dirty:", "Referenced:", "Anonymous:", - "KSM:", "LazyFree:", "AnonHugePages:", - "ShmemPmdMapped:", - "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", - "Swap:", "SwapPss:", "Locked:", - "THPeligible:", "ProtectionKey:", "VmFlags:", + "Size:", "KernelPageSize:", "MMUPageSize:", "Rss:", + "Pss:", "Pss_Dirty:", "Shared_Clean:", "Shared_Dirty:", + "Private_Clean:", "Private_Dirty:", "Referenced:", "Anonymous:", + "KSM:", "LazyFree:", "AnonHugePages:", "ShmemPmdMapped:", + "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", "Swap:", + "SwapPss:", "Locked:", "THPeligible:", "ProtectionKey:", + "VmFlags:", }; #define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) @@ -80,7 +78,8 @@ static bool parse_token(const char **cursor, char *token, size_t token_size) return true; } -static bool parse_unsigned_token(const char *token, int base, +static bool parse_unsigned_token(const char *token, + int base, unsigned long long *value) { if (!token[0]) @@ -179,7 +178,8 @@ static bool parse_header(const char *line, smaps_vma_t *vma) * Both smaps field families share the same label/whitespace/overflow rules; * only the trailing unit differs ("kB" for the first group, none for the * remaining numeric fields). */ -static bool parse_numeric_field(const char *line, const char *label, +static bool parse_numeric_field(const char *line, + const char *label, const char *suffix, unsigned long long *value) { @@ -247,15 +247,14 @@ static void skip_optional_fields(const char *line, size_t *field_index) (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || !strncmp(line, "VmFlags:", 8))) (*field_index)++; - if (*field_index == SMAPS_FIELD_COUNT - 2 && - !strncmp(line, "VmFlags:", 8)) + if (*field_index == SMAPS_FIELD_COUNT - 2 && !strncmp(line, "VmFlags:", 8)) (*field_index)++; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) { - smaps_vma_t *vmas = realloc(info->vmas, - (info->count + 1) * sizeof(*info->vmas)); + smaps_vma_t *vmas = + realloc(info->vmas, (info->count + 1) * sizeof(*info->vmas)); if (!vmas) return false; vmas[info->count++] = *vma; @@ -298,8 +297,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) goto fail; } if (info->count > 0 && - (header.start < previous_end || header.start <= - info->vmas[info->count - 1].start)) + (header.start < previous_end || + header.start <= info->vmas[info->count - 1].start)) goto fail; current = header; have_current = true; @@ -347,8 +346,7 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) } if (!have_current || !finish_vma(¤t, field_index) || - !append_vma(info, ¤t) || - info->count == 0) + !append_vma(info, ¤t) || info->count == 0) goto fail; return true; @@ -386,7 +384,8 @@ static const smaps_vma_t *find_vma(const smaps_info_t *info, uintptr_t address) return NULL; } -static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, +static size_t count_vmas_in_range(const smaps_info_t *info, + uintptr_t start, uintptr_t end) { size_t count = 0; @@ -397,8 +396,10 @@ static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, return count; } -static bool validate_layout(const smaps_info_t *info, uintptr_t target, - uintptr_t stress, size_t page_size, +static bool validate_layout(const smaps_info_t *info, + uintptr_t target, + uintptr_t stress, + size_t page_size, size_t stress_size) { if (info->count <= 256) @@ -421,11 +422,11 @@ static bool validate_layout(const smaps_info_t *info, uintptr_t target, strcmp(last->perms, "rw-p")) return false; if (first->size_kb != page_kb || middle->size_kb != page_kb || - last->size_kb != page_kb || !first->vmflags_wr || - middle->vmflags_wr || !last->vmflags_wr || - first->kernel_page_kb != 4 || first->mmu_page_kb != 4 || - middle->kernel_page_kb != 4 || middle->mmu_page_kb != 4 || - last->kernel_page_kb != 4 || last->mmu_page_kb != 4) + last->size_kb != page_kb || !first->vmflags_wr || middle->vmflags_wr || + !last->vmflags_wr || first->kernel_page_kb != 4 || + first->mmu_page_kb != 4 || middle->kernel_page_kb != 4 || + middle->mmu_page_kb != 4 || last->kernel_page_kb != 4 || + last->mmu_page_kb != 4) return false; /* Every stress-map page alternates permissions, so each page must remain @@ -461,7 +462,9 @@ static bool read_exact(int fd, void *data, size_t len) return true; } -static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, +static int child_probe(uintptr_t target, + uintptr_t stress, + size_t page_size, size_t stress_size) { char pid_path[64]; @@ -472,8 +475,8 @@ static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, smaps_info_t info; if (!load_smaps(paths[i], &info)) return 1; - bool ok = validate_layout(&info, target, stress, page_size, - stress_size); + bool ok = + validate_layout(&info, target, stress, page_size, stress_size); const smaps_vma_t *first = find_vma(&info, target); const smaps_vma_t *middle = find_vma(&info, target + page_size); const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); @@ -536,7 +539,8 @@ int main(void) FAIL("mmap/mprotect fixture"); } else { char pid_path[64]; - snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", + (long) getpid()); const char *paths[] = {"/proc/self/smaps", pid_path}; bool ok = true; size_t expected_count = 0; @@ -544,8 +548,8 @@ int main(void) smaps_info_t info; bool parsed = load_smaps(paths[i], &info); if (!parsed || - !validate_layout(&info, (uintptr_t) target, - (uintptr_t) stress, page_size, stress_size)) { + !validate_layout(&info, (uintptr_t) target, (uintptr_t) stress, + page_size, stress_size)) { ok = false; } else if (i == 0) { expected_count = info.count; @@ -584,8 +588,8 @@ int main(void) close(pipefd[0]); int status = 0; bool waited = waitpid(pid, &status, 0) == pid; - EXPECT_TRUE(received && waited && result == 0 && WIFEXITED(status) && - WEXITSTATUS(status) == 0, + EXPECT_TRUE(received && waited && result == 0 && + WIFEXITED(status) && WEXITSTATUS(status) == 0, "fork Shared_Dirty accounting"); } } diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index e926bb1b..91f642d6 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -116,7 +116,7 @@ static void test_growth_preserves_content(void) assert(string_builder_init(&builder, 1) == 0); for (size_t i = 0; i < COUNT; i++) { - expected[i] = (char)('A' + (i % 26)); + expected[i] = (char) ('A' + (i % 26)); char chunk[2] = {expected[i], '\0'}; assert(string_builder_append(&builder, chunk) == 0); } @@ -147,7 +147,8 @@ static void test_overflow_preserves_content(void) char snapshot[32]; assert(string_builder_length(&builder) < sizeof(snapshot)); - memcpy(snapshot, string_builder_data_const(&builder), string_builder_length(&builder)); + memcpy(snapshot, string_builder_data_const(&builder), + string_builder_length(&builder)); size_t old_length = string_builder_length(&builder); size_t old_capacity = string_builder_capacity(&builder); @@ -156,7 +157,8 @@ static void test_overflow_preserves_content(void) assert(errno == EOVERFLOW); assert(string_builder_length(&builder) == old_length); assert(string_builder_capacity(&builder) == old_capacity); - assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); assert(string_builder_data_const(&builder)[old_length] == '\0'); errno = 0; @@ -164,7 +166,8 @@ static void test_overflow_preserves_content(void) assert(errno == EILSEQ); assert(string_builder_length(&builder) == old_length); assert(string_builder_capacity(&builder) == old_capacity); - assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); assert(string_builder_data_const(&builder)[old_length] == '\0'); string_builder_destroy(&builder); } diff --git a/tests/test-util.h b/tests/test-util.h index abe5e433..3357ef66 100644 --- a/tests/test-util.h +++ b/tests/test-util.h @@ -68,7 +68,8 @@ static inline ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) /* Read a file whose size is not available from st_size (for example a proc * file) until EOF, growing the buffer and appending a NUL terminator. */ -static inline ssize_t read_file_dynamic_nul(const char *path, char **buf_out, +static inline ssize_t read_file_dynamic_nul(const char *path, + char **buf_out, size_t *len_out) { if (!path || !buf_out || !len_out) { From a13ac757bc0bd36a96bad2232bc9973bbfd1e1b9 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:38:41 +0800 Subject: [PATCH 14/26] Address smaps review comments Apply review feedback to the smaps emulation and its regression tests. --- src/dynamic-array.c | 45 +++++---- src/dynamic-array.h | 19 +++- src/runtime/procemu.c | 96 +++++++++++------- src/string-builder.c | 163 ++++++++++--------------------- src/string-builder.h | 6 +- tests/test-dynamic-array-host.c | 11 +++ tests/test-string-builder-host.c | 28 ++++++ 7 files changed, 201 insertions(+), 167 deletions(-) diff --git a/src/dynamic-array.c b/src/dynamic-array.c index aad9dda4..50db79c9 100644 --- a/src/dynamic-array.c +++ b/src/dynamic-array.c @@ -81,27 +81,31 @@ static int dynamic_array_source_offset(const dynamic_array_t *array, return 1; } -/* Initialize only metadata; allocate storage separately when requested. */ +/* Initialize only metadata; allocate storage separately when requested. + * + * This is a fresh-initialization operation and deliberately does not inspect + * prior object contents, so an automatic, uninitialized object is safe. Use + * dynamic_array_destroy before reinitializing an array that already owns + * storage; otherwise that allocation is intentionally abandoned. + */ int dynamic_array_init(dynamic_array_t *array, size_t element_size) { if (array == NULL || element_size == 0) return dynamic_array_invalid(); - /* There is no allocation to fail here. Preserve storage only when the - * caller is reinitializing an array with the same element type. */ - if (array->data != NULL && array->element_size != element_size) { - errno = EINVAL; - return -1; - } - if (array->data == NULL) { - array->count = 0; - array->capacity = 0; - } - array->element_size = element_size; + *array = (dynamic_array_t) { + .element_size = element_size, + }; return 0; } -/* Initialize and reserve storage for an initial number of elements. */ +/* Initialize and reserve storage for an initial number of elements. + * + * Like dynamic_array_init, this is a fresh-initialization operation that does + * not inspect prior object contents. In particular, do not free an + * indeterminate pointer from an automatic object; destroy an existing array + * before reinitializing it. + */ int dynamic_array_init_with_capacity(dynamic_array_t *array, size_t element_size, size_t initial_capacity) @@ -109,6 +113,11 @@ int dynamic_array_init_with_capacity(dynamic_array_t *array, if (array == NULL || element_size == 0) return dynamic_array_invalid(); + /* Establish a safe zero state before any fallible allocation. This is a + * fresh initializer, so an existing allocation must have been destroyed + * by the caller rather than silently leaked here. */ + *array = (dynamic_array_t) {0}; + size_t bytes; if (initial_capacity != 0 && element_size > SIZE_MAX / initial_capacity) { errno = EOVERFLOW; @@ -124,11 +133,11 @@ int dynamic_array_init_with_capacity(dynamic_array_t *array, } } - free(array->data); - array->data = storage; - array->count = 0; - array->capacity = initial_capacity; - array->element_size = element_size; + *array = (dynamic_array_t) { + .data = storage, + .capacity = initial_capacity, + .element_size = element_size, + }; return 0; } diff --git a/src/dynamic-array.h b/src/dynamic-array.h index 3899194a..fb4c7e3c 100644 --- a/src/dynamic-array.h +++ b/src/dynamic-array.h @@ -43,10 +43,14 @@ static inline int dynamic_array_typed_prepare(dynamic_array_t *array, return was_uninitialized; } -/* Initialize an array for elements of element_size bytes without allocation. */ +/* Initialize an array for elements of element_size bytes without allocation. + * This fresh initializer is safe on an uninitialized automatic object; destroy + * an existing array before reinitializing it. */ int dynamic_array_init(dynamic_array_t *array, size_t element_size); -/* Initialize and reserve initial_capacity element slots. */ +/* Initialize and reserve initial_capacity element slots. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing array before reinitializing it. */ int dynamic_array_init_with_capacity(dynamic_array_t *array, size_t element_size, size_t initial_capacity); @@ -94,19 +98,26 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); /* Generate a small type-safe facade over the raw container. The facade owns * no additional state; all growth and copying remains in dynamic-array.c. + * A typed object must be zero-initialized before first-touch operations such + * as append, insert, reserve, or resize; the explicit init functions are safe + * on a fresh automatic object and establish the metadata themselves. */ #define DYNAMIC_ARRAY_DEFINE(name, type) \ typedef struct name { \ dynamic_array_t raw; \ } name##_t; \ \ - /* Initialize the typed array with no preallocated storage. */ \ + /* Initialize the typed array with no preallocated storage. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ DYNAMIC_ARRAY_INLINE int name##_init(name##_t *array) \ { \ return dynamic_array_init(array != NULL ? &array->raw : NULL, \ sizeof(type)); \ } \ - /* Initialize typed array and preallocate initial slots. */ \ + /* Initialize typed array and preallocate initial slots. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ DYNAMIC_ARRAY_INLINE int name##_init_with_capacity( \ name##_t *array, size_t initial_capacity) \ { \ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index ccc7ba5d..17adafda 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -19,7 +19,8 @@ * bounded. */ #define MAPS_ENTRY_INITIAL_CAP 64 -#define MAPS_ENTRY_MAX (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED) +#define MAPS_ENTRY_MAX \ + (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED * (GUEST_MAX_REGIONS + 1)) /* Bound the transient host-PID snapshot used by /proc/net enumeration. This * is an output-work limit, not the dynamically growing lifecycle-table cap. @@ -171,6 +172,58 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) (void) maps_entries_resize(entries, out + 1); } +/* Add only the portions of a preannounced interval not covered by live VMAs. + * A shadow VMA must never overlap a realized VMA: strict smaps consumers treat + * overlapping headers as a malformed snapshot. Both inputs are page-rounded + * because that is the granularity exposed by /proc/self/maps. */ +static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, + const guest_region_t *shadow, + const guest_region_t *live_regions, + int nlive) +{ + uint64_t shadow_start = shadow->start & ~0xFFFULL; + uint64_t shadow_end = (shadow->end + 0xFFFULL) & ~0xFFFULL; + if (shadow_end <= shadow_start) + return 0; + + uint64_t cursor = shadow_start; + for (int i = 0; i < nlive && cursor < shadow_end; i++) { + uint64_t live_start = live_regions[i].start & ~0xFFFULL; + uint64_t live_end = (live_regions[i].end + 0xFFFULL) & ~0xFFFULL; + if (live_end <= live_start || live_end <= cursor) + continue; + if (live_start >= shadow_end) + break; + + if (live_start > cursor) { + uint64_t gap_end = + live_start < shadow_end ? live_start : shadow_end; + uint64_t offset = shadow->offset; + uint64_t delta = cursor - shadow_start; + if (delta <= UINT64_MAX - offset) + offset += delta; + if (maps_entries_insert_sorted(entries, cursor, gap_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) + return -1; + } + if (live_end > cursor) + cursor = live_end; + } + + if (cursor < shadow_end) { + uint64_t offset = shadow->offset; + uint64_t delta = cursor - shadow_start; + if (delta <= UINT64_MAX - offset) + offset += delta; + if (maps_entries_insert_sorted(entries, cursor, shadow_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) + return -1; + } + return 0; +} + /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily * on first access (Java GC, Go runtime, libnuma probe these to size thread * pools). Layout matches the minimal subset Linux exposes: @@ -2401,9 +2454,8 @@ static int pty_open_master(int linux_flags) } /* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges - * contiguous regions[] runs that came from one mmap, then folds in - * preannounced[] shadow entries whose advertised interval is not yet fully - * covered by live regions. + * contiguous regions[] runs that came from one mmap, then folds in the + * uncovered pieces of preannounced[] shadow entries around live coverage. * * The region and preannounced arrays are mutable from guest mmap/mprotect/ * munmap operations. Snapshot and merge while mmap_lock is held, then release @@ -2450,12 +2502,10 @@ static int proc_build_maps_entries(const guest_t *g, goto out_unlock; } - /* Add preannounced entries only while they still have an uncovered tail. - * Once the union of live regions covers the full advertised interval, - * suppress the shadow entry so /proc/self/maps shows only the realized - * split VMAs. A partial union must stay visible because some - * reserved-but-not-realized span remains to advertise. - */ + /* Add only uncovered portions of each preannounced interval. Keeping the + * shadow VMA whole when a live mapping realizes its middle produces + * overlapping maps/smaps headers; subtract every covered live interval + * instead, preserving any reserved-but-not-realized gaps. */ int npreannounced = g->npreannounced; if (npreannounced < 0) npreannounced = 0; @@ -2463,30 +2513,8 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - bool shadowed = false; - uint64_t covered_end = r->start; - - for (int j = 0; j < nregions; j++) { - const guest_region_t *live = &g->regions[j]; - - if (live->end <= covered_end) - continue; - if (live->start > covered_end) - break; - - covered_end = live->end; - if (covered_end >= r->end) { - shadowed = true; - break; - } - } - - if (shadowed) - continue; - - if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, - r->flags, r->offset, r->name) < 0) + if (maps_entries_insert_shadow_gaps(&entries, r, g->regions, nregions) < + 0) goto out_unlock; } maps_entries_merge_adjacent(&entries); diff --git a/src/string-builder.c b/src/string-builder.c index fa134065..c90b8b40 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -20,13 +21,16 @@ static int string_builder_invalid(void) return -1; } -/* Initialize storage and establish an empty, NUL-terminated builder. */ +/* Initialize storage and establish an empty, NUL-terminated builder. This + * fresh initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. + */ int string_builder_init(string_builder_t *builder, size_t initial_capacity) { if (builder == NULL) return string_builder_invalid(); + builder->storage = (string_builder_storage_t) {0}; if (initial_capacity == 0) { - string_builder_destroy(builder); return 0; } if (string_builder_storage_init_with_capacity(&builder->storage, @@ -131,90 +135,19 @@ static int string_builder_format_failure(void) return -1; } -/* Call vsnprintf with a copied argument list for the current sizing pass. */ -static int string_builder_vsnprintf(char *destination, - size_t available, - const char *format, - va_list arguments) -{ - va_list pass; - va_copy(pass, arguments); - errno = 0; -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" - int result = vsnprintf(destination, available, format, pass); -#pragma clang diagnostic pop - va_end(pass); - return result; -} - -typedef enum string_builder_format_status { - STRING_BUILDER_FORMAT_ERROR = -1, - STRING_BUILDER_FORMAT_FIT = 0, - STRING_BUILDER_FORMAT_TRUNCATED = 1 -} string_builder_format_status_t; - -/* Ensure the builder has a writable tail and capture its current length. */ -static int string_builder_prepare_appendf(string_builder_t *builder, - size_t *old_len) -{ - if (string_builder_capacity(builder) == 0 && - string_builder_reserve(builder, 1) < 0) - return -1; - - char *data = string_builder_data(builder); - size_t capacity = string_builder_capacity(builder); - *old_len = string_builder_storage_count(&builder->storage); - if (data == NULL || *old_len >= capacity) - return string_builder_invalid(); - return 0; -} - -/* Format into the current tail and report whether the result was truncated. */ -static string_builder_format_status_t string_builder_try_format( - string_builder_t *builder, - size_t old_len, - const char *format, - va_list arguments, - size_t *written) -{ - char *data = string_builder_data(builder); - size_t available = string_builder_capacity(builder) - old_len; - int formatted = - string_builder_vsnprintf(data + old_len, available, format, arguments); - if (formatted < 0) - return STRING_BUILDER_FORMAT_ERROR; - - size_t visible = strlen(data + old_len); - if ((size_t) formatted < available || visible < available - 1) { - *written = visible; - return STRING_BUILDER_FORMAT_FIT; - } - - *written = (size_t) formatted; - return STRING_BUILDER_FORMAT_TRUNCATED; -} - -/* Restore the terminator after a failed formatting attempt. */ -static void string_builder_rollback_appendf(string_builder_t *builder, - size_t old_len) -{ - char *data = string_builder_data(builder); - if (data != NULL && old_len < string_builder_capacity(builder)) - data[old_len] = '\0'; -} - -/* Append printf-formatted text, growing and retrying when the first pass is - * truncated. +/* Format into separate storage before touching the builder. Besides avoiding + * writes through an aliased format string, this keeps %s arguments that point + * into the builder valid even when appending the result grows the allocation. */ int string_builder_appendf(string_builder_t *builder, const char *format, ...) { - int result = -1; int saved_errno; - size_t old_len = 0; - size_t written = 0; - string_builder_format_status_t status; + int formatted_len; + size_t visible_len; + char *formatted = NULL; va_list arguments; + va_list sizing; + va_list rendering; if (builder == NULL || format == NULL) return string_builder_invalid(); @@ -222,45 +155,57 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) saved_errno = errno; va_start(arguments, format); - if (string_builder_prepare_appendf(builder, &old_len) < 0) + va_copy(sizing, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + formatted_len = vsnprintf(NULL, 0, format, sizing); +#pragma clang diagnostic pop + va_end(sizing); + if (formatted_len < 0) { + string_builder_format_failure(); goto out; + } - status = string_builder_try_format(builder, old_len, format, arguments, - &written); - if (status == STRING_BUILDER_FORMAT_ERROR) { - string_builder_format_failure(); - goto rollback; + formatted = malloc((size_t) formatted_len + 1); + if (formatted == NULL) { + errno = ENOMEM; + goto out; } - if (status == STRING_BUILDER_FORMAT_TRUNCATED) { - if (string_builder_reserve(builder, written) < 0) - goto rollback; - status = string_builder_try_format(builder, old_len, format, arguments, - &written); - if (status == STRING_BUILDER_FORMAT_ERROR) { - string_builder_format_failure(); - goto rollback; - } - if (status == STRING_BUILDER_FORMAT_TRUNCATED) { - errno = EOVERFLOW; - goto rollback; - } + va_copy(rendering, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + int rendered_len = + vsnprintf(formatted, (size_t) formatted_len + 1, format, rendering); +#pragma clang diagnostic pop + va_end(rendering); + if (rendered_len < 0 || rendered_len != formatted_len) { + string_builder_format_failure(); + goto out; } - if (string_builder_commit_append( - builder, string_builder_data(builder) + old_len, written) < 0) - goto rollback; + /* Preserve the builder's C-string semantics for formatted NUL bytes. */ + visible_len = strlen(formatted); + if (visible_len == 0) { + errno = saved_errno; + free(formatted); + va_end(arguments); + return 0; + } + if (string_builder_commit_append(builder, formatted, visible_len) < 0) + goto out; errno = saved_errno; - result = 0; - goto out; - -rollback: - string_builder_rollback_appendf(builder, old_len); + free(formatted); + va_end(arguments); + return 0; out: + free(formatted); va_end(arguments); - return result; + return -1; } /* Return mutable storage for the builder, if it has been allocated. */ diff --git a/src/string-builder.h b/src/string-builder.h index f9fb1c0c..c0942071 100644 --- a/src/string-builder.h +++ b/src/string-builder.h @@ -22,8 +22,10 @@ typedef struct string_builder { } string_builder_t; /* Initialize a builder, reserving initial_capacity bytes including the NUL. - * A zero capacity leaves storage unallocated for lazy growth. Returns 0 on - * success or -1 with errno set on invalid input or allocation failure. + * A zero capacity leaves storage unallocated for lazy growth. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. Returns 0 on success or -1 with + * errno set on invalid input or allocation failure. */ int string_builder_init(string_builder_t *builder, size_t initial_capacity); diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c index 6a3ea3ce..0ca09eb1 100644 --- a/tests/test-dynamic-array-host.c +++ b/tests/test-dynamic-array-host.c @@ -23,6 +23,17 @@ DYNAMIC_ARRAY_DEFINE(odd_array, odd_t) static void test_zero_and_init(void) { + int_array_t fresh; + assert(int_array_init_with_capacity(&fresh, 4) == 0); + assert(int_array_capacity(&fresh) >= 4); + int_array_destroy(&fresh); + + int_array_t fresh_lazy; + assert(int_array_init(&fresh_lazy) == 0); + assert(int_array_append_value(&fresh_lazy, 11) == 0); + assert(*int_array_at(&fresh_lazy, 0) == 11); + int_array_destroy(&fresh_lazy); + int_array_t values = {0}; int value = 7; assert(int_array_count(&values) == 0); diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index 91f642d6..073958ed 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -29,10 +29,24 @@ static void expect_string(const string_builder_t *builder, const char *expected) static void test_zero_and_initial_capacity(void) { + /* Fresh automatic objects need not be manually zeroed before init. */ + string_builder_t fresh; + assert(string_builder_init(&fresh, 0) == 0); + assert(string_builder_append(&fresh, "fresh") == 0); + expect_string(&fresh, "fresh"); + string_builder_destroy(&fresh); + + string_builder_t fresh_capacity; + assert(string_builder_init(&fresh_capacity, 32) == 0); + assert(string_builder_capacity(&fresh_capacity) >= 32); + string_builder_destroy(&fresh_capacity); + string_builder_t zero = {0}; assert(string_builder_data(&zero) == NULL); assert(string_builder_length(&zero) == 0); assert(string_builder_capacity(&zero) == 0); + assert(string_builder_appendf(&zero, "%c", '\0') == 0); + assert(string_builder_data(&zero) == NULL); /* The public API also accepts a plain {0} value without an init call. */ assert(string_builder_append(&zero, "zero") == 0); expect_string(&zero, "zero"); @@ -139,6 +153,19 @@ static void test_alias_append(void) string_builder_destroy(&builder); } +static void test_formatted_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "x%s") == 0); + const char *alias = string_builder_data_const(&builder); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + assert(string_builder_appendf(&builder, alias, alias) == 0); +#pragma clang diagnostic pop + expect_string(&builder, "x%sxx%s"); + string_builder_destroy(&builder); +} + static void test_overflow_preserves_content(void) { string_builder_t builder = {0}; @@ -179,6 +206,7 @@ int main(void) test_c_string_semantics(); test_growth_preserves_content(); test_alias_append(); + test_formatted_alias_append(); test_overflow_preserves_content(); puts("test-string-builder-host: PASS"); return 0; From 68512ff526c131f267f6f77c14039c8e17354428 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:45:08 +0800 Subject: [PATCH 15/26] Fix string builder capacity warning Grow formatted string storage safely and silence the Infer warning exposed by the smaps changes. --- src/string-builder.c | 8 +++++++- tests/test-string-builder-host.c | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/string-builder.c b/src/string-builder.c index c90b8b40..97d7bec9 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -148,6 +148,7 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) va_list arguments; va_list sizing; va_list rendering; + char sizing_sink; if (builder == NULL || format == NULL) return string_builder_invalid(); @@ -159,7 +160,10 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) errno = 0; #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wformat-nonliteral" - formatted_len = vsnprintf(NULL, 0, format, sizing); + /* A non-NULL destination keeps static analyzers from treating the + * standards-sanctioned n == 0 sizing call as a null dereference. The + * destination is never written when its size is zero. */ + formatted_len = vsnprintf(&sizing_sink, 0, format, sizing); #pragma clang diagnostic pop va_end(sizing); if (formatted_len < 0) { @@ -194,6 +198,8 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) va_end(arguments); return 0; } + if (string_builder_reserve(builder, visible_len) < 0) + goto out; if (string_builder_commit_append(builder, formatted, visible_len) < 0) goto out; diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index 073958ed..e404680c 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -122,6 +122,22 @@ static void test_c_string_semantics(void) string_builder_destroy(&fit); } +static void test_formatted_append_reserves_terminator(void) +{ + string_builder_t builder = {0}; + + /* A formatted append must reserve one byte beyond the visible payload for + * the builder's trailing NUL. This exact-length payload used to make the + * generic array allocate only the payload bytes before the terminator was + * written. + */ + assert(string_builder_appendf(&builder, "%s", "12345678") == 0); + expect_string(&builder, "12345678"); + assert(string_builder_capacity(&builder) >= + string_builder_length(&builder) + 1); + string_builder_destroy(&builder); +} + static void test_growth_preserves_content(void) { enum { COUNT = 4096 }; @@ -204,6 +220,7 @@ int main(void) test_zero_and_initial_capacity(); test_text_and_formatted_append(); test_c_string_semantics(); + test_formatted_append_reserves_terminator(); test_growth_preserves_content(); test_alias_append(); test_formatted_alias_append(); From 5163c26ca7a4c6dc0d7b25df67dca8580bb89784 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:50:26 +0800 Subject: [PATCH 16/26] Use unique names for concurrent shm tests Avoid collisions when concurrent shared-memory tests create temporary names. --- tests/test-dev-shm-paths.c | 53 ++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/tests/test-dev-shm-paths.c b/tests/test-dev-shm-paths.c index 728aae9d..bcb68252 100644 --- a/tests/test-dev-shm-paths.c +++ b/tests/test-dev-shm-paths.c @@ -51,19 +51,42 @@ static char shm_dir[128]; static char shm_fifo[128]; static char shm_exec[128]; static char victim_path[128]; +static char fixture_suffix[16]; -static void name_fixtures(void) +/* Guest PIDs restart from the same value in each independent elfuse process. + * Reserve a host-unique suffix so concurrent runtime jobs cannot unlink or + * replace one another's /dev/shm fixtures. */ +static int name_fixtures(void) { - int pid = (int) getpid(); - snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%d", pid); - snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%d", pid); - snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%d", pid); - snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%d", pid); - snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%d", pid); - snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%d", pid); - snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%d", pid); - snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%d", - pid); + char seed[] = "/tmp/elfuse-shm-seed-XXXXXX"; + int fd = mkstemp(seed); + if (fd < 0) + return -1; + close(fd); + (void) unlink(seed); + + const char *suffix = strrchr(seed, '-'); + if (suffix == NULL || suffix[1] == '\0') + return -1; + snprintf(fixture_suffix, sizeof(fixture_suffix), "%s", suffix + 1); + + snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%s", + fixture_suffix); + snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%s", + fixture_suffix); + snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%s", + fixture_suffix); + snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%s", + fixture_suffix); + snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", + fixture_suffix); + snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%s", + fixture_suffix); + snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%s", + fixture_suffix); + snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%s", + fixture_suffix); + return 0; } static void cleanup_fixtures(void) @@ -724,7 +747,7 @@ static void test_dotdot_bearing_names_allowed(void) static const char *names[] = {"a..b", "..a", "a..", "..."}; for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { char p[128]; - snprintf(p, sizeof(p), SHM_DIR "elfuse_%d_%s", (int) getpid(), + snprintf(p, sizeof(p), SHM_DIR "elfuse_%s_%s", fixture_suffix, names[i]); int fd = open(p, O_CREAT | O_EXCL | O_RDWR, 0600); if (fd < 0) { @@ -775,7 +798,11 @@ int main(int argc, char **argv) printf("test-dev-shm-paths: /dev/shm path-syscall consistency\n"); - name_fixtures(); + if (name_fixtures() < 0) { + FAIL("reserve unique fixture suffix"); + SUMMARY("test-dev-shm-paths"); + return 1; + } cleanup_fixtures(); if (test_open_then_chmod() == 0) { From 486c124f352129b89c7179f454c93ddd7f5143d3 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:52:35 +0800 Subject: [PATCH 17/26] Format concurrent shm fixture setup Keep concurrent shared-memory fixture setup consistent with repository formatting conventions. --- tests/test-dev-shm-paths.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test-dev-shm-paths.c b/tests/test-dev-shm-paths.c index bcb68252..dff9e9ff 100644 --- a/tests/test-dev-shm-paths.c +++ b/tests/test-dev-shm-paths.c @@ -78,8 +78,7 @@ static int name_fixtures(void) fixture_suffix); snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%s", fixture_suffix); - snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", - fixture_suffix); + snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", fixture_suffix); snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%s", fixture_suffix); snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%s", From 8fe5d24355b7d9e6528bf324398dac3ba6a2d276 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 11:34:18 +0800 Subject: [PATCH 18/26] Optimize fragmented proc snapshots Append live and shadow VMAs, then sort and merge once. This keeps fragmented maps and smaps snapshots from quadratic insertion work. --- src/runtime/procemu.c | 73 +++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 17adafda..91fccb61 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -104,19 +104,24 @@ typedef struct { int prot, flags; uint64_t offset; char name[64]; + /* Preserve the producer order for equal-start entries when qsort() is + * used below. Existing snapshots placed equal-start entries after one + * another in append order, and keeping that order avoids changing the + * handling of malformed/overlapping shadow metadata. */ + size_t order; } maps_entry_t; /* A growable VMA snapshot. The generated facade keeps element-size and * allocation bookkeeping in the generic array implementation. */ DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) -static int maps_entries_insert_sorted(maps_entries_t *entries, - uint64_t start, - uint64_t end, - int prot, - int flags, - uint64_t offset, - const char *name) +static int maps_entries_append_entry(maps_entries_t *entries, + uint64_t start, + uint64_t end, + int prot, + int flags, + uint64_t offset, + const char *name) { if (end <= start) return 0; @@ -134,17 +139,34 @@ static int maps_entries_insert_sorted(maps_entries_t *entries, .prot = prot, .flags = flags, .offset = offset, + .order = maps_entries_count(entries), }; if (name && name[0]) str_copy_trunc(value.name, name, sizeof(value.name)); else value.name[0] = '\0'; - size_t i = maps_entries_count(entries); - while (i > 0 && maps_entries_at(entries, i - 1)->start > start) { - i--; - } - return maps_entries_insert(entries, i, &value); + return maps_entries_append_value(entries, value); +} + +/* The live-region and shadow-gap producers are each ordered, but their + * outputs interleave. Append both streams while holding mmap_lock and sort + * once after all gaps have been generated. This avoids shifting an already + * populated array for every split shadow gap (the old insertion path was + * quadratic for fragmented snapshots). */ +static int maps_entries_compare_start(const void *lhs, const void *rhs) +{ + const maps_entry_t *a = lhs; + const maps_entry_t *b = rhs; + if (a->start < b->start) + return -1; + if (a->start > b->start) + return 1; + if (a->order < b->order) + return -1; + if (a->order > b->order) + return 1; + return 0; } static void maps_entries_merge_adjacent(maps_entries_t *entries) @@ -176,7 +198,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) * A shadow VMA must never overlap a realized VMA: strict smaps consumers treat * overlapping headers as a malformed snapshot. Both inputs are page-rounded * because that is the granularity exposed by /proc/self/maps. */ -static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, +static int maps_entries_append_shadow_gaps(maps_entries_t *entries, const guest_region_t *shadow, const guest_region_t *live_regions, int nlive) @@ -202,9 +224,9 @@ static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_insert_sorted(entries, cursor, gap_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, gap_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) return -1; } if (live_end > cursor) @@ -216,9 +238,9 @@ static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_insert_sorted(entries, cursor, shadow_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, shadow_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) return -1; } return 0; @@ -2456,6 +2478,8 @@ static int pty_open_master(int linux_flags) /* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges * contiguous regions[] runs that came from one mmap, then folds in the * uncovered pieces of preannounced[] shadow entries around live coverage. + * Producers append entries while the lock is held; one sort/merge pass puts + * the interleaved live and shadow streams back into VMA order. * * The region and preannounced arrays are mutable from guest mmap/mprotect/ * munmap operations. Snapshot and merge while mmap_lock is held, then release @@ -2497,8 +2521,8 @@ static int proc_build_maps_entries(const guest_t *g, last->end = end; continue; } - if (maps_entries_insert_sorted(&entries, start, end, r->prot, r->flags, - r->offset, r->name) < 0) + if (maps_entries_append_entry(&entries, start, end, r->prot, r->flags, + r->offset, r->name) < 0) goto out_unlock; } @@ -2513,10 +2537,13 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - if (maps_entries_insert_shadow_gaps(&entries, r, g->regions, nregions) < - 0) + if (maps_entries_append_shadow_gaps(&entries, r, g->regions, + nregions) < 0) goto out_unlock; } + if (maps_entries_count(&entries) > 1) + qsort(maps_entries_data(&entries), maps_entries_count(&entries), + sizeof(maps_entry_t), maps_entries_compare_start); maps_entries_merge_adjacent(&entries); result = (int) maps_entries_count(&entries); From 8fcb1af6840ce6da9e7be219f7e011970ec42fed Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 11:43:13 +0800 Subject: [PATCH 19/26] Format proc VMA snapshot calls Apply repository formatting conventions to proc VMA snapshot calls. --- src/runtime/procemu.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 91fccb61..d4bd5b7b 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -238,9 +238,8 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_append_entry(entries, cursor, shadow_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, shadow_end, shadow->prot, + shadow->flags, offset, shadow->name) < 0) return -1; } return 0; @@ -2537,8 +2536,8 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - if (maps_entries_append_shadow_gaps(&entries, r, g->regions, - nregions) < 0) + if (maps_entries_append_shadow_gaps(&entries, r, g->regions, nregions) < + 0) goto out_unlock; } if (maps_entries_count(&entries) > 1) From 8b32fd4de39df94bae8b5fd205c211021fde5cde Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 15:47:31 +0800 Subject: [PATCH 20/26] Make smaps Shared_Dirty fork-aware Track whether each guest VMA existed at the fork snapshot so post-fork mappings are excluded from the synthetic Shared_Dirty compatibility signal. Propagate the metadata through fork IPC and memory-region transformations, and add a regression test for post-fork mappings. --- docs/internals.md | 15 +-- src/core/guest.c | 16 +++- src/core/guest.h | 15 ++- src/runtime/fork-state.c | 7 ++ src/runtime/fork-state.h | 2 +- src/runtime/procemu.c | 86 ++++++++++++----- src/syscall/mem.c | 138 ++++++++++++++++++++++------ tests/test-fork-ipc-protocol-host.c | 7 +- tests/test-matrix.sh | 4 +- tests/test-proc-smap.c | 22 ++++- 10 files changed, 237 insertions(+), 75 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index abb49fd2..3059fc32 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -809,18 +809,21 @@ ProtectionKey, VmFlags `Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a -non-zero value: in a fork child, writable private anonymous VMAs report their -full VMA size because they are logically shared with the parent's CoW -snapshot. Every other numeric counter is emitted as zero, including -`THPeligible` and `ProtectionKey`. `VmFlags` is evidence-based and contains -only the permission/sharing flags represented by the tracked VMA (`rd`, `wr`, +non-zero value: in a fork child, writable private anonymous VMAs that existed +in the parent's CoW snapshot report their full VMA size because they are +logically shared with that snapshot. Every other numeric counter, including +`THPeligible` and `ProtectionKey`, is emitted as zero. `VmFlags` is +evidence-based and contains only the permission/sharing flags represented by +the tracked VMA (`rd`, `wr`, `ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing against a real Linux kernel should tolerate either conditional field being absent. These are coarse VMA-level values, not host page-residency, dirty-bit, or proportional-sharing accounting, so they are suitable for fork-safety checks -but not precise memory profiling. `smaps_rollup` is not implemented. +but not precise memory profiling. The fork-child marker is tracked per VMA, so +writable private anonymous mappings created after fork are excluded from the +compatibility signal. `smaps_rollup` is not implemented. Synthetic proc directories have explicit snapshot boundaries. The backing trees reached by opening `/proc` or `/proc/self` are materialized once, on the diff --git a/src/core/guest.c b/src/core/guest.c index addc79f4..575d5d8d 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1870,6 +1870,8 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return false; if (a->backing_ro != b->backing_ro) return false; + if (a->inherited_at_fork != b->inherited_at_fork) + return false; if (a->overlay_active || b->overlay_active) return false; if (strcmp(a->name, b->name) != 0) @@ -1981,7 +1983,7 @@ int guest_region_add_ex(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false); } int guest_region_add_ex_gpa(guest_t *g, @@ -2002,7 +2004,7 @@ int guest_region_add_ex_gpa(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, gpa_base, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false); } int guest_region_add_ex_owned(guest_t *g, @@ -2012,10 +2014,12 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork) { return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, + inherited_at_fork); } int guest_region_add_ex_owned_gpa(guest_t *g, @@ -2026,7 +2030,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2055,6 +2060,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->shared = (flags & LINUX_MAP_SHARED) != 0; r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; + r->inherited_at_fork = inherited_at_fork; guest_region_clear_overlay(r); if (name) { str_copy_trunc(r->name, name, sizeof(r->name)); diff --git a/src/core/guest.h b/src/core/guest.h index 5e80d9ee..65267cda 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -242,6 +242,11 @@ typedef struct { * later PROT_WRITE request against it with EACCES, * matching a real kernel's VMA max_prot tracking. */ + bool inherited_at_fork; /* Region existed at the most recent fork + * snapshot. Used by synthetic smaps to report + * only VMAs that can participate in that fork's + * CoW snapshot as Shared_Dirty. + */ bool overlay_active; /* Region has a live host MAP_FIXED|MAP_SHARED overlay * of backing_fd at host_base+start. The kernel's page * cache keeps it coherent with the file and with peer @@ -1176,7 +1181,9 @@ int guest_region_add_ex_gpa(guest_t *g, int backing_fd); /* Like guest_region_add_ex, but consumes owned_backing_fd on success or - * failure. + * failure. inherited_at_fork identifies bytes copied from the latest fork + * snapshot; ordinary mmap/exec callers pass false, while fork/mremap restore + * paths preserve the source VMA's value. */ int guest_region_add_ex_owned(guest_t *g, uint64_t start, @@ -1185,7 +1192,8 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork); int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t start, uint64_t end, @@ -1194,7 +1202,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork); /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 43b75d49..9ff5a799 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -926,6 +926,13 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) return -1; g->nregions = (int) recv_regions; + /* Every VMA present in the serialized parent snapshot is inherited by + * this child, regardless of whether the parent itself created it after an + * earlier fork. New mappings added in this process start unmarked through + * guest_region_add_ex_owned[_gpa]. + */ + for (int i = 0; i < g->nregions; i++) + g->regions[i].inherited_at_fork = true; g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index d4a5663e..b6f8331e 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464EU /* "ELFN" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464FU /* "ELFO" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index d4bd5b7b..4aa46b9b 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -103,6 +103,7 @@ typedef struct { uint64_t start, end; int prot, flags; uint64_t offset; + bool inherited_at_fork; char name[64]; /* Preserve the producer order for equal-start entries when qsort() is * used below. Existing snapshots placed equal-start entries after one @@ -115,13 +116,45 @@ typedef struct { * allocation bookkeeping in the generic array implementation. */ DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) +/* Round a VMA endpoint up to the 4 KiB granularity exposed by procfs without + * allowing the addition to wrap. There is no representable page-aligned + * endpoint above UINT64_MAX - 0xFFF, so saturate to UINT64_MAX and let the + * caller's normal end <= start validation handle an empty interval. + */ +static uint64_t maps_align_up_page(uint64_t value) +{ + const uint64_t mask = 0xFFFULL; + if (value > UINT64_MAX - mask) + return UINT64_MAX; + return (value + mask) & ~mask; +} + +/* Translate a shadow VMA's cursor into its file offset. A wrapped offset would + * produce a syntactically valid but semantically wrong maps entry, so surface + * the overflow to the intercepted open instead. + */ +static int maps_shadow_offset(const guest_region_t *shadow, + uint64_t shadow_start, + uint64_t cursor, + uint64_t *offset_out) +{ + uint64_t delta = cursor - shadow_start; + if (delta > UINT64_MAX - shadow->offset) { + errno = EOVERFLOW; + return -1; + } + *offset_out = shadow->offset + delta; + return 0; +} + static int maps_entries_append_entry(maps_entries_t *entries, uint64_t start, uint64_t end, int prot, int flags, uint64_t offset, - const char *name) + const char *name, + bool inherited_at_fork) { if (end <= start) return 0; @@ -139,6 +172,7 @@ static int maps_entries_append_entry(maps_entries_t *entries, .prot = prot, .flags = flags, .offset = offset, + .inherited_at_fork = inherited_at_fork, .order = maps_entries_count(entries), }; if (name && name[0]) @@ -183,6 +217,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) current->prot == previous->prot && current->flags == previous->flags && current->offset == previous->offset && + current->inherited_at_fork == previous->inherited_at_fork && strcmp(current->name, previous->name) == 0) { previous->end = current->end; continue; @@ -204,14 +239,14 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, int nlive) { uint64_t shadow_start = shadow->start & ~0xFFFULL; - uint64_t shadow_end = (shadow->end + 0xFFFULL) & ~0xFFFULL; + uint64_t shadow_end = maps_align_up_page(shadow->end); if (shadow_end <= shadow_start) return 0; uint64_t cursor = shadow_start; for (int i = 0; i < nlive && cursor < shadow_end; i++) { uint64_t live_start = live_regions[i].start & ~0xFFFULL; - uint64_t live_end = (live_regions[i].end + 0xFFFULL) & ~0xFFFULL; + uint64_t live_end = maps_align_up_page(live_regions[i].end); if (live_end <= live_start || live_end <= cursor) continue; if (live_start >= shadow_end) @@ -220,13 +255,12 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, if (live_start > cursor) { uint64_t gap_end = live_start < shadow_end ? live_start : shadow_end; - uint64_t offset = shadow->offset; - uint64_t delta = cursor - shadow_start; - if (delta <= UINT64_MAX - offset) - offset += delta; - if (maps_entries_append_entry(entries, cursor, gap_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; + if (maps_entries_append_entry( + entries, cursor, gap_end, shadow->prot, shadow->flags, + offset, shadow->name, shadow->inherited_at_fork) < 0) return -1; } if (live_end > cursor) @@ -234,12 +268,12 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, } if (cursor < shadow_end) { - uint64_t offset = shadow->offset; - uint64_t delta = cursor - shadow_start; - if (delta <= UINT64_MAX - offset) - offset += delta; + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; if (maps_entries_append_entry(entries, cursor, shadow_end, shadow->prot, - shadow->flags, offset, shadow->name) < 0) + shadow->flags, offset, shadow->name, + shadow->inherited_at_fork) < 0) return -1; } return 0; @@ -2495,6 +2529,7 @@ static int proc_build_maps_entries(const guest_t *g, maps_entries_t entries = {0}; int result = -1; + int saved_errno = 0; pthread_mutex_lock(&mmap_lock); @@ -2510,18 +2545,20 @@ static int proc_build_maps_entries(const guest_t *g, for (int i = 0; i < nregions; i++) { const guest_region_t *r = &g->regions[i]; uint64_t start = r->start & ~0xFFFULL; - uint64_t end = (r->end + 0xFFFULL) & ~0xFFFULL; + uint64_t end = maps_align_up_page(r->end); size_t count = maps_entries_count(&entries); maps_entry_t *last = count > 0 ? maps_entries_at(&entries, count - 1) : NULL; if (last != NULL && last->end == start && last->prot == r->prot && last->flags == r->flags && last->offset == r->offset && + last->inherited_at_fork == r->inherited_at_fork && !strcmp(last->name, r->name)) { last->end = end; continue; } if (maps_entries_append_entry(&entries, start, end, r->prot, r->flags, - r->offset, r->name) < 0) + r->offset, r->name, + r->inherited_at_fork) < 0) goto out_unlock; } @@ -2547,10 +2584,11 @@ static int proc_build_maps_entries(const guest_t *g, result = (int) maps_entries_count(&entries); out_unlock: + saved_errno = errno; pthread_mutex_unlock(&mmap_lock); if (result < 0) { maps_entries_destroy(&entries); - errno = ENOMEM; + errno = saved_errno; return -1; } *entries_out = entries; @@ -2655,9 +2693,10 @@ static int proc_open_self_maps(const guest_t *g) /* Emit a Linux-shaped /proc/self/smaps approximation. The runtime tracks * guest VMAs and logical fork snapshots, but it cannot observe kernel page - * residency or dirty bits on the host. For a fork child, writable private - * anonymous VMAs therefore report their full VMA size as Shared_Dirty; all - * other fields that require kernel page accounting are stable zeroes. + * residency or dirty bits on the host. Writable private anonymous VMAs that + * were present in the most recent fork snapshot report their full VMA size as + * Shared_Dirty; newly-created VMAs are excluded. All other fields that + * require kernel page accounting are stable zeroes. */ static int proc_open_self_smaps(const guest_t *g) { @@ -2672,7 +2711,6 @@ static int proc_open_self_smaps(const guest_t *g) if (string_builder_init(&builder, initial_capacity) < 0) goto out; - bool fork_child = proc_get_ppid() != 0; for (int i = 0; i < nentries; i++) { const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char header[256]; @@ -2686,8 +2724,8 @@ static int proc_open_self_smaps(const guest_t *g) bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && anonymous && !shared; - bool logical_shared_dirty = - fork_child && private_anon && (e->prot & LINUX_PROT_WRITE); + bool logical_shared_dirty = e->inherited_at_fork && private_anon && + (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; char vmflags[64]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index a5477c5e..8fc47d90 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -83,6 +83,7 @@ typedef struct { uint64_t overlay_start; uint64_t overlay_end; bool backing_ro; + bool inherited_at_fork; char name[sizeof(((guest_region_t *) 0)->name)]; } region_snapshot_t; @@ -207,19 +208,67 @@ static void mark_overlay_metadata_range(guest_t *g, } } -/* Mark the region spanning exactly [start, end) as backed by a fd that lost - * write access, so sys_mprotect rejects a later PROT_WRITE upgrade. Exact match - * (not overlap) because callers use this right after installing a single - * freshly-added region. +/* Mark every region overlapping [start, end) as backed by a fd that lost write + * access, so sys_mprotect rejects a later PROT_WRITE upgrade. mremap can split + * an inherited VMA at the fork boundary, so callers must not require one + * exact region match here. */ static void mark_region_backing_ro(guest_t *g, uint64_t start, uint64_t end) { for (int i = 0; i < g->nregions; i++) { - if (g->regions[i].start == start && g->regions[i].end == end) { - g->regions[i].backing_ro = true; + if (g->regions[i].start >= end) break; + if (g->regions[i].end <= start) + continue; + g->regions[i].backing_ro = true; + } +} + +/* Track an mremap result without losing the fork boundary inside an in-place + * growth or a moved mapping. Bytes copied from the old VMA were present at the + * fork snapshot; an extension is new child-private address space and must + * remain unmarked. Keep the two portions as separate VMAs even when all other + * metadata matches so the synthetic smaps view can distinguish them. + */ +static int add_mremap_region(guest_t *g, + uint64_t start, + uint64_t old_size, + uint64_t new_size, + int prot, + int flags, + uint64_t offset, + const char *name, + int backing_fd, + bool inherited_at_fork) +{ + if (inherited_at_fork && old_size > 0 && old_size < new_size) { + int tail_backing_fd = -1; + if (backing_fd >= 0) { + tail_backing_fd = dup(backing_fd); + if (tail_backing_fd < 0) { + close(backing_fd); + return -1; + } + } + if (guest_region_add_ex_owned(g, start, start + old_size, prot, flags, + offset, name, backing_fd, true) < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return -1; + } + + if (guest_region_add_ex_owned(g, start + old_size, start + new_size, + prot, flags, offset + old_size, name, + tail_backing_fd, false) < 0) { + guest_region_remove(g, start, start + old_size); + return -1; } + return 0; } + + return guest_region_add_ex_owned(g, start, start + new_size, prot, flags, + offset, name, backing_fd, + inherited_at_fork && old_size > 0); } static void region_clip_overlay(guest_region_t *r) @@ -851,8 +900,8 @@ static int64_t sys_mmap_high_va(guest_t *g, replaced_region_removed = true; } if (guest_region_add_ex_owned_gpa(g, addr, addr + length, gpa_base, prot, - flags, offset, NULL, - track_backing_fd) < 0) + flags, offset, NULL, track_backing_fd, + false) < 0) goto fail; /* Ownership of track_backing_fd is now held by the new region. The fail * handler below skips closing when track_backing_fd < 0, so subsequent @@ -1133,6 +1182,7 @@ static int capture_region_snapshots(guest_t *g, snap->overlay_start = r->overlay_start; snap->overlay_end = r->overlay_end; snap->backing_ro = r->backing_ro; + snap->inherited_at_fork = r->inherited_at_fork; str_copy_trunc(snap->name, r->name, sizeof(snap->name)); } @@ -1239,7 +1289,7 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) if (guest_region_add_ex_owned_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd) < 0) { + snap->backing_fd, snap->inherited_at_fork) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -1829,7 +1879,26 @@ int64_t sys_brk(guest_t *g, uint64_t addr) for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start == g->brk_base && !strcmp(g->regions[i].name, "[heap]")) { - g->regions[i].end = new_off; + guest_region_t *heap = &g->regions[i]; + uint64_t old_heap_end = heap->end; + if (new_off > old_heap_end && heap->inherited_at_fork) { + /* Keep the fork-snapshot portion separate from pages + * materialized by this post-fork brk growth. If the + * semantic table is full, retain the historical + * conservative range and mark it stale rather than + * failing an otherwise successful brk syscall. + */ + if (guest_region_add( + g, old_heap_end, new_off, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, + "[heap]") < 0) { + heap->end = new_off; + g->regions_tracker_stale = true; + } + } else { + heap->end = new_off; + } found = true; break; } @@ -2559,7 +2628,7 @@ int64_t sys_mmap(guest_t *g, */ if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, track_backing_fd) < 0) { + NULL, track_backing_fd, false) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -2760,8 +2829,6 @@ int64_t sys_mremap(guest_t *g, {old_off, old_end}, {new_off, new_end}, }; - if (!region_has_capacity_after_removes(g, removed, 2, 1)) - return -LINUX_ENOMEM; /* Capture old region metadata BEFORE modifying any regions. If mremap * removed destination first, an overlapping source would lose its @@ -2779,6 +2846,14 @@ int64_t sys_mremap(guest_t *g, return -LINUX_ENOMEM; bool source_overlay = old_reg && region_has_live_overlay(old_reg); bool source_backing_ro = old_reg && old_reg->backing_ro; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; + int added_regions = + source_inherited_at_fork && old_size < new_size ? 2 : 1; + if (!region_has_capacity_after_removes(g, removed, 2, added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + return -LINUX_ENOMEM; + } uint64_t source_file_off = old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; char track_name[sizeof(old_reg->name)] = {0}; @@ -2948,9 +3023,9 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; } - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) { + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork) < 0) { (void) restore_region_snapshots(g, dest_snaps, dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); @@ -2993,9 +3068,6 @@ int64_t sys_mremap(guest_t *g, if (can_grow) { remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) - return -LINUX_ENOMEM; - /* Extend in place */ const guest_region_t *old_reg = guest_region_find(g, old_off); int prot = old_reg ? old_reg->prot @@ -3011,6 +3083,15 @@ int64_t sys_mremap(guest_t *g, uint64_t old_overlay_end = old_overlay ? old_reg->overlay_end : 0; bool old_backing_ro = old_reg && old_reg->backing_ro; + bool old_inherited_at_fork = + old_reg && old_reg->inherited_at_fork; + int added_regions = old_inherited_at_fork ? 2 : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, + added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + return -LINUX_ENOMEM; + } if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) return -LINUX_ENOMEM; char track_name[sizeof(old_reg->name)] = {0}; @@ -3028,10 +3109,10 @@ int64_t sys_mremap(guest_t *g, /* Update region tracking: remove old, add extended */ guest_region_remove(g, old_off, old_off + old_size); - if (guest_region_add_ex_owned(g, old_off, old_off + new_size, - prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, - track_backing_fd) < 0) + if (add_mremap_region( + g, old_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, old_inherited_at_fork) < 0) return -LINUX_ENOMEM; if (old_overlay) mark_overlay_metadata_range(g, old_off, old_off + old_size, @@ -3073,6 +3154,7 @@ int64_t sys_mremap(guest_t *g, source_overlay ? old_reg->overlay_start : 0; uint64_t source_overlay_end = source_overlay ? old_reg->overlay_end : 0; bool source_backing_ro = old_reg && old_reg->backing_ro; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; uint64_t source_file_off = old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; uint64_t source_overlay_file_off = @@ -3104,7 +3186,9 @@ int64_t sys_mremap(guest_t *g, } remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) { + int added_regions = + source_inherited_at_fork && old_size < new_size ? 2 : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); return -LINUX_ENOMEM; @@ -3189,9 +3273,9 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; /* Track new region */ - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork) < 0) return -LINUX_ENOMEM; if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 8dc4cc88..0f420098 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -19,9 +19,10 @@ #define LEGACY_ELFK_MAGIC 0x454C464BU #define PREVIOUS_ELFL_MAGIC 0x454C464CU #define PREVIOUS_ELFM_MAGIC 0x454C464DU +#define PREVIOUS_ELFN_MAGIC 0x454C464EU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464EU, - "fork IPC protocol magic must remain ELFN until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464FU, + "fork IPC protocol magic must remain ELFO until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -31,6 +32,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFL_MAGIC, "NOFILE header fields require rejecting ELFL peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFM_MAGIC, "start_stack header field requires rejecting ELFM peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFN_MAGIC, + "region fork metadata requires rejecting ELFN peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 1c2e16a2..f0f5ab54 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -363,8 +363,8 @@ QEMU_SKIP=" # under elfuse, while a real kernel allows the open and only rejects the # write -- a genuine behavioral difference worth reviewing on its own, # not just an environment artifact. -# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including a -# coarse Shared_Dirty compatibility signal for untouched writable VMAs. Real +# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including +# per-VMA Shared_Dirty inheritance and exclusion of post-fork VMAs. Real # Linux smaps exposes kernel-owned VMA/page accounting instead, so this is # intentionally not a reference-kernel invariant. diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 97a02e77..3563b22c 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -467,14 +467,21 @@ static int child_probe(uintptr_t target, size_t page_size, size_t stress_size) { + void *postfork = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (postfork == MAP_FAILED) + return 1; + char pid_path[64]; snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); const char *paths[] = {"/proc/self/smaps", pid_path}; for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { smaps_info_t info; - if (!load_smaps(paths[i], &info)) + if (!load_smaps(paths[i], &info)) { + munmap(postfork, page_size); return 1; + } bool ok = validate_layout(&info, target, stress, page_size, stress_size); const smaps_vma_t *first = find_vma(&info, target); @@ -482,15 +489,20 @@ static int child_probe(uintptr_t target, const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); const smaps_vma_t *stress_ro = find_vma(&info, stress); const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || - first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || - last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || - stress_rw->shared_dirty_kb == 0) { + !postfork_vma || first->shared_dirty_kb == 0 || + middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || + stress_ro->shared_dirty_kb != 0 || + stress_rw->shared_dirty_kb == 0 || + postfork_vma->shared_dirty_kb != 0) { free_smaps(&info); + munmap(postfork, page_size); return 1; } free_smaps(&info); } + munmap(postfork, page_size); return 0; } @@ -562,7 +574,7 @@ int main(void) EXPECT_TRUE(ok, "smaps parser/layout/completeness"); } - TEST("fork Shared_Dirty positive and read-only negative"); + TEST("fork Shared_Dirty inheritance and post-fork exclusion"); if (!fixture_ok) { FAIL("fixture unavailable"); } else { From 86f3c0ab017f272fd841b3a541187458b52bd007 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 23:46:58 +0800 Subject: [PATCH 21/26] Extend fork-aware VMA tracking Preserve VMA lineage across fork-split mappings and keep synthetic smaps accounting consistent with per-VMA fork state. Add file-backed mremap regression coverage and wire the tests into the build and documentation. --- Makefile | 5 + docs/testing.md | 8 +- mk/tests.mk | 6 + src/core/guest.c | 143 ++++- src/core/guest.h | 42 +- src/syscall/mem.c | 967 +++++++++++++++++++++++------- tests/manifest.txt | 1 + tests/test-mremap-fork-tracking.c | 842 ++++++++++++++++++++++++++ tests/test-mremap-tail-emfile.c | 235 ++++++++ tests/test-proc-smap.c | 74 ++- 10 files changed, 2066 insertions(+), 257 deletions(-) create mode 100644 tests/test-mremap-fork-tracking.c create mode 100644 tests/test-mremap-tail-emfile.c diff --git a/Makefile b/Makefile index 299762d6..57fb835e 100644 --- a/Makefile +++ b/Makefile @@ -477,6 +477,11 @@ endif endif +## Build the libc-based file-backed mremap EMFILE regression probe +$(BUILD_DIR)/test-mremap-tail-emfile: tests/test-mremap-tail-emfile.c | $(BUILD_DIR) + @echo " CROSS $<" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< + include mk/tests.mk include mk/analysis.mk include mk/help.mk diff --git a/docs/testing.md b/docs/testing.md index e4d52f34..a7d80d1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -126,7 +126,8 @@ What they do: - the unit suite from `tests/manifest.txt` -- deliberately narrow: only tests that assert elfuse-internal implementation details with no real Linux counterpart (the EL1 shim fast-path suite, `test-mremap-infra`, - `test-oom-proc`), plus whatever `mk/tests.mk`'s `SANITIZER_SECTIONS` + `test-mremap-fork-tracking`, `test-oom-proc`), plus whatever + `mk/tests.mk`'s `SANITIZER_SECTIONS` needs for the `check-{asan,ubsan,tsan}` lanes. Everything that is meaningful to cross-check against a real Linux kernel lives exclusively in `tests/test-matrix.sh`'s `run_unit_tests` instead (see Test Matrix @@ -327,8 +328,9 @@ surface -- every binary that is meaningful to run against a real kernel, which is almost everything. It deliberately excludes only the handful of tests that assert elfuse-internal implementation details with no meaningful counterpart on a real kernel (the EL1 shim fast-path suite, `test-mremap-infra`, -`test-oom-proc` -- these live solely in `tests/manifest.txt` / `make check`, -see that file's header for the full split rationale). There is no separate +`test-mremap-fork-tracking`, `test-oom-proc` -- these live solely in +`tests/manifest.txt` / `make check`, see that file's header for the full split +rationale). There is no separate "core" vs "extended" test set inside the matrix; a test that has a real, understood divergence from the qemu reference kernel is listed in `QEMU_SKIP` with a comment explaining why instead -- see that variable in diff --git a/mk/tests.mk b/mk/tests.mk index eff296dc..1345054f 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -16,6 +16,7 @@ test-vcpu-run-hooks-host test-identity-override-host \ test-dynamic-array-host \ test-string-builder-host \ + test-mremap-tail-emfile \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-launch-flags \ test-fuse-alpine \ @@ -43,6 +44,11 @@ test-hello: $(ELFUSE_BIN) $(TEST_HELLO_DEP) @printf "$(BLUE)▸ Running$(RESET) test-hello\n" $(ELFUSE_BIN) $(TEST_DIR)/test-hello +## Run the libc-based file-backed region removal EMFILE regression probe +test-mremap-tail-emfile: $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + @printf "$(BLUE)▸ Running$(RESET) test-mremap-tail-emfile\n" + $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + ## Verify dispatch.tbl coverage of the kernel-supported syscall set check-syscall-coverage: @python3 scripts/check-syscall-coverage.py diff --git a/src/core/guest.c b/src/core/guest.c index 575d5d8d..7620dcb2 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1840,15 +1840,18 @@ int guest_get_used_regions(const guest_t *g, /* Semantic region tracking. * - * Check whether two adjacent regions can be merged. They must be contiguous in - * address space, have identical protection/flags/name, and have contiguous file - * offsets (so the merged region still represents valid mapping). For anonymous - * regions the offset is meaningless (always 0, but may become non-zero after - * split/trim), so the contiguity check is skipped. Without this, adjacent - * anonymous mmaps (common in megablock-style allocators) each create separate - * entries that exhaust the region table. + * Check whether two adjacent regions have merge-compatible layouts. An actual + * merge additionally requires a shared vma_id; a new same-generation mapping + * may adopt its compatible neighbor's ID below. Regions must be contiguous in + * address space, have identical protection/flags/name, and have contiguous + * file offsets (so the merged region still represents valid mapping). For + * anonymous regions the offset is meaningless (always 0, but may become + * non-zero after split/trim), so the contiguity check is skipped. Without this, + * adjacent anonymous mmaps (common in megablock-style allocators) each create + * separate entries that exhaust the region table. */ -static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +static bool regions_mergeable_layout(const guest_region_t *a, + const guest_region_t *b) { if (a->end != b->start) return false; @@ -1886,6 +1889,11 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return a->offset + (a->end - a->start) == b->offset; } +static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +{ + return a->vma_id == b->vma_id && regions_mergeable_layout(a, b); +} + /* First region whose start is >= start. regions[] is sorted by start. */ static int region_lower_bound_start(const guest_t *g, uint64_t start) { @@ -1983,7 +1991,8 @@ int guest_region_add_ex(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd, false); + offset, name, owned_backing_fd, false, + 0); } int guest_region_add_ex_gpa(guest_t *g, @@ -2004,7 +2013,31 @@ int guest_region_add_ex_gpa(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, gpa_base, prot, flags, - offset, name, owned_backing_fd, false); + offset, name, owned_backing_fd, false, + 0); +} + +static uint64_t allocate_vma_id(guest_t *g) +{ + uint64_t candidate = g->next_vma_id; + + for (;;) { + candidate++; + if (candidate == 0) + candidate = 1; + + bool in_use = false; + for (int i = 0; i < g->nregions; i++) { + if (g->regions[i].vma_id == candidate) { + in_use = true; + break; + } + } + if (!in_use) { + g->next_vma_id = candidate; + return candidate; + } + } } int guest_region_add_ex_owned(guest_t *g, @@ -2015,11 +2048,12 @@ int guest_region_add_ex_owned(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t vma_id) { return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, offset, name, owned_backing_fd, - inherited_at_fork); + inherited_at_fork, vma_id); } int guest_region_add_ex_owned_gpa(guest_t *g, @@ -2031,7 +2065,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t vma_id) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2044,6 +2079,12 @@ int guest_region_add_ex_owned_gpa(guest_t *g, return -1; } + bool new_vma = !vma_id; + if (new_vma) + vma_id = allocate_vma_id(g); + else if (vma_id > g->next_vma_id) + g->next_vma_id = vma_id; + /* Find insertion point (keep sorted by start address). */ int i = region_lower_bound_start(g, start); memmove(&g->regions[i + 1], &g->regions[i], @@ -2053,6 +2094,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->start = start; r->end = end; r->gpa_base = gpa_base; + r->vma_id = vma_id; r->prot = prot; r->flags = flags; r->offset = offset; @@ -2069,6 +2111,20 @@ int guest_region_add_ex_owned_gpa(guest_t *g, } g->nregions++; + /* Preserve the historical coalescing of compatible same-generation + * anonymous mmap calls: Linux may merge those into one VMA and the region + * tracker relies on that to stay below GUEST_MAX_REGIONS. Never adopt a + * neighbor's lineage across an inherited/private boundary, which is the + * provenance distinction find_mremap_source() must retain after fork. + */ + if (new_vma) { + if (i > 0 && regions_mergeable_layout(&g->regions[i - 1], r)) + r->vma_id = g->regions[i - 1].vma_id; + else if (i + 1 < g->nregions && + regions_mergeable_layout(r, &g->regions[i + 1])) + r->vma_id = g->regions[i + 1].vma_id; + } + /* Try to merge with adjacent regions to reduce table pressure. Merge right * first, then left (order matters: right merge does not change the index of * the left neighbor). @@ -2111,10 +2167,44 @@ int guest_preannounce(guest_t *g, return 0; } -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd) { + if (!reserved_backing_fd) + return -1; + *reserved_backing_fd = -1; if (end <= start) - return; + return 0; + + int first = guest_region_first_end_above(g, start); + for (int i = first; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->start < start && r->end > end) { + /* A full table follows the existing stale-tracker fallback and + * does not publish a right-hand record, so no fd is required. */ + if (g->nregions >= GUEST_MAX_REGIONS || r->backing_fd < 0) + return 0; + *reserved_backing_fd = dup(r->backing_fd); + return *reserved_backing_fd >= 0 ? 0 : -1; + } + } + return 0; +} + +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd) +{ + if (end <= start) { + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; + } /* In-place compaction: 'out' is the next output slot, 'in' is the next * input slot. Since the prefix [0, first) is untouched (it sorts strictly @@ -2174,18 +2264,16 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) right.gpa_base += trimmed; right.start = end; if (orig.backing_fd >= 0) { - right.backing_fd = dup(orig.backing_fd); - if (right.backing_fd < 0) - log_error( - "guest: dup() failed for region split " - "backing fd %d: %s", - orig.backing_fd, strerror(errno)); + right.backing_fd = reserved_backing_fd; + reserved_backing_fd = -1; } guest_region_clip_overlay(&right); g->regions[out + 1] = right; g->nregions = out + 2 + suffix_count; - return; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; } } @@ -2223,6 +2311,17 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) memmove(&g->regions[out], &g->regions[in], tail * sizeof(guest_region_t)); g->nregions = out + tail; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; +} + +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +{ + int reserved_backing_fd = -1; + if (guest_region_remove_prepare(g, start, end, &reserved_backing_fd) < 0) + return -1; + return guest_region_remove_reserved(g, start, end, reserved_backing_fd); } const guest_region_t *guest_region_find(const guest_t *g, uint64_t addr) diff --git a/src/core/guest.h b/src/core/guest.h index 65267cda..0645e70b 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -230,6 +230,12 @@ typedef struct { * identity-mapped regions; differs for high-VA guest * mappings whose VA and GPA diverge. */ + uint64_t vma_id; /* Stable logical-VMA lineage. Tracker splits and + * mremap moves preserve it; compatible same-generation + * mappings may share it when the tracker coalesces + * them. Unlike inherited_at_fork, this ID remains + * meaningful across subsequent forks. + */ int prot; /* LINUX_PROT_* flags */ int flags; /* LINUX_MAP_* flags (for /proc/self/maps display) */ uint64_t offset; /* File offset (for /proc/self/maps display) */ @@ -508,7 +514,8 @@ typedef struct { /* Semantic region tracking for munmap/mprotect/proc-self-maps */ guest_region_t regions[GUEST_MAX_REGIONS]; - int nregions; /* Number of active regions */ + int nregions; /* Number of active regions */ + uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */ /* Sticky flag set when guest_region_set_prot could not honor a request * because the region table was full. After this point the tracker no longer * faithfully reflects PTE state, so the mprotect fast path must fall back @@ -1182,8 +1189,8 @@ int guest_region_add_ex_gpa(guest_t *g, /* Like guest_region_add_ex, but consumes owned_backing_fd on success or * failure. inherited_at_fork identifies bytes copied from the latest fork - * snapshot; ordinary mmap/exec callers pass false, while fork/mremap restore - * paths preserve the source VMA's value. + * snapshot. vma_id preserves logical-VMA provenance across tracker splits and + * mremap moves; pass 0 for a new VMA. */ int guest_region_add_ex_owned(guest_t *g, uint64_t start, @@ -1193,7 +1200,8 @@ int guest_region_add_ex_owned(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork); + bool inherited_at_fork, + uint64_t vma_id); int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t start, uint64_t end, @@ -1203,7 +1211,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork); + bool inherited_at_fork, + uint64_t vma_id); /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest @@ -1223,10 +1232,29 @@ int guest_preannounce(guest_t *g, uint64_t offset, const char *name); +/* Reserve any backing fd needed by an interior split in [start, end). + * The reservation must be consumed by guest_region_remove_reserved(). + * Returns 0 on success, -1 when the backing fd cannot be duplicated. + */ +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd); + /* Remove all region coverage in [start, end). Regions fully contained are - * deleted; partially overlapping regions are trimmed or split. + * deleted; partially overlapping regions are trimmed or split. Any required + * backing fd is reserved before the first metadata mutation. + */ +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end); + +/* Commit a removal using a backing fd reserved by + * guest_region_remove_prepare(). Ownership of reserved_backing_fd is consumed + * even when this call does not need an interior split. */ -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end); +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd); /* Find the region containing addr. * diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 8fc47d90..45943aa5 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -75,6 +75,7 @@ typedef struct { uint64_t start; uint64_t end; uint64_t gpa_base; + uint64_t vma_id; int prot; int flags; uint64_t offset; @@ -230,6 +231,168 @@ static void mark_region_backing_ro(guest_t *g, uint64_t start, uint64_t end) * remain unmarked. Keep the two portions as separate VMAs even when all other * metadata matches so the synthetic smaps view can distinguish them. */ +static bool mremap_backings_match(const guest_region_t *a, + const guest_region_t *b) +{ + if (a->backing_fd < 0 || b->backing_fd < 0) + return a->backing_fd < 0 && b->backing_fd < 0; + if (a->backing_fd == b->backing_fd) + return true; + + struct stat sa, sb; + if (fstat(a->backing_fd, &sa) < 0 || fstat(b->backing_fd, &sb) < 0) + return false; + return sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t gpa_base; + uint64_t offset; + int backing_fd; /* borrowed from the region tracker */ + bool overlay_active; + uint64_t overlay_start; + uint64_t overlay_end; +} mremap_source_segment_t; + +typedef struct { + const guest_region_t *first; + uint64_t inherited_prefix; + int nsegments; + mremap_source_segment_t *segments; +} mremap_source_t; + +static void dispose_mremap_source(mremap_source_t *source) +{ + if (!source) + return; + free(source->segments); + memset(source, 0, sizeof(*source)); +} + +static int64_t finish_mremap(mremap_source_t *source, int64_t result) +{ + dispose_mremap_source(source); + return result; +} + +/* Resolve a logical mremap source. Fork-aware growth intentionally leaves an + * inherited prefix and a child-private tail as separate records; those two + * records are still one VMA for mremap purposes. The stable vma_id proves that + * provenance even after another fork changes inherited_at_fork on both + * records. Reject any boundary with a different lineage so unrelated adjacent + * mappings cannot be copied as one source. + */ +static int find_mremap_source(const guest_t *g, + uint64_t start, + uint64_t size, + bool collect_segments, + mremap_source_t *source) +{ + uint64_t end = start + size; + const guest_region_t *first = guest_region_find(g, start); + if (!first) + return -LINUX_EFAULT; + + memset(source, 0, sizeof(*source)); + source->first = first; + + int index = (int) (first - g->regions); + if (index < 0 || index >= g->nregions) + goto invalid; + + uint64_t cursor = start; + uint64_t expected_gpa = first->gpa_base + (start - first->start); + uint64_t expected_offset = first->offset + (start - first->start); + uint64_t inherited_prefix = 0; + bool prefix_inherited = true; + int nsegments = 0; + for (int scan = index; scan < g->nregions && cursor < end; scan++) { + const guest_region_t *r = &g->regions[scan]; + if (r->start > cursor || r->end <= cursor) + goto invalid; + if (r != first) { + const guest_region_t *previous = &g->regions[scan - 1]; + if (previous->end != r->start || previous->prot != r->prot || + previous->flags != r->flags || previous->shared != r->shared || + previous->noreserve != r->noreserve || + previous->backing_ro != r->backing_ro || + strcmp(previous->name, r->name) != 0 || + !mremap_backings_match(previous, r)) + goto invalid; + if (!previous->vma_id || previous->vma_id != r->vma_id) + goto invalid; + } + + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + if (r->gpa_base + (cursor - r->start) != expected_gpa) + goto invalid; + if (!(r->flags & LINUX_MAP_ANONYMOUS) && + r->offset + (cursor - r->start) != expected_offset) + goto invalid; + + if (nsegments >= GUEST_MAX_REGIONS) + goto invalid; + nsegments++; + + if (prefix_inherited && r->inherited_at_fork) + inherited_prefix += segment_len; + else + prefix_inherited = false; + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + if (cursor != end) + goto invalid; + + /* Validation also gives the exact allocation size. Same-size non-fixed + * mremap stops here, so its no-op success cannot be turned into ENOMEM by + * segment bookkeeping. + */ + source->inherited_prefix = inherited_prefix; + source->nsegments = nsegments; + if (!collect_segments) + return 0; + + source->segments = + malloc((size_t) nsegments * sizeof(*source->segments)); + if (!source->segments) { + dispose_mremap_source(source); + return -LINUX_ENOMEM; + } + + cursor = start; + expected_gpa = first->gpa_base + (start - first->start); + expected_offset = first->offset + (start - first->start); + for (int segment_index = 0; segment_index < nsegments; segment_index++) { + const guest_region_t *r = &g->regions[index + segment_index]; + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + mremap_source_segment_t *segment = + &source->segments[segment_index]; + segment->start = cursor; + segment->end = segment_end; + segment->gpa_base = expected_gpa; + segment->offset = expected_offset; + segment->backing_fd = r->backing_fd; + segment->overlay_active = region_has_live_overlay(r); + segment->overlay_start = r->overlay_start; + segment->overlay_end = r->overlay_end; + + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + return 0; + +invalid: + dispose_mremap_source(source); + return -LINUX_EFAULT; +} + static int add_mremap_region(guest_t *g, uint64_t start, uint64_t old_size, @@ -239,36 +402,45 @@ static int add_mremap_region(guest_t *g, uint64_t offset, const char *name, int backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t inherited_size, + int tail_backing_fd, + uint64_t vma_id) { - if (inherited_at_fork && old_size > 0 && old_size < new_size) { - int tail_backing_fd = -1; - if (backing_fd >= 0) { - tail_backing_fd = dup(backing_fd); - if (tail_backing_fd < 0) { - close(backing_fd); - return -1; - } + if (!inherited_at_fork) + inherited_size = 0; + if (inherited_size > old_size) + inherited_size = old_size; + if (inherited_size > new_size) + inherited_size = new_size; + + if (inherited_size > 0 && inherited_size < new_size) { + if (backing_fd >= 0 && tail_backing_fd < 0) { + close(backing_fd); + return -1; } - if (guest_region_add_ex_owned(g, start, start + old_size, prot, flags, - offset, name, backing_fd, true) < 0) { + if (guest_region_add_ex_owned(g, start, start + inherited_size, prot, + flags, offset, name, backing_fd, true, + vma_id) < 0) { if (tail_backing_fd >= 0) close(tail_backing_fd); return -1; } - - if (guest_region_add_ex_owned(g, start + old_size, start + new_size, - prot, flags, offset + old_size, name, - tail_backing_fd, false) < 0) { - guest_region_remove(g, start, start + old_size); + if (guest_region_add_ex_owned(g, start + inherited_size, + start + new_size, prot, flags, + offset + inherited_size, name, + tail_backing_fd, false, vma_id) < 0) { + guest_region_remove(g, start, start + inherited_size); return -1; } return 0; } - return guest_region_add_ex_owned(g, start, start + new_size, prot, flags, - offset, name, backing_fd, - inherited_at_fork && old_size > 0); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return guest_region_add_ex_owned( + g, start, start + new_size, prot, flags, offset, name, backing_fd, + inherited_size == new_size && inherited_size > 0, vma_id); } static void region_clip_overlay(guest_region_t *r) @@ -290,43 +462,51 @@ static void region_clip_overlay(guest_region_t *r) region_clear_overlay(r); } -static void split_regions_at_boundary(guest_t *g, uint64_t boundary) +static int split_regions_at_boundary(guest_t *g, uint64_t boundary) { if (boundary == 0) - return; + return 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (boundary <= r->start) break; if (boundary >= r->end) continue; if (g->nregions >= GUEST_MAX_REGIONS) { log_error( - "guest: region table full, cleanup split skipped at " + "guest: region table full, region split skipped at " "0x%llx", (unsigned long long) boundary); - return; + return -LINUX_ENOMEM; } + guest_region_t left = *r; + guest_region_t right = *r; + if (right.backing_fd >= 0) { + right.backing_fd = dup(right.backing_fd); + if (right.backing_fd < 0) { + log_error("guest: dup() failed for region split: %s", + strerror(errno)); + return -LINUX_ENOMEM; + } + } + + left.end = boundary; + right.offset += boundary - right.start; + right.gpa_base += boundary - right.start; + right.start = boundary; + region_clip_overlay(&left); + region_clip_overlay(&right); + memmove(&g->regions[i + 1], &g->regions[i], (g->nregions - i) * sizeof(guest_region_t)); + g->regions[i] = left; + g->regions[i + 1] = right; g->nregions++; - - g->regions[i].end = boundary; - g->regions[i + 1].offset += (boundary - g->regions[i + 1].start); - g->regions[i + 1].gpa_base += (boundary - g->regions[i + 1].start); - g->regions[i + 1].start = boundary; - if (g->regions[i + 1].backing_fd >= 0) { - g->regions[i + 1].backing_fd = dup(g->regions[i + 1].backing_fd); - if (g->regions[i + 1].backing_fd < 0) - log_error("guest: dup() failed for cleanup split: %s", - strerror(errno)); - } - region_clip_overlay(&g->regions[i]); - region_clip_overlay(&g->regions[i + 1]); - return; + return 0; } + return 0; } static uint64_t find_free_gap_inner(const guest_t *g, @@ -901,7 +1081,7 @@ static int64_t sys_mmap_high_va(guest_t *g, } if (guest_region_add_ex_owned_gpa(g, addr, addr + length, gpa_base, prot, flags, offset, NULL, track_backing_fd, - false) < 0) + false, 0) < 0) goto fail; /* Ownership of track_backing_fd is now held by the new region. The fail * handler below skips closing when track_backing_fd < 0, so subsequent @@ -1101,6 +1281,167 @@ static int restore_file_overlay_range(guest_t *g, return 0; } +static bool mremap_source_has_overlay(const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + if (source->segments[i].overlay_active) + return true; + } + return false; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t offset; + int backing_fd; + bool active; + uint64_t overlay_start; + uint64_t overlay_end; +} saved_overlay_t; + +typedef void (*saved_overlay_getter_t)(const void *saved, + int index, + saved_overlay_t *overlay); + +static uint64_t saved_overlay_file_offset(const saved_overlay_t *overlay) +{ + if (overlay->overlay_start >= overlay->start) + return overlay->offset + (overlay->overlay_start - overlay->start); + return overlay->offset - (overlay->start - overlay->overlay_start); +} + +/* Reinstall each distinct host overlay once, then restore metadata on every + * tracker fragment that shared it. cleanup_overlays_in_range() can fail after + * removing only a subset, so replaying the complete saved set is rollback-safe. + */ +static int restore_saved_overlays_in_place(guest_t *g, + const void *saved, + int n, + saved_overlay_getter_t get) +{ + for (int i = 0; i < n; i++) { + saved_overlay_t overlay; + get(saved, i, &overlay); + if (!overlay.active || overlay.backing_fd < 0) + continue; + + uint64_t file_off = saved_overlay_file_offset(&overlay); + bool first = true; + for (int j = 0; j < i; j++) { + saved_overlay_t previous; + get(saved, j, &previous); + if (previous.active && previous.backing_fd >= 0 && + previous.overlay_start == overlay.overlay_start && + previous.overlay_end == overlay.overlay_end && + saved_overlay_file_offset(&previous) == file_off) { + first = false; + break; + } + } + + if (first) { + int err = restore_file_overlay_range( + g, overlay.start, overlay.end, overlay.overlay_start, + overlay.overlay_end, overlay.backing_fd, file_off); + if (err < 0) + return err; + } else { + mark_overlay_metadata_range(g, overlay.start, overlay.end, + overlay.overlay_start, + overlay.overlay_end); + } + } + return 0; +} + +static void get_mremap_source_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const mremap_source_t *source = saved; + const mremap_source_segment_t *segment = &source->segments[index]; + *overlay = (saved_overlay_t) { + .start = segment->start, + .end = segment->end, + .offset = segment->offset, + .backing_fd = segment->backing_fd, + .active = segment->overlay_active, + .overlay_start = segment->overlay_start, + .overlay_end = segment->overlay_end, + }; +} + +static int restore_mremap_source_overlays_in_place( + guest_t *g, + const mremap_source_t *source) +{ + return restore_saved_overlays_in_place(g, source, source->nsegments, + get_mremap_source_overlay); +} + +/* The host overlay stays installed during an in-place growth; only the region + * records are replaced. Reapply the saved metadata to the corresponding new + * records without remapping the host VA. + */ +static void mark_mremap_source_overlay_metadata(guest_t *g, + const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + if (segment->overlay_active) + mark_overlay_metadata_range(g, segment->start, segment->end, + segment->overlay_start, + segment->overlay_end); + } +} + +/* Copy each source segment according to its own backing state. Live-overlay + * bytes must be refreshed from the file after the overlay is removed; private + * fork-grown bytes remain in the slab and must be copied from their GPA. + */ +static int copy_mremap_source(guest_t *g, + uint64_t dest_gpa, + uint64_t source_start, + uint64_t length, + const mremap_source_t *source) +{ + uint64_t source_end = source_start + length; + uint64_t cursor = source_start; + + for (int i = 0; i < source->nsegments && cursor < source_end; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + uint64_t start = segment->start > cursor ? segment->start : cursor; + uint64_t end = segment->end < source_end ? segment->end : source_end; + if (end <= start) + continue; + if (start != cursor) + return -LINUX_EFAULT; + + uint64_t len = end - start; + uint64_t dest = dest_gpa + (start - source_start); + if (segment->overlay_active) { + if (segment->backing_fd < 0) + return -LINUX_EFAULT; + int err = read_file_range_to_guest( + g, dest, segment->backing_fd, + segment->offset + (start - segment->start), len); + if (err < 0) + return err; + } else { + uint8_t *dest_ptr = host_ptr_for_gpa(g, dest); + uint8_t *source_ptr = host_ptr_for_gpa( + g, segment->gpa_base + (start - segment->start)); + if (!dest_ptr || !source_ptr) + return -LINUX_EFAULT; + memmove(dest_ptr, source_ptr, len); + } + cursor = end; + } + + return cursor == source_end ? 0 : -LINUX_EFAULT; +} + typedef struct { uint64_t overlay_start; uint64_t overlay_len; @@ -1148,8 +1489,12 @@ static int capture_region_snapshots(guest_t *g, region_snapshot_t *snaps, int max_snaps) { - split_regions_at_boundary(g, start); - split_regions_at_boundary(g, end); + int split_err = split_regions_at_boundary(g, start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, end); + if (split_err < 0) + return split_err; int n = 0; for (int i = 0; i < g->nregions; i++) { @@ -1167,6 +1512,7 @@ static int capture_region_snapshots(guest_t *g, snap->start = r->start; snap->end = r->end; snap->gpa_base = r->gpa_base; + snap->vma_id = r->vma_id; snap->prot = r->prot; snap->flags = r->flags; snap->offset = r->offset; @@ -1189,48 +1535,58 @@ static int capture_region_snapshots(guest_t *g, return n; } -static int restore_snapshot_overlays_in_place(guest_t *g, - const region_snapshot_t *snaps, - int n) +/* MREMAP_FIXED may remove a destination fragment that used to share the same + * tracker backing fd as a source fragment. Rebind file-backed source segments + * to the owned source snapshots before destination removal, so later overlay + * restore and pread-based copies cannot observe a closed borrowed fd. */ +static int rebind_mremap_source_backings(mremap_source_t *source, + const region_snapshot_t *snaps, + int n) { - for (int i = 0; i < n; i++) { - const region_snapshot_t *snap = &snaps[i]; - if (!snap->overlay_active || snap->backing_fd < 0) + for (int i = 0; i < source->nsegments; i++) { + mremap_source_segment_t *segment = &source->segments[i]; + if (segment->backing_fd < 0) continue; - bool first = true; - uint64_t snap_file_off = - snap->offset + (snap->overlay_start - snap->start); - for (int j = 0; j < i; j++) { - const region_snapshot_t *prev = &snaps[j]; - if (!prev->overlay_active || prev->backing_fd < 0) - continue; - uint64_t prev_file_off = - prev->offset + (prev->overlay_start - prev->start); - if (prev->overlay_start == snap->overlay_start && - prev->overlay_end == snap->overlay_end && - prev_file_off == snap_file_off) { - first = false; + int stable_fd = -1; + for (int j = 0; j < n; j++) { + if (snaps[j].start <= segment->start && + segment->start < snaps[j].end && snaps[j].backing_fd >= 0) { + stable_fd = snaps[j].backing_fd; break; } } - - if (first) { - int err = restore_file_overlay_range( - g, snap->start, snap->end, snap->overlay_start, - snap->overlay_end, snap->backing_fd, snap_file_off); - if (err < 0) - return err; - continue; - } - - mark_overlay_metadata_range(g, snap->start, snap->end, - snap->overlay_start, snap->overlay_end); + if (stable_fd < 0) + return -LINUX_EFAULT; + segment->backing_fd = stable_fd; } - return 0; } +static void get_region_snapshot_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const region_snapshot_t *snap = &((const region_snapshot_t *) saved)[index]; + *overlay = (saved_overlay_t) { + .start = snap->start, + .end = snap->end, + .offset = snap->offset, + .backing_fd = snap->backing_fd, + .active = snap->overlay_active, + .overlay_start = snap->overlay_start, + .overlay_end = snap->overlay_end, + }; +} + +static int restore_snapshot_overlays_in_place(guest_t *g, + const region_snapshot_t *snaps, + int n) +{ + return restore_saved_overlays_in_place(g, snaps, n, + get_region_snapshot_overlay); +} + static bool snapshot_has_materialized_ptes(const region_snapshot_t *snap) { return snap->prot != LINUX_PROT_NONE && @@ -1289,7 +1645,7 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) if (guest_region_add_ex_owned_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd, snap->inherited_at_fork) < 0) { + snap->backing_fd, snap->inherited_at_fork, snap->vma_id) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -1768,8 +2124,12 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) uint64_t host_start = ALIGN_DOWN(start, hps); uint64_t host_end = ALIGN_UP(end, hps); - split_regions_at_boundary(g, host_start); - split_regions_at_boundary(g, host_end); + int split_err = split_regions_at_boundary(g, host_start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, host_end); + if (split_err < 0) + return split_err; /* Snapshot affected ranges first; the host-side mmap calls below do not * touch the region array, but a future caller invariant is to allow this @@ -1826,6 +2186,22 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) /* Memory syscalls (tightly coupled to guest.h). */ +static bool heap_tail_can_extend(const guest_region_t *tail, + const guest_region_t *heap, + uint64_t old_brk) +{ + const int heap_flags = LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS; + + return tail->start == heap->end && tail->end == old_brk && + tail->end > tail->start && tail->gpa_base == tail->start && + tail->vma_id == heap->vma_id && + tail->prot == (LINUX_PROT_READ | LINUX_PROT_WRITE) && + tail->flags == heap_flags && tail->offset == 0 && + tail->backing_fd < 0 && !tail->shared && !tail->noreserve && + !tail->backing_ro && !tail->inherited_at_fork && + !region_has_live_overlay(tail) && !strcmp(tail->name, "[heap]"); +} + int64_t sys_brk(guest_t *g, uint64_t addr) { /* brk addresses as seen by the guest are IPA-based */ @@ -1847,6 +2223,16 @@ int64_t sys_brk(guest_t *g, uint64_t addr) return (int64_t) ipa_brk; } + /* Shrinking can split a file-backed semantic region when a fork child has + * grown and reshaped its heap metadata. Reserve the right-hand region's + * backing fd before publishing the new break, so EMFILE leaves both the + * break and region table unchanged. + */ + int shrink_remove_fd = -1; + if (new_off < old_brk && + guest_region_remove_prepare(g, new_off, old_brk, &shrink_remove_fd) < 0) + return (int64_t) ipa_brk; + /* Materialize any newly exposed heap pages. This must handle both: * 1. growth into brand-new 2 MiB blocks, and * 2. growth within an already-split block where finalize_block_perms() @@ -1874,7 +2260,15 @@ int64_t sys_brk(guest_t *g, uint64_t addr) * avoids the remove+add gap where a concurrent /proc/self/maps reader could * see no heap region. */ - if (new_off > g->brk_base) { + if (new_off < old_brk) { + /* Trim every semantic heap segment covered by the released suffix. + * In a fork child this may shorten or remove the private tail while + * leaving the inherited prefix intact. Keeping the tracker end equal + * to brk_current prevents a later regrowth from overlapping stale + * tail metadata. + */ + guest_region_remove_reserved(g, new_off, old_brk, shrink_remove_fd); + } else if (new_off > g->brk_base) { bool found = false; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start == g->brk_base && @@ -1883,17 +2277,31 @@ int64_t sys_brk(guest_t *g, uint64_t addr) uint64_t old_heap_end = heap->end; if (new_off > old_heap_end && heap->inherited_at_fork) { /* Keep the fork-snapshot portion separate from pages - * materialized by this post-fork brk growth. If the - * semantic table is full, retain the historical - * conservative range and mark it stale rather than - * failing an otherwise successful brk syscall. + * materialized by post-fork brk growth. On later growths, + * extend the existing child-private tail rather than + * adding an overlapping range from the old boundary. */ - if (guest_region_add( - g, old_heap_end, new_off, - LINUX_PROT_READ | LINUX_PROT_WRITE, - LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, - "[heap]") < 0) { - heap->end = new_off; + guest_region_t *right = + i + 1 < g->nregions ? &g->regions[i + 1] : NULL; + guest_region_t *tail = + right && heap_tail_can_extend(right, heap, old_brk) + ? right + : NULL; + if (tail) { + if (new_off > tail->end) + tail->end = new_off; + } else if (new_off > old_brk && + guest_region_add_ex_owned( + g, old_brk, new_off, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, + "[heap]", -1, false, heap->vma_id) < 0) { + /* Widening the inherited prefix would either overlap + * an incompatible child-private tail or mislabel new + * pages as inherited. Keep the original boundary; brk + * memory already grew successfully, so only the + * semantic tracker becomes stale. + */ g->regions_tracker_stale = true; } } else { @@ -2628,7 +3036,7 @@ int64_t sys_mmap(guest_t *g, */ if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, track_backing_fd, false) < 0) { + NULL, track_backing_fd, false, 0) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -2758,12 +3166,20 @@ int64_t sys_mremap(guest_t *g, if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; - /* Verify the whole source range is covered by one tracked VMA. mremap() - * must not copy holes or unrelated adjacent mappings. + /* Verify the whole source range is covered by one logical VMA. A + * fork-aware growth can split that VMA at the inherited/private boundary, + * but no unrelated adjacent mapping may be included. */ - const guest_region_t *src_reg = guest_region_find(g, old_off); - if (!src_reg || src_reg->end - old_off < old_size) - return -LINUX_EFAULT; + mremap_source_t source; + bool collect_source_segments = + old_size != new_size || (flags & LINUX_MREMAP_FIXED); + int source_err = find_mremap_source(g, old_off, old_size, + collect_source_segments, &source); + if (source_err < 0) + return source_err; + const guest_region_t *src_reg = source.first; + uint64_t source_inherited_size = source.inherited_prefix; + uint64_t source_vma_id = src_reg->vma_id; /* Capture the source region's GPA layout before any region mutation below * invalidates src_reg. src_gpa_base + (va_off - src_start) is the backing @@ -2776,41 +3192,48 @@ int64_t sys_mremap(guest_t *g, /* Same size: nothing to do */ if (old_size == new_size && !(flags & LINUX_MREMAP_FIXED)) - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); /* Shrinking mremap keeps the base address and releases only the tail. */ if (new_size < old_size && !(flags & LINUX_MREMAP_FIXED)) { uint64_t tail_off = old_off + new_size, tail_end = old_off + old_size; + int tail_remove_fd = -1; + if (guest_region_remove_prepare(g, tail_off, tail_end, + &tail_remove_fd) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); /* Restore slab backing under any tail overlay before zeroing so the * memset does not write zeros into a file. */ int cleanup_err = cleanup_overlays_in_range(g, tail_off, tail_end); - if (cleanup_err < 0) - return cleanup_err; + if (cleanup_err < 0) { + if (tail_remove_fd >= 0) + close(tail_remove_fd); + return finish_mremap(&source, cleanup_err); + } /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, tail_end - tail_off); - guest_region_remove(g, tail_off, tail_end); + guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) g->mmap_rx_gap_hint = tail_off; - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } /* MREMAP_FIXED: move to a specific new address */ if (flags & LINUX_MREMAP_FIXED) { if (new_addr & 4095) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); uint64_t new_off = new_addr - g->ipa_base; /* MREMAP_FIXED dest stays primary-only for the same reason as the * source check above. */ if (new_off > g->guest_size || new_size > g->guest_size - new_off) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Same infrastructure protection as the source range: the move tail * removes any existing dest region and rewrites PTEs, which would @@ -2818,12 +3241,12 @@ int64_t sys_mremap(guest_t *g, * infra. */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; if (old_off < new_end && new_off < old_end) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); remove_range_t removed[] = { {old_off, old_end}, @@ -2835,27 +3258,38 @@ int64_t sys_mremap(guest_t *g, * metadata. The overlap check above prevents this case, but capturing * first is still the safe ordering. */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; int added_regions = - source_inherited_at_fork && old_size < new_size ? 2 : 1; + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; if (!region_has_capacity_after_removes(g, removed, 2, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; char track_name[sizeof(old_reg->name)] = {0}; /* Heap-allocated to avoid blowing the ~512 KiB default macOS thread * stack: each region_snapshot_t array is GUEST_MAX_REGIONS * @@ -2876,7 +3310,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } source_nsnaps = capture_region_snapshots( @@ -2886,7 +3322,20 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return source_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, source_nsnaps); + } + int rebind_err = + rebind_mremap_source_backings(&source, source_snaps, source_nsnaps); + if (rebind_err < 0) { + dispose_region_snapshots(&source_snaps, &source_nsnaps); + free(dest_snaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, rebind_err); } dest_nsnaps = capture_region_snapshots(g, new_off, new_off + new_size, dest_snaps, GUEST_MAX_REGIONS); @@ -2895,7 +3344,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return dest_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, dest_nsnaps); } if (source_overlay) { @@ -2908,7 +3359,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } } @@ -2922,7 +3375,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2930,7 +3385,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { @@ -2941,7 +3398,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2949,7 +3408,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } /* Remove existing mappings at the destination after all fallible @@ -2957,19 +3418,19 @@ int64_t sys_mremap(guest_t *g, */ guest_region_remove(g, new_off, new_off + new_size); - /* Copy data (use memmove for potential overlap). If the source has a - * live overlay, the read side of the memmove pulls live file content; - * the destination receives a private snapshot at mremap time (no + /* Copy each logical source segment according to its own backing state. + * The destination receives a private snapshot at mremap time (no * overlay reapplied), and msync's emulated pwrite-the-diff path keeps * subsequent writes consistent. */ uint64_t copy_len = old_size < new_size ? old_size : new_size; if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, copy_len); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, copy_len, &source); if (copy_err < 0) { int restore_err = restore_snapshot_overlays_in_place( g, source_snaps, source_nsnaps); @@ -2990,21 +3451,14 @@ int64_t sys_mremap(guest_t *g, restore_err = pt_err; if (track_backing_fd >= 0) close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); if (restore_err < 0) - return restore_err; - return copy_err; + return finish_mremap(&source, restore_err); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA (identity for primary sources, - * overflow/mapping backing for high-VA). The destination is always - * a fresh primary-window range, so it never overlaps the source and - * the copy direction does not matter. - */ - memmove((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - copy_len); } /* Zero any extension beyond old data */ if (new_size > old_size) @@ -3025,17 +3479,19 @@ int64_t sys_mremap(guest_t *g, if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, source_inherited_at_fork) < 0) { + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) { (void) restore_region_snapshots(g, dest_snaps, dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Grow in place: try to extend without moving */ @@ -3048,7 +3504,7 @@ int64_t sys_mremap(guest_t *g, * it. */ if (guest_range_hits_infra(g, grow_off, grow_off + grow_len)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Check if the space after the old region is free (overflow-safe) */ if (grow_off <= g->guest_size && grow_len <= g->guest_size - grow_off) { @@ -3069,31 +3525,54 @@ int64_t sys_mremap(guest_t *g, if (can_grow) { remove_range_t removed = {old_off, old_off + old_size}; /* Extend in place */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); - bool old_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t old_overlay_start = - old_overlay ? old_reg->overlay_start : 0; - uint64_t old_overlay_end = - old_overlay ? old_reg->overlay_end : 0; + int tail_backing_fd = -1; + if (source_inherited_size > 0 && + source_inherited_size < new_size && track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } bool old_backing_ro = old_reg && old_reg->backing_ro; bool old_inherited_at_fork = old_reg && old_reg->inherited_at_fork; - int added_regions = old_inherited_at_fork ? 2 : 1; + int added_regions = source_inherited_size > 0 && + source_inherited_size < new_size + ? 2 + : 1; if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + if (old_reg && old_reg->backing_fd >= 0 && + track_backing_fd < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, @@ -3102,22 +3581,26 @@ int64_t sys_mremap(guest_t *g, if (mremap_extend_range(g, grow_off, grow_len, prot) < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); /* Update region tracking: remove old, add extended */ - guest_region_remove(g, old_off, old_off + old_size); - if (add_mremap_region( - g, old_off, old_size, new_size, prot, track_flags, - track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, old_inherited_at_fork) < 0) - return -LINUX_ENOMEM; - if (old_overlay) - mark_overlay_metadata_range(g, old_off, old_off + old_size, - old_overlay_start, - old_overlay_end); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); + if (add_mremap_region(g, old_off, old_size, new_size, prot, + track_flags, track_offset, + track_name[0] ? track_name : NULL, + track_backing_fd, old_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + mark_mremap_source_overlay_metadata(g, &source); if (old_backing_ro) mark_region_backing_ro(g, old_off, old_off + new_size); @@ -3131,36 +3614,37 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } } /* Growth in place failed; MREMAP_MAYMOVE is required */ if (!(flags & LINUX_MREMAP_MAYMOVE)) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Allocate a new region and move */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t source_overlay_start = - source_overlay ? old_reg->overlay_start : 0; - uint64_t source_overlay_end = source_overlay ? old_reg->overlay_end : 0; + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; - uint64_t source_overlay_file_off = - source_overlay - ? old_reg->offset + (source_overlay_start - old_reg->start) - : 0; char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, sizeof(track_name)); @@ -3182,56 +3666,85 @@ int64_t sys_mremap(guest_t *g, if (new_off == UINT64_MAX) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } remove_range_t removed = {old_off, old_off + old_size}; int added_regions = - source_inherited_at_fork && old_size < new_size ? 2 : 1; + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_overlay) { int cleanup_err = cleanup_overlays_in_range(g, old_off, old_off + old_size); if (cleanup_err < 0) { + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + if (restore_err < 0) + return finish_mremap(&source, restore_err); + return finish_mremap(&source, cleanup_err); } } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { if (source_overlay) { - int restore_err = restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (restore_err < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, restore_err); } } if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - /* Copy old data, zero extension. The new range was just allocated from - * a free gap so it has no overlays to clean up; the source may have an - * overlay, which is read transparently by the memcpy before its - * underlying slab is restored below. + /* Copy each source segment according to its own backing state, then + * zero the extension. The new range is a fresh gap and receives no + * live overlay. */ if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, old_size); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, old_size, &source); if (copy_err < 0) { /* Roll back both sides: re-apply the source overlay so the * caller's MAP_SHARED is not silently demoted to a slab @@ -3239,23 +3752,16 @@ int64_t sys_mremap(guest_t *g, * allocated via mremap_extend_range so the guest does not see * phantom zero pages where the failed mremap landed. */ - (void) restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + (void) restore_mremap_source_overlays_in_place(g, &source); guest_invalidate_ptes(g, new_off, new_off + new_size); if (track_backing_fd >= 0) close(track_backing_fd); - return copy_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA so high-VA sources copy from - * their real backing (identity for primary: == host_base + - * old_off). The destination is a fresh primary-window gap. - */ - memcpy((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - old_size); } memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); @@ -3265,7 +3771,8 @@ int64_t sys_mremap(guest_t *g, */ memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, old_size); - guest_region_remove(g, old_off, old_off + old_size); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; @@ -3275,8 +3782,10 @@ int64_t sys_mremap(guest_t *g, /* Track new region */ if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, source_inherited_at_fork) < 0) - return -LINUX_ENOMEM; + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); @@ -3290,11 +3799,11 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Should not reach here */ - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); } /* sys_madvise. */ @@ -3496,20 +4005,34 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + /* An interior removal from a file-backed region needs a second owned fd + * for the surviving right half. Reserve it before changing overlays, + * page tables, or host memory so descriptor exhaustion is failure-atomic. + */ + int remove_fd = -1; + if (guest_region_remove_prepare(g, unmap_off, end, &remove_fd) < 0) + return -LINUX_ENOMEM; + /* Restore slab backing under any active MAP_SHARED file overlay before * zeroing the host VA. Without this, the memset below would write zeros * directly into the file. */ int cleanup_err = cleanup_overlays_in_range(g, unmap_off, end); - if (cleanup_err < 0) + if (cleanup_err < 0) { + if (remove_fd >= 0) + close(remove_fd); return cleanup_err; + } /* Invalidate PTEs first. This may need to split a 2MiB block which can fail * if the page table pool is exhausted. Failing before region removal keeps * metadata consistent. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); return -LINUX_ENOMEM; + } for (int i = 0; i < g->nregions; i++) { guest_region_t *r = &g->regions[i]; if (r->start >= end) @@ -3522,7 +4045,7 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) uint64_t zend = (r->end < end) ? r->end : end; memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); } - guest_region_remove(g, unmap_off, end); + guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = unmap_off; if (unmap_off < g->mmap_rx_gap_hint) @@ -3576,9 +4099,17 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { if (region_range_overlaps(g, addr, addr + length)) { - if (guest_invalidate_ptes(g, addr, addr + length) < 0) + int remove_fd = -1; + if (guest_region_remove_prepare(g, addr, addr + length, + &remove_fd) < 0) return -LINUX_ENOMEM; - guest_region_remove(g, addr, addr + length); + if (guest_invalidate_ptes(g, addr, addr + length) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + guest_region_remove_reserved(g, addr, addr + length, + remove_fd); } return 0; } diff --git a/tests/manifest.txt b/tests/manifest.txt index 1e3948f6..4e379b07 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -110,6 +110,7 @@ test-mmap-hint [section] mremap tests test-mremap test-mremap-infra +test-mremap-fork-tracking test-shim-cred-race [section] msync MAP_SHARED tests diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c new file mode 100644 index 00000000..0bdf9b31 --- /dev/null +++ b/tests/test-mremap-fork-tracking.c @@ -0,0 +1,842 @@ +/* + * elfuse-internal mremap fork-tracking tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * These cases exercise elfuse's inherited-at-fork region bookkeeping, not a + * portable Linux ABI contract. Do not add this binary to test-matrix.sh: + * Linux can merge the adjacent file mappings below, and extends MAP_SHARED + * mappings directly from the backing file rather than creating elfuse's + * child-private tracking tail. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif +#ifndef MREMAP_FIXED +#define MREMAP_FIXED 2 +#endif + +static bool set_program_break(uintptr_t address) +{ + return brk((void *) address) == 0 && sbrk(0) == (void *) address; +} + +static ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) +{ + if (bufsz == 0) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + ssize_t total = 0; + while ((size_t) total < bufsz - 1) { + ssize_t n = read(fd, buf + total, bufsz - 1 - (size_t) total); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + int saved_errno = errno; + (void) close(fd); + errno = saved_errno; + return -1; + } + if (n == 0) + break; + total += n; + } + buf[total] = '\0'; + (void) close(fd); + return total; +} + +static void *reserve_then_map_fixed(size_t reserve_length, + size_t mapping_length, + int prot, + int flags, + int fd, + off_t offset) +{ + void *base = mmap(NULL, reserve_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) + return MAP_FAILED; + if (munmap(base, reserve_length) != 0) { + int saved_errno = errno; + (void) munmap(base, reserve_length); + errno = saved_errno; + return MAP_FAILED; + } + return mmap(base, mapping_length, prot, flags | MAP_FIXED, fd, offset); +} + +static bool parse_maps_hex(const char **cursor, + const char *end, + char delimiter, + uintptr_t *value) +{ + const char *start = *cursor; + uintptr_t parsed = 0; + while (*cursor < end && **cursor != delimiter) { + unsigned int digit; + char c = **cursor; + if (c >= '0' && c <= '9') + digit = (unsigned int) (c - '0'); + else if (c >= 'a' && c <= 'f') + digit = (unsigned int) (c - 'a' + 10); + else + return false; + parsed = parsed * 16 + digit; + (*cursor)++; + } + if (*cursor == start || *cursor >= end) + return false; + *value = parsed; + return true; +} + +static bool mapping_is_rw_private(uintptr_t address) +{ + char maps[64 * 1024]; + ssize_t length = read_file_nul("/proc/self/maps", maps, sizeof(maps)); + if (length <= 0) + return false; + + const char *cursor = maps; + const char *end = maps + length; + while (cursor < end) { + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&cursor, end, '-', &start)) + return false; + cursor++; + if (!parse_maps_hex(&cursor, end, ' ', &limit)) + return false; + cursor++; + if (cursor + 4 > end) + return false; + if (start <= address && address < limit) { + return cursor[0] == 'r' && cursor[1] == 'w' && cursor[2] == '-' && + cursor[3] == 'p'; + } + while (cursor < end && *cursor != '\n') + cursor++; + if (cursor < end) + cursor++; + } + return false; +} + +static char maps_buffer[1024 * 1024]; + +static bool read_maps_snapshot(ssize_t *length_out) +{ + ssize_t length = + read_file_nul("/proc/self/maps", maps_buffer, sizeof(maps_buffer)); + if (length <= 0 || (size_t) length == sizeof(maps_buffer) - 1) + return false; + *length_out = length; + return true; +} + +static bool count_maps_entries(int *count_out) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + int count = 0; + for (ssize_t i = 0; i < length; i++) { + if (maps_buffer[i] == '\n') + count++; + } + *count_out = count; + return true; +} + +static bool heap_headers_do_not_overlap(int *heap_count) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + const char heap_name[] = "[heap]"; + const char *cursor = maps_buffer; + const char *end = maps_buffer + length; + uintptr_t previous_end = 0; + int count = 0; + + while (cursor < end) { + const char *line = cursor; + while (cursor < end && *cursor != '\n') + cursor++; + const char *line_end = cursor; + if (cursor < end) + cursor++; + + bool is_heap = false; + for (const char *p = line; p + sizeof(heap_name) - 1 <= line_end; p++) { + if (memcmp(p, heap_name, sizeof(heap_name) - 1) == 0) { + is_heap = true; + break; + } + } + if (!is_heap) + continue; + + const char *field = line; + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&field, line_end, '-', &start)) + return false; + field++; + if (!parse_maps_hex(&field, line_end, ' ', &limit)) + return false; + if (count > 0 && start < previous_end) + return false; + previous_end = limit; + count++; + } + + *heap_count = count; + return true; +} + +static void test_postfork_adjacent_anon_rejected(void) +{ + TEST("mremap rejects unrelated post-fork tail"); + + const size_t span = 64 * 1024; + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (first == MAP_FAILED) { + FAIL("inherited anonymous mmap failed"); + return; + } + void *base = first; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(first, span); + return; + } + if (pid == 0) { + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (second != (char *) base + span) + _exit(30); + + errno = 0; + void *shrunk = mremap(base, 2 * span, span, 0); + if (shrunk == MAP_FAILED && errno == EFAULT) + _exit(0); + _exit(31); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("mremap accepted unrelated post-fork mapping"); + + munmap(first, span); +} + +static void test_adjacent_file_vmas_rejected(void) +{ + TEST("mremap rejects adjacent independent file VMAs"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-adjacent-XXXXXX"; + int fd1 = mkstemp(tmpl); + if (fd1 < 0) { + FAIL("mkstemp failed"); + return; + } + int fd2 = open(tmpl, O_RDWR); + unlink(tmpl); + if (fd2 < 0 || ftruncate(fd1, (off_t) (2 * span)) != 0) { + FAIL("file setup failed"); + if (fd2 >= 0) + close(fd2); + close(fd1); + return; + } + + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd1, 0); + void *base = first; + if (first == MAP_FAILED) { + FAIL("address reservation or first file mmap failed"); + close(fd2); + close(fd1); + return; + } + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd2, (off_t) span); + if (first != base || second != (char *) base + span) { + FAIL("adjacent file mmap failed"); + if (first == base) + munmap(first, span); + if (second == (char *) base + span) + munmap(second, span); + close(fd2); + close(fd1); + return; + } + + errno = 0; + void *q = mremap(base, 2 * span, 2 * span, 0); + if (q == MAP_FAILED && errno == EFAULT) + PASS(); + else + FAIL("mremap accepted two independent tracker records"); + + munmap(first, span); + munmap(second, span); + close(fd2); + close(fd1); +} + +static void test_file_backed_fork_split_move(void) +{ + TEST("MAP_SHARED file: mremap across fork-split source"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-cf-mremap-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0 || pwrite(fd, "F", 1, 0) != 1 || + pwrite(fd, "X", 1, (off_t) span) != 1) { + FAIL("file setup"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap"); + close(fd); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, span); + close(fd); + return; + } + + if (pid == 0) { + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(40); + if (grown[0] != 'F' || grown[span] != 0) + _exit(41); + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(42); + + /* A second fork marks both tracker records inherited. Their stable + * logical-VMA lineage must still allow the grandchild to mremap the + * complete source. + */ + pid_t grandchild = fork(); + if (grandchild < 0) + _exit(43); + if (grandchild == 0) { + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved == MAP_FAILED) + _exit(44); + if (moved == grown) + _exit(45); + if (moved[0] != 'F' || moved[span] != 0 || moved[2 * span] != 0) + _exit(46); + + munmap(moved, 3 * span); + munmap(blocker, span); + _exit(0); + } + + int grandchild_status = 0; + if (waitpid(grandchild, &grandchild_status, 0) < 0 || + !WIFEXITED(grandchild_status)) + _exit(47); + if (WEXITSTATUS(grandchild_status) != 0) + _exit(WEXITSTATUS(grandchild_status)); + munmap(grown, 2 * span); + munmap(blocker, span); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid"); + } else if (!WIFEXITED(status)) { + FAIL("child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "child mremap failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_file_backed_mprotect_fragments_move(void) +{ + TEST("MAP_SHARED file: mremap across restored mprotect fragments"); + + const size_t span = 64 * 1024; + const size_t old_size = 3 * span; + const size_t new_size = 4 * span; + char tmpl[] = "/tmp/elfuse-mremap-fragments-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) new_size) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(5 * span, old_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'A'; + p[span] = 'B'; + p[2 * span] = 'C'; + + if (mprotect(p + span, span, PROT_READ) != 0 || + mprotect(p + span, span, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect split and restore failed"); + munmap(p, old_size); + close(fd); + return; + } + + void *blocker = mmap(p + old_size, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != p + old_size) { + FAIL("move blocker mmap failed"); + munmap(p, old_size); + close(fd); + return; + } + + char *moved = mremap(p, old_size, new_size, MREMAP_MAYMOVE); + if (moved == MAP_FAILED || moved == p || moved[0] != 'A' || + moved[span] != 'B' || moved[2 * span] != 'C' || moved[3 * span] != 0) { + FAIL("fragmented logical VMA did not move intact"); + if (moved == MAP_FAILED) + munmap(p, old_size); + else + munmap(moved, new_size); + } else { + moved[3 * span] = 'D'; + PASS(); + munmap(moved, new_size); + } + + munmap(blocker, span); + close(fd); +} + +static void test_file_backed_fixed_move_from_same_vma(void) +{ + TEST("MAP_SHARED file: fixed subrange move keeps source fd live"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-fixed-source-fd-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (3 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'D'; + p[span] = 'M'; + p[2 * span] = 'S'; + + char *moved = + mremap(p + 2 * span, span, span, MREMAP_MAYMOVE | MREMAP_FIXED, p); + if (moved != p || moved[0] != 'S' || moved[span] != 'M') + FAIL("fixed subrange move lost its source backing fd"); + else + PASS(); + + munmap(p, 3 * span); + close(fd); +} + +static void test_file_backed_mremap_emfile_atomic(void) +{ + TEST("MAP_SHARED fork growth: mremap EMFILE preserves source"); + + const size_t span = 64 * 1024; + const size_t page_size = 4096; + const int fill_limit = 4096; + char tmpl[] = "/tmp/elfuse-mremap-emfile-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(p, span); + close(fd); + return; + } + if (pid == 0) { + const size_t filler_reservation_size = + (size_t) fill_limit * 2 * page_size; + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(60); + grown[span] = 'T'; + grown[2 * span - 1] = 'Z'; + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(61); + + char *fixed_probe = + reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, (off_t) span); + if (fixed_probe == MAP_FAILED) + _exit(72); + fixed_probe[0] = 'D'; + fixed_probe[2 * span] = 'S'; + + int last_dup = -1; + for (;;) { + int duplicated = dup(fd); + if (duplicated < 0) + break; + last_dup = duplicated; + } + if (errno != EMFILE || last_dup < 0) + _exit(62); + /* Keep one guest slot available for the /proc/self/maps proof below. + * The close also releases one host descriptor; the file-backed filler + * loop consumes it again before finding the actual host/table limit. */ + if (close(last_dup) != 0) + _exit(69); + + void *filler_base = mmap(NULL, filler_reservation_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (filler_base == MAP_FAILED || + munmap(filler_base, filler_reservation_size) != 0) + _exit(70); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < fill_limit; filler_count++) { + void *filler = mmap( + (char *) filler_base + (size_t) filler_count * 2 * page_size, + page_size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, fd, 0); + if (filler == MAP_FAILED) + break; + last_filler = filler; + } + if (filler_count == 0 || filler_count == fill_limit || errno != ENOMEM) + _exit(63); + + /* The first failed filler proves no host descriptor remains. Freeing + * one owned tracker fd lets mremap's prefix dup succeed and forces the + * second, tail-owned dup to hit EMFILE. + */ + if (munmap(last_filler, page_size) != 0) + _exit(64); + + /* With one host and one guest descriptor available, maps must be + * readable. Near the fixed tracker capacity, VMA count alone cannot + * prove whether tracker pressure or host descriptors ended the filler + * loop, so skip instead of attributing an ambiguous ENOMEM to the + * second mremap dup. */ + int maps_count = 0; + if (!count_maps_entries(&maps_count) || maps_count >= 4000) + _exit(77); + + errno = 0; + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved != MAP_FAILED || errno != ENOMEM) + _exit(65); + + volatile char *source = grown; + if (source[0] != 'I' || source[span] != 'T' || + source[2 * span - 1] != 'Z') + _exit(66); + if (mprotect(grown, 2 * span, PROT_READ) != 0 || source[0] != 'I' || + source[span] != 'T' || + mprotect(grown, 2 * span, PROT_READ | PROT_WRITE) != 0) + _exit(67); + source[1] = 'J'; + source[span + 1] = 'U'; + if (source[1] != 'J' || source[span + 1] != 'U') + _exit(68); + + /* Exactly one host descriptor is free again. A fixed subrange move + * consumes it for target tracking, then must fail atomically when the + * source-boundary snapshot cannot duplicate its backing fd. */ + errno = 0; + char *fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != MAP_FAILED || errno != ENOMEM || fixed_probe[0] != 'D' || + fixed_probe[2 * span] != 'S') + _exit(73); + + /* Release guest dup slots and filler tracker descriptors, then retry. + * A failed split that published backing_fd=-1 would make this retry + * fail even though descriptor capacity is now available. */ + for (int guest_fd = 0; guest_fd < 4096; guest_fd++) { + if (guest_fd != fd) + (void) close(guest_fd); + } + if (munmap(filler_base, filler_reservation_size) != 0) + _exit(74); + + fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != fixed_probe || fixed[0] != 'S') + _exit(75); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("EMFILE child terminated abnormally"); + } else if (WEXITSTATUS(status) == 77) { + printf("SKIP: fd exhaustion and region-table pressure are ambiguous\n"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "EMFILE child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_heap_tail_mprotect_then_grow(void) +{ + TEST("brk growth does not reuse protected heap tail"); + + const uintptr_t page_size = 4096; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(50); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(51); + if (!set_program_break(final_end)) + _exit(52); + + if (!mapping_is_rw_private(protected_end)) + _exit(54); + + if (!set_program_break(protected_end)) + _exit(55); + if (!set_program_break(final_end)) + _exit(56); + if (!mapping_is_rw_private(protected_end)) + _exit(57); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("new brk page inherited stale tail protection"); + + (void) set_program_break(original); +} + +static void test_heap_growth_with_full_region_table(void) +{ + TEST("brk growth with full region table has no overlapping heap headers"); + + const uintptr_t page_size = 4096; + const int fill_limit = 8192; + const int minimum_fill = 2048; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(80); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(81); + + int filled = 0; + for (; filled < fill_limit; filled++) { + int prot = (filled & 1) ? PROT_NONE : PROT_READ; + void *q = + mmap(NULL, page_size, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) + break; + } + if (filled < minimum_fill || filled == fill_limit || errno != ENOMEM) + _exit(82); + + if (!set_program_break(final_end)) + _exit(83); + volatile char *new_page = (char *) protected_end; + new_page[0] = 'H'; + if (new_page[0] != 'H') + _exit(84); + + int heap_count = 0; + if (!heap_headers_do_not_overlap(&heap_count) || heap_count < 2) + _exit(85); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("region-table child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[96]; + snprintf(buf, sizeof(buf), "region-table child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + (void) set_program_break(original); +} + +int main(void) +{ + printf("test-mremap-fork-tracking: elfuse mremap tracker tests\n"); + + test_postfork_adjacent_anon_rejected(); + test_adjacent_file_vmas_rejected(); + test_file_backed_fork_split_move(); + test_file_backed_mprotect_fragments_move(); + test_file_backed_fixed_move_from_same_vma(); + test_file_backed_mremap_emfile_atomic(); + test_heap_tail_mprotect_then_grow(); + test_heap_growth_with_full_region_table(); + + SUMMARY("test-mremap-fork-tracking"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-mremap-tail-emfile.c b/tests/test-mremap-tail-emfile.c new file mode 100644 index 00000000..99ce2466 --- /dev/null +++ b/tests/test-mremap-tail-emfile.c @@ -0,0 +1,235 @@ +/* + * test-mremap-tail-emfile exercises file-backed mremap bookkeeping while the + * guest and host descriptor tables are exhausted. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This probe deliberately uses libc interfaces so it can serve as the + * repository's portable C implementation of the regression. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif + +static const size_t PAGE_SIZE = 4096; +static const size_t SPAN = 64 * 1024; +static const int FILL_LIMIT = 4096; + +static void child_exit(int status) +{ + _exit(status); +} + +static void child_fail(int status) +{ + child_exit(status); +} + +static void child_probe(int file_fd, unsigned char *base) +{ + void *result = mremap(base, SPAN, 3 * SPAN, 0); + if (result == MAP_FAILED || result != base) + child_fail(10); + + base[SPAN] = 'T'; + base[2 * SPAN] = 'R'; + base[SPAN + PAGE_SIZE] = 'S'; + base[SPAN + 3 * PAGE_SIZE] = 'M'; + + /* Exhaust both descriptor tables, then leave one slot available. */ + int last_dup = -1; + int dup_error = 0; + for (;;) { + int duplicated = dup(file_fd); + if (duplicated < 0) { + dup_error = errno; + break; + } + last_dup = duplicated; + } + if (dup_error != EMFILE || last_dup < 0) + child_fail(11); + if (close(last_dup) != 0) + child_fail(12); + + /* Fill the region table with one-page file mappings separated by holes. */ + const size_t filler_length = (size_t) FILL_LIMIT * 2 * PAGE_SIZE; + void *reserved = mmap(NULL, filler_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) + child_fail(13); + if (munmap(reserved, filler_length) != 0) + child_fail(13); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < FILL_LIMIT; filler_count++) { + void *address = (char *) reserved + + (size_t) filler_count * 2 * PAGE_SIZE; + void *filler = mmap(address, PAGE_SIZE, PROT_NONE, + MAP_PRIVATE | MAP_FIXED, file_fd, 0); + if (filler == MAP_FAILED) { + int filler_errno = errno; + if (filler_errno != ENOMEM || filler_count == 0) + child_fail(filler_errno == ENOMEM ? 16 : 14); + break; + } + if (filler != address) + child_fail(15); + last_filler = filler; + } + if (filler_count == FILL_LIMIT || last_filler == MAP_FAILED) + child_fail(16); + if (munmap(last_filler, PAGE_SIZE) != 0) + child_fail(17); + + /* Consume the one descriptor released by the last filler mapping. */ + int probe_fd = dup(file_fd); + if (probe_fd < 0) + child_fail(22); + + /* Splitting either boundary of the child-private tail must fail before + * changing memory or PTEs when no descriptor is available. */ + errno = 0; + result = mremap(base, SPAN + 2 * PAGE_SIZE, SPAN + PAGE_SIZE, 0); + int remap_errno = errno; + if (result != MAP_FAILED || remap_errno != ENOMEM || + base[SPAN + PAGE_SIZE] != 'S') + child_fail(23); + + errno = 0; + int munmap_result = munmap(base + SPAN + 3 * PAGE_SIZE, PAGE_SIZE); + int munmap_errno = errno; + if (munmap_result != -1 || munmap_errno != ENOMEM || + base[SPAN + 3 * PAGE_SIZE] != 'M') + child_fail(25); + + if (close(probe_fd) != 0) + child_fail(27); + + /* old_size ends inside the child-private tail. */ + errno = 0; + result = mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, + MREMAP_MAYMOVE); + remap_errno = errno; + + /* Drop all duplicate guest slots, retaining the original file fd. */ + for (int fd = 0; fd < FILL_LIMIT; fd++) { + if (fd != file_fd) + (void) close(fd); + } + if (munmap(reserved, filler_length) != 0) + child_fail(18); + + /* A successful move owns a target mapping; release it before msync. */ + if (result != MAP_FAILED) { + if (munmap(result, 2 * SPAN + PAGE_SIZE) != 0) + child_fail(21); + } else if (remap_errno != ENOMEM) { + child_fail(21); + } + + if (msync(base + 2 * SPAN, SPAN, MS_SYNC) != 0) + child_fail(19); + + unsigned char byte = 0; + if (pread(file_fd, &byte, 1, 2 * SPAN) != 1 || byte != 'R') + child_fail(20); + + child_exit(0); +} + +static void test_file_backed_mremap_tail_emfile(void) +{ + TEST("MAP_SHARED mremap tail EMFILE preserves source"); + + char path[] = "/tmp/elfuse-mremap-tail-emfile-XXXXXX"; + int file_fd = mkstemp(path); + if (file_fd < 0) { + FAIL("create temp file failed"); + return; + } + (void) unlink(path); + + const size_t file_length = 4 * SPAN; + if (ftruncate(file_fd, (off_t) file_length) != 0) { + FAIL("truncate temp file failed"); + close(file_fd); + return; + } + + const size_t reservation_length = file_length + PAGE_SIZE; + void *reserved = mmap(NULL, reservation_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED || munmap(reserved, reservation_length) != 0) { + FAIL("reserve source range failed"); + close(file_fd); + return; + } + + void *source_address = (char *) reserved + PAGE_SIZE; + void *mapped = mmap(source_address, SPAN, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file_fd, 0); + if (mapped == MAP_FAILED || mapped != source_address) { + FAIL("map source file failed"); + close(file_fd); + return; + } + + unsigned char *base = mapped; + base[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) munmap(base, SPAN); + close(file_fd); + return; + } + if (pid == 0) + child_probe(file_fd, base); + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("wait failed"); + } else if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + char message[96]; + if (WIFEXITED(status)) + snprintf(message, sizeof(message), "child failed at step %d", + WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) + snprintf(message, sizeof(message), "child killed by signal %d", + WTERMSIG(status)); + else + snprintf(message, sizeof(message), "child ended unexpectedly"); + FAIL(message); + } else { + PASS(); + } + + (void) munmap(base, SPAN); + close(file_fd); +} + +int main(void) +{ + printf("test-mremap-tail-emfile: file-backed mremap atomicity\n"); + test_file_backed_mremap_tail_emfile(); + SUMMARY("test-mremap-tail-emfile"); + return fails != 0; +} diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 3563b22c..5be4802d 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -19,7 +19,6 @@ #include #include "test-harness.h" -#include "test-util.h" int passes = 0, fails = 0; @@ -367,11 +366,16 @@ static void free_smaps(smaps_info_t *info) * on both success and failure. */ static bool load_smaps(const char *path, smaps_info_t *info) { + FILE *file = fopen(path, "r"); + if (!file) + return false; + char *buf = NULL; - size_t len = 0; - bool ok = read_file_dynamic_nul(path, &buf, &len) >= 0 && - parse_smaps(buf, len, info); + size_t capacity = 0; + ssize_t length = getdelim(&buf, &capacity, '\0', file); + bool ok = length >= 0 && parse_smaps(buf, (size_t) length, info); free(buf); + (void) fclose(file); return ok; } @@ -462,13 +466,69 @@ static bool read_exact(int fd, void *data, size_t len) return true; } +static bool write_exact(int fd, const void *data, size_t len) +{ + const char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = write(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, size_t stress_size) { - void *postfork = mmap(NULL, page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *postfork = MAP_FAILED; + uintptr_t target_end = target + 3 * page_size; + uintptr_t stress_end = stress + stress_size; + +#ifdef MAP_FIXED_NOREPLACE + /* Keep the probe in a known gap so the synthetic smaps builder cannot + * merge it with the target's final rw page or the stress fixture. */ + uintptr_t fixed_hint = 0x700000000000ULL; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) fixed_hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (candidate != MAP_FAILED) { + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + } + fixed_hint += 16 * page_size; + } +#endif + + if (postfork == MAP_FAILED) { + uintptr_t hint = stress_end + 64 * page_size; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (candidate == MAP_FAILED) + break; + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + hint += 64 * page_size; + } + } if (postfork == MAP_FAILED) return 1; @@ -590,7 +650,7 @@ int main(void) close(pipefd[0]); int result = child_probe((uintptr_t) target, (uintptr_t) stress, page_size, stress_size); - (void) write_fd_all(pipefd[1], &result, sizeof(result)); + (void) write_exact(pipefd[1], &result, sizeof(result)); close(pipefd[1]); _exit(result); } else { From e448861b7a88b6cf91022ec7503d8c442097251e Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 08:27:04 +0800 Subject: [PATCH 22/26] Format memory remap code Apply consistent line wrapping in the memory syscall implementation and its mremap regression test without changing behavior. --- src/syscall/mem.c | 9 +++------ tests/test-mremap-tail-emfile.c | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 45943aa5..7ea6b2ef 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -357,8 +357,7 @@ static int find_mremap_source(const guest_t *g, if (!collect_segments) return 0; - source->segments = - malloc((size_t) nsegments * sizeof(*source->segments)); + source->segments = malloc((size_t) nsegments * sizeof(*source->segments)); if (!source->segments) { dispose_mremap_source(source); return -LINUX_ENOMEM; @@ -371,8 +370,7 @@ static int find_mremap_source(const guest_t *g, const guest_region_t *r = &g->regions[index + segment_index]; uint64_t segment_end = r->end < end ? r->end : end; uint64_t segment_len = segment_end - cursor; - mremap_source_segment_t *segment = - &source->segments[segment_index]; + mremap_source_segment_t *segment = &source->segments[segment_index]; segment->start = cursor; segment->end = segment_end; segment->gpa_base = expected_gpa; @@ -4108,8 +4106,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) close(remove_fd); return -LINUX_ENOMEM; } - guest_region_remove_reserved(g, addr, addr + length, - remove_fd); + guest_region_remove_reserved(g, addr, addr + length, remove_fd); } return 0; } diff --git a/tests/test-mremap-tail-emfile.c b/tests/test-mremap-tail-emfile.c index 99ce2466..ce94e872 100644 --- a/tests/test-mremap-tail-emfile.c +++ b/tests/test-mremap-tail-emfile.c @@ -79,8 +79,8 @@ static void child_probe(int file_fd, unsigned char *base) void *last_filler = MAP_FAILED; int filler_count = 0; for (; filler_count < FILL_LIMIT; filler_count++) { - void *address = (char *) reserved + - (size_t) filler_count * 2 * PAGE_SIZE; + void *address = + (char *) reserved + (size_t) filler_count * 2 * PAGE_SIZE; void *filler = mmap(address, PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_FIXED, file_fd, 0); if (filler == MAP_FAILED) { @@ -124,8 +124,8 @@ static void child_probe(int file_fd, unsigned char *base) /* old_size ends inside the child-private tail. */ errno = 0; - result = mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, - MREMAP_MAYMOVE); + result = + mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, MREMAP_MAYMOVE); remap_errno = errno; /* Drop all duplicate guest slots, retaining the original file fd. */ From a0171ddc8e077a57a4929235fd44940a11ed627a Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 15:29:47 +0800 Subject: [PATCH 23/26] Harden fork-aware smaps and mremap Preserve fork lineage and smaps consistency when restored regions receive new allocations. Flush shared-file contents before mremap removes source mappings. Make region-boundary preparation transactional so allocation or descriptor failures leave metadata unchanged. Add regressions for repeated post-fork allocations and misaligned moves. Cover PROT_NONE smaps output as well. Related: #259 --- docs/internals.md | 27 +++-- src/core/guest.c | 10 ++ src/core/guest.h | 5 + src/runtime/fork-state.c | 1 + src/runtime/procemu.c | 27 +++-- src/syscall/mem.c | 189 +++++++++++++++++++++++++++++- tests/test-mremap-fork-tracking.c | 147 +++++++++++++++++++++++ tests/test-proc-smap.c | 82 ++++++++----- 8 files changed, 430 insertions(+), 58 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index 3059fc32..9c30cfcf 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -797,28 +797,29 @@ under `/proc`, `/dev`, and a few Linux-expected compatibility files: `/proc/self/smaps` and `/proc//smaps` are generated from the same tracked VMA list as `/proc/self/maps`. The complete field set currently emitted for -each VMA is the maps header followed by these 25 fields, in this order: +each VMA is the maps header followed by these 24 fields, in this order: ```text Size, KernelPageSize, MMUPageSize, Rss, Pss, Pss_Dirty, Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty, Referenced, Anonymous, KSM, LazyFree, AnonHugePages, ShmemPmdMapped, FilePmdMapped, Shared_Hugetlb, Private_Hugetlb, Swap, SwapPss, Locked, THPeligible, -ProtectionKey, VmFlags +VmFlags ``` `Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are -reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a -non-zero value: in a fork child, writable private anonymous VMAs that existed -in the parent's CoW snapshot report their full VMA size because they are -logically shared with that snapshot. Every other numeric counter, including -`THPeligible` and `ProtectionKey`, is emitted as zero. `VmFlags` is -evidence-based and contains only the permission/sharing flags represented by -the tracked VMA (`rd`, `wr`, -`ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. -elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing -against a real Linux kernel should tolerate either conditional field being -absent. +reported as 4 KiB. In a fork child, writable private anonymous VMAs that +existed in the parent's CoW snapshot report their full VMA size for +`Shared_Dirty`, `Rss`, and `Pss`, keeping those coarse counters internally +consistent. Newly-created VMAs are excluded from that signal. Every other +numeric counter, including `THPeligible`, is emitted as a stable zero. +`ProtectionKey` (a Linux pkeys field added in Linux 4.9) is intentionally +omitted because the macOS host has no equivalent; consumers comparing against a +real Linux kernel should treat it as optional. `VmFlags` is evidence-based and +contains only the permission/sharing flags represented by the tracked VMA +(`rd`, `wr`, `ex`, `sh`, and `nr` when applicable); no untracked kernel flags +are invented. A VMA with no such evidence (for example `PROT_NONE`) still +prints the field's separating space as `VmFlags: `. These are coarse VMA-level values, not host page-residency, dirty-bit, or proportional-sharing accounting, so they are suitable for fork-safety checks but not precise memory profiling. The fork-child marker is tracked per VMA, so diff --git a/src/core/guest.c b/src/core/guest.c index 7620dcb2..d3d2ebf2 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -2040,6 +2040,16 @@ static uint64_t allocate_vma_id(guest_t *g) } } +void guest_reseed_next_vma_id(guest_t *g) +{ + uint64_t max_id = g->next_vma_id; + for (int i = 0; i < g->nregions; i++) { + if (g->regions[i].vma_id > max_id) + max_id = g->regions[i].vma_id; + } + g->next_vma_id = max_id; +} + int guest_region_add_ex_owned(guest_t *g, uint64_t start, uint64_t end, diff --git a/src/core/guest.h b/src/core/guest.h index 0645e70b..c16dca56 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -1214,6 +1214,11 @@ int guest_region_add_ex_owned_gpa(guest_t *g, bool inherited_at_fork, uint64_t vma_id); +/* Re-seed the logical-VMA allocator from a restored region snapshot. Fork IPC + * restores regions by value, so next_vma_id must be advanced past every + * serialized lineage before child-private mappings are added. */ +void guest_reseed_next_vma_id(guest_t *g); + /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest * MAP_FIXED_NOREPLACE reservations. diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 9ff5a799..49f0c6ac 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -933,6 +933,7 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) */ for (int i = 0; i < g->nregions; i++) g->regions[i].inherited_at_fork = true; + guest_reseed_next_vma_id(g); g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 4aa46b9b..b3f2aab8 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -2695,8 +2695,10 @@ static int proc_open_self_maps(const guest_t *g) * guest VMAs and logical fork snapshots, but it cannot observe kernel page * residency or dirty bits on the host. Writable private anonymous VMAs that * were present in the most recent fork snapshot report their full VMA size as - * Shared_Dirty; newly-created VMAs are excluded. All other fields that - * require kernel page accounting are stable zeroes. + * Shared_Dirty, Rss, and Pss; keeping those counters aligned avoids a + * self-contradictory snapshot while retaining a coarse fork-compatibility + * signal. Newly-created VMAs are excluded. All other fields that require + * kernel page accounting are stable zeroes. */ static int proc_open_self_smaps(const guest_t *g) { @@ -2727,15 +2729,22 @@ static int proc_open_self_smaps(const guest_t *g) bool logical_shared_dirty = e->inherited_at_fork && private_anon && (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; + uint64_t rss_kb = shared_dirty_kb; + uint64_t pss_kb = shared_dirty_kb; - char vmflags[64]; - size_t vmflags_len = 0; + /* Linux writes a separating space after VmFlags:, even when no + * evidence-based flags are available (for example a PROT_NONE VMA). + * Start with that space so the empty form is exactly "VmFlags: \n". + */ + char vmflags[64] = {' '}; + size_t vmflags_len = 1; #define APPEND_VMFLAG(flag) \ do { \ const char *token = (flag); \ size_t token_len = strlen(token); \ if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ - vmflags[vmflags_len++] = ' '; \ + if (vmflags_len > 1) \ + vmflags[vmflags_len++] = ' '; \ memcpy(vmflags + vmflags_len, token, token_len); \ vmflags_len += token_len; \ } \ @@ -2763,8 +2772,8 @@ static int proc_open_self_smaps(const guest_t *g) "Size: %llu kB\n" "KernelPageSize: 4 kB\n" "MMUPageSize: 4 kB\n" - "Rss: 0 kB\n" - "Pss: 0 kB\n" + "Rss: %llu kB\n" + "Pss: %llu kB\n" "Pss_Dirty: 0 kB\n" "Shared_Clean: 0 kB\n" "Shared_Dirty: %llu kB\n" @@ -2783,9 +2792,9 @@ static int proc_open_self_smaps(const guest_t *g) "SwapPss: 0 kB\n" "Locked: 0 kB\n" "THPeligible: 0\n" - "ProtectionKey: 0\n" "VmFlags:%s\n", header_len, header, (unsigned long long) size_kb, + (unsigned long long) rss_kb, (unsigned long long) pss_kb, (unsigned long long) shared_dirty_kb, vmflags) < 0) goto out; } @@ -3400,7 +3409,7 @@ int proc_intercept_open(const guest_t *g, return proc_open_self_maps(g); /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata - * and the coarse fork Shared_Dirty compatibility signal. + * and the coarse fork Shared_Dirty/Rss/Pss compatibility signal. */ if (!strcmp(path, "/proc/self/smaps")) return proc_open_self_smaps(g); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 7ea6b2ef..2d18ed8b 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -105,6 +105,10 @@ static int restore_snapshot_page_tables(guest_t *g, static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n); +static int64_t sync_shared_aliases_range(guest_t *g, + int backing_fd, + uint64_t file_start, + uint64_t file_end); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -251,6 +255,7 @@ typedef struct { uint64_t gpa_base; uint64_t offset; int backing_fd; /* borrowed from the region tracker */ + bool shared_non_overlay; bool overlay_active; uint64_t overlay_start; uint64_t overlay_end; @@ -376,6 +381,8 @@ static int find_mremap_source(const guest_t *g, segment->gpa_base = expected_gpa; segment->offset = expected_offset; segment->backing_fd = r->backing_fd; + segment->shared_non_overlay = + r->shared && r->backing_fd >= 0 && !region_has_live_overlay(r); segment->overlay_active = region_has_live_overlay(r); segment->overlay_start = r->overlay_start; segment->overlay_end = r->overlay_end; @@ -460,6 +467,71 @@ static void region_clip_overlay(guest_region_t *r) region_clear_overlay(r); } +/* Region-boundary edits are a prepare step for metadata snapshots. Keep a + * private copy of the tracker while the boundaries are split so an allocation + * failure (table capacity or backing-fd dup) cannot publish a half-split + * tracker. Split-created fds are closed on rollback; the original region fds + * remain owned by the original array and are restored verbatim. + */ +typedef struct { + guest_region_t *regions; + int nregions; +} region_array_txn_t; + +static int begin_region_array_txn(const guest_t *g, region_array_txn_t *txn) +{ + memset(txn, 0, sizeof(*txn)); + txn->nregions = g->nregions; + if (txn->nregions == 0) + return 0; + + txn->regions = malloc((size_t) txn->nregions * sizeof(*txn->regions)); + if (!txn->regions) { + txn->nregions = 0; + return -LINUX_ENOMEM; + } + memcpy(txn->regions, g->regions, + (size_t) txn->nregions * sizeof(*txn->regions)); + return 0; +} + +static bool txn_original_fd_present(const region_array_txn_t *txn, int fd) +{ + if (fd < 0) + return true; + for (int i = 0; i < txn->nregions; i++) { + if (txn->regions[i].backing_fd == fd) + return true; + } + return false; +} + +static void rollback_region_array_txn(guest_t *g, region_array_txn_t *txn) +{ + if (!txn || !txn->regions) + return; + + /* split_regions_at_boundary() only creates new fds; it never closes an + * original one. Close those new descriptors before restoring the copy. + */ + for (int i = 0; i < g->nregions; i++) { + int fd = g->regions[i].backing_fd; + if (fd >= 0 && !txn_original_fd_present(txn, fd)) + close(fd); + } + memcpy(g->regions, txn->regions, + (size_t) txn->nregions * sizeof(*txn->regions)); + g->nregions = txn->nregions; +} + +static void finish_region_array_txn(region_array_txn_t *txn) +{ + if (!txn) + return; + free(txn->regions); + memset(txn, 0, sizeof(*txn)); +} + static int split_regions_at_boundary(guest_t *g, uint64_t boundary) { if (boundary == 0) @@ -1288,6 +1360,34 @@ static bool mremap_source_has_overlay(const mremap_source_t *source) return false; } +/* A snapshot-style MAP_SHARED source may contain guest writes that have not + * reached its backing file yet. mremap destroys the source VMA (and may zero + * its slab backing) after copying, so publish those dirty bytes before any + * source cleanup. Live file overlays are excluded: the host page cache already + * owns their coherence and cleanup restores the slab before the source is + * removed. + */ +static int64_t flush_mremap_source_shared(guest_t *g, + const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + if (!segment->shared_non_overlay || segment->backing_fd < 0) + continue; + + uint64_t len = segment->end - segment->start; + uint64_t file_end = segment->offset + len; + if (file_end < segment->offset) + return -LINUX_EFAULT; + + int64_t err = sync_shared_aliases_range(g, segment->backing_fd, + segment->offset, file_end); + if (err < 0) + return err; + } + return 0; +} + typedef struct { uint64_t start; uint64_t end; @@ -1487,12 +1587,26 @@ static int capture_region_snapshots(guest_t *g, region_snapshot_t *snaps, int max_snaps) { + /* Split and snapshot as one metadata transaction. A failed boundary split, + * descriptor dup, or snapshot-capacity check must leave regions[] and its + * owned fds exactly as they were on entry. */ + region_array_txn_t txn; + int txn_err = begin_region_array_txn(g, &txn); + if (txn_err < 0) + return txn_err; + int split_err = split_regions_at_boundary(g, start); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return split_err; + } split_err = split_regions_at_boundary(g, end); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return split_err; + } int n = 0; for (int i = 0; i < g->nregions; i++) { @@ -1503,6 +1617,8 @@ static int capture_region_snapshots(guest_t *g, continue; if (n >= max_snaps) { close_region_snapshots(snaps, n); + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return -LINUX_ENOMEM; } @@ -1519,6 +1635,8 @@ static int capture_region_snapshots(guest_t *g, snap->backing_fd = dup(r->backing_fd); if (snap->backing_fd < 0) { close_region_snapshots(snaps, n); + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return -LINUX_ENOMEM; } } @@ -1530,6 +1648,7 @@ static int capture_region_snapshots(guest_t *g, str_copy_trunc(snap->name, r->name, sizeof(snap->name)); } + finish_region_array_txn(&txn); return n; } @@ -2122,12 +2241,23 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) uint64_t host_start = ALIGN_DOWN(start, hps); uint64_t host_end = ALIGN_UP(end, hps); + region_array_txn_t split_txn; + int txn_err = begin_region_array_txn(g, &split_txn); + if (txn_err < 0) + return txn_err; int split_err = split_regions_at_boundary(g, host_start); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &split_txn); + finish_region_array_txn(&split_txn); return split_err; + } split_err = split_regions_at_boundary(g, host_end); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &split_txn); + finish_region_array_txn(&split_txn); return split_err; + } + finish_region_array_txn(&split_txn); /* Snapshot affected ranges first; the host-side mmap calls below do not * touch the region array, but a future caller invariant is to allow this @@ -3313,9 +3443,27 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, -LINUX_ENOMEM); } + /* Keep both boundary captures in one transaction. The per-capture + * helper rolls back its own edits, while this outer guard also undoes + * a successful source capture if destination capture or the preflight + * shared-file flush fails afterward. */ + region_array_txn_t capture_txn; + int capture_txn_err = begin_region_array_txn(g, &capture_txn); + if (capture_txn_err < 0) { + free(source_snaps); + free(dest_snaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, capture_txn_err); + } + source_nsnaps = capture_region_snapshots( g, old_off, old_off + old_size, source_snaps, GUEST_MAX_REGIONS); if (source_nsnaps < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); free(source_snaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3327,6 +3475,8 @@ int64_t sys_mremap(guest_t *g, int rebind_err = rebind_mremap_source_backings(&source, source_snaps, source_nsnaps); if (rebind_err < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); dispose_region_snapshots(&source_snaps, &source_nsnaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3338,6 +3488,8 @@ int64_t sys_mremap(guest_t *g, dest_nsnaps = capture_region_snapshots(g, new_off, new_off + new_size, dest_snaps, GUEST_MAX_REGIONS); if (dest_nsnaps < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); dispose_region_snapshots(&source_snaps, &source_nsnaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3347,6 +3499,20 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, dest_nsnaps); } + int64_t flush_err = flush_mremap_source_shared(g, &source); + if (flush_err < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); + dispose_region_snapshots(&dest_snaps, &dest_nsnaps); + dispose_region_snapshots(&source_snaps, &source_nsnaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, flush_err); + } + finish_region_array_txn(&capture_txn); + if (source_overlay) { int cleanup_err = cleanup_overlays_in_range(g, old_off, old_off + old_size); @@ -3681,6 +3847,15 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, -LINUX_ENOMEM); } + int64_t flush_err = flush_mremap_source_shared(g, &source); + if (flush_err < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, flush_err); + } + int source_remove_fd = -1; if (guest_region_remove_prepare(g, old_off, old_off + old_size, &source_remove_fd) < 0) { @@ -4312,8 +4487,8 @@ static int64_t sync_shared_aliases_range(guest_t *g, uint8_t original[4096]; for (uint64_t chunk_start = file_start; chunk_start < file_end;) { - uint64_t chunk_end = ALIGN_DOWN(chunk_start + sizeof(original), 4096); - if (chunk_end <= chunk_start || chunk_end > file_end) + uint64_t chunk_end = chunk_start + sizeof(original); + if (chunk_end < chunk_start || chunk_end > file_end) chunk_end = file_end; size_t chunk_len = (size_t) (chunk_end - chunk_start); @@ -4329,6 +4504,8 @@ static int64_t sync_shared_aliases_range(guest_t *g, const guest_region_t *src = &g->regions[i]; if (!src->shared || src->backing_fd < 0) continue; + if (src->overlay_active) + continue; if (!(src->prot & LINUX_PROT_WRITE)) continue; if (!same_backing_file(backing_fd, src->backing_fd)) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index 0bdf9b31..bac30b2e 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -261,6 +261,55 @@ static void test_postfork_adjacent_anon_rejected(void) munmap(first, span); } +static void test_postfork_repeated_allocations_keep_lineage(void) +{ + TEST("repeated post-fork allocations never alias inherited VMA IDs"); + + const size_t span = 64 * 1024; + const int attempts = 256; + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (first == MAP_FAILED) { + FAIL("inherited anonymous mmap failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(first, span); + return; + } + if (pid == 0) { + for (int i = 0; i < attempts; i++) { + void *tail = + mmap((char *) first + span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (tail != (char *) first + span) + _exit(90); + + /* If child allocation reused the inherited source's vma_id, + * find_mremap_source would incorrectly accept the adjacent pair. + * A reseeded allocator keeps the logical lineages distinct. */ + errno = 0; + void *same = mremap(first, 2 * span, 2 * span, 0); + if (same != MAP_FAILED || errno != EFAULT) + _exit(91); + if (munmap(tail, span) != 0) + _exit(92); + } + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("post-fork VMA IDs collided during repeated allocation"); + munmap(first, span); +} + static void test_adjacent_file_vmas_rejected(void) { TEST("mremap rejects adjacent independent file VMAs"); @@ -317,6 +366,102 @@ static void test_adjacent_file_vmas_rejected(void) close(fd1); } +static void test_misaligned_shared_mremap_writeback(void) +{ + TEST("misaligned MAP_SHARED mremap writes back after move"); + + const size_t page = 4096; + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-misaligned-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (2 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + /* Deliberately place the source one host page into its reservation. This + * keeps the guest mapping page-aligned while making it non-2MiB-aligned, + * so the MAP_SHARED overlay/mremap path exercises a split HVF segment. */ + size_t reservation_length = 2 * span + page; + void *reservation = mmap(NULL, reservation_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED || + munmap(reservation, reservation_length) != 0) { + FAIL("source reservation failed"); + if (reservation != MAP_FAILED) + (void) munmap(reservation, reservation_length); + close(fd); + return; + } + char *source_address = (char *) reservation + page; + char *source = mmap(source_address, span, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (source == MAP_FAILED || source != source_address) { + FAIL("misaligned MAP_SHARED mmap failed"); + close(fd); + return; + } + source[0] = 'A'; + source[page + 7] = 'B'; + + /* Block in-place growth so mremap must move the mapping and leave the + * destination on the snapshot-style shared-writeback path. */ + void *blocker = mmap(source + span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != source + span) { + FAIL("mremap blocker mmap failed"); + munmap(source, span); + close(fd); + return; + } + + char *moved = mremap(source, span, 2 * span, MREMAP_MAYMOVE); + if (moved == MAP_FAILED || moved == source) { + FAIL("misaligned MAP_SHARED mremap did not move"); + if (moved != MAP_FAILED) + munmap(moved, 2 * span); + else + munmap(source, span); + munmap(blocker, span); + close(fd); + return; + } + if (moved[0] != 'A' || moved[page + 7] != 'B') { + FAIL("mremap move corrupted source bytes"); + munmap(moved, 2 * span); + munmap(blocker, span); + close(fd); + return; + } + + moved[page + 7] = 'C'; + moved[span + 19] = 'D'; + errno = 0; + int sync_result = msync(moved, 2 * span, MS_SYNC); + int sync_errno = errno; + unsigned char first = 0, extension = 0; + bool writeback_ok = sync_result == 0 && + pread(fd, &first, 1, (off_t) (page + 7)) == 1 && + pread(fd, &extension, 1, (off_t) (span + 19)) == 1 && + first == 'C' && extension == 'D'; + if (writeback_ok) + PASS(); + else { + errno = sync_errno; + FAIL("mremap destination did not write back MAP_SHARED bytes"); + } + + munmap(moved, 2 * span); + munmap(blocker, span); + close(fd); +} + static void test_file_backed_fork_split_move(void) { TEST("MAP_SHARED file: mremap across fork-split source"); @@ -829,7 +974,9 @@ int main(void) printf("test-mremap-fork-tracking: elfuse mremap tracker tests\n"); test_postfork_adjacent_anon_rejected(); + test_postfork_repeated_allocations_keep_lineage(); test_adjacent_file_vmas_rejected(); + test_misaligned_shared_mremap_writeback(); test_file_backed_fork_split_move(); test_file_backed_mprotect_fragments_move(); test_file_backed_fixed_move_from_same_vma(); diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 5be4802d..0096c443 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -30,8 +30,12 @@ typedef struct { unsigned long long size_kb; unsigned long long kernel_page_kb; unsigned long long mmu_page_kb; + unsigned long long rss_kb; + unsigned long long pss_kb; unsigned long long shared_dirty_kb; bool have_size; + bool have_rss; + bool have_pss; bool have_shared_dirty; bool have_vmflags; bool vmflags_wr; @@ -43,8 +47,7 @@ static const char *const smaps_fields[] = { "Private_Clean:", "Private_Dirty:", "Referenced:", "Anonymous:", "KSM:", "LazyFree:", "AnonHugePages:", "ShmemPmdMapped:", "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", "Swap:", - "SwapPss:", "Locked:", "THPeligible:", "ProtectionKey:", - "VmFlags:", + "SwapPss:", "Locked:", "THPeligible:", "VmFlags:", }; #define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) @@ -165,8 +168,12 @@ static bool parse_header(const char *line, smaps_vma_t *vma) vma->size_kb = 0; vma->kernel_page_kb = 0; vma->mmu_page_kb = 0; + vma->rss_kb = 0; + vma->pss_kb = 0; vma->shared_dirty_kb = 0; vma->have_size = false; + vma->have_rss = false; + vma->have_pss = false; vma->have_shared_dirty = false; vma->have_vmflags = false; vma->vmflags_wr = false; @@ -211,6 +218,12 @@ static bool parse_vmflags(const char *line, bool *writable) return false; const char *p = line + label_len; + if (*p && !isspace((unsigned char) *p)) + return false; + /* Keep the provider's stable spelling for an empty flag set: the Linux + * field always has a separating space, even when no token follows. */ + if (!*p) + return false; bool has_wr = false; while (*(p = skip_space(p))) { const char *start = p; @@ -233,21 +246,8 @@ static bool parse_vmflags(const char *line, bool *writable) static bool finish_vma(smaps_vma_t *vma, size_t field_index) { return field_index == SMAPS_FIELD_COUNT && vma->have_size && - vma->have_shared_dirty && vma->have_vmflags; -} - -/* THPeligible and ProtectionKey are conditional in real Linux kernels. The - * elfuse provider always emits both, but qemu may legitimately omit either; - * when the parser reaches one of those labels, advance over any missing - * optional fields before parsing VmFlags. */ -static void skip_optional_fields(const char *line, size_t *field_index) -{ - if (*field_index == SMAPS_FIELD_COUNT - 3 && - (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || - !strncmp(line, "VmFlags:", 8))) - (*field_index)++; - if (*field_index == SMAPS_FIELD_COUNT - 2 && !strncmp(line, "VmFlags:", 8)) - (*field_index)++; + vma->have_rss && vma->have_pss && vma->have_shared_dirty && + vma->have_vmflags; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) @@ -278,14 +278,10 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) goto fail; *next = '\0'; - /* Linux normally places headers back-to-back; the synthetic proc - * provider separates records with one blank line. Accept that - * separator only after a complete record. */ + /* smaps records are contiguous: a blank line is not a field and is + * rejected so malformed snapshots cannot hide an out-of-order VMA. */ if (!*line) { - if (!have_current || field_index != SMAPS_FIELD_COUNT) - goto fail; - line = next + 1; - continue; + goto fail; } smaps_vma_t header; @@ -306,7 +302,6 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) } else { if (!have_current) goto fail; - skip_optional_fields(line, &field_index); if (field_index >= SMAPS_FIELD_COUNT) goto fail; unsigned long long value; @@ -320,10 +315,18 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.kernel_page_kb = value; if (field_index == 2) current.mmu_page_kb = value; + if (field_index == 3) + current.rss_kb = value; + if (field_index == 4) + current.pss_kb = value; if (field_index == 7) current.shared_dirty_kb = value; if (field_index == 0) current.have_size = true; + if (field_index == 3) + current.have_rss = true; + if (field_index == 4) + current.have_pss = true; if (field_index == 7) current.have_shared_dirty = true; field_index++; @@ -432,10 +435,16 @@ static bool validate_layout(const smaps_info_t *info, middle->mmu_page_kb != 4 || last->kernel_page_kb != 4 || last->mmu_page_kb != 4) return false; + if (first->rss_kb != first->shared_dirty_kb || + first->pss_kb != first->shared_dirty_kb || middle->rss_kb != 0 || + middle->pss_kb != 0 || last->rss_kb != last->shared_dirty_kb || + last->pss_kb != last->shared_dirty_kb) + return false; - /* Every stress-map page alternates permissions, so each page must remain - * its own VMA. Requiring the full count makes a short read or a producer - * cap observable instead of merely checking that some blocks exceed 256. + /* Every stress-map page has deliberately distinct permissions (with one + * PROT_NONE probe), so each page must remain its own VMA. Requiring the + * full count makes a short read or a producer cap observable instead of + * merely checking that some blocks exceed 256. */ size_t expected_stress_vmas = stress_size / page_size; if (expected_stress_vmas <= 256 || @@ -448,6 +457,11 @@ static bool validate_layout(const smaps_info_t *info, page->end != stress + (i + 1) * page_size) return false; } + const smaps_vma_t *stress_none = find_vma(info, stress + 2 * page_size); + if (!stress_none || strcmp(stress_none->perms, "---p") || + stress_none->vmflags_wr || stress_none->shared_dirty_kb != 0 || + stress_none->rss_kb != 0 || stress_none->pss_kb != 0) + return false; return true; } @@ -549,13 +563,16 @@ static int child_probe(uintptr_t target, const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); const smaps_vma_t *stress_ro = find_vma(&info, stress); const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + const smaps_vma_t *stress_none = + find_vma(&info, stress + 2 * page_size); const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || - !postfork_vma || first->shared_dirty_kb == 0 || + !stress_none || !postfork_vma || first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || stress_rw->shared_dirty_kb == 0 || - postfork_vma->shared_dirty_kb != 0) { + stress_none->shared_dirty_kb != 0 || stress_none->rss_kb != 0 || + stress_none->pss_kb != 0 || postfork_vma->shared_dirty_kb != 0) { free_smaps(&info); munmap(postfork, page_size); return 1; @@ -604,6 +621,11 @@ int main(void) break; } } + /* Keep one page inaccessible so the parser and formatter exercise + * PROT_NONE coverage and the required empty VmFlags spelling. */ + if (fixture_ok && + mprotect(stress + 2 * page_size, page_size, PROT_NONE) < 0) + fixture_ok = false; } TEST("smaps headers, order, fields, and completeness"); From b09c74542e39a2ac65ae77ba7d1c2aea4a2f86b9 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 16:11:09 +0800 Subject: [PATCH 24/26] Align smaps dirty accounting and mremap flushes Keep synthesized Pss_Dirty consistent with fork-inherited Shared_Dirty, and avoid flushing writable aliases when moving a read-only MAP_SHARED snapshot. Add regressions for both cases. --- src/runtime/procemu.c | 10 ++-- src/syscall/mem.c | 17 ++++--- tests/test-mremap-fork-tracking.c | 82 +++++++++++++++++++++++++++++++ tests/test-proc-smap.c | 31 +++++++++--- 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index b3f2aab8..68a67715 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -2695,8 +2695,8 @@ static int proc_open_self_maps(const guest_t *g) * guest VMAs and logical fork snapshots, but it cannot observe kernel page * residency or dirty bits on the host. Writable private anonymous VMAs that * were present in the most recent fork snapshot report their full VMA size as - * Shared_Dirty, Rss, and Pss; keeping those counters aligned avoids a - * self-contradictory snapshot while retaining a coarse fork-compatibility + * Shared_Dirty, Rss, Pss, and Pss_Dirty; keeping those counters aligned avoids + * a self-contradictory snapshot while retaining a coarse fork-compatibility * signal. Newly-created VMAs are excluded. All other fields that require * kernel page accounting are stable zeroes. */ @@ -2731,6 +2731,7 @@ static int proc_open_self_smaps(const guest_t *g) uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; uint64_t rss_kb = shared_dirty_kb; uint64_t pss_kb = shared_dirty_kb; + uint64_t pss_dirty_kb = shared_dirty_kb; /* Linux writes a separating space after VmFlags:, even when no * evidence-based flags are available (for example a PROT_NONE VMA). @@ -2774,7 +2775,7 @@ static int proc_open_self_smaps(const guest_t *g) "MMUPageSize: 4 kB\n" "Rss: %llu kB\n" "Pss: %llu kB\n" - "Pss_Dirty: 0 kB\n" + "Pss_Dirty: %llu kB\n" "Shared_Clean: 0 kB\n" "Shared_Dirty: %llu kB\n" "Private_Clean: 0 kB\n" @@ -2795,6 +2796,7 @@ static int proc_open_self_smaps(const guest_t *g) "VmFlags:%s\n", header_len, header, (unsigned long long) size_kb, (unsigned long long) rss_kb, (unsigned long long) pss_kb, + (unsigned long long) pss_dirty_kb, (unsigned long long) shared_dirty_kb, vmflags) < 0) goto out; } @@ -3409,7 +3411,7 @@ int proc_intercept_open(const guest_t *g, return proc_open_self_maps(g); /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata - * and the coarse fork Shared_Dirty/Rss/Pss compatibility signal. + * and the coarse fork Shared_Dirty/Rss/Pss/Pss_Dirty compatibility signal. */ if (!strcmp(path, "/proc/self/smaps")) return proc_open_self_smaps(g); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 2d18ed8b..df463efc 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -255,6 +255,7 @@ typedef struct { uint64_t gpa_base; uint64_t offset; int backing_fd; /* borrowed from the region tracker */ + /* Snapshot-style MAP_SHARED source that may contain guest writes. */ bool shared_non_overlay; bool overlay_active; uint64_t overlay_start; @@ -382,7 +383,8 @@ static int find_mremap_source(const guest_t *g, segment->offset = expected_offset; segment->backing_fd = r->backing_fd; segment->shared_non_overlay = - r->shared && r->backing_fd >= 0 && !region_has_live_overlay(r); + r->shared && (r->prot & LINUX_PROT_WRITE) && r->backing_fd >= 0 && + !region_has_live_overlay(r); segment->overlay_active = region_has_live_overlay(r); segment->overlay_start = r->overlay_start; segment->overlay_end = r->overlay_end; @@ -1360,12 +1362,13 @@ static bool mremap_source_has_overlay(const mremap_source_t *source) return false; } -/* A snapshot-style MAP_SHARED source may contain guest writes that have not - * reached its backing file yet. mremap destroys the source VMA (and may zero - * its slab backing) after copying, so publish those dirty bytes before any - * source cleanup. Live file overlays are excluded: the host page cache already - * owns their coherence and cleanup restores the slab before the source is - * removed. +/* A writable snapshot-style MAP_SHARED source may contain guest writes that + * have not reached its backing file yet. mremap destroys the source VMA (and + * may zero its slab backing) after copying, so publish those dirty bytes before + * any source cleanup. Read-only sources cannot contain writes and therefore do + * not trigger this alias scan. Live file overlays are excluded: the host page + * cache already owns their coherence and cleanup restores the slab before the + * source is removed. */ static int64_t flush_mremap_source_shared(guest_t *g, const mremap_source_t *source) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index bac30b2e..7649a180 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -462,6 +462,87 @@ static void test_misaligned_shared_mremap_writeback(void) close(fd); } +/* Force the snapshot-style MAP_SHARED path by placing a guest-page mapping + * one page into an otherwise unused reservation. Apple hosts use larger host + * pages than the guest's 4 KiB pages, so this address cannot receive a live + * file overlay. */ +static void *map_misaligned_shared_fixed(size_t length, + int prot, + int fd, + off_t offset) +{ + const size_t page = 4096; + void *reservation = mmap(NULL, length + page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return MAP_FAILED; + + void *address = (char *) reservation + page; + if (munmap(reservation, length + page) != 0) + return MAP_FAILED; + return mmap(address, length, prot, MAP_SHARED | MAP_FIXED, fd, offset); +} + +static void test_readonly_shared_mremap_does_not_flush_alias(void) +{ + TEST("read-only MAP_SHARED mremap does not flush writable alias"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-readonly-alias-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (2 * span)) != 0 || pwrite(fd, "F", 1, 0) != 1) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *writer = + map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, 0); + char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0); + if (writer == MAP_FAILED || source == MAP_FAILED) { + FAIL("snapshot MAP_SHARED mappings failed"); + if (writer != MAP_FAILED) + munmap(writer, span); + if (source != MAP_FAILED) + munmap(source, span); + close(fd); + return; + } + writer[0] = 'W'; + + void *blocker = mmap(source + span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != source + span) { + FAIL("mremap blocker mmap failed"); + munmap(writer, span); + munmap(source, span); + close(fd); + return; + } + + char *moved = mremap(source, span, 2 * span, MREMAP_MAYMOVE); + unsigned char file_byte = 0; + bool unchanged = moved != MAP_FAILED && moved != source && + pread(fd, &file_byte, 1, 0) == 1 && file_byte == 'F'; + if (unchanged) + PASS(); + else + FAIL("read-only source mremap flushed writable alias"); + + if (moved != MAP_FAILED) + munmap(moved, 2 * span); + else + munmap(source, span); + munmap(writer, span); + munmap(blocker, span); + close(fd); +} + static void test_file_backed_fork_split_move(void) { TEST("MAP_SHARED file: mremap across fork-split source"); @@ -977,6 +1058,7 @@ int main(void) test_postfork_repeated_allocations_keep_lineage(); test_adjacent_file_vmas_rejected(); test_misaligned_shared_mremap_writeback(); + test_readonly_shared_mremap_does_not_flush_alias(); test_file_backed_fork_split_move(); test_file_backed_mprotect_fragments_move(); test_file_backed_fixed_move_from_same_vma(); diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 0096c443..d39551b8 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -32,10 +32,12 @@ typedef struct { unsigned long long mmu_page_kb; unsigned long long rss_kb; unsigned long long pss_kb; + unsigned long long pss_dirty_kb; unsigned long long shared_dirty_kb; bool have_size; bool have_rss; bool have_pss; + bool have_pss_dirty; bool have_shared_dirty; bool have_vmflags; bool vmflags_wr; @@ -170,10 +172,12 @@ static bool parse_header(const char *line, smaps_vma_t *vma) vma->mmu_page_kb = 0; vma->rss_kb = 0; vma->pss_kb = 0; + vma->pss_dirty_kb = 0; vma->shared_dirty_kb = 0; vma->have_size = false; vma->have_rss = false; vma->have_pss = false; + vma->have_pss_dirty = false; vma->have_shared_dirty = false; vma->have_vmflags = false; vma->vmflags_wr = false; @@ -246,8 +250,8 @@ static bool parse_vmflags(const char *line, bool *writable) static bool finish_vma(smaps_vma_t *vma, size_t field_index) { return field_index == SMAPS_FIELD_COUNT && vma->have_size && - vma->have_rss && vma->have_pss && vma->have_shared_dirty && - vma->have_vmflags; + vma->have_rss && vma->have_pss && vma->have_pss_dirty && + vma->have_shared_dirty && vma->have_vmflags; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) @@ -319,6 +323,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.rss_kb = value; if (field_index == 4) current.pss_kb = value; + if (field_index == 5) + current.pss_dirty_kb = value; if (field_index == 7) current.shared_dirty_kb = value; if (field_index == 0) @@ -327,6 +333,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.have_rss = true; if (field_index == 4) current.have_pss = true; + if (field_index == 5) + current.have_pss_dirty = true; if (field_index == 7) current.have_shared_dirty = true; field_index++; @@ -436,9 +444,12 @@ static bool validate_layout(const smaps_info_t *info, last->mmu_page_kb != 4) return false; if (first->rss_kb != first->shared_dirty_kb || - first->pss_kb != first->shared_dirty_kb || middle->rss_kb != 0 || - middle->pss_kb != 0 || last->rss_kb != last->shared_dirty_kb || - last->pss_kb != last->shared_dirty_kb) + first->pss_kb != first->shared_dirty_kb || + first->pss_dirty_kb != first->shared_dirty_kb || middle->rss_kb != 0 || + middle->pss_kb != 0 || middle->pss_dirty_kb != 0 || + last->rss_kb != last->shared_dirty_kb || + last->pss_kb != last->shared_dirty_kb || + last->pss_dirty_kb != last->shared_dirty_kb) return false; /* Every stress-map page has deliberately distinct permissions (with one @@ -460,7 +471,8 @@ static bool validate_layout(const smaps_info_t *info, const smaps_vma_t *stress_none = find_vma(info, stress + 2 * page_size); if (!stress_none || strcmp(stress_none->perms, "---p") || stress_none->vmflags_wr || stress_none->shared_dirty_kb != 0 || - stress_none->rss_kb != 0 || stress_none->pss_kb != 0) + stress_none->rss_kb != 0 || stress_none->pss_kb != 0 || + stress_none->pss_dirty_kb != 0) return false; return true; } @@ -568,11 +580,16 @@ static int child_probe(uintptr_t target, const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || !stress_none || !postfork_vma || first->shared_dirty_kb == 0 || + first->pss_dirty_kb != first->shared_dirty_kb || middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || + last->pss_dirty_kb != last->shared_dirty_kb || stress_ro->shared_dirty_kb != 0 || stress_rw->shared_dirty_kb == 0 || + stress_rw->pss_dirty_kb != stress_rw->shared_dirty_kb || stress_none->shared_dirty_kb != 0 || stress_none->rss_kb != 0 || - stress_none->pss_kb != 0 || postfork_vma->shared_dirty_kb != 0) { + stress_none->pss_kb != 0 || stress_none->pss_dirty_kb != 0 || + postfork_vma->shared_dirty_kb != 0 || + postfork_vma->pss_dirty_kb != 0) { free_smaps(&info); munmap(postfork, page_size); return 1; From b61b433add9b92cd173814d8f46b0cfd6198459e Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 16:37:46 +0800 Subject: [PATCH 25/26] Preserve failed mmap reservations Keep failed reservation addresses visible to callers when munmap fails. This lets MAP_FAILED cleanup release still-live mappings instead of leaking them. --- tests/test-mremap-fork-tracking.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index 7649a180..10c2175a 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -469,17 +469,25 @@ static void test_misaligned_shared_mremap_writeback(void) static void *map_misaligned_shared_fixed(size_t length, int prot, int fd, - off_t offset) + off_t offset, + void **reservation_out) { const size_t page = 4096; + *reservation_out = NULL; + void *reservation = mmap(NULL, length + page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (reservation == MAP_FAILED) return MAP_FAILED; void *address = (char *) reservation + page; - if (munmap(reservation, length + page) != 0) + if (munmap(reservation, length + page) != 0) { + /* Keep ownership visible to the caller when the reservation cannot + * be released. Returning MAP_FAILED alone would make the live + * mapping indistinguishable from an allocation that never happened. */ + *reservation_out = reservation; return MAP_FAILED; + } return mmap(address, length, prot, MAP_SHARED | MAP_FIXED, fd, offset); } @@ -487,6 +495,7 @@ static void test_readonly_shared_mremap_does_not_flush_alias(void) { TEST("read-only MAP_SHARED mremap does not flush writable alias"); + const size_t page = 4096; const size_t span = 64 * 1024; char tmpl[] = "/tmp/elfuse-mremap-readonly-alias-XXXXXX"; int fd = mkstemp(tmpl); @@ -501,15 +510,22 @@ static void test_readonly_shared_mremap_does_not_flush_alias(void) return; } - char *writer = - map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, 0); - char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0); + void *writer_reservation = NULL; + void *source_reservation = NULL; + char *writer = map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, + 0, &writer_reservation); + char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0, + &source_reservation); if (writer == MAP_FAILED || source == MAP_FAILED) { FAIL("snapshot MAP_SHARED mappings failed"); if (writer != MAP_FAILED) munmap(writer, span); if (source != MAP_FAILED) munmap(source, span); + if (writer_reservation != NULL) + munmap(writer_reservation, span + page); + if (source_reservation != NULL) + munmap(source_reservation, span + page); close(fd); return; } From bc9fdeefadd86a6697fc52432294913d4d4f882a Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 5 Aug 2026 14:47:47 +0200 Subject: [PATCH 26/26] Add redis real-workload CI lane Boot redis:7-alpine under HVF with redis-server foreground as the guest process and drive it from a second guest running redis-cli over the shared host loopback: PING, a SET/GET round-trip, then BGSAVE polled through INFO persistence to rdb_bgsave_in_progress:0 with rdb_last_bgsave_status:ok asserted. BGSAVE forks the server and snapshots the dataset copy-on-write, so the lane pins the exact path /proc/self/smaps exists to keep safe. redis cannot announce an ephemeral port (--port 0 disables TCP), so the host probes a free loopback port before boot instead of reading one back as the node lane does. Readiness keeps the node lane's two-stage shape (process alive, then socket accepts), shutdown is an in-band SHUTDOWN NOSAVE, and the server's own exit status is asserted host-side. This leg fails for now: redis-server's ARM64 COW safety check needs /proc/self/smaps (issue #258), which elfuse does not yet synthesize; upstream PR #259 adds it. The lane is staged in advance so that once that lands and this branch is rebased onto it, the leg turns green with no further changes and keeps the fork/COW path covered from then on. Verified both ways: red on this tree, green with the smaps branch merged locally. --- .github/actions/hvf-elfuse-setup/action.yml | 3 +- .github/workflows/main.yml | 5 +- docs/testing.md | 9 +- scripts/ci/oci-workload.sh | 108 +++++++++++++++++--- scripts/ci/workloads/redis-driver.sh | 76 ++++++++++++++ 5 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 scripts/ci/workloads/redis-driver.sh diff --git a/.github/actions/hvf-elfuse-setup/action.yml b/.github/actions/hvf-elfuse-setup/action.yml index 629e2ada..1d689b17 100644 --- a/.github/actions/hvf-elfuse-setup/action.yml +++ b/.github/actions/hvf-elfuse-setup/action.yml @@ -43,7 +43,8 @@ runs: cache: true # Reuse the arm64 elfuse binary built + entitlement-checked by build-macos - # instead of rebuilding the C project on the self-hosted runner five times. + # instead of rebuilding the C project on the self-hosted runner once per + # workload leg. # Mach-O code signatures (and their embedded HVF entitlement) travel inside # the binary, so they survive the artifact zip round-trip; only the execute # bit is lost and restored here. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9d5d8a9..9ecacc74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,8 +13,8 @@ # (crane/skopeo/umoci) on Linux # oci-image-macos : elfuse-oci darwin build, unit tests, sparsebundle # round-trip, and a run-less image-lifecycle smoke -# workload : per-image real-workload smokes (python/node/go/jvm/c, one -# matrix leg each) that boot the image under HVF on +# workload : per-image real-workload smokes (python/node/go/jvm/c/redis, +# one matrix leg each) that boot the image under HVF on # self-hosted Apple Silicon and drive its characteristic # operations # @@ -857,6 +857,7 @@ jobs: # gcc:14 is the largest first pull and the compile bursts (make plus # a heavier single TU) dominate the wall time. - { key: c, name: C, timeout: 45 } + - { key: redis, name: Redis, timeout: 30 } steps: - name: Checkout uses: actions/checkout@v7 diff --git a/docs/testing.md b/docs/testing.md index a7d80d1f..e3b42b8a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -260,7 +260,8 @@ CI splits this coverage by what each runner can do: cold cache with the image, then a `run --keep` cache that rmi refuses without `--force` and `--force` detaches and drops). Besides those, a per-image workload suite - (the `workload` job, one matrix leg per image: python, node, go, jvm, c) + (the `workload` job, one matrix leg per image: python, node, go, jvm, c, + redis) drives each image through its characteristic operations. The shared driver is `scripts/ci/oci-workload.sh `; each image's guest workload lives in `scripts/ci/workloads/`: @@ -277,6 +278,12 @@ CI splits this coverage by what each runner can do: file I/O, SHA-256, an 8-thread pool, and a subprocess. - **c** (`gcc:14`): a small multi-file `make` project plus a larger single translation unit compiled with `gcc -O1`. + - **redis** (`redis:7-alpine`): redis-server runs in the foreground as + the guest process, while a second guest drives redis-cli over the host + loopback. Several actions are performed: PING, a SET/GET round-trip, + and a BGSAVE polled until `INFO persistence` reports + `rdb_last_bgsave_status:ok`. BGSAVE forks the server and saves the + dataset from the copy-on-write child. `gcc:14` and `eclipse-temurin:21` are Debian/Ubuntu-based and ship the shadow suite, so these jobs also exercise the unpack setuid/setgid degrade end to diff --git a/scripts/ci/oci-workload.sh b/scripts/ci/oci-workload.sh index b7b7d754..e7baa7cb 100755 --- a/scripts/ci/oci-workload.sh +++ b/scripts/ci/oci-workload.sh @@ -4,14 +4,14 @@ # One key per image. `run` pulls on demand, so a warm persistent # ELFUSE_OCI_STORE keeps reruns network-free. # -# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-workload.sh +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-workload.sh # shellcheck source=scripts/ci/oci-lib.sh . "$(dirname "$0")/oci-lib.sh" require_bin : "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" export ELFUSE_OCI_STORE -key="${1:?usage: oci-workload.sh }" +key="${1:?usage: oci-workload.sh }" WL="$OCI_CI_DIR/workloads" # assert_sentinel SENTINEL DESC OUTPUT: OUTPUT must contain the fixed SENTINEL. @@ -32,14 +32,21 @@ run_capture() { assert_sentinel "$sentinel" "$desc" "$out" } -# Background guest bookkeeping for the node server phase; reap_guest (in -# oci-lib.sh) keeps a failed assertion from leaking the guest. +# Background guest bookkeeping for the node and redis server phases; +# reap_guest (in oci-lib.sh) keeps a failed assertion from leaking the guest. guest="" -node_outfile="" +srv_outfile="" on_exit() { rc=$? reap_guest - [ -n "$node_outfile" ] && rm -f "$node_outfile" + # On failure the server's own log often holds the only evidence (a fork + # error during BGSAVE never reaches the driver guest); a duplicate dump + # on the paths that already cat it beats a missing one. + if [ "$rc" -ne 0 ] && [ -s "$srv_outfile" ]; then + echo "--- server guest output ---" >&2 + cat "$srv_outfile" >&2 + fi + [ -n "$srv_outfile" ] && rm -f "$srv_outfile" exit "$rc" } trap on_exit EXIT @@ -66,9 +73,9 @@ run_node() { case "$reqs" in '' | *[!0-9]* | 0) fail "WL_NODE_REQUESTS must be a positive integer, got '$reqs'" ;; esac - node_outfile="$(mktemp)" + srv_outfile="$(mktemp)" "$BIN" run --entrypoint /usr/local/bin/node node:22-alpine \ - -e "$(cat "$WL/node-server.js")" >"$node_outfile" 2>&1 & + -e "$(cat "$WL/node-server.js")" >"$srv_outfile" 2>&1 & guest=$! # Wait for the server to announce its ephemeral port. Poll rather than @@ -76,10 +83,10 @@ run_node() { # its own captured output instead of an opaque timeout. local waited=0 port="" while [ "$waited" -lt 120 ]; do - port="$(awk -F= '/^PORT=/{print $2; exit}' "$node_outfile")" + port="$(awk -F= '/^PORT=/{print $2; exit}' "$srv_outfile")" [ -n "$port" ] && break if ! kill -0 "$guest" 2>/dev/null; then - cat "$node_outfile" >&2 + cat "$srv_outfile" >&2 guest="" fail "node server exited before announcing a port" fi @@ -87,7 +94,7 @@ run_node() { waited=$((waited + 1)) done if [ -z "$port" ]; then - cat "$node_outfile" >&2 + cat "$srv_outfile" >&2 fail "node server did not announce a port within 60s" fi # The PORT= line is the guest flushing stdout, not proof the socket accepts @@ -123,6 +130,82 @@ run_node() { guest="" } +run_redis() { + # redis-server runs foreground as the guest process and is driven from + # outside by a second redis-cli guest over the shared host loopback: + # an in-guest backgrounded daemon risks the backgrounded-fork wait + # livelock, and per-probe redis-cli guests would pay a boot each. + # --entrypoint bypasses the image's docker-entrypoint.sh, whose user + # switching this smoke does not need. + # + # redis cannot announce an ephemeral port (--port 0 disables TCP), so the + # host picks one: probe the shared loopback until a port refuses, in a + # range below the OS ephemeral allocator so another lane's port-0 bind + # cannot land on it. The probe-to-bind window stays unguarded; losing + # that race surfaces as the exited-early dump below. + local port="" try=0 + while [ "$try" -lt 10 ]; do + port=$((20000 + RANDOM % 20000)) + if ! nc -z 127.0.0.1 "$port" 2>/dev/null; then + break + fi + port="" + try=$((try + 1)) + done + [ -n "$port" ] || fail "no free loopback port for redis after 10 probes" + + # --save '' disables periodic snapshots so the only fork is the BGSAVE + # the driver issues; --dir /data names a path present in the rootfs. + srv_outfile="$(mktemp)" + "$BIN" run --entrypoint /usr/local/bin/redis-server redis:7-alpine \ + --bind 127.0.0.1 --port "$port" --save '' --dir /data \ + >"$srv_outfile" 2>&1 & + guest=$! + + # Wait for the server's readiness log line. Poll rather than wait_for so + # a guest that dies (the ARM64 COW safety check aborting because + # /proc/self/smaps is not synthesized) surfaces its own captured output + # instead of an opaque timeout. + local waited=0 ready="" + while [ "$waited" -lt 120 ]; do + if grep -F "Ready to accept connections" "$srv_outfile" >/dev/null; then + ready=1 + break + fi + if ! kill -0 "$guest" 2>/dev/null; then + cat "$srv_outfile" >&2 + guest="" + fail "redis server exited before reporting readiness" + fi + sleep 0.5 + waited=$((waited + 1)) + done + if [ -z "$ready" ]; then + cat "$srv_outfile" >&2 + fail "redis server not ready within 60s" + fi + # The log line is the guest flushing stdout, not proof the socket accepts + # connections yet; probe it before booting the driver guest. + wait_for 30 "redis server on 127.0.0.1:$port" nc -z 127.0.0.1 "$port" + printf 'redis server on 127.0.0.1:%s\n' "$port" + + # One driver guest runs the whole PING/SET/GET/BGSAVE sequence. The port + # rides in as $1 so the script text stays fixed. + run_capture elfuse-oci-redis-workload-ok redis-driver \ + --entrypoint /bin/sh redis:7-alpine \ + -c "$(cat "$WL/redis-driver.sh")" sh "$port" + + # The driver ends with SHUTDOWN NOSAVE; redis exits 0 on it. Bound the + # wait so a lost shutdown fails here and the EXIT trap reaps the guest + # instead of idling to the job's timeout-minutes. + wait_for 10 "redis server exit after SHUTDOWN" guest_gone + if ! wait "$guest"; then + guest="" + fail "redis server exited non-zero after SHUTDOWN" + fi + guest="" +} + case "$key" in python) run_capture elfuse-oci-python-workload-ok python \ @@ -130,6 +213,7 @@ case "$key" in -c "$(cat "$WL/python-workload.py")" ;; node) run_node ;; + redis) run_redis ;; go) run_capture elfuse-oci-go-workload-ok go \ golang:1.23-alpine /bin/sh -c "$(cat "$WL/go-workload.sh")" @@ -142,7 +226,7 @@ case "$key" in run_capture elfuse-oci-c-workload-ok c \ gcc:14 /bin/sh -c "$(cat "$WL/c-workload.sh")" ;; - *) fail "unknown workload key: $key (want python|node|go|jvm|c)" ;; + *) fail "unknown workload key: $key (want python|node|go|jvm|c|redis)" ;; esac # Keep a persistent store bounded: a moved pin strands the old digest's diff --git a/scripts/ci/workloads/redis-driver.sh b/scripts/ci/workloads/redis-driver.sh new file mode 100644 index 00000000..5390dc47 --- /dev/null +++ b/scripts/ci/workloads/redis-driver.sh @@ -0,0 +1,76 @@ +# shellcheck shell=sh +# Redis image workload driver, run in a second guest via `/bin/sh -c` while +# the first guest runs redis-server in the foreground. elfuse forwards socket +# syscalls to host sockets and does no netns isolation, so 127.0.0.1 here is +# the same loopback the server bound; the target port arrives as $1. BGSAVE +# is the point of this lane: it forks the server and snapshots the dataset +# copy-on-write, the path redis's ARM64 safety check reads /proc/self/smaps +# to vet, so a regression there turns the BGSAVE stanza red. +# Prints one sentinel token on success. POSIX sh (busybox ash). +set -e +port="$1" + +r() { + redis-cli -h 127.0.0.1 -p "$port" "$@" +} + +# Every reply is string-compared: redis-cli exits 0 even when the server +# answers with an (error) reply, so exit status alone asserts nothing. +out=$(r PING) +if [ "$out" != PONG ]; then + echo "PING returned '$out'" >&2 + exit 1 +fi + +out=$(r SET elfuse:workload elfuse-redis-value) +if [ "$out" != OK ]; then + echo "SET returned '$out'" >&2 + exit 1 +fi + +out=$(r GET elfuse:workload) +if [ "$out" != elfuse-redis-value ]; then + echo "GET returned '$out'" >&2 + exit 1 +fi + +# The reply pins that the fork itself happened; completion and the child's +# fate are only visible through INFO persistence. rdb_last_bgsave_status +# reads "ok" from boot, so checking it without this reply would pass with +# no save ever run. +out=$(r BGSAVE) +if [ "$out" != "Background saving started" ]; then + echo "BGSAVE returned '$out'" >&2 + exit 1 +fi + +# INFO lines end in \r\n, so substring greps only, never full-line matches. +i=0 +while [ "$i" -lt 30 ]; do + if r INFO persistence | grep -q 'rdb_bgsave_in_progress:0'; then + break + fi + sleep 1 + i=$((i + 1)) +done + +# Re-assert completion outside the loop: falling out on timeout with the +# save still in flight must not reach the status check, whose boot-time +# default would read as a pass. +if ! r INFO persistence | grep -q 'rdb_bgsave_in_progress:0'; then + r INFO persistence >&2 + echo "BGSAVE still in progress after 30s" >&2 + exit 1 +fi +if ! r INFO persistence | grep -q 'rdb_last_bgsave_status:ok'; then + r INFO persistence >&2 + echo "BGSAVE did not finish with status ok" >&2 + exit 1 +fi + +echo "elfuse-oci-redis-workload-ok" + +# Sentinel first, shutdown last: the server closes the connection while +# acknowledging SHUTDOWN, so redis-cli's status is noise here; the host +# asserts the server's own exit code instead. +r SHUTDOWN NOSAVE || true