From a95bd60d66a2e6d573bf1a1193a438f5cc940fea Mon Sep 17 00:00:00 2001 From: Max042004 Date: Tue, 14 Jul 2026 00:54:35 +0800 Subject: [PATCH 1/9] Optimize anonymous mmap and page-fault paths Anonymous memory is reworked on a lazy foundation: sys_mmap only records the region, while page tables, zeroing and host-memory commit are deferred to first touch. Two optimization layers sit on top. mmap: the common shape (addr == 0, MAP_PRIVATE|MAP_ANONYMOUS, RW) is served entirely at EL1 with no vCPU exit and no lock. The shim stays stateless: it bump-allocates from a host-carved per-vCPU VA arena and reports each grant through a per-vCPU SPSC ring that the host drains whenever it takes mmap_lock, so no cross-vCPU synchronization exists. Arenas recycle freed address space through the gap allocator, refill automatically when a request does not fit (the new arena sized to cover the triggering request, with an ARENA_MAX guard so oversized requests fall back without churning the arena), and decay back to workload scale based on served-length history. Page fault: a per-2MiB-block dirty map lets materialization skip zeroing blocks whose slab bytes are known zero, moving the host page commit off the fault and out of mmap_lock onto the guest's own first writes. The TLBI protocol emits a single range op per materialized block instead of per-page invalidations, and a repeat fault on an already-valid block returns without re-zeroing. Host-side access to untouched lazy memory (I/O buffers, futex words, signal frames, shmat/shmdt copies) materializes on demand with lock-order-safe pre-faulting, and PROT_NONE stays a pure faulting reservation. tests/test-mmap-lazy.c and tests/test-mmap-fastpath.c pin the contract: zero-on-reuse, huge sparse reservations, host access to untouched memory, PROT_NONE round trips, fork, racing first touch, and arena refill/revocation. Close #165 --- Makefile | 22 + docs/usage.md | 9 + src/core/bootstrap.c | 30 ++ src/core/guest.c | 539 ++++++++++++++++++-- src/core/guest.h | 204 ++++++-- src/core/mmap-fastpath.h | 95 ++++ src/core/shim-globals.c | 120 ++++- src/core/shim-globals.h | 7 +- src/core/shim.S | 152 +++++- src/main.c | 9 + src/runtime/fork-state.c | 14 +- src/runtime/fork-state.h | 3 +- src/runtime/forkipc.c | 49 +- src/runtime/futex.c | 15 + src/syscall/exec.c | 10 + src/syscall/internal.h | 7 + src/syscall/mem.c | 676 ++++++++++++++++++++++--- src/syscall/proc.c | 39 +- src/syscall/signal.c | 24 + src/syscall/syscall.c | 65 ++- src/syscall/sysvipc.c | 13 +- tests/bench-mmap-lazy.c | 111 +++++ tests/manifest.txt | 4 + tests/test-fork-ipc-protocol-host.c | 7 +- tests/test-mmap-dirty-stats.sh | 45 ++ tests/test-mmap-fastpath-stats.sh | 113 +++++ tests/test-mmap-fastpath.c | 278 +++++++++++ tests/test-mmap-lazy.c | 738 ++++++++++++++++++++++++++++ tests/test-tlbi-encoder-host.c | 53 +- 29 files changed, 3237 insertions(+), 214 deletions(-) create mode 100644 src/core/mmap-fastpath.h create mode 100644 tests/bench-mmap-lazy.c create mode 100755 tests/test-mmap-dirty-stats.sh create mode 100755 tests/test-mmap-fastpath-stats.sh create mode 100644 tests/test-mmap-fastpath.c create mode 100644 tests/test-mmap-lazy.c diff --git a/Makefile b/Makefile index 9a0ed40f..f38de266 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,28 @@ $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread +# test-mmap-lazy races concurrent first touch from several threads. +$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-lazy +test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-lazy + +# EL1 consumer-mmap integration/stress test. +$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-fastpath +test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-fastpath + # test-thread-churn creates >64 threads to force thread-table slot reuse. $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/docs/usage.md b/docs/usage.md index bef8aca2..f92af358 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,6 +29,15 @@ only bounds a single `hv_vcpu_run()` iteration before the host regains control, which is what allows host-side timers and signals to be observed promptly. Setting `--timeout 0` disables this watchdog for long-running CPU-bound guests. +### mmap call fast path + +The aarch64 EL1 consumer fast path is enabled by default for +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)`. +Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted +arenas, and full consumption rings fall back to the normal host syscall path. +Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host +path so observability and translated-guest behavior are unchanged. + ## Common Launch Patterns Run a statically linked guest binary: diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 70bddfd5..f31c56ff 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -36,6 +36,7 @@ #include "syscall/signal.h" #include "debug/log.h" +#include "core/mmap-fastpath.h" /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter. @@ -136,6 +137,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len) guest_invalidate_ptes(g, 0, 0x1000); } +/* ELF/shim/stack bytes are populated directly in the slab before their page + * tables become live. Mark their semantic backing regardless of requested + * permissions: a read-only file segment is still nonzero and must be scrubbed + * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block. + * Synthetic page-table coverage for unallocated mmap space has no semantic + * region and therefore remains clean. + */ +static void mark_registered_backing_dirty(guest_t *g) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->end <= r->start) + continue; + uint64_t len = r->end - r->start; + if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base) + continue; + guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len); + } +} + int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info) { memset(info, 0, sizeof(*info)); @@ -547,6 +568,7 @@ int guest_bootstrap_prepare(guest_t *g, } register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); startup_trace_step("register_regions", t0); log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0, @@ -732,6 +754,13 @@ int guest_bootstrap_create_vcpu(guest_t *g, */ shim_globals_set_singleton(g); + /* Publish the main vCPU's first arena only after shim_globals_init has + * cleared every recycled control slot. Verbose tracing keeps all shim + * syscall fast paths on HVC so the trace remains complete. + */ + if (!verbose) + mmap_fastpath_prepare_vcpu(g, current_thread); + /* CNTKCTL_EL1.EL0VCTEN | EL0PCTEN: allow EL0 to read {CNTVCT,CNTPCT}_EL0. * Required by the vDSO clock_gettime fast path (and is the default on * native Linux), without which the guest gets 0 back from MRS. @@ -843,6 +872,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g, g->rosetta_guest_base - g->rosetta_va_base, ROSETTA_PATH); register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); int rosetta_argc = 0; const char **rosetta_argv = NULL; diff --git a/src/core/guest.c b/src/core/guest.c index 02250cea..e6c119cd 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -43,10 +43,11 @@ #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" -#include "runtime/futex.h" /* futex_interrupt_request */ -#include "runtime/thread.h" /* thread_destroy_all_vcpus */ -#include "syscall/poll.h" /* wakeup_pipe_signal */ -#include "syscall/proc.h" /* proc_request_exit_group */ +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" /* thread_destroy_all_vcpus */ +#include "syscall/internal.h" /* mmap_lock (lazy fault-in) */ +#include "syscall/poll.h" /* wakeup_pipe_signal */ +#include "syscall/proc.h" /* proc_request_exit_group */ /* Per-vCPU pending TLBI request. Zero-initialized in every host pthread by * virtue of TLS default-zeroing, which maps to TLBI_NONE. @@ -474,6 +475,7 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); return 0; } @@ -586,6 +588,7 @@ int guest_init_from_shm(guest_t *g, */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); log_debug( "guest: CoW fork: mapped %llu GiB from shm " @@ -707,6 +710,7 @@ void guest_destroy(guest_t *g) close(g->shm_fd); g->shm_fd = -1; } + pthread_cond_destroy(&g->materialize_cond); } /* Check whether a candidate IPA range [gpa, gpa+size) overlaps the primary @@ -1441,11 +1445,11 @@ static uint64_t gva_contiguous_avail(const guest_t *g, * (MEM_PERM_R/W/X bitmask). The walk continues across adjacent L2/L3 entries * until a mapping, permission, or physical-contiguity break is found. */ -static void *gva_resolve_perm(const guest_t *g, - uint64_t gva, - uint64_t *avail, - int required_perms, - uint64_t avail_limit) +static void *gva_resolve_perm_walk(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit) { /* Always walk page tables to enforce permissions. The guest slab is * identity-mapped (GVA == GPA == offset), but L2 block descriptors carry @@ -1491,14 +1495,114 @@ static void *gva_resolve_perm(const guest_t *g, return NULL; } +/* Host-access fault-in for lazy (deferred-PTE) regions. + * + * A syscall may target guest memory the guest itself has never touched: a + * fresh calloc()-style arena handed straight to read(2), a futex word inside + * an untouched mapping, an iovec into a new heap chunk. The guest-fault path + * (guest_materialize_lazy via the EL1 shim) never runs for those, so the + * page-table walk in gva_resolve_perm_walk fails even though the access is + * legal. Materialize the touched blocks here, then let the caller re-walk. + * + * Locking: takes mmap_lock; callers of the resolve API that already hold + * mmap_lock must use the _nofault variants. Do not call while holding any + * lock that ranks after mmap_lock in the ordering (syscall/internal.h); + * callers that resolve under such locks (futex bucket paths) pre-fault at + * their lock-free entry points so the hook never engages there. + * + * TLBI: guest_materialize_lazy accumulates TLBI requests in the calling + * thread's per-vCPU slot. On a vCPU thread the syscall epilogue emits them. + * On non-vCPU threads the request is lost, which is self-healing: a vCPU + * that still holds a stale negative TLB entry re-faults, and the + * already-valid early-return in guest_materialize_lazy re-issues a page + * TLBI for it without re-zeroing. + * + * Returns 0 if at least one block in [gva, gva+len) is now materialized (or + * already was), -1 if the range intersects no materializable lazy region. + */ +int guest_lazy_faultin_locked(const guest_t *cg, uint64_t gva, uint64_t len) +{ + /* The lazy machinery mutates page tables; the const on the resolve API + * reflects the pure-walk fast path, not this slow path. + */ + guest_t *g = (guest_t *) (uintptr_t) cg; + + if (gva >= g->guest_size) + return -1; /* High-VA / non-identity ranges are never lazy. */ + if (len == 0) + len = 1; + uint64_t end = (len > g->guest_size - gva) ? g->guest_size : gva + len; + + int rc = -1; + for (uint64_t b = gva & ~(uint64_t) (BLOCK_2MIB - 1); b < end; + b += BLOCK_2MIB) { + uint64_t probe = (b > gva) ? b : gva; + if (guest_materialize_lazy(g, probe) == 0) + rc = 0; + } + return rc; +} + +static int gva_lazy_faultin(const guest_t *cg, + uint64_t gva, + uint64_t len, + int required_perms) +{ + (void) required_perms; /* Region prot gates the retry walk, not this. */ + + int rc; + mmap_lock_acquire((guest_t *) (uintptr_t) cg); + rc = guest_lazy_faultin_locked(cg, gva, len); + mmap_lock_release(); + return rc; +} + +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len) +{ + return gva_lazy_faultin(g, gva, len, MEM_PERM_R); +} + +static void *gva_resolve_perm(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit, + bool allow_faultin) +{ + void *ptr = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (!allow_faultin) + return ptr; + + /* Window the fault-in to what the caller actually needs. Length-less + * resolves (guest_ptr / guest_ptr_avail) materialize a single block; + * their callers iterate and re-enter here per chunk. + */ + uint64_t want = (avail_limit == UINT64_MAX) ? 1 : avail_limit; + if (!ptr) { + if (gva_lazy_faultin(g, gva, want, required_perms) < 0) + return NULL; + return gva_resolve_perm_walk(g, gva, avail, required_perms, + avail_limit); + } + if (avail && *avail < want && gva <= UINT64_MAX - *avail && + gva_lazy_faultin(g, gva + *avail, want - *avail, required_perms) == 0) { + void *again = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (again) + ptr = again; + } + return ptr; +} + void *guest_ptr(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX, true); } void *guest_ptr_w(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX, true); } void *guest_ptr_avail(const guest_t *g, @@ -1506,7 +1610,19 @@ void *guest_ptr_avail(const guest_t *g, uint64_t *avail, int required_perms) { - return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX); + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, true); +} + +/* Pure page-table walk without lazy fault-in. For callers that already hold + * mmap_lock (e.g. the stale-TLB re-walk in the EL0 fault handler, which runs + * after guest_materialize_lazy has already been consulted). + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms) +{ + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, false); } void *guest_ptr_bound(const guest_t *g, @@ -1515,7 +1631,7 @@ void *guest_ptr_bound(const guest_t *g, int required_perms, uint64_t len_limit) { - return gva_resolve_perm(g, gva, avail, required_perms, len_limit); + return gva_resolve_perm(g, gva, avail, required_perms, len_limit, true); } static inline int guest_copy(const guest_t *g, @@ -1523,7 +1639,8 @@ static inline int guest_copy(const guest_t *g, void *dst, const void *src, size_t len, - int required_perms) + int required_perms, + bool allow_faultin) { if (len == 0) return 0; @@ -1537,7 +1654,7 @@ static inline int guest_copy(const guest_t *g, while (copied < len) { uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, required_perms, - (uint64_t) (len - copied)); + (uint64_t) (len - copied), allow_faultin); if (!ptr) return -1; size_t chunk = len - copied; @@ -1554,7 +1671,7 @@ static inline int guest_copy(const guest_t *g, int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len) { - return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R); + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, true); } int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) @@ -1571,7 +1688,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len) { - return guest_copy(g, gva, NULL, src, len, MEM_PERM_W); + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, true); +} + +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len) +{ + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, false); } int guest_write_small(guest_t *g, uint64_t gva, const void *src, size_t len) @@ -1597,7 +1719,7 @@ int guest_read_str(const guest_t *g, uint64_t gva, char *dst, size_t max) break; uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, MEM_PERM_R, - (uint64_t) (limit - copied)); + (uint64_t) (limit - copied), true); if (!ptr) break; @@ -1668,6 +1790,7 @@ void guest_reset(guest_t *g) if (gpa > g->guest_size || len > g->guest_size - gpa) continue; /* backing lies outside the primary slab */ memset((uint8_t *) g->host_base + gpa, 0, len); + guest_dirty_clear_zeroed_range(g, gpa, gpa + len); } /* Zero page table pool (not tracked in region array) */ @@ -2681,6 +2804,11 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) if (!finalize_block_perms(g, regions, n)) return 0; + for (int r = 0; r < n; r++) { + if (regions[r].perms & MEM_PERM_W) + guest_dirty_mark_range(g, regions[r].gpa_start, regions[r].gpa_end); + } + guest_pt_gen_bump(g); return ttbr0; } @@ -2816,6 +2944,47 @@ int guest_extend_page_tables(guest_t *g, return 0; } +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || !g->ttbr0) + return end; + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + if (!l0) + return end; + + va &= ~(uint64_t) (BLOCK_2MIB - 1); + while (va < end) { + uint64_t ipa = base + va; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return end; + if (!(l0[l0_idx] & PT_VALID)) { + uint64_t next_ipa = (uint64_t) (l0_idx + 1) * 512ULL * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; /* wrap guard */ + va = next_ipa - base; + continue; + } + uint64_t *l1 = pt_at(g, (l0[l0_idx] & 0xFFFFFFFFF000ULL) - base); + if (!l1) + return end; + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + if (!(l1[l1_idx] & PT_VALID)) { + uint64_t next_ipa = (ipa / BLOCK_1GIB + 1) * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; + va = next_ipa - base; + continue; + } + return va; + } + return end; +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) @@ -3020,8 +3189,13 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) for (uint64_t addr = start; addr < end;) { uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 entry (already unmapped); skip this 2MiB block */ - addr = ALIGN_2MIB_UP(addr + 1); + /* No L2 table (L0/L1 slot absent): nothing to invalidate in this + * block. Skip whole absent 1GiB/512GiB slots at once; a lazy + * multi-GiB mmap invalidates its stale range on every allocation + * and would otherwise pay one four-level walk per 2MiB of empty + * address space. + */ + addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3240,6 +3414,8 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) addr = page_end; } + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, start, end); guest_pt_gen_bump(g); return 0; } @@ -3321,15 +3497,131 @@ int guest_install_va_pages(guest_t *g, if (!bcast && changed_hi > changed_lo) tlbi_request_range(changed_lo, changed_hi); + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, gpa, gpa + length); guest_pt_gen_bump(g); return 0; } -/* Lazy page materialization for MAP_NORESERVE. */ +/* Lazy page materialization for deferred-PTE (private anonymous / + * MAP_NORESERVE) regions. + */ -int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start) +{ + if (!g || block_start >= g->guest_size) + return true; + uint64_t block = block_start / BLOCK_2MIB; + return (g->dirty_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = start / BLOCK_2MIB; + uint64_t last = (end - 1) / BLOCK_2MIB; + for (uint64_t block = first; block <= last; block++) + g->dirty_blocks[block >> 6] |= 1ULL << (block & 63); +} + +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = ALIGN_2MIB_UP(start); + uint64_t last = ALIGN_2MIB_DOWN(end); + for (uint64_t addr = first; addr < last; addr += BLOCK_2MIB) { + uint64_t block = addr / BLOCK_2MIB; + g->dirty_blocks[block >> 6] &= ~(1ULL << (block & 63)); + } +} + +static bool materialize_claim_overlaps(const guest_materialize_claim_t *claim, + uint64_t start, + uint64_t end) +{ + return claim->active && start < claim->end && end > claim->start; +} + +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (;;) { + bool overlap = false; + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], start, + end)) { + overlap = true; + break; + } + } + if (!overlap) + return; + mmap_lock_cond_wait(g, &g->materialize_cond); + } +} + +void guest_materialize_wait_all_locked(guest_t *g) +{ + guest_materialize_wait_range_locked(g, 0, UINT64_MAX); +} + +static int materialize_claim_alloc_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + guest_materialize_wait_range_locked(g, start, end); + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + guest_materialize_claim_t *claim = &g->materialize_claims[i]; + if (!claim->active) { + claim->start = start; + claim->end = end; + claim->active = true; + return i; + } + } + return -1; +} + +static void materialize_claim_release_locked(guest_t *g, int slot) +{ + if (slot < 0) + return; + g->materialize_claims[slot].active = false; + pthread_cond_broadcast(&g->materialize_cond); +} + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Callers + * must hold mmap_lock; used to detect blocks a concurrent fault already + * materialized. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va) { - /* Find the noreserve region containing this offset */ + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + return false; + if ((*l2_entry & 3) == 1) + return true; /* 2MiB block descriptor */ + uint64_t *l3 = pt_at(g, (*l2_entry & 0xFFFFFFFFF000ULL) - g->ipa_base); + if (!l3) + return false; + unsigned l3_idx = + (unsigned) (((g->ipa_base + va) % BLOCK_2MIB) / PAGE_SIZE); + return (l3[l3_idx] & PT_VALID) != 0; +} + +static int guest_materialize_lazy_one(guest_t *g, uint64_t fault_offset) +{ +retry:; + /* Find the lazy region containing this offset */ const guest_region_t *region = NULL; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start <= fault_offset && @@ -3340,7 +3632,35 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) } if (!region) - return -1; /* Not a noreserve region */ + return -1; /* Not a lazy region */ + + /* PROT_NONE is a reservation, not a mapping: a fault inside it is a + * genuine SIGSEGV, never a materialization request. Without this check a + * PROT_NONE|MAP_NORESERVE region would be silently granted read + * permission by the perms fallback below. + */ + if (region->prot == LINUX_PROT_NONE) + return -1; + + uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); + uint64_t block_end = block_start + BLOCK_2MIB; + if (block_end > g->guest_size) + block_end = g->guest_size; + + /* Already materialized: another thread (concurrent guest fault or a + * host-side fault-in on a syscall path) completed this block while this + * vCPU was queued on mmap_lock, and the guest may have written real data + * through the new PTEs since. Running the memset below again would wipe + * those writes. The fault that got us here is then either a stale + * negative TLB entry or an in-flight retry. Invalidate the whole + * materialization block so this path follows the same one-RVAE-per-block + * contract as a newly installed block. + */ + if (guest_va_pte_valid(g, fault_offset)) { + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + return 0; + } /* Materialize one 2MiB block containing the fault address. This is the * smallest granule that guest_extend_page_tables works with. For the common @@ -3348,12 +3668,17 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) * trade-off: it avoids over-committing the large reservation while keeping * the fault rate manageable. */ - uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); - uint64_t block_end = block_start + BLOCK_2MIB; - - /* Clamp to guest size */ - if (block_end > g->guest_size) - block_end = g->guest_size; + /* A sibling may be zeroing another window in this block without the lock. + * Wait before inspecting regions/PTEs, then restart because a mutator that + * was itself waiting may have changed the region layout first. + */ + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], block_start, + block_end)) { + guest_materialize_wait_range_locked(g, block_start, block_end); + goto retry; + } + } uint64_t materialize_start = (block_start > region->start) ? block_start : region->start; @@ -3374,18 +3699,70 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) if (perms == 0) perms = MEM_PERM_R; /* At minimum readable */ + /* Zero the window BEFORE any PTE becomes valid. The moment a descriptor + * is published, sibling vCPUs with no stale TLB entry can write through + * it without ever faulting; zeroing afterwards (the historical order) + * would wipe such a write. The slab is host memory, so zeroing needs no + * PTEs, and every writer that could touch the window first goes through + * mmap_lock (guest faults and host-side fault-in alike), so nothing can + * write between this memset and the descriptor stores below. Skip pages + * that are already valid: they belong to a previously materialized + * neighbor in the same block and may hold live data. + */ + int claim_slot = -1; + bool dirty = guest_block_may_be_dirty(g, block_start); + if (dirty) { + uint64_t zero_pages[8] = {0}; + bool any_valid = false; + for (uint64_t pg = materialize_start; pg < materialize_end; + pg += PAGE_SIZE) { + unsigned page = (unsigned) ((pg - block_start) / PAGE_SIZE); + if (guest_va_pte_valid(g, pg)) + any_valid = true; + else + zero_pages[page >> 6] |= 1ULL << (page & 63); + } + + claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); + if (claim_slot >= 0) + mmap_lock_release(); + for (unsigned page = 0; page < 512;) { + if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { + page++; + continue; + } + unsigned first = page; + do { + page++; + } while (page < 512 && + (zero_pages[page >> 6] & (1ULL << (page & 63)))); + memset((uint8_t *) g->host_base + block_start + + (uint64_t) first * PAGE_SIZE, + 0, (uint64_t) (page - first) * PAGE_SIZE); + } + if (claim_slot >= 0) + mmap_lock_acquire(g); + if (!any_valid && materialize_start == block_start && + materialize_end == block_end) + guest_dirty_clear_zeroed_range(g, block_start, block_end); + } + /* Create page table entries. guest_extend_page_tables creates L2 block * descriptors but skips existing table descriptors (L2->L3 splits). * guest_update_perms handles the L3 case: if guest_invalidate_ptes * previously split the block and invalidated the L3 entries, update_perms * recreates them with correct perms. */ - if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) + if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (partial_block) { - if (guest_split_block(g, block_start) < 0) + if (guest_split_block(g, block_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } /* If this block had no page-table entry before the lazy fault, * guest_extend_page_tables() necessarily created a full 2MiB block. @@ -3395,25 +3772,99 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) */ if (!had_mapping) { if (block_start < materialize_start && - guest_invalidate_ptes(g, block_start, materialize_start) < 0) + guest_invalidate_ptes(g, block_start, materialize_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (materialize_end < block_end && - guest_invalidate_ptes(g, materialize_end, block_end) < 0) + guest_invalidate_ptes(g, materialize_end, block_end) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } } } guest_update_perms(g, materialize_start, materialize_end, perms); + /* One 2MiB materialization gets one block-sized RVAE1IS. This is cheaper + * than per-page invalidation and covers every negative entry that may have + * been cached for the block while its descriptors were invalid. + */ + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + g->materialize_stats[dirty ? GUEST_MATERIALIZE_DIRTY_MEMSET + : GUEST_MATERIALIZE_CLEAN_SKIP]++; + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += + materialize_end - materialize_start; + materialize_claim_release_locked(g, claim_slot); + return 0; +} - /* Zero the materialized memory. Only zero within the region boundaries to - * avoid clobbering adjacent data. - */ - if (materialize_end > materialize_start) - memset((uint8_t *) g->host_base + materialize_start, 0, - materialize_end - materialize_start); +int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +{ + return guest_materialize_lazy_one(g, fault_offset); +} - /* The page-table helpers above already requested the matching TLBI; no - * additional flush is needed here. - */ - return 0; +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset) +{ + typedef struct { + guest_t *guest; + uint64_t next_block; + unsigned streak; + } fault_around_state_t; + static _Thread_local fault_around_state_t state; + + uint64_t block = ALIGN_2MIB_DOWN(fault_offset); + if (state.guest == g && block == state.next_block) { + if (state.streak < 4) + state.streak++; + } else { + state.guest = g; + state.streak = 0; + } + + const guest_region_t *region = guest_region_find(g, fault_offset); + if (!region || !region->noreserve || region->prot == LINUX_PROT_NONE) + return -1; + + unsigned blocks = 1U << state.streak; + if (blocks > 16) + blocks = 16; + uint64_t region_last = ALIGN_2MIB_UP(region->end); + if (region_last > g->guest_size) + region_last = g->guest_size; + + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + if (guest_block_may_be_dirty(g, ahead) && blocks > 4) { + blocks = 4; + break; + } + } + + int result = -1; + uint64_t last = block; + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + /* The current block must probe the actual FAR. A fast-path mmap can + * extend an already materialized region within the same 2MiB block; + * probing the region/block start would see an older valid page and + * return without installing the newly faulted page. Ahead blocks have + * no FAR, so use their first address inside the region. + */ + uint64_t probe = (i == 0) + ? fault_offset + : (ahead < region->start ? region->start : ahead); + int rc = guest_materialize_lazy_one(g, probe); + if (i == 0) + result = rc; + if (rc < 0) + break; + last = ahead; + } + if (result == 0) + state.next_block = last + BLOCK_2MIB; + return result; } diff --git a/src/core/guest.h b/src/core/guest.h index 78c6200f..ee30725c 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -234,7 +235,11 @@ typedef struct { uint64_t offset; /* File offset (for /proc/self/maps display) */ int backing_fd; /* Duplicated host fd for file-backed mappings, or -1 */ bool shared; /* MAP_SHARED (writes should propagate) */ - bool noreserve; /* MAP_NORESERVE: PTEs deferred until fault */ + bool noreserve; /* Lazy region: PTEs and zeroing deferred until first + * touch. Set for MAP_NORESERVE and for all private + * anonymous mappings (sys_mmap folds the latter into + * the tracked MAP_NORESERVE bit; not guest visible). + */ bool backing_ro; /* MAP_SHARED region whose backing_fd was opened * without write access, so its Linux max_prot is * capped to PROT_READ. sys_mprotect must reject any @@ -266,6 +271,8 @@ typedef struct { * TLBI_BROADCAST -> X8 = 1 (TLBI VMALLE1IS, broadest) * TLBI_RANGE -> X8 = 3, X9 = start VA, X10 = page count * (TLBI VAE1IS loop preserves unrelated TLB entries) + * TLBI_RANGE_LARGE -> X8 = 4, X9 = encoded range operand + * (single TLBI RVAE1IS) * X8 = 2 is reserved for the execve drop-frame marker the shim handles * separately; it is never produced by the accumulator. */ @@ -287,14 +294,12 @@ typedef enum { */ #define TLBI_SELECTIVE_MAX_PAGES 16 -/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. With SCALE=0 the - * RVAE1IS operand encoding covers (NUM+1)*2 pages with NUM in [0..31], so a - * single instruction reaches 64 pages == 256 KiB. Beyond that the host would - * need SCALE=1 (NUM*64 step), which over-invalidates for the typical - * dynamic-linker RELRO / glibc-bring-up storm sizes seen in practice; stay at - * SCALE=0 for now and broadcast above 64 pages. +/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. SCALE=0 covers up to + * 64 pages, SCALE=1 up to 2048 pages, and SCALE=2 much larger ranges. 32768 + * pages (128 MiB) covers the lazy fault-around window with one instruction + * while still fitting tlbi_request_t.pages in uint16_t. */ -#define TLBI_RVAE_MAX_PAGES 64 +#define TLBI_RVAE_MAX_PAGES 32768 /* TLBI RVAE1IS operand bit-field constants. Per ARM ARM DDI 0487J.a D8.7.6 the * operand layout is: @@ -310,28 +315,37 @@ typedef enum { */ #define RVAE_OPERAND_BADDR_MASK ((1ULL << 37) - 1) #define RVAE_OPERAND_NUM_SHIFT 39 +#define RVAE_OPERAND_SCALE_SHIFT 44 #define RVAE_OPERAND_TG_4KB (1ULL << 46) /* Pure encoder: build the TLBI RVAE1IS Xt operand from a 4 KiB-aligned VA and a - * page count in the SCALE=0 range (1..TLBI_RVAE_MAX_PAGES). Lives in the header - * as static inline so tlbi_request_emit_to_vcpu and any future caller - * (host-side unit tests included) compile to the same expression. NUM = - * ceil(pages / 2) - 1 over-invalidates odd page counts by exactly one page, - * which is a perf-only side effect (the extra invalidation evicts a neighbour - * TLB entry that the guest's next access reloads). pages < 2 is clamped to 2 - * because SCALE=0 NUM=0 means 2 pages -- the encoder cannot represent a single - * page through RVAE1IS; single-page callers go through the per-page VAE1IS path - * instead, but the clamp keeps the encoder total in any pathological input. + * page count in the supported SCALE=0..2 range. Lives in the header so the + * emit path and host unit tests use the same expression. Each NUM step covers + * 2^(5*SCALE+1) pages; the normalizer aligns and widens callers to that unit. + * pages < 2 is clamped to 2 because SCALE=0 NUM=0 is the smallest encodable + * range. Single-page callers normally use VAE1IS instead. */ +static inline uint64_t tlbi_rvae_unit_pages(uint64_t pages) +{ + if (pages <= 64) + return 2; /* SCALE=0 */ + if (pages <= 2048) + return 64; /* SCALE=1 */ + return 2048; /* SCALE=2 */ +} + static inline uint64_t tlbi_rvae1is_operand(uint64_t start_va, uint16_t pages) { if (pages < 2) pages = 2; + uint64_t scale = pages <= 64 ? 0 : (pages <= 2048 ? 1 : 2); + uint64_t unit_pages = 1ULL << (5 * scale + 1); uint64_t baddr = (start_va >> 12) & RVAE_OPERAND_BADDR_MASK; - uint64_t num = ((pages + 1) / 2) - 1; + uint64_t num = ((pages + unit_pages - 1) / unit_pages) - 1; if (num > 31) num = 31; - return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | RVAE_OPERAND_TG_4KB; + return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | + (scale << RVAE_OPERAND_SCALE_SHIFT) | RVAE_OPERAND_TG_4KB; } /* Runtime feature flag: TRUE when the host PE implements FEAT_TLBIRANGE @@ -346,8 +360,8 @@ typedef struct { * visible to EL0, so the shim must IC IALLU * after the TLBI sequence. 0 = data-only * change, skip the I-cache invalidation. */ - uint16_t pages; /* Page count when kind == TLBI_RANGE (1..MAX) */ - uint64_t start; /* Page-aligned VA when kind == TLBI_RANGE */ + uint16_t pages; /* Page count for either range kind (1..MAX) */ + uint64_t start; /* Page-aligned VA for either range kind */ } tlbi_request_t; /* Layout contract: 16 bytes (1+1+2+4 padding+8). Documents the padding and pins * the TLS slot size so future field additions surface as a build break rather @@ -399,6 +413,32 @@ typedef struct { uint64_t next; /* Bump offset; (next + BLOCK_2MIB) > size means full */ } guest_overflow_t; +/* One conservative "may contain nonzero bytes" bit per 2 MiB primary-slab + * block. The largest supported slab is 1 TiB, so this costs 64 KiB per guest. + * All access is serialized by mmap_lock. + */ +#define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) +#define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) + +enum { + GUEST_MATERIALIZE_CLEAN_SKIP = 0, + GUEST_MATERIALIZE_DIRTY_MEMSET, + GUEST_MATERIALIZE_ALREADY_VALID, + GUEST_MATERIALIZE_WINDOW_BYTES, + GUEST_MATERIALIZE_STATS_N, +}; + +/* Dirty-block zeroing claims. A claim makes its block's invalid PTE window + * stable while the expensive host memset runs without mmap_lock. Waiters use + * one guest-wide condition variable; the fixed table bounds host allocation + * and is ample for the vCPU limit. + */ +#define GUEST_MATERIALIZE_CLAIMS 64 +typedef struct { + uint64_t start, end; + bool active; +} guest_materialize_claim_t; + /* Guest state. */ typedef struct { void *host_base; /* Host pointer to allocated guest memory */ @@ -522,6 +562,11 @@ typedef struct { */ _Atomic uint64_t pt_gen; + uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; + uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; + guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; + pthread_cond_t materialize_cond; + /* Optional HVC 6 embedder extension hook. * * Native AArch64 guests reach this through HVC 6. When the build enables @@ -643,8 +688,8 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) hv_vcpu_set_reg(vcpu, HV_REG_X11, cpu_tlbi_req.icache_flush ? 1 : 0); break; case TLBI_RANGE_LARGE: { - /* Single-shot TLBI RVAE1IS for ranges in (16..64] pages. The operand - * format and the SCALE=0 / TG=01 / ASID=0 assumptions are documented at + /* Single-shot TLBI RVAE1IS for ranges above the selective cap. The + * SCALE/NUM format and TG=01 assumption are documented at * tlbi_rvae1is_operand above. ASID stays 0 because the shim runs * single-ASID (TCR_EL1.A1=0, TTBR0 ASID=0; rosetta does not allocate a * separate ASID). If a future change introduces non-zero ASIDs, the @@ -666,6 +711,31 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) tlbi_request_clear(); } +/* RVAE1IS requires BaseADDR alignment to its SCALE granule. Widen a requested + * interval to that granule, repeating if widening crosses a SCALE threshold. + * Over-invalidation is architecturally harmless and preserves unrelated TLB + * entries far better than a VMALLE1IS broadcast. + */ +static inline bool tlbi_rvae_normalize(uint64_t *start, uint64_t *end) +{ + for (int pass = 0; pass < 3; pass++) { + uint64_t pages = (*end - *start) >> 12; + if (pages <= TLBI_SELECTIVE_MAX_PAGES) + return true; + uint64_t unit_pages = tlbi_rvae_unit_pages(pages); + uint64_t unit_bytes = unit_pages << 12; + uint64_t s = *start & ~(unit_bytes - 1); + if (*end > UINT64_MAX - (unit_bytes - 1)) + return false; + uint64_t e = (*end + unit_bytes - 1) & ~(unit_bytes - 1); + *start = s; + *end = e; + if (((e - s) >> 12) <= unit_pages * 32) + return ((e - s) >> 12) <= TLBI_RVAE_MAX_PAGES; + } + return false; +} + static inline void tlbi_request_range(uint64_t start, uint64_t end) { if (cpu_tlbi_req.kind == TLBI_BROADCAST) @@ -693,6 +763,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) */ uint64_t large_cap = g_tlbi_range_supported ? TLBI_RVAE_MAX_PAGES : TLBI_SELECTIVE_MAX_PAGES; + if (g_tlbi_range_supported && n > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&s, &e)) { + tlbi_request_broadcast(); + return; + } + n = (e - s) >> 12; + } if (n > large_cap) { tlbi_request_broadcast(); return; @@ -722,6 +799,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) uint64_t us = s < es ? s : es; uint64_t ue = e > ee ? e : ee; uint64_t un = (ue - us) >> 12; + if (g_tlbi_range_supported && un > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&us, &ue)) { + tlbi_request_broadcast(); + return; + } + un = (ue - us) >> 12; + } if (un > large_cap) { tlbi_request_broadcast(); return; @@ -978,6 +1062,38 @@ static inline bool guest_kbuf_user_va_overlap(uint64_t va, uint64_t size) */ void *guest_ptr(const guest_t *g, uint64_t gva); +/* Like guest_ptr_avail but never triggers lazy fault-in. For callers that + * already hold mmap_lock. + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms); + +/* Materialize any lazy (deferred-PTE) blocks intersecting [gva, gva+len) so + * later resolves under locks that rank after mmap_lock (futex buckets) do + * not have to. Takes mmap_lock; call only from lock-free context. Returns 0 + * if anything was (or already is) materialized, -1 otherwise; callers that + * merely pre-fault can ignore the result. + */ +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); + +/* Same as guest_lazy_faultin for callers that already hold mmap_lock (e.g. + * SC_LOCKED syscall handlers about to guest_read/guest_write a lazy range: + * the resolve-time hook would self-deadlock re-acquiring mmap_lock, so they + * must materialize up front through this variant). + */ +int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); + +/* Smallest block-aligned va' in [va, end) whose 1GiB L1 slot is present in + * the page tables, or end if none. Lets range walkers skip absent 1GiB / + * 512GiB slots in O(1) instead of probing every 2MiB block. Locking: callers + * MUST hold mmap_lock. + */ +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end); + /* Get a host pointer for a guest virtual address (write access). * Returns NULL if gva is out of bounds or not writable. */ @@ -1023,6 +1139,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len); */ int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len); +/* Same copy without lazy materialization. Callers that already hold mmap_lock + * can use this after guest_lazy_faultin_locked(); an invalid destination then + * fails instead of recursively trying to acquire mmap_lock. + */ +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len); + /* Optimized host-to-guest copy for small fixed-size outputs. Uses a direct * guest pointer when the full range is contiguous and writable, otherwise falls * back to guest_write() for boundary-crossing safety. @@ -1245,12 +1367,34 @@ bool guest_region_range_has_ro_shared_backing(const guest_t *g, uint64_t start, uint64_t end); -/* Try to materialize a lazy (MAP_NORESERVE) page at the given offset. Called - * from the data/instruction abort handler when the faulting address falls - * within a noreserve region. Creates page table entries for one 2MiB block - * containing the fault address, zeros the memory, and clears the noreserve flag - * for the materialized sub-range. - * Returns 0 on success (caller should TLBI and retry), -1 if the offset is not - * in a noreserve region. +/* Try to materialize a lazy (deferred-PTE: private anonymous or + * MAP_NORESERVE) page at the given offset. Called from the data/instruction + * abort handler when the faulting address falls within a lazy region, and + * from the host-access fault-in path when a syscall targets a lazy range the + * guest has not touched yet. Creates page table entries for one 2MiB block + * containing the fault address. A slab block known to be clean skips zeroing; + * a possibly-dirty block zeros invalid pages before publishing them. A block + * that is already valid returns success without re-zeroing + * (concurrent-fault idempotence). + * Returns 0 on success (caller should TLBI and retry), -1 if the offset is + * not in a lazy region or the region is PROT_NONE. + * Locking: callers MUST hold mmap_lock. */ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset); + +/* Guest-fault variant with per-vCPU sequential fault-around. */ +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset); + +/* Dirty-map and in-flight-claim helpers. Callers hold mmap_lock. */ +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start); +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end); +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end); +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end); +void guest_materialize_wait_all_locked(guest_t *g); + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. + * Locking: callers MUST hold mmap_lock. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va); diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h new file mode 100644 index 00000000..8b2a8fbe --- /dev/null +++ b/src/core/mmap-fastpath.h @@ -0,0 +1,95 @@ +/* + * Per-vCPU EL1 anonymous-mmap consumer rings. + * + * The host is the sole producer of arenas and the sole consumer of ring + * entries. EL1 only bump-allocates VA and appends descriptions. Control + * blocks live in the EL1-only shim-data mapping and are selected from SP_EL1's + * per-thread stack slot, so no guest-visible register ABI is consumed. + */ + +#pragma once + +#include +#include +#include + +#include "core/guest.h" + +typedef struct thread_entry thread_entry_t; + +#define SHIM_MMAP_CONTROL_BASE 0x20000u +#define SHIM_MMAP_CONTROL_STRIDE 0x800u +#define SHIM_MMAP_RING_SIZE 16u +#define SHIM_MMAP_CTRL_ENABLED 0x1u + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (1ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_HISTORY_MULTIPLIER 16u + +enum { + SHIM_MMAP_COUNTER_SHAPE_MISS = 0, + SHIM_MMAP_COUNTER_CAPACITY_MISS, + SHIM_MMAP_COUNTER_RING_FULL, + SHIM_MMAP_COUNTER_GENERATION_STALE, + SHIM_MMAP_COUNTER_ATTENTION, + SHIM_MMAP_COUNTER_HIT, + SHIM_MMAP_COUNTERS_N, +}; + +typedef struct { + uint64_t addr; + uint64_t len; + uint64_t prot; +} shim_mmap_entry_t; + +typedef struct { + _Atomic uint32_t generation; /* host publish word */ + _Atomic uint32_t consumer_generation; /* EL1 generation ack */ + _Atomic uint32_t flags; /* host-owned enable bits */ + _Atomic uint32_t head; /* host consumer cursor */ + _Atomic uint32_t tail; /* EL1 producer cursor */ + uint32_t _pad0; + _Atomic uint64_t arena_base; + _Atomic uint64_t arena_limit; + _Atomic uint64_t cursor; /* EL1 bump cursor */ + uint64_t next_arena_size; /* most recently selected generation size */ + uint64_t max_len_seen; /* outgoing-generation request history */ + shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; + _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; + uint64_t refill_count; + uint64_t recycle_count; + uint64_t peak_arena_size; +} shim_mmap_control_t; + +/* Called while initializing a vCPU, before it can enter guest code. */ +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); + +/* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ +void mmap_fastpath_drain_locked(guest_t *g); + +/* Refill the current vCPU after an eligible mmap slow-path. request_len is + * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. + * Caller holds mmap_lock. + */ +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); + +/* Give an explicit slow-path hint precedence over this stopped vCPU's + * unconsumed arena tail. Caller holds mmap_lock. + */ +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length); + +/* Revoke all arenas while sibling vCPUs are quiesced. Caller holds mmap_lock. + */ +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); + +/* Disable the feature before first guest entry (debugger/observability). */ +void mmap_fastpath_disable(guest_t *g); + +/* Advance *start past an EL1 arena reservation that overlaps length bytes. */ +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index d5fd53d7..02351e1b 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -18,6 +19,7 @@ #include "hvutil.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/vdso.h" #include "debug/log.h" @@ -95,6 +97,42 @@ _Static_assert(SHIM_GLOBALS_SIZE <= BLOCK_2MIB, _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); +_Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, + "shim.S mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x800, + "shim.S mmap fast path hard-codes control stride 0x800"); +_Static_assert(SHIM_MMAP_RING_SIZE == 16, + "shim.S mmap fast path hard-codes 16 ring entries"); +_Static_assert(offsetof(shim_mmap_control_t, generation) == 0, + "shim.S mmap generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, + "shim.S mmap consumer-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, flags) == 8, + "shim.S mmap flags offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, head) == 12, + "shim.S mmap head offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, tail) == 16, + "shim.S mmap tail offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, + "shim.S mmap arena-base offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, + "shim.S mmap arena-limit offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, + "shim.S mmap cursor offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, + "mmap next-arena-size offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, max_len_seen) == 56, + "mmap max-len-seen offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, ring) == 64, + "shim.S mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x1C0, + "shim.S mmap counter offset drift"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "per-vCPU mmap control exceeds its shim-data stride"); +_Static_assert(SHIM_MMAP_CONTROL_BASE + + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE <= + BLOCK_2MIB - MAX_THREADS * 4096, + "mmap controls overlap per-vCPU EL1 stacks"); static uint8_t *cache_base(const guest_t *g) { @@ -125,7 +163,13 @@ static void urandom_ring_unlock(uint32_t *lock_p) void shim_globals_init(guest_t *g) { - memset(cache_base(g), 0, SHIM_GLOBALS_SIZE); + /* mmap controls occupy a separate low shim-data range. Init/exec/fork + * child all call this while no sibling can execute, so clearing the whole + * control array also prevents a recycled SP_EL1 slot from inheriting an + * arena published to its previous owner. + */ + memset(cache_base(g), 0, + SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) @@ -435,11 +479,10 @@ static const char *const counter_names[SHIM_COUNTERS_N] = { [SHIM_COUNTER_URANDOM_HIT] = "URANDOM_HIT", [SHIM_COUNTER_GETRANDOM_HIT] = "GETRANDOM_HIT", [SHIM_COUNTER_PGSID_HIT] = "PGSID_HIT", - /* Slots 12..15 (SHIM_COUNTERS_N == 16) are intentionally unnamed; the dump - * prints "(reserved)" so they appear in the output when non-zero, which - * would flag an out-of-band increment. Bind a name here when a future EL1 - * service claims one of these slots. - */ + [SHIM_COUNTER_FAULT_MATERIALIZE] = "FAULT_MATERIALIZE", + [SHIM_COUNTER_FAULT_TLBI_VAE] = "FAULT_TLBI_VAE", + [SHIM_COUNTER_FAULT_TLBI_RVAE] = "FAULT_TLBI_RVAE", + [SHIM_COUNTER_FAULT_TLBI_BCAST] = "FAULT_TLBI_BCAST", }; uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) @@ -452,6 +495,15 @@ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) return __atomic_load_n(slot_p, __ATOMIC_RELAXED); } +void shim_globals_counter_inc(guest_t *g, unsigned slot) +{ + if (!shim_globals_stats_enabled() || slot >= SHIM_COUNTERS_N) + return; + uint8_t *page = (uint8_t *) g->host_base + g->shim_data_base; + uint64_t *slot_p = (uint64_t *) (page + SHIM_COUNTERS_OFF) + slot; + __atomic_fetch_add(slot_p, 1, __ATOMIC_RELAXED); +} + void shim_globals_counters_dump(const guest_t *g) { fprintf(stderr, "shim-stats (pid=%lld)\n", (long long) proc_get_pid()); @@ -463,6 +515,62 @@ void shim_globals_counters_dump(const guest_t *g) fprintf(stderr, " %-20s %llu\n", name ? name : "(reserved)", (unsigned long long) v); } + + static const char *const mmap_counter_names[SHIM_MMAP_COUNTERS_N] = { + [SHIM_MMAP_COUNTER_SHAPE_MISS] = "MMAP_SHAPE_MISS", + [SHIM_MMAP_COUNTER_CAPACITY_MISS] = "MMAP_CAPACITY_MISS", + [SHIM_MMAP_COUNTER_RING_FULL] = "MMAP_RING_FULL", + [SHIM_MMAP_COUNTER_GENERATION_STALE] = "MMAP_GENERATION_STALE", + [SHIM_MMAP_COUNTER_ATTENTION] = "MMAP_ATTENTION", + [SHIM_MMAP_COUNTER_HIT] = "MMAP_HIT", + }; + uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; + uint64_t refill_count = 0, recycle_count = 0; + uint64_t current_max = 0, peak_max = 0; + const uint8_t *shim_data = + (const uint8_t *) g->host_base + g->shim_data_base; + for (int slot = 0; slot < MAX_THREADS; slot++) { + const shim_mmap_control_t *c = + (const shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + mmap_counters[i] += + atomic_load_explicit(&c->counters[i], memory_order_relaxed); + refill_count += c->refill_count; + recycle_count += c->recycle_count; + if (c->next_arena_size > current_max) + current_max = c->next_arena_size; + if (c->peak_arena_size > peak_max) + peak_max = c->peak_arena_size; + } + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], + (unsigned long long) mmap_counters[i]); + fprintf(stderr, " %-20s %llu\n", "MMAP_REFILL", + (unsigned long long) refill_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", + (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", + (unsigned long long) current_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", + (unsigned long long) peak_max); + uint64_t high_water = + g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; + fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", + (unsigned long long) high_water); + fprintf(stderr, " %-20s %llu\n", "FAULT_CLEAN_SKIP", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]); + fprintf(stderr, " %-20s %llu\n", "FAULT_DIRTY_MEMSET", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_DIRTY_MEMSET]); + fprintf(stderr, " %-20s %llu\n", "FAULT_ALREADY_VALID", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]); + fprintf(stderr, " %-20s %llu\n", "FAULT_WINDOW_BYTES", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES]); } static pthread_once_t stats_once = PTHREAD_ONCE_INIT; diff --git a/src/core/shim-globals.h b/src/core/shim-globals.h index 14cb24f1..049775d4 100644 --- a/src/core/shim-globals.h +++ b/src/core/shim-globals.h @@ -165,7 +165,7 @@ * not in urandom bitmap, len zero, len over inline cap, ring fill below * request, ring wrap, EL0 buffer probe failure). Slots 8..11 record fast-path * hits so bail rates can be computed against a hit denominator. Slots 12..15 - * are reserved. + * attribute lazy-fault materializations and the TLBI wire mode they emit. * * The shim hardcodes the byte offset of each slot; the static_asserts in * shim-globals.c keep the C-side macros and the assembly in sync. @@ -185,6 +185,10 @@ #define SHIM_COUNTER_URANDOM_HIT 9 #define SHIM_COUNTER_GETRANDOM_HIT 10 #define SHIM_COUNTER_PGSID_HIT 11 +#define SHIM_COUNTER_FAULT_MATERIALIZE 12 +#define SHIM_COUNTER_FAULT_TLBI_VAE 13 +#define SHIM_COUNTER_FAULT_TLBI_RVAE 14 +#define SHIM_COUNTER_FAULT_TLBI_BCAST 15 /* Extended identity slots: pgid and sid. * @@ -390,6 +394,7 @@ void shim_globals_refill_urandom_ring(guest_t *g); * when ELFUSE_SHIM_STATS is set. */ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot); +void shim_globals_counter_inc(guest_t *g, unsigned slot); void shim_globals_counters_dump(const guest_t *g); /* ELFUSE_SHIM_STATS env-var gate (idempotent / cached). When enabled the exit diff --git a/src/core/shim.S b/src/core/shim.S index 4c6a1b86..6f1a15c7 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -251,6 +251,28 @@ .Lctr_skip_\@: .endm +/* Per-vCPU mmap counters live after the 16 x 24-byte publication ring in the + * mmap control block. Each vCPU is the sole writer of its own slots. Keep the + * shared stats gate so normal fast paths do not touch diagnostic cache lines. + */ +.equ SHIM_MMAP_COUNTERS_OFF, 0x1C0 +.equ MC_SHAPE_MISS, 0 +.equ MC_CAPACITY_MISS, 1 +.equ MC_RING_FULL, 2 +.equ MC_GENERATION_STALE, 3 +.equ MC_ATTENTION, 4 +.equ MC_HIT, 5 + +.macro MMAP_COUNTER_INC slot + mrs x29, tpidr_el1 + ldrb w30, [x29, #SHIM_STATS_EN_OFF] + cbz w30, .Lmmap_ctr_skip_\@ + ldr x29, [x12, #(SHIM_MMAP_COUNTERS_OFF + 8 * \slot)] + add x29, x29, #1 + str x29, [x12, #(SHIM_MMAP_COUNTERS_OFF + 8 * \slot)] +.Lmmap_ctr_skip_\@: +.endm + /* BAD_VEC: vector-table entry that reports an unexpected exception. * Each table slot is 128 bytes; the leading .align 7 places this entry at the * next 128-byte boundary. @@ -440,6 +462,118 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #222 /* SYS_mmap? */ + b.eq mmap_anon_fast + b handle_svc_0 + +/* mmap_anon_fast: consume a host-prepared, per-vCPU VA arena for the exact + * anonymous RW shape used by allocators: + * + * mmap(NULL, len, PROT_READ|PROT_WRITE, + * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) + * + * The control is selected without another system register. SP_EL1 stack tops + * are shim_data_end - slot*4KiB; controls are shim_data+0x20000 + slot*2KiB, + * so (shim_data_end - ALIGN_UP(sp,4KiB))/2 is the control-array displacement. + * Each vCPU is the sole producer of its ring and cursor. The host acquire- + * drains all rings whenever it takes mmap_lock. + */ +mmap_anon_fast: + mrs x12, tpidr_el1 /* shared shim-data base */ + ldar w13, [x12] /* tracing/signal attention gate */ + + add x14, sp, #0xfff + and x14, x14, #~0xfff /* this vCPU's stack top */ + add x15, x12, #0x200, lsl #12 /* shim_data + 2MiB */ + sub x15, x15, x14 /* slot * 4KiB */ + lsr x15, x15, #1 /* slot * 2KiB */ + add x12, x12, x15 + add x12, x12, #0x20, lsl #12 /* control base + 0x20000 */ + + cbnz w13, mmap_attention_bail + + ldr x14, [sp, #0] /* addr */ + cbnz x14, mmap_shape_bail + ldr x16, [sp, #16] /* prot */ + cmp x16, #3 /* PROT_READ|PROT_WRITE */ + b.ne mmap_shape_bail + ldr x17, [sp, #24] /* flags */ + bic x18, x17, #0x4000 /* tolerate MAP_NORESERVE */ + cmp x18, #0x22 /* MAP_PRIVATE|MAP_ANONYMOUS */ + b.ne mmap_shape_bail + + ldr x15, [sp, #8] /* len */ + cbz x15, mmap_shape_bail + adds x15, x15, #0xfff + b.cs mmap_shape_bail /* PAGE_ALIGN_UP overflow */ + and x15, x15, #~0xfff + + ldar w13, [x12] /* descriptor generation */ + ldr w14, [x12, #4] /* consumer generation */ + cmp w13, w14 + b.ne mmap_generation_bail + ldr w13, [x12, #8] /* SHIM_MMAP_CTRL_ENABLED */ + tbz w13, #0, mmap_capacity_bail + + ldr x19, [x12, #40] /* private bump cursor */ + mov x20, #0x200000 + cmp x15, x20 + b.lo 1f + sub x21, x20, #1 + adds x19, x19, x21 + b.cs mmap_capacity_bail + bic x19, x19, x21 /* >=2MiB: 2MiB-align */ +1: ldr x20, [x12, #32] /* arena limit */ + adds x21, x19, x15 /* new cursor */ + b.cs mmap_capacity_bail + cmp x21, x20 + b.hi mmap_capacity_bail + + add x23, x12, #12 /* &head */ + ldar w22, [x23] + ldr w23, [x12, #16] /* producer-owned tail */ + sub w24, w23, w22 + cmp w24, #16 /* SHIM_MMAP_RING_SIZE */ + b.hs mmap_ring_full_bail + + and w24, w23, #15 + add x24, x24, x24, lsl #1 /* index * 3 */ + add x25, x12, #64 /* ring base */ + add x25, x25, x24, lsl #3 /* index * 24 */ + str x19, [x25, #0] + str x15, [x25, #8] + str x16, [x25, #16] + str x21, [x12, #40] /* publish cursor before entry */ + add w23, w23, #1 + add x25, x12, #16 + stlr w23, [x25] /* release-publish ring entry */ + mov x0, x19 + MMAP_COUNTER_INC MC_HIT + b svc_restore_eret + +mmap_shape_bail: + MMAP_COUNTER_INC MC_SHAPE_MISS + b handle_svc_0 + +mmap_capacity_bail: + MMAP_COUNTER_INC MC_CAPACITY_MISS + b handle_svc_0 + +mmap_ring_full_bail: + MMAP_COUNTER_INC MC_RING_FULL + b handle_svc_0 + +mmap_attention_bail: + MMAP_COUNTER_INC MC_ATTENTION + b attn_bail + +mmap_generation_bail: + /* A revocation deliberately leaves the consumer generation stale. Ack + * only after the acquire load above, then make this syscall take HVC; the + * host will either leave the control disabled or publish a fresh arena. + */ + MMAP_COUNTER_INC MC_GENERATION_STALE + str w13, [x12, #4] b handle_svc_0 identity_class_fast: @@ -1203,6 +1337,7 @@ handle_el0_fault: .Lel0_fault_tlbi_full: /* Broadcast TLB + conditional I-cache flush. X11=0 skips IC IALLU. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Lel0_fault_full_no_ic @@ -1223,6 +1358,7 @@ handle_el0_fault: mov x13, x11 ubfx x11, x9, #12, #44 mov x12, x10 + dsb ishst 4: tlbi vae1is, x11 add x11, x11, #1 subs x12, x12, #1 @@ -1237,7 +1373,9 @@ handle_el0_fault: .Lel0_fault_tlbi_rvae: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE). X9 carries the pre-encoded - * operand (baddr | NUM<<39 | TG=01<<46); X11 the I-cache hint. */ + * operand (baddr | NUM<<39 | SCALE<<44 | TG=01<<46); X11 the I-cache + * hint. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Lel0_fault_rvae_no_ic @@ -1281,6 +1419,7 @@ tlbi_restore_eret: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x0, x0, #12, #44 + dsb ishst tlbi vae1is, x0 dsb ish ic iallu @@ -1356,6 +1495,7 @@ handle_svc_0: * 1 = broadcast TLBI VMALLE1IS * 2 = execve replaced register state (drop frame + flush) * 3 = selective TLBI VAE1IS over X10 pages starting at X9 + * 4 = single-shot TLBI RVAE1IS with encoded operand in X9 * 5. Resume vCPU (execution continues below) */ hvc #5 @@ -1384,6 +1524,7 @@ tlbi_full: * include exec), so the shim must IC IALLU; zero means a data-only * PT change and the I-cache invalidation is skipped. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Ltlbi_full_skip_ic @@ -1419,6 +1560,7 @@ tlbi_selective: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x11, x9, #12, #44 /* x11 = VA[55:12] (current page operand) */ mov x12, x10 /* x12 = remaining page counter */ + dsb ishst 3: tlbi vae1is, x11 add x11, x11, #1 /* next page (operand is in 4 KiB units) */ subs x12, x12, #1 @@ -1434,11 +1576,11 @@ tlbi_selective: tlbi_range_large: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE, ARMv8.4+). The host has * encoded the full operand in X9: baddr (VA >> 12), TTL=0, NUM in bits - * [43:39], SCALE=0, ASID=0. One instruction covers up to 64 pages, - * avoiding the broadcast TLBI VMALLE1IS that the prior selective cap - * forced for 17..64-page ranges. X11 carries the I-cache hint as in - * tlbi_full / tlbi_selective. + * [43:39], SCALE in bits [45:44], ASID=0. One instruction covers up to + * TLBI_RVAE_MAX_PAGES. X11 carries the I-cache hint as in tlbi_full / + * tlbi_selective. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Ltlbi_rvae_skip_ic diff --git a/src/main.c b/src/main.c index 62fac804..9127e598 100644 --- a/src/main.c +++ b/src/main.c @@ -33,6 +33,7 @@ #include "core/rosetta.h" #include "core/shim-globals.h" #include "core/sysroot.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/futex.h" /* futex_interrupt_request */ @@ -654,6 +655,14 @@ int main(int argc, char **argv) return 1; } + /* Tracing/debuggers require every mmap to reach host dispatch so syscall + * observation and region state advance in lockstep. This also prevents a + * trace-gated shim from leaving invisible arena reservations that perturb + * the host slow path's address-hint behavior. + */ + if (verbose || gdb_port > 0) + mmap_fastpath_disable(&g); + /* GDB setup must happen before the first run so entry-stop and hardware * breakpoints can affect the initial vCPU. */ diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index b5b6bc2b..d9ca7a3e 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -181,9 +181,9 @@ int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) #define MAX_USED_REGIONS 16 used_region_t used[MAX_USED_REGIONS]; unsigned int shim_sz = proc_get_shim_size(); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire((guest_t *) (uintptr_t) g); int nregions = guest_get_used_regions(g, shim_sz, used, MAX_USED_REGIONS); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint32_t num_regions = (uint32_t) nregions; if (fork_ipc_write_all(ipc_sock, &num_regions, sizeof(num_regions)) < 0) @@ -695,6 +695,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced) { @@ -747,6 +748,9 @@ int fork_ipc_send_process_state(int ipc_sock, fork_ipc_write_all(ipc_sock, regions_snapshot, num_guest_regions * sizeof(guest_region_t)) < 0) return -1; + if (fork_ipc_write_all(ipc_sock, dirty_blocks_snapshot, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) + return -1; if (fork_ipc_write_all(ipc_sock, &num_preannounced, sizeof(num_preannounced)) < 0) @@ -958,6 +962,12 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); + if (fork_ipc_read_all(ipc_fd, g->dirty_blocks, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) { + log_error("fork-child: failed to read dirty block bitmap"); + return -1; + } + uint32_t num_preannounced = 0; if (fork_ipc_read_all(ipc_fd, &num_preannounced, sizeof(num_preannounced)) < 0) { diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 87a59555..572b4d40 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464CU /* "ELFL" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464DU /* "ELFM" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -127,6 +127,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced); int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig); diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index d00acf70..427c96fe 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -32,6 +32,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/fork-state.h" @@ -438,6 +439,11 @@ int fork_child_main(int ipc_fd, */ shim_globals_rebuild_urandom_bitmap(); + if (!verbose) + mmap_fastpath_prepare_vcpu(&g, current_thread); + else + mmap_fastpath_disable(&g); + /* Now that current_thread is set, apply signal state. This must happen * after thread_register_main() so the per-thread blocked mask and altstack * are properly restored to the thread entry. @@ -498,7 +504,7 @@ typedef struct { vcpu_simd_state_t simd_state; } thread_create_args_t; -static void resolve_clone_stack_range(const guest_t *g, +static void resolve_clone_stack_range(guest_t *g, uint64_t child_stack, uint64_t *start_out, uint64_t *end_out) @@ -514,14 +520,22 @@ static void resolve_clone_stack_range(const guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; + /* EL1 consumer-mmap entries are drained into regions[] under mmap_lock. + * A sibling worker can drain and merge the array while clone resolves its + * new stack, so take the same lock and copy the bounds before releasing it. + */ + mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); - if (!r) + if (!r) { + mmap_lock_release(); return; + } if (start_out) *start_out = r->start; if (end_out) *end_out = r->end; + mmap_lock_release(); } /* Forward declaration: worker entry runs after sys_clone_thread */ @@ -947,6 +961,8 @@ startup_ok:; */ thread_fork_barrier_check(); + mmap_fastpath_prepare_vcpu(g, t); + log_debug("thread tid=%lld starting on vCPU", (long long) t->guest_tid); vcpu_run_loop(vcpu, vexit, g, verbose, 0); @@ -962,10 +978,12 @@ startup_ok:; * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain any deferred munmap of this thread's stack before waking the - * joiner: the parent may reuse the freed VA as soon as it returns from - * pthread_join, and reuse must not race with the deferred unmap. + * Drain any deferred munmap before publishing clear_child_tid. A joiner + * may observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * freed VA immediately; ordering only the wake after cleanup leaves a + * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -980,7 +998,6 @@ startup_ok:; (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1230,15 +1247,17 @@ static void *vm_clone_thread_run(void *arg) /* Set per-thread TLS pointer and enter worker run loop */ current_thread = t; thread_fork_barrier_check(); + mmap_fastpath_prepare_vcpu(g, t); log_debug("vm_clone tid=%lld starting on vCPU", (long long) t->guest_tid); int exit_code = vcpu_run_loop(vcpu, vexit, g, verbose, 0); - /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: drain - * deferred stack munmaps before waking the joiner so the parent does not - * reuse the VA before it is released. + /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: the zero + * itself, not just the futex wake, releases a joiner, so publish it only + * after the deferred stack mapping is gone. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1253,7 +1272,6 @@ static void *vm_clone_thread_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1534,6 +1552,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, mmap_fork_anon_shared_txn_t *anon_shared_txn = NULL; guest_region_t *regions_snapshot = NULL; + uint64_t *dirty_blocks_snapshot = NULL; guest_region_t preannounced_snapshot[GUEST_MAX_PREANNOUNCED]; /* APFS clone fd for the CoW snapshot sent to the child. Declared up front * so early goto fail_snapshot exits do not read an uninitialized local. @@ -1749,6 +1768,10 @@ int64_t sys_clone(hv_vcpu_t vcpu, } memcpy(regions_snapshot, g->regions, snap_sz); } + dirty_blocks_snapshot = malloc(sizeof(g->dirty_blocks)); + if (!dirty_blocks_snapshot) + goto fail_snapshot; + memcpy(dirty_blocks_snapshot, g->dirty_blocks, sizeof(g->dirty_blocks)); int npreannounced_snapshot = g->npreannounced; if (npreannounced_snapshot > 0) { memcpy(preannounced_snapshot, g->preannounced, @@ -1773,8 +1796,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, uint32_t num_preannounced = (uint32_t) npreannounced_snapshot; if (fork_ipc_send_process_state( ipc_sock, regions_snapshot, num_guest_regions, - regions_tracker_stale_snapshot, preannounced_snapshot, - num_preannounced) < 0) { + regions_tracker_stale_snapshot, dirty_blocks_snapshot, + preannounced_snapshot, num_preannounced) < 0) { log_error("clone: failed to send process state"); goto fail_snapshot; } @@ -1837,12 +1860,14 @@ int64_t sys_clone(hv_vcpu_t vcpu, child_host_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); return child_guest_pid; fail_snapshot: free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); /* Roll back the in-place anon-shared overlay conversion while siblings are diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 826d1a1f..871af561 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -1493,6 +1493,16 @@ int64_t sys_futex(guest_t *g, { int cmd = op & FUTEX_CMD_MASK; + /* Pre-fault lazy mappings before any bucket lock is taken. The word + * resolves below run under per-bucket locks, which rank after mmap_lock; + * materializing there would invert the lock order. A futex word in a + * mapping the guest never touched reads as zero, matching Linux. + */ + guest_lazy_faultin(g, uaddr, sizeof(uint32_t)); + if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE || + cmd == FUTEX_WAKE_OP) + guest_lazy_faultin(g, uaddr2, sizeof(uint32_t)); + switch (cmd) { case FUTEX_WAIT: #if ELFUSE_HAVE_OS_SYNC_WAIT_ON_ADDRESS @@ -1734,6 +1744,11 @@ int64_t sys_futex_waitv(guest_t *g, */ if (!futex_uaddr_is_aligned(elts[i].uaddr)) return -LINUX_EINVAL; + /* Pre-fault lazy mappings: the word resolves below run with every + * bucket lock held, where materializing would invert the lock order + * against mmap_lock. + */ + guest_lazy_faultin(g, elts[i].uaddr, sizeof(uint32_t)); } waitv_shared_t shared; diff --git a/src/syscall/exec.c b/src/syscall/exec.c index d694ba74..a31a0976 100644 --- a/src/syscall/exec.c +++ b/src/syscall/exec.c @@ -668,6 +668,14 @@ int64_t sys_execve(hv_vcpu_t vcpu, return err; } + /* Input copying above may fault in argv/env strings from lazy anonymous + * mappings, so it must run without mmap_lock held. Serialize only after + * all recoverable validation is complete and immediately before replacing + * the guest address space. From this point every failure is fatal and both + * successful return paths release the lock explicitly. + */ + mmap_lock_acquire(g); + /* Point of no return. guest_reset() zeroes all guest memory. The old * process image is gone. All validation that can fail gracefully MUST * happen above this line. Failures below are unrecoverable; elfuse exits @@ -883,6 +891,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, free(temp_str); exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } @@ -1187,6 +1196,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; too_many_regions: diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 35882e9c..176a7d3f 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -50,6 +50,13 @@ typedef int host_fd_t; extern pthread_mutex_t mmap_lock; /* Lock order: 1, mmap/brk + page tables */ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ +/* The only supported mmap_lock entry/exit API. Acquire drains the per-vCPU + * EL1 mmap rings before any caller can inspect semantic region state. + */ +void mmap_lock_acquire(guest_t *g); +void mmap_lock_release(void); +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); + /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index a5477c5e..3c791c2f 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -21,8 +21,10 @@ #include #include "debug/log.h" +#include "debug/syscall-hist.h" #include "utils.h" +#include "core/mmap-fastpath.h" #include "runtime/thread.h" #include "syscall/abi.h" #include "syscall/fuse.h" @@ -35,6 +37,392 @@ */ pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ +static pthread_once_t mmap_fastpath_env_once = PTHREAD_ONCE_INIT; +static bool mmap_fastpath_env_enabled; +static _Atomic bool mmap_fastpath_forced_off; + +static uint64_t find_free_gap_inner(const guest_t *g, + uint64_t length, + uint64_t min_addr, + uint64_t max_addr, + uint64_t align); + +static void mmap_fastpath_read_env(void) +{ + const char *v = getenv("ELFUSE_MMAP_FASTPATH"); + mmap_fastpath_env_enabled = + !v || (strcmp(v, "0") != 0 && strcmp(v, "false") != 0); +} + +static bool mmap_fastpath_available(const guest_t *g) +{ + pthread_once(&mmap_fastpath_env_once, mmap_fastpath_read_env); + return mmap_fastpath_env_enabled && !g->is_rosetta && + !atomic_load_explicit(&mmap_fastpath_forced_off, + memory_order_acquire) && + !syscall_hist_enabled(); +} + +static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) +{ + if (!g || !g->host_base || slot < 0 || slot >= MAX_THREADS) + return NULL; + return (shim_mmap_control_t *) ((uint8_t *) g->host_base + + g->shim_data_base + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); +} + +static void mmap_fastpath_disable_control(shim_mmap_control_t *c) +{ + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); + atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + c->next_arena_size = MMAP_FAST_ARENA_MIN; + c->max_len_seen = 0; + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_RING_SIZE) { + log_fatal( + "mmap fast path: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t arena_limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + while (head != tail) { + const shim_mmap_entry_t *e = + &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; + uint64_t addr = e->addr; + uint64_t len = e->len; + if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || + (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + e->prot != (LINUX_PROT_READ | LINUX_PROT_WRITE)) { + log_fatal( + "mmap fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx arena=0x%llx..0x%llx)", + slot, (unsigned long long) addr, (unsigned long long) len, + (unsigned long long) arena_base, + (unsigned long long) arena_limit); + } + if (guest_region_add_ex(g, addr, addr + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) { + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a + * false SIGSEGV, so fail closed on the violated provisioning + * invariant instead of silently corrupting process state. + */ + log_fatal( + "mmap fast path: region metadata exhausted while " + "draining vCPU slot %d", + slot); + } + if (len > c->max_len_seen) + c->max_len_seen = len; + head++; + } + atomic_store_explicit(&c->head, head, memory_order_release); + } +} + +void mmap_lock_acquire(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_release(void) +{ + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) +{ + pthread_cond_wait(cond, &mmap_lock); + /* pthread_cond_wait reacquires mmap_lock directly, so preserve the + * drain-before-region-read invariant of mmap_lock_acquire(). + */ + mmap_fastpath_drain_locked(g); +} + +static bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + return start <= limit && len <= limit - start; +} + +static uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + value--; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + return value + 1; +} + +static uint64_t mmap_fastpath_arena_size(uint64_t max_len_seen, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (max_len_seen) { + uint64_t target = + max_len_seen > MMAP_FAST_ARENA_MAX / MMAP_FAST_HISTORY_MULTIPLIER + ? MMAP_FAST_ARENA_MAX + : max_len_seen * MMAP_FAST_HISTORY_MULTIPLIER; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} + +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len) +{ + if (!t || t->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = mmap_fastpath_control(g, t->sp_el1_slot); + if (!c) + return; + if (!mmap_fastpath_available(g)) { + mmap_fastpath_disable_control(c); + return; + } + + /* Giant requests are deliberately slow-path-only. Do not abandon a useful + * small-request arena or poison its adaptive history. + */ + if (request_len > MMAP_FAST_ARENA_MAX) + return; + + if (request_len > c->max_len_seen) + c->max_len_seen = request_len; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + if ((flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len)) + return; + + /* The owner is parked in HVC. Make the stranded tail immediately + * recyclable before the gap scan; mappings already served from the prefix + * were drained into regions[] on mmap_lock acquisition. + */ + if (flags & SHIM_MMAP_CTRL_ENABLED) + atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); + + uint64_t arena_size = + mmap_fastpath_arena_size(c->max_len_seen, request_len); + + /* Prefer a real hole below the current high-water mark. Active sibling + * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap + * allocator. Only grow mmap_next when no recyclable hole fits. + */ + uint64_t high = g->mmap_next; + if (high > g->mmap_limit) + high = g->mmap_limit; + uint64_t base = UINT64_MAX; + bool recycled = false; + if (high > MMAP_BASE) { + base = find_free_gap_inner(g, arena_size, MMAP_BASE, high, BLOCK_2MIB); + recycled = base != UINT64_MAX; + } + + if (!recycled) { + if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { + mmap_fastpath_disable_control(c); + return; + } + base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); + if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { + mmap_fastpath_disable_control(c); + return; + } + } + uint64_t new_limit = base + arena_size; + + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. + */ + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(c); + return; + } + if (!recycled) { + g->mmap_next = new_limit; + if (g->mmap_rw_gap_hint < new_limit) + g->mmap_rw_gap_hint = new_limit; + } + + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + c->next_arena_size = arena_size; + c->max_len_seen = 0; + c->refill_count++; + if (recycled) + c->recycle_count++; + if (arena_size > c->peak_arena_size) + c->peak_arena_size = arena_size; + atomic_store_explicit(&c->flags, SHIM_MMAP_CTRL_ENABLED, + memory_order_relaxed); + /* This vCPU is stopped in HVC (or has never run), so host may acknowledge + * the freshly published descriptor on its behalf. Revocation deliberately + * does not do this, making an in-flight stale generation bail once. + */ + atomic_store_explicit(&c->consumer_generation, generation, + memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) +{ + mmap_fastpath_refill_thread_locked(g, current_thread, request_len); +} + +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length) +{ + if (!current_thread || current_thread->sp_el1_slot < 0 || !length || + addr > UINT64_MAX - length) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor >= limit || addr >= limit || addr + length <= cursor) + return; + + /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race + * this descriptor update. Revoke its whole unconsumed tail: the explicit + * hint must remain semantically free, and the post-syscall refill will + * provision a new non-overlapping arena. + */ + mmap_fastpath_disable_control(c); +} + +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) +{ + mmap_lock_acquire(g); + mmap_fastpath_refill_thread_locked(g, t, 0); + mmap_lock_release(); +} + +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) +{ + mmap_fastpath_drain_locked(g); + for (int slot = 0; slot < MAX_THREADS; slot++) + mmap_fastpath_disable_control(mmap_fastpath_control(g, slot)); + + if (!shrink_high_water) + return; + uint64_t high = MMAP_BASE; + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= MMAP_BASE && r->start < g->mmap_limit && r->end > high) + high = r->end; + } + g->mmap_next = high; + if (g->mmap_rw_gap_hint > high) + g->mmap_rw_gap_hint = high; +} + +void mmap_fastpath_disable(guest_t *g) +{ + atomic_store_explicit(&mmap_fastpath_forced_off, true, + memory_order_release); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, true); + mmap_lock_release(); +} + +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr) +{ + if (!g || !start || !length) + return; + bool advanced; + do { + advanced = false; + if (*start > max_addr || length > max_addr - *start) + return; + uint64_t end = *start + length; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t cursor = + atomic_load_explicit(&c->cursor, memory_order_acquire); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor < limit && *start < limit && end > cursor) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + } + } while (advanced); +} + /* Host kernel page size (16 KiB on Apple Silicon, typically 4 KiB on Intel * macOS). MAP_FIXED requires addr/length/offset multiples of this, so an * overlay onto a guest 4 KiB-aligned IPA is only applicable when the IPA @@ -103,6 +491,11 @@ static int restore_snapshot_page_tables(guest_t *g, static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n); +static int read_file_range_to_guest(guest_t *g, + uint64_t gpa, + int fd, + uint64_t file_off, + uint64_t len); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -298,6 +691,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, * allocation patterns. */ uint64_t gap_start = ALIGN_UP(min_addr, align); + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* Skip the prefix of regions entirely below gap_start in O(log n). After a * successful allocation the gap hint advances near or past the existing @@ -306,6 +700,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, */ for (int i = guest_region_first_end_above(g, gap_start); i < g->nregions; i++) { + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* A region can still slip below gap_start after the ALIGN_UP advance * below skips past a smaller adjacent region; keep the cheap guard. */ @@ -335,6 +730,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, } /* Check trailing space after all regions */ + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); if (gap_start <= max_addr && length <= max_addr - gap_start) return gap_start; return UINT64_MAX; /* No suitable gap found */ @@ -714,6 +1110,7 @@ static int64_t sys_mmap_high_va(guest_t *g, if (!host) goto fail; memset(host, 0, BLOCK_2MIB); + guest_dirty_clear_zeroed_range(g, gpa, gpa + BLOCK_2MIB); /* Detect freshness BEFORE guest_map_va_range so the decision is not * confused by a prior high-VA mmap into the same 2 MiB block. A fresh @@ -787,27 +1184,11 @@ static int64_t sys_mmap_high_va(guest_t *g, } else if (prot != LINUX_PROT_NONE) { memset(map_host, 0, length); replaced_bytes_dirty = replacing_existing; - uint8_t *dst = map_host; - size_t remaining = length; - off_t file_off = (off_t) offset; - while (remaining > 0) { - ssize_t nr = pread(host_backing_fd, dst, remaining, file_off); - if (nr < 0) { - if (errno == EINTR) - continue; - /* Real host I/O failure (not EINTR); previously the loop broke - * without setting ret and the syscall returned a "successful" - * partially-zero mapping. - */ - ret = linux_errno(); - goto fail; - } - if (nr == 0) - break; - dst += nr; - remaining -= (size_t) nr; - file_off += nr; - } + uint64_t gpa_for_addr = backing_gpa_start + (addr - va_start); + ret = read_file_range_to_guest(g, gpa_for_addr, host_backing_fd, offset, + length); + if (ret < 0) + goto fail; } /* Install L3 PTEs for the actual mapped range. Fresh blocks were fully @@ -1017,6 +1398,11 @@ static int read_file_range_to_guest(guest_t *g, uint8_t *dst = host_ptr_for_gpa(g, gpa); if (!dst) return -LINUX_EFAULT; + /* A short read, EOF, or later error may still leave nonzero bytes in the + * destination. Mark before the first pread so every exit is conservative. + */ + if (len <= UINT64_MAX - gpa) + guest_dirty_mark_range(g, gpa, gpa + len); size_t remaining = len; while (remaining > 0) { @@ -1639,6 +2025,8 @@ static int hvf_apply_file_overlay(guest_t *g, thread_quiesce_siblings(); int err = hvf_apply_file_overlay_quiesced(g, ipa, len, fd, file_off); thread_resume_siblings(); + if (err == 0 && len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); return err; } @@ -1677,6 +2065,12 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, hvf_remap_segments_best_effort(g, segments, nsegments); return err; } + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. + * The following munmap/MAP_FIXED path will clear the bit only after it has + * actually zeroed a complete 2 MiB block. + */ + if (len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); for (int i = 0; i < nsegments; i++) { if (hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, @@ -1860,6 +2254,19 @@ int64_t sys_mmap(guest_t *g, bool needs_exec = (prot & LINUX_PROT_EXEC) != 0; bool is_prot_none = (prot == LINUX_PROT_NONE); bool is_noreserve = is_anon && (flags & LINUX_MAP_NORESERVE) != 0; + /* Anonymous mappings defer page-table creation and zeroing to first touch + * (guest fault or host-side access), like MAP_NORESERVE always has. This + * keeps mmap()/munmap() cost independent of length: a multi-GiB + * reservation costs neither an eager PTE walk nor a full-length memset, + * and never-touched blocks consume no page-table pool. PROT_NONE stays a + * pure reservation (faults deliver SIGSEGV, not materialization), and + * MAP_FIXED keeps the eager path because it must atomically replace live + * mappings. Shared anonymous memory stays eager unless the caller opted + * into MAP_NORESERVE (the historical lazy set), since deferred zeroing + * has never been exercised against the fork snapshot paths for it. + */ + bool is_lazy = is_anon && !is_prot_none && + ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); host_fd_ref_t backing_ref = {.fd = -1, .owned = 0}; int host_backing_fd = -1, track_backing_fd = -1; /* Tracks whether hvf_apply_file_overlay has installed a host @@ -1887,13 +2294,24 @@ int64_t sys_mmap(guest_t *g, region_snapshot_t *replaced_snaps = NULL; int replaced_nsnaps = 0; bool replaced_regions_removed = false; + /* Linux kernel rejects MAP_FIXED with non-page-aligned address (checked + * below); the flag itself is needed early because it gates the lazy path. + */ + bool is_fixed = + (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); + if (is_fixed) + is_lazy = false; int track_flags = ((flags & LINUX_MAP_SHARED) ? LINUX_MAP_SHARED : LINUX_MAP_PRIVATE); if (is_anon) track_flags |= LINUX_MAP_ANONYMOUS; - /* Preserve MAP_NORESERVE in region metadata before merge checks run. */ - if (is_noreserve) + /* Preserve MAP_NORESERVE in region metadata before merge checks run. The + * same bit doubles as the internal lazy marker: guest_region_add_ex + * derives the region's deferred-PTE flag from it, and it is not guest + * visible (/proc/self/maps prints only prot and shared/private). + */ + if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; /* The memory syscall layer handles all mmap variants. Aligned file-backed @@ -1928,9 +2346,17 @@ int64_t sys_mmap(guest_t *g, if (length == 0) return -LINUX_ENOMEM; + /* A non-fixed nonzero address is a strong Linux hint. If it lands in the + * current (stopped) vCPU's invisible arena tail, release that tail before + * gap finding so implementation-only VA preparation does not perturb the + * address the application observes. Sibling arenas stay immutable without + * quiesce; their disjoint high-water placement makes self-overlap the + * normal and important case (allocator hinting near its previous result). + */ + if (!is_fixed && addr != 0) + mmap_fastpath_release_current_hint_locked(g, addr, length); + /* Linux kernel rejects MAP_FIXED with non-page-aligned address */ - bool is_fixed = - (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); if (is_fixed && (addr & 4095)) return -LINUX_EINVAL; @@ -1979,6 +2405,7 @@ int64_t sys_mmap(guest_t *g, uint64_t fix_end = off + length; if (guest_range_hits_infra(g, off, fix_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, off, fix_end); result_off = off; @@ -2376,7 +2803,7 @@ int64_t sys_mmap(guest_t *g, guest_invalidate_ptes(g, result_off, result_off + length); } - if (!is_prot_none && !is_fixed && !is_noreserve) { + if (!is_prot_none && !is_fixed && !is_lazy) { /* Extend page tables for this specific allocation range only. * guest_extend_page_tables skips already-mapped blocks, so calling it * on pre-mapped regions is a no-op. This avoids creating entries for @@ -2435,16 +2862,24 @@ int64_t sys_mmap(guest_t *g, g->mmap_end = ext_end; } - /* Zero the mapped region */ + /* Zero the mapped region. RX mappings cannot be dirtied through their + * published PTEs, so a complete-block zero can make them clean again. + * Other mappings currently use writable stage-1 entries and must stay + * conservatively dirty even if their requested Linux prot is read-only. + */ memset((uint8_t *) g->host_base + result_off, 0, length); + if (needs_exec && !(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, result_off, result_off + length); } - /* MAP_NORESERVE: invalidate any stale PTEs (like PROT_NONE path) but track - * the region for lazy materialization on first fault. Page table entries - * will be created by guest_materialize_lazy() when the guest first touches - * a page in this range. + /* Lazy (private anonymous, incl. MAP_NORESERVE): invalidate any stale PTEs + * (like the PROT_NONE path) but track the region for lazy materialization + * on first fault. Page table entries will be created by + * guest_materialize_lazy() when the guest first touches a page in this + * range, or by the host-access fault-in path when a syscall targets the + * range before the guest ever touches it. */ - if (is_noreserve && !is_fixed) { + if (is_lazy) { guest_invalidate_ptes(g, result_off, result_off + length); } @@ -2512,6 +2947,7 @@ int64_t sys_mmap(guest_t *g, overlay_ipa = result_off; overlay_len = nf_overlay_len; } else { + guest_dirty_mark_range(g, result_off, result_off + length); uint8_t *dst = (uint8_t *) g->host_base + result_off; size_t remaining = length; off_t file_off = offset; @@ -2688,6 +3124,11 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; + if (old_off < g->guest_size) + guest_materialize_wait_range_locked(g, old_off, + old_size > g->guest_size - old_off + ? g->guest_size + : old_off + old_size); /* Verify the whole source range is covered by one tracked VMA. mremap() * must not copy holes or unrelated adjacent mappings. @@ -2721,10 +3162,13 @@ int64_t sys_mremap(guest_t *g, /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ - memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, - tail_end - tail_off); + uint64_t tail_gpa = src_gpa_base + (tail_off - src_start); + if (guest_invalidate_ptes(g, tail_off, tail_end) < 0) + return -LINUX_ENOMEM; + memset(host_ptr_for_gpa(g, tail_gpa), 0, tail_end - tail_off); + guest_dirty_clear_zeroed_range(g, tail_gpa, + tail_gpa + (tail_end - tail_off)); guest_region_remove(g, tail_off, tail_end); - guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) @@ -2750,6 +3194,7 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, new_off, new_off + new_size); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; @@ -2936,12 +3381,20 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping */ if (old_size > 0) { - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove(g, old_off, old_off + old_size); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3025,6 +3478,9 @@ int64_t sys_mremap(guest_t *g, } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); + if (!(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, grow_off, + grow_off + grow_len); /* Update region tracking: remove old, add extended */ guest_region_remove(g, old_off, old_off + old_size); @@ -3176,13 +3632,21 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping. Any live source overlay was already torn down * before the destination range was touched. */ - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove(g, old_off, old_off + old_size); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3293,6 +3757,8 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) */ if (!madvise_range_mapped(g, off, length)) return -LINUX_ENOMEM; + if (in_primary) + guest_materialize_wait_range_locked(g, off, off + length); uint64_t end = off + length; for (int i = 0; i < g->nregions; i++) { @@ -3406,11 +3872,38 @@ static int compare_range_pair(const void *a, const void *b) return 0; } +/* Coalesced sub-ranges of a munmap that must be zeroed. Sized so that even a + * pathologically fragmented lazy mapping (alternating materialized and + * untouched blocks) rarely overflows; on overflow the remainder of the region + * overlap is zeroed wholesale, which is always correct (zeroing already-zero + * slab bytes), just slower. + */ +#define MUNMAP_ZERO_RANGES_MAX 128 + +typedef struct { + uint64_t lo, hi; +} zero_range_t; + +static void zero_range_push(zero_range_t *ranges, + int *n, + uint64_t lo, + uint64_t hi) +{ + if (lo >= hi) + return; + if (*n > 0 && ranges[*n - 1].hi == lo) { + ranges[*n - 1].hi = hi; + return; + } + ranges[(*n)++] = (zero_range_t) {lo, hi}; +} + static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) { /* Reject munmap targeting VM infrastructure regions. */ if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, unmap_off, end); /* Restore slab backing under any active MAP_SHARED file overlay before * zeroing the host VA. Without this, the memset below would write zeros @@ -3420,14 +3913,19 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (cleanup_err < 0) return cleanup_err; - /* Invalidate PTEs first. This may need to split a 2MiB block which can fail - * if the page table pool is exhausted. Failing before region removal keeps - * metadata consistent. + /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below + * destroys the evidence. Eager regions are zeroed across the whole + * overlap, as before. Lazy (deferred-PTE) regions only need their + * materialized 2MiB blocks zeroed: a block with no L2 mapping was never + * touched through PTEs, host-side fault-in materializes before writing, + * and the previous unmap of that slab range zeroed it -- so its bytes are + * still zero. This keeps munmap cost proportional to memory actually + * touched instead of to the mapping length. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) - return -LINUX_ENOMEM; + zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; + int nzr = 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (r->start >= end) break; if (r->end <= unmap_off) @@ -3436,7 +3934,50 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; - memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); + if (!r->noreserve) { + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: widen the last range instead of dropping any + * span that must be zeroed. Everything between ranges lies + * inside [unmap_off, end) and is being unmapped, so zeroing + * the gap as well is harmless. + */ + zr[nzr - 1].hi = zend; + continue; + } + zero_range_push(zr, &nzr, zstart, zend); + continue; + } + for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { + if (!guest_va_block_mapped(g, b)) { + /* Skip absent 1GiB/512GiB slots wholesale; a huge untouched + * reservation would otherwise pay one walk per 2MiB. + */ + b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); + continue; + } + uint64_t lo = (b > zstart) ? b : zstart; + uint64_t hi = (b + BLOCK_2MIB < zend) ? b + BLOCK_2MIB : zend; + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: fold the remainder of this overlap into the + * last range and stop scanning blocks. + */ + zr[nzr - 1].hi = zend; + break; + } + zero_range_push(zr, &nzr, lo, hi); + b += BLOCK_2MIB; + } + } + + /* Invalidate PTEs first. This may need to split a 2MiB block which can fail + * if the page table pool is exhausted. Failing before region removal keeps + * metadata consistent. + */ + if (guest_invalidate_ptes(g, unmap_off, end) < 0) + return -LINUX_ENOMEM; + for (int i = 0; i < nzr; i++) { + memset((uint8_t *) g->host_base + zr[i].lo, 0, zr[i].hi - zr[i].lo); + guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove(g, unmap_off, end); if (unmap_off < g->mmap_rw_gap_hint) @@ -3461,7 +4002,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) if (nranges <= 0) return; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < nranges; i++) { int rc = munmap_guest_range(g, starts[i], ends[i]); if (rc < 0) { @@ -3474,7 +4015,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) } thread_drop_deferred_stack_unmap(t, starts[i], ends[i]); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* sys_munmap. */ @@ -3624,6 +4165,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) */ if (guest_range_hits_infra(g, mprot_off, mprot_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, mprot_off, mprot_end); /* Same max_prot check as the high-VA branch above. */ if ((prot & LINUX_PROT_WRITE) && @@ -3641,6 +4183,18 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (prot != LINUX_PROT_NONE) { int page_perms = prot_to_perms(prot); + /* Materialize lazy blocks in the range at their region's + * current prot before the block-granular extend below. The + * extend stamps whole 2MiB blocks with page_perms; on an + * unmaterialized lazy region that would hand every neighbor + * page OUTSIDE [mprot_off, mprot_end) the sub-range's + * permissions (region says RW, PTE says R-only, host-side + * writes EFAULT). guest_materialize_lazy covers block-within- + * region at region prot, so after this the extend is a no-op + * for lazy regions and update_perms below adjusts only the + * requested range. + */ + guest_lazy_faultin_locked(g, mprot_off, mprot_end - mprot_off); if (guest_extend_page_tables(g, mprot_off, mprot_end, page_perms) < 0) return -LINUX_ENOMEM; @@ -3934,7 +4488,13 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + /* fork callers have quiesced siblings. Drain their last publications, + * revoke every descriptor, and trim never-consumed arena tails before the + * legacy [MMAP_BASE,mmap_next) snapshot range is computed. + */ + mmap_fastpath_revoke_all_locked(g, true); + guest_materialize_wait_all_locked(g); size_t hps = host_page_size_cached(); @@ -4081,7 +4641,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return -LINUX_ENOMEM; } @@ -4094,7 +4654,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return nsnaps; } @@ -4134,7 +4694,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, close(fd); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return 0; } @@ -4153,7 +4713,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, mmap_fork_anon_shared_txn_t *txn = *txn_ptr; int rc = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = txn->noverlays - 1; i >= 0; i--) { const fork_overlay_snapshot_t *ovl = &txn->overlays[i]; @@ -4222,7 +4782,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); mmap_fork_dispose_anon_shared_txn(txn_ptr); return rc; } @@ -4236,7 +4796,7 @@ int mmap_fork_restore_overlays(guest_t *g, const uint64_t *parent_ovl_start, const uint64_t *parent_ovl_end) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int rc = 0; for (int i = 0; i < g->nregions; i++) { @@ -4325,6 +4885,6 @@ int mmap_fork_restore_overlays(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return rc; } diff --git a/src/syscall/proc.c b/src/syscall/proc.c index fe2aa533..003a202a 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -2168,7 +2168,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, * lookups to prevent races with concurrent * mmap/mprotect/munmap from other vCPU threads. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); /* Check if this is a genuine permission violation (not a * W^X toggle). If the guest region lacks the required @@ -2182,7 +2182,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, int required = (type == 1) ? LINUX_PROT_WRITE : LINUX_PROT_EXEC; if (reg && !(reg->prot & required)) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint64_t esr; hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_ESR_EL1, &esr); signal_set_fault_info(LINUX_SEGV_ACCERR, far, esr); @@ -2210,7 +2210,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, int sr = guest_split_block(g, block_start); int ur = guest_update_perms(g, page_start, page_end, new_perms); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (verbose && (sr < 0 || ur < 0)) log_warn( "%s: W^X toggle FAILED " @@ -2377,9 +2377,9 @@ int vcpu_run_loop(hv_vcpu_t vcpu, uint32_t fsc_type = (fsc >> 2) & 0xF; if (fsc_type == 0x01) { uint64_t fault_off = far_addr - g->ipa_base; - pthread_mutex_lock(&mmap_lock); - int mat = guest_materialize_lazy(g, fault_off); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + int mat = guest_materialize_lazy_fault(g, fault_off); + mmap_lock_release(); if (mat == 0) { /* Page materialized; the helpers inside * guest_materialize_lazy populated the per-vCPU @@ -2393,6 +2393,25 @@ int vcpu_run_loop(hv_vcpu_t vcpu, * re-fault on the retry, looping until the entry * self-evicts. */ + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_MATERIALIZE); + switch ((tlbi_kind_t) cpu_tlbi_req.kind) { + case TLBI_RANGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_VAE); + break; + case TLBI_RANGE_LARGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_RVAE); + break; + case TLBI_BROADCAST: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_BCAST); + break; + case TLBI_NONE: + default: + break; + } tlbi_request_emit_to_vcpu(vcpu); break; } @@ -2446,10 +2465,10 @@ int vcpu_run_loop(hv_vcpu_t vcpu, uint64_t live_avail = 0; void *live_pt = NULL; if (stale_plausible) { - pthread_mutex_lock(&mmap_lock); - live_pt = guest_ptr_avail(g, far_addr, &live_avail, - want_perm); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + live_pt = guest_ptr_avail_nofault( + g, far_addr, &live_avail, want_perm); + mmap_lock_release(); } if (live_pt) { /* Bound per vCPU and per (page, faulting PC). A diff --git a/src/syscall/signal.c b/src/syscall/signal.c index d6f628a9..bb8957d5 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -1748,8 +1748,31 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, return 1; } +/* Pre-fault the candidate signal-frame windows (current stack and altstack + * top) before sig_lock is taken. The frame write in deliver_signal_locked + * runs under sig_lock; letting it materialize lazy stack pages there would + * acquire mmap_lock in descending lock order. The pre-fault is advisory -- + * the write path still faults in as a backstop -- but it makes the + * under-lock engagement unreachable in practice. Reading the altstack + * fields without sig_lock is benign for the same reason. + */ +static void signal_prefault_frame(hv_vcpu_t vcpu, guest_t *g) +{ + uint64_t need = sizeof(linux_rt_sigframe_t) + 512; + uint64_t sp = 0; + hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_SP_EL0, &sp); + if (sp > need && sp <= g->guest_size) + guest_lazy_faultin(g, sp - need, need); + thread_entry_t *thr = current_thread; + if (thr && thr->altstack_sp != 0 && + !(thr->altstack_flags & LINUX_SS_DISABLE) && thr->altstack_size > need) + guest_lazy_faultin(g, thr->altstack_sp + thr->altstack_size - need, + need); +} + int signal_deliver(hv_vcpu_t vcpu, guest_t *g, int *exit_code) { + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); uint64_t *blocked = thread_blocked_ptr(); /* Consider this thread's private (thread-directed) set plus the shared @@ -1811,6 +1834,7 @@ int signal_deliver_fault(hv_vcpu_t vcpu, guest_t *g, int signum, int *exit_code) * threads faulting on the same signal collapse into one bit so one fault is * lost. Deliver directly here, never touching sig_state.pending. */ + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); /* Linux force_sig_info_to_task(): a forced synchronous fault cannot be diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index bac5168b..95ade75f 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -66,6 +66,7 @@ #include "syscall/time.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "debug/syscall-hist.h" @@ -174,9 +175,9 @@ typedef int64_t (*syscall_handler_t)(guest_t *g, { \ (void) g; (void) x0; (void) x1; (void) x2; \ (void) x3; (void) x4; (void) x5; (void) verbose; \ - pthread_mutex_lock(&mmap_lock); \ + mmap_lock_acquire(g); \ int64_t r = (body); \ - pthread_mutex_unlock(&mmap_lock); \ + mmap_lock_release(); \ return r; \ } @@ -484,16 +485,16 @@ static void sc_sync_regions_inline(guest_t *g) * position) cannot make us skip an entry permanently. */ for (int i = 0;; i++) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); if (i >= g->nregions) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); break; } const guest_region_t *r = &g->regions[i]; int duped = -1; if (r->shared && r->backing_fd >= 0) duped = dup(r->backing_fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (duped < 0) continue; (void) fsync(duped); @@ -524,7 +525,7 @@ static int64_t sc_sync_impl(guest_t *g) } pthread_mutex_unlock(&fd_lock); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < g->nregions && n < (int) cap; i++) { const guest_region_t *r = &g->regions[i]; if (!r->shared || r->backing_fd < 0) @@ -534,7 +535,7 @@ static int64_t sc_sync_impl(guest_t *g) continue; hosts[n++] = duped; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); /* fsync each dup outside both locks so a slow disk does not stall * concurrent FD or memory operations on other threads. @@ -708,7 +709,7 @@ static int64_t sc_mincore(guest_t *g, * never early-returns on a hole. */ uint8_t chunk[512]; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int ri = guest_region_first_end_above(g, addr); for (uint64_t done = 0; done < npages;) { uint64_t batch = npages - done; @@ -723,13 +724,19 @@ static int64_t sc_mincore(guest_t *g, if (!mapped) has_hole = true; } - if (guest_write(g, vec + done, chunk, batch) < 0) { - pthread_mutex_unlock(&mmap_lock); + /* sc_mincore holds mmap_lock while regions[] is swept. Materialize a + * valid lazy output block through the locked entry point, then use a + * no-fault copy so an invalid vec returns EFAULT instead of trying to + * acquire mmap_lock recursively. + */ + (void) guest_lazy_faultin_locked(g, vec + done, batch); + if (guest_write_nofault(g, vec + done, chunk, batch) < 0) { + mmap_lock_release(); return -LINUX_EFAULT; } done += batch; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return has_hole ? -LINUX_ENOMEM : 0; } @@ -942,6 +949,19 @@ static int64_t sc_set_tid_address(guest_t *g, return proc_get_pid(); } +static uint64_t mmap_fastpath_eligible_length(uint64_t addr, + uint64_t length, + uint64_t prot, + uint64_t flags) +{ + if (addr != 0 || prot != (LINUX_PROT_READ | LINUX_PROT_WRITE) || + (flags & ~(uint64_t) LINUX_MAP_NORESERVE) != + (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS) || + length == 0 || length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return 0; + return (length + GUEST_PAGE_SIZE - 1) & ~(GUEST_PAGE_SIZE - 1); +} + static int64_t sc_mmap(guest_t *g, uint64_t x0, uint64_t x1, @@ -951,9 +971,12 @@ static int64_t sc_mmap(guest_t *g, uint64_t x5, bool verbose) { - pthread_mutex_lock(&mmap_lock); + uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + mmap_lock_acquire(g); int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - pthread_mutex_unlock(&mmap_lock); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) (uint64_t) r); @@ -970,9 +993,9 @@ static int64_t sc_mremap(guest_t *g, bool verbose) { (void) x5; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) x2, (int) x3, @@ -2090,10 +2113,7 @@ static int64_t sc_execve(guest_t *g, (void) x3; (void) x4; (void) x5; - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); - pthread_mutex_unlock(&mmap_lock); - return r; + return sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); } static int64_t sc_execveat(guest_t *g, @@ -2109,8 +2129,9 @@ static int64_t sc_execveat(guest_t *g, hv_vcpu_t vcpu = current_thread->vcpu; int dirfd = (int) x0, flags = (int) x4; - /* Resolve the target path before taking mmap_lock (path resolution may call - * fd_to_host / openat which do not need mmap_lock). + /* Resolve the target path before entering the exec transaction. Path + * resolution may call fd_to_host / openat and does not need mmap_lock; + * sys_execve takes it at the point of no return. */ uint64_t path_gva = x1; char resolved[LINUX_PATH_MAX]; @@ -2150,7 +2171,6 @@ static int64_t sc_execveat(guest_t *g, need_resolve = true; } - pthread_mutex_lock(&mmap_lock); int64_t r; if (need_resolve) { /* Use the host-resolved path directly so execveat does not copy a host @@ -2160,7 +2180,6 @@ static int64_t sc_execveat(guest_t *g, } else { r = sys_execve(vcpu, g, path_gva, x2, x3, verbose, NULL); } - pthread_mutex_unlock(&mmap_lock); return r; } diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 25ad7280..fb659f9a 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -253,7 +253,12 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) return gva; /* propagate mmap error */ } - /* Copy host shm content into guest memory */ + /* Copy host shm content into guest memory. sys_shmat runs under + * mmap_lock (SC_LOCKED), so the resolve-time lazy fault-in inside + * guest_write would self-deadlock on it; materialize the fresh anonymous + * mapping through the locked variant first. + */ + guest_lazy_faultin_locked(g, (uint64_t) gva, seg_size); if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { shmdt(host_addr); return -LINUX_EFAULT; @@ -312,7 +317,11 @@ int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) /* Write back guest modifications to host shm (unless read-only) */ if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ + /* Read guest memory back to host shm buffer. Same SC_LOCKED + * self-deadlock hazard as the shmat copy-in: pages the guest never + * touched may still be unmaterialized. + */ + guest_lazy_faultin_locked(g, entry.guest_gva, entry.size); guest_read(g, entry.guest_gva, entry.host_addr, entry.size); } diff --git a/tests/bench-mmap-lazy.c b/tests/bench-mmap-lazy.c new file mode 100644 index 00000000..377aae70 --- /dev/null +++ b/tests/bench-mmap-lazy.c @@ -0,0 +1,111 @@ +/* + * Guest microbenchmark for anonymous private mmap latency vs size. + * + * Measures mmap(), first-touch, and munmap() latency for MAP_PRIVATE | + * MAP_ANONYMOUS mappings from 4 KiB to 32 GiB. A lazy (deferred page-table) + * implementation should show size-independent mmap/munmap cost; an eager + * implementation scales linearly with length and exhausts resources on the + * multi-GiB sizes. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void bench_size(uint64_t size, int iters) +{ + uint64_t t_map = 0, t_touch = 0, t_unmap = 0; + int ok = 0; + + for (int i = 0; i < iters; i++) { + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = now_ns(); + if (p == MAP_FAILED) { + printf("%10llu KiB: mmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + /* First touch: one write at the start and one mid-mapping. */ + volatile char *c = p; + c[0] = 1; + c[size / 2] = 1; + uint64_t t2 = now_ns(); + int rc = munmap(p, size); + uint64_t t3 = now_ns(); + if (rc != 0) { + printf("%10llu KiB: munmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + t_map += t1 - t0; + t_touch += t2 - t1; + t_unmap += t3 - t2; + ok++; + } + printf( + "%10llu KiB: mmap %10llu ns touch2 %10llu ns munmap %10llu ns " + "(%d iters)\n", + (unsigned long long) (size >> 10), + (unsigned long long) (t_map / (uint64_t) ok), + (unsigned long long) (t_touch / (uint64_t) ok), + (unsigned long long) (t_unmap / (uint64_t) ok), ok); +} + +int main(int argc, char **argv) +{ + static const struct { + uint64_t size; + int iters; + } cases[] = { + {4ull << 10, 200}, {64ull << 10, 200}, {2ull << 20, 100}, + {64ull << 20, 20}, {512ull << 20, 10}, {2ull << 30, 5}, + {8ull << 30, 3}, {16ull << 30, 3}, {32ull << 30, 3}, + {64ull << 30, 1}, {128ull << 30, 1}, {256ull << 30, 1}, + }; + /* Optional argv[1]: cap size in GiB (eager implementations commit host + * memory for every byte mapped; the full matrix would thrash small hosts). + */ + uint64_t cap = ~0ull; + if (argc > 1) + cap = (uint64_t) atoll(argv[1]) << 30; + + setvbuf(stdout, NULL, _IONBF, 0); + printf("anon private mmap latency vs size\n"); + for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (cases[i].size <= cap) + bench_size(cases[i].size, cases[i].iters); + + /* Full-touch throughput sanity: 64 MiB written end to end. */ + uint64_t size = 64ull << 20; + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("full-touch mmap failed: %s\n", strerror(errno)); + return 1; + } + memset(p, 0xa5, size); + uint64_t t1 = now_ns(); + munmap(p, size); + printf("mmap+memset 64MiB: %llu ns (%.2f GiB/s)\n", + (unsigned long long) (t1 - t0), + (double) size / 1.073741824 / (double) (t1 - t0)); + return 0; +} diff --git a/tests/manifest.txt b/tests/manifest.txt index 871557d3..4c0de5f5 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -105,6 +105,10 @@ test-fork-synthetic-fd test-guard-page test-mmap-hint +[section] Lazy anonymous mmap tests +test-mmap-lazy +test-mmap-fastpath + [section] mremap tests test-mremap test-mremap-infra diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 115ce16a..b17f45c5 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -17,14 +17,17 @@ #include "runtime/fork-state.h" #define LEGACY_ELFK_MAGIC 0x454C464BU +#define LEGACY_ELFL_MAGIC 0x454C464CU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464CU, - "fork IPC protocol magic must remain ELFL until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464DU, + "fork IPC protocol magic must remain ELFM until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != LEGACY_ELFK_MAGIC, "current protocol must reject old ELFK children/parents"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != LEGACY_ELFL_MAGIC, + "dirty-bitmap wire must reject old ELFL children/parents"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh new file mode 100755 index 00000000..d7ad0b0c --- /dev/null +++ b/tests/test-mmap-dirty-stats.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Counter-backed dirty-map materialization integration checks. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-lazy} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ + >"$TMPDIR_CASE/out" 2>"$TMPDIR_CASE/err" + +counter() +{ + key=$1 + awk -v key="$key" \ + '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/err" +} + +require_ge() +{ + key=$1 + floor=$2 + value=$(counter "$key") || { + printf 'missing dirty-map counter %s\n' "$key" >&2 + return 1 + } + if [ "$value" -lt "$floor" ]; then + printf '%s=%s, expected >= %s\n' "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_ge FAULT_CLEAN_SKIP 1 +require_ge FAULT_DIRTY_MEMSET 1 +require_ge FAULT_ALREADY_VALID 1 +require_ge FAULT_WINDOW_BYTES 2097152 + +printf ' clean-block zero skip OK\n' +printf ' dirty-block selective memset OK\n' +printf ' already-valid early return OK\n' +printf ' materialized-window bytes OK\n' +printf 'test-mmap-dirty-stats: PASS\n' diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh new file mode 100755 index 00000000..4075bc05 --- /dev/null +++ b/tests/test-mmap-fastpath-stats.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# Counter-backed refill, adaptive sizing, giant-request guard, and VA recycle +# integration checks for the EL1 anonymous-mmap consumer fast path. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-fastpath} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-mmap-stats.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +run_case() +{ + case_name=$1 + out="$TMPDIR_CASE/$case_name.out" + err="$TMPDIR_CASE/$case_name.err" + ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ + >"$out" 2>"$err" +} + +counter() +{ + case_name=$1 + key=$2 + value=$(awk -v key="$key" '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/$case_name.err") || { + printf 'missing counter %s in case %s\n' "$key" "$case_name" >&2 + return 1 + } + printf '%s\n' "$value" +} + +require_ge() +{ + case_name=$1 + key=$2 + floor=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -lt "$floor" ]; then + printf '%s: %s=%s, expected >= %s\n' \ + "$case_name" "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_eq() +{ + case_name=$1 + key=$2 + expected=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -ne "$expected" ]; then + printf '%s: %s=%s, expected %s\n' \ + "$case_name" "$key" "$value" "$expected" >&2 + return 1 + fi +} + +require_le() +{ + case_name=$1 + key=$2 + ceiling=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -gt "$ceiling" ]; then + printf '%s: %s=%s, expected <= %s\n' \ + "$case_name" "$key" "$value" "$ceiling" >&2 + return 1 + fi +} + +run_case np2-10m +require_ge np2-10m MMAP_HIT 80 +require_ge np2-10m MMAP_CAPACITY_MISS 1 +require_ge np2-10m MMAP_RING_FULL 1 +printf ' sustained 10 MiB stream OK\n' + +run_case np2-48m +require_ge np2-48m MMAP_HIT 40 +require_ge np2-48m MMAP_CAPACITY_MISS 1 +printf ' sustained 48 MiB stream OK\n' + +run_case np2-100m +require_ge np2-100m MMAP_HIT 24 +require_ge np2-100m MMAP_CAPACITY_MISS 1 +printf ' sustained 100 MiB stream OK\n' + +run_case escalation +require_ge escalation MMAP_HIT 45 +require_eq escalation MMAP_ARENA_CURRENT 1073741824 +printf ' 10 MiB -> 512 MiB escalation OK\n' + +run_case giant-guard +require_ge giant-guard MMAP_HIT 34 +require_le giant-guard MMAP_ARENA_PEAK 536870912 +printf ' >1 GiB giant request guard OK\n' + +run_case adaptive-small +require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-small MMAP_ARENA_PEAK 67108864 +printf ' small-stream arena floor OK\n' + +run_case adaptive-decay +require_eq adaptive-decay MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-decay MMAP_ARENA_PEAK 1073741824 +printf ' one-generation arena decay OK\n' + +run_case recycle +require_ge recycle MMAP_RECYCLE 1 +require_le recycle MMAP_HIGH_WATER 201326592 +printf ' arena VA recycling OK\n' + +printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c new file mode 100644 index 00000000..4a9aa21e --- /dev/null +++ b/tests/test-mmap-fastpath.c @@ -0,0 +1,278 @@ +/* + * EL1 consumer-mmap fast-path integration tests. + * + * Run through the dedicated make target without an ELFUSE_MMAP_FASTPATH + * override so the default-enabled configuration is exercised. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static int maps_extent_for(uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + int fd = open("/proc/self/maps", O_RDONLY); + if (fd < 0) + return -1; + char buf[16384]; + ssize_t n = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + + char *line = buf; + while (*line) { + unsigned long long lo, hi; + if (sscanf(line, "%llx-%llx", &lo, &hi) == 2 && needle >= lo && + needle < hi) { + *lo_out = (uintptr_t) lo; + *hi_out = (uintptr_t) hi; + return 0; + } + char *nl = strchr(line, '\n'); + if (!nl) + break; + line = nl + 1; + } + return -1; +} + +static void test_fidelity(void) +{ + TEST("unconsumed arena is absent and faults"); + struct sigaction sa = {.sa_handler = segv_handler}; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, NULL) != 0) { + FAIL("sigaction"); + return; + } + + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[0] = 0x5a; /* drains the publication through the fault-side lock */ + + volatile uint8_t *unconsumed = p + 4096; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *unconsumed; + FAIL("wild read into unconsumed arena did not SIGSEGV"); + munmap(p, 4096); + return; + } + + uintptr_t lo = 0, hi = 0; + if (maps_extent_for((uintptr_t) p, &lo, &hi) < 0 || lo != (uintptr_t) p || + hi != (uintptr_t) p + 4096) { + FAIL("/proc/self/maps exposed more than the consumed page"); + munmap(p, 4096); + return; + } + munmap(p, 4096); + PASS(); +} + +static void test_exhaustion_fallback(void) +{ + TEST("arena exhaustion falls back to host mmap"); + const size_t len = 80ULL << 20; /* larger than the first 64MiB arena */ + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("80MiB mmap"); + return; + } + if (p[0] != 0 || p[len - 1] != 0) { + FAIL("fallback mapping was not zero-filled"); + munmap(p, len); + return; + } + p[0] = 1; + p[len - 1] = 2; + if (munmap(p, len) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +typedef struct { + int iterations; + _Atomic int *failed; +} storm_arg_t; + +static void *storm_worker(void *opaque) +{ + storm_arg_t *arg = opaque; + for (int i = 0; i < arg->iterations; i++) { + size_t len = (size_t) ((i & 7) + 1) * 4096; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + p[0] = (uint8_t) i; + p[len - 1] = (uint8_t) (i ^ 0x5a); + if (munmap(p, len) != 0) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + } + return NULL; +} + +static void test_mt_storm_and_fork_exec(void) +{ + TEST("multi-vCPU mmap storm with fork+exec revocation"); + enum { NTHREADS = 8 }; + pthread_t threads[NTHREADS]; + _Atomic int failed = 0; + storm_arg_t arg = {.iterations = 400, .failed = &failed}; + + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&threads[made], NULL, storm_worker, &arg) != 0) { + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + break; + } + } + + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = {(char *) "/proc/self/exe", NULL}; + char *const envp[] = {(char *) "ELFUSE_FASTPATH_EXEC_CHILD=1", NULL}; + execve(argv[0], argv, envp); + _exit(111); + } + if (pid < 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + + for (int i = 0; i < made; i++) + pthread_join(threads[i], NULL); + + if (pid > 0) { + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + } + + /* The parent's arenas were revoked for the fork snapshot. This pair makes + * the first call take the generation fallback and verifies service resumes. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + else { + p[0] = 7; + munmap(p, 4096); + } + + if (__atomic_load_n(&failed, __ATOMIC_RELAXED)) { + FAIL("storm/fork/exec worker failure"); + return; + } + PASS(); +} + +static int stats_stream(size_t len, int iterations, bool release_each) +{ + for (int i = 0; i < iterations; i++) { + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (release_each && munmap(p, len) != 0) + return 1; + } + return 0; +} + +static int run_stats_case(const char *name) +{ + if (strcmp(name, "np2-10m") == 0) + return stats_stream(10ULL << 20, 96, false); + if (strcmp(name, "np2-48m") == 0) + return stats_stream(48ULL << 20, 48, false); + if (strcmp(name, "np2-100m") == 0) + return stats_stream(100ULL << 20, 30, false); + if (strcmp(name, "escalation") == 0) { + if (stats_stream(10ULL << 20, 48, false) != 0) + return 1; + return stats_stream(512ULL << 20, 3, false); + } + if (strcmp(name, "giant-guard") == 0) { + if (stats_stream(10ULL << 20, 32, false) != 0) + return 1; + for (int i = 0; i < 6; i++) { + if (stats_stream(2ULL << 30, 1, false) != 0 || + stats_stream(10ULL << 20, 1, false) != 0) + return 1; + } + return 0; + } + if (strcmp(name, "adaptive-small") == 0) + return stats_stream(64ULL << 10, 1100, false); + if (strcmp(name, "adaptive-decay") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0 || + stats_stream(500ULL << 20, 1, false) != 0) + return 1; + /* Ring-full fallbacks do not consume the arena cursor, so exceed the + * nominal 16384 pages enough to force a true 1GiB capacity rollover. + */ + return stats_stream(64ULL << 10, 18000, false); + } + if (strcmp(name, "recycle") == 0) + return stats_stream(64ULL << 10, 6000, true); + return 2; +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strncmp(argv[1], "--stats-", 8) == 0) + return run_stats_case(argv[1] + 8); + + if (getenv("ELFUSE_FASTPATH_EXEC_CHILD")) { + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 0xa5; + return p[0] == 0xa5 ? 0 : 1; + } + + test_fidelity(); + test_exhaustion_fallback(); + test_mt_storm_and_fork_exec(); + + printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, + fails ? "FAIL" : "PASS"); + return fails ? 1 : 0; +} diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c new file mode 100644 index 00000000..b483cd26 --- /dev/null +++ b/tests/test-mmap-lazy.c @@ -0,0 +1,738 @@ +/* + * Lazy anonymous mmap regression tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Private anonymous mappings defer page-table creation and zeroing to first + * touch. These tests pin down the guest-visible contract of that laziness: + * huge reservations succeed and read as zeros, address reuse never leaks + * stale bytes, host-side syscall access (read/write/futex) works on memory + * the guest never touched, PROT_NONE stays a faulting reservation, data + * survives PROT_NONE round trips and fork, and concurrent first touch from + * multiple threads never loses a write to the deferred zeroing. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#define BLOCK_2MIB (2ULL << 20) + +#ifndef FUTEX_WAIT +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#endif + +/* Largest plain anonymous RW mapping the kernel grants. On elfuse the lazy + * path must take this well past physical memory; on real Linux the result + * depends on the overcommit heuristic, so the tests only require >= 1 GiB + * and probe downward. + */ +static void *map_largest(size_t *out_size) +{ + static const size_t sizes[] = { + 64ULL << 30, + 16ULL << 30, + 4ULL << 30, + 1ULL << 30, + }; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + void *p = mmap(NULL, sizes[i], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + *out_size = sizes[i]; + return p; + } + } + return NULL; +} + +static void test_huge_sparse(void) +{ + TEST("huge mmap + sparse touch"); + size_t size = 0; + volatile uint8_t *p = map_largest(&size); + if (!p || size < (1ULL << 30)) { + FAIL("no >=1GiB anonymous mapping granted"); + return; + } + /* Sparse probes: start, one per size/8 stride, last page. All must read + * zero and accept writes. + */ + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != 0) { + FAIL("fresh mapping reads nonzero"); + munmap((void *) p, size); + return; + } + p[off] = (uint8_t) (i + 1); + } + if (p[size - 1] != 0) { + FAIL("last page reads nonzero"); + munmap((void *) p, size); + return; + } + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != (uint8_t) (i + 1)) { + FAIL("sparse write lost"); + munmap((void *) p, size); + return; + } + } + if (munmap((void *) p, size) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +static void test_zero_reuse(void) +{ + TEST("address reuse reads zero"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xa5, size); + munmap(p, size); + uint8_t *q = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + /* The allocator typically reuses the freed range; either way no byte may + * be nonzero. Check one page per 2MiB block plus both ends. + */ + for (size_t off = 0; off < size; off += 4096) { + if (q[off] != 0) { + FAIL("stale data after reuse"); + munmap(q, size); + return; + } + } + munmap(q, size); + PASS(); +} + +static void test_partial_block_reuse(void) +{ + TEST("partial-block reuse preserves neighbor"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[17] = 0xa5; + p[half + 17] = 0x5a; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED || q != p) { + FAIL("freed half was not reused at hint"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + if (q[17] != 0 || q[half - 1] != 0 || p[half + 17] != 0x5a) { + FAIL("partial zero clobbered neighbor or leaked stale data"); + munmap(q, half); + munmap(p + half, half); + return; + } + munmap(q, half); + munmap(p + half, half); + PASS(); +} + +static void test_fork_clean_reuse(void) +{ + TEST("fork child sees zero on clean-block reuse"); + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xcc, BLOCK_2MIB); + if (munmap(p, BLOCK_2MIB) != 0) { + FAIL("munmap"); + return; + } + uint8_t *q = mmap(p, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + pid_t pid = fork(); + if (pid == 0) { + if (q[0] != 0 || q[BLOCK_2MIB - 1] != 0) + _exit(1); + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + int st = 0; + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) { + FAIL("fork clean-block state"); + munmap(q, BLOCK_2MIB); + return; + } + munmap(q, BLOCK_2MIB); + PASS(); +} + +static void test_file_overlay_reuse(void) +{ + TEST("file overlay teardown then lazy reuse"); + char path[] = "/tmp/elfuse-dirty-map.XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, BLOCK_2MIB) != 0) { + FAIL("ftruncate"); + close(fd); + return; + } + uint8_t first = 0xa7, last = 0x5c; + if (pwrite(fd, &first, 1, 17) != 1 || + pwrite(fd, &last, 1, BLOCK_2MIB - 1) != 1) { + FAIL("pwrite"); + close(fd); + return; + } + + uint8_t *reserve = mmap(NULL, 2 * BLOCK_2MIB, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserve == MAP_FAILED) { + FAIL("reserve"); + close(fd); + return; + } + uintptr_t aligned = + ((uintptr_t) reserve + BLOCK_2MIB - 1) & ~(BLOCK_2MIB - 1); + munmap(reserve, 2 * BLOCK_2MIB); + uint8_t *target = (uint8_t *) aligned; + + uint8_t *file = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (file != target || file[17] != first || file[BLOCK_2MIB - 1] != last) { + FAIL("file mmap"); + if (file != MAP_FAILED) + munmap(file, BLOCK_2MIB); + close(fd); + return; + } + file[BLOCK_2MIB / 2] = 0xe1; + if (munmap(file, BLOCK_2MIB) != 0) { + FAIL("file munmap"); + close(fd); + return; + } + close(fd); + + uint8_t *anon = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon != target || anon[17] != 0 || anon[BLOCK_2MIB / 2] != 0 || + anon[BLOCK_2MIB - 1] != 0) { + FAIL("stale file bytes after lazy reuse"); + if (anon != MAP_FAILED) + munmap(anon, BLOCK_2MIB); + return; + } + munmap(anon, BLOCK_2MIB); + PASS(); +} + +static void test_read_into_lazy(void) +{ + TEST("read() into untouched mapping"); + size_t size = 6ULL << 20; + uint8_t *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (buf == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + static const char msg[] = "lazy-host-access-payload"; + /* Unaligned target crossing into the mapping's third 2MiB block. */ + size_t off = (4ULL << 20) + 123; + if (write(fds[1], msg, sizeof(msg)) != (ssize_t) sizeof(msg) || + read(fds[0], buf + off, sizeof(msg)) != (ssize_t) sizeof(msg)) { + FAIL("pipe copy through untouched buffer"); + goto out; + } + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload corrupted"); + goto out; + } + /* A guest touch elsewhere in the same 2MiB block must not re-zero the + * host-written payload (deferred-zeroing idempotence). + */ + buf[(4ULL << 20) + 64 * 1024] = 7; + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload clobbered by later fault in same block"); + goto out; + } + /* Untouched parts of the mapping still read zero. */ + for (size_t i = 0; i < 4096; i++) { + if (buf[i] != 0) { + FAIL("nonzero byte in untouched block"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(buf, size); +} + +static void test_write_from_lazy(void) +{ + TEST("write() from untouched mapping"); + size_t size = 2ULL << 20; + uint8_t *src = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (src == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + uint8_t back[512]; + memset(back, 0xff, sizeof(back)); + if (write(fds[1], src + 4096, sizeof(back)) != (ssize_t) sizeof(back) || + read(fds[0], back, sizeof(back)) != (ssize_t) sizeof(back)) { + FAIL("pipe copy from untouched buffer"); + goto out; + } + for (size_t i = 0; i < sizeof(back); i++) { + if (back[i] != 0) { + FAIL("untouched buffer sent nonzero bytes"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(src, size); +} + +static void test_prot_none_roundtrip(void) +{ + TEST("mprotect NONE round trip keeps data"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x5c, 8192); + p[size - 1] = 0x77; + if (mprotect(p, size, PROT_NONE) != 0 || + mprotect(p, size, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect"); + munmap(p, size); + return; + } + if (p[0] != 0x5c || p[8191] != 0x5c || p[size - 1] != 0x77 || + p[16384] != 0) { + FAIL("data lost or stale bytes after round trip"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_reserve_commit(void) +{ + TEST("PROT_NONE reserve + mprotect commit"); + size_t size = 1ULL << 30; + uint8_t *p = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + FAIL("reserve"); + return; + } + uint8_t *slab = p + (512ULL << 20); + if (mprotect(slab, 8ULL << 20, PROT_READ | PROT_WRITE) != 0) { + FAIL("commit"); + munmap(p, size); + return; + } + for (size_t off = 0; off < (8ULL << 20); off += 4096) { + if (slab[off] != 0) { + FAIL("committed slab reads nonzero"); + munmap(p, size); + return; + } + } + slab[0] = 1; + slab[(8ULL << 20) - 1] = 2; + if (slab[0] != 1 || slab[(8ULL << 20) - 1] != 2) { + FAIL("committed slab write lost"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static void test_prot_none_faults(void) +{ + TEST("PROT_NONE|NORESERVE still faults"); + size_t size = 16ULL << 20; + volatile uint8_t *p = + mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + struct sigaction sa = {0}, old_sa; + sa.sa_handler = segv_handler; + sigaction(SIGSEGV, &sa, &old_sa); + int faulted = 0; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) p[BLOCK_2MIB + 5]; + } else { + faulted = 1; + } + sigaction(SIGSEGV, &old_sa, NULL); + munmap((void *) p, size); + /* A lazy materializer that ignores prot would silently hand the guest a + * readable zero page here instead of SIGSEGV. + */ + EXPECT_TRUE(faulted, "read from PROT_NONE reservation did not fault"); +} + +static void test_fork_lazy(void) +{ + TEST("fork with partially touched mapping"); + size_t size = 8ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x42, 4096); /* touch only block 0 */ + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, size); + return; + } + if (pid == 0) { + /* Child: inherited data intact, untouched block reads zero and is + * privately writable. + */ + if (p[0] != 0x42 || p[4095] != 0x42) + _exit(1); + if (p[4ULL << 20] != 0) + _exit(2); + p[4ULL << 20] = 0x99; + if (p[(4ULL << 20) + 1] != 0) + _exit(3); + _exit(0); + } + int st = 0; + if (waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || WEXITSTATUS(st) != 0) { + FAIL("child saw wrong memory"); + munmap(p, size); + return; + } + /* Parent: child's private write must not leak back. */ + if (p[4ULL << 20] != 0) { + FAIL("child write leaked into parent"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_futex_untouched(void) +{ + TEST("futex on untouched mapping"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + uint32_t *word = (uint32_t *) (p + (2ULL << 20) + 256); + /* WAKE on never-touched memory: no waiters, must not fault. */ + long r = syscall(SYS_futex, word, FUTEX_WAKE, 1, NULL, NULL, 0); + if (r != 0) { + FAIL("FUTEX_WAKE on untouched word"); + munmap(p, size); + return; + } + /* WAIT with expected=1: the word reads as zero, so EAGAIN. */ + r = syscall(SYS_futex, word, FUTEX_WAIT, 1, NULL, NULL, 0); + if (!(r == -1 && errno == EAGAIN)) { + FAIL("FUTEX_WAIT did not read zero from untouched word"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +/* Concurrent first touch: every thread writes its own slot in the same fresh + * 2MiB block, racing the deferred zeroing. A materializer that re-zeros an + * already-populated block loses some slots. + */ +#define MT_THREADS 4 +#define MT_ITERS 64 + +typedef struct { + uint8_t *base; + int idx; + pthread_barrier_t *barrier; +} mt_arg_t; + +static void *mt_touch(void *argp) +{ + mt_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[a->idx * 64] = (uint8_t) (a->idx + 1); + /* Also touch a private block so several materializations race. */ + a->base[BLOCK_2MIB * (unsigned) (a->idx + 1) + 17] = + (uint8_t) (0x10 + a->idx); + return NULL; +} + +static void test_mt_first_touch(void) +{ + TEST("concurrent first touch"); + for (int iter = 0; iter < MT_ITERS; iter++) { + size_t size = BLOCK_2MIB * (MT_THREADS + 2); + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS); + pthread_t th[MT_THREADS]; + mt_arg_t args[MT_THREADS]; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (mt_arg_t) {p, i, &barrier}; + if (pthread_create(&th[i], NULL, mt_touch, &args[i]) != 0) { + FAIL("pthread_create"); + return; + } + } + for (int i = 0; i < MT_THREADS; i++) + pthread_join(th[i], NULL); + pthread_barrier_destroy(&barrier); + for (int i = 0; i < MT_THREADS; i++) { + if (p[i * 64] != (uint8_t) (i + 1) || + p[BLOCK_2MIB * (unsigned) (i + 1) + 17] != + (uint8_t) (0x10 + i)) { + FAIL("write lost to concurrent materialization"); + munmap(p, size); + return; + } + } + munmap(p, size); + } + PASS(); +} + +typedef struct { + uint8_t *base; + size_t half; + int idx; + pthread_barrier_t *barrier; + int *error; +} claim_race_arg_t; + +static void *claim_race_touch(void *argp) +{ + claim_race_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[64 * (unsigned) a->idx] = (uint8_t) (a->idx + 1); + return NULL; +} + +static void *claim_race_mutate(void *argp) +{ + claim_race_arg_t *a = argp; + uint8_t *neighbor = a->base + a->half; + uint8_t *hole = neighbor + 4096; + pthread_barrier_wait(a->barrier); + for (int i = 0; i < 16; i++) { + if (mprotect(neighbor, a->half, PROT_NONE) != 0 || + mprotect(neighbor, a->half, PROT_READ) != 0 || + munmap(hole, 4096) != 0) { + *a->error = 1; + return NULL; + } + void *r = mmap(hole, 4096, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (r != hole) { + *a->error = 1; + return NULL; + } + } + return NULL; +} + +static void test_claim_mutation_race(void) +{ + TEST("first-touch claim vs adjacent mutations"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[half + 17] = 0x6d; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + FAIL("reuse"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS + 1); + pthread_t workers[MT_THREADS], mutator; + claim_race_arg_t args[MT_THREADS + 1]; + int error = 0; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (claim_race_arg_t) {q, half, i, &barrier, &error}; + pthread_create(&workers[i], NULL, claim_race_touch, &args[i]); + } + args[MT_THREADS] = (claim_race_arg_t) {q, half, 0, &barrier, &error}; + pthread_create(&mutator, NULL, claim_race_mutate, &args[MT_THREADS]); + for (int i = 0; i < MT_THREADS; i++) + pthread_join(workers[i], NULL); + pthread_join(mutator, NULL); + pthread_barrier_destroy(&barrier); + + for (int i = 0; i < MT_THREADS; i++) { + if (q[64 * (unsigned) i] != (uint8_t) (i + 1)) + error = 1; + } + if (p[half + 17] != 0x6d) + error = 1; + munmap(q, half); + munmap(p + half, half); + if (error) { + FAIL("claim/mutation race corrupted data"); + return; + } + PASS(); +} + +static void test_adjacent_region_extension(void) +{ + TEST("adjacent fast-mmap region extension"); + enum { N_PAGES = 64 }; + uint8_t *pages[N_PAGES]; + int allocated = 0; + bool ok = true; + + for (int i = 0; i < N_PAGES; i++) { + pages[i] = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages[i] == MAP_FAILED) { + ok = false; + break; + } + allocated++; + pages[i][0] = (uint8_t) (i + 1); + } + for (int i = 0; ok && i < allocated; i++) { + if (pages[i][0] != (uint8_t) (i + 1)) + ok = false; + } + for (int i = 0; i < allocated; i++) + munmap(pages[i], 4096); + + if (!ok) { + FAIL("adjacent lazy mappings did not materialize independently"); + return; + } + PASS(); +} + +int main(void) +{ + test_huge_sparse(); + test_zero_reuse(); + test_partial_block_reuse(); + test_fork_clean_reuse(); + test_file_overlay_reuse(); + test_read_into_lazy(); + test_write_from_lazy(); + test_prot_none_roundtrip(); + test_reserve_commit(); + test_prot_none_faults(); + test_fork_lazy(); + test_futex_untouched(); + test_mt_first_touch(); + test_claim_mutation_race(); + test_adjacent_region_extension(); + + SUMMARY("test-mmap-lazy"); + return fails ? 1 : 0; +} diff --git a/tests/test-tlbi-encoder-host.c b/tests/test-tlbi-encoder-host.c index 4757a2f0..accc26b7 100644 --- a/tests/test-tlbi-encoder-host.c +++ b/tests/test-tlbi-encoder-host.c @@ -47,12 +47,12 @@ static void check_field(const char *label, uint64_t got, uint64_t expect) /* Decompose the operand per ARM ARM D8.7.6 and compare each field against the * expected value. baseADDR is VA>>12 masked to 37 bits; TG must be 01 (4 KiB); - * SCALE must be 0; TTL must be 0; ASID must be 0. NUM derives from the page - * count via the ceil(pages/2) - 1 SCALE=0 encoding. + * TTL and ASID must be 0. NUM derives from the selected SCALE unit. */ static void verify_operand(uint64_t start_va, uint16_t pages, - uint64_t expect_num) + uint64_t expect_num, + uint64_t expect_scale) { uint64_t op = tlbi_rvae1is_operand(start_va, pages); @@ -76,7 +76,7 @@ static void verify_operand(uint64_t start_va, check_field(label, num, expect_num); snprintf(label, sizeof(label), "SCALE (pages=%u)", (unsigned) pages); - check_field(label, scale, 0); + check_field(label, scale, expect_scale); snprintf(label, sizeof(label), "TG (start=0x%llx)", (unsigned long long) start_va); @@ -100,30 +100,47 @@ int main(void) * pages 63 -> NUM 31 (covers 64) * pages 64 -> NUM 31 (covers 64) */ - verify_operand(0x10000000ULL, 2, 0); - verify_operand(0x10000000ULL, 3, 1); - verify_operand(0x10000000ULL, 16, 7); - verify_operand(0x10000000ULL, 17, 8); - verify_operand(0x10000000ULL, 32, 15); - verify_operand(0x10000000ULL, 63, 31); - verify_operand(0x10000000ULL, 64, 31); + verify_operand(0x10000000ULL, 2, 0, 0); + verify_operand(0x10000000ULL, 3, 1, 0); + verify_operand(0x10000000ULL, 16, 7, 0); + verify_operand(0x10000000ULL, 17, 8, 0); + verify_operand(0x10000000ULL, 32, 15, 0); + verify_operand(0x10000000ULL, 63, 31, 0); + verify_operand(0x10000000ULL, 64, 31, 0); + + /* SCALE=1 covers 64 pages per NUM step; SCALE=2 covers 2048. A lazy + * 2 MiB block is 512 pages and must therefore encode as SCALE=1, NUM=7. + */ + verify_operand(0x10000000ULL, 512, 7, 1); + verify_operand(0x10000000ULL, 2048, 31, 1); + verify_operand(0x10000000ULL, 8192, 3, 2); /* Boundary VAs. 4 KiB-aligned, low-VA, MMAP_BASE (8 GiB), high-VA just * below the 48-bit BaseADDR truncation point. */ - verify_operand(0x00000000ULL, 32, 15); /* zero base */ - verify_operand(0x200000000ULL, 32, 15); /* MMAP_BASE */ - verify_operand(0x800000000000ULL, 32, 15); /* Rosetta image */ - verify_operand(0x0000FFFFF0000000ULL, 32, 15); /* KBUF_USER_VA */ + verify_operand(0x00000000ULL, 32, 15, 0); /* zero base */ + verify_operand(0x200000000ULL, 32, 15, 0); /* MMAP_BASE */ + verify_operand(0x800000000000ULL, 32, 15, 0); /* Rosetta image */ + verify_operand(0x0000FFFFF0000000ULL, 32, 15, 0); /* KBUF_USER_VA */ /* Pathological inputs the clamp must catch: * pages = 0 -> clamped to 2 -> NUM 0 * pages = 1 -> clamped to 2 -> NUM 0 (callers never reach here) * pages = UINT16_MAX -> NUM clamped to 31 (saturating) */ - verify_operand(0x10000000ULL, 0, 0); - verify_operand(0x10000000ULL, 1, 0); - verify_operand(0x10000000ULL, UINT16_MAX, 31); + verify_operand(0x10000000ULL, 0, 0, 0); + verify_operand(0x10000000ULL, 1, 0, 0); + verify_operand(0x10000000ULL, UINT16_MAX, 31, 2); + + /* The accumulator widens a 2 MiB request to the SCALE=1 granule and keeps + * it on the single-shot RVAE path instead of degrading to broadcast. + */ + g_tlbi_range_supported = true; + tlbi_request_clear(); + tlbi_request_range(0x200000000ULL, 0x200200000ULL); + check_field("2MiB accumulator kind", cpu_tlbi_req.kind, TLBI_RANGE_LARGE); + check_field("2MiB accumulator pages", cpu_tlbi_req.pages, 512); + check_field("2MiB accumulator start", cpu_tlbi_req.start, 0x200000000ULL); /* TG bit is the architectural lynchpin -- if the encoder ever drops it the * integration tests on Apple Silicon would still pass. Pin a direct bit-46 From d217c963e1033096207aba37b02be1a67540f877 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 15 Jul 2026 15:57:36 +0800 Subject: [PATCH 2/9] Preserve neighbor PTEs on cross-block extend mremap_extend_range() applied guest_update_perms() over the whole 2 MiB- aligned extension, zeroing the lazy PTEs of unrelated mappings that shared an edge block. Snapshot per-block presence before extending, then split and selectively invalidate only the non-kept subranges of freshly created blocks; pre-existing blocks keep their PTEs. A single idempotent fail label rolls back on error, restoring the page-table high-water mark and re-invalidating the touched range. sys_mremap()'s ENOMEM path now restores the destination snapshot page tables so a failed grow leaves no half-mapped range. Fold the five open-coded RX-vs-RW arena tests and four copy-pasted high-water bumps into mmap_pt_end_for_off() and mmap_bump_next(). guest_va_block_mapped() now reports L3 table descriptors as mapped, not only L2 block descriptors, so the presence check sees split blocks. Add tests/bench-mmap.c, a fair mmap microbenchmark: make build/bench-mmap build/elfuse build/bench-mmap Timing reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN), so a sample costs an isb+mrs rather than a clock_gettime SVC that would swamp sub-microsecond operations. The 24 MHz counter (~41.7 ns/tick) is amortized over an adaptive batch for sub-tick effective resolution. Every case takes one untimed warmup and reports the median (and min) over repeated runs. Sections: A. mmap/munmap latency vs size, 4 KiB to 32 GiB, steady state. B. fresh bump-tail mmap, isolating the deferred-PTE path: a bounded run allocated without freeing so every mapping lands above the arena high-water; every result failure-checked. C. first-touch fault cost, one byte per page at 16 KiB stride so no two touches share a macOS host page (every touch is a real fault). D. mprotect split, 2 MiB block into 512 L3 pages. E. mremap grow, in-place vs forced move. F. multi-threaded fresh mmap under mmap_lock (2 and 4 threads); the worker times with clock_gettime because CNTVCT_EL0 needs per-vCPU EL0VCTEN. A/B two builds of the same section: build elfuse with and without the change, run the identical benchmark binary under each, and read the delta on the affected section. Overlapping median ranges across repeated runs mean no measurable effect; a consistent gap that scales with size means a real one. This is how the once-present fresh-range invalidate skip was shown to be a no-op (the invalidate is already free on an unmapped range) and dropped. --- Makefile | 5 + src/core/guest.h | 4 +- src/syscall/mem.c | 204 ++++++++++---- tests/bench-mmap.c | 455 ++++++++++++++++++++++++++++++ tests/test-mmap-dirty-stats.sh | 2 +- tests/test-mmap-fastpath-stats.sh | 2 +- tests/test-mmap-lazy.c | 33 +++ tests/test-mremap.c | 102 +++++++ 8 files changed, 741 insertions(+), 66 deletions(-) create mode 100644 tests/bench-mmap.c diff --git a/Makefile b/Makefile index f38de266..abd57059 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,11 @@ $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread +# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread. +$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + # test-mmap-lazy races concurrent first touch from several threads. $(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/src/core/guest.h b/src/core/guest.h index ee30725c..d9b3f481 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -1035,8 +1035,8 @@ int guest_install_va_pages(guest_t *g, uint64_t gpa, int perms); -/* Query whether a 2 MiB TTBR0 VA block already has a leaf mapping. - * Returns true only for a present L2 block descriptor. +/* Query whether a 2 MiB TTBR0 VA block already has any L2 entry, either a + * block descriptor or an L3 table descriptor. */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 3c791c2f..9365ec07 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -128,9 +128,9 @@ void mmap_fastpath_drain_locked(guest_t *g) LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, NULL, -1) < 0) { - /* EL1 already returned this address to the guest. Continuing - * without semantic metadata would turn first touch into a - * false SIGSEGV, so fail closed on the violated provisioning + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a false + * SIGSEGV, so fail closed on the violated provisioning * invariant instead of silently corrupting process state. */ log_fatal( @@ -250,9 +250,9 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, mmap_fastpath_request_fits(cursor, limit, request_len)) return; - /* The owner is parked in HVC. Make the stranded tail immediately - * recyclable before the gap scan; mappings already served from the prefix - * were drained into regions[] on mmap_lock acquisition. + /* The owner is parked in HVC. Make the stranded tail immediately recyclable + * before the gap scan; mappings already served from the prefix were drained + * into regions[] on mmap_lock acquisition. */ if (flags & SHIM_MMAP_CTRL_ENABLED) atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); @@ -287,12 +287,15 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, } uint64_t new_limit = base + arena_size; - /* Carve VA only. Clearing stale descriptors once here makes every later - * bump allocation PTE-free without putting page-table work in EL1. + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. Fresh + * bump-tail arenas beyond mmap_end cannot contain stale descriptors. */ - if (guest_invalidate_ptes(g, base, new_limit) < 0) { - mmap_fastpath_disable_control(c); - return; + if (recycled || base < g->mmap_end) { + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(c); + return; + } } if (!recycled) { g->mmap_next = new_limit; @@ -317,7 +320,7 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->flags, SHIM_MMAP_CTRL_ENABLED, memory_order_relaxed); /* This vCPU is stopped in HVC (or has never run), so host may acknowledge - * the freshly published descriptor on its behalf. Revocation deliberately + * the freshly published descriptor on its behalf. Revocation deliberately * does not do this, making an in-flight stale generation bail once. */ atomic_store_explicit(&c->consumer_generation, generation, @@ -1349,11 +1352,39 @@ static int64_t sys_mmap_high_va(guest_t *g, return ret; } +/* Page-table high-water slot (mmap_rx_end / mmap_end) for a TTBR0 mmap offset, + * or NULL if @off is not in either mmap arena. + */ +static uint64_t *mmap_pt_end_for_off(guest_t *g, uint64_t off) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) + return &g->mmap_rx_end; + if (off >= MMAP_BASE) + return &g->mmap_end; + return NULL; +} + +/* Advance the allocation high-water mark (mmap_rx_next / mmap_next) that fork + * IPC state transfer replays, for whichever arena @off belongs to. + */ +static void mmap_bump_next(guest_t *g, uint64_t off, uint64_t end) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) { + if (end > g->mmap_rx_next) + g->mmap_rx_next = end; + } else if (off >= MMAP_BASE) { + if (end > g->mmap_next) + g->mmap_next = end; + } +} + static int mremap_extend_range(guest_t *g, uint64_t off, uint64_t size, int prot) { + uint64_t *pt_end = mmap_pt_end_for_off(g, off); + if (prot == LINUX_PROT_NONE) { guest_invalidate_ptes(g, off, off + size); return 0; @@ -1364,10 +1395,67 @@ static int mremap_extend_range(guest_t *g, uint64_t ext_end = ALIGN_UP(off + size, BLOCK_2MIB); if (ext_end > g->guest_size) ext_end = g->guest_size; - if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) + size_t nblocks = pt_end ? (size_t) ((ext_end - ext_start) / BLOCK_2MIB) : 0; + bool *block_preexisting = NULL; + if (nblocks) { + block_preexisting = calloc(nblocks, sizeof(*block_preexisting)); + if (!block_preexisting) + return -1; + for (size_t i = 0; i < nblocks; i++) + block_preexisting[i] = + guest_va_block_mapped(g, ext_start + (uint64_t) i * BLOCK_2MIB); + } + if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) { + free(block_preexisting); return -1; - guest_update_perms(g, off, off + size, page_perms); + } + uint64_t saved_pt_end = pt_end ? *pt_end : 0; + if (pt_end && ext_end > *pt_end) + *pt_end = ext_end; + + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + uint64_t keep_start = off > b ? off : b; + uint64_t keep_end = off + size < bend ? off + size : bend; + if (keep_start <= b && keep_end >= bend) + continue; + if (guest_split_block(g, b) < 0) + goto fail; + if (b < keep_start && guest_invalidate_ptes(g, b, keep_start) < 0) + goto fail; + if (keep_end < bend && guest_invalidate_ptes(g, keep_end, bend) < 0) + goto fail; + } + + if (guest_update_perms(g, off, off + size, page_perms) < 0) + goto fail; + free(block_preexisting); return 0; + + /* Roll back: re-invalidate every fresh block whole (idempotent -- the + * forward pass already cleared the non-kept subranges) plus the kept [off, + * off+size) span, returning the range to its pre-extend state. + */ +fail: + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + (void) guest_invalidate_ptes(g, b, bend); + } + (void) guest_invalidate_ptes(g, off, off + size); + if (pt_end) + *pt_end = saved_pt_end; + free(block_preexisting); + return -1; } static int hvf_apply_file_overlay(guest_t *g, @@ -1691,12 +1779,16 @@ static int rollback_fresh_mmap_allocation(guest_t *g, { if (overlay_installed) hvf_remove_file_overlay(g, overlay_ipa, overlay_len); - if (guest_invalidate_ptes(g, start, start + length) < 0) + uint64_t end = start + length; + uint64_t cur_mmap_end = g->mmap_end; + uint64_t cur_mmap_rx_end = g->mmap_rx_end; + if (guest_invalidate_ptes(g, start, end) < 0) return -LINUX_ENOMEM; g->mmap_next = saved_mmap_next; - g->mmap_end = saved_mmap_end; + g->mmap_end = cur_mmap_end > saved_mmap_end ? cur_mmap_end : saved_mmap_end; g->mmap_rx_next = saved_mmap_rx_next; - g->mmap_rx_end = saved_mmap_rx_end; + g->mmap_rx_end = cur_mmap_rx_end > saved_mmap_rx_end ? cur_mmap_rx_end + : saved_mmap_rx_end; g->mmap_rw_gap_hint = saved_rw_gap_hint; g->mmap_rx_gap_hint = saved_rx_gap_hint; return 0; @@ -2065,8 +2157,8 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, hvf_remap_segments_best_effort(g, segments, nsegments); return err; } - /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. - * The following munmap/MAP_FIXED path will clear the bit only after it has + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. The + * following munmap/MAP_FIXED path will clear the bit only after it has * actually zeroed a complete 2 MiB block. */ if (len <= UINT64_MAX - ipa) @@ -2256,14 +2348,14 @@ int64_t sys_mmap(guest_t *g, bool is_noreserve = is_anon && (flags & LINUX_MAP_NORESERVE) != 0; /* Anonymous mappings defer page-table creation and zeroing to first touch * (guest fault or host-side access), like MAP_NORESERVE always has. This - * keeps mmap()/munmap() cost independent of length: a multi-GiB - * reservation costs neither an eager PTE walk nor a full-length memset, - * and never-touched blocks consume no page-table pool. PROT_NONE stays a - * pure reservation (faults deliver SIGSEGV, not materialization), and - * MAP_FIXED keeps the eager path because it must atomically replace live - * mappings. Shared anonymous memory stays eager unless the caller opted - * into MAP_NORESERVE (the historical lazy set), since deferred zeroing - * has never been exercised against the fork snapshot paths for it. + * keeps mmap()/munmap() cost independent of length: a multi-GiB reservation + * costs neither an eager PTE walk nor a full-length memset, and + * never-touched blocks consume no page-table pool. PROT_NONE stays a pure + * reservation (faults deliver SIGSEGV, not materialization), and MAP_FIXED + * keeps the eager path because it must atomically replace live mappings. + * Shared anonymous memory stays eager unless the caller opted into + * MAP_NORESERVE (the historical lazy set), since deferred zeroing has never + * been exercised against the fork snapshot paths for it. */ bool is_lazy = is_anon && !is_prot_none && ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); @@ -2307,9 +2399,9 @@ int64_t sys_mmap(guest_t *g, track_flags |= LINUX_MAP_ANONYMOUS; /* Preserve MAP_NORESERVE in region metadata before merge checks run. The - * same bit doubles as the internal lazy marker: guest_region_add_ex - * derives the region's deferred-PTE flag from it, and it is not guest - * visible (/proc/self/maps prints only prot and shared/private). + * same bit doubles as the internal lazy marker: guest_region_add_ex derives + * the region's deferred-PTE flag from it, and it is not guest visible + * (/proc/self/maps prints only prot and shared/private). */ if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; @@ -2715,9 +2807,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rx_hwm = result_off + length; - if (rx_hwm > g->mmap_rx_next) - g->mmap_rx_next = rx_hwm; + mmap_bump_next(g, result_off, result_off + length); } else { /* RW (or PROT_NONE, or PROT_READ): allocate from main mmap region. * Honor the address hint if provided and within bounds. Some @@ -2776,9 +2866,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rw_hwm = result_off + length; - if (rw_hwm > g->mmap_next) - g->mmap_next = rw_hwm; + mmap_bump_next(g, result_off, result_off + length); } if (!region_has_capacity_after_removes(g, NULL, 0, 1)) { host_fd_ref_close(&backing_ref); @@ -3315,10 +3403,16 @@ int64_t sys_mremap(guest_t *g, } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); + int pt_err = restore_snapshot_page_tables( + g, new_off, new_off + new_size, dest_snaps, dest_nsnaps); + if (pt_err < 0) + restore_err = pt_err; dispose_region_snapshots(&dest_snaps, &dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); + if (restore_err < 0) + return restore_err; return -LINUX_ENOMEM; } @@ -3497,14 +3591,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, old_off, old_off + new_size); /* Update high-water marks */ - uint64_t hwm = old_off + new_size; - if (old_off >= MMAP_RX_BASE && old_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (old_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, old_off, old_off + new_size); return (int64_t) old_addr; } @@ -3661,14 +3748,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, new_off, new_off + new_size); /* Update high-water marks */ - uint64_t hwm = new_off + new_size; - if (new_off >= MMAP_RX_BASE && new_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (new_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, new_off, new_off + new_size); return (int64_t) guest_ipa(g, new_off); } @@ -3914,13 +3994,13 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) return cleanup_err; /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below - * destroys the evidence. Eager regions are zeroed across the whole - * overlap, as before. Lazy (deferred-PTE) regions only need their - * materialized 2MiB blocks zeroed: a block with no L2 mapping was never - * touched through PTEs, host-side fault-in materializes before writing, - * and the previous unmap of that slab range zeroed it -- so its bytes are - * still zero. This keeps munmap cost proportional to memory actually - * touched instead of to the mapping length. + * destroys the evidence. Eager regions are zeroed across the whole overlap, + * as before. Lazy (deferred-PTE) regions only need their materialized 2MiB + * blocks zeroed: a block with no L2 mapping was never touched through PTEs, + * host-side fault-in materializes before writing, and the previous unmap of + * that slab range zeroed it -- so its bytes are still zero. This keeps + * munmap cost proportional to memory actually touched instead of to the + * mapping length. */ zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; int nzr = 0; @@ -3938,8 +4018,8 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (nzr >= MUNMAP_ZERO_RANGES_MAX) { /* Out of slots: widen the last range instead of dropping any * span that must be zeroed. Everything between ranges lies - * inside [unmap_off, end) and is being unmapped, so zeroing - * the gap as well is harmless. + * inside [unmap_off, end) and is being unmapped, so zeroing the + * gap as well is harmless. */ zr[nzr - 1].hi = zend; continue; @@ -4489,7 +4569,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, return -LINUX_ENOMEM; mmap_lock_acquire(g); - /* fork callers have quiesced siblings. Drain their last publications, + /* fork callers have quiesced siblings. Drain their last publications, * revoke every descriptor, and trim never-consumed arena tails before the * legacy [MMAP_BASE,mmap_next) snapshot range is computed. */ diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c new file mode 100644 index 00000000..c57e9d03 --- /dev/null +++ b/tests/bench-mmap.c @@ -0,0 +1,455 @@ +/* + * Comprehensive anonymous-mmap microbenchmark for elfuse. + * + * Measures the guest-visible cost of the mmap subsystem in isolation: + * allocation, teardown, first-touch faults, permission splitting, and remap. It + * is self-contained -- no external harness -- and is meant to be run under + * elfuse (./build/elfuse ./build/bench-mmap) but also runs on any aarch64-linux + * host for a ground-truth comparison. + * + * Timing: reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN in + * bootstrap.c), so a measurement costs an isb + mrs, not a clock_gettime SVC. + * On Apple Silicon CNTFRQ is ~24 MHz (~41.7 ns/tick); amortizing over an + * adaptive batch drives the effective resolution well below one tick. This is + * the key fairness property: clock_gettime on a static guest falls through to + * the ~2 us SVC path and swamps any sub-us operation. + * + * Every case takes one untimed warmup pass (to pay the one-time arena carve and + * page-table extension) and reports the median and min over ITERS runs. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +/* CNTVCT timing */ + +static double ns_per_tick; + +static inline uint64_t rd(void) +{ + uint64_t v; + __asm__ volatile("isb\n\tmrs %0, cntvct_el0" : "=r"(v)); + return v; +} + +static void clock_init(void) +{ + uint64_t f; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(f)); + if (f == 0) + f = 24000000; /* defensive: assume 24 MHz if RES0 */ + ns_per_tick = 1e9 / (double) f; +} + +static double ns(uint64_t ticks) +{ + return (double) ticks * ns_per_tick; +} + +static int cmp_d(const void *a, const void *b) +{ + double x = *(const double *) a, y = *(const double *) b; + return (x > y) - (x < y); +} + +static double median(double *v, int n) +{ + qsort(v, n, sizeof(*v), cmp_d); + return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +#define ITERS 15 +#define MAXB 64 +#define KIB (1ULL << 10) +#define MIB (1ULL << 20) +#define GIB (1ULL << 30) + +/* Batch size: amortize the coarse counter over B ops while bounding the live + * address footprint of one timed batch to ~256 MiB. + */ +static int batch_for(uint64_t size) +{ + uint64_t b = (256 * MIB) / size; + if (b < 1) + b = 1; + if (b > MAXB) + b = MAXB; + return (int) b; +} + +static const char *human(uint64_t s, char *buf) +{ + if (s >= GIB) + sprintf(buf, "%llu GiB", (unsigned long long) (s / GIB)); + else if (s >= MIB) + sprintf(buf, "%llu MiB", (unsigned long long) (s / MIB)); + else + sprintf(buf, "%llu KiB", (unsigned long long) (s / KIB)); + return buf; +} + +/* A. mmap + munmap latency vs size (steady state) NULL-hint allocate then free, + * batched. Iterations after the first reuse freed address space, so this is the + * realistic repeated-allocation number a workload sees, not the one-shot fresh + * case (that is section B). + */ +static void bench_size_sweep(void) +{ + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + printf( + "== A. mmap / munmap latency vs size (steady state, NULL hint) ==\n"); + printf("%-10s %6s %12s %12s\n", "size", "batch", "mmap ns", "munmap ns"); + void *ptr[MAXB]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int b = batch_for(size); + double mm[ITERS], um[ITERS]; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t t0 = rd(); + for (int i = 0; i < b; i++) + ptr[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + for (int i = 0; i < b; i++) + if (ptr[i] == MAP_FAILED) + ok = 0; + if (!ok) + break; + uint64_t t2 = rd(); + for (int i = 0; i < b; i++) + munmap(ptr[i], size); + uint64_t t3 = rd(); + if (it >= 0) { + mm[it] = ns(t1 - t0) / b; + um[it] = ns(t3 - t2) / b; + } + } + char hb[16]; + if (!ok) { + printf("%-10s %6d %12s %12s\n", human(size, hb), b, "FAILED", "-"); + continue; + } + printf("%-10s %6d %12.1f %12.1f\n", human(size, hb), b, + median(mm, ITERS), median(um, ITERS)); + } + printf("\n"); +} + +/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path) Allocate a + * bounded sequential run WITHOUT freeing, so every mapping lands at or above + * the arena high-water -- exactly the case lazy_fresh_range skips the stale-PTE + * scan for. The run is kept small enough (<= 1000 regions, well under + * GUEST_MAX_REGIONS, footprint <= 2 GiB) that region bookkeeping and page-table + * extension do not dominate, and every result is failure-checked. Run this + * binary against an opt-off build to read the skip's contribution as the + * difference on this identical code path -- a MAP_FIXED "recycled" compare + * would instead measure the region-snapshot replacement path, not the skip. + */ +static void bench_fresh(void) +{ + static const uint64_t sizes[] = {4 * KIB, 64 * KIB, MIB, + 2 * MIB, 8 * MIB, 128 * MIB, + GIB, 8 * GIB, 32 * GIB}; + printf( + "== B. fresh bump-tail mmap, per-mmap ns (lazy_fresh_range path) ==\n"); + printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); + void *run[1000]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int n = (int) (2 * GIB / size); + if (n < 1) + n = 1; + if (n > 1000) + n = 1000; + /* warmup one fresh mapping so the arena high-water is already primed */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + char hb[16]; + if (failed) { + printf("%-10s %8d %14s (%d failed)\n", human(size, hb), n, + "PARTIAL", failed); + continue; + } + printf("%-10s %8d %14.1f\n", human(size, hb), n, ns(t1 - t0) / n); + } + printf("\n"); +} + +/* C. first-touch page-fault cost Touch one byte per page at a 16 KiB stride so + * no two touches share a macOS host page; every touch is a genuine fault (HVC + * #11 -> host fault handler -> page-table install + zero). Reports per-fault + * ns. + */ +static void bench_fault(void) +{ + const uint64_t stride = 16 * KIB; + const int pages = 512; + uint64_t size = stride * (uint64_t) (pages + 1); + printf("== C. first-touch fault cost (16 KiB stride, %d pages) ==\n", + pages); + double per[ITERS]; + for (int it = -1; it < ITERS; it++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED: %s\n\n", strerror(errno)); + return; + } + uint64_t t0 = rd(); + for (int i = 0; i < pages; i++) + p[(uint64_t) i * stride] = 1; + uint64_t t1 = rd(); + munmap((void *) p, size); + if (it >= 0) + per[it] = ns(t1 - t0) / pages; + } + printf(" per-fault: median %.1f ns min %.1f ns\n\n", median(per, ITERS), + per[0]); +} + +/* D. mprotect split cost Flip the middle 4 KiB of a 2 MiB RW block to + * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 + * pages. Restore between iterations so each run does a fresh split. + */ +static void bench_mprotect_split(void) +{ + printf( + "== D. mprotect split (2 MiB block -> L3, protect middle 4 KiB) ==\n"); + double sp[ITERS]; + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint8_t *mid = p + MIB; + uint64_t t0 = rd(); + int rc = mprotect(mid, 4 * KIB, PROT_READ); + uint64_t t1 = rd(); + munmap(p, 2 * MIB); + if (rc != 0) { + printf(" mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + sp[it] = ns(t1 - t0); + } + printf(" split: median %.1f ns min %.1f ns\n\n", median(sp, ITERS), + sp[0]); +} + +/* E. mremap grow: in-place vs forced move */ +static void bench_mremap(void) +{ + printf("== E. mremap grow 4 KiB -> 8 KiB ==\n"); + double inp[ITERS], mov[ITERS]; + + /* In-place: no blocker, the following page is free. */ + for (int it = -1; it < ITERS; it++) { + void *p = mmap(NULL, 4 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + munmap(p, 4 * KIB); + printf(" mremap in-place FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (it >= 0) + inp[it] = ns(t1 - t0); + } + + /* Forced move: a PROT_READ blocker sits immediately after, so the grow must + * relocate. + */ + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 8 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + /* free the tail page and pin it read-only so in-place growth is blocked + * but the head is still a 4 KiB mapping. + */ + munmap(p + 4 * KIB, 4 * KIB); + void *blk = mmap(p + 4 * KIB, 4 * KIB, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + printf(" mremap move FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (blk != MAP_FAILED) + munmap(blk, 4 * KIB); + if (it >= 0) + mov[it] = ns(t1 - t0); + } + printf(" in-place: median %.1f ns min %.1f ns\n", median(inp, ITERS), + inp[0]); + printf(" move: median %.1f ns min %.1f ns\n\n", median(mov, ITERS), + mov[0]); +} + +/* F. multi-threaded fresh mmap under mmap_lock Several threads hammer fresh + * bump-tail mmaps concurrently. mmap serializes on mmap_lock, so this exposes + * both lock contention and any per-mmap TLBI shootdown cost -- the one place a + * "skip the invalidate on fresh ranges" optimization could pay off that a + * single-threaded run cannot see. Threads do not free during the timed run + * (every mapping stays fresh); total live regions are capped under + * GUEST_MAX_REGIONS. Compare against an opt-off build to read the skip's + * multi-threaded contribution. + */ +typedef struct { + uint64_t size; + int n; + void **buf; + double per_op_ns; + int failed; +} mt_arg_t; + +static pthread_barrier_t mt_barrier; + +/* CNTVCT_EL0 reads a constant on worker vCPUs (EL0VCTEN is set for the main + * vCPU only), so the MT worker brackets its whole loop with clock_gettime and + * amortizes the one SVC pair over n mmaps. + */ +static uint64_t mono_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void *mt_worker(void *p) +{ + mt_arg_t *a = p; + pthread_barrier_wait(&mt_barrier); + uint64_t t0 = mono_ns(); + for (int i = 0; i < a->n; i++) + a->buf[i] = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = mono_ns(); + a->per_op_ns = (double) (t1 - t0) / a->n; + for (int i = 0; i < a->n; i++) { + if (a->buf[i] == MAP_FAILED) + a->failed++; + else + munmap(a->buf[i], a->size); + } + return NULL; +} + +#define MT_MAX_THREADS 4 +#define MT_REGION_CAP 3000 /* keep T*n well under GUEST_MAX_REGIONS (4096) */ + +static void bench_mt(void) +{ + static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; + static const int threads[] = {2, 4}; + printf( + "== F. multi-threaded fresh mmap, per-op ns (mmap_lock contention) " + "==\n"); + printf("%-10s %8s %12s %12s\n", "size", "threads", "mean ns", "max ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) { + int T = threads[t]; + int n = MT_REGION_CAP / T; + if (n < 1) + n = 1; + mt_arg_t arg[MT_MAX_THREADS]; + pthread_t th[MT_MAX_THREADS]; + int ok = 1; + for (int i = 0; i < T; i++) { + arg[i].size = size; + arg[i].n = n; + arg[i].per_op_ns = 0; + arg[i].failed = 0; + arg[i].buf = calloc(n, sizeof(void *)); + if (!arg[i].buf) + ok = 0; + } + /* prime the arena high-water so the timed run is genuinely fresh */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + pthread_barrier_init(&mt_barrier, NULL, (unsigned) T); + for (int i = 0; i < T && ok; i++) + if (pthread_create(&th[i], NULL, mt_worker, &arg[i]) != 0) + ok = 0; + double sum = 0, mx = 0; + int failed = 0; + for (int i = 0; i < T; i++) { + pthread_join(th[i], NULL); + sum += arg[i].per_op_ns; + if (arg[i].per_op_ns > mx) + mx = arg[i].per_op_ns; + failed += arg[i].failed; + free(arg[i].buf); + } + pthread_barrier_destroy(&mt_barrier); + char hb[16]; + if (!ok || failed) + printf("%-10s %8d %12s\n", human(size, hb), T, "FAILED"); + else + printf("%-10s %8d %12.1f %12.1f\n", human(size, hb), T, sum / T, + mx); + } + } + printf("\n"); +} + +int main(void) +{ + clock_init(); + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_size_sweep(); + bench_fresh(); + bench_fault(); + bench_mprotect_split(); + bench_mremap(); + bench_mt(); + return 0; +} diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh index d7ad0b0c..9d8e2715 100755 --- a/tests/test-mmap-dirty-stats.sh +++ b/tests/test-mmap-dirty-stats.sh @@ -9,7 +9,7 @@ TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ - >"$TMPDIR_CASE/out" 2>"$TMPDIR_CASE/err" + > "$TMPDIR_CASE/out" 2> "$TMPDIR_CASE/err" counter() { diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh index 4075bc05..fa92a8a7 100755 --- a/tests/test-mmap-fastpath-stats.sh +++ b/tests/test-mmap-fastpath-stats.sh @@ -15,7 +15,7 @@ run_case() out="$TMPDIR_CASE/$case_name.out" err="$TMPDIR_CASE/$case_name.err" ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ - >"$out" 2>"$err" + > "$out" 2> "$err" } counter() diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c index b483cd26..8541b819 100644 --- a/tests/test-mmap-lazy.c +++ b/tests/test-mmap-lazy.c @@ -134,6 +134,38 @@ static void test_zero_reuse(void) PASS(); } +static void test_hinted_tail_zero(void) +{ + TEST("hinted tail mmap reads zero"); + size_t size = 2ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap base"); + return; + } + p[0] = 0x3c; + p[size - 1] = 0xc3; + + uint8_t *hint = p + size; + uint8_t *q = mmap(hint, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap hint"); + munmap(p, size); + return; + } + if (q[0] != 0 || q[size - 1] != 0 || p[0] != 0x3c || p[size - 1] != 0xc3) { + FAIL("hinted tail leaked stale bytes or clobbered neighbor"); + munmap(q, size); + munmap(p, size); + return; + } + munmap(q, size); + munmap(p, size); + PASS(); +} + static void test_partial_block_reuse(void) { TEST("partial-block reuse preserves neighbor"); @@ -719,6 +751,7 @@ int main(void) { test_huge_sparse(); test_zero_reuse(); + test_hinted_tail_zero(); test_partial_block_reuse(); test_fork_clean_reuse(); test_file_overlay_reuse(); diff --git a/tests/test-mremap.c b/tests/test-mremap.c index dc97321b..bf8abe33 100644 --- a/tests/test-mremap.c +++ b/tests/test-mremap.c @@ -120,6 +120,106 @@ static void test_grow_maymove(void) munmap(q, 4096 * 4); } +static void test_grow_move_adjacent_fault(void) +{ + TEST("mremap fixed move keeps adjacent page unmapped"); + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *dest = + mmap(NULL, 4096 * 3, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap dest"); + munmap(p, 4096); + return; + } + if (munmap(dest + 8192, 4096) < 0) { + FAIL("munmap guard"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + ((char *) p)[0] = 0x5a; + + void *q = mremap(p, 4096, 8192, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (q == MAP_FAILED) { + FAIL("mremap move"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + if (q != dest || ((char *) q)[0] != 0x5a || ((char *) q)[4096] != 0) { + FAIL("mremap data"); + munmap(q, 8192); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + volatile unsigned char value = *((volatile unsigned char *) q + 8192); + (void) value; + _exit(1); + } + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid) { + FAIL("fork/wait"); + munmap(q, 8192); + return; + } + if ((WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV) || + (WIFEXITED(status) && WEXITSTATUS(status) == 139)) + PASS(); + else + FAIL("adjacent page became readable"); + munmap(q, 8192); +} + +static void test_fixed_preserves_neighbor_l3(void) +{ + TEST("mremap fixed preserves neighboring lazy PTEs"); + char *source = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (source == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *anchor = mmap(NULL, 8192, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anchor == MAP_FAILED) { + FAIL("mmap anchor"); + munmap(source, 4096); + return; + } + char *dest = anchor + 4096; + if (munmap(dest, 4096) < 0) { + FAIL("munmap dest"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + source[0] = 0x11; + anchor[0] = 0x7e; + + void *moved = + mremap(source, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (moved == MAP_FAILED) { + FAIL("mremap fixed"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + if (moved != dest || dest[0] != 0x11 || anchor[0] != 0x7e) + FAIL("neighboring lazy page changed"); + else + PASS(); + + munmap(dest, 4096); + munmap(anchor, 4096); +} + /* Test 3: grow without MAYMOVE fails if blocked */ static void test_grow_no_maymove(void) @@ -403,6 +503,8 @@ int main(void) test_shrink(); test_grow_maymove(); + test_grow_move_adjacent_fault(); + test_fixed_preserves_neighbor_l3(); test_grow_no_maymove(); test_fixed(); test_same_size(); From be12128518ee9e8e66c54d28493cba7ed2e6858e Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 15 Jul 2026 15:57:36 +0800 Subject: [PATCH 3/9] Set CNTKCTL_EL1 on worker vCPUs for clock fastpath Bootstrap sets CNTKCTL_EL1.EL0VCTEN|EL0PCTEN on the primary vCPU so EL0 can read CNTVCT_EL0, but the CLONE_THREAD and CLONE_VM worker vCPU bring-up paths copied VBAR/MAIR/TCR/TTBR0/CPACR/SCTLR/SP/TPIDR/ELR/SPSR and omitted CNTKCTL. Worker EL0 read 0 from CNTVCT_EL0, so the vDSO clock_gettime trampoline's seqlock/anchor check failed and silently fell back to the SVC clock on every non-main thread. Set CNTKCTL_EL1 = 0x3 in both worker paths; the value is not thread-specific. Measured with a dynamic glibc guest (which uses the vDSO trampoline), clock_gettime(CLOCK_MONOTONIC) averaged over 200k calls: no fix with fix main 8.1 ns 9.2 ns worker 2898.5 ns 9.2 ns Static glibc guests SVC clock_gettime even on the main thread, which is why this stayed hidden. To reproduce: build a dynamic clock_gettime loop, run it under elfuse --sysroot with the loop on a worker thread, and compare against a build with the two CNTKCTL sets removed. --- src/runtime/forkipc.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index 427c96fe..7cf6973c 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -520,9 +520,9 @@ static void resolve_clone_stack_range(guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; - /* EL1 consumer-mmap entries are drained into regions[] under mmap_lock. - * A sibling worker can drain and merge the array while clone resolves its - * new stack, so take the same lock and copy the bounds before releasing it. + /* EL1 consumer-mmap entries are drained into regions[] under mmap_lock. A + * sibling worker can drain and merge the array while clone resolves its new + * stack, so take the same lock and copy the bounds before releasing it. */ mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); @@ -706,11 +706,10 @@ static int64_t sys_clone_thread(hv_vcpu_t parent_vcpu, return -LINUX_EFAULT; } - /* Create the host pthread (joinable; the main thread joins all live - * workers via thread_join_workers before guest teardown, and a worker - * that exits on its own is joined when its table slot is reused by - * thread_alloc). Threads clean up their TID address via - * CLONE_CHILD_CLEARTID + futex wake. + /* Create the host pthread (joinable; the main thread joins all live workers + * via thread_join_workers before guest teardown, and a worker that exits on + * its own is joined when its table slot is reused by thread_alloc). Threads + * clean up their TID address via CLONE_CHILD_CLEARTID + futex wake. */ pthread_t host_thread; pthread_attr_t attr; @@ -860,6 +859,13 @@ static void *thread_create_and_run(void *arg) WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TTBR0_EL1, tca->ttbr0)); WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CPACR_EL1, tca->cpacr)); + /* CNTKCTL_EL1.EL0VCTEN|EL0PCTEN: worker vCPUs need EL0 counter access too, + * or the vDSO clock_gettime fast path reads 0 from CNTVCT_EL0 and every + * non-main thread silently drops to the SVC clock. Same constant bootstrap + * sets on the primary vCPU; not thread-specific. + */ + WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, 0x3ULL)); + /* All worker vCPUs in the process share the same shim_globals base (one VM * per process); a fresh TPIDR_EL1 set is still required because HVF created * this vCPU empty. CONTEXTIDR_EL1 holds the per-thread tid that the gettid @@ -978,8 +984,8 @@ startup_ok:; * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain any deferred munmap before publishing clear_child_tid. A joiner - * may observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * Drain any deferred munmap before publishing clear_child_tid. A joiner may + * observe the zero without ever sleeping in FUTEX_WAIT, then reuse the * freed VA immediately; ordering only the wake after cleanup leaves a * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ @@ -1211,6 +1217,10 @@ static void *vm_clone_thread_run(void *arg) HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TCR_EL1, tca->tcr)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TTBR0_EL1, tca->ttbr0)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CPACR_EL1, tca->cpacr)); + /* EL0 counter access for the vDSO clock fast path; see + * thread_create_and_run. + */ + HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, 0x3ULL)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SCTLR_EL1, tca->sctlr)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SP_EL1, tca->sp_el1)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SP_EL0, tca->child_stack)); @@ -1306,9 +1316,9 @@ static void *vm_clone_thread_run(void *arg) * harmless no-op. */ thread_interrupt_all(); - /* thread_interrupt_all only reaches threads inside hv_vcpu_run. - * Peers parked on fork_cond or another slot's ptrace_cond/resume_cond - * see neither the vCPU kick nor this exit, so broadcast separately. + /* thread_interrupt_all only reaches threads inside hv_vcpu_run. Peers + * parked on fork_cond or another slot's ptrace_cond/resume_cond see + * neither the vCPU kick nor this exit, so broadcast separately. */ thread_wake_exit_waiters(); } From 9cf91e773aeebc34525df5449787e43850513526 Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 14:18:33 +0800 Subject: [PATCH 4/9] Implement mmap and deferred munmap fast paths Add adaptive per-vCPU mmap arenas, EL1 anonymous munmap retirement with broadcast TLBI, deferred host metadata cleanup, and large backing replacement. Extend synchronization, lifecycle handling, tests, and latency benchmarks for the complete fast-path design. --- docs/internals.md | 8 + docs/usage.md | 3 +- src/core/bootstrap.c | 9 +- src/core/guest.c | 252 +++++++++++- src/core/guest.h | 34 +- src/core/mmap-fastpath.h | 82 +++- src/core/shim-globals.c | 38 +- src/core/shim.S | 505 +++++++++++++++++++++++- src/runtime/forkipc.c | 12 + src/syscall/internal.h | 5 + src/syscall/mem.c | 634 +++++++++++++++++++++++++++++- src/syscall/proc.c | 16 + src/syscall/signal.c | 9 + src/syscall/syscall.c | 14 +- tests/bench-mmap-fresh | 78 ++++ tests/bench-mmap.c | 315 +++++++++++++-- tests/test-mmap-fastpath-stats.sh | 20 +- tests/test-mmap-fastpath.c | 213 +++++++++- tests/test-mmap-lazy.c | 75 ++++ 19 files changed, 2221 insertions(+), 101 deletions(-) create mode 100755 tests/bench-mmap-fresh diff --git a/docs/internals.md b/docs/internals.md index fa285303..4829c238 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -426,6 +426,14 @@ goes above the structured area, never below. Post-push masking ### `mmap` Notes +Private anonymous mappings are lazy at 2 MiB materialization granularity. A +host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any +valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled +fast-path arenas use this index to visit only materialized blocks, so untouched +multi-GiB reservations have length-independent teardown. When every mapping in +a per-vCPU arena has been released and the index confirms that no PTE remains, +the arena cursor rewinds in place instead of taking a refill HVC. + Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so the kernel page cache keeps the mapping coherent with the file (and diff --git a/docs/usage.md b/docs/usage.md index f92af358..7e41f093 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -32,7 +32,8 @@ Setting `--timeout 0` disables this watchdog for long-running CPU-bound guests. ### mmap call fast path The aarch64 EL1 consumer fast path is enabled by default for -`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)`. +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to +32 GiB per request. Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted arenas, and full consumption rings fall back to the normal host syscall path. Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index f31c56ff..b69c8f71 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -310,7 +310,14 @@ static bool build_boot_regions(mem_region_t *regions, * to the vDSO page when splitting the block; otherwise vdso_build cannot * write into it through guest_ptr. */ - if (!append_boot_region(regions, nregions, g->shim_base, + /* EL1 fast munmap walks and atomically clears the live TTBR0 tree. Give + * the page-table pool an identity VA visible only to EL1; EL0 remains + * unable to inspect or corrupt descriptors, and every guest syscall still + * rejects the encompassing infrastructure range. + */ + if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end, + MEM_PERM_RW_EL1_ONLY) || + !append_boot_region(regions, nregions, g->shim_base, g->shim_base + shim_bin_len, MEM_PERM_RX) || /* shim_data is EL1-only: the guest must not directly read or write the * identity cache, attention flag, urandom bitmap, or ring, any of which diff --git a/src/core/guest.c b/src/core/guest.c index e6c119cd..c7665ada 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -40,6 +40,7 @@ #include #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" @@ -87,6 +88,7 @@ static void guest_region_clear(guest_t *g); /* Forward declaration (defined in the page table section below) */ static int desc_to_perms(uint64_t desc); +static uint64_t *find_l2_entry(guest_t *g, uint64_t va); /* Page table pool allocator. */ @@ -253,6 +255,49 @@ static inline void pte_store_release(uint64_t *entry, uint64_t desc) __atomic_store_n(entry, desc, __ATOMIC_RELEASE); } +/* Low-VA TTBR0 occupancy index. All mutators run under mmap_lock (or during + * single-threaded bootstrap), so plain bitmap operations are sufficient. The + * page-table descriptors themselves remain release-published for lock-free + * guest walkers; this host-only index is never consulted by a vCPU. + */ +static inline bool guest_pte_present_index(uint64_t va, uint64_t *block_out) +{ + if (va >= GUEST_PTE_PRESENT_LIMIT) + return false; + *block_out = va / BLOCK_2MIB; + return true; +} + +static inline void guest_pte_present_set(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] |= 1ULL << (block & 63); + g->pte_present_summary[word >> 6] |= 1ULL << (word & 63); +} + +static inline void guest_pte_present_clear(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] &= ~(1ULL << (block & 63)); + if (g->pte_present_blocks[word] == 0) + g->pte_present_summary[word >> 6] &= ~(1ULL << (word & 63)); +} + +static inline bool guest_pte_present_test(const guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return false; + return (g->pte_present_blocks[block >> 6] & + (1ULL << (block & 63))) != 0; +} + /* Public API */ /* FEAT_TLBIRANGE probe -- runs exactly once via pthread_once. ARMv8.4 @@ -1135,9 +1180,11 @@ int guest_map_va_range(guest_t *g, * guest_split_block instead. Skip silently to mirror upstream's * sys_mmap_high_va "reuse existing GPA" behavior. */ + guest_pte_present_set(g, va); continue; } pte_store_release(&l2[l2_idx], make_block_desc(cur_gpa, perms)); + guest_pte_present_set(g, va); if (!bcast) { if (va < changed_lo) changed_lo = va; @@ -1827,6 +1874,8 @@ void guest_reset(guest_t *g) g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; g->ttbr0 = 0; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); tlbi_request_clear(); g->elf_load_min = ELF_DEFAULT_BASE; @@ -2696,6 +2745,9 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) { uint64_t base = g->ipa_base; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + /* Allocate L0 table */ uint64_t l0_gpa = pt_alloc_page(g); if (!l0_gpa) @@ -2791,6 +2843,8 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) * to the primary-buffer GPA where the bytes actually are. */ l2[l2_idx] = make_block_desc(output_ipa, block_perms); + if (lookup_addr >= base) + guest_pte_present_set(g, lookup_addr - base); } } @@ -2926,9 +2980,12 @@ int guest_extend_page_tables(guest_t *g, * an explicit PT_VALID test so the intent survives a future * descriptor-bit renumbering. */ - if (l2[l2_idx] & PT_VALID) + if (l2[l2_idx] & PT_VALID) { + guest_pte_present_set(g, addr); continue; + } pte_store_release(&l2[l2_idx], make_block_desc(ipa, perms)); + guest_pte_present_set(g, addr); if (!bcast) { if (addr < changed_lo) changed_lo = addr; @@ -2944,9 +3001,9 @@ int guest_extend_page_tables(guest_t *g, return 0; } -uint64_t guest_va_next_present_block(const guest_t *g, - uint64_t va, - uint64_t end) +static uint64_t guest_va_next_page_table_block(const guest_t *g, + uint64_t va, + uint64_t end) { if (!g || !g->ttbr0) return end; @@ -2985,10 +3042,81 @@ uint64_t guest_va_next_present_block(const guest_t *g, return end; } +static uint64_t guest_va_next_indexed_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (va >= end || va >= GUEST_PTE_PRESENT_LIMIT) + return end; + if (end > GUEST_PTE_PRESENT_LIMIT) + end = GUEST_PTE_PRESENT_LIMIT; + + uint64_t first_block = va / BLOCK_2MIB; + uint64_t last_block = (end - 1) / BLOCK_2MIB; + uint64_t first_word = first_block >> 6; + uint64_t last_word = last_block >> 6; + uint64_t bits = g->pte_present_blocks[first_word] & + (~0ULL << (first_block & 63)); + if (first_word == last_word && (last_block & 63) != 63) + bits &= (1ULL << ((last_block & 63) + 1)) - 1; + if (bits) + return (first_word * 64 + (uint64_t) __builtin_ctzll(bits)) * + BLOCK_2MIB; + + uint64_t word = first_word + 1; + while (word <= last_word) { + uint64_t summary_word = word >> 6; + uint64_t summary_last = last_word >> 6; + uint64_t summary = g->pte_present_summary[summary_word] & + (~0ULL << (word & 63)); + if (summary_word == summary_last && (last_word & 63) != 63) + summary &= (1ULL << ((last_word & 63) + 1)) - 1; + if (summary) { + uint64_t present_word = summary_word * 64 + + (uint64_t) __builtin_ctzll(summary); + uint64_t present = g->pte_present_blocks[present_word]; + if (present_word == last_word && (last_block & 63) != 63) + present &= (1ULL << ((last_block & 63) + 1)) - 1; + if (present) + return (present_word * 64 + + (uint64_t) __builtin_ctzll(present)) * BLOCK_2MIB; + } + if (summary_word == summary_last) + break; + word = (summary_word + 1) * 64; + } + return end; +} + +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || va >= end) + return end; + + if (va < GUEST_PTE_PRESENT_LIMIT) { + uint64_t low_end = end < GUEST_PTE_PRESENT_LIMIT + ? end + : GUEST_PTE_PRESENT_LIMIT; + uint64_t next = guest_va_next_indexed_block(g, va, low_end); + if (next < low_end || end <= GUEST_PTE_PRESENT_LIMIT) + return next; + va = GUEST_PTE_PRESENT_LIMIT; + } + + /* Non-identity/high-VA mappings live outside the compact low-VA index. + * Preserve the page-table walker for those uncommon ranges. + */ + return guest_va_next_page_table_block(g, va, end); +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) return false; + if (va < GUEST_PTE_PRESENT_LIMIT) + return guest_pte_present_test(g, va); uint64_t base = g->ipa_base; uint64_t *l0 = pt_at(g, g->ttbr0 - base); @@ -3017,6 +3145,71 @@ bool guest_va_block_mapped(const guest_t *g, uint64_t va) return (l2[l2_idx] & PT_VALID) != 0; } +void guest_rebuild_pte_present(guest_t *g) +{ + if (!g) + return; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + if (!g->ttbr0) + return; + + uint64_t limit = g->guest_size; + if (limit > GUEST_PTE_PRESENT_LIMIT) + limit = GUEST_PTE_PRESENT_LIMIT; + for (uint64_t va = 0; va < limit; va += BLOCK_2MIB) { + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + continue; + if ((*l2_entry & 3) == 1) { + guest_pte_present_set(g, va); + continue; + } + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (!l3) + continue; + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + guest_pte_present_set(g, va); + break; + } + } + } +} + +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t block = ALIGN_2MIB_DOWN(start); + while (block < end) { + bool present = false; + uint64_t *l2_entry = find_l2_entry(g, block); + if (l2_entry && (*l2_entry & PT_VALID)) { + if ((*l2_entry & 3) == 1) { + present = true; + } else { + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (l3) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (pte_load_acquire(&l3[i]) & PT_VALID) { + present = true; + break; + } + } + } + } + } + if (present) + guest_pte_present_set(g, block); + else + guest_pte_present_clear(g, block); + block += BLOCK_2MIB; + } +} + /* L3 page table splitting. */ /* L3 page descriptor: bits[1:0]=11 = valid page at level 3. This is distinct @@ -3173,6 +3366,7 @@ int guest_split_block(guest_t *g, uint64_t block_gpa) int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) { uint64_t base = g->ipa_base; + bool any_changed = false; /* Page-align the range. The ALIGN_UP step on end could wrap to 0 for inputs * within PAGE_SIZE-1 of UINT64_MAX, silently turning the invalidation into @@ -3187,14 +3381,20 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) return 0; for (uint64_t addr = start; addr < end;) { + uint64_t indexed_block = ALIGN_2MIB_DOWN(addr); + if (indexed_block < GUEST_PTE_PRESENT_LIMIT && + !guest_pte_present_test(g, indexed_block)) { + addr = guest_va_next_present_block( + g, indexed_block + BLOCK_2MIB, end); + continue; + } uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 table (L0/L1 slot absent): nothing to invalidate in this - * block. Skip whole absent 1GiB/512GiB slots at once; a lazy - * multi-GiB mmap invalidates its stale range on every allocation - * and would otherwise pay one four-level walk per 2MiB of empty - * address space. + /* No L2 table: nothing to invalidate in this block. The low-VA + * occupancy index jumps directly to the next block containing a + * valid PTE; high VA retains the page-table hierarchy fallback. */ + guest_pte_present_clear(g, indexed_block); addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3204,6 +3404,7 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) /* Not mapped at all: skip */ if (!(*l2_entry & 1)) { + guest_pte_present_clear(g, block_start); addr = block_end; continue; } @@ -3217,6 +3418,8 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) * broadcast. */ pte_store_release(l2_entry, 0); + guest_pte_present_clear(g, block_start); + any_changed = true; tlbi_request_range(base + block_start, base + block_end); addr = block_end; continue; @@ -3242,12 +3445,14 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) uint64_t page_end = (end < block_end) ? end : block_end; uint64_t changed_lo = UINT64_MAX, changed_hi = 0; bool bcast = tlbi_request_is_broadcast(); + bool block_changed = false; for (uint64_t pa = page_start; pa < page_end; pa += PAGE_SIZE) { unsigned l3_idx = (unsigned) (((base + pa) % BLOCK_2MIB) / PAGE_SIZE); if (l3[l3_idx] != 0) { pte_store_release(&l3[l3_idx], 0); /* Invalid descriptor */ + block_changed = true; if (!bcast) { if (pa < changed_lo) changed_lo = pa; @@ -3259,10 +3464,27 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + if (block_changed) + any_changed = true; + + bool block_present = false; + if (page_start > block_start || page_end < block_end) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + block_present = true; + break; + } + } + } + if (block_present) + guest_pte_present_set(g, block_start); + else + guest_pte_present_clear(g, block_start); addr = page_end; } - guest_pt_gen_bump(g); + if (any_changed) + guest_pt_gen_bump(g); return 0; } @@ -3338,6 +3560,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) pte_store_release(l2_entry, make_block_desc(ipa, perms)); tlbi_request_range(base + block_start, base + block_end); } + guest_pte_present_set(g, block_start); addr = block_end; continue; } @@ -3411,6 +3634,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + guest_pte_present_set(g, block_start); addr = page_end; } @@ -3493,6 +3717,7 @@ int guest_install_va_pages(guest_t *g, changed_hi = v + PAGE_SIZE; } } + guest_pte_present_set(g, v); } if (!bcast && changed_hi > changed_lo) @@ -3657,6 +3882,7 @@ retry:; * contract as a newly installed block. */ if (guest_va_pte_valid(g, fault_offset)) { + mmap_fastpath_note_materialized_locked(g, block_start, block_end); g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); return 0; @@ -3725,7 +3951,7 @@ retry:; claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); if (claim_slot >= 0) - mmap_lock_release(); + mmap_lock_drop_keep_gate(); for (unsigned page = 0; page < 512;) { if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { page++; @@ -3741,7 +3967,7 @@ retry:; 0, (uint64_t) (page - first) * PAGE_SIZE); } if (claim_slot >= 0) - mmap_lock_acquire(g); + mmap_lock_reacquire_with_gate(g); if (!any_valid && materialize_start == block_start && materialize_end == block_end) guest_dirty_clear_zeroed_range(g, block_start, block_end); @@ -3794,6 +4020,8 @@ retry:; : GUEST_MATERIALIZE_CLEAN_SKIP]++; g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += materialize_end - materialize_start; + mmap_fastpath_note_materialized_locked(g, materialize_start, + materialize_end); materialize_claim_release_locked(g, claim_slot); return 0; } diff --git a/src/core/guest.h b/src/core/guest.h index d9b3f481..b9580224 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -420,6 +420,19 @@ typedef struct { #define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) #define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) +/* Host-side occupancy index for low-VA TTBR0 mappings. One bit per 2 MiB + * block records whether that block contains at least one valid L2/L3 PTE; a + * second-level bitmap records which occupancy words are non-zero. This lets + * huge lazy mmap/munmap ranges skip untouched address space without walking + * one page-table slot per GiB. The index is separate from dirty_blocks: a + * read-only mapping can have valid PTEs without dirty backing bytes. + */ +#define GUEST_PTE_PRESENT_BLOCKS_MAX GUEST_DIRTY_BLOCKS_MAX +#define GUEST_PTE_PRESENT_WORDS (GUEST_PTE_PRESENT_BLOCKS_MAX / 64) +#define GUEST_PTE_PRESENT_SUMMARY_WORDS (GUEST_PTE_PRESENT_WORDS / 64) +#define GUEST_PTE_PRESENT_LIMIT \ + (GUEST_PTE_PRESENT_BLOCKS_MAX * BLOCK_2MIB) + enum { GUEST_MATERIALIZE_CLEAN_SKIP = 0, GUEST_MATERIALIZE_DIRTY_MEMSET, @@ -562,6 +575,8 @@ typedef struct { */ _Atomic uint64_t pt_gen; + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; + uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; @@ -1040,6 +1055,17 @@ int guest_install_va_pages(guest_t *g, */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); +/* Rebuild the low-VA PTE occupancy index from TTBR0. Used after fork restores + * page-table pages into a freshly initialized guest_t. + */ +void guest_rebuild_pte_present(guest_t *g); + +/* Reconcile the host-only 2 MiB occupancy index after EL1 has invalidated + * descriptors in [start,end). This observes PTEs only; it never writes a + * descriptor or requests another TLBI. Caller holds mmap_lock. + */ +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end); + /* Returns true when the VA range [va, va+size) overlaps the user-VA kbuf alias * window [KBUF_USER_VA, KBUF_USER_VA+KBUF_SIZE). Callers that install TTBR0 * mappings (the future rosetta_finalize, sys_mmap MAP_FIXED touching this @@ -1085,10 +1111,10 @@ int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); */ int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); -/* Smallest block-aligned va' in [va, end) whose 1GiB L1 slot is present in - * the page tables, or end if none. Lets range walkers skip absent 1GiB / - * 512GiB slots in O(1) instead of probing every 2MiB block. Locking: callers - * MUST hold mmap_lock. +/* Smallest 2MiB-block-aligned va' in [va, end) containing at least one valid + * TTBR0 PTE, or end if none. Low VA uses the host-side hierarchical occupancy + * bitmap; non-identity/high VA falls back to the page-table hierarchy. Locking: + * callers MUST hold mmap_lock. */ uint64_t guest_va_next_present_block(const guest_t *g, uint64_t va, diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index 8b2a8fbe..2c4b695a 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -10,6 +10,7 @@ #pragma once #include +#include #include #include @@ -19,12 +20,28 @@ typedef struct thread_entry thread_entry_t; #define SHIM_MMAP_CONTROL_BASE 0x20000u #define SHIM_MMAP_CONTROL_STRIDE 0x800u -#define SHIM_MMAP_RING_SIZE 16u +#define SHIM_MMAP_RING_SIZE 32u #define SHIM_MMAP_CTRL_ENABLED 0x1u +#define SHIM_MMAP_CTRL_TLBIRANGE 0x2u + +/* Host page-table writers set this gate before changing an arena descriptor or + * a stage-1 PTE. Each EL1 producer announces itself in its private control + * before rechecking the gate. This is a writer-vs-per-vCPU-reader handshake, + * not an allocator lock: fast munmap producers never write a shared cache + * line or wait for one another. + */ +#define SHIM_MMAP_PT_GATE_OFF 0x1160u + +#define SHIM_MUNMAP_RETIRE_RING_SIZE 32u +#define SHIM_MUNMAP_RETIRE_OFF 0x400u +#define SHIM_MUNMAP_RETIRE_BYTES_SOFT (256ULL * 1024 * 1024) +#define SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK 0x3fu +#define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u +#define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u #define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) -#define MMAP_FAST_ARENA_MAX (1ULL * 1024 * 1024 * 1024) -#define MMAP_FAST_HISTORY_MULTIPLIER 16u +#define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_ARENA_TARGET_ENTRIES 32u enum { SHIM_MMAP_COUNTER_SHAPE_MISS = 0, @@ -42,6 +59,24 @@ typedef struct { uint64_t prot; } shim_mmap_entry_t; +typedef struct munmap_retire_entry { + uint64_t addr; + uint64_t length; + uint32_t arena_generation; + uint32_t flags; +} munmap_retire_entry_t; + +typedef struct munmap_retire_ring { + _Atomic uint32_t head; /* host consumer */ + _Atomic uint32_t tail; /* EL1 producer */ + _Atomic uint64_t produced_bytes; /* EL1 producer, monotonic */ + _Atomic uint64_t consumed_bytes; /* host consumer, monotonic */ + _Atomic uint32_t producer_active; + /* Advisory; never forces the producer to exit. */ + _Atomic uint32_t cleanup_requested; + munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; +} munmap_retire_ring_t; + typedef struct { _Atomic uint32_t generation; /* host publish word */ _Atomic uint32_t consumer_generation; /* EL1 generation ack */ @@ -53,26 +88,65 @@ typedef struct { _Atomic uint64_t arena_limit; _Atomic uint64_t cursor; /* EL1 bump cursor */ uint64_t next_arena_size; /* most recently selected generation size */ - uint64_t max_len_seen; /* outgoing-generation request history */ + uint64_t max_len_seen; /* outgoing-generation maximum request length */ shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; uint64_t refill_count; uint64_t recycle_count; uint64_t peak_arena_size; + _Atomic uint32_t materialized_generation; + uint32_t _pad1; + _Atomic uint64_t materialized_start; + _Atomic uint64_t materialized_end; + uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; + munmap_retire_ring_t retire; } shim_mmap_control_t; +_Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, + "shim.S retire-ring offset ABI"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "mmap control exceeds per-vCPU stride"); + /* Called while initializing a vCPU, before it can enter guest code. */ void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); /* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ void mmap_fastpath_drain_locked(guest_t *g); +/* Opportunistically drain publications and retirements at a natural VM exit. + * Safe before every exit handler; it acquires mmap_lock internally. + */ +void mmap_fastpath_drain_vmexit(guest_t *g); + +/* True when the stopped current vCPU was interrupted in the middle of its EL1 + * producer critical section. A cancellation exit must resume it before host + * drain can close the PT gate. + */ +bool mmap_fastpath_current_producer_active(const guest_t *g); + +/* Mark every arena intersecting a successfully materialized lazy range. Caller + * holds mmap_lock. EL1 may skip its PTE walk only while this marker differs + * from the arena's current generation. + */ +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end); + /* Refill the current vCPU after an eligible mmap slow-path. request_len is * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. * Caller holds mmap_lock. */ void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); +/* Fulfil an eligible mmap that reached HVC (capacity/ring/generation fallback) + * directly from the current vCPU's refilled arena. Metadata is committed by + * the host immediately, so no mmap publication entry is needed. Caller holds + * mmap_lock. + */ +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); + /* Give an explicit slow-path hint precedence over this stopped vCPU's * unconsumed arena tail. Caller holds mmap_lock. */ diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 02351e1b..642bfaa6 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -101,8 +101,8 @@ _Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, "shim.S mmap fast path hard-codes control base 0x20000"); _Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x800, "shim.S mmap fast path hard-codes control stride 0x800"); -_Static_assert(SHIM_MMAP_RING_SIZE == 16, - "shim.S mmap fast path hard-codes 16 ring entries"); +_Static_assert(SHIM_MMAP_RING_SIZE == 32, + "shim.S mmap fast path hard-codes 32 ring entries"); _Static_assert(offsetof(shim_mmap_control_t, generation) == 0, "shim.S mmap generation offset drift"); _Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, @@ -125,8 +125,40 @@ _Static_assert(offsetof(shim_mmap_control_t, max_len_seen) == 56, "mmap max-len-seen offset drift"); _Static_assert(offsetof(shim_mmap_control_t, ring) == 64, "shim.S mmap ring offset drift"); -_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x1C0, +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x340, "shim.S mmap counter offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_generation) == 0x388, + "shim.S mmap materialized-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_start) == 0x390 && + offsetof(shim_mmap_control_t, materialized_end) == 0x398, + "shim.S mmap materialized bounds offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, retire) == 0x400, + "shim.S munmap retire offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, produced_bytes) == 8, + "shim.S munmap produced-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, consumed_bytes) == 16, + "shim.S munmap consumed-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, producer_active) == 24, + "shim.S munmap active offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, cleanup_requested) == 28, + "shim.S munmap cleanup-request offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, entries) == 32, + "shim.S munmap entries offset drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_RING_SIZE == 32 && MAX_THREADS == 64, + "shim.S munmap ring/arena scan constants drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, + "shim.S munmap byte advisory threshold drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, + "shim.S munmap retire flag encoding drift"); +_Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= + (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), + "munmap retire flags cannot encode maximum arena charge"); +_Static_assert(SHIM_MMAP_PT_GATE_OFF >= SHIM_GLOBALS_SIZE && + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) <= + SHIM_MMAP_CONTROL_BASE, + "host PT gate overlaps shim globals or mmap controls"); _Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, "per-vCPU mmap control exceeds its shim-data stride"); _Static_assert(SHIM_MMAP_CONTROL_BASE + diff --git a/src/core/shim.S b/src/core/shim.S index 6f1a15c7..849c79c1 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -26,7 +26,7 @@ * 4 = single-shot TLBI RVAE1IS (FEAT_TLBIRANGE); * X9 carries the pre-encoded RVAE1IS operand * (baddr | NUM<<39 | SCALE<<44 | TTL<<37 | - * ASID<<48; SCALE=0, TTL=0, ASID=0 today) + * ASID<<48; TTL=0 and ASID=0) * X11 carries the I-cache hint for X8 in {1, 3, 4}: * 1 = IC IALLU after the TLBI sequence (new * executable content visible to EL0), 0 = skip the @@ -251,11 +251,11 @@ .Lctr_skip_\@: .endm -/* Per-vCPU mmap counters live after the 16 x 24-byte publication ring in the +/* Per-vCPU mmap counters live after the 32 x 24-byte publication ring in the * mmap control block. Each vCPU is the sole writer of its own slots. Keep the * shared stats gate so normal fast paths do not touch diagnostic cache lines. */ -.equ SHIM_MMAP_COUNTERS_OFF, 0x1C0 +.equ SHIM_MMAP_COUNTERS_OFF, 0x340 .equ MC_SHAPE_MISS, 0 .equ MC_CAPACITY_MISS, 1 .equ MC_RING_FULL, 2 @@ -462,6 +462,8 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #215 /* SYS_munmap? */ + b.eq munmap_anon_fast cmp x10, #222 /* SYS_mmap? */ b.eq mmap_anon_fast b handle_svc_0 @@ -533,10 +535,10 @@ mmap_anon_fast: ldar w22, [x23] ldr w23, [x12, #16] /* producer-owned tail */ sub w24, w23, w22 - cmp w24, #16 /* SHIM_MMAP_RING_SIZE */ + cmp w24, #32 /* SHIM_MMAP_RING_SIZE */ b.hs mmap_ring_full_bail - and w24, w23, #15 + and w24, w23, #31 add x24, x24, x24, lsl #1 /* index * 3 */ add x25, x12, #64 /* ring base */ add x25, x25, x24, lsl #3 /* index * 24 */ @@ -576,6 +578,499 @@ mmap_generation_bail: str w13, [x12, #4] b handle_svc_0 +/* munmap_anon_fast: invalidate an anonymous arena range synchronously at EL1, + * then release-publish a metadata retirement to this vCPU's SPSC ring. The + * host consumes it at a later natural VM exit. No shared allocator cursor is + * modified here. + * + * The two-pass page-table walk is deliberate. Pass one proves every partial + * 2 MiB block already has an L3 table, so pass two cannot discover a need to + * split after it has cleared an earlier descriptor. Whole L2 blocks and L3 + * leaves are the only descriptors this path writes. + */ +munmap_anon_fast: + mrs x12, tpidr_el1 /* shim-data base */ + ldar w13, [x12] /* attention gate */ + + add x20, sp, #0xfff + and x20, x20, #~0xfff /* this vCPU's stack top */ + add x21, x12, #0x200, lsl #12 + sub x21, x21, x20 + lsr x21, x21, #1 + add x21, x12, x21 + add x21, x21, #0x20, lsl #12 /* current mmap control */ + + cbnz w13, munmap_pre_bail + ldr x14, [sp, #0] /* addr */ + ldr x15, [sp, #8] /* length */ + cbz x15, munmap_pre_bail + tst x14, #0xfff + b.ne munmap_pre_bail + tst x15, #0xfff + b.ne munmap_pre_bail + adds x16, x14, x15 /* exclusive end */ + b.cs munmap_pre_bail + + add x17, x21, #0x400 /* private retire ring */ + add x20, x17, #0 + ldar w20, [x20] /* host head */ + ldr w19, [x17, #4] /* producer tail */ + sub w22, w19, w20 + cmp w22, #31 /* near-full => batched HVC drain */ + b.hs munmap_pre_bail + ldr x18, [x17, #8] /* producer byte sequence */ + + /* Join the per-vCPU side of the host-writer handshake. The second gate + * read closes the check-vs-announce race: if host closed it in between, + * withdraw without touching a PTE. + */ + mrs x20, tpidr_el1 + add x20, x20, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, munmap_pre_bail + mov w22, #1 + add x25, x17, #24 + stlr w22, [x25] /* producer_active = 1 */ + ldar w22, [x20] + cbnz w22, munmap_active_bail + + /* Locate the one arena generation containing the whole range. Scanning + * 64 cache-cold descriptors is only the cross-vCPU case; the owner's + * control normally matches early. generation is re-read after bounds so + * a descriptor publication can never be observed torn. + */ + mrs x25, tpidr_el1 + add x25, x25, #0x20, lsl #12 /* control[0] */ + mov w24, #0 /* arena slot */ +1: add x26, x25, #8 + ldar w23, [x26] /* arena flags */ + tbz w23, #0, 2f + ldar w22, [x25] /* generation snapshot */ + ldr x26, [x25, #24] /* arena base */ + cmp x14, x26 + b.lo 2f + add x27, x25, #40 + ldar x27, [x27] /* consumed cursor */ + cmp x16, x27 + b.hi 2f + ldar w28, [x25] + cmp w22, w28 + b.eq munmap_arena_found +2: add w24, w24, #1 + add x25, x25, #0x800 + cmp w24, #64 + b.lo 1b + b munmap_active_bail + +munmap_arena_found: + /* Reserve and fill, but do not advance tail until PTE+TLBI completion. */ + and w20, w19, #31 + add x20, x20, x20, lsl #1 /* index * 3 */ + add x26, x17, #32 /* retire.entries */ + add x26, x26, x20, lsl #3 /* index * 24 */ + str x14, [x26, #0] + str x15, [x26, #8] + str w22, [x26, #16] + str w24, [x26, #20] + + /* Refill invalidates every stale descriptor before publishing a new + * generation. Until the host records a lazy materialization in that + * generation, the entire arena is known PTE-empty and no walk/TLBI is + * needed. The host PT gate makes the generation marker stable here. + */ + add x20, x25, #0x388 + ldar w20, [x20] + cmp w20, w22 + b.ne munmap_publish + ldr x11, [x25, #0x390] /* materialized lower bound */ + ldr x12, [x25, #0x398] /* materialized upper bound */ + cmp x11, x14 + csel x11, x11, x14, hi /* max(addr, materialized_start) */ + cmp x12, x16 + csel x12, x12, x16, lo /* min(end, materialized_end) */ + cmp x11, x12 + b.hs munmap_publish + + /* Common large-anonymous case: the materialized envelope is 2 MiB + * aligned and consists only of L2 block descriptors (or holes retired by + * a sibling). Walk L0 once, then L1 once per 1 GiB window and scan the L2 + * entries linearly. The generic path below remains the fallback for any + * L3 table or unusual upper-level shape. + */ + mov w10, #0 /* generic mutation by default */ + orr x20, x11, x12 + tst x20, #0x1fffff + b.ne munmap_validate_generic + sub x20, x12, #1 + eor x20, x20, x11 + lsr x20, x20, #39 /* must stay in one L0 slot */ + cbnz x20, munmap_validate_generic + + mov x13, #0 + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 /* L0 table */ + ubfx x26, x11, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_l2_fast_done + cmp x28, #3 + b.ne munmap_validate_generic + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 /* L1 table */ + + mov x20, x11 +munmap_validate_l2_fast_window: + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + lsr x29, x20, #30 + add x29, x29, #1 + lsl x30, x29, #30 /* next 1 GiB boundary */ + cmp x30, x12 + csel x30, x30, x12, lo /* this window's end */ + cbz x28, munmap_validate_l2_fast_next_window + cmp x28, #3 + b.ne munmap_validate_generic + ubfx x26, x27, #12, #36 + lsl x26, x26, #12 /* L2 table */ + ubfx x29, x20, #21, #9 + add x26, x26, x29, lsl #3 +munmap_validate_l2_fast_entry: + ldr x27, [x26] + and x28, x27, #3 + cbz x28, 1f + cmp x28, #1 /* L2 block, never L3 table */ + b.ne munmap_validate_generic + add x13, x13, #512 +1: add x26, x26, #8 + add x20, x20, #0x200000 + cmp x20, x30 + b.lo munmap_validate_l2_fast_entry + b 2f +munmap_validate_l2_fast_next_window: + mov x20, x30 +2: cmp x20, x12 + b.lo munmap_validate_l2_fast_window +munmap_validate_l2_fast_done: + mov w10, #1 + b munmap_clear_begin + + /* Validation pass. x20 is the current VA; x13 counts valid target pages. + * Pending bytes are charged by materialized backing rather than virtual + * length. Crossing the soft threshold records cleanup_requested but never + * forces this producer to HVC; another natural exit will consume the ring. + */ +munmap_validate_generic: + mov w10, #0 + mov x13, #0 + mov x20, x11 +munmap_validate_block: + cmp x20, x12 + b.hs munmap_clear_begin + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.ne munmap_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.ne munmap_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #21, #9 + add x26, x25, x26, lsl #3 /* &L2[index] */ + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.eq munmap_validate_l3 + cmp x28, #1 + b.ne munmap_active_bail + /* An L2 block can only be cleared as a whole. */ + lsr x29, x20, #21 + lsl x29, x29, #21 /* block start */ + cmp x14, x29 + b.hi munmap_active_bail + add x30, x29, #0x200000 + cmp x16, x30 + b.lo munmap_active_bail + add x13, x13, #512 + b munmap_validate_advance + +munmap_validate_l3: + ubfx x25, x27, #12, #36 /* L3 table VA */ + lsl x25, x25, #12 + lsr x29, x20, #21 + lsl x29, x29, #21 + add x30, x29, #0x200000 + cmp x30, x12 + csel x30, x30, x12, lo +1: ubfx x26, x20, #12, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + tbz x27, #0, 2f + add x13, x13, #1 +2: add x20, x20, #0x1000 + cmp x20, x30 + b.lo 1b + b munmap_validate_block + +munmap_validate_absent: + lsr x29, x20, #21 + add x29, x29, #1 + lsl x20, x29, #21 + b munmap_validate_block +munmap_validate_advance: + mov x20, x30 + b munmap_validate_block + + /* Mutation pass. Host writers are gated; sibling fast munmaps can only + * change the same descriptors to zero, so repeated invalidation is safe. + */ +munmap_clear_begin: + /* Encode the charged 4 KiB page count in the retire flags so the host's + * single-writer consumed sequence advances by exactly the same amount. + */ + and w20, w19, #31 + add x20, x20, x20, lsl #1 + add x26, x17, #32 + add x26, x26, x20, lsl #3 + lsl w29, w13, #6 + orr w29, w29, w24 + str w29, [x26, #20] + cbz x13, munmap_publish + + add x20, x17, #16 + ldar x20, [x20] /* host-consumed byte sequence */ + sub x29, x18, x20 /* pending materialized bytes */ + lsl x30, x13, #12 /* this retirement's charge */ + adds x29, x29, x30 + b.cs munmap_active_bail + adds x18, x18, x30 /* next producer byte sequence */ + b.cs munmap_active_bail + mov x20, #0x10000000 /* 256 MiB soft advisory */ + cmp x29, x20 + b.ls 1f + mov w20, #1 + add x25, x17, #28 + stlr w20, [x25] /* never makes this vCPU exit */ +1: + + cbnz w10, munmap_clear_l2_fast_begin + mov x20, x11 +munmap_clear_block: + cmp x20, x12 + b.hs munmap_tlbi + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne munmap_clear_next_block + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne munmap_clear_next_block + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #21, #9 + add x26, x25, x26, lsl #3 /* &L2[index] */ + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_clear_next_block + cmp x28, #1 + b.eq munmap_clear_l2 + cmp x28, #3 + b.ne munmap_active_bail /* validation invariant */ + + ubfx x25, x27, #12, #36 /* L3 table VA */ + lsl x25, x25, #12 + lsr x29, x20, #21 + lsl x29, x29, #21 /* block start */ + add x30, x29, #0x200000 + cmp x30, x12 + csel x30, x30, x12, lo /* page-loop end */ +munmap_clear_l3_page: + ubfx x26, x20, #12, #9 + add x26, x25, x26, lsl #3 + stlr xzr, [x26] + add x20, x20, #0x1000 + cmp x20, x30 + b.lo munmap_clear_l3_page + b munmap_clear_block + +munmap_clear_l2: + stlr xzr, [x26] +munmap_clear_next_block: + lsr x29, x20, #21 + add x29, x29, #1 + lsl x20, x29, #21 + b munmap_clear_block + +munmap_clear_l2_fast_begin: + /* Validation proved one L0 table and L2-block-only leaves. Re-walk L1 once + * per 1 GiB window, then use ordinary aligned 64-bit stores. Those stores + * are atomic; the DSB ISHST below supplies the required publication order, + * so a release store on every descriptor is unnecessary. + */ + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x11, #39, #9 + add x26, x25, x26, lsl #3 + ldr x27, [x26] + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 /* L1 table */ + mov x20, x11 +munmap_clear_l2_fast_window: + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldr x27, [x26] + and x28, x27, #3 + lsr x29, x20, #30 + add x29, x29, #1 + lsl x30, x29, #30 + cmp x30, x12 + csel x30, x30, x12, lo + cbz x28, munmap_clear_l2_fast_next_window + ubfx x26, x27, #12, #36 + lsl x26, x26, #12 + ubfx x29, x20, #21, #9 + add x26, x26, x29, lsl #3 +munmap_clear_l2_fast_entry: + /* Validation plus the closed host gate proves this slot is a block or + * already zero. A sibling producer can only move it toward zero, so an + * unconditional aligned store is safe and avoids a second load/branch. + */ + str xzr, [x26] + add x26, x26, #8 + add x20, x20, #0x200000 + cmp x20, x30 + b.lo munmap_clear_l2_fast_entry + b 2f +munmap_clear_l2_fast_next_window: + mov x20, x30 +2: cmp x20, x12 + b.lo munmap_clear_l2_fast_window + b munmap_tlbi + +munmap_tlbi: + /* Descriptor stores -> broadcast invalidation -> completed visibility, + * before the retire tail release makes metadata cleanup eligible. + */ + dsb ishst + sub x20, x12, x11 + lsr x20, x20, #12 /* changed-page envelope */ + cmp x20, #8 + b.hi munmap_tlbi_large + ubfx x25, x11, #12, #44 +3: tlbi vae1is, x25 + add x25, x25, #1 + subs x20, x20, #1 + b.ne 3b + b munmap_tlbi_done + +munmap_tlbi_large: + /* Encode SCALE=0..3 RVAE1IS batches directly from the changed envelope. + * Each instruction covers up to 32 scale units: 64 pages at SCALE=0, + * 2048 at SCALE=1, 65536 at SCALE=2, and 2097152 (8 GiB) at SCALE=3. + * Widening to the scale-unit boundary only invalidates neighboring TLB + * entries; it cannot change mappings. Ranges larger than one instruction + * are emitted as adjacent batches while retaining a single completion DSB. + */ + tbz w23, #1, munmap_tlbi_full + cmp x20, #64 + b.hi 1f + mov x27, #13 /* 2 pages = 8 KiB */ + mov x28, #0 /* SCALE=0 */ + b munmap_tlbi_rvae_normalize +1: cmp x20, #0x800 + b.hi 2f + mov x27, #18 /* 64 pages = 256 KiB */ + mov x28, #1 /* SCALE=1 */ + b munmap_tlbi_rvae_normalize +2: mov x29, #0x10000 + cmp x20, x29 + b.hi 3f + mov x27, #23 /* 2048 pages = 8 MiB */ + mov x28, #2 /* SCALE=2 */ + b munmap_tlbi_rvae_normalize +3: mov x27, #28 /* 65536 pages = 256 MiB */ + mov x28, #3 /* SCALE=3 */ + +munmap_tlbi_rvae_normalize: + mov x29, #1 + lsl x29, x29, x27 + sub x29, x29, #1 /* scale-unit byte mask */ + bic x25, x11, x29 /* widened start */ + adds x26, x12, x29 + b.cs munmap_tlbi_full + bic x26, x26, x29 /* widened end */ +munmap_tlbi_rvae_batch: + sub x20, x26, x25 + lsrv x20, x20, x27 /* remaining scale units */ + mov x30, #32 + cmp x20, x30 + csel x20, x20, x30, lo /* this instruction's units */ + ubfx x29, x25, #12, #37 /* BaseADDR */ + sub x30, x20, #1 /* NUM */ + lsl x30, x30, #39 + orr x29, x29, x30 + lsl x30, x28, #44 /* SCALE */ + orr x29, x29, x30 + mov x30, #1 + lsl x30, x30, #46 /* TG=01 (4 KiB) */ + orr x29, x29, x30 + tlbi rvae1is, x29 + lslv x20, x20, x27 + add x25, x25, x20 + cmp x25, x26 + b.lo munmap_tlbi_rvae_batch + b munmap_tlbi_done +munmap_tlbi_full: + tlbi vmalle1is +munmap_tlbi_done: + dsb ish + isb + +munmap_publish: + str x18, [x17, #8] /* publish producer byte sequence */ + add w19, w19, #1 + add x20, x17, #4 + stlr w19, [x20] /* publish retire record */ + add x20, x17, #24 + stlr wzr, [x20] /* leave writer handshake */ + mov x0, #0 + b svc_restore_eret + +munmap_active_bail: + add x20, x17, #24 + stlr wzr, [x20] +munmap_pre_bail: + b handle_svc_0 + identity_class_fast: mrs x12, tpidr_el1 /* shim-globals base */ ldar w13, [x12] /* attention flag, acquire */ diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index 7cf6973c..1e395b55 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -260,6 +260,7 @@ int fork_child_main(int ipc_fd, guest_destroy(&g); return 1; } + guest_rebuild_pte_present(&g); if (fork_ipc_recv_fd_table(ipc_fd, &g) < 0) { log_error("fork-child: failed to receive fd table"); @@ -1396,6 +1397,17 @@ int64_t sys_clone(hv_vcpu_t vcpu, if ((flags & ~(uint64_t) 0xff) & LINUX_CLONE3_NS_FLAGS) return -LINUX_EINVAL; + /* Once an anonymous arena allocation becomes a live thread stack, munmap + * must pass through thread_collect_and_defer_stack_ranges(). Revoke arena + * generations before publishing the stack to the thread table so no EL1 + * fast munmap can bypass that lifetime rule. + */ + if (child_stack != 0) { + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); + } + /* CLONE_THREAD: create a new thread in the same VM (not a new process) */ if (flags & LINUX_CLONE_THREAD) { return sys_clone_thread(vcpu, g, flags, child_stack, stack_map_start, diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 176a7d3f..6f598c7f 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -56,6 +56,11 @@ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ void mmap_lock_acquire(guest_t *g); void mmap_lock_release(void); void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); +/* Temporarily drop mmap_lock while retaining the host PT-gate reference, then + * reacquire without taking a second reference. Used only by lazy zeroing. + */ +void mmap_lock_drop_keep_gate(void); +void mmap_lock_reacquire_with_gate(guest_t *g); /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 9365ec07..2a4d3b33 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "debug/log.h" #include "debug/syscall-hist.h" @@ -46,6 +47,20 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t min_addr, uint64_t max_addr, uint64_t align); +static void mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, shim_mmap_control_t *c); + +/* A host-VA replacement has a fixed cost: sibling quiesce plus an exact HVF + * stage-2 unmap/remap. Native measurements on Apple M4 leave a comfortable + * margin over memset once at least 32 MiB of materialized backing can be + * discarded at once. This is independent of EL1's pending-byte soft advisory: + * crossing that threshold never makes the munmap producer exit. + */ +#define MUNMAP_HVF_REPLACE_THRESHOLD (32ULL * 1024 * 1024) + +static int hvf_replace_slab_zero_range_quiesced(guest_t *g, + uint64_t ipa, + uint64_t len); static void mmap_fastpath_read_env(void) { @@ -72,6 +87,53 @@ static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); } +static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) +{ + if (!g || !g->host_base) + return NULL; + return (_Atomic uint32_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_GATE_OFF); +} + +/* mmap_lock serializes host writers. The gate extends that exclusion to EL1 + * fast munmap without making the per-vCPU producers contend with each other: + * after publishing gate=closed, wait for each producer's private active word. + */ +static void mmap_fastpath_host_gate_close(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate) + return; + uint32_t previous = + atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); + if (previous != 0) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + while (atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0) + sched_yield(); + } +} + +static void mmap_fastpath_host_gate_open(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (gate) { + uint32_t count = atomic_load_explicit(gate, memory_order_relaxed); + while (count != 0 && !atomic_compare_exchange_weak_explicit( + gate, &count, count - 1, memory_order_release, + memory_order_relaxed)) { + } + /* exec resets the entire shim-data page while holding mmap_lock, + * including this implementation-only counter. Seeing zero here is + * therefore an already-open gate, not an underflow. + */ + } +} + +static _Thread_local guest_t *mmap_lock_guest; + static void mmap_fastpath_disable_control(shim_mmap_control_t *c) { uint32_t generation = @@ -82,12 +144,16 @@ static void mmap_fastpath_disable_control(shim_mmap_control_t *c) atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, + memory_order_relaxed); c->next_arena_size = MMAP_FAST_ARENA_MIN; c->max_len_seen = 0; atomic_store_explicit(&c->generation, generation, memory_order_release); } -void mmap_fastpath_drain_locked(guest_t *g) +static void mmap_fastpath_drain_publications_locked(guest_t *g) { if (!g || !g->host_base) return; @@ -146,26 +212,364 @@ void mmap_fastpath_drain_locked(guest_t *g) } } +/* Return true only when every semantic mapping of this backing range is part + * of the VA retirement being committed. Fast mmap arenas are identity mapped + * today, but retaining this check prevents a future high-VA alias from having + * its backing replaced underneath a still-live PTE. + */ +static bool munmap_retire_backing_is_exclusive(const guest_t *g, + uint64_t backing_start, + uint64_t backing_end, + uint64_t retire_start, + uint64_t retire_end) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + uint64_t rlen = r->end - r->start; + if (r->gpa_base > UINT64_MAX - rlen) + return false; + uint64_t r_gpa_end = r->gpa_base + rlen; + uint64_t lo = backing_start > r->gpa_base ? backing_start + : r->gpa_base; + uint64_t hi = backing_end < r_gpa_end ? backing_end : r_gpa_end; + if (hi <= lo) + continue; + + uint64_t mapped_start = r->start + (lo - r->gpa_base); + uint64_t mapped_end = mapped_start + (hi - lo); + if (mapped_start < retire_start || mapped_end > retire_end) + return false; + } + return true; +} + +static void munmap_retire_commit_locked(guest_t *g, + const munmap_retire_entry_t *e, + uint64_t backing_start, + uint64_t backing_end, + uint64_t charged_bytes) +{ + uint64_t start = e->addr; + uint64_t end = start + e->length; + + /* Publication drain ran first, so every mapping causally preceding this + * retirement is now represented in regions[]. A non-anonymous overlay in + * an arena indicates a missing revocation and must fail closed: EL1 has + * already invalidated the PTEs, so silently retaining such metadata would + * permit a later fault path to recreate them. + */ + for (int i = guest_region_first_end_above(g, start); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= start) + continue; + if (!(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) { + log_fatal( + "munmap retire: non-fast mapping in arena " + "[0x%llx-0x%llx)", + (unsigned long long) start, (unsigned long long) end); + } + } + + guest_materialize_wait_range_locked(g, start, end); + + /* The PTE occupancy evidence was consumed by EL1, so use the conservative + * dirty bitmap to avoid touching huge never-materialized reservations. + * Full, contiguous dirty runs at or above 32 MiB are cheaper to discard by + * replacing their zero-fill backing while their HVF stage-2 segments are + * detached. Partial blocks and shorter runs retain the inline memset path. + * charged_bytes is the EL1 producer's page-accurate materialized count, so + * a sparse virtual retirement cannot trigger replacement merely because + * its address span is large. + */ + bool allow_replace = charged_bytes >= MUNMAP_HVF_REPLACE_THRESHOLD; + bool siblings_quiesced = false; + uint64_t block = ALIGN_DOWN(backing_start, BLOCK_2MIB); + while (block < backing_end) { + uint64_t block_end = block + BLOCK_2MIB; + if (guest_block_may_be_dirty(g, block)) { + uint64_t lo = backing_start > block ? backing_start : block; + uint64_t hi = + backing_end < block_end ? backing_end : block_end; + + if (allow_replace && lo == block && hi == block_end) { + uint64_t run_end = block_end; + while (run_end < backing_end && + run_end <= UINT64_MAX - BLOCK_2MIB && + guest_block_may_be_dirty(g, run_end)) + run_end += BLOCK_2MIB; + + if (run_end - block >= MUNMAP_HVF_REPLACE_THRESHOLD && + munmap_retire_backing_is_exclusive( + g, block, run_end, start, end)) { + if (!siblings_quiesced) { + thread_quiesce_siblings(); + siblings_quiesced = true; + } + if (hvf_replace_slab_zero_range_quiesced( + g, block, run_end - block) == 0) { + guest_dirty_clear_zeroed_range(g, block, run_end); + block = run_end; + continue; + } + } + } + + memset((uint8_t *) g->host_base + lo, 0, hi - lo); + guest_dirty_clear_zeroed_range(g, lo, hi); + } + block = block_end; + } + if (siblings_quiesced) + thread_resume_siblings(); + + guest_region_remove(g, start, end); + if (backing_end > backing_start) + guest_retire_ptes_committed(g, backing_start, backing_end); + if (start < g->mmap_rw_gap_hint) + g->mmap_rw_gap_hint = start; + if (start < g->mmap_rx_gap_hint) + g->mmap_rx_gap_hint = start; +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + /* Acquire-snapshot every retirement tail before consuming any mmap + * publication. This is the cross-vCPU causal ordering required for + * "A mmap; publish pointer; B munmap": the acquire observes B's retire, + * then publication drain establishes A's semantic region before removal. + */ + uint32_t retire_tails[MAX_THREADS]; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + retire_tails[slot] = + atomic_load_explicit(&c->retire.tail, memory_order_acquire); + } + + mmap_fastpath_drain_publications_locked(g); + + bool retired_any = false; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = retire_tails[slot]; + if ((uint32_t) (tail - head) > SHIM_MUNMAP_RETIRE_RING_SIZE) { + log_fatal( + "munmap retire: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + while (head != tail) { + const munmap_retire_entry_t *e = + &producer->retire + .entries[head & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + uint32_t arena_slot = + e->flags & SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK; + uint64_t charged_pages = + (e->flags & SHIM_MUNMAP_RETIRE_F_CHARGE_MASK) >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT; + uint64_t charged_bytes = charged_pages * GUEST_PAGE_SIZE; + if (arena_slot >= MAX_THREADS || !e->length || + (e->addr & (GUEST_PAGE_SIZE - 1)) || + (e->length & (GUEST_PAGE_SIZE - 1)) || + e->addr > UINT64_MAX - e->length || + charged_bytes > e->length) { + log_fatal("munmap retire: invalid entry in vCPU slot %d", slot); + } + + shim_mmap_control_t *arena = + mmap_fastpath_control(g, (int) arena_slot); + uint32_t generation = + atomic_load_explicit(&arena->generation, memory_order_acquire); + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + uint64_t end = e->addr + e->length; + if (generation != e->arena_generation || e->addr < base || + end > cursor) { + log_fatal( + "munmap retire: stale arena generation/range " + "(producer=%d arena=%u gen=%u/%u)", + slot, arena_slot, e->arena_generation, generation); + } + + uint64_t backing_start = 0, backing_end = 0; + if (charged_bytes != 0) { + backing_start = atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed); + backing_end = atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed); + if (backing_start < e->addr) + backing_start = e->addr; + if (backing_end > end) + backing_end = end; + if (backing_end <= backing_start) + log_fatal( + "munmap retire: charged entry has no materialized " + "bounds"); + } + + munmap_retire_commit_locked(g, e, backing_start, backing_end, + charged_bytes); + /* If that retirement removed the arena's final live descriptor, + * restore the whole-arena PTE-empty proof. This prevents historical + * materialization bounds from growing across sequential sparse + * mappings in the same bump arena. + */ + if (guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + if (current_thread && + arena_slot == (uint32_t) current_thread->sp_el1_slot) + mmap_fastpath_rewind_control_if_clean_locked(g, arena); + uint64_t consumed = atomic_load_explicit( + &producer->retire.consumed_bytes, memory_order_relaxed); + atomic_store_explicit(&producer->retire.consumed_bytes, + consumed + charged_bytes, + memory_order_release); + head++; + retired_any = true; + } + atomic_store_explicit(&producer->retire.head, head, + memory_order_release); + /* The PT gate is closed while draining, so no producer can race this + * acknowledgement. Ring fullness remains the only hard per-vCPU + * backpressure; byte pressure is deliberately advisory. + */ + atomic_store_explicit(&producer->retire.cleanup_requested, 0, + memory_order_release); + } + + /* EL1 PTE stores cannot update guest_t's host-only cache generation. One + * bump per batch invalidates every host GVA translation cache after all + * retirement entries have committed. + */ + if (retired_any) + guest_pt_gen_bump(g); +} + void mmap_lock_acquire(guest_t *g) { pthread_mutex_lock(&mmap_lock); + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; mmap_fastpath_drain_locked(g); } void mmap_lock_release(void) { + mmap_fastpath_host_gate_open(mmap_lock_guest); + mmap_lock_guest = NULL; pthread_mutex_unlock(&mmap_lock); } void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) { + mmap_fastpath_host_gate_open(g); + mmap_lock_guest = NULL; pthread_cond_wait(cond, &mmap_lock); /* pthread_cond_wait reacquires mmap_lock directly, so preserve the * drain-before-region-read invariant of mmap_lock_acquire(). */ + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; mmap_fastpath_drain_locked(g); } +void mmap_lock_drop_keep_gate(void) +{ + /* Dirty lazy-materialization drops mmap_lock around a potentially large + * memset. Retain this thread's gate reference so EL1 cannot retire the + * invalid PTE window and let the materializer recreate it afterwards. + * Another host thread may temporarily acquire mmap_lock; the refcounted + * gate remains closed until this owner finishes the materialization. + */ + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_reacquire_with_gate(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_lock_guest = g; + /* EL1 mmap publication does not need the PT gate and may have progressed + * during the memset, so refresh semantic metadata before resuming. + */ + mmap_fastpath_drain_locked(g); +} + +void mmap_fastpath_drain_vmexit(guest_t *g) +{ + mmap_lock_acquire(g); + mmap_lock_release(); +} + +bool mmap_fastpath_current_producer_active(const guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return false; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + return c && atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0; +} + +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start >= limit || end <= base) + continue; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed); + uint64_t lo = start > base ? start : base; + uint64_t hi = end < limit ? end : limit; + uint32_t materialized = atomic_load_explicit( + &c->materialized_generation, memory_order_relaxed); + if (materialized == generation) { + uint64_t old_lo = atomic_load_explicit( + &c->materialized_start, memory_order_relaxed); + uint64_t old_hi = atomic_load_explicit( + &c->materialized_end, memory_order_relaxed); + if (old_lo < lo) + lo = old_lo; + if (old_hi > hi) + hi = old_hi; + } + atomic_store_explicit(&c->materialized_start, lo, + memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, hi, + memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, generation, + memory_order_release); + } +} + static bool mmap_fastpath_request_fits(uint64_t cursor, uint64_t limit, uint64_t len) @@ -202,10 +606,10 @@ static uint64_t mmap_fastpath_arena_size(uint64_t max_len_seen, { uint64_t adaptive = MMAP_FAST_ARENA_MIN; if (max_len_seen) { - uint64_t target = - max_len_seen > MMAP_FAST_ARENA_MAX / MMAP_FAST_HISTORY_MULTIPLIER - ? MMAP_FAST_ARENA_MAX - : max_len_seen * MMAP_FAST_HISTORY_MULTIPLIER; + const uint64_t target_entries = MMAP_FAST_ARENA_TARGET_ENTRIES; + uint64_t target = max_len_seen > MMAP_FAST_ARENA_MAX / target_entries + ? MMAP_FAST_ARENA_MAX + : max_len_seen * target_entries; adaptive = mmap_fastpath_pow2_clamped(target); } @@ -310,6 +714,10 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, + memory_order_relaxed); c->next_arena_size = arena_size; c->max_len_seen = 0; c->refill_count++; @@ -317,8 +725,10 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, c->recycle_count++; if (arena_size > c->peak_arena_size) c->peak_arena_size = arena_size; - atomic_store_explicit(&c->flags, SHIM_MMAP_CTRL_ENABLED, - memory_order_relaxed); + uint32_t control_flags = SHIM_MMAP_CTRL_ENABLED; + if (g_tlbi_range_supported) + control_flags |= SHIM_MMAP_CTRL_TLBIRANGE; + atomic_store_explicit(&c->flags, control_flags, memory_order_relaxed); /* This vCPU is stopped in HVC (or has never run), so host may acknowledge * the freshly published descriptor on its behalf. Revocation deliberately * does not do this, making an in-flight stale generation bail once. @@ -333,6 +743,47 @@ void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) mmap_fastpath_refill_thread_locked(g, current_thread, request_len); } +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_fastpath_refill_thread_locked(g, current_thread, request_len); + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + return false; + uint64_t end = start + request_len; + + if (guest_region_add_ex(g, start, end, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) + return false; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + return true; +} + void mmap_fastpath_release_current_hint_locked(guest_t *g, uint64_t addr, uint64_t length) @@ -359,6 +810,57 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, mmap_fastpath_disable_control(c); } +/* Reuse a fully released arena in place. mmap_lock acquisition has drained + * this vCPU's publication ring, and sys_munmap has removed the last semantic + * region before calling here. The PTE occupancy index is the final guard: an + * arena is rewound only when no live metadata and no valid descriptor remain. + * The owner is stopped in HVC, so resetting its private bump cursor cannot + * race EL1. Keeping base/limit/generation unchanged avoids a host refill on + * the next mmap -- especially important when one 32 GiB request consumes the + * entire maximum-sized arena. + */ +static void mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, shim_mmap_control_t *c) +{ + if (!g || !c) + return; + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + if (base >= limit || cursor <= base) + return; + + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= limit) + break; + if (r->end > base) + return; + } + if (guest_va_next_present_block(g, base, limit) < limit) + return; + + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, + memory_order_relaxed); + c->recycle_count++; +} + +static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return; + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); +} + void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) { mmap_lock_acquire(g); @@ -2009,6 +2511,101 @@ static void hvf_remap_segments_best_effort(guest_t *g, } } +/* Replace a retired slab range with fresh zero-fill pages and refresh HVF's + * cached host VA->PA translation. Split only the two range boundaries, just + * as the file-overlay path does, so the expensive detach/remap covers the + * retired bytes rather than an initial slab-sized HVF segment. Reusing the + * same arena bounds is a no-op on later calls. If many distinct boundaries + * exhaust GUEST_MAX_HVF_SEGMENTS, splitting fails safely and the caller falls + * back to memset. + * + * The shared-slab case punches a hole in the backing file before re-establishing + * its MAP_SHARED host mapping. This keeps future clonefile fork snapshots in + * sync with the new zero state. A private slab can be replaced directly by a + * fresh MAP_ANON mapping. + * + * Caller holds mmap_lock, has invalidated every stage-1 PTE in the target, and + * has quiesced sibling vCPUs. + */ +static int hvf_replace_slab_zero_range_quiesced(guest_t *g, + uint64_t ipa, + uint64_t len) +{ + if (!g || !len || (ipa & (BLOCK_2MIB - 1)) || + (len & (BLOCK_2MIB - 1)) || ipa >= g->guest_size || + len > g->guest_size - ipa) + return -LINUX_EINVAL; + + uint64_t end = ipa + len; + hvf_segment_t segments[GUEST_MAX_HVF_SEGMENTS]; + int err = hvf_segment_split_range_boundaries(g, ipa, end); + if (err < 0) + return err; + int nsegments = hvf_segment_collect_range( + g, ipa, end, segments, GUEST_MAX_HVF_SEGMENTS); + if (nsegments < 0) + return nsegments; + + int unmapped = 0; + for (int i = 0; i < nsegments; i++) { + if (hv_vm_unmap(segments[i].ipa, segments[i].len) != HV_SUCCESS) { + hvf_remap_segments_best_effort(g, segments, unmapped); + return -LINUX_EIO; + } + unmapped++; + } + + err = 0; + if (g->shm_fd >= 0) { + struct fpunchhole hole = { + .fp_flags = 0, + .reserved = 0, + .fp_offset = (off_t) ipa, + .fp_length = (off_t) len, + }; + if (fcntl(g->shm_fd, F_PUNCHHOLE, &hole) < 0) + err = (int) linux_errno(); + } + + if (err == 0) { + void *target = (uint8_t *) g->host_base + ipa; + int flags = MAP_FIXED; + int fd = -1; + off_t offset = 0; + if (g->shm_fd >= 0) { + flags |= MAP_SHARED; + fd = g->shm_fd; + offset = (off_t) ipa; + } else { + flags |= MAP_ANON | MAP_PRIVATE; + } + if (mmap(target, len, PROT_READ | PROT_WRITE, flags, fd, offset) == + MAP_FAILED) + err = (int) linux_errno(); + } + + /* Whether replacement succeeded or failed, restore every detached segment. + * On a remap failure, retry the failed and remaining segments best-effort; + * already-restored prefixes must not be mapped twice. + */ + for (int i = 0; i < nsegments; i++) { + hv_return_t r = + hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, + segments[i].ipa, segments[i].len, HVF_SEGMENT_FLAGS); + if (r == HV_SUCCESS) + continue; + log_error( + "munmap retire: hv_vm_map(0x%llx, 0x%llx) failed with 0x%x " + "after backing replacement", + (unsigned long long) segments[i].ipa, + (unsigned long long) segments[i].len, (int) r); + hvf_remap_segments_best_effort(g, &segments[i], nsegments - i); + return -LINUX_EIO; + } + + return err; +} + /* Apply a real MAP_SHARED file overlay at [ipa, ipa+len) backed by [fd, * file_off). The IPA range may be sub-2 MiB; the containing 2 MiB segment is * split out first if it is not already isolated. Caller holds mmap_lock and has @@ -2457,6 +3054,14 @@ int64_t sys_mmap(guest_t *g, */ bool is_noreplace = (flags & LINUX_MAP_FIXED_NOREPLACE) != 0; + /* A fixed mapping may replace an address previously handed out by an EL1 + * arena with a file/shared/stack-like mapping. Revoke all descriptors + * before making that semantic transition so a later fast munmap cannot + * classify it from the stale arena bounds. + */ + if (is_fixed) + mmap_fastpath_revoke_all_locked(g, false); + uint64_t result_off; /* Result as offset (0-based) */ if (is_fixed) { /* Addresses above TASK_SIZE (bit 63 set or beyond user VA range) are @@ -4029,8 +4634,9 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) } for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { if (!guest_va_block_mapped(g, b)) { - /* Skip absent 1GiB/512GiB slots wholesale; a huge untouched - * reservation would otherwise pay one walk per 2MiB. + /* Jump through the PTE occupancy index to the next materialized + * block. A huge untouched reservation therefore does no work + * proportional to its virtual length. */ b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); continue; @@ -4056,7 +4662,8 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (guest_invalidate_ptes(g, unmap_off, end) < 0) return -LINUX_ENOMEM; for (int i = 0; i < nzr; i++) { - memset((uint8_t *) g->host_base + zr[i].lo, 0, zr[i].hi - zr[i].lo); + memset((uint8_t *) g->host_base + zr[i].lo, 0, + zr[i].hi - zr[i].lo); guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove(g, unmap_off, end); @@ -4163,6 +4770,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) thread_finish_deferred_stack_ranges(txns, nranges); } } + mmap_fastpath_rewind_current_if_clean_locked(g); return 0; } @@ -4190,6 +4798,12 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; + /* Permission and VMA-shape changes are slow-path boundaries. Retire any + * already-published unmaps, then invalidate arena generations before the + * metadata/PTE edit so EL1 cannot act on the old anonymous classification. + */ + mmap_fastpath_revoke_all_locked(g, false); + if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { uint64_t mprot_end = addr + length; diff --git a/src/syscall/proc.c b/src/syscall/proc.c index 003a202a..13eb275e 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -37,6 +37,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/futex.h" @@ -1830,6 +1831,21 @@ int vcpu_run_loop(hv_vcpu_t vcpu, drain_external_guest_signal(); + /* Every return from HVF is a natural retirement point. Drain before + * dispatching syscalls, page faults, MAP_FIXED, fork/exec, signals, or + * exit so no host path can consult pre-munmap region metadata and + * rematerialize an EL1-invalidated page. The helper also drains mmap + * publications before the acquire-snapshotted retire entries. + */ + bool munmap_producer_active = mmap_fastpath_current_producer_active(g); + if (!munmap_producer_active) + mmap_fastpath_drain_vmexit(g); + else if (vexit->reason == HV_EXIT_REASON_EXCEPTION) + log_error( + "%s: exception exit interrupted an active EL1 munmap " + "producer", + prefix); + /* Main: disarm timeout */ if (is_main) alarm(0); diff --git a/src/syscall/signal.c b/src/syscall/signal.c index bb8957d5..51f23a59 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -30,12 +30,14 @@ #include "hvutil.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/thread.h" #include "syscall/abi.h" #include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/internal.h" #include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" /* proc_get_pid, proc_get_uid, SYSCALL_EXEC_HAPPENED */ #include "syscall/signal.h" @@ -1369,6 +1371,13 @@ int64_t signal_sigaltstack(guest_t *g, uint64_t ss_gva, uint64_t old_ss_gva) } else { if (ss.ss_size < LINUX_MINSIGSTKSZ) return -LINUX_ENOMEM; + /* Alternate stacks have the same lifetime sensitivity as clone + * stacks: once registered, their unmap must take the host path + * instead of being classified only as an anonymous arena range. + */ + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); t->altstack_sp = ss.ss_sp; t->altstack_flags = 0; t->altstack_size = ss.ss_size; diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index 95ade75f..df3e715d 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -973,9 +973,16 @@ static int64_t sc_mmap(guest_t *g, { uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); mmap_lock_acquire(g); - int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - if (r >= 0 && refill_len) - mmap_fastpath_refill_current_locked(g, refill_len); + uint64_t arena_addr = 0; + int64_t r; + if (refill_len && + mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { + r = (int64_t) arena_addr; + } else { + r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + } mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, @@ -994,6 +1001,7 @@ static int64_t sc_mremap(guest_t *g, { (void) x5; mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", diff --git a/tests/bench-mmap-fresh b/tests/bench-mmap-fresh new file mode 100755 index 00000000..554b0b34 --- /dev/null +++ b/tests/bench-mmap-fresh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Accumulate fresh bump-tail mmap timing across independent elfuse processes. +# One process cannot retain enough large fresh mappings to overcome CNTVCT's +# 41.7-ns tick without exhausting guest VA. Each invocation below gets a new +# guest, while its in-guest timer excludes process startup from the measurement. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +# 1024 ticks = about 42.7 us on Apple Silicon. This bounds the aggregate +# counter quantization to roughly 0.1%, while keeping the largest cases timely. +TARGET_TICKS="${BENCH_MMAP_FRESH_TARGET_TICKS:-1024}" +MAX_RUNS="${BENCH_MMAP_FRESH_MAX_RUNS:-8192}" + +# size in bytes : mappings per fresh guest run +CASES=( + "4096:1000" + "65536:1000" + "1048576:1000" + "2097152:1000" + "8388608:256" + "134217728:16" + "1073741824:8" + "8589934592:4" + "34359738368:2" +) + +# Optional positional cases use the same size:count spelling as CASES, e.g. +# tests/bench-mmap-fresh 1073741824:8 8589934592:4 +if [ "$#" -gt 0 ]; then + CASES=("$@") +fi + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +printf 'fresh bump-tail mmap: target %s aggregate CNTVCT ticks\n' "$TARGET_TICKS" +printf '%-10s %8s %6s %14s\n' size count runs 'fresh mmap ns' + +for case in "${CASES[@]}"; do + size="${case%%:*}" + count="${case##*:}" + total_ticks=0 + total_ops=0 + runs=0 + ns_per_tick= + + while awk -v ticks="$total_ticks" -v target="$TARGET_TICKS" \ + 'BEGIN { exit !(ticks < target) }'; do + if [ "$runs" -ge "$MAX_RUNS" ]; then + echo "fresh benchmark exceeded ${MAX_RUNS} runs for ${size}" >&2 + exit 1 + fi + if ! output="$("$ELFUSE" "$BENCH" fresh "$size" "$count" 2>&1)"; then + echo "fresh guest run failed for size=${size}, count=${count}:" >&2 + echo "$output" >&2 + exit 1 + fi + raw_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ticks=/) {sub(/^ticks=/, "", $i); print $i}}' <<< "$output")" + read_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^read_ticks=/) {sub(/^read_ticks=/, "", $i); print $i}}' <<< "$output")" + ns_per_tick="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ns_per_tick=/) {sub(/^ns_per_tick=/, "", $i); print $i}}' <<< "$output")" + if [ -z "$raw_ticks" ] || [ -z "$read_ticks" ] || [ -z "$ns_per_tick" ]; then + echo "unexpected benchmark output: $output" >&2 + exit 1 + fi + total_ticks="$(awk -v total="$total_ticks" -v raw="$raw_ticks" -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + total_ops=$((total_ops + count)) + runs=$((runs + 1)) + done + + ns_per_op="$(awk -v ticks="$total_ticks" -v ops="$total_ops" -v ns="$ns_per_tick" 'BEGIN { printf "%.2f", ticks * ns / ops }')" + human="$(awk -v n="$size" 'BEGIN { if (n >= 1073741824) printf "%g GiB", n / 1073741824; else if (n >= 1048576) printf "%g MiB", n / 1048576; else printf "%g KiB", n / 1024 }')" + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" "$ns_per_op" +done diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index c57e9d03..d75e6c3e 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -15,7 +15,8 @@ * the ~2 us SVC path and swamps any sub-us operation. * * Every case takes one untimed warmup pass (to pay the one-time arena carve and - * page-table extension) and reports the median and min over ITERS runs. + * page-table extension). Most sections report aggregate samples; section H + * retains every operation so its normal latency and long tail remain visible. * * Copyright 2026 elfuse contributors * SPDX-License-Identifier: Apache-2.0 @@ -70,23 +71,45 @@ static double median(double *v, int n) return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); } +/* R-7/sample quantile, matching the common (n - 1) * p interpolation. The + * input must already be sorted. */ +static double sorted_quantile(const double *v, unsigned n, double p) +{ + double pos = (double) (n - 1) * p; + unsigned lo = (unsigned) pos; + unsigned hi = lo + (lo + 1 < n); + return v[lo] + (v[hi] - v[lo]) * (pos - lo); +} + #define ITERS 15 -#define MAXB 64 +#define MIN_TIMED_TICKS 4096 #define KIB (1ULL << 10) #define MIB (1ULL << 20) #define GIB (1ULL << 30) -/* Batch size: amortize the coarse counter over B ops while bounding the live - * address footprint of one timed batch to ~256 MiB. +/* The counter is too coarse to time a single mmap reliably. A section-A + * sample therefore repeats independent mmap -> munmap pairs until the + * aggregate interval is long enough. This avoids batching altogether: each + * allocation is immediately returned to the steady-state free list. + * + * Per-operation timestamps introduce one rd()/rd() interval per result. Its + * median cost is calibrated over long runs and subtracted, so a one-tick + * counter cannot turn a 1-GiB mmap into a spurious 83.3-ns (two-tick) result. */ -static int batch_for(uint64_t size) +static double rd_pair_ticks(void) { - uint64_t b = (256 * MIB) / size; - if (b < 1) - b = 1; - if (b > MAXB) - b = MAXB; - return (int) b; + enum { CAL_SAMPLES = 15, CAL_OPS = 4096 }; + double samples[CAL_SAMPLES]; + for (int sample = 0; sample < CAL_SAMPLES; sample++) { + uint64_t total = 0; + for (int op = 0; op < CAL_OPS; op++) { + uint64_t t0 = rd(); + uint64_t t1 = rd(); + total += t1 - t0; + } + samples[sample] = (double) total / CAL_OPS; + } + return median(samples, CAL_SAMPLES); } static const char *human(uint64_t s, char *buf) @@ -100,10 +123,10 @@ static const char *human(uint64_t s, char *buf) return buf; } -/* A. mmap + munmap latency vs size (steady state) NULL-hint allocate then free, - * batched. Iterations after the first reuse freed address space, so this is the - * realistic repeated-allocation number a workload sees, not the one-shot fresh - * case (that is section B). +/* A. mmap + munmap latency vs size (steady state). Each operation allocates + * with a NULL hint and immediately frees the result. This measures the + * repeated-allocation path a workload sees, not the one-shot fresh case + * (section B), while avoiding a size-dependent batch policy. */ static void bench_size_sweep(void) { @@ -113,52 +136,66 @@ static void bench_size_sweep(void) }; printf( "== A. mmap / munmap latency vs size (steady state, NULL hint) ==\n"); - printf("%-10s %6s %12s %12s\n", "size", "batch", "mmap ns", "munmap ns"); - void *ptr[MAXB]; + double timer_ticks = rd_pair_ticks(); + printf("%-10s %8s %12s %12s\n", "size", "ops", "mmap ns", "munmap ns"); for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { uint64_t size = sizes[s]; - int b = batch_for(size); double mm[ITERS], um[ITERS]; + unsigned reported_ops = 0; int ok = 1; for (int it = -1; it < ITERS && ok; it++) { - uint64_t t0 = rd(); - for (int i = 0; i < b; i++) - ptr[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - uint64_t t1 = rd(); - for (int i = 0; i < b; i++) - if (ptr[i] == MAP_FAILED) + uint64_t mmap_ticks = 0, munmap_ticks = 0; + unsigned ops = 0; + do { + uint64_t t0 = rd(); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + mmap_ticks += t1 - t0; + if (ptr == MAP_FAILED) { + ok = 0; + break; + } + t0 = rd(); + int rc = munmap(ptr, size); + t1 = rd(); + munmap_ticks += t1 - t0; + if (rc != 0) { ok = 0; + break; + } + ops++; + } while (ok && + (mmap_ticks < MIN_TIMED_TICKS || munmap_ticks < MIN_TIMED_TICKS)); if (!ok) break; - uint64_t t2 = rd(); - for (int i = 0; i < b; i++) - munmap(ptr[i], size); - uint64_t t3 = rd(); if (it >= 0) { - mm[it] = ns(t1 - t0) / b; - um[it] = ns(t3 - t2) / b; + mm[it] = ((double) mmap_ticks / ops - timer_ticks) * ns_per_tick; + um[it] = + ((double) munmap_ticks / ops - timer_ticks) * ns_per_tick; + reported_ops = ops; } } char hb[16]; if (!ok) { - printf("%-10s %6d %12s %12s\n", human(size, hb), b, "FAILED", "-"); + printf("%-10s %8s %12s %12s\n", human(size, hb), "-", "FAILED", + "-"); continue; } - printf("%-10s %6d %12.1f %12.1f\n", human(size, hb), b, + printf("%-10s %8u %12.1f %12.1f\n", human(size, hb), reported_ops, median(mm, ITERS), median(um, ITERS)); } printf("\n"); } -/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path) Allocate a - * bounded sequential run WITHOUT freeing, so every mapping lands at or above - * the arena high-water -- exactly the case lazy_fresh_range skips the stale-PTE - * scan for. The run is kept small enough (<= 1000 regions, well under - * GUEST_MAX_REGIONS, footprint <= 2 GiB) that region bookkeeping and page-table - * extension do not dominate, and every result is failure-checked. Run this - * binary against an opt-off build to read the skip's contribution as the - * difference on this identical code path -- a MAP_FIXED "recycled" compare +/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path). Allocate a + * sequential run WITHOUT freeing, so every mapping lands at or above the arena + * high-water -- exactly the case lazy_fresh_range skips the stale-PTE scan for. + * Small mappings use the original 2-GiB footprint cap; large mappings use a + * minimum count chosen to retain multiple samples without exceeding 64 GiB of + * live fresh VA. + * Run this binary against an opt-off build to read the skip's contribution as + * the difference on this identical code path -- a MAP_FIXED "recycled" compare * would instead measure the region-snapshot replacement path, not the skip. */ static void bench_fresh(void) @@ -177,6 +214,12 @@ static void bench_fresh(void) n = 1; if (n > 1000) n = 1000; + if (size == GIB) + n = 8; + else if (size == 8 * GIB) + n = 4; + else if (size == 32 * GIB) + n = 2; /* warmup one fresh mapping so the arena high-water is already primed */ void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -205,6 +248,44 @@ static void bench_fresh(void) printf("\n"); } +/* One fresh bump-tail run for the host-side driver. A new elfuse process is + * used for each invocation, so the driver can accumulate many counter ticks + * without exhausting one guest's VA space. */ +static int bench_fresh_one(uint64_t size, int n) +{ + void *run[1000]; + if (n < 1 || n > (int) (sizeof(run) / sizeof(run[0]))) + return 2; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED) + return 1; + munmap(warmup, size); + + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + if (failed) + return 1; + + printf("fresh size=%llu count=%d ticks=%llu read_ticks=%.6f " + "ns_per_tick=%.12f\n", + (unsigned long long) size, n, (unsigned long long) (t1 - t0), + rd_pair_ticks(), ns_per_tick); + return 0; +} + /* C. first-touch page-fault cost Touch one byte per page at a 16 KiB stride so * no two touches share a macOS host page; every touch is a genuine fault (HVC * #11 -> host fault handler -> page-table install + zero). Reports per-fault @@ -441,9 +522,157 @@ static void bench_mt(void) printf("\n"); } -int main(void) +/* G. Teardown after materialization. The store is deliberately outside the + * timed interval: it takes the lazy first-touch fault and installs the first + * page, then the counter brackets only munmap(). The size sweep matches A, + * so each row compares a wholly untouched mapping with one that has exactly + * one materialized 4-KiB page; touching every page would instead benchmark + * faulting and zeroing gigabytes of memory. */ +static void bench_munmap_materialized(void) +{ + double timer_ticks = rd_pair_ticks(); + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + + printf("== G. munmap after materializing one 4 KiB page ==\n"); + printf("%-10s %8s %12s\n", "size", "ops", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + double unmap_ns[ITERS]; + unsigned reported_ops = 0; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t unmap_ticks = 0; + unsigned ops = 0; + do { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + p[0] = 1; /* materialize before starting the timed interval */ + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + unmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (unmap_ticks < MIN_TIMED_TICKS); + + if (it >= 0 && ok) { + unmap_ns[it] = + ((double) unmap_ticks / ops - timer_ticks) * ns_per_tick; + reported_ops = ops; + } + } + + char hb[16]; + if (!ok) + printf("%-10s %8s %12s\n", human(size, hb), "-", "FAILED"); + else + printf("%-10s %8u %12.1f\n", human(size, hb), reported_ops, + median(unmap_ns, ITERS)); + } + printf("\n"); +} + +/* H. Teardown after every page was dirtied. Page stores happen before the + * timed interval, so this reports only munmap's handling of the materialized, + * dirty mapping. Each size has a fixed operation count and every munmap is + * retained separately. In particular, one slow first operation cannot end an + * adaptive batch and become the whole sample. Counts decrease with size to + * bound total dirtying work. The 1-GiB case uses eight operations so its + * untimed page stores remain below elfuse's vCPU watchdog; interpolated p95 + * still remains distinct from the separately reported maximum. + */ +static void bench_munmap_dirty(void) +{ + static const struct { + uint64_t size; + unsigned ops; + } cases[] = { + {4 * KIB, 2048}, {16 * KIB, 2048}, {64 * KIB, 2048}, + {256 * KIB, 1024}, {MIB, 512}, {2 * MIB, 256}, + {8 * MIB, 128}, {64 * MIB, 32}, {256 * MIB, 24}, + {GIB, 8}, + }; + double timer_ticks = rd_pair_ticks(); + + printf("== H. munmap after dirtying every 4 KiB page ==\n"); + printf("%-10s %8s %12s %12s %12s\n", "size", "ops", "p50 ns", + "p95 ns", "max ns"); + for (unsigned s = 0; s < sizeof(cases) / sizeof(cases[0]); s++) { + uint64_t size = cases[s].size; + unsigned ops = cases[s].ops; + double *unmap_ns = malloc((size_t) ops * sizeof(*unmap_ns)); + int ok = 1; + if (!unmap_ns) + ok = 0; + + /* One full untimed warmup pays setup without consuming an observation. */ + for (int64_t op = -1; op < (int64_t) ops && ok; op++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + for (uint64_t off = 0; off < size; off += 4 * KIB) + p[off] = (uint8_t) (off >> 12); + + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + if (rc != 0) { + ok = 0; + break; + } + if (op >= 0) { + double ticks = (double) (t1 - t0) - timer_ticks; + if (ticks < 0.0) + ticks = 0.0; + unmap_ns[op] = ticks * ns_per_tick; + } + } + + char hb[16]; + if (!ok) { + printf("%-10s %8u %12s %12s %12s\n", human(size, hb), ops, + "FAILED", "-", "-"); + } else { + qsort(unmap_ns, ops, sizeof(*unmap_ns), cmp_d); + printf("%-10s %8u %12.1f %12.1f %12.1f\n", human(size, hb), ops, + sorted_quantile(unmap_ns, ops, 0.50), + sorted_quantile(unmap_ns, ops, 0.95), unmap_ns[ops - 1]); + } + free(unmap_ns); + } + printf("\n"); +} + +int main(int argc, char **argv) { clock_init(); + if (argc == 4 && strcmp(argv[1], "fresh") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long count = strtol(argv[3], &end, 0); + if (errno || !end || *end || count < 1 || count > 1000) + return 2; + return bench_fresh_one(size, (int) count); + } + if (argc != 1) + return 2; printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); bench_size_sweep(); bench_fresh(); @@ -451,5 +680,7 @@ int main(void) bench_mprotect_split(); bench_mremap(); bench_mt(); + bench_munmap_materialized(); + bench_munmap_dirty(); return 0; } diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh index fa92a8a7..4a735583 100755 --- a/tests/test-mmap-fastpath-stats.sh +++ b/tests/test-mmap-fastpath-stats.sh @@ -69,10 +69,14 @@ require_le() fi } +run_case ring-full +require_ge ring-full MMAP_HIT 32 +require_ge ring-full MMAP_RING_FULL 1 +printf ' 32-entry ring fallback OK\n' + run_case np2-10m require_ge np2-10m MMAP_HIT 80 require_ge np2-10m MMAP_CAPACITY_MISS 1 -require_ge np2-10m MMAP_RING_FULL 1 printf ' sustained 10 MiB stream OK\n' run_case np2-48m @@ -87,23 +91,23 @@ printf ' sustained 100 MiB stream OK\n' run_case escalation require_ge escalation MMAP_HIT 45 -require_eq escalation MMAP_ARENA_CURRENT 1073741824 +require_eq escalation MMAP_ARENA_CURRENT 17179869184 printf ' 10 MiB -> 512 MiB escalation OK\n' run_case giant-guard require_ge giant-guard MMAP_HIT 34 -require_le giant-guard MMAP_ARENA_PEAK 536870912 -printf ' >1 GiB giant request guard OK\n' +require_eq giant-guard MMAP_ARENA_PEAK 34359738368 +printf ' 2 GiB request uses fast path OK\n' run_case adaptive-small require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 require_eq adaptive-small MMAP_ARENA_PEAK 67108864 printf ' small-stream arena floor OK\n' -run_case adaptive-decay -require_eq adaptive-decay MMAP_ARENA_CURRENT 67108864 -require_eq adaptive-decay MMAP_ARENA_PEAK 1073741824 -printf ' one-generation arena decay OK\n' +run_case adaptive-retention +require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 +require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 +printf ' large arena retained OK\n' run_case recycle require_ge recycle MMAP_RECYCLE 1 diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c index 4a9aa21e..41cf1544 100644 --- a/tests/test-mmap-fastpath.c +++ b/tests/test-mmap-fastpath.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,8 @@ int passes = 0, fails = 0; static sigjmp_buf segv_jmp; +static inline void spin_hint(void); + static void segv_handler(int sig) { (void) sig; @@ -64,8 +67,9 @@ static void test_fidelity(void) { TEST("unconsumed arena is absent and faults"); struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) != 0) { + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { FAIL("sigaction"); return; } @@ -74,6 +78,7 @@ static void test_fidelity(void) MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); return; } p[0] = 0x5a; /* drains the publication through the fault-side lock */ @@ -83,6 +88,7 @@ static void test_fidelity(void) (void) *unconsumed; FAIL("wild read into unconsumed arena did not SIGSEGV"); munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); return; } @@ -91,9 +97,26 @@ static void test_fidelity(void) hi != (uintptr_t) p + 4096) { FAIL("/proc/self/maps exposed more than the consumed page"); munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + if (munmap(p, 4096) != 0) { + FAIL("munmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene between munmap and this load: EL1 must have + * invalidated the Stage-1 descriptor and completed broadcast TLBI before + * returning, even though host region cleanup is still deferred. + */ + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *(volatile uint8_t *) p; + FAIL("access immediately after munmap did not SIGSEGV"); + sigaction(SIGSEGV, &old_sa, NULL); return; } - munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); PASS(); } @@ -121,6 +144,177 @@ static void test_exhaustion_fallback(void) PASS(); } +typedef struct { + volatile uint8_t *p; + _Atomic int ready; + _Atomic int go; + _Atomic int result; +} large_tlbi_arg_t; + +static void *large_tlbi_worker(void *opaque) +{ + large_tlbi_arg_t *arg = opaque; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) arg->p[0]; /* seed a translation on this sibling vCPU */ + atomic_store_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + (void) arg->p[0]; + atomic_store_explicit(&arg->result, -1, memory_order_release); + } else { + atomic_store_explicit(&arg->result, 1, memory_order_release); + } + return NULL; +} + +static void test_large_l2_range_tlbi(void) +{ + TEST("SCALE=3 RVAE1IS invalidates sibling L2 translation"); + const size_t len = 320ULL << 20; /* exceeds SCALE=2's 256MiB maximum */ + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + /* One touch per 2MiB materializes L2 block descriptors without making the + * test resident at every guest page. The resulting TLBI envelope requires + * SCALE=3 when FEAT_TLBIRANGE is enabled. + */ + for (size_t off = 0; off < len; off += 2ULL << 20) + p[off] = (uint8_t) (off >> 21); + + large_tlbi_arg_t arg = {.p = p}; + pthread_t worker; + if (pthread_create(&worker, NULL, large_tlbi_worker, &arg) != 0) { + FAIL("pthread_create"); + munmap((void *) p, len); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + atomic_store_explicit(&arg.go, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene here: the sibling must observe EL1's broadcast + * invalidation before any host drain gets a chance to remove metadata. + */ + atomic_store_explicit(&arg.go, 1, memory_order_release); + int result; + while (!(result = atomic_load_explicit(&arg.result, + memory_order_acquire))) + spin_hint(); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (result < 0) { + FAIL("stale sibling translation survived large-range TLBI"); + return; + } + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int ready; + _Atomic int done; + _Atomic int release; +} handoff_arg_t; + +static inline void spin_hint(void) +{ + __asm__ volatile("yield" ::: "memory"); +} + +static void *handoff_worker(void *opaque) +{ + handoff_arg_t *arg = opaque; + atomic_store_explicit(&arg->ready, 1, memory_order_release); + uintptr_t ptr; + while (!(ptr = atomic_load_explicit(&arg->ptr, memory_order_acquire))) + spin_hint(); + int rc = munmap((void *) ptr, 4096); + atomic_store_explicit(&arg->done, rc == 0 ? 1 : -1, + memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + return NULL; +} + +static void test_cross_vcpu_handoff(void) +{ + TEST("cross-vCPU mmap publication then munmap retirement"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + handoff_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, handoff_worker, &arg) != 0) { + FAIL("pthread_create"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + /* Keep the worker alive after munmap so its next thread-exit syscall + * cannot drain either ring. The fault below is the first natural VM exit + * after A's mmap publication and B's retirement. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + atomic_store_explicit(&arg.ptr, (uintptr_t) p, memory_order_release); + int done; + while (!(done = atomic_load_explicit(&arg.done, memory_order_acquire))) + spin_hint(); + + int faulted = 0; + if (done > 0 && sigsetjmp(segv_jmp, 1) == 0) + (void) *(volatile uint8_t *) p; + else if (done > 0) + faulted = 1; + + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (done < 0) { + FAIL("worker munmap"); + return; + } + if (!faulted) { + FAIL("retired cross-vCPU mapping remained accessible"); + return; + } + PASS(); +} + typedef struct { int iterations; _Atomic int *failed; @@ -217,6 +411,8 @@ static int stats_stream(size_t len, int iterations, bool release_each) static int run_stats_case(const char *name) { + if (strcmp(name, "ring-full") == 0) + return stats_stream(64ULL << 10, 40, false); if (strcmp(name, "np2-10m") == 0) return stats_stream(10ULL << 20, 96, false); if (strcmp(name, "np2-48m") == 0) @@ -240,14 +436,13 @@ static int run_stats_case(const char *name) } if (strcmp(name, "adaptive-small") == 0) return stats_stream(64ULL << 10, 1100, false); - if (strcmp(name, "adaptive-decay") == 0) { - if (stats_stream(64ULL << 10, 1100, false) != 0 || - stats_stream(500ULL << 20, 1, false) != 0) + if (strcmp(name, "adaptive-retention") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0) return 1; - /* Ring-full fallbacks do not consume the arena cursor, so exceed the - * nominal 16384 pages enough to force a true 1GiB capacity rollover. + /* The first request selects a 16GiB arena, the next 32 consume it, and + * the last forces a capacity rollover that must retain the target. */ - return stats_stream(64ULL << 10, 18000, false); + return stats_stream(500ULL << 20, 34, false); } if (strcmp(name, "recycle") == 0) return stats_stream(64ULL << 10, 6000, true); @@ -270,6 +465,8 @@ int main(int argc, char **argv) test_fidelity(); test_exhaustion_fallback(); + test_large_l2_range_tlbi(); + test_cross_vcpu_handoff(); test_mt_storm_and_fork_exec(); printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c index 8541b819..173b850a 100644 --- a/tests/test-mmap-lazy.c +++ b/tests/test-mmap-lazy.c @@ -747,6 +747,80 @@ static void test_adjacent_region_extension(void) PASS(); } +static void test_large_retire_backing_replacement(void) +{ + TEST("large retire replacement preserves neighbors and fork zeroes"); + const size_t body_len = 48ULL << 20; + const size_t total_len = body_len + (6ULL << 20); + uint8_t *p = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uintptr_t aligned = ((uintptr_t) p + BLOCK_2MIB * 2 - 1) & + ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *body = (uint8_t *) aligned; + size_t left_len = (size_t) (body - p); + size_t right_len = total_len - left_len - body_len; + if (left_len < BLOCK_2MIB || right_len < BLOCK_2MIB) { + FAIL("guard alignment"); + munmap(p, total_len); + return; + } + + memset(p, 0xa5, total_len); + if (munmap(body, body_len) != 0) { + FAIL("retire body"); + munmap(p, total_len); + return; + } + + /* MAP_FIXED is a metadata-reading slow path, so it must first drain the + * EL1 retirement. The 48 MiB aligned dirty body exercises the 32 MiB HVF + * backing-replacement policy rather than the short-range memset fallback. + */ + uint8_t *q = mmap(body, body_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != body) { + FAIL("fixed reuse"); + munmap(p, left_len); + munmap(body + body_len, right_len); + return; + } + + bool ok = p[0] == 0xa5 && body[-1] == 0xa5 && + body[body_len] == 0xa5 && p[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < body_len; off += 4096) + ok = q[off] == 0; + + pid_t pid = -1; + int st = 0; + if (ok) + pid = fork(); + if (pid == 0) { + for (size_t off = 0; off < body_len; off += BLOCK_2MIB) { + if (q[off] != 0) + _exit(1); + } + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) + ok = false; + + munmap(p, left_len); + munmap(q, body_len); + munmap(body + body_len, right_len); + if (!ok) { + FAIL("replacement leaked data, clobbered a neighbor, or broke fork"); + return; + } + PASS(); +} + int main(void) { test_huge_sparse(); @@ -765,6 +839,7 @@ int main(void) test_mt_first_touch(); test_claim_mutation_race(); test_adjacent_region_extension(); + test_large_retire_backing_replacement(); SUMMARY("test-mmap-lazy"); return fails ? 1 : 0; From ca1e97790c4951d1d9c0e6b79c50d603fa413e6c Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 18:51:32 +0800 Subject: [PATCH 5/9] Reduce mmap and munmap latency jitter Grow fully rewound mmap arenas to their adaptive target instead of repeatedly retaining undersized arenas. Isolate dirty-munmap timing from deferred cleanup and long no-exit intervals, and add regression coverage for rewind-driven arena growth. --- src/syscall/mem.c | 23 ++++++++++++++----- tests/bench-mmap.c | 37 +++++++++++++++++++++++++++---- tests/test-mmap-fastpath-stats.sh | 5 +++++ tests/test-mmap-fastpath.c | 7 ++++++ 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 2a4d3b33..f68282b0 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -647,12 +647,28 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, c->max_len_seen = request_len; uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + uint64_t arena_size = + mmap_fastpath_arena_size(c->max_len_seen, request_len); if ((flags & SHIM_MMAP_CTRL_ENABLED) && - mmap_fastpath_request_fits(cursor, limit, request_len)) - return; + mmap_fastpath_request_fits(cursor, limit, request_len)) { + /* A capacity miss enters HVC, whose drain can rewind a completely + * retired arena before this refill check. Retaining that undersized + * arena merely because one more request fits makes the same miss recur + * every few operations and turns mmap latency into a periodic sawtooth. + * Grow an empty arena to the adaptive target; never relocate one that + * still contains allocations served by the current generation. + */ + bool empty = cursor == arena_base; + bool target_sized = arena_base <= limit && + limit - arena_base >= arena_size; + if (!empty || target_sized) + return; + } /* The owner is parked in HVC. Make the stranded tail immediately recyclable * before the gap scan; mappings already served from the prefix were drained @@ -661,9 +677,6 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, if (flags & SHIM_MMAP_CTRL_ENABLED) atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); - uint64_t arena_size = - mmap_fastpath_arena_size(c->max_len_seen, request_len); - /* Prefer a real hole below the current high-water mark. Active sibling * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap * allocator. Only grow mmap_next when no recyclable hole fits. diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index d75e6c3e..29699522 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -26,6 +26,7 @@ #define _GNU_SOURCE #endif #include +#include #include #include #include @@ -86,6 +87,7 @@ static double sorted_quantile(const double *v, unsigned n, double p) #define KIB (1ULL << 10) #define MIB (1ULL << 20) #define GIB (1ULL << 30) +#define DIRTY_VMEXIT_STRIDE (8 * MIB) /* The counter is too coarse to time a single mmap reliably. A section-A * sample therefore repeats independent mmap -> munmap pairs until the @@ -587,9 +589,9 @@ static void bench_munmap_materialized(void) * dirty mapping. Each size has a fixed operation count and every munmap is * retained separately. In particular, one slow first operation cannot end an * adaptive batch and become the whole sample. Counts decrease with size to - * bound total dirtying work. The 1-GiB case uses eight operations so its - * untimed page stores remain below elfuse's vCPU watchdog; interpolated p95 - * still remains distinct from the separately reported maximum. + * bound total dirtying work. The 1-GiB case uses eight operations, and the + * untimed store loop takes periodic VM exits so every vCPU-run interval stays + * below elfuse's watchdog; interpolated p95 remains distinct from max. */ static void bench_munmap_dirty(void) { @@ -623,8 +625,23 @@ static void bench_munmap_dirty(void) ok = 0; break; } - for (uint64_t off = 0; off < size; off += 4 * KIB) + for (uint64_t off = 0; off < size; off += 4 * KIB) { p[off] = (uint8_t) (off >> 12); + /* Large already-backed runs can execute in EL0 long enough + * for the benchmark's 10-second vCPU watchdog to fire under + * macOS memory pressure. This deliberately failing fcntl is + * a guaranteed, side-effect-free HVC outside the timed region. + * It also drains the preceding retirement in bounded chunks. + */ + if (off != 0 && (off & (DIRTY_VMEXIT_STRIDE - 1)) == 0) + (void) fcntl(-1, F_GETFD); + } + + /* Establish an identical clean host-retirement boundary before + * every measurement. The interval below then contains EL1's + * descriptor/TLBI path, never the previous operation's cleanup. + */ + (void) fcntl(-1, F_GETFD); uint64_t t0 = rd(); int rc = munmap((void *) p, size); @@ -659,6 +676,18 @@ static void bench_munmap_dirty(void) int main(int argc, char **argv) { clock_init(); + if (argc == 2 && strcmp(argv[1], "a") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_size_sweep(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "h") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_munmap_dirty(); + return 0; + } if (argc == 4 && strcmp(argv[1], "fresh") == 0) { char *end = NULL; errno = 0; diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh index 4a735583..0b07df9d 100755 --- a/tests/test-mmap-fastpath-stats.sh +++ b/tests/test-mmap-fastpath-stats.sh @@ -109,6 +109,11 @@ require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 printf ' large arena retained OK\n' +run_case adaptive-rewind-growth +require_ge adaptive-rewind-growth MMAP_CAPACITY_MISS 1 +require_eq adaptive-rewind-growth MMAP_ARENA_CURRENT 268435456 +printf ' rewound arena grows to target OK\n' + run_case recycle require_ge recycle MMAP_RECYCLE 1 require_le recycle MMAP_HIGH_WATER 201326592 diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c index 41cf1544..a0ac2d15 100644 --- a/tests/test-mmap-fastpath.c +++ b/tests/test-mmap-fastpath.c @@ -444,6 +444,13 @@ static int run_stats_case(const char *name) */ return stats_stream(500ULL << 20, 34, false); } + if (strcmp(name, "adaptive-rewind-growth") == 0) + /* The first 64MiB arena holds eight 8MiB mappings. The ninth mmap + * takes the capacity fallback after the matching munmaps let host + * drain rewind the arena; refill must grow it to the 32-entry target + * instead of retaining an arena that will miss every eight calls. + */ + return stats_stream(8ULL << 20, 41, true); if (strcmp(name, "recycle") == 0) return stats_stream(64ULL << 10, 6000, true); return 2; From e3a18f651c4f3dc6bc28bbf74d97a3a7fc14628d Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 21:51:57 +0800 Subject: [PATCH 6/9] Improve EL1 mmap retirement and reuse Return metadata-committed extents to per-vCPU EL1 allocators, defer dirty backing cleanup, and avoid global PT-gate rendezvous for publication-only mmap handoffs. Add mixed-size reuse, duplicate retirement, latency, and contention coverage. --- src/core/mmap-fastpath.h | 39 ++- src/core/shim-globals.c | 34 ++- src/core/shim.S | 137 ++++++++-- src/syscall/mem.c | 423 ++++++++++++++++++++++++++---- src/syscall/syscall.c | 6 +- tests/bench-mmap.c | 65 ++++- tests/test-mmap-fastpath-stats.sh | 7 + tests/test-mmap-fastpath.c | 109 ++++++++ 8 files changed, 727 insertions(+), 93 deletions(-) diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index 2c4b695a..2166d20e 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -1,10 +1,11 @@ /* * Per-vCPU EL1 anonymous-mmap consumer rings. * - * The host is the sole producer of arenas and the sole consumer of ring - * entries. EL1 only bump-allocates VA and appends descriptions. Control - * blocks live in the EL1-only shim-data mapping and are selected from SP_EL1's - * per-thread stack slot, so no guest-visible register ABI is consumed. + * The host produces arenas, consumes mmap/munmap publications, and returns + * metadata-committed holes through a reverse SPSC ring. EL1 first-fit consumes + * those private extents before bump-allocating fresh VA. Control blocks live + * in the EL1-only shim-data mapping and are selected from SP_EL1's per-thread + * stack slot, so no guest-visible register ABI is consumed. */ #pragma once @@ -19,7 +20,7 @@ typedef struct thread_entry thread_entry_t; #define SHIM_MMAP_CONTROL_BASE 0x20000u -#define SHIM_MMAP_CONTROL_STRIDE 0x800u +#define SHIM_MMAP_CONTROL_STRIDE 0x1000u #define SHIM_MMAP_RING_SIZE 32u #define SHIM_MMAP_CTRL_ENABLED 0x1u #define SHIM_MMAP_CTRL_TLBIRANGE 0x2u @@ -39,6 +40,9 @@ typedef struct thread_entry thread_entry_t; #define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u #define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u +#define SHIM_MMAP_REUSE_RING_SIZE 32u +#define SHIM_MMAP_REUSE_OFF 0x720u + #define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) #define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) #define MMAP_FAST_ARENA_TARGET_ENTRIES 32u @@ -77,6 +81,25 @@ typedef struct munmap_retire_ring { munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; } munmap_retire_ring_t; +typedef struct mmap_reuse_entry { + uint64_t addr; + uint64_t length; +} mmap_reuse_entry_t; + +/* Host producer -> EL1 consumer. Entries transfer committed, quarantined VA + * extents back to their owning arena only after semantic metadata and PTE + * retirement have completed. EL1 may shorten an acquired entry in place; + * the host does not touch its slot again until the release-published head has + * advanced past it. */ +typedef struct mmap_reuse_ring { + _Atomic uint32_t head; /* EL1 consumer */ + _Atomic uint32_t tail; /* host producer */ + _Atomic uint64_t published; + _Atomic uint64_t dropped; + _Atomic uint64_t hits; /* EL1 producer */ + mmap_reuse_entry_t entries[SHIM_MMAP_REUSE_RING_SIZE]; +} mmap_reuse_ring_t; + typedef struct { _Atomic uint32_t generation; /* host publish word */ _Atomic uint32_t consumer_generation; /* EL1 generation ack */ @@ -100,10 +123,13 @@ typedef struct { _Atomic uint64_t materialized_end; uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; munmap_retire_ring_t retire; + mmap_reuse_ring_t reuse; } shim_mmap_control_t; _Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, "shim.S retire-ring offset ABI"); +_Static_assert(offsetof(shim_mmap_control_t, reuse) == SHIM_MMAP_REUSE_OFF, + "shim.S reuse-ring offset ABI"); _Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, "mmap control exceeds per-vCPU stride"); @@ -146,6 +172,9 @@ void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); bool mmap_fastpath_allocate_current_locked(guest_t *g, uint64_t request_len, uint64_t *addr_out); +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); /* Give an explicit slow-path hint precedence over this stopped vCPU's * unconsumed arena tail. Caller holds mmap_lock. diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 642bfaa6..6fce6844 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -99,8 +99,8 @@ _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= "counter array must not overlap the PGID slot"); _Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, "shim.S mmap fast path hard-codes control base 0x20000"); -_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x800, - "shim.S mmap fast path hard-codes control stride 0x800"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x1000, + "shim.S mmap fast path hard-codes control stride 0x1000"); _Static_assert(SHIM_MMAP_RING_SIZE == 32, "shim.S mmap fast path hard-codes 32 ring entries"); _Static_assert(offsetof(shim_mmap_control_t, generation) == 0, @@ -151,6 +151,15 @@ _Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, _Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, "shim.S munmap retire flag encoding drift"); +_Static_assert(offsetof(shim_mmap_control_t, reuse) == 0x720, + "shim.S mmap reuse offset drift"); +_Static_assert(offsetof(mmap_reuse_ring_t, published) == 8 && + offsetof(mmap_reuse_ring_t, dropped) == 16 && + offsetof(mmap_reuse_ring_t, hits) == 24 && + offsetof(mmap_reuse_ring_t, entries) == 32, + "shim.S mmap reuse-ring layout drift"); +_Static_assert(SHIM_MMAP_REUSE_RING_SIZE == 32, + "shim.S mmap reuse-ring size drift"); _Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), @@ -558,6 +567,8 @@ void shim_globals_counters_dump(const guest_t *g) }; uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; uint64_t refill_count = 0, recycle_count = 0; + uint64_t reuse_published = 0, reuse_dropped = 0, reuse_hits = 0; + uint64_t reuse_pending = 0; uint64_t current_max = 0, peak_max = 0; const uint8_t *shim_data = (const uint8_t *) g->host_base + g->shim_data_base; @@ -571,6 +582,17 @@ void shim_globals_counters_dump(const guest_t *g) atomic_load_explicit(&c->counters[i], memory_order_relaxed); refill_count += c->refill_count; recycle_count += c->recycle_count; + reuse_published += atomic_load_explicit(&c->reuse.published, + memory_order_relaxed); + reuse_dropped += atomic_load_explicit(&c->reuse.dropped, + memory_order_relaxed); + reuse_hits += + atomic_load_explicit(&c->reuse.hits, memory_order_relaxed); + uint32_t reuse_head = + atomic_load_explicit(&c->reuse.head, memory_order_relaxed); + uint32_t reuse_tail = + atomic_load_explicit(&c->reuse.tail, memory_order_relaxed); + reuse_pending += (uint32_t) (reuse_tail - reuse_head); if (c->next_arena_size > current_max) current_max = c->next_arena_size; if (c->peak_arena_size > peak_max) @@ -583,6 +605,14 @@ void shim_globals_counters_dump(const guest_t *g) (unsigned long long) refill_count); fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_PUBLISH", + (unsigned long long) reuse_published); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_DROP", + (unsigned long long) reuse_dropped); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_HIT", + (unsigned long long) reuse_hits); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_PENDING", + (unsigned long long) reuse_pending); fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", (unsigned long long) current_max); fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", diff --git a/src/core/shim.S b/src/core/shim.S index 849c79c1..34b2b6cc 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -475,10 +475,11 @@ svc_handler: * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) * * The control is selected without another system register. SP_EL1 stack tops - * are shim_data_end - slot*4KiB; controls are shim_data+0x20000 + slot*2KiB, - * so (shim_data_end - ALIGN_UP(sp,4KiB))/2 is the control-array displacement. - * Each vCPU is the sole producer of its ring and cursor. The host acquire- - * drains all rings whenever it takes mmap_lock. + * are shim_data_end - slot*4KiB; controls are shim_data+0x20000 + slot*4KiB, + * so shim_data_end - ALIGN_UP(sp,4KiB) is the control-array displacement. + * Each vCPU is the sole producer of its mmap publication ring and cursor, and + * the sole consumer of its host-committed reuse ring. The host acquire-drains + * all publications whenever it takes mmap_lock. */ mmap_anon_fast: mrs x12, tpidr_el1 /* shared shim-data base */ @@ -488,7 +489,6 @@ mmap_anon_fast: and x14, x14, #~0xfff /* this vCPU's stack top */ add x15, x12, #0x200, lsl #12 /* shim_data + 2MiB */ sub x15, x15, x14 /* slot * 4KiB */ - lsr x15, x15, #1 /* slot * 2KiB */ add x12, x12, x15 add x12, x12, #0x20, lsl #12 /* control base + 0x20000 */ @@ -510,6 +510,22 @@ mmap_anon_fast: b.cs mmap_shape_bail /* PAGE_ALIGN_UP overflow */ and x15, x15, #~0xfff + /* Join the same host-writer handshake as fast munmap. Arena revoke, + * refill, and reuse publication all run with the gate closed; announcing + * this per-vCPU producer prevents the host from invalidating a descriptor + * between our generation check and mmap publication. */ + mrs x20, tpidr_el1 + add x20, x20, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, mmap_gate_bail + add x24, x12, #0x400 + add x24, x24, #24 /* retire.producer_active */ + mov w22, #1 + stlr w22, [x24] + ldar w22, [x20] + cbnz w22, mmap_active_capacity_bail + ldar w13, [x12] /* descriptor generation */ ldr w14, [x12, #4] /* consumer generation */ cmp w13, w14 @@ -517,6 +533,58 @@ mmap_anon_fast: ldr w13, [x12, #8] /* SHIM_MMAP_CTRL_ENABLED */ tbz w13, #0, mmap_capacity_bail + /* Reserve publication capacity before mutating either the bump cursor or + * a host-committed reuse extent. */ + add x23, x12, #12 /* &mmap publication head */ + ldar w22, [x23] + ldr w23, [x12, #16] /* producer-owned tail */ + sub w24, w23, w22 + cmp w24, #32 /* SHIM_MMAP_RING_SIZE */ + b.hs mmap_ring_full_bail + + /* Prefer committed holes returned by the host. The host release-publishes + * immutable entries through reuse.tail; after acquiring tail, this EL1 + * consumer owns every pending slot and may shorten it in place. First-fit + * handles mixed allocation sizes. Large requests retain the bump path's + * 2 MiB start-alignment promise; an unaligned extent is simply skipped. */ + add x26, x12, #0x720 /* host -> EL1 reuse ring */ + ldr w27, [x26, #0] /* consumer-owned head */ + add x25, x26, #4 + ldar w28, [x25] /* host-published tail */ + mov w29, w27 +mmap_reuse_scan: + cmp w29, w28 + b.eq mmap_reuse_miss + and w30, w29, #31 + add x25, x26, #32 /* reuse.entries */ + add x25, x25, x30, lsl #4 + ldr x19, [x25, #0] /* extent address */ + ldr x24, [x25, #8] /* extent length; zero = consumed */ + cbz x24, mmap_reuse_next + cmp x24, x15 + b.lo mmap_reuse_next + mov x20, #0x200000 + cmp x15, x20 + b.lo mmap_reuse_fit + sub x20, x20, #1 + tst x19, x20 + b.ne mmap_reuse_next +mmap_reuse_fit: + adds x21, x19, x15 + b.cs mmap_reuse_next /* corrupt extent cannot be used */ + sub x24, x24, x15 + str x21, [x25, #0] /* retain suffix in this slot */ + str x24, [x25, #8] /* zero marks full consumption */ + ldr x20, [x26, #24] /* per-vCPU reuse hits */ + add x20, x20, #1 + str x20, [x26, #24] + mov w28, #0 /* address came from reuse ring */ + b mmap_addr_ready +mmap_reuse_next: + add w29, w29, #1 + b mmap_reuse_scan + +mmap_reuse_miss: ldr x19, [x12, #40] /* private bump cursor */ mov x20, #0x200000 cmp x15, x20 @@ -530,14 +598,9 @@ mmap_anon_fast: b.cs mmap_capacity_bail cmp x21, x20 b.hi mmap_capacity_bail + mov w28, #1 /* bump cursor must advance */ - add x23, x12, #12 /* &head */ - ldar w22, [x23] - ldr w23, [x12, #16] /* producer-owned tail */ - sub w24, w23, w22 - cmp w24, #32 /* SHIM_MMAP_RING_SIZE */ - b.hs mmap_ring_full_bail - +mmap_addr_ready: and w24, w23, #31 add x24, x24, x24, lsl #1 /* index * 3 */ add x25, x12, #64 /* ring base */ @@ -545,10 +608,38 @@ mmap_anon_fast: str x19, [x25, #0] str x15, [x25, #8] str x16, [x25, #16] - str x21, [x12, #40] /* publish cursor before entry */ + cbz w28, 2f + str x21, [x12, #40] /* publish bump cursor before entry */ +2: add w23, w23, #1 add x25, x12, #16 stlr w23, [x25] /* release-publish ring entry */ + + /* Only after the mmap publication is visible may fully consumed reuse + * slots be released to the host producer. Skip zero tombstones at the + * head; first-fit may have consumed a later slot, which will be reclaimed + * when all earlier entries have drained. */ + add x26, x12, #0x720 + ldr w27, [x26, #0] + add x25, x26, #4 + ldar w29, [x25] + mov w30, w27 +3: cmp w30, w29 + b.eq 5f + and w24, w30, #31 + add x25, x26, #32 + add x25, x25, x24, lsl #4 + ldr x24, [x25, #8] + cbnz x24, 5f + add w30, w30, #1 + b 3b +5: cmp w30, w27 + b.eq 6f + stlr w30, [x26] +6: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] /* producer_active = 0 */ mov x0, x19 MMAP_COUNTER_INC MC_HIT b svc_restore_eret @@ -558,13 +649,27 @@ mmap_shape_bail: b handle_svc_0 mmap_capacity_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] MMAP_COUNTER_INC MC_CAPACITY_MISS b handle_svc_0 mmap_ring_full_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] MMAP_COUNTER_INC MC_RING_FULL b handle_svc_0 +mmap_active_capacity_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] +mmap_gate_bail: + MMAP_COUNTER_INC MC_CAPACITY_MISS + b handle_svc_0 + mmap_attention_bail: MMAP_COUNTER_INC MC_ATTENTION b attn_bail @@ -576,6 +681,9 @@ mmap_generation_bail: */ MMAP_COUNTER_INC MC_GENERATION_STALE str w13, [x12, #4] + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] b handle_svc_0 /* munmap_anon_fast: invalidate an anonymous arena range synchronously at EL1, @@ -596,7 +704,6 @@ munmap_anon_fast: and x20, x20, #~0xfff /* this vCPU's stack top */ add x21, x12, #0x200, lsl #12 sub x21, x21, x20 - lsr x21, x21, #1 add x21, x12, x21 add x21, x21, #0x20, lsl #12 /* current mmap control */ @@ -658,7 +765,7 @@ munmap_anon_fast: cmp w22, w28 b.eq munmap_arena_found 2: add w24, w24, #1 - add x25, x25, #0x800 + add x25, x25, #0x1000 cmp w24, #64 b.lo 1b b munmap_active_bail diff --git a/src/syscall/mem.c b/src/syscall/mem.c index f68282b0..af070ca7 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -47,7 +47,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t min_addr, uint64_t max_addr, uint64_t align); -static void mmap_fastpath_rewind_control_if_clean_locked( +static bool mmap_fastpath_rewind_control_if_clean_locked( guest_t *g, shim_mmap_control_t *c); /* A host-VA replacement has a fixed cost: sibling quiesce plus an exact HVF @@ -87,6 +87,115 @@ static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); } +static void mmap_fastpath_reuse_reset(shim_mmap_control_t *c) +{ + if (!c) + return; + atomic_store_explicit(&c->reuse.head, 0, memory_order_relaxed); + atomic_store_explicit(&c->reuse.tail, 0, memory_order_relaxed); +} + +/* Publish a semantically removed, PTE-retired range back to its owning arena. + * mmap_lock serializes host producers; EL1 is the sole consumer. A full ring + * is only a performance miss: the range remains unavailable to EL1 until a + * later arena refill lets the ordinary host gap allocator recover it. */ +static void mmap_fastpath_publish_reuse_locked(shim_mmap_control_t *c, + uint32_t generation, + uint64_t start, + uint64_t end) +{ + if (!c || end <= start || + !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->generation, memory_order_acquire) != + generation) + return; + + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start < base || end > limit) + return; + + uint32_t head = + atomic_load_explicit(&c->reuse.head, memory_order_acquire); + uint32_t tail = + atomic_load_explicit(&c->reuse.tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_REUSE_RING_SIZE) { + log_fatal("mmap reuse: corrupt ring (head=%u tail=%u)", head, tail); + return; + } + + /* mmap_fastpath_host_gate_close() has stopped every EL1 allocator before + * drain reaches here, so the host may compact the consumer-owned pending + * set in place. Fold tombstones and adjacent extents on every publication; + * otherwise mixed-size prefix splits would fill a FIFO with tiny suffixes + * even though the arena had ample aggregate free space. */ + mmap_reuse_entry_t extents[SHIM_MMAP_REUSE_RING_SIZE + 1]; + unsigned count = 0; + bool touches_existing = false; + for (uint32_t seq = head; seq != tail; seq++) { + mmap_reuse_entry_t e = + c->reuse.entries[seq & (SHIM_MMAP_REUSE_RING_SIZE - 1)]; + if (!e.length) + continue; + if (e.addr < base || e.addr > limit || e.length > limit - e.addr) { + log_fatal("mmap reuse: invalid pending extent"); + continue; + } + uint64_t e_end = e.addr + e.length; + if (end >= e.addr && start <= e_end) + touches_existing = true; + extents[count++] = e; + } + if (count == SHIM_MMAP_REUSE_RING_SIZE && !touches_existing) { + atomic_fetch_add_explicit(&c->reuse.dropped, 1, + memory_order_relaxed); + return; + } + extents[count++] = + (mmap_reuse_entry_t) {.addr = start, .length = end - start}; + + for (unsigned i = 1; i < count; i++) { + mmap_reuse_entry_t key = extents[i]; + unsigned j = i; + while (j > 0 && extents[j - 1].addr > key.addr) { + extents[j] = extents[j - 1]; + j--; + } + extents[j] = key; + } + + unsigned merged = 0; + for (unsigned i = 0; i < count; i++) { + uint64_t e_end = extents[i].addr + extents[i].length; + if (merged != 0) { + mmap_reuse_entry_t *last = &extents[merged - 1]; + uint64_t last_end = last->addr + last->length; + if (extents[i].addr <= last_end) { + if (e_end > last_end) + last->length = e_end - last->addr; + continue; + } + } + extents[merged++] = extents[i]; + } + if (merged > SHIM_MMAP_REUSE_RING_SIZE) { + atomic_fetch_add_explicit(&c->reuse.dropped, 1, + memory_order_relaxed); + return; + } + + for (unsigned i = 0; i < merged; i++) + c->reuse.entries[i] = extents[i]; + for (unsigned i = merged; i < SHIM_MMAP_REUSE_RING_SIZE; i++) + c->reuse.entries[i].length = 0; + atomic_store_explicit(&c->reuse.head, 0, memory_order_relaxed); + atomic_fetch_add_explicit(&c->reuse.published, 1, memory_order_relaxed); + atomic_store_explicit(&c->reuse.tail, merged, memory_order_release); +} + static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) { if (!g || !g->host_base) @@ -148,6 +257,7 @@ static void mmap_fastpath_disable_control(shim_mmap_control_t *c) atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + mmap_fastpath_reuse_reset(c); c->next_arena_size = MMAP_FAST_ARENA_MIN; c->max_len_seen = 0; atomic_store_explicit(&c->generation, generation, memory_order_release); @@ -280,48 +390,58 @@ static void munmap_retire_commit_locked(guest_t *g, * dirty bitmap to avoid touching huge never-materialized reservations. * Full, contiguous dirty runs at or above 32 MiB are cheaper to discard by * replacing their zero-fill backing while their HVF stage-2 segments are - * detached. Partial blocks and shorter runs retain the inline memset path. + * detached. For shorter or partial runs, retain the dirty bits and let a + * future lazy materialization zero only the backing that is actually + * reused. In particular, do not charge an unrelated VM exit (often the + * next mapping's first fault) with an eager memset of the retired range. + * The replacement mapping cannot observe stale bytes: EL1 has already + * invalidated the old descriptors, and guest_materialize_lazy_one() zeros + * dirty backing before publishing any new descriptor. + * * charged_bytes is the EL1 producer's page-accurate materialized count, so * a sparse virtual retirement cannot trigger replacement merely because * its address span is large. */ bool allow_replace = charged_bytes >= MUNMAP_HVF_REPLACE_THRESHOLD; bool siblings_quiesced = false; - uint64_t block = ALIGN_DOWN(backing_start, BLOCK_2MIB); - while (block < backing_end) { - uint64_t block_end = block + BLOCK_2MIB; - if (guest_block_may_be_dirty(g, block)) { + if (allow_replace) { + uint64_t block = ALIGN_DOWN(backing_start, BLOCK_2MIB); + while (block < backing_end) { + uint64_t block_end = block + BLOCK_2MIB; + if (!guest_block_may_be_dirty(g, block)) { + block = block_end; + continue; + } uint64_t lo = backing_start > block ? backing_start : block; uint64_t hi = backing_end < block_end ? backing_end : block_end; + if (lo != block || hi != block_end) { + block = block_end; + continue; + } - if (allow_replace && lo == block && hi == block_end) { - uint64_t run_end = block_end; - while (run_end < backing_end && - run_end <= UINT64_MAX - BLOCK_2MIB && - guest_block_may_be_dirty(g, run_end)) - run_end += BLOCK_2MIB; - - if (run_end - block >= MUNMAP_HVF_REPLACE_THRESHOLD && - munmap_retire_backing_is_exclusive( - g, block, run_end, start, end)) { - if (!siblings_quiesced) { - thread_quiesce_siblings(); - siblings_quiesced = true; - } - if (hvf_replace_slab_zero_range_quiesced( - g, block, run_end - block) == 0) { - guest_dirty_clear_zeroed_range(g, block, run_end); - block = run_end; - continue; - } + uint64_t run_end = block_end; + while (run_end < backing_end && + run_end <= UINT64_MAX - BLOCK_2MIB && + guest_block_may_be_dirty(g, run_end)) + run_end += BLOCK_2MIB; + + if (run_end - block >= MUNMAP_HVF_REPLACE_THRESHOLD && + munmap_retire_backing_is_exclusive( + g, block, run_end, start, end)) { + if (!siblings_quiesced) { + thread_quiesce_siblings(); + siblings_quiesced = true; + } + if (hvf_replace_slab_zero_range_quiesced( + g, block, run_end - block) == 0) { + guest_dirty_clear_zeroed_range(g, block, run_end); + block = run_end; + continue; } } - - memset((uint8_t *) g->host_base + lo, 0, hi - lo); - guest_dirty_clear_zeroed_range(g, lo, hi); + block = block_end; } - block = block_end; } if (siblings_quiesced) thread_resume_siblings(); @@ -415,27 +535,56 @@ void mmap_fastpath_drain_locked(guest_t *g) if (backing_end <= backing_start) log_fatal( "munmap retire: charged entry has no materialized " - "bounds"); + "bounds (producer=%d arena=%u range=0x%llx..0x%llx " + "marker=0x%llx..0x%llx charged=0x%llx)", + slot, arena_slot, (unsigned long long) e->addr, + (unsigned long long) end, + (unsigned long long) atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed), + (unsigned long long) atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed), + (unsigned long long) charged_bytes); + } + + /* Capture only bytes that are still semantically mapped. Linux + * permits munmap across holes (and repeated munmap of a hole), so + * publishing the raw retire range would double-free VA. Adjacent + * anonymous regions are coalesced to preserve reuse-ring capacity. + * More than one ring's worth is safely left to the host gap + * allocator after a later arena rollover. */ + mmap_reuse_entry_t reusable[SHIM_MMAP_REUSE_RING_SIZE]; + unsigned reusable_count = 0; + for (int i = guest_region_first_end_above(g, e->addr); + i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= e->addr || !r->noreserve || + !(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) + continue; + uint64_t lo = r->start > e->addr ? r->start : e->addr; + uint64_t hi = r->end < end ? r->end : end; + if (reusable_count != 0 && + reusable[reusable_count - 1].addr + + reusable[reusable_count - 1].length == + lo) { + reusable[reusable_count - 1].length += hi - lo; + } else if (reusable_count < SHIM_MMAP_REUSE_RING_SIZE) { + reusable[reusable_count++] = + (mmap_reuse_entry_t) {.addr = lo, + .length = hi - lo}; + } } munmap_retire_commit_locked(g, e, backing_start, backing_end, charged_bytes); - /* If that retirement removed the arena's final live descriptor, - * restore the whole-arena PTE-empty proof. This prevents historical - * materialization bounds from growing across sequential sparse - * mappings in the same bump arena. - */ - if (guest_va_next_present_block(g, base, cursor) >= cursor) { - atomic_store_explicit(&arena->materialized_start, 0, - memory_order_relaxed); - atomic_store_explicit(&arena->materialized_end, 0, - memory_order_relaxed); - atomic_store_explicit(&arena->materialized_generation, 0, - memory_order_release); + for (unsigned i = 0; i < reusable_count; i++) { + mmap_fastpath_publish_reuse_locked( + arena, generation, reusable[i].addr, + reusable[i].addr + reusable[i].length); } - if (current_thread && - arena_slot == (uint32_t) current_thread->sp_el1_slot) - mmap_fastpath_rewind_control_if_clean_locked(g, arena); uint64_t consumed = atomic_load_explicit( &producer->retire.consumed_bytes, memory_order_relaxed); atomic_store_explicit(&producer->retire.consumed_bytes, @@ -454,6 +603,39 @@ void mmap_fastpath_drain_locked(guest_t *g) memory_order_release); } + /* A stopped owner whose whole arena retired can collapse all published + * sub-extents back into its bump cursor. Like envelope reset, this must + * wait for the complete snapshot: overlapping retire records from sibling + * producers may otherwise observe a prematurely reset arena. */ + if (current_thread && current_thread->sp_el1_slot >= 0) + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); + + /* EL1 may publish several charged retirements before this drain. Their + * PTEs are all already invalid, so clearing an arena's materialized + * envelope after the first commit would make later entries in the same + * snapshot lose their backing bounds. Restore the PTE-empty proof only + * after every snapshotted retirement has consumed the old envelope. */ + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + if (base < cursor && + guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + } + /* EL1 PTE stores cannot update guest_t's host-only cache generation. One * bump per batch invalidates every host GVA translation cache after all * retirement entries have committed. @@ -567,6 +749,10 @@ void mmap_fastpath_note_materialized_locked(guest_t *g, memory_order_relaxed); atomic_store_explicit(&c->materialized_generation, generation, memory_order_release); + /* Fast-path arenas are allocated from disjoint VA ranges. Once this + * block has updated its owner, no later vCPU control can overlap it; + * avoid another 63 control-page probes on the single-vCPU hot path. */ + break; } } @@ -731,6 +917,7 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + mmap_fastpath_reuse_reset(c); c->next_arena_size = arena_size; c->max_len_seen = 0; c->refill_count++; @@ -797,6 +984,93 @@ bool mmap_fastpath_allocate_current_locked(guest_t *g, return true; } +/* Service mmap publication-ring backpressure without turning it into a + * global PT-gate rendezvous. The calling vCPU is stopped in HVC, so the host + * may advance only that vCPU's existing bump cursor after draining its prior + * publications. Sibling EL1 allocators own disjoint arenas and may continue + * publishing concurrently. + * + * This path deliberately refuses every operation that could require a host + * writer transaction: a pending retirement, a closed gate, arena refill or + * generation change all fall back to mmap_lock_acquire(). The acquire + * snapshots of every retire tail preserve the usual drain-before-metadata + * ordering for causally prior munmaps. */ +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!g || !addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + pthread_mutex_lock(&mmap_lock); + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) + goto miss; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&producer->retire.head, + memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&producer->retire.tail, + memory_order_acquire); + if (head != tail) + goto miss; + } + + mmap_fastpath_drain_publications_locked(g); + + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_acquire); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->consumer_generation, memory_order_relaxed) != + generation) + goto miss; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start == base) { + uint64_t target = + mmap_fastpath_arena_size(c->max_len_seen, request_len); + if (base > limit || limit - base < target) + goto miss; + } + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + goto miss; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + goto miss; + uint64_t end = start + request_len; + + if (guest_region_add_ex(g, start, end, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) + goto miss; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + if (request_len > c->max_len_seen) + c->max_len_seen = request_len; + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + pthread_mutex_unlock(&mmap_lock); + return true; + +miss: + pthread_mutex_unlock(&mmap_lock); + return false; +} + void mmap_fastpath_release_current_hint_locked(guest_t *g, uint64_t addr, uint64_t length) @@ -809,16 +1083,17 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) return; - uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); - if (cursor >= limit || addr >= limit || addr + length <= cursor) + if (base >= limit || addr >= limit || addr + length <= base) return; /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race - * this descriptor update. Revoke its whole unconsumed tail: the explicit - * hint must remain semantically free, and the post-syscall refill will - * provision a new non-overlapping arena. + * this descriptor update. Revoke its bump tail and committed free extents: + * the explicit hint must take precedence over every EL1-reserved hole, and + * the post-syscall refill will provision a new non-overlapping arena. */ mmap_fastpath_disable_control(c); } @@ -832,38 +1107,42 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, * the next mmap -- especially important when one 32 GiB request consumes the * entire maximum-sized arena. */ -static void mmap_fastpath_rewind_control_if_clean_locked( +static bool mmap_fastpath_rewind_control_if_clean_locked( guest_t *g, shim_mmap_control_t *c) { if (!g || !c) - return; + return false; if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) - return; + return false; uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); if (base >= limit || cursor <= base) - return; + return false; for (int i = 0; i < g->nregions; i++) { const guest_region_t *r = &g->regions[i]; if (r->start >= limit) break; if (r->end > base) - return; + return false; } if (guest_va_next_present_block(g, base, limit) < limit) - return; + return false; + /* The stopped owner can safely discard pending sub-extents because the + * whole arena is becoming one bump-allocatable extent again. */ + mmap_fastpath_reuse_reset(c); atomic_store_explicit(&c->cursor, base, memory_order_relaxed); atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); c->recycle_count++; + return true; } static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) @@ -937,6 +1216,36 @@ void mmap_fastpath_skip_reserved(const guest_t *g, advanced = true; break; } + + /* mmap_lock acquisition closed the producer gate and drained mmap + * publications before a host gap scan reaches here. Pending reuse + * entries are therefore stable: reserve them precisely instead of + * hoarding the whole arena after a vCPU becomes idle. */ + uint32_t reuse_head = + atomic_load_explicit(&c->reuse.head, memory_order_acquire); + uint32_t reuse_tail = + atomic_load_explicit(&c->reuse.tail, memory_order_acquire); + if ((uint32_t) (reuse_tail - reuse_head) > + SHIM_MMAP_REUSE_RING_SIZE) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + for (uint32_t seq = reuse_head; seq != reuse_tail; seq++) { + const mmap_reuse_entry_t *e = + &c->reuse + .entries[seq & (SHIM_MMAP_REUSE_RING_SIZE - 1)]; + if (!e->length || e->addr > UINT64_MAX - e->length) + continue; + uint64_t reuse_end = e->addr + e->length; + if (*start < reuse_end && end > e->addr) { + *start = ALIGN_UP(reuse_end, align); + advanced = true; + break; + } + } + if (advanced) + break; } } while (advanced); } diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index df3e715d..2a0ac4d1 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -972,8 +972,12 @@ static int64_t sc_mmap(guest_t *g, bool verbose) { uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); - mmap_lock_acquire(g); uint64_t arena_addr = 0; + if (refill_len && mmap_fastpath_allocate_current_publication_only( + g, refill_len, &arena_addr)) + return (int64_t) arena_addr; + + mmap_lock_acquire(g); int64_t r; if (refill_len && mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index 29699522..a9c73f4e 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -288,19 +288,20 @@ static int bench_fresh_one(uint64_t size, int n) return 0; } -/* C. first-touch page-fault cost Touch one byte per page at a 16 KiB stride so - * no two touches share a macOS host page; every touch is a genuine fault (HVC - * #11 -> host fault handler -> page-table install + zero). Reports per-fault - * ns. +/* C. first-touch cost. Touch one byte per macOS 16 KiB host page. Each access + * demands a distinct physical backing page, but it is not necessarily a + * distinct HVC: elfuse installs Stage-1 descriptors in 2 MiB windows and the + * fault-around policy may install several windows per exit. Report both the + * whole sweep and its per-host-page amortization; calling the latter a + * "per-fault" cost would substantially overcount guest translation faults. */ -static void bench_fault(void) +static void bench_fault(int pages, int drain_between) { const uint64_t stride = 16 * KIB; - const int pages = 512; uint64_t size = stride * (uint64_t) (pages + 1); - printf("== C. first-touch fault cost (16 KiB stride, %d pages) ==\n", - pages); - double per[ITERS]; + printf("== C. first-touch fault cost (16 KiB stride, %d pages%s) ==\n", + pages, drain_between ? ", forced retire drain" : ""); + double sweep[ITERS]; for (int it = -1; it < ITERS; it++) { volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -313,11 +314,20 @@ static void bench_fault(void) p[(uint64_t) i * stride] = 1; uint64_t t1 = rd(); munmap((void *) p, size); + if (drain_between) + (void) fcntl(-1, F_GETFD); if (it >= 0) - per[it] = ns(t1 - t0) / pages; + sweep[it] = ns(t1 - t0); } - printf(" per-fault: median %.1f ns min %.1f ns\n\n", median(per, ITERS), - per[0]); + qsort(sweep, ITERS, sizeof(*sweep), cmp_d); + printf(" sweep: p50 %.1f us p95 %.1f us max %.1f us\n", + sorted_quantile(sweep, ITERS, 0.50) / 1000.0, + sorted_quantile(sweep, ITERS, 0.95) / 1000.0, + sweep[ITERS - 1] / 1000.0); + printf(" amortized/touch: p50 %.1f ns p95 %.1f ns max %.1f ns\n\n", + sorted_quantile(sweep, ITERS, 0.50) / pages, + sorted_quantile(sweep, ITERS, 0.95) / pages, + sweep[ITERS - 1] / pages); } /* D. mprotect split cost Flip the middle 4 KiB of a 2 MiB RW block to @@ -688,6 +698,35 @@ int main(int argc, char **argv) bench_munmap_dirty(); return 0; } + if (argc == 2 && strcmp(argv[1], "f") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_mt(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault(512, 0); + return 0; + } + if (argc == 3 && strcmp(argv[1], "c") == 0) { + char *end = NULL; + errno = 0; + long pages = strtol(argv[2], &end, 0); + if (errno || !end || *end || pages < 1 || pages > 1048576) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault((int) pages, 0); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c-drain") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault(512, 1); + return 0; + } if (argc == 4 && strcmp(argv[1], "fresh") == 0) { char *end = NULL; errno = 0; @@ -705,7 +744,7 @@ int main(int argc, char **argv) printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); bench_size_sweep(); bench_fresh(); - bench_fault(); + bench_fault(512, 0); bench_mprotect_split(); bench_mremap(); bench_mt(); diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh index 0b07df9d..e557d720 100755 --- a/tests/test-mmap-fastpath-stats.sh +++ b/tests/test-mmap-fastpath-stats.sh @@ -119,4 +119,11 @@ require_ge recycle MMAP_RECYCLE 1 require_le recycle MMAP_HIGH_WATER 201326592 printf ' arena VA recycling OK\n' +run_case reuse-mixed +require_ge reuse-mixed MMAP_REUSE_PUBLISH 100 +require_ge reuse-mixed MMAP_REUSE_HIT 100 +require_eq reuse-mixed MMAP_REUSE_DROP 0 +require_le reuse-mixed MMAP_HIGH_WATER 134217728 +printf ' mixed-size committed reuse OK\n' + printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c index a0ac2d15..12e4eb1a 100644 --- a/tests/test-mmap-fastpath.c +++ b/tests/test-mmap-fastpath.c @@ -315,6 +315,89 @@ static void test_cross_vcpu_handoff(void) PASS(); } +static void test_mixed_size_committed_reuse(void) +{ + TEST("mixed-size committed extents split and read zero"); + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + + for (int i = 0; i < 128; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + if (p[0] != 0 || p[len / 2] != 0 || p[len - 1] != 0) { + munmap((void *) p, len); + FAIL("reused extent exposed stale bytes"); + return; + } + p[0] = (uint8_t) (i + 1); + p[len / 2] = (uint8_t) (i ^ 0x5a); + p[len - 1] = (uint8_t) (i ^ 0xa5); + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + return; + } + } + PASS(); +} + +static void test_repeated_munmap_not_double_reused(void) +{ + TEST("repeated munmap does not double-publish an extent"); + const size_t len = 64ULL << 10; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("initial mmap"); + return; + } + p[0] = 1; + if (munmap((void *) p, len) != 0 || munmap((void *) p, len) != 0) { + FAIL("repeated munmap"); + return; + } + + /* This fault drains both retire records. Only the first still has region + * coverage and may publish a reusable extent. */ + volatile uint8_t *bridge = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bridge == MAP_FAILED) { + FAIL("bridge mmap"); + return; + } + bridge[0] = 2; + + volatile uint8_t *a = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + volatile uint8_t *b = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED || b == MAP_FAILED || a == b) { + if (a != MAP_FAILED) + munmap((void *) a, len); + if (b != MAP_FAILED && b != a) + munmap((void *) b, len); + munmap((void *) bridge, len); + FAIL("duplicate extent allocation"); + return; + } + a[0] = 0x31; + b[0] = 0x42; + if (a[0] != 0x31 || b[0] != 0x42) { + FAIL("distinct mappings aliased"); + return; + } + munmap((void *) a, len); + munmap((void *) b, len); + munmap((void *) bridge, len); + PASS(); +} + typedef struct { int iterations; _Atomic int *failed; @@ -409,6 +492,28 @@ static int stats_stream(size_t len, int iterations, bool release_each) return 0; } +static int stats_mixed_reuse(void) +{ + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + for (int i = 0; i < 160; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (p[0] != 0 || p[len - 1] != 0) + return 1; + p[0] = 1; + p[len - 1] = 2; + if (munmap((void *) p, len) != 0) + return 1; + } + return 0; +} + static int run_stats_case(const char *name) { if (strcmp(name, "ring-full") == 0) @@ -453,6 +558,8 @@ static int run_stats_case(const char *name) return stats_stream(8ULL << 20, 41, true); if (strcmp(name, "recycle") == 0) return stats_stream(64ULL << 10, 6000, true); + if (strcmp(name, "reuse-mixed") == 0) + return stats_mixed_reuse(); return 2; } @@ -474,6 +581,8 @@ int main(int argc, char **argv) test_exhaustion_fallback(); test_large_l2_range_tlbi(); test_cross_vcpu_handoff(); + test_mixed_size_committed_reuse(); + test_repeated_munmap_not_double_reused(); test_mt_storm_and_fork_exec(); printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, From 7874eaeaf3f9c02307c8ea0c7e89e98f3f198822 Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 22:34:02 +0800 Subject: [PATCH 7/9] Optimize mprotect block splitting --- src/core/guest.c | 28 +++++++++++++++++++--------- src/core/mmap-fastpath.h | 8 ++++++++ src/syscall/mem.c | 29 ++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/core/guest.c b/src/core/guest.c index c7665ada..1b705f29 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -171,11 +171,13 @@ static int compute_infra_layout(guest_t *g) return 0; } -/* Allocate a zeroed 4KiB page from the page table pool. - * Returns GPA of the page, or 0 on pool exhaustion. Acquires pt_lock - * internally. Caller typically holds mmap_lock. +/* Allocate a 4KiB page from the page table pool. Returns its GPA, or 0 on + * exhaustion. Acquires pt_lock internally; caller typically holds mmap_lock. + * Most callers need a zero-filled page because they publish only some entries. + * split_l2_block instead overwrites all 512 entries before publication and can + * skip the redundant clear. */ -static uint64_t pt_alloc_page(guest_t *g) +static uint64_t pt_alloc_page_impl(guest_t *g, bool clear) { pthread_mutex_lock(&pt_lock); if (g->pt_pool_next + PAGE_SIZE > g->pt_pool_end) { @@ -202,14 +204,22 @@ static uint64_t pt_alloc_page(guest_t *g) pt_pool_warned = true; } - /* Zero the page while still holding the lock so no other thread can observe - * a partially-zeroed page table page. - */ - memset((uint8_t *) g->host_base + gpa, 0, PAGE_SIZE); + if (clear) + memset((uint8_t *) g->host_base + gpa, 0, PAGE_SIZE); pthread_mutex_unlock(&pt_lock); return gpa; } +static uint64_t pt_alloc_page(guest_t *g) +{ + return pt_alloc_page_impl(g, true); +} + +static uint64_t pt_alloc_page_uninitialized(guest_t *g) +{ + return pt_alloc_page_impl(g, false); +} + /* Get host pointer to a page table entry array at a given GPA */ static uint64_t *pt_at(const guest_t *g, uint64_t gpa) { @@ -3333,7 +3343,7 @@ static int split_l2_block(guest_t *g, uint64_t *l2_entry) int old_perms = desc_to_perms(*l2_entry); - uint64_t l3_gpa = pt_alloc_page(g); + uint64_t l3_gpa = pt_alloc_page_uninitialized(g); if (!l3_gpa) return -1; uint64_t *l3 = pt_at(g, l3_gpa); diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index 2166d20e..1992a42e 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -187,6 +187,14 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, */ void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); +/* Revoke only arenas whose reserved VA span intersects [start, end). The + * caller holds mmap_lock, whose acquisition has already drained publication + * and retirement rings and closed the EL1 page-table gate. + */ +void mmap_fastpath_revoke_range_locked(guest_t *g, + uint64_t start, + uint64_t end); + /* Disable the feature before first guest entry (debugger/observability). */ void mmap_fastpath_disable(guest_t *g); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index af070ca7..0e684f2e 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -1179,6 +1179,33 @@ void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) g->mmap_rw_gap_hint = high; } +void mmap_fastpath_revoke_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + + /* mmap_lock_acquire() closed the producer gate and drained every ring + * before the syscall reached this point. An arena outside the edited VA + * range retains the same anonymous/noreserve classification and can keep + * serving its owner; only an overlapping arena could let a later EL1 + * munmap act on metadata changed by this mprotect. + */ + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start < limit && end > base) + mmap_fastpath_disable_control(c); + } +} + void mmap_fastpath_disable(guest_t *g) { atomic_store_explicit(&mmap_fastpath_forced_off, true, @@ -5124,7 +5151,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) * already-published unmaps, then invalidate arena generations before the * metadata/PTE edit so EL1 cannot act on the old anonymous classification. */ - mmap_fastpath_revoke_all_locked(g, false); + mmap_fastpath_revoke_range_locked(g, addr, addr + length); if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { From cbdb49cf353b3ee5bb162dcb22363458bec8897e Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 23:12:03 +0800 Subject: [PATCH 8/9] Optimize single-page mprotect fast path --- docs/internals.md | 11 +++ src/core/guest.c | 2 + src/core/guest.h | 7 ++ src/core/mmap-fastpath.h | 18 +++-- src/core/shim-globals.c | 2 + src/core/shim.S | 146 +++++++++++++++++++++++++++++++++++ src/syscall/mem.c | 161 ++++++++++++++++++++++++++------------- tests/bench-mmap.c | 43 ++++++++++- tests/test-negative.c | 43 +++++++++++ 9 files changed, 373 insertions(+), 60 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index 4829c238..136fc3a4 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -226,6 +226,17 @@ Splitting is triggered by: block needs splitting, splits it if so, then updates the affected L3 page entries. Whole-block permission changes are done in place without splitting. +After a private anonymous mmap-arena block already has an L3 table, the EL1 +shim specializes a one-page `PROT_READ` ↔ `PROT_READ|PROT_WRITE` transition: +it validates the current arena generation, updates only AP[2:1], broadcasts +`TLBI VAE1IS`, and returns without an HVC exit. A tagged entry in the existing +mmap publication ring defers the matching region-metadata update to the next +host drain. The host always drains that ring before consulting `regions[]`; +the producer-active/PT-gate handshake prevents a host page-table writer from +racing the EL1 update. Lazy materialization, L2 splitting, executable or +`PROT_NONE` transitions, multi-page ranges, and non-arena mappings retain the +host path. + `mmap` itself uses a gap-finding allocator that walks the sorted region array to find free address space. `PROT_EXEC` requests go to the RX region (`MMAP_RX_BASE = 0x10000000`); other requests go to the RW region diff --git a/src/core/guest.c b/src/core/guest.c index 1b705f29..135f9e76 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1883,6 +1883,8 @@ void guest_reset(guest_t *g) g->mmap_rx_end = MMAP_RX_INITIAL_END; g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; + atomic_store_explicit(&g->mmap_fastpath_active_slots, 0, + memory_order_relaxed); g->ttbr0 = 0; memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); diff --git a/src/core/guest.h b/src/core/guest.h index b9580224..12f8d182 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -575,6 +575,13 @@ typedef struct { */ _Atomic uint64_t pt_gen; + /* Host-only index of shim mmap controls that are currently enabled. A bit + * is published only after the control descriptor is complete and cleared + * only while mmap_lock has closed the EL1 producer gate. This lets the + * common one-vCPU path avoid touching 63 cold control pages per drain. + */ + _Atomic uint64_t mmap_fastpath_active_slots; + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index 1992a42e..2b7cfb8a 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -24,6 +24,11 @@ typedef struct thread_entry thread_entry_t; #define SHIM_MMAP_RING_SIZE 32u #define SHIM_MMAP_CTRL_ENABLED 0x1u #define SHIM_MMAP_CTRL_TLBIRANGE 0x2u +/* Host mprotect changed region/PTE shape inside this arena. mmap remains safe, + * and EL1 mprotect validates L3 directly, but fast munmap falls back so its + * cleanup is not deferred across the changed metadata. + */ +#define SHIM_MMAP_CTRL_NO_FAST_MUNMAP 0x4u /* Host page-table writers set this gate before changing an arena descriptor or * a stage-1 PTE. Each EL1 producer announces itself in its private control @@ -43,6 +48,11 @@ typedef struct thread_entry thread_entry_t; #define SHIM_MMAP_REUSE_RING_SIZE 32u #define SHIM_MMAP_REUSE_OFF 0x720u +/* The mmap publication ring also carries completed EL1 single-page mprotect + * metadata updates. The tag lives outside the Linux PROT_* namespace. + */ +#define SHIM_MMAP_ENTRY_MPROTECT (1ULL << 63) + #define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) #define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) #define MMAP_FAST_ARENA_TARGET_ENTRIES 32u @@ -187,14 +197,6 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, */ void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); -/* Revoke only arenas whose reserved VA span intersects [start, end). The - * caller holds mmap_lock, whose acquisition has already drained publication - * and retirement rings and closed the EL1 page-table gate. - */ -void mmap_fastpath_revoke_range_locked(guest_t *g, - uint64_t start, - uint64_t end); - /* Disable the feature before first guest entry (debugger/observability). */ void mmap_fastpath_disable(guest_t *g); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 6fce6844..12dee166 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -211,6 +211,8 @@ void shim_globals_init(guest_t *g) */ memset(cache_base(g), 0, SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); + atomic_store_explicit(&g->mmap_fastpath_active_slots, 0, + memory_order_relaxed); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) diff --git a/src/core/shim.S b/src/core/shim.S index 34b2b6cc..8cf8e9d2 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -466,6 +466,151 @@ svc_handler: b.eq munmap_anon_fast cmp x10, #222 /* SYS_mmap? */ b.eq mmap_anon_fast + cmp x10, #226 /* SYS_mprotect? */ + b.eq mprotect_page_fast + b handle_svc_0 + +/* mprotect_page_fast: change one already-materialized L3 page between R and + * RW without exiting to the host. The mapping must belong to this vCPU's + * enabled anonymous arena. PTE + broadcast TLBI complete synchronously here; + * a tagged mmap-ring entry defers only the host region metadata update. + * Anything requiring L2 splitting, lazy materialization, PROT_NONE, execute + * permission, or broader validation falls back to HVC. + */ +mprotect_page_fast: + mrs x12, tpidr_el1 + ldar w13, [x12] /* attention / tracing gate */ + cbnz w13, handle_svc_0 + + add x20, sp, #0xfff + and x20, x20, #~0xfff /* this vCPU's stack top */ + add x21, x12, #0x200, lsl #12 + sub x21, x21, x20 + add x21, x12, x21 + add x21, x21, #0x20, lsl #12 /* current mmap control */ + + ldr x14, [sp, #0] /* addr */ + tst x14, #0xfff + b.ne handle_svc_0 + ldr x15, [sp, #8] /* Linux accepts len 1..4096 */ + cbz x15, handle_svc_0 + cmp x15, #0x1000 + b.hi handle_svc_0 + ldr x16, [sp, #16] /* prot */ + cmp x16, #1 /* PROT_READ */ + b.eq 1f + cmp x16, #3 /* PROT_READ|PROT_WRITE */ + b.ne handle_svc_0 + +1: /* Join the host-writer gate before validating the control or PTE. */ + add x20, x12, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, handle_svc_0 + add x24, x21, #0x400 + add x24, x24, #24 /* retire.producer_active */ + mov w22, #1 + stlr w22, [x24] + ldar w22, [x20] + cbnz w22, mprotect_active_bail + + ldar w22, [x21] /* generation */ + ldr w23, [x21, #4] /* consumer generation */ + cmp w22, w23 + b.ne mprotect_active_bail + ldr w23, [x21, #8] /* control flags */ + tbz w23, #0, mprotect_active_bail + + ldr x25, [x21, #24] /* arena base */ + cmp x14, x25 + b.lo mprotect_active_bail + ldr x25, [x21, #40] /* consumed cursor */ + add x17, x14, #0x1000 + cmp x17, x25 + b.hi mprotect_active_bail + + /* Reserve the same SPSC publication ring used by fast mmap. */ + add x23, x21, #12 /* host head */ + ldar w22, [x23] + ldr w23, [x21, #16] /* producer tail */ + sub w25, w23, w22 + cmp w25, #32 + b.hs mprotect_active_bail + + /* Walk TTBR0 and require L0/L1/L2 table descriptors plus a valid L3 page. + * A block or invalid descriptor needs host materialization/splitting. + */ + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne mprotect_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne mprotect_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #21, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne mprotect_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x28, x14, #12, #9 + add x26, x25, x28, lsl #3 /* target L3 entry */ + ldar x27, [x26] + tbz x27, #0, mprotect_active_bail + + /* Preserve output address and attributes; replace only AP[2:1]. */ + bic x27, x27, #0xc0 + cmp x16, #3 + mov x28, #0xc0 /* AP=11: EL0 read-only */ + mov x29, #0x40 /* AP=01: EL0 read-write */ + csel x28, x29, x28, eq + orr x27, x27, x28 + stlr x27, [x26] + + dsb ishst + ubfx x25, x14, #12, #44 + tlbi vae1is, x25 + dsb ish + isb + + /* Tag the publication as metadata-only mprotect. Host drains it before + * consulting regions[] on any later VM exit. + */ + and w22, w23, #31 + add x22, x22, x22, lsl #1 + add x25, x21, #64 + add x25, x25, x22, lsl #3 + str x14, [x25, #0] + mov x22, #0x1000 + str x22, [x25, #8] + mov x22, #1 + lsl x22, x22, #63 + orr x22, x22, x16 + str x22, [x25, #16] + add w23, w23, #1 + add x25, x21, #16 + stlr w23, [x25] + + stlr wzr, [x24] /* producer_active = 0 */ + mov x0, #0 + b svc_restore_eret + +mprotect_active_bail: + stlr wzr, [x24] b handle_svc_0 /* mmap_anon_fast: consume a host-prepared, per-vCPU VA arena for the exact @@ -753,6 +898,7 @@ munmap_anon_fast: 1: add x26, x25, #8 ldar w23, [x26] /* arena flags */ tbz w23, #0, 2f + tbnz w23, #2, 2f /* host mprotect => slow munmap */ ldar w22, [x25] /* generation snapshot */ ldr x26, [x25, #24] /* arena base */ cmp x14, x26 diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 0e684f2e..817340ca 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -87,6 +87,20 @@ static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); } +static uint64_t mmap_fastpath_active_slots(const guest_t *g) +{ + return g ? atomic_load_explicit(&g->mmap_fastpath_active_slots, + memory_order_acquire) + : 0; +} + +static int mmap_fastpath_pop_slot(uint64_t *slots) +{ + int slot = __builtin_ctzll(*slots); + *slots &= *slots - 1; + return slot; +} + static void mmap_fastpath_reuse_reset(shim_mmap_control_t *c) { if (!c) @@ -217,7 +231,9 @@ static void mmap_fastpath_host_gate_close(guest_t *g) atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); if (previous != 0) return; - for (int slot = 0; slot < MAX_THREADS; slot++) { + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *c = mmap_fastpath_control(g, slot); while (atomic_load_explicit(&c->retire.producer_active, memory_order_acquire) != 0) @@ -243,8 +259,11 @@ static void mmap_fastpath_host_gate_open(guest_t *g) static _Thread_local guest_t *mmap_lock_guest; -static void mmap_fastpath_disable_control(shim_mmap_control_t *c) +static void mmap_fastpath_disable_control(guest_t *g, int slot) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!c) + return; uint32_t generation = atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; if (generation == 0) @@ -261,14 +280,39 @@ static void mmap_fastpath_disable_control(shim_mmap_control_t *c) c->next_arena_size = MMAP_FAST_ARENA_MIN; c->max_len_seen = 0; atomic_store_explicit(&c->generation, generation, memory_order_release); + atomic_fetch_and_explicit(&g->mmap_fastpath_active_slots, ~BIT64(slot), + memory_order_release); +} + +static void mmap_fastpath_mark_mprotect_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start < limit && end > base) + atomic_fetch_or_explicit(&c->flags, + SHIM_MMAP_CTRL_NO_FAST_MUNMAP, + memory_order_release); + } } -static void mmap_fastpath_drain_publications_locked(guest_t *g) +static void mmap_fastpath_drain_publications_locked(guest_t *g, + uint64_t slots) { if (!g || !g->host_base) return; - for (int slot = 0; slot < MAX_THREADS; slot++) { + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *c = mmap_fastpath_control(g, slot); uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); @@ -288,6 +332,27 @@ static void mmap_fastpath_drain_publications_locked(guest_t *g) &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; uint64_t addr = e->addr; uint64_t len = e->len; + if (e->prot & SHIM_MMAP_ENTRY_MPROTECT) { + uint64_t prot = e->prot & ~SHIM_MMAP_ENTRY_MPROTECT; + if ((addr & (GUEST_PAGE_SIZE - 1)) || + len != GUEST_PAGE_SIZE || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + (prot != LINUX_PROT_READ && + prot != (LINUX_PROT_READ | LINUX_PROT_WRITE))) { + log_fatal( + "mprotect fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx prot=0x%llx)", + slot, (unsigned long long) addr, + (unsigned long long) len, + (unsigned long long) prot); + } + guest_region_set_prot(g, addr, addr + len, (int) prot); + if (prot & LINUX_PROT_WRITE) + guest_dirty_mark_range(g, addr, addr + len); + guest_pt_gen_bump(g); + head++; + continue; + } if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || addr > arena_limit || len > arena_limit - addr || @@ -466,16 +531,21 @@ void mmap_fastpath_drain_locked(guest_t *g) * then publication drain establishes A's semantic region before removal. */ uint32_t retire_tails[MAX_THREADS]; - for (int slot = 0; slot < MAX_THREADS; slot++) { + uint64_t active_slots = mmap_fastpath_active_slots(g); + uint64_t slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *c = mmap_fastpath_control(g, slot); retire_tails[slot] = atomic_load_explicit(&c->retire.tail, memory_order_acquire); } - mmap_fastpath_drain_publications_locked(g); + mmap_fastpath_drain_publications_locked(g, active_slots); bool retired_any = false; - for (int slot = 0; slot < MAX_THREADS; slot++) { + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); uint32_t head = atomic_load_explicit(&producer->retire.head, memory_order_relaxed); @@ -616,7 +686,9 @@ void mmap_fastpath_drain_locked(guest_t *g) * envelope after the first commit would make later entries in the same * snapshot lose their backing bounds. Restore the PTE-empty proof only * after every snapshotted retirement has consumed the old envelope. */ - for (int slot = 0; slot < MAX_THREADS; slot++) { + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) @@ -716,7 +788,9 @@ void mmap_fastpath_note_materialized_locked(guest_t *g, { if (!g || end <= start) return; - for (int slot = 0; slot < MAX_THREADS; slot++) { + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *c = mmap_fastpath_control(g, slot); if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) @@ -815,11 +889,12 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, { if (!t || t->sp_el1_slot < 0) return; - shim_mmap_control_t *c = mmap_fastpath_control(g, t->sp_el1_slot); + int slot = t->sp_el1_slot; + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); if (!c) return; if (!mmap_fastpath_available(g)) { - mmap_fastpath_disable_control(c); + mmap_fastpath_disable_control(g, slot); return; } @@ -879,12 +954,12 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, if (!recycled) { if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { - mmap_fastpath_disable_control(c); + mmap_fastpath_disable_control(g, slot); return; } base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { - mmap_fastpath_disable_control(c); + mmap_fastpath_disable_control(g, slot); return; } } @@ -896,7 +971,7 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, */ if (recycled || base < g->mmap_end) { if (guest_invalidate_ptes(g, base, new_limit) < 0) { - mmap_fastpath_disable_control(c); + mmap_fastpath_disable_control(g, slot); return; } } @@ -936,6 +1011,8 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->consumer_generation, generation, memory_order_relaxed); atomic_store_explicit(&c->generation, generation, memory_order_release); + atomic_fetch_or_explicit(&g->mmap_fastpath_active_slots, BIT64(slot), + memory_order_release); } void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) @@ -1008,7 +1085,10 @@ bool mmap_fastpath_allocate_current_publication_only(guest_t *g, if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) goto miss; - for (int slot = 0; slot < MAX_THREADS; slot++) { + uint64_t slots = mmap_fastpath_active_slots(g); + uint64_t pending_slots = slots; + while (pending_slots) { + int slot = mmap_fastpath_pop_slot(&pending_slots); shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); uint32_t head = atomic_load_explicit(&producer->retire.head, memory_order_relaxed); @@ -1018,7 +1098,7 @@ bool mmap_fastpath_allocate_current_publication_only(guest_t *g, goto miss; } - mmap_fastpath_drain_publications_locked(g); + mmap_fastpath_drain_publications_locked(g, slots); shim_mmap_control_t *c = mmap_fastpath_control(g, current_thread->sp_el1_slot); @@ -1095,7 +1175,7 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, * the explicit hint must take precedence over every EL1-reserved hole, and * the post-syscall refill will provision a new non-overlapping arena. */ - mmap_fastpath_disable_control(c); + mmap_fastpath_disable_control(g, current_thread->sp_el1_slot); } /* Reuse a fully released arena in place. mmap_lock acquisition has drained @@ -1163,8 +1243,11 @@ void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) { mmap_fastpath_drain_locked(g); - for (int slot = 0; slot < MAX_THREADS; slot++) - mmap_fastpath_disable_control(mmap_fastpath_control(g, slot)); + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + mmap_fastpath_disable_control(g, slot); + } if (!shrink_high_water) return; @@ -1179,33 +1262,6 @@ void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) g->mmap_rw_gap_hint = high; } -void mmap_fastpath_revoke_range_locked(guest_t *g, - uint64_t start, - uint64_t end) -{ - if (!g || end <= start) - return; - - /* mmap_lock_acquire() closed the producer gate and drained every ring - * before the syscall reached this point. An arena outside the edited VA - * range retains the same anonymous/noreserve classification and can keep - * serving its owner; only an overlapping arena could let a later EL1 - * munmap act on metadata changed by this mprotect. - */ - for (int slot = 0; slot < MAX_THREADS; slot++) { - shim_mmap_control_t *c = mmap_fastpath_control(g, slot); - if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & - SHIM_MMAP_CTRL_ENABLED)) - continue; - uint64_t base = - atomic_load_explicit(&c->arena_base, memory_order_relaxed); - uint64_t limit = - atomic_load_explicit(&c->arena_limit, memory_order_relaxed); - if (start < limit && end > base) - mmap_fastpath_disable_control(c); - } -} - void mmap_fastpath_disable(guest_t *g) { atomic_store_explicit(&mmap_fastpath_forced_off, true, @@ -1229,7 +1285,9 @@ void mmap_fastpath_skip_reserved(const guest_t *g, if (*start > max_addr || length > max_addr - *start) return; uint64_t end = *start + length; - for (int slot = 0; slot < MAX_THREADS; slot++) { + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); shim_mmap_control_t *c = mmap_fastpath_control(g, slot); if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & SHIM_MMAP_CTRL_ENABLED)) @@ -5147,11 +5205,12 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; - /* Permission and VMA-shape changes are slow-path boundaries. Retire any - * already-published unmaps, then invalidate arena generations before the - * metadata/PTE edit so EL1 cannot act on the old anonymous classification. + /* mmap_lock acquisition already drained EL1 publications. Permission-only + * edits do not change an arena mapping's private/anonymous/noreserve + * classification, so keep its control enabled: once this call has created + * an L3 table, later single-page R<->RW transitions can stay in EL1. */ - mmap_fastpath_revoke_range_locked(g, addr, addr + length); + mmap_fastpath_mark_mprotect_locked(g, addr, addr + length); if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index a9c73f4e..a48282ac 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -358,10 +358,45 @@ static void bench_mprotect_split(void) if (it >= 0) sp[it] = ns(t1 - t0); } - printf(" split: median %.1f ns min %.1f ns\n\n", median(sp, ITERS), + printf(" split: median %.1f ns min %.1f ns\n", median(sp, ITERS), sp[0]); } +/* Once the block has an L3 table, R<->RW on one anonymous arena page can stay + * in EL1 and defer only region metadata through the publication ring. + */ +static void bench_mprotect_fast_leaf(void) +{ + double leaf[ITERS]; + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" fast leaf mmap FAILED\n\n"); + return; + } + if (mprotect(p + MIB, 4 * KIB, PROT_READ) != 0) { + munmap(p, 2 * MIB); + printf(" fast leaf setup FAILED: %s\n\n", strerror(errno)); + return; + } + for (int it = -1; it < ITERS; it++) { + int prot = (it & 1) ? PROT_READ : PROT_READ | PROT_WRITE; + uint64_t t0 = rd(); + int rc = mprotect(p + MIB, 4 * KIB, prot); + uint64_t t1 = rd(); + if (rc != 0) { + munmap(p, 2 * MIB); + printf(" fast leaf mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + leaf[it] = ns(t1 - t0); + } + munmap(p, 2 * MIB); + printf(" fast leaf: median %.1f ns min %.1f ns\n\n", + median(leaf, ITERS), leaf[0]); +} + /* E. mremap grow: in-place vs forced move */ static void bench_mremap(void) { @@ -686,6 +721,11 @@ static void bench_munmap_dirty(void) int main(int argc, char **argv) { clock_init(); + if (argc == 2 && strcmp(argv[1], "d") == 0) { + bench_mprotect_split(); + bench_mprotect_fast_leaf(); + return 0; + } if (argc == 2 && strcmp(argv[1], "a") == 0) { printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); @@ -746,6 +786,7 @@ int main(int argc, char **argv) bench_fresh(); bench_fault(512, 0); bench_mprotect_split(); + bench_mprotect_fast_leaf(); bench_mremap(); bench_mt(); bench_munmap_materialized(); diff --git a/tests/test-negative.c b/tests/test-negative.c index 3470147e..edbc9a49 100644 --- a/tests/test-negative.c +++ b/tests/test-negative.c @@ -208,6 +208,49 @@ static void test_mmap_prot(void) munmap(p, 4096); EXPECT_TRUE(saw_segv, "write to read-only mapping succeeded"); } + + TEST("fast single-page mprotect drains publication ring"); + { + volatile int *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap failed"); + return; + } + *p = 1; /* Materialize a partial block, which installs an L3 table. */ + for (int i = 0; i < 96; i++) { + int prot = (i & 1) ? PROT_READ : PROT_READ | PROT_WRITE; + if (mprotect((void *) p, 4096, prot) != 0) { + munmap((void *) p, 4096); + FAIL("mprotect toggle failed"); + return; + } + } + + struct sigaction old_sa; + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = on_sigsegv; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + munmap((void *) p, 4096); + FAIL("sigaction failed"); + return; + } + saw_segv = 0; + if (sigsetjmp(segv_jmp, 1) == 0) + *p = 2; + sigaction(SIGSEGV, &old_sa, NULL); + + int restored = mprotect((void *) p, 4096, + PROT_READ | PROT_WRITE) == 0; + if (restored) + *p = 3; + int final = *p; + munmap((void *) p, 4096); + EXPECT_TRUE(saw_segv && restored && final == 3, + "fast mprotect permissions or ring drain were stale"); + } } /* Test 7: fcntl on invalid FD */ From a58f5bbc5f229168ca8759b01cc07e1fc39d954e Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 23:56:41 +0800 Subject: [PATCH 9/9] Accelerate initial mprotect block splits --- docs/internals.md | 30 +++-- src/core/guest.c | 82 ++++++++++++ src/core/guest.h | 13 ++ src/core/mmap-fastpath.h | 24 ++++ src/core/shim-globals.c | 46 ++++++- src/core/shim.S | 263 +++++++++++++++++++++++++++++++++++++-- src/syscall/mem.c | 131 ++++++++++++++++--- tests/bench-mmap.c | 26 +++- tests/test-negative.c | 65 +++++++++- 9 files changed, 629 insertions(+), 51 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index 136fc3a4..d25a78bb 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -226,16 +226,26 @@ Splitting is triggered by: block needs splitting, splits it if so, then updates the affected L3 page entries. Whole-block permission changes are done in place without splitting. -After a private anonymous mmap-arena block already has an L3 table, the EL1 -shim specializes a one-page `PROT_READ` ↔ `PROT_READ|PROT_WRITE` transition: -it validates the current arena generation, updates only AP[2:1], broadcasts -`TLBI VAE1IS`, and returns without an HVC exit. A tagged entry in the existing -mmap publication ring defers the matching region-metadata update to the next -host drain. The host always drains that ring before consulting `regions[]`; -the producer-active/PT-gate handshake prevents a host page-table writer from -racing the EL1 update. Lazy materialization, L2 splitting, executable or -`PROT_NONE` transitions, multi-page ranges, and non-arena mappings retain the -host path. +For a private anonymous mmap-arena block, the EL1 shim specializes a one-page +`PROT_READ` ↔ `PROT_READ|PROT_WRITE` transition. Arena refill prepares the +upper TTBR0 chain, a per-vCPU batch of 32 L3 table pages, and a clean-backing +bitmap for a rolling window of 32 two-megabyte blocks. A slow-path miss moves +that window to the current allocation area. If an undrained mmap publication +proves that one anonymous RW mapping owns a complete clean block, EL1 may +populate a prepared L3 page and install the first table descriptor itself. A +previously retired, empty L3 table is reused in place. Existing valid L3 leaves +need only an AP[2:1] update. + +The shim issues `TLBI RVAE1IS` for first materialization or `TLBI VAE1IS` for +an existing leaf and returns without an HVC exit. A tagged entry in the mmap +publication ring defers region metadata plus host-only PTE/dirty-index +reconciliation to the next drain. The producer-active/PT-gate handshake +prevents a host page-table writer from racing the EL1 update. Dirty backing, +but still-lazy target pages use a metadata-only tag and keep their PTE invalid; +their eventual fault drains the new protection before materializing data. +Partial ownership, an already-consumed publication, an exhausted table cache, +executable or `PROT_NONE` transitions, multi-page ranges, and non-arena mappings +retain the host path. `mmap` itself uses a gap-finding allocator that walks the sorted region array to find free address space. `PROT_EXEC` requests go to the RX region diff --git a/src/core/guest.c b/src/core/guest.c index 135f9e76..7cd93bab 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -220,6 +220,38 @@ static uint64_t pt_alloc_page_uninitialized(guest_t *g) return pt_alloc_page_impl(g, false); } +uint64_t guest_reserve_pt_pages_uninitialized(guest_t *g, unsigned count) +{ + if (!g || count == 0) + return 0; + uint64_t bytes = (uint64_t) count * PAGE_SIZE; + + pthread_mutex_lock(&pt_lock); + if (g->pt_pool_next > g->pt_pool_end || + bytes > g->pt_pool_end - g->pt_pool_next) { + log_error( + "guest: page table pool exhausted reserving %u pages " + "(used %llu / %llu bytes)", + count, (unsigned long long) (g->pt_pool_next - g->pt_pool_base), + (unsigned long long) (g->pt_pool_end - g->pt_pool_base)); + pthread_mutex_unlock(&pt_lock); + return 0; + } + uint64_t gpa = g->pt_pool_next; + g->pt_pool_next += bytes; + + uint64_t used = g->pt_pool_next - g->pt_pool_base; + uint64_t total = g->pt_pool_end - g->pt_pool_base; + if (!pt_pool_warned && used > (total * 4 / 5)) { + log_debug("guest: page table pool at %llu%% (%llu / %llu bytes)", + (unsigned long long) (used * 100 / total), + (unsigned long long) used, (unsigned long long) total); + pt_pool_warned = true; + } + pthread_mutex_unlock(&pt_lock); + return gpa; +} + /* Get host pointer to a page table entry array at a given GPA */ static uint64_t *pt_at(const guest_t *g, uint64_t gpa) { @@ -3013,6 +3045,56 @@ int guest_extend_page_tables(guest_t *g, return 0; } +int guest_prepare_l2_tables(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || !g->ttbr0 || start >= end || end > g->guest_size || + end > UINT64_MAX - (BLOCK_2MIB - 1)) + return -1; + + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + uint64_t addr_start = ALIGN_2MIB_DOWN(start); + uint64_t addr_end = ALIGN_2MIB_UP(end); + + for (uint64_t addr = addr_start; addr < addr_end;) { + uint64_t ipa = base + addr; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return -1; + + if (!(pte_load_acquire(&l0[l0_idx]) & PT_VALID)) { + uint64_t l1_gpa = pt_alloc_page(g); + if (!l1_gpa) + return -1; + pte_store_release(&l0[l0_idx], + (base + l1_gpa) | PT_VALID | PT_TABLE); + } + uint64_t l0e = pte_load_acquire(&l0[l0_idx]); + if ((l0e & 3) != (PT_VALID | PT_TABLE)) + return -1; + uint64_t *l1 = pt_at(g, (l0e & 0xFFFFFFFFF000ULL) - base); + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + + if (!(pte_load_acquire(&l1[l1_idx]) & PT_VALID)) { + uint64_t l2_gpa = pt_alloc_page(g); + if (!l2_gpa) + return -1; + pte_store_release(&l1[l1_idx], + (base + l2_gpa) | PT_VALID | PT_TABLE); + } + uint64_t l1e = pte_load_acquire(&l1[l1_idx]); + if ((l1e & 3) != (PT_VALID | PT_TABLE)) + return -1; + + uint64_t next = ALIGN_UP(addr + 1, BLOCK_1GIB); + if (next <= addr || next > addr_end) + next = addr_end; + addr = next; + } + return 0; +} + static uint64_t guest_va_next_page_table_block(const guest_t *g, uint64_t va, uint64_t end) diff --git a/src/core/guest.h b/src/core/guest.h index 12f8d182..1c2c5ca2 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -1216,6 +1216,19 @@ int guest_extend_page_tables(guest_t *g, uint64_t end, int perms); +/* Prepare only the upper TTBR0 chain needed to reach L2 entries in + * [start,end); all L2 leaf descriptors remain untouched/invalid. This lets + * the host provision an arena for a later EL1-owned L3 install without ever + * transiently mapping unallocated user memory. + */ +int guest_prepare_l2_tables(guest_t *g, uint64_t start, uint64_t end); + +/* Reserve a contiguous run of uninitialized 4 KiB pages from the page-table + * pool. The caller/EL1 producer must overwrite every entry before publishing + * any page as a table descriptor. Returns the first GPA, or 0 on exhaustion. + */ +uint64_t guest_reserve_pt_pages_uninitialized(guest_t *g, unsigned count); + /* Split a 2MiB block descriptor into 512 x 4KiB L3 page descriptors. block_gpa * must be within a currently-mapped 2MiB block. The block's permissions are * inherited by all 512 page entries. If the block is already split (L2 entry is diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index 2b7cfb8a..df1178d2 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -52,6 +52,22 @@ typedef struct thread_entry thread_entry_t; * metadata updates. The tag lives outside the Linux PROT_* namespace. */ #define SHIM_MMAP_ENTRY_MPROTECT (1ULL << 63) +/* The mprotect also installed the first L3 table for a clean, fully-owned + * 2 MiB lazy block. The host must reconcile its PTE/dirty indexes when the + * deferred metadata entry drains. + */ +#define SHIM_MMAP_ENTRY_MPROTECT_SPLIT (1ULL << 62) +/* Permission metadata changed while the target PTE remained lazy/invalid. + * The next fault drains the entry before consulting regions[]. + */ +#define SHIM_MMAP_ENTRY_MPROTECT_LAZY (1ULL << 61) + +/* A cache batch covers a rolling 32-block (64 MiB) arena window. Unused pages + * stay attached to the vCPU across arena generations, so refills do not leak a + * batch each time the VA reservation rolls over. + */ +#define SHIM_MMAP_L3_CACHE_PAGES 32u +#define SHIM_MMAP_SPLIT_CLEAN_BLOCKS 32u #define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) #define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) @@ -134,6 +150,14 @@ typedef struct { uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; munmap_retire_ring_t retire; mmap_reuse_ring_t reuse; + _Atomic uint64_t l3_cache_next; /* EL1-owned bump cursor (GPA) */ + uint64_t l3_cache_end; /* host-published exclusive end */ + uint64_t split_clean_base; /* 2 MiB-aligned arena window */ + uint64_t split_clean_bitmap; /* one clean-backing bit per block */ + uint64_t split_prep_miss; + uint64_t split_owner_miss; + uint64_t split_hit; + uint64_t lazy_mprotect_hit; } shim_mmap_control_t; _Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 12dee166..eef3374b 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -160,6 +160,19 @@ _Static_assert(offsetof(mmap_reuse_ring_t, published) == 8 && "shim.S mmap reuse-ring layout drift"); _Static_assert(SHIM_MMAP_REUSE_RING_SIZE == 32, "shim.S mmap reuse-ring size drift"); +_Static_assert(offsetof(shim_mmap_control_t, l3_cache_next) == 0x940 && + offsetof(shim_mmap_control_t, l3_cache_end) == 0x948 && + offsetof(shim_mmap_control_t, split_clean_base) == 0x950 && + offsetof(shim_mmap_control_t, split_clean_bitmap) == 0x958, + "shim.S mmap L3-cache layout drift"); +_Static_assert(offsetof(shim_mmap_control_t, split_prep_miss) == 0x960 && + offsetof(shim_mmap_control_t, split_owner_miss) == 0x968 && + offsetof(shim_mmap_control_t, split_hit) == 0x970 && + offsetof(shim_mmap_control_t, lazy_mprotect_hit) == 0x978, + "shim.S mmap L3 diagnostic layout drift"); +_Static_assert(SHIM_MMAP_L3_CACHE_PAGES == 32 && + SHIM_MMAP_SPLIT_CLEAN_BLOCKS == 32, + "shim.S mmap L3-cache constants drift"); _Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), @@ -572,6 +585,9 @@ void shim_globals_counters_dump(const guest_t *g) uint64_t reuse_published = 0, reuse_dropped = 0, reuse_hits = 0; uint64_t reuse_pending = 0; uint64_t current_max = 0, peak_max = 0; + uint64_t l3_cache_available = 0, split_clean_blocks = 0; + uint64_t split_prep_miss = 0, split_owner_miss = 0, split_hit = 0; + uint64_t lazy_mprotect_hit = 0; const uint8_t *shim_data = (const uint8_t *) g->host_base + g->shim_data_base; for (int slot = 0; slot < MAX_THREADS; slot++) { @@ -584,10 +600,10 @@ void shim_globals_counters_dump(const guest_t *g) atomic_load_explicit(&c->counters[i], memory_order_relaxed); refill_count += c->refill_count; recycle_count += c->recycle_count; - reuse_published += atomic_load_explicit(&c->reuse.published, - memory_order_relaxed); - reuse_dropped += atomic_load_explicit(&c->reuse.dropped, - memory_order_relaxed); + reuse_published += + atomic_load_explicit(&c->reuse.published, memory_order_relaxed); + reuse_dropped += + atomic_load_explicit(&c->reuse.dropped, memory_order_relaxed); reuse_hits += atomic_load_explicit(&c->reuse.hits, memory_order_relaxed); uint32_t reuse_head = @@ -599,6 +615,16 @@ void shim_globals_counters_dump(const guest_t *g) current_max = c->next_arena_size; if (c->peak_arena_size > peak_max) peak_max = c->peak_arena_size; + uint64_t l3_next = + atomic_load_explicit(&c->l3_cache_next, memory_order_relaxed); + if (l3_next < c->l3_cache_end) + l3_cache_available += (c->l3_cache_end - l3_next) / 4096; + split_clean_blocks += + (uint64_t) __builtin_popcountll(c->split_clean_bitmap); + split_prep_miss += c->split_prep_miss; + split_owner_miss += c->split_owner_miss; + split_hit += c->split_hit; + lazy_mprotect_hit += c->lazy_mprotect_hit; } for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], @@ -619,6 +645,18 @@ void shim_globals_counters_dump(const guest_t *g) (unsigned long long) current_max); fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", (unsigned long long) peak_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_L3_AVAILABLE", + (unsigned long long) l3_cache_available); + fprintf(stderr, " %-20s %llu\n", "MMAP_SPLIT_CLEAN", + (unsigned long long) split_clean_blocks); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_PREP_MISS", + (unsigned long long) split_prep_miss); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_OWNER_MISS", + (unsigned long long) split_owner_miss); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_HIT", + (unsigned long long) split_hit); + fprintf(stderr, " %-20s %llu\n", "MPROT_LAZY_HIT", + (unsigned long long) lazy_mprotect_hit); uint64_t high_water = g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", diff --git a/src/core/shim.S b/src/core/shim.S index 8cf8e9d2..73a390b5 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -273,6 +273,16 @@ .Lmmap_ctr_skip_\@: .endm +.macro MPROTECT_COUNTER_INC byte_off + mrs x29, tpidr_el1 + ldrb w30, [x29, #SHIM_STATS_EN_OFF] + cbz w30, .Lmprotect_ctr_skip_\@ + ldr x29, [x21, #\byte_off] + add x29, x29, #1 + str x29, [x21, #\byte_off] +.Lmprotect_ctr_skip_\@: +.endm + /* BAD_VEC: vector-table entry that reports an unexpected exception. * Each table slot is 128 bytes; the leading .align 7 places this entry at the * next 128-byte boundary. @@ -470,12 +480,13 @@ svc_handler: b.eq mprotect_page_fast b handle_svc_0 -/* mprotect_page_fast: change one already-materialized L3 page between R and - * RW without exiting to the host. The mapping must belong to this vCPU's - * enabled anonymous arena. PTE + broadcast TLBI complete synchronously here; - * a tagged mmap-ring entry defers only the host region metadata update. - * Anything requiring L2 splitting, lazy materialization, PROT_NONE, execute - * permission, or broader validation falls back to HVC. +/* mprotect_page_fast: change one L3 page between R and RW without exiting to + * the host. Besides an existing L3 leaf, EL1 may consume a host-prepared table + * page to split an existing L2 block or materialize a clean 2 MiB lazy block. + * The latter is allowed only while an undrained mmap publication proves that + * this vCPU owns the complete block; this prevents neighboring arena holes + * from becoming accessible. PTE + broadcast TLBI complete synchronously here; + * a tagged mmap-ring entry defers host region/index reconciliation. */ mprotect_page_fast: mrs x12, tpidr_el1 @@ -537,8 +548,8 @@ mprotect_page_fast: cmp w25, #32 b.hs mprotect_active_bail - /* Walk TTBR0 and require L0/L1/L2 table descriptors plus a valid L3 page. - * A block or invalid descriptor needs host materialization/splitting. + /* Walk TTBR0. Arena refill has prepared the upper chain for the first + * clean window, while older/out-of-window mappings may still bail here. */ mrs x25, ttbr0_el1 ubfx x25, x25, #12, #36 @@ -548,7 +559,7 @@ mprotect_page_fast: ldar x27, [x26] and x28, x27, #3 cmp x28, #3 - b.ne mprotect_active_bail + b.ne mprotect_split_prep_bail ubfx x25, x27, #12, #36 lsl x25, x25, #12 ubfx x26, x14, #30, #9 @@ -556,21 +567,196 @@ mprotect_page_fast: ldar x27, [x26] and x28, x27, #3 cmp x28, #3 - b.ne mprotect_active_bail + b.ne mprotect_split_prep_bail ubfx x25, x27, #12, #36 lsl x25, x25, #12 ubfx x26, x14, #21, #9 - add x26, x25, x26, lsl #3 + add x26, x25, x26, lsl #3 /* target L2 entry */ ldar x27, [x26] and x28, x27, #3 cmp x28, #3 - b.ne mprotect_active_bail + b.eq mprotect_existing_l3 + cmp x28, #1 + b.eq mprotect_install_l3_block + cbnz x28, mprotect_active_bail + mov w20, #0 /* needs a prepared table page */ + +mprotect_validate_lazy_block: + /* An invalid L2 slot can be materialized only when the host's snapshot + * says the identity backing is clean and a still-pending mmap publication + * covers this whole 2 MiB block. + */ + lsr x11, x14, #21 + lsl x11, x11, #21 /* block start */ + mov x17, x11 /* retain exact block for bookkeeping */ + ldr x12, [x21, #0x950] /* clean-window base */ + cmp x11, x12 + b.lo mprotect_split_prep_bail + sub x13, x11, x12 + lsr x13, x13, #21 /* clean-window bit */ + cmp x13, #32 + b.hs mprotect_split_prep_bail + ldr x12, [x21, #0x958] /* clean bitmap */ + lsrv x12, x12, x13 + and w9, w12, #1 /* full-block materialization is safe */ + + mov w10, w22 /* host head snapshot */ +mprotect_find_full_block_mapping: + cmp w10, w23 + b.eq mprotect_split_owner_bail + and w12, w10, #31 + add x12, x12, x12, lsl #1 + add x13, x21, #64 + add x12, x13, x12, lsl #3 + ldr x13, [x12, #16] /* mmap prot / entry tag */ + tbnz x13, #63, mprotect_find_full_block_next + cmp x13, #3 + b.ne mprotect_find_full_block_next + ldr x13, [x12, #0] /* mapping start */ + cmp x14, x13 + b.lo mprotect_find_full_block_next + ldr x12, [x12, #8] /* mapping length */ + adds x12, x13, x12 + b.cs mprotect_active_bail + add x15, x14, #0x1000 + cmp x12, x15 + b.lo mprotect_find_full_block_next + cbz w9, mprotect_publish_lazy_only + cmp x11, x13 + b.lo mprotect_find_full_block_next + add x13, x11, #0x200000 + cmp x12, x13 + b.hs mprotect_install_l3_lazy +mprotect_find_full_block_next: + add w10, w10, #1 + b mprotect_find_full_block_mapping + +mprotect_install_l3_block: + mov w18, #0 /* split preserves materialization */ + mov x10, x27 + orr x10, x10, #2 /* L2 block -> L3 page type */ + b mprotect_install_l3_common + +mprotect_install_l3_lazy: + mov w18, #1 /* first lazy materialization */ + mov x10, #0x767 /* AF|ISH|NS|ATTR1|RW_EL0|PAGE */ + movk x10, #0x60, lsl #48 /* UXN|PXN */ + orr x10, x10, x11 /* first identity page */ + cbnz w20, mprotect_fill_existing_empty_l3 + +mprotect_install_l3_common: + ldr x25, [x21, #0x940] /* prepared L3 page */ + ldr x29, [x21, #0x948] + cmp x25, x29 + b.hs mprotect_split_prep_bail + add x29, x25, #0x1000 + str x29, [x21, #0x940] /* sole EL1 consumer */ + MPROTECT_COUNTER_INC 0x970 + mov x19, x26 /* preserve target L2 entry */ + mov x12, x25 /* L3 fill cursor */ + mov x13, #512 +mprotect_fill_l3: + str x10, [x12], #8 + add x10, x10, #0x1000 + subs x13, x13, #1 + b.ne mprotect_fill_l3 + + /* Tighten the requested page before publishing the table. */ + ubfx x28, x14, #12, #9 + add x26, x25, x28, lsl #3 + ldr x27, [x26] + bic x27, x27, #0xc0 + cmp x16, #3 + mov x28, #0xc0 /* AP=11: EL0 read-only */ + mov x29, #0x40 /* AP=01: EL0 read-write */ + csel x28, x29, x28, eq + orr x27, x27, x28 + str x27, [x26] + orr x27, x25, #3 /* L2 table descriptor */ + stlr x27, [x19] + + b mprotect_lazy_l3_installed + +mprotect_fill_existing_empty_l3: + /* A previous materialize/retire cycle may leave an empty L3 table in the + * L2 slot. Prove every entry is zero before reusing it. The target page is + * installed with its final AP first so a sibling can never observe a + * transient RW permission while this mprotect requests read-only. + */ + mov x12, x25 + mov x13, #512 +1: ldr x27, [x12], #8 + cbnz x27, mprotect_split_prep_bail + subs x13, x13, #1 + b.ne 1b + + ubfx x28, x14, #12, #9 /* target page index */ + lsl x29, x28, #12 + add x27, x10, x29 /* target page descriptor */ + bic x27, x27, #0xc0 + cmp x16, #3 + mov x29, #0xc0 + mov x19, #0x40 + csel x29, x19, x29, eq + orr x27, x27, x29 + add x26, x25, x28, lsl #3 + stlr x27, [x26] + + mov x12, x25 + mov x13, #0 /* current page index */ +2: cmp x13, x28 + b.eq 3f + str x10, [x12] +3: add x12, x12, #8 + add x10, x10, #0x1000 + add x13, x13, #1 + cmp x13, #512 + b.lo 2b + MPROTECT_COUNTER_INC 0x970 + +mprotect_lazy_l3_installed: + + cbz w18, mprotect_pte_changed + + /* Make the materialized envelope visible before a following fast munmap, + * and consume the clean proof so it cannot authorize a second install. + */ + ldr w22, [x21] + ldr w12, [x21, #0x388] + add x13, x11, #0x200000 + cmp w12, w22 + b.ne 1f + ldr x10, [x21, #0x390] + ldr x12, [x21, #0x398] + cmp x10, x11 + csel x11, x10, x11, lo + cmp x12, x13 + csel x13, x12, x13, hi +1: str x11, [x21, #0x390] + str x13, [x21, #0x398] + add x10, x21, #0x388 + stlr w22, [x10] + ldr x10, [x21, #0x950] + sub x10, x17, x10 + lsr x10, x10, #21 + mov x12, #1 + lsl x12, x12, x10 + ldr x13, [x21, #0x958] + bic x13, x13, x12 + str x13, [x21, #0x958] + b mprotect_pte_changed + +mprotect_existing_l3: ubfx x25, x27, #12, #36 lsl x25, x25, #12 ubfx x28, x14, #12, #9 add x26, x25, x28, lsl #3 /* target L3 entry */ ldar x27, [x26] - tbz x27, #0, mprotect_active_bail + tbnz x27, #0, 1f + mov w20, #1 /* reuse this published empty table */ + b mprotect_validate_lazy_block +1: + mov w18, #0 /* Preserve output address and attributes; replace only AP[2:1]. */ bic x27, x27, #0xc0 @@ -581,12 +767,35 @@ mprotect_page_fast: orr x27, x27, x28 stlr x27, [x26] +mprotect_pte_changed: dsb ishst + cbnz w18, mprotect_tlbi_lazy_block ubfx x25, x14, #12, #44 tlbi vae1is, x25 + b mprotect_tlbi_done +mprotect_tlbi_lazy_block: + ldr w25, [x21, #8] + tbz w25, #1, mprotect_tlbi_full + lsr x25, x14, #21 + lsl x25, x25, #9 /* block VA >> 12 */ + mov x27, #7 /* 8 x 256 KiB scale units */ + lsl x27, x27, #39 /* NUM */ + orr x25, x25, x27 + mov x27, #1 + lsl x27, x27, #44 /* SCALE=1 */ + orr x25, x25, x27 + mov x27, #1 + lsl x27, x27, #46 /* TG=01 (4 KiB) */ + orr x25, x25, x27 + tlbi rvae1is, x25 + b mprotect_tlbi_done +mprotect_tlbi_full: + tlbi vmalle1is +mprotect_tlbi_done: dsb ish isb +mprotect_publish_entry: /* Tag the publication as metadata-only mprotect. Host drains it before * consulting regions[] on any later VM exit. */ @@ -600,6 +809,17 @@ mprotect_page_fast: mov x22, #1 lsl x22, x22, #63 orr x22, x22, x16 + cmp w18, #1 + b.ne 1f + mov x26, #1 + lsl x26, x26, #62 + orr x22, x22, x26 +1: cmp w18, #2 + b.ne 2f + mov x26, #1 + lsl x26, x26, #61 + orr x22, x22, x26 +2: str x22, [x25, #16] add w23, w23, #1 add x25, x21, #16 @@ -609,10 +829,27 @@ mprotect_page_fast: mov x0, #0 b svc_restore_eret +mprotect_publish_lazy_only: + /* The target belongs to a pending anonymous mapping but its backing is not + * proven clean. Keep the PTE invalid and defer only the VMA permission + * update; a later fault drains this entry before materializing the page. + */ + MPROTECT_COUNTER_INC 0x978 + mov w18, #2 + b mprotect_publish_entry + mprotect_active_bail: stlr wzr, [x24] b handle_svc_0 +mprotect_split_prep_bail: + MPROTECT_COUNTER_INC 0x960 + b mprotect_active_bail + +mprotect_split_owner_bail: + MPROTECT_COUNTER_INC 0x968 + b mprotect_active_bail + /* mmap_anon_fast: consume a host-prepared, per-vCPU VA arena for the exact * anonymous RW shape used by allocators: * diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 817340ca..be6e4054 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -101,6 +101,59 @@ static int mmap_fastpath_pop_slot(uint64_t *slots) return slot; } +static bool mmap_fastpath_provision_l3_cache_locked(guest_t *g, + shim_mmap_control_t *c) +{ + uint64_t next = + atomic_load_explicit(&c->l3_cache_next, memory_order_relaxed); + uint64_t end = c->l3_cache_end; + if (next < end) + return true; + if (end == UINT64_MAX) + return false; + + uint64_t first = + guest_reserve_pt_pages_uninitialized(g, SHIM_MMAP_L3_CACHE_PAGES); + if (!first) { + atomic_store_explicit(&c->l3_cache_next, UINT64_MAX, + memory_order_relaxed); + c->l3_cache_end = UINT64_MAX; + return false; + } + c->l3_cache_end = + first + (uint64_t) SHIM_MMAP_L3_CACHE_PAGES * GUEST_PAGE_SIZE; + atomic_store_explicit(&c->l3_cache_next, first, memory_order_release); + return true; +} + +static void mmap_fastpath_prepare_split_window_locked(guest_t *g, + shim_mmap_control_t *c, + uint64_t base, + uint64_t limit) +{ + c->split_clean_base = base; + c->split_clean_bitmap = 0; + if (base >= limit || (base & (BLOCK_2MIB - 1)) || + !mmap_fastpath_provision_l3_cache_locked(g, c)) + return; + + uint64_t window_end = limit; + uint64_t max_window = (uint64_t) SHIM_MMAP_SPLIT_CLEAN_BLOCKS * BLOCK_2MIB; + if (window_end - base > max_window) + window_end = base + max_window; + if (guest_prepare_l2_tables(g, base, window_end) < 0) + return; + + uint64_t clean = 0; + unsigned bit = 0; + for (uint64_t block = base; block < window_end; + block += BLOCK_2MIB, bit++) { + if (!guest_block_may_be_dirty(g, block)) + clean |= BIT64(bit); + } + c->split_clean_bitmap = clean; +} + static void mmap_fastpath_reuse_reset(shim_mmap_control_t *c) { if (!c) @@ -276,6 +329,8 @@ static void mmap_fastpath_disable_control(guest_t *g, int slot) atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->split_clean_base = 0; + c->split_clean_bitmap = 0; mmap_fastpath_reuse_reset(c); c->next_arena_size = MMAP_FAST_ARENA_MIN; c->max_len_seen = 0; @@ -298,15 +353,26 @@ static void mmap_fastpath_mark_mprotect_locked(guest_t *g, atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); - if (start < limit && end > base) - atomic_fetch_or_explicit(&c->flags, - SHIM_MMAP_CTRL_NO_FAST_MUNMAP, + if (start < limit && end > base) { + /* A long-lived adaptive arena can move its cursor many GiB beyond + * the 32-block window installed at refill. The current mprotect + * has already fallen back under the closed PT gate, so use that + * natural exit to roll the prepared window forward. Subsequent + * nearby allocator permission changes can then stay in EL1. + */ + uint64_t block = ALIGN_DOWN(start, BLOCK_2MIB); + uint64_t split_base = c->split_clean_base; + uint64_t split_span = + (uint64_t) SHIM_MMAP_SPLIT_CLEAN_BLOCKS * BLOCK_2MIB; + if (block < split_base || block - split_base >= split_span) + mmap_fastpath_prepare_split_window_locked(g, c, block, limit); + atomic_fetch_or_explicit(&c->flags, SHIM_MMAP_CTRL_NO_FAST_MUNMAP, memory_order_release); + } } } -static void mmap_fastpath_drain_publications_locked(guest_t *g, - uint64_t slots) +static void mmap_fastpath_drain_publications_locked(guest_t *g, uint64_t slots) { if (!g || !g->host_base) return; @@ -333,22 +399,38 @@ static void mmap_fastpath_drain_publications_locked(guest_t *g, uint64_t addr = e->addr; uint64_t len = e->len; if (e->prot & SHIM_MMAP_ENTRY_MPROTECT) { - uint64_t prot = e->prot & ~SHIM_MMAP_ENTRY_MPROTECT; - if ((addr & (GUEST_PAGE_SIZE - 1)) || - len != GUEST_PAGE_SIZE || addr < arena_base || - addr > arena_limit || len > arena_limit - addr || + bool installed_l3 = + (e->prot & SHIM_MMAP_ENTRY_MPROTECT_SPLIT) != 0; + bool stayed_lazy = + (e->prot & SHIM_MMAP_ENTRY_MPROTECT_LAZY) != 0; + uint64_t prot = e->prot & ~(SHIM_MMAP_ENTRY_MPROTECT | + SHIM_MMAP_ENTRY_MPROTECT_SPLIT | + SHIM_MMAP_ENTRY_MPROTECT_LAZY); + if ((addr & (GUEST_PAGE_SIZE - 1)) || len != GUEST_PAGE_SIZE || + addr < arena_base || addr > arena_limit || + len > arena_limit - addr || (prot != LINUX_PROT_READ && prot != (LINUX_PROT_READ | LINUX_PROT_WRITE))) { log_fatal( "mprotect fast path: invalid entry in vCPU slot %d " "(addr=0x%llx len=0x%llx prot=0x%llx)", slot, (unsigned long long) addr, - (unsigned long long) len, - (unsigned long long) prot); + (unsigned long long) len, (unsigned long long) prot); } guest_region_set_prot(g, addr, addr + len, (int) prot); - if (prot & LINUX_PROT_WRITE) + if (installed_l3) { + uint64_t block = ALIGN_DOWN(addr, BLOCK_2MIB); + guest_retire_ptes_committed(g, block, block + BLOCK_2MIB); + mmap_fastpath_note_materialized_locked(g, block, + block + BLOCK_2MIB); + /* The newly published table exposes the rest of the + * full-block anonymous mapping as RW. Conservatively mark + * all of its backing dirty before any later retirement. + */ + guest_dirty_mark_range(g, block, block + BLOCK_2MIB); + } else if (!stayed_lazy && (prot & LINUX_PROT_WRITE)) { guest_dirty_mark_range(g, addr, addr + len); + } guest_pt_gen_bump(g); head++; continue; @@ -714,6 +796,16 @@ void mmap_fastpath_drain_locked(guest_t *g) */ if (retired_any) guest_pt_gen_bump(g); + + /* Replenish only exhausted per-vCPU batches. The gate is closed and every + * active producer is quiescent, so EL1 cannot race the cursor publication. + */ + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + mmap_fastpath_provision_l3_cache_locked(g, + mmap_fastpath_control(g, slot)); + } } void mmap_lock_acquire(guest_t *g) @@ -990,8 +1082,8 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->cursor, base, memory_order_relaxed); atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); - atomic_store_explicit(&c->materialized_generation, 0, - memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + mmap_fastpath_prepare_split_window_locked(g, c, base, new_limit); mmap_fastpath_reuse_reset(c); c->next_arena_size = arena_size; c->max_len_seen = 0; @@ -1219,8 +1311,15 @@ static bool mmap_fastpath_rewind_control_if_clean_locked( atomic_store_explicit(&c->cursor, base, memory_order_relaxed); atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); - atomic_store_explicit(&c->materialized_generation, 0, - memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + /* Host retirement has now removed every live region/PTE and zeroed dirty + * backing. Refresh the EL1 clean proof for the rewound generation and + * allow fast munmap again; the PTE-shape concern that set the bit no longer + * survives this full-arena reset. + */ + mmap_fastpath_prepare_split_window_locked(g, c, base, limit); + atomic_fetch_and_explicit(&c->flags, ~SHIM_MMAP_CTRL_NO_FAST_MUNMAP, + memory_order_release); c->recycle_count++; return true; } diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index a48282ac..a7da49f5 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -330,34 +330,47 @@ static void bench_fault(int pages, int drain_between) sweep[ITERS - 1] / pages); } -/* D. mprotect split cost Flip the middle 4 KiB of a 2 MiB RW block to - * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 - * pages. Restore between iterations so each run does a fresh split. +/* D. mprotect split cost. Touch the mapping before timing so lazy mmap has an + * actual L2 block descriptor; otherwise the EL1 lazy-metadata specialization + * can correctly defer PTE creation and this would measure a 1-2 tick metadata + * publication rather than a block-to-table split. Flip the middle 4 KiB to + * PROT_READ, forcing conversion into 512 L3 pages. Keep every mapping live + * until the samples finish so allocator reuse cannot hand a later iteration + * an already-existing (but empty) L3 table. */ static void bench_mprotect_split(void) { printf( "== D. mprotect split (2 MiB block -> L3, protect middle 4 KiB) ==\n"); double sp[ITERS]; + uint8_t *held[ITERS + 1]; + int nheld = 0; for (int it = -1; it < ITERS; it++) { uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); printf(" mmap FAILED\n\n"); return; } + held[nheld++] = p; + p[0] = 1; /* materialize a real 2 MiB L2 block before timing */ uint8_t *mid = p + MIB; uint64_t t0 = rd(); int rc = mprotect(mid, 4 * KIB, PROT_READ); uint64_t t1 = rd(); - munmap(p, 2 * MIB); if (rc != 0) { + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); printf(" mprotect FAILED: %s\n\n", strerror(errno)); return; } if (it >= 0) sp[it] = ns(t1 - t0); } + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); printf(" split: median %.1f ns min %.1f ns\n", median(sp, ITERS), sp[0]); } @@ -374,6 +387,7 @@ static void bench_mprotect_fast_leaf(void) printf(" fast leaf mmap FAILED\n\n"); return; } + p[0] = 1; /* ensure setup below creates an L3 table, not lazy metadata */ if (mprotect(p + MIB, 4 * KIB, PROT_READ) != 0) { munmap(p, 2 * MIB); printf(" fast leaf setup FAILED: %s\n\n", strerror(errno)); @@ -393,8 +407,8 @@ static void bench_mprotect_fast_leaf(void) leaf[it] = ns(t1 - t0); } munmap(p, 2 * MIB); - printf(" fast leaf: median %.1f ns min %.1f ns\n\n", - median(leaf, ITERS), leaf[0]); + printf(" fast leaf: median %.1f ns min %.1f ns\n\n", median(leaf, ITERS), + leaf[0]); } /* E. mremap grow: in-place vs forced move */ diff --git a/tests/test-negative.c b/tests/test-negative.c index edbc9a49..3d5ef814 100644 --- a/tests/test-negative.c +++ b/tests/test-negative.c @@ -242,8 +242,7 @@ static void test_mmap_prot(void) *p = 2; sigaction(SIGSEGV, &old_sa, NULL); - int restored = mprotect((void *) p, 4096, - PROT_READ | PROT_WRITE) == 0; + int restored = mprotect((void *) p, 4096, PROT_READ | PROT_WRITE) == 0; if (restored) *p = 3; int final = *p; @@ -251,6 +250,68 @@ static void test_mmap_prot(void) EXPECT_TRUE(saw_segv && restored && final == 3, "fast mprotect permissions or ring drain were stale"); } + + TEST("first lazy 2MiB block mprotect materializes safely"); + { + const size_t block_size = 2 * 1024 * 1024; + volatile unsigned char *seed = + mmap(NULL, block_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (seed == MAP_FAILED) { + FAIL("seed mmap failed"); + return; + } + seed[1024 * 1024] = 0x7f; + munmap((void *) seed, block_size); + /* Force retirement so the next mmap can reuse backing whose dirty bit + * remains conservative. mprotect must update lazy metadata without + * exposing the stale byte; the later read performs the zeroing fault. + */ + (void) fcntl(-1, F_GETFD); + + volatile unsigned char *p = + mmap(NULL, block_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap failed"); + return; + } + volatile unsigned char *mid = p + 1024 * 1024; + if (mprotect((void *) mid, 4096, PROT_READ) != 0) { + munmap((void *) p, block_size); + FAIL("first-block mprotect failed"); + return; + } + + int zeroed = p[0] == 0 && mid[0] == 0 && p[block_size - 1] == 0; + p[0] = 0x31; + p[block_size - 1] = 0x32; + + struct sigaction old_sa; + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = on_sigsegv; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + munmap((void *) p, block_size); + FAIL("sigaction failed"); + return; + } + saw_segv = 0; + if (sigsetjmp(segv_jmp, 1) == 0) + mid[0] = 0x33; + sigaction(SIGSEGV, &old_sa, NULL); + + int restored = + mprotect((void *) mid, 4096, PROT_READ | PROT_WRITE) == 0; + if (restored) + mid[0] = 0x34; + int neighbors = p[0] == 0x31 && p[block_size - 1] == 0x32; + int target = mid[0] == 0x34; + munmap((void *) p, block_size); + EXPECT_TRUE(zeroed && saw_segv && restored && neighbors && target, + "first-block split exposed stale bytes or wrong perms"); + } } /* Test 7: fcntl on invalid FD */