Optimize anonymous mmap and page-fault paths

Makefile

@@ -197,6 +197,33 @@ $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR)
 	@echo "  CROSS   $< (with -lpthread)"
 	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
 
+# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread.
+$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR)
+	@echo "  CROSS   $< (with -lpthread)"
+	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
+
+# test-mmap-lazy races concurrent first touch from several threads.
+$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR)
+	@echo "  CROSS   $< (with -lpthread)"
+	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
+
+.PHONY: test-mmap-lazy
+test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy
+	@$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy
+	@sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \
+		$(BUILD_DIR)/test-mmap-lazy
+
+# EL1 consumer-mmap integration/stress test.
+$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR)
+	@echo "  CROSS   $< (with -lpthread)"
+	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
+
+.PHONY: test-mmap-fastpath
+test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath
+	@$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath
+	@sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \
+		$(BUILD_DIR)/test-mmap-fastpath
+
 # test-thread-churn creates >64 threads to force thread-table slot reuse.
 $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR)
 	@echo "  CROSS   $< (with -lpthread)"

docs/internals.md

@@ -226,6 +226,27 @@ Splitting is triggered by:
 block needs splitting, splits it if so, then updates the affected L3 page
 entries. Whole-block permission changes are done in place without splitting.
 
+For a private anonymous mmap-arena block, the EL1 shim specializes a one-page
+`PROT_READ` ↔ `PROT_READ|PROT_WRITE` transition. Arena refill prepares the
+upper TTBR0 chain, a per-vCPU batch of 32 L3 table pages, and a clean-backing
+bitmap for a rolling window of 32 two-megabyte blocks. A slow-path miss moves
+that window to the current allocation area. If an undrained mmap publication
+proves that one anonymous RW mapping owns a complete clean block, EL1 may
+populate a prepared L3 page and install the first table descriptor itself. A
+previously retired, empty L3 table is reused in place. Existing valid L3 leaves
+need only an AP[2:1] update.
+
+The shim issues `TLBI RVAE1IS` for first materialization or `TLBI VAE1IS` for
+an existing leaf and returns without an HVC exit. A tagged entry in the mmap
+publication ring defers region metadata plus host-only PTE/dirty-index
+reconciliation to the next drain. The producer-active/PT-gate handshake
+prevents a host page-table writer from racing the EL1 update. Dirty backing,
+but still-lazy target pages use a metadata-only tag and keep their PTE invalid;
+their eventual fault drains the new protection before materializing data.
+Partial ownership, an already-consumed publication, an exhausted table cache,
+executable or `PROT_NONE` transitions, multi-page ranges, and non-arena mappings
+retain the host path.
+
 `mmap` itself uses a gap-finding allocator that walks the sorted region array
 to find free address space. `PROT_EXEC` requests go to the RX region
 (`MMAP_RX_BASE = 0x10000000`); other requests go to the RW region
@@ -426,6 +447,14 @@ goes above the structured area, never below. Post-push masking
 
 ### `mmap` Notes
 
+Private anonymous mappings are lazy at 2 MiB materialization granularity. A
+host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any
+valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled
+fast-path arenas use this index to visit only materialized blocks, so untouched
+multi-GiB reservations have length-independent teardown. When every mapping in
+a per-vCPU arena has been released and the index confirms that no PTE remains,
+the arena cursor rewinds in place instead of taking a refill HVC.
+
 Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real
 host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so
 the kernel page cache keeps the mapping coherent with the file (and

docs/usage.md

@@ -29,6 +29,16 @@ only bounds a single `hv_vcpu_run()` iteration before the host regains control,
 which is what allows host-side timers and signals to be observed promptly.
 Setting `--timeout 0` disables this watchdog for long-running CPU-bound guests.
 
+### mmap call fast path
+
+The aarch64 EL1 consumer fast path is enabled by default for
+`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to
+32 GiB per request.
+Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted
+arenas, and full consumption rings fall back to the normal host syscall path.
+Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host
+path so observability and translated-guest behavior are unchanged.
+
 ## Common Launch Patterns
 
 Run a statically linked guest binary:

src/core/bootstrap.c

@@ -36,6 +36,7 @@
 #include "syscall/signal.h"
 
 #include "debug/log.h"
+#include "core/mmap-fastpath.h"
 
 /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap
  * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter.
@@ -136,6 +137,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len)
     guest_invalidate_ptes(g, 0, 0x1000);
 }
 
+/* ELF/shim/stack bytes are populated directly in the slab before their page
+ * tables become live. Mark their semantic backing regardless of requested
+ * permissions: a read-only file segment is still nonzero and must be scrubbed
+ * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block.
+ * Synthetic page-table coverage for unallocated mmap space has no semantic
+ * region and therefore remains clean.
+ */
+static void mark_registered_backing_dirty(guest_t *g)
+{
+    for (int i = 0; i < g->nregions; i++) {
+        const guest_region_t *r = &g->regions[i];
+        if (r->end <= r->start)
+            continue;
+        uint64_t len = r->end - r->start;
+        if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base)
+            continue;
+        guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len);
+    }
+}
+
 int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info)
 {
     memset(info, 0, sizeof(*info));
@@ -289,7 +310,14 @@ static bool build_boot_regions(mem_region_t *regions,
      * to the vDSO page when splitting the block; otherwise vdso_build cannot
      * write into it through guest_ptr.
      */
-    if (!append_boot_region(regions, nregions, g->shim_base,
+    /* EL1 fast munmap walks and atomically clears the live TTBR0 tree.  Give
+     * the page-table pool an identity VA visible only to EL1; EL0 remains
+     * unable to inspect or corrupt descriptors, and every guest syscall still
+     * rejects the encompassing infrastructure range.
+     */
+    if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end,
+                            MEM_PERM_RW_EL1_ONLY) ||
+        !append_boot_region(regions, nregions, g->shim_base,
                             g->shim_base + shim_bin_len, MEM_PERM_RX) ||
         /* shim_data is EL1-only: the guest must not directly read or write the
          * identity cache, attention flag, urandom bitmap, or ring, any of which
@@ -547,6 +575,7 @@ int guest_bootstrap_prepare(guest_t *g,
     }
 
     register_runtime_regions(g, shim_bin_len);
+    mark_registered_backing_dirty(g);
     startup_trace_step("register_regions", t0);
 
     log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0,
@@ -732,6 +761,13 @@ int guest_bootstrap_create_vcpu(guest_t *g,
      */
     shim_globals_set_singleton(g);
 
+    /* Publish the main vCPU's first arena only after shim_globals_init has
+     * cleared every recycled control slot. Verbose tracing keeps all shim
+     * syscall fast paths on HVC so the trace remains complete.
+     */
+    if (!verbose)
+        mmap_fastpath_prepare_vcpu(g, current_thread);
+
     /* CNTKCTL_EL1.EL0VCTEN | EL0PCTEN: allow EL0 to read {CNTVCT,CNTPCT}_EL0.
      * Required by the vDSO clock_gettime fast path (and is the default on
      * native Linux), without which the guest gets 0 back from MRS.
@@ -843,6 +879,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g,
                                  g->rosetta_guest_base - g->rosetta_va_base,
                                  ROSETTA_PATH);
     register_runtime_regions(g, shim_bin_len);
+    mark_registered_backing_dirty(g);
 
     int rosetta_argc = 0;
     const char **rosetta_argv = NULL;