Skip to content

Commit 6906118

Browse files
kalyazinclaude
authored andcommitted
perf(orch): drop dead per-pause snapshot structures behind flags
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> GitOrigin-RevId: 7aaeed323a2a49c0a4ebb0ecf5b47a78b2ba5fa9
1 parent 44932b6 commit 6906118

13 files changed

Lines changed: 1192 additions & 47 deletions

File tree

‎packages/orchestrator/pkg/sandbox/block/memfd.go‎

Lines changed: 68 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ type DedupedMemfdCache struct {
291291
done *utils.SetOnce[*Cache]
292292

293293
inflight bool
294+
// freeIndex frees the packed index with the memfd when it is released.
295+
freeIndex bool
294296
// swapped is resolved by MarkSwapped once the local template has swapped
295297
// off the provisional header onto the deduped one. runDedup waits on it
296298
// (bounded) before releasing the memfd, so a provisional read can't hit a
@@ -334,6 +336,31 @@ var inflightServePagesCounter = utils.Must(otel.Meter("github.com/e2b-dev/infra/
334336
Int64Counter("orchestrator.memfd.inflight_serve_pages",
335337
metric.WithDescription("Guest pages served from the still-mapped memfd during dedup (in-flight memfd serving)")))
336338

339+
// deadStructureOutcomeCounter records what every memfd release did with the
340+
// packed index, and deadStructureBytesCounter the index's size wherever there
341+
// was one to free. They are the free-index flag's engagement and size signals.
342+
var (
343+
deadStructureOutcomeCounter = utils.Must(telemetry.GetCounter(
344+
otel.Meter("github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block"),
345+
telemetry.OrchestratorDeadStructureOutcomeCounterName))
346+
deadStructureBytesCounter = utils.Must(telemetry.GetCounter(
347+
otel.Meter("github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block"),
348+
telemetry.OrchestratorDeadStructureBytesCounterName))
349+
)
350+
351+
var (
352+
dedupIndexDroppedAttr = dedupIndexOutcomeAttr("dropped")
353+
dedupIndexFlagOffAttr = dedupIndexOutcomeAttr("flag_off")
354+
dedupIndexNoneAttr = dedupIndexOutcomeAttr("none")
355+
)
356+
357+
func dedupIndexOutcomeAttr(outcome string) metric.MeasurementOption {
358+
return metric.WithAttributeSet(attribute.NewSet(
359+
attribute.String("structure", "dedup_index"),
360+
attribute.String("outcome", outcome),
361+
))
362+
}
363+
337364
// inflightServe{Provisional,Drain}Attr tag which serving path recorded a page:
338365
// the provisional compare window (ServeMemfd, identity offsets) or the in-flight
339366
// drain window (tryInflightRead, packed offsets). Precomputed as attribute-set
@@ -356,17 +383,19 @@ func NewCacheFromMemfdDeduped(
356383
inputEmpty *roaring.Bitmap,
357384
metaOut *utils.SetOnce[*header.DiffMetadata],
358385
inflightServe bool,
386+
freeIndex bool,
359387
) (*DedupedMemfdCache, error) {
360388
if blockSize%header.PageSize != 0 {
361389
return nil, fmt.Errorf("diff block size %d not a multiple of dedup page size %d", blockSize, header.PageSize)
362390
}
363391
drainCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
364392
d := &DedupedMemfdCache{
365-
outPath: outPath,
366-
cancel: cancel,
367-
done: utils.NewSetOnce[*Cache](),
368-
swapped: utils.NewSetOnce[struct{}](),
369-
inflight: inflightServe,
393+
outPath: outPath,
394+
cancel: cancel,
395+
done: utils.NewSetOnce[*Cache](),
396+
swapped: utils.NewSetOnce[struct{}](),
397+
inflight: inflightServe,
398+
freeIndex: freeIndex,
370399
// Publish the memfd up front (guarded by mu) so a provisional-header
371400
// resume can serve dirty pages via ServeMemfd during the compare window,
372401
// before metaOut/the deduped header resolves. runDedup frees it under mu
@@ -456,7 +485,7 @@ func (d *DedupedMemfdCache) runDedup(
456485
compareDur := time.Since(compareStart)
457486
if err != nil {
458487
logSetOnceErr(ctx, "dedup metaOut", metaOut.SetError(err))
459-
logSetOnceErr(ctx, "dedup done", d.done.SetError(errors.Join(err, d.releaseMemfd())))
488+
logSetOnceErr(ctx, "dedup done", d.done.SetError(errors.Join(err, d.releaseMemfd(ctx))))
460489

461490
return
462491
}
@@ -514,7 +543,7 @@ func (d *DedupedMemfdCache) runDedup(
514543
logger.L().Warn(ctx, "memfd swap grace elapsed; releasing memfd without a swap signal")
515544
}
516545
}
517-
if closeErr := d.releaseMemfd(); closeErr != nil {
546+
if closeErr := d.releaseMemfd(ctx); closeErr != nil {
518547
logger.L().Warn(ctx, "close memfd after dedup drain", zap.Error(closeErr))
519548
}
520549
}
@@ -603,8 +632,10 @@ func (d *DedupedMemfdCache) Slice(off, length int64) ([]byte, error) {
603632
}
604633

605634
// releaseMemfd closes the memfd exactly once, under the write lock so it can't
606-
// be unmapped beneath an in-flight ServeMemfd/tryInflightRead reader.
607-
func (d *DedupedMemfdCache) releaseMemfd() error {
635+
// be unmapped beneath an in-flight ServeMemfd/tryInflightRead reader. With
636+
// freeIndex set, the packed index goes with it: tryInflightRead is its only
637+
// reader and needs the memfd too, so no read can use it afterwards.
638+
func (d *DedupedMemfdCache) releaseMemfd(ctx context.Context) error {
608639
d.mu.Lock()
609640
defer d.mu.Unlock()
610641
if d.memfd == nil {
@@ -613,9 +644,37 @@ func (d *DedupedMemfdCache) releaseMemfd() error {
613644
err := d.memfd.Close()
614645
d.memfd = nil
615646

647+
d.releaseIndex(ctx)
648+
616649
return err
617650
}
618651

652+
// packedSegBytes is the size of one packedSeg: three int64 fields.
653+
const packedSegBytes = 24
654+
655+
// releaseIndex frees the packed index if freeIndex allows it, and records the
656+
// outcome. Called with d.mu held for writing.
657+
func (d *DedupedMemfdCache) releaseIndex(ctx context.Context) {
658+
if d.index == nil {
659+
deadStructureOutcomeCounter.Add(ctx, 1, dedupIndexNoneAttr)
660+
661+
return
662+
}
663+
664+
// The whole backing array goes, so its capacity is what is freed.
665+
size := int64(cap(d.index)) * packedSegBytes
666+
if !d.freeIndex {
667+
deadStructureOutcomeCounter.Add(ctx, 1, dedupIndexFlagOffAttr)
668+
deadStructureBytesCounter.Add(ctx, size, dedupIndexFlagOffAttr)
669+
670+
return
671+
}
672+
673+
d.index = nil
674+
deadStructureOutcomeCounter.Add(ctx, 1, dedupIndexDroppedAttr)
675+
deadStructureBytesCounter.Add(ctx, size, dedupIndexDroppedAttr)
676+
}
677+
619678
// ServeMemfd copies [off, off+len(b)) from the still-mapped memfd using
620679
// identity (device) addressing, for a provisional local header that attributes
621680
// dirty pages to the memfd at their device offset. Returns BytesNotAvailableError

0 commit comments

Comments
 (0)