feat(sandbox): pre-pause guest reclaim via envd

packages/orchestrator/cmd/resume-build/main.go

@@ -72,8 +72,15 @@ func main() {
 	cmdSignalPause := flag.String("cmd-signal-pause", "", "execute command in sandbox, then wait for SIGUSR1 before pausing")
 	optimize := flag.Bool("optimize", false, "collect fresh prefetch mapping after pause (resumes snapshot to record page faults)")
 
+	// Pause-time reclaim override; 0 = use LD default (off).
+	reclaimTimeoutMs := flag.Int("reclaim-timeout-ms", 0, "override reclaim-on-pause-timeout-ms LD flag (0 = use LD default)")
+
 	flag.Parse()
 
+	if *reclaimTimeoutMs > 0 {
+		featureflags.NewIntFlag("reclaim-on-pause-timeout-ms", *reclaimTimeoutMs)
+	}
+
 	if *fromBuild == "" {
 		log.Fatal("-from-build required")
 	}

packages/orchestrator/pkg/sandbox/reclaim.go

@@ -0,0 +1,88 @@
+package sandbox
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"connectrpc.com/connect"
+	"go.uber.org/zap"
+
+	"github.com/e2b-dev/infra/packages/shared/pkg/consts"
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
+	"github.com/e2b-dev/infra/packages/shared/pkg/grpc"
+	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process"
+	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process/processconnect"
+	"github.com/e2b-dev/infra/packages/shared/pkg/logger"
+)
+
+type reclaimStep struct {
+	flag featureflags.IntFlag
+	cmd  string
+}
+
+// Order matters: sync makes drop_caches more effective; drop_caches gives
+// compact_memory more headroom; fstrim wants a stable FS view.
+var reclaimSteps = []reclaimStep{
+	{featureflags.ReclaimSyncTimeoutMs, "sync"},
+	{featureflags.ReclaimDropCachesTimeoutMs, "echo 3 > /proc/sys/vm/drop_caches"},
+	{featureflags.ReclaimCompactMemoryTimeoutMs, "echo 1 > /proc/sys/vm/compact_memory"},
+	{featureflags.ReclaimFstrimTimeoutMs, "fstrim -av"},
+}
+
+// buildReclaimScript composes a chain where each step has its own hard ceiling
+// (`timeout -s KILL`). Steps with cap=0 are skipped; the script is empty when
+// every step is disabled.
+func (s *Sandbox) buildReclaimScript(ctx context.Context) string {
+	var parts []string
+	for _, st := range reclaimSteps {
+		ms := s.featureFlags.IntFlag(ctx, st.flag)
+		if ms <= 0 {
+			continue
+		}
+		parts = append(parts, fmt.Sprintf("timeout -s KILL %dms sh -c %q 2>/dev/null; true", ms, st.cmd))
+	}
+
+	return strings.Join(parts, " ")
+}
+
+// bestEffortReclaim asks envd to reclaim guest memory + disk before pause.
+// All failures are swallowed.
+func (s *Sandbox) bestEffortReclaim(ctx context.Context, timeout time.Duration) {
+	ctx, span := tracer.Start(ctx, "envd-reclaim")
+	defer span.End()
+
+	script := s.buildReclaimScript(ctx)
+	if script == "" {
+		return
+	}
+
+	rcCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	addr := fmt.Sprintf("http://%s:%d", s.Slot.HostIPString(), consts.DefaultEnvdServerPort)
+	pc := processconnect.NewProcessClient(&http.Client{Transport: sandboxHttpClient.Transport}, addr)
+
+	req := connect.NewRequest(&process.StartRequest{
+		Process: &process.ProcessConfig{Cmd: "/bin/bash", Args: []string{"-c", script}},
+	})
+	req.Header().Set("Connect-Timeout-Ms", strconv.FormatInt(int64(timeout/time.Millisecond), 10))
+	if s.Config.Envd.AccessToken != nil {
+		req.Header().Set("X-Access-Token", *s.Config.Envd.AccessToken)
+	}
+	grpc.SetUserHeader(req.Header(), "root")
+
+	stream, err := pc.Start(rcCtx, req)
+	if err != nil {
+		logger.L().Warn(ctx, "envd reclaim failed", logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err))
+
+		return
+	}
+	defer stream.Close()
+
+	for stream.Receive() {
+	}
+}

packages/orchestrator/pkg/sandbox/sandbox.go

@@ -217,6 +217,8 @@ type Sandbox struct {
 	files   *storage.SandboxFiles
 	cleanup *Cleanup
 
+	featureFlags *featureflags.Client
+
 	process      *fc.Process
 	cgroupHandle *cgroup.CgroupHandle
 
@@ -457,7 +459,8 @@ func (f *Factory) CreateSandbox(
 		files:    sandboxFiles,
 		process:  fcHandle,
 
-		cleanup: cleanup,
+		cleanup:      cleanup,
+		featureFlags: f.featureFlags,
 
 		APIStoredConfig: apiConfigToStore,
 
@@ -797,7 +800,8 @@ func (f *Factory) ResumeSandbox(
 		files:    sandboxFiles,
 		process:  fcHandle,
 
-		cleanup: cleanup,
+		cleanup:      cleanup,
+		featureFlags: f.featureFlags,
 
 		APIStoredConfig: apiConfigToStore,
 		CABundle:        f.egressProxy.CABundle(),
@@ -1051,6 +1055,12 @@ func (s *Sandbox) Pause(
 	// Stop the health check before pausing the VM
 	s.Checks.Stop()
 
+	// Best-effort pre-pause guest reclaim (sync, drop_caches, compact_memory,
+	// fstrim) on the live VM via envd. Non-fatal. Timeout=0 disables it.
+	if t := time.Duration(s.featureFlags.IntFlag(ctx, featureflags.ReclaimOnPauseTimeoutMs)) * time.Millisecond; t > 0 {
+		s.bestEffortReclaim(ctx, t)
+	}
+
 	if err := s.process.Pause(ctx); err != nil {
 		return nil, fmt.Errorf("failed to pause VM: %w", err)
 	}

packages/shared/pkg/featureflags/flags.go

@@ -157,7 +157,17 @@ var (
 	BestOfKMaxOvercommit          = NewIntFlag("best-of-k-max-overcommit", 400)              // Default R=4 (stored as percentage, max over-commit ratio)
 	BestOfKAlpha                  = NewIntFlag("best-of-k-alpha", 50)                        // Default Alpha=0.5 (stored as percentage for int flag, current usage weight)
 	EnvdInitTimeoutMilliseconds   = NewIntFlag("envd-init-request-timeout-milliseconds", 50) // Timeout for envd init request in milliseconds
-	HostStatsSamplingInterval     = NewIntFlag("host-stats-sampling-interval", 5000)         // Host stats sampling interval in milliseconds (default 5s)
+	// ReclaimOnPauseTimeoutMs gates a pre-pause guest reclaim run via envd and
+	// caps the whole chain. 0 disables the feature entirely.
+	ReclaimOnPauseTimeoutMs = NewIntFlag("reclaim-on-pause-timeout-ms", 0)
+	// Per-step ceilings for the reclaim chain. Each step is wrapped in
+	// `timeout -s KILL`; 0 skips that step. A stuck step cannot starve the
+	// rest, so compact_memory always runs (as long as its own cap is > 0).
+	ReclaimSyncTimeoutMs          = NewIntFlag("reclaim-sync-timeout-ms", 500)
+	ReclaimDropCachesTimeoutMs    = NewIntFlag("reclaim-drop-caches-timeout-ms", 200)
+	ReclaimCompactMemoryTimeoutMs = NewIntFlag("reclaim-compact-memory-timeout-ms", 1000)
+	ReclaimFstrimTimeoutMs        = NewIntFlag("reclaim-fstrim-timeout-ms", 500)
+	HostStatsSamplingInterval     = NewIntFlag("host-stats-sampling-interval", 5000) // Host stats sampling interval in milliseconds (default 5s)
 	MaxCacheWriterConcurrencyFlag = NewIntFlag("max-cache-writer-concurrency", 10)
 
 	// BuildCacheMaxUsagePercentage the maximum percentage of the cache disk storage