🤖 fix: stop background status fetch from converting repos into partial clones

src/browser/stores/GitStatusStore.ts

@@ -48,6 +48,13 @@ const MAX_CONCURRENT_GIT_OPS = 5;
 // Fetch configuration - aggressive intervals for fresh data
 const FETCH_BASE_INTERVAL_MS = 3 * 1000; // 3 seconds
 const FETCH_MAX_INTERVAL_MS = 60 * 1000; // 60 seconds
+// Background fetches are unfiltered (see GIT_FETCH_SCRIPT) and may run a
+// one-time full --refetch to heal repos poisoned into promisor/partial
+// clones, so transfers can be much larger than the old blob-filtered ones.
+// Killing a slow-but-progressing fetch wastes the entire transfer and leaves
+// ahead/behind state permanently stale behind retry backoff, so budget for a
+// full-object transfer instead.
+const FETCH_TIMEOUT_SECS = 300; // 5 minutes
 
 interface FetchState {
   lastFetch: number;
@@ -925,7 +932,7 @@ export class GitStatusStore {
       // Passive fetches use the runtime path because git fetch / git ls-remote
       // may need remote credentials that only exist inside the runtime. These
       // background fetches are only scheduled when that runtime is already running.
-      options: repoRootBashOptions(30, repoRootProjectPath),
+      options: repoRootBashOptions(FETCH_TIMEOUT_SECS, repoRootProjectPath),
     });
 
     if (!result.success) {

src/common/utils/git/gitStatus.fetch.test.ts

@@ -59,4 +59,85 @@ describe("GIT_FETCH_SCRIPT", () => {
       await rm(tempDir, { recursive: true, force: true });
     }
   }, 20000);
+
+  test("heals a repo poisoned into a promisor partial clone even when up to date", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "mux-git-heal-"));
+    const originDir = path.join(tempDir, "origin.git");
+    const seedDir = path.join(tempDir, "seed");
+    const workspaceDir = path.join(tempDir, "workspace");
+
+    const run = (cmd: string, cwd?: string) =>
+      execSync(cmd, { cwd, stdio: "pipe" }).toString().trim();
+    const configureIdentity = (cwd: string) => {
+      run('git config user.email "test@example.com"', cwd);
+      run('git config user.name "Test User"', cwd);
+      run("git config commit.gpgsign false", cwd);
+    };
+
+    try {
+      run(`git init --bare ${originDir}`);
+      // Local-path remotes reject --filter unless the server side opts in.
+      run(`git -C ${originDir} config uploadpack.allowFilter true`);
+
+      // Seed main via a separate clone so the workspace clone below stays
+      // unaware of later blobs.
+      run(`git clone ${originDir} ${seedDir}`);
+      configureIdentity(seedDir);
+      await writeFile(path.join(seedDir, "README.md"), "init\n");
+      run("git add README.md", seedDir);
+      run('git commit -m "init"', seedDir);
+      run("git branch -M main", seedDir);
+      run("git push -u origin main", seedDir);
+      run("git symbolic-ref HEAD refs/heads/main", originDir);
+
+      // Full (healthy) clone of the workspace.
+      run(`git clone ${originDir} ${workspaceDir}`);
+      configureIdentity(workspaceDir);
+
+      // Advance origin/main with a commit whose blob the workspace lacks.
+      await writeFile(path.join(seedDir, "data.txt"), "poisoned blob content\n");
+      run("git add data.txt", seedDir);
+      run('git commit -m "add data"', seedDir);
+      run("git push origin main", seedDir);
+
+      // Reproduce the poisoning done by previous versions of the script: a
+      // single filtered fetch persists promisor config and skips the new blob.
+      run("git fetch origin --filter=blob:none", workspaceDir);
+      expect(run("git config --local --get remote.origin.partialclonefilter", workspaceDir)).toBe(
+        "blob:none"
+      );
+      // rev-list reports missing objects without lazy-fetching them.
+      const missingBefore = run(
+        "git rev-list --objects --missing=print origin/main | grep -c '^?' || true",
+        workspaceDir
+      );
+      expect(Number(missingBefore)).toBeGreaterThan(0);
+
+      // The filtered fetch already updated the tracking ref, so the script's
+      // LOCAL_SHA/REMOTE_SHA early-exit is hit: the heal must run before it.
+      const script = GIT_FETCH_SCRIPT;
+      const output = run(script, workspaceDir);
+      expect(output).toContain("HEAL: backfilling promisor partial clone");
+
+      // Promisor config removed and previously missing blobs backfilled.
+      expect(
+        run("git config --local --get remote.origin.partialclonefilter || echo GONE", workspaceDir)
+      ).toBe("GONE");
+      expect(
+        run("git config --local --get remote.origin.promisor || echo GONE", workspaceDir)
+      ).toBe("GONE");
+      const missingAfter = run(
+        "git rev-list --objects --missing=print origin/main | grep -c '^?' || true",
+        workspaceDir
+      );
+      expect(Number(missingAfter)).toBe(0);
+
+      // Heal is one-shot: a second run must skip without re-fetching.
+      const secondOutput = run(script, workspaceDir);
+      expect(secondOutput).not.toContain("HEAL:");
+      expect(secondOutput).toContain("SKIP: Remote SHA already fetched");
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  }, 20000);
 });

src/common/utils/git/gitStatus.ts

@@ -176,6 +176,19 @@ export function parseGitStatusScriptOutput(output: string): ParsedGitStatusOutpu
   };
 }
 
+/**
+ * Git config keys that mark a repo as a promisor/partial clone. Previous
+ * versions of GIT_FETCH_SCRIPT fetched with --filter=blob:none, which made
+ * git persist this state (poisoning the repo: every later fetch stayed
+ * filtered and checkouts lazy-fetched blobs from the network). The fetch
+ * script's heal block and SSHRuntime's base-repo hygiene both unset these.
+ */
+export const PROMISOR_CONFIG_KEYS = [
+  "remote.origin.promisor",
+  "remote.origin.partialclonefilter",
+  "extensions.partialclone",
+] as const;
+
 /**
  * Smart git fetch script that minimizes lock contention.
  *
@@ -212,6 +225,46 @@ if [ -z "$REMOTE_SHA" ]; then
   exit 0
 fi
 
+# One-time heal for repos that previous versions of this script converted
+# into promisor/partial clones. --no-filter (used below) stops the damage but
+# does not remove the persisted promisor config, nor backfill the blobs that
+# earlier filtered fetches omitted, and up-to-date repos would skip the fetch
+# entirely via the early-exit below. Left unhealed, "git worktree add"
+# (workspace creation) lazy-fetches those old blobs mid-checkout and fails on
+# transient network errors. Only repos whose filter is exactly the
+# "blob:none" this script used to write are healed.
+if [ "$(git config --local --get remote.origin.partialclonefilter 2>/dev/null)" = "blob:none" ]; then
+  # Lock in the shared git dir: sibling worktrees share repo config, so this
+  # keeps them from starting concurrent full refetches. A stale lock (killed
+  # heal) expires after an hour, which also rate-limits retries when the
+  # refetch keeps failing (e.g. huge repo on a slow link).
+  COMMON_DIR=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)
+  HEAL_LOCK="$COMMON_DIR/xum-promisor-heal.lock"
+  if [ -n "$COMMON_DIR" ] && [ -n "$(find "$HEAL_LOCK" -maxdepth 0 -type d -mmin +60 2>/dev/null)" ]; then
+    rmdir "$HEAL_LOCK" 2>/dev/null
+  fi
+  if [ -n "$COMMON_DIR" ] && mkdir "$HEAL_LOCK" 2>/dev/null; then
+    echo "HEAL: backfilling promisor partial clone"
+    # --refetch (git >= 2.36) negotiates as if the repo had nothing, so the
+    # server re-sends every object including previously filtered-out blobs.
+    # Unset the promisor config only after a successful refetch: stripping
+    # first would leave missing blobs with no lazy-fetch fallback, breaking
+    # checkouts outright instead of healing them.
+    if git -c protocol.version=2 \\
+        fetch origin \\
+        --refetch \\
+        --no-filter \\
+        --prune \\
+        --no-tags \\
+        --no-recurse-submodules \\
+        --no-write-fetch-head \\
+        2>&1; then
+${PROMISOR_CONFIG_KEYS.map((key) => `      git config --local --unset-all ${key} 2>/dev/null`).join("\n")}
+    fi
+    rmdir "$HEAL_LOCK" 2>/dev/null
+  fi
+fi
+
 # Check current local remote-tracking ref (no lock)
 LOCAL_SHA=$(git rev-parse --verify "refs/remotes/origin/$PRIMARY_BRANCH" 2>/dev/null || echo "")
 
@@ -222,13 +275,25 @@ if [ "$LOCAL_SHA" = "$REMOTE_SHA" ]; then
 fi
 
 # Remote has new commits or ref moved - fetch updates
+#
+# --no-filter (NOT --filter=blob:none): a filtered fetch permanently converts
+# the repo into a promisor/partial clone (git writes remote.origin.promisor +
+# remote.origin.partialclonefilter on the first filtered fetch, and the
+# configured filter then applies to every subsequent plain fetch). That leaves
+# every commit fetched by this background loop without its blobs, so a later
+# "git worktree add" (workspace creation) must lazy-fetch blobs from the
+# remote mid-checkout and any transient network failure aborts it with
+# "fatal: could not fetch <oid> from promisor remote". --no-filter avoids
+# poisoning healthy repos and keeps this fetch unfiltered even in a repo that
+# is still poisoned (already-converted repos are backfilled and cleaned up by
+# the one-time heal block above).
 git -c protocol.version=2 \\
     -c fetch.negotiationAlgorithm=skipping \\
     fetch origin \\
     --prune \\
     --no-tags \\
     --no-recurse-submodules \\
     --no-write-fetch-head \\
-    --filter=blob:none \\
+    --no-filter \\
     2>&1
 `;

src/node/runtime/SSHRuntime.ts

@@ -39,6 +39,7 @@ import { expandTildeForSSH, cdCommandForSSH } from "./tildeExpansion";
 import { sleepWithAbort } from "@/node/utils/abort";
 import { execBuffered } from "@/node/utils/runtime/helpers";
 import { getErrorMessage } from "@/common/utils/errors";
+import { PROMISOR_CONFIG_KEYS } from "@/common/utils/git/gitStatus";
 import {
   type SSHRuntimeConfig,
   getControlPath,
@@ -106,11 +107,7 @@ const BASE_REPO_MAINTENANCE_WAIT_TIMEOUT_SECONDS = 30 * 60;
  *  `repo_has_promisor_remote()`. Unsetting all three is what makes
  *  receive-pack's `check_connected()` skip the buggy partial-clone fast
  *  path on subsequent pushes (see `stripBaseRepoPromisorConfig`). */
-const BASE_REPO_PROMISOR_CONFIG_KEYS = [
-  "remote.origin.promisor",
-  "remote.origin.partialclonefilter",
-  "extensions.partialclone",
-] as const;
+const BASE_REPO_PROMISOR_CONFIG_KEYS = PROMISOR_CONFIG_KEYS;
 const BASE_REPO_FRAGMENTED_PACK_THRESHOLD = 25;
 const PROJECT_SYNC_MAX_ATTEMPTS = 3;
 const PROJECT_SYNC_RETRYABLE_ERRORS = [
@@ -152,7 +149,11 @@ function isUnresolvedDeltaPushFailure(errorMsg: string): boolean {
 }
 
 function isMissingObjectCheckoutFailure(message: string): boolean {
-  return /unable to read sha1 file|Could not reset index file|missing (blob|tree|commit)|bad object|unable to read tree|object file .* is empty|loose object .* is corrupt/i.test(
+  // "could not fetch ... from promisor remote": the checkout needed objects the
+  // repo does not have and a lazy fetch from upstream failed (e.g. transient
+  // network drop). The objects are still missing locally, so the same
+  // repair-from-local path applies.
+  return /unable to read sha1 file|Could not reset index file|missing (blob|tree|commit)|bad object|unable to read tree|object file .* is empty|loose object .* is corrupt|could not fetch .* from promisor remote/i.test(
     message
   );
 }
@@ -2657,6 +2658,18 @@ export class SSHRuntime extends RemoteRuntime {
       // path where ensureBaseRepo() has retry/error handling instead of risking
       // materializing a worktree from still-poisoned shared config.
       `git --git-dir=${baseRepoPathArg} symbolic-ref HEAD ${baseRepoUnbornHeadArg} 2>/dev/null || { echo WARM_MISS:base-head-normalization-failed; exit 0; }`,
+      // Best-effort promisor strip, mirroring ensureBaseRepo()'s epilogue. The
+      // warm path skips ensureBaseRepo(), and background status fetches that
+      // ran `git fetch --filter=blob:none` inside sibling worktrees register
+      // the shared base repo as a promisor remote (remote.origin.promisor +
+      // partialclonefilter). Left in place, `git worktree add` below would
+      // lazy-fetch missing blobs from upstream mid-checkout, so a transient
+      // network drop aborts workspace creation with "could not fetch <oid>
+      // from promisor remote" instead of the repairable missing-objects path.
+      ...BASE_REPO_PROMISOR_CONFIG_KEYS.map(
+        (key) =>
+          `git --git-dir=${baseRepoPathArg} config --local --unset-all ${shescape.quote(key)} 2>/dev/null || true`
+      ),
     ];
 
     const originPreamble = originUrlArg
@@ -2706,7 +2719,7 @@ export class SSHRuntime extends RemoteRuntime {
       "wt_status=$?",
       'if [ "$wt_status" -ne 0 ]; then',
       '  case "$wt_output" in',
-      '    *"unable to read sha1 file"*|*"Could not reset index file"*|*"missing blob"*|*"missing tree"*|*"missing commit"*|*"bad object"*|*"unable to read tree"*) wt_reason=missing-objects ;;',
+      '    *"unable to read sha1 file"*|*"Could not reset index file"*|*"missing blob"*|*"missing tree"*|*"missing commit"*|*"bad object"*|*"unable to read tree"*|*"from promisor remote"*) wt_reason=missing-objects ;;',
       "    *) wt_reason=worktree-add-failed ;;",
       "  esac",
       `  git -C ${baseRepoPathArg} worktree remove --force ${workspacePathArg} >/dev/null 2>&1 || rm -rf ${workspacePathArg}`,

tests/runtime/runtime.test.ts

@@ -2206,6 +2206,12 @@ describeIntegration("Runtime integration tests", () => {
             `git --git-dir="${baseRepoPath}" config --local core.bare true`,
             `git --git-dir="${baseRepoPath}" config --local core.worktree "${bogusWorktreePath}"`,
             `git --git-dir="${baseRepoPath}" symbolic-ref HEAD refs/heads/main`,
+            // Simulate what a background `git fetch --filter=blob:none` from a
+            // sibling worktree registers in the shared gitdir. A promisor base
+            // repo lazy-fetches missing blobs over the network mid-checkout, so
+            // the warm path must strip these before `git worktree add`.
+            `git --git-dir="${baseRepoPath}" config --local remote.origin.promisor true`,
+            `git --git-dir="${baseRepoPath}" config --local remote.origin.partialclonefilter blob:none`,
           ].join(" && ")
         );
         expect(poisonResult.exitCode).toBe(0);
@@ -2243,6 +2249,20 @@ describeIntegration("Runtime integration tests", () => {
         );
         expect(baseRepoCoreWorktreeCheck.exitCode).toBe(1);
 
+        // Promisor/partial-clone registration must be stripped so worktree
+        // materialization never lazy-fetches blobs over the network.
+        const baseRepoPromisorCheck = await execSSH(
+          runtime,
+          `git --git-dir="${baseRepoPath}" config --get remote.origin.promisor`
+        );
+        expect(baseRepoPromisorCheck.exitCode).toBe(1);
+
+        const baseRepoFilterCheck = await execSSH(
+          runtime,
+          `git --git-dir="${baseRepoPath}" config --get remote.origin.partialclonefilter`
+        );
+        expect(baseRepoFilterCheck.exitCode).toBe(1);
+
         const baseHeadSymbolicCheck = await execSSH(
           runtime,
           `git --git-dir="${baseRepoPath}" symbolic-ref -q HEAD`