🤖 fix: stop background status fetch from converting repos into partial clones

src/browser/stores/GitStatusStore.ts

@@ -48,6 +48,13 @@ const MAX_CONCURRENT_GIT_OPS = 5;
 // Fetch configuration - aggressive intervals for fresh data
 const FETCH_BASE_INTERVAL_MS = 3 * 1000; // 3 seconds
 const FETCH_MAX_INTERVAL_MS = 60 * 1000; // 60 seconds
+// Background fetches are unfiltered (see GIT_FETCH_SCRIPT) and may run a
+// one-time full --refetch to heal repos poisoned into promisor/partial
+// clones, so transfers can be much larger than the old blob-filtered ones.
+// Killing a slow-but-progressing fetch wastes the entire transfer and leaves
+// ahead/behind state permanently stale behind retry backoff, so budget for a
+// full-object transfer instead.
+const FETCH_TIMEOUT_SECS = 300; // 5 minutes
 
 interface FetchState {
   lastFetch: number;
@@ -925,7 +932,7 @@ export class GitStatusStore {
       // Passive fetches use the runtime path because git fetch / git ls-remote
       // may need remote credentials that only exist inside the runtime. These
       // background fetches are only scheduled when that runtime is already running.
-      options: repoRootBashOptions(30, repoRootProjectPath),
+      options: repoRootBashOptions(FETCH_TIMEOUT_SECS, repoRootProjectPath),
     });
 
     if (!result.success) {

src/common/utils/git/gitStatus.fetch.test.ts

@@ -59,4 +59,152 @@ describe("GIT_FETCH_SCRIPT", () => {
       await rm(tempDir, { recursive: true, force: true });
     }
   }, 20000);
+
+  test("heals a repo poisoned into a promisor partial clone even when up to date", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "mux-git-heal-"));
+    const originDir = path.join(tempDir, "origin.git");
+    const seedDir = path.join(tempDir, "seed");
+    const workspaceDir = path.join(tempDir, "workspace");
+
+    const run = (cmd: string, cwd?: string) =>
+      execSync(cmd, { cwd, stdio: "pipe" }).toString().trim();
+    const configureIdentity = (cwd: string) => {
+      run('git config user.email "test@example.com"', cwd);
+      run('git config user.name "Test User"', cwd);
+      run("git config commit.gpgsign false", cwd);
+    };
+
+    try {
+      run(`git init --bare ${originDir}`);
+      // Local-path remotes reject --filter unless the server side opts in.
+      run(`git -C ${originDir} config uploadpack.allowFilter true`);
+
+      // Seed main via a separate clone so the workspace clone below stays
+      // unaware of later blobs.
+      run(`git clone ${originDir} ${seedDir}`);
+      configureIdentity(seedDir);
+      await writeFile(path.join(seedDir, "README.md"), "init\n");
+      run("git add README.md", seedDir);
+      run('git commit -m "init"', seedDir);
+      run("git branch -M main", seedDir);
+      run("git push -u origin main", seedDir);
+      run("git symbolic-ref HEAD refs/heads/main", originDir);
+
+      // Full (healthy) clone of the workspace.
+      run(`git clone ${originDir} ${workspaceDir}`);
+      configureIdentity(workspaceDir);
+
+      // Advance origin/main with a commit whose blob the workspace lacks.
+      await writeFile(path.join(seedDir, "data.txt"), "poisoned blob content\n");
+      run("git add data.txt", seedDir);
+      run('git commit -m "add data"', seedDir);
+      run("git push origin main", seedDir);
+
+      // Reproduce the poisoning done by previous versions of the script: a
+      // single filtered fetch persists promisor config and skips the new blob.
+      run("git fetch origin --filter=blob:none", workspaceDir);
+      expect(run("git config --local --get remote.origin.partialclonefilter", workspaceDir)).toBe(
+        "blob:none"
+      );
+      // rev-list reports missing objects without lazy-fetching them.
+      const missingBefore = run(
+        "git rev-list --objects --missing=print origin/main | grep -c '^?' || true",
+        workspaceDir
+      );
+      expect(Number(missingBefore)).toBeGreaterThan(0);
+
+      // The filtered fetch already updated the tracking ref, so the script's
+      // LOCAL_SHA/REMOTE_SHA early-exit is hit: the heal must run before it.
+      const script = GIT_FETCH_SCRIPT;
+      const output = run(script, workspaceDir);
+      expect(output).toContain("HEAL: backfilling promisor partial clone");
+
+      // Promisor config removed and previously missing blobs backfilled.
+      expect(
+        run("git config --local --get remote.origin.partialclonefilter || echo GONE", workspaceDir)
+      ).toBe("GONE");
+      expect(
+        run("git config --local --get remote.origin.promisor || echo GONE", workspaceDir)
+      ).toBe("GONE");
+      const missingAfter = run(
+        "git rev-list --objects --missing=print origin/main | grep -c '^?' || true",
+        workspaceDir
+      );
+      expect(Number(missingAfter)).toBe(0);
+
+      // Heal is one-shot: a second run must skip without re-fetching.
+      const secondOutput = run(script, workspaceDir);
+      expect(secondOutput).not.toContain("HEAL:");
+      expect(secondOutput).toContain("SKIP: Remote SHA already fetched");
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  }, 20000);
+
+  test("keeps promisor config when refetch cannot restore locally referenced blobs", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "mux-git-heal-incomplete-"));
+    const originDir = path.join(tempDir, "origin.git");
+    const seedDir = path.join(tempDir, "seed");
+    const workspaceDir = path.join(tempDir, "workspace");
+
+    const run = (cmd: string, cwd?: string) =>
+      execSync(cmd, { cwd, stdio: "pipe" }).toString().trim();
+    const configureIdentity = (cwd: string) => {
+      run('git config user.email "test@example.com"', cwd);
+      run('git config user.name "Test User"', cwd);
+      run("git config commit.gpgsign false", cwd);
+    };
+
+    try {
+      run(`git init --bare ${originDir}`);
+      run(`git -C ${originDir} config uploadpack.allowFilter true`);
+
+      run(`git clone ${originDir} ${seedDir}`);
+      configureIdentity(seedDir);
+      await writeFile(path.join(seedDir, "README.md"), "init\n");
+      run("git add README.md", seedDir);
+      run('git commit -m "init"', seedDir);
+      run("git branch -M main", seedDir);
+      run("git push -u origin main", seedDir);
+      run("git symbolic-ref HEAD refs/heads/main", originDir);
+
+      run(`git clone ${originDir} ${workspaceDir}`);
+      configureIdentity(workspaceDir);
+
+      // Push a feature branch whose blob the workspace will only ever see
+      // through a filtered fetch.
+      run("git checkout -b feature", seedDir);
+      await writeFile(path.join(seedDir, "orphan.txt"), "blob that will be orphaned upstream\n");
+      run("git add orphan.txt", seedDir);
+      run('git commit -m "orphan"', seedDir);
+      run("git push origin feature", seedDir);
+
+      // Poison the workspace and pin the blobless commit with a local branch.
+      run("git fetch origin --filter=blob:none", workspaceDir);
+      run("git branch keep origin/feature", workspaceDir);
+
+      // Delete the branch upstream: --refetch can no longer re-send its blob.
+      run("git push origin :feature", seedDir);
+
+      const output = run(GIT_FETCH_SCRIPT, workspaceDir);
+      expect(output).toContain(
+        "HEAL: objects still missing after refetch; keeping promisor config"
+      );
+
+      // Promisor config retained so the lazy-fetch fallback keeps working.
+      expect(run("git config --local --get remote.origin.partialclonefilter", workspaceDir)).toBe(
+        "blob:none"
+      );
+      // Incomplete-heal marker set: retries are throttled to daily.
+      expect(
+        Number(run("git config --local --get xum.promisorHealIncompleteAt", workspaceDir))
+      ).toBeGreaterThan(0);
+
+      // Within the daily window a second run must not attempt another refetch.
+      const secondOutput = run(GIT_FETCH_SCRIPT, workspaceDir);
+      expect(secondOutput).not.toContain("HEAL:");
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  }, 20000);
 });

src/common/utils/git/gitStatus.ts

@@ -176,6 +176,19 @@ export function parseGitStatusScriptOutput(output: string): ParsedGitStatusOutpu
   };
 }
 
+/**
+ * Git config keys that mark a repo as a promisor/partial clone. Previous
+ * versions of GIT_FETCH_SCRIPT fetched with --filter=blob:none, which made
+ * git persist this state (poisoning the repo: every later fetch stayed
+ * filtered and checkouts lazy-fetched blobs from the network). The fetch
+ * script's heal block and SSHRuntime's base-repo hygiene both unset these.
+ */
+export const PROMISOR_CONFIG_KEYS = [
+  "remote.origin.promisor",
+  "remote.origin.partialclonefilter",
+  "extensions.partialclone",
+] as const;
+
 /**
  * Smart git fetch script that minimizes lock contention.
  *
@@ -196,6 +209,75 @@ export GIT_ASKPASS=echo
 export SSH_ASKPASS=echo
 export GIT_SSH_COMMAND="\${GIT_SSH_COMMAND:-ssh} -o BatchMode=yes -o StrictHostKeyChecking=accept-new"
 
+# One-time heal for repos that previous versions of this script converted
+# into promisor/partial clones. --no-filter (used below) stops the damage but
+# does not remove the persisted promisor config, nor backfill the blobs that
+# earlier filtered fetches omitted. Left unhealed, "git worktree add"
+# (workspace creation) lazy-fetches those old blobs mid-checkout and fails on
+# transient network errors. Only repos whose filter is exactly the
+# "blob:none" this script used to write are healed. This block runs before
+# any ls-remote/primary-branch gating on purpose: a stale origin/HEAD (e.g.
+# default branch renamed upstream) makes the checks below exit early, which
+# must not leave the repo poisoned forever.
+if [ "$(git config --local --get remote.origin.partialclonefilter 2>/dev/null)" = "blob:none" ]; then
+  COMMON_DIR=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)
+  NOW=$(date +%s)
+  # A refetch can succeed while objects stay missing (see the completeness
+  # check below); that outcome may never improve, so retry it at most daily
+  # instead of hammering the network with a full refetch every poll.
+  LAST_INCOMPLETE=$(git config --local --get xum.promisorHealIncompleteAt 2>/dev/null || echo 0)
+  [ -n "$LAST_INCOMPLETE" ] || LAST_INCOMPLETE=0
+  if [ -n "$COMMON_DIR" ] && [ $((NOW - LAST_INCOMPLETE)) -ge 86400 ]; then
+    # mkdir is the atomic claim: sibling worktrees share repo config, so this
+    # keeps them from starting concurrent full refetches. Staleness comes
+    # from a timestamp file inside the lock (portable, unlike find/stat
+    # mtime probing): a lock left behind by a killed heal expires after an
+    # hour, which also rate-limits retries after a failed refetch (the
+    # failure path below keeps the lock in place for that reason).
+    HEAL_LOCK="$COMMON_DIR/xum-promisor-heal.lock"
+    LOCK_TS=$(cat "$HEAL_LOCK/started" 2>/dev/null || echo 0)
+    [ -n "$LOCK_TS" ] || LOCK_TS=0
+    if [ -d "$HEAL_LOCK" ] && [ $((NOW - LOCK_TS)) -gt 3600 ]; then
+      rm -rf "$HEAL_LOCK"
+    fi
+    if mkdir "$HEAL_LOCK" 2>/dev/null; then
+      echo "$NOW" > "$HEAL_LOCK/started"
+      echo "HEAL: backfilling promisor partial clone"
+      # --refetch (git >= 2.36) negotiates as if the repo had nothing, so the
+      # server re-sends every object reachable from the fetch refspec,
+      # including previously filtered-out blobs.
+      if git -c protocol.version=2 \\
+          fetch origin \\
+          --refetch \\
+          --no-filter \\
+          --prune \\
+          --no-tags \\
+          --no-recurse-submodules \\
+          --no-write-fetch-head \\
+          2>&1; then
+        # A successful refetch is not proof of completeness: it only re-sends
+        # objects reachable from the *remote's current refs*. Blobs referenced
+        # only by local refs into upstream-deleted branches can remain
+        # missing, and unsetting the promisor config then would turn a
+        # recoverable partial clone into a repo whose checkouts hard-fail
+        # ("unable to read sha1 file") with no lazy-fetch fallback. Only
+        # unset once every locally reachable object is actually present.
+        if git rev-list --objects --missing=print --all 2>/dev/null | grep -q '^?'; then
+          echo "HEAL: objects still missing after refetch; keeping promisor config"
+          git config --local xum.promisorHealIncompleteAt "$NOW" 2>/dev/null
+        else
+${PROMISOR_CONFIG_KEYS.map((key) => `          git config --local --unset-all ${key} 2>/dev/null`).join("\n")}
+          git config --local --unset-all xum.promisorHealIncompleteAt 2>/dev/null
+        fi
+        rm -rf "$HEAL_LOCK"
+      fi
+      # On refetch failure the lock (with its timestamp) stays in place so the
+      # next attempt waits out the 1h staleness window instead of re-running a
+      # full refetch on every status poll.
+    fi
+  fi
+fi
+
 # Get primary branch name
 PRIMARY_BRANCH=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's@^refs/remotes/origin/@@')
 if [ -z "$PRIMARY_BRANCH" ]; then
@@ -222,13 +304,25 @@ if [ "$LOCAL_SHA" = "$REMOTE_SHA" ]; then
 fi
 
 # Remote has new commits or ref moved - fetch updates
+#
+# --no-filter (NOT --filter=blob:none): a filtered fetch permanently converts
+# the repo into a promisor/partial clone (git writes remote.origin.promisor +
+# remote.origin.partialclonefilter on the first filtered fetch, and the
+# configured filter then applies to every subsequent plain fetch). That leaves
+# every commit fetched by this background loop without its blobs, so a later
+# "git worktree add" (workspace creation) must lazy-fetch blobs from the
+# remote mid-checkout and any transient network failure aborts it with
+# "fatal: could not fetch <oid> from promisor remote". --no-filter avoids
+# poisoning healthy repos and keeps this fetch unfiltered even in a repo that
+# is still poisoned (already-converted repos are backfilled and cleaned up by
+# the one-time heal block above).
 git -c protocol.version=2 \\
     -c fetch.negotiationAlgorithm=skipping \\
     fetch origin \\
     --prune \\
     --no-tags \\
     --no-recurse-submodules \\
     --no-write-fetch-head \\
-    --filter=blob:none \\
+    --no-filter \\
     2>&1
 `;

src/node/runtime/SSHRuntime.ts

@@ -39,6 +39,7 @@ import { expandTildeForSSH, cdCommandForSSH } from "./tildeExpansion";
 import { sleepWithAbort } from "@/node/utils/abort";
 import { execBuffered } from "@/node/utils/runtime/helpers";
 import { getErrorMessage } from "@/common/utils/errors";
+import { PROMISOR_CONFIG_KEYS } from "@/common/utils/git/gitStatus";
 import {
   type SSHRuntimeConfig,
   getControlPath,
@@ -106,11 +107,7 @@ const BASE_REPO_MAINTENANCE_WAIT_TIMEOUT_SECONDS = 30 * 60;
  *  `repo_has_promisor_remote()`. Unsetting all three is what makes
  *  receive-pack's `check_connected()` skip the buggy partial-clone fast
  *  path on subsequent pushes (see `stripBaseRepoPromisorConfig`). */
-const BASE_REPO_PROMISOR_CONFIG_KEYS = [
-  "remote.origin.promisor",
-  "remote.origin.partialclonefilter",
-  "extensions.partialclone",
-] as const;
+const BASE_REPO_PROMISOR_CONFIG_KEYS = PROMISOR_CONFIG_KEYS;
 const BASE_REPO_FRAGMENTED_PACK_THRESHOLD = 25;
 const PROJECT_SYNC_MAX_ATTEMPTS = 3;
 const PROJECT_SYNC_RETRYABLE_ERRORS = [
@@ -152,7 +149,11 @@ function isUnresolvedDeltaPushFailure(errorMsg: string): boolean {
 }
 
 function isMissingObjectCheckoutFailure(message: string): boolean {
-  return /unable to read sha1 file|Could not reset index file|missing (blob|tree|commit)|bad object|unable to read tree|object file .* is empty|loose object .* is corrupt/i.test(
+  // "could not fetch ... from promisor remote": the checkout needed objects the
+  // repo does not have and a lazy fetch from upstream failed (e.g. transient
+  // network drop). The objects are still missing locally, so the same
+  // repair-from-local path applies.
+  return /unable to read sha1 file|Could not reset index file|missing (blob|tree|commit)|bad object|unable to read tree|object file .* is empty|loose object .* is corrupt|could not fetch .* from promisor remote/i.test(
     message
   );
 }
@@ -2657,6 +2658,18 @@ export class SSHRuntime extends RemoteRuntime {
       // path where ensureBaseRepo() has retry/error handling instead of risking
       // materializing a worktree from still-poisoned shared config.
       `git --git-dir=${baseRepoPathArg} symbolic-ref HEAD ${baseRepoUnbornHeadArg} 2>/dev/null || { echo WARM_MISS:base-head-normalization-failed; exit 0; }`,
+      // Best-effort promisor strip, mirroring ensureBaseRepo()'s epilogue. The
+      // warm path skips ensureBaseRepo(), and background status fetches that
+      // ran `git fetch --filter=blob:none` inside sibling worktrees register
+      // the shared base repo as a promisor remote (remote.origin.promisor +
+      // partialclonefilter). Left in place, `git worktree add` below would
+      // lazy-fetch missing blobs from upstream mid-checkout, so a transient
+      // network drop aborts workspace creation with "could not fetch <oid>
+      // from promisor remote" instead of the repairable missing-objects path.
+      ...BASE_REPO_PROMISOR_CONFIG_KEYS.map(
+        (key) =>
+          `git --git-dir=${baseRepoPathArg} config --local --unset-all ${shescape.quote(key)} 2>/dev/null || true`
+      ),
     ];
 
     const originPreamble = originUrlArg
@@ -2706,7 +2719,7 @@ export class SSHRuntime extends RemoteRuntime {
       "wt_status=$?",
       'if [ "$wt_status" -ne 0 ]; then',
       '  case "$wt_output" in',
-      '    *"unable to read sha1 file"*|*"Could not reset index file"*|*"missing blob"*|*"missing tree"*|*"missing commit"*|*"bad object"*|*"unable to read tree"*) wt_reason=missing-objects ;;',
+      '    *"unable to read sha1 file"*|*"Could not reset index file"*|*"missing blob"*|*"missing tree"*|*"missing commit"*|*"bad object"*|*"unable to read tree"*|*"from promisor remote"*) wt_reason=missing-objects ;;',
       "    *) wt_reason=worktree-add-failed ;;",
       "  esac",
       `  git -C ${baseRepoPathArg} worktree remove --force ${workspacePathArg} >/dev/null 2>&1 || rm -rf ${workspacePathArg}`,

tests/runtime/runtime.test.ts

@@ -2206,6 +2206,12 @@ describeIntegration("Runtime integration tests", () => {
             `git --git-dir="${baseRepoPath}" config --local core.bare true`,
             `git --git-dir="${baseRepoPath}" config --local core.worktree "${bogusWorktreePath}"`,
             `git --git-dir="${baseRepoPath}" symbolic-ref HEAD refs/heads/main`,
+            // Simulate what a background `git fetch --filter=blob:none` from a
+            // sibling worktree registers in the shared gitdir. A promisor base
+            // repo lazy-fetches missing blobs over the network mid-checkout, so
+            // the warm path must strip these before `git worktree add`.
+            `git --git-dir="${baseRepoPath}" config --local remote.origin.promisor true`,
+            `git --git-dir="${baseRepoPath}" config --local remote.origin.partialclonefilter blob:none`,
           ].join(" && ")
         );
         expect(poisonResult.exitCode).toBe(0);
@@ -2243,6 +2249,20 @@ describeIntegration("Runtime integration tests", () => {
         );
         expect(baseRepoCoreWorktreeCheck.exitCode).toBe(1);
 
+        // Promisor/partial-clone registration must be stripped so worktree
+        // materialization never lazy-fetches blobs over the network.
+        const baseRepoPromisorCheck = await execSSH(
+          runtime,
+          `git --git-dir="${baseRepoPath}" config --get remote.origin.promisor`
+        );
+        expect(baseRepoPromisorCheck.exitCode).toBe(1);
+
+        const baseRepoFilterCheck = await execSSH(
+          runtime,
+          `git --git-dir="${baseRepoPath}" config --get remote.origin.partialclonefilter`
+        );
+        expect(baseRepoFilterCheck.exitCode).toBe(1);
+
         const baseHeadSymbolicCheck = await execSSH(
           runtime,
           `git --git-dir="${baseRepoPath}" symbolic-ref -q HEAD`