fix(admin): explain Trino recovery blockers

controlplane/admin/README.md

@@ -192,6 +192,11 @@ The **Trino cell → Instance recovery** panel lets admins select a shared-pool
 instance, review its exact identity, explicitly authorize failure retirement,
 and follow progress. It reuses the existing recovery API. It does not verify
 live workload or health, cancel an accepted recovery, or bypass a conflict.
+The recovery action remains visible but disabled when prerequisites fail. The
+preview identifies the exact phase, frozen-pool, identity-field, and stored
+capacity blockers instead of hiding the form. Existing requests and retired
+instances retain their progress or terminal status and cannot start another
+recovery operation.
 See the [recovery runbook](../../docs/runbooks/trino-pool-admin-recovery.md)
 for prerequisites, result-loss risks, and identical-request retry rules.
 

controlplane/admin/ui/src/lib/trinoRecovery.ts

@@ -19,14 +19,33 @@ export function recoveryIdentity(identity: TrinoRecoveryIdentity): TrinoRecovery
 }
 
 export function validRecoveryIdentity(identity: TrinoRecoveryIdentity): boolean {
-  return Number.isSafeInteger(identity.expected_generation) && identity.expected_generation > 0 &&
-    [identity.incarnation, identity.pod_uid, identity.boot_id, identity.node_id, identity.coordinator_id]
-      .every((value) => typeof value === "string" && value.trim().length > 0);
+  return invalidRecoveryIdentityFields(identity).length === 0;
+}
+
+function invalidRecoveryIdentityFields(identity: TrinoRecoveryIdentity): string[] {
+  const fields: string[] = [];
+  if (!Number.isSafeInteger(identity.expected_generation) || identity.expected_generation <= 0) fields.push("expected_generation");
+  for (const field of ["incarnation", "pod_uid", "boot_id", "node_id", "coordinator_id"] as const) {
+    if (typeof identity[field] !== "string" || !identity[field].trim()) fields.push(field);
+  }
+  return fields;
 }
 
 export function recoveryAllowed(preview: TrinoRecoveryPreview): boolean {
-  return !preview.request && !preview.capacity.frozen && validRecoveryIdentity(preview.instance) &&
-    preview.capacity.stored_serving >= preview.capacity.min_serving && preview.instance.phase === "DRAINING";
+  return recoveryBlockers(preview).length === 0;
+}
+
+export function recoveryBlockers(preview: TrinoRecoveryPreview): string[] {
+  const blockers: string[] = [];
+  if (preview.request) blockers.push(`Recovery operation ${preview.request.operation_id} is already recorded. A new request is not allowed.`);
+  if (preview.instance.phase !== "DRAINING") blockers.push(`Instance phase is ${preview.instance.phase}; recovery requires DRAINING.`);
+  if (preview.capacity.frozen) blockers.push("The pool is frozen. Recovery cannot proceed while it is frozen.");
+  const invalidIdentity = invalidRecoveryIdentityFields(preview.instance);
+  if (invalidIdentity.length) blockers.push(`Missing or invalid admitted identity fields: ${invalidIdentity.join(", ")}.`);
+  if (!(preview.capacity.stored_serving >= preview.capacity.min_serving)) {
+    blockers.push(`Stored serving count is ${preview.capacity.stored_serving}; minimum required is ${preview.capacity.min_serving}. Recovery cannot proceed below this minimum.`);
+  }
+  return blockers;
 }
 
 export function recoveryReviewKey(preview: TrinoRecoveryPreview): string {

controlplane/admin/ui/src/pages/TrinoRecovery.test.tsx

@@ -132,7 +132,7 @@ describe("Trino recovery", () => {
     fireEvent.click(button);
     fireEvent.click(button);
     await waitFor(() => expect(mocks.requestTrinoRecovery).toHaveBeenCalledTimes(1));
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
   });
 
   it("restores an unknown request after remount with the identical operation ID and body", async () => {
@@ -146,7 +146,7 @@ describe("Trino recovery", () => {
     firstPage.unmount();
     mount();
     await selectInstance();
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
     await userEvent.click(screen.getByRole("button", { name: "Retry identical request" }));
     await waitFor(() => expect(mocks.requestTrinoRecovery).toHaveBeenCalledTimes(2));
     expect(mocks.requestTrinoRecovery.mock.calls[1]).toEqual(first);
@@ -166,7 +166,7 @@ describe("Trino recovery", () => {
     firstPage.unmount();
     mount();
     await selectInstance();
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
     expect(screen.queryByRole("button", { name: "Review a new preview" })).not.toBeInTheDocument();
     expect(mocks.requestTrinoRecovery).toHaveBeenCalledTimes(2);
   });
@@ -196,7 +196,7 @@ describe("Trino recovery", () => {
     await selectInstance();
     expect(screen.getByText(/cannot be cancelled or amended/)).toBeInTheDocument();
     expect(screen.getByText("operation-existing")).toBeInTheDocument();
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
     expect(mocks.requestTrinoRecovery).not.toHaveBeenCalled();
   });
 
@@ -207,7 +207,7 @@ describe("Trino recovery", () => {
     await userEvent.selectOptions(screen.getByRole("combobox", { name: "Trino instance" }), "instance-a");
     await screen.findByText(/Check your sign-in and permissions/);
     expect(screen.getByRole("button", { name: "Refresh preview" })).toBeDisabled();
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
   });
 
   it("blocks writes when identity refresh failed even if the cached role is admin", () => {
@@ -226,7 +226,7 @@ describe("Trino recovery", () => {
     await screen.findByText(/identity or recorded intent changed/i);
     expect(mocks.requestTrinoRecovery).toHaveBeenCalledTimes(1);
     expect(screen.queryByRole("button", { name: "Retry identical request" })).not.toBeInTheDocument();
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
     await waitFor(() => expect(screen.getByRole("button", { name: "Review a new preview" })).toBeEnabled());
     await userEvent.click(screen.getByRole("button", { name: "Review a new preview" }));
     expect(await screen.findByLabelText("Type the instance ID")).toHaveValue("");
@@ -255,7 +255,7 @@ describe("Trino recovery", () => {
     await userEvent.click(screen.getByRole("button", { name: "Refresh preview" }));
     await screen.findByText(/retirement completed|recovery completed/i);
     expect(screen.getByRole("combobox", { name: "Trino instance" })).toHaveValue("instance-a");
-    expect(screen.queryByRole("button", { name: "Request destructive recovery" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
   });
 
   it("isolates a cell switch from an old in-flight response", async () => {
@@ -439,4 +439,53 @@ describe("Trino recovery", () => {
     expect(mocks.trinoInstances).toHaveBeenCalledTimes(2);
     expect(screen.getByText("No active shared-pool instances.")).toBeInTheDocument();
   });
+
+  it("shows why a draining instance cannot recover when stored capacity is zero", async () => {
+    const blocked = preview();
+    blocked.capacity.stored_serving = 0;
+    blocked.capacity.min_serving = 3;
+    mocks.trinoRecovery.mockResolvedValue(blocked);
+    mount();
+    await selectInstance();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
+    expect(screen.getByText("Stored serving count is 0; minimum required is 3. Recovery cannot proceed below this minimum.")).toBeInTheDocument();
+    expect(mocks.requestTrinoRecovery).not.toHaveBeenCalled();
+  });
+
+  it("lists every failed eligibility condition instead of a generic missing button", async () => {
+    const blocked = preview();
+    blocked.instance.phase = "CREATING";
+    blocked.instance.boot_id = "";
+    blocked.instance.pod_uid = "";
+    blocked.instance.expected_generation = 0;
+    blocked.capacity.frozen = true;
+    blocked.capacity.stored_serving = 1;
+    mocks.trinoRecovery.mockResolvedValue(blocked);
+    mount();
+    await selectInstance();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
+    expect(screen.getByText("Instance phase is CREATING; recovery requires DRAINING.")).toBeInTheDocument();
+    expect(screen.getByText("The pool is frozen. Recovery cannot proceed while it is frozen.")).toBeInTheDocument();
+    expect(screen.getByText("Missing or invalid admitted identity fields: expected_generation, pod_uid, boot_id.")).toBeInTheDocument();
+    expect(screen.getByText("Stored serving count is 1; minimum required is 2. Recovery cannot proceed below this minimum.")).toBeInTheDocument();
+    expect(mocks.requestTrinoRecovery).not.toHaveBeenCalled();
+  });
+
+  it("requires confirmation again after a capacity blocker clears", async () => {
+    mount();
+    await selectInstance();
+    await confirmRecovery();
+    const blocked = preview();
+    blocked.capacity.stored_serving = 0;
+    mocks.trinoRecovery.mockResolvedValue(blocked);
+    await userEvent.click(screen.getByRole("button", { name: "Refresh preview" }));
+    expect(await screen.findByText(/Stored serving count is 0/)).toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
+    mocks.trinoRecovery.mockResolvedValue(preview());
+    await userEvent.click(screen.getByRole("button", { name: "Refresh preview" }));
+    await waitFor(() => expect(screen.getByLabelText("Type the instance ID")).toHaveValue(""));
+    expect(screen.getByRole("button", { name: "Request destructive recovery" })).toBeDisabled();
+    expect(screen.getByRole("checkbox", { name: /retained results/ })).not.toBeChecked();
+    expect(mocks.requestTrinoRecovery).not.toHaveBeenCalled();
+  });
 });

controlplane/admin/ui/src/pages/TrinoRecovery.tsx

@@ -7,7 +7,7 @@ import { Input } from "@/components/ui/input";
 import { api, ApiError } from "@/lib/api";
 import { POLL } from "@/lib/query";
 import {
-  RECOVERY_PREVIEW_MAX_AGE_MS, readRecoveryRequest, recoveryAccessDenied, recoveryAllowed, recoveryComplete,
+  RECOVERY_PREVIEW_MAX_AGE_MS, readRecoveryRequest, recoveryAccessDenied, recoveryAllowed, recoveryBlockers, recoveryComplete,
   recoveryError, recoveryIdentity, recoveryPollingPaused, recoveryReviewKey, recoveryStorageKey, validRecoveryReason,
 } from "@/lib/trinoRecovery";
 import type { TrinoRecoveryBody } from "@/types/api";
@@ -78,6 +78,7 @@ function RecoveryInstance({ cell, instance, actor }: { cell: string; instance: s
     refetchInterval: (query) => recoveryComplete(query.state.data?.instance.phase ?? "") || recoveryPollingPaused(query.state.error) ? false : POLL.normal,
   });
   const snapshot = preview.data?.cell === cell && preview.data.instance.instance_id === instance ? preview.data : undefined;
+  const blockers = snapshot ? recoveryBlockers(snapshot) : [];
   const mutation = useMutation({
     mutationFn: ({ body }: { body: TrinoRecoveryBody; previouslyUnknown: boolean }) => api.requestTrinoRecovery(instance, body, cell), retry: false,
     onSettled: async (_data, error, { body, previouslyUnknown }) => {
@@ -178,6 +179,10 @@ function RecoveryInstance({ cell, instance, actor }: { cell: string; instance: s
     {preview.error && <p role="alert">{recoveryError(preview.error)} The previous preview cannot authorize a new request.</p>}
     {localError && <p role="alert">{localError}</p>}
     {notice && <p role="status">{notice}</p>}
+    {!snapshot && <div className="space-y-2">
+      <p>A valid preview is required before requesting recovery.</p>
+      <Button variant="destructive" disabled>Request destructive recovery</Button>
+    </div>}
     {snapshot && <>
       <p className="rounded border border-warning/40 p-3">This stored snapshot does not verify live workload or capacity. Check workload and remaining capacity before requesting recovery. The operator verifies its safety gates separately.</p>
       <dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
@@ -216,10 +221,21 @@ function RecoveryInstance({ cell, instance, actor }: { cell: string; instance: s
           onClick={() => submit(pending)}>Retry identical request</Button>}
         {rejected && !recorded && <Button variant="outline" disabled={!fresh} onClick={reviewAgain}>Review a new preview</Button>}
       </div>}
-      {!pending && !recorded && !complete && (recoveryAllowed(snapshot) ?
-        <RecoveryForm key={reviewKey} instance={instance} disabled={denied || checking || !!localError}
-          submitDisabled={!fresh} onSubmit={start} /> :
-        <p>Recovery requires a DRAINING instance with a complete admitted identity, an unfrozen pool, and stored serving capacity at or above the minimum.</p>)}
+      {!pending && !recorded && !complete ? <>
+        {blockers.length > 0 && <div className="space-y-1" role="status">
+          <p>Recovery is unavailable for this instance:</p>
+          <ul className="list-disc space-y-1 pl-5" aria-label="Recovery blockers">
+            {blockers.map((blocker) => <li key={blocker}>{blocker}</li>)}
+          </ul>
+        </div>}
+        <RecoveryForm key={reviewKey} instance={instance} disabled={denied || checking || !!localError || blockers.length > 0}
+          submitDisabled={!fresh} onSubmit={start} />
+      </> : <div className="space-y-2">
+        <p>{complete ? "This instance is already retired. It cannot receive a new recovery request." : recorded ?
+          "An immutable recovery request already exists for this instance. Follow its progress above." :
+          "Resolve the existing request before creating a new recovery request. Its status and available actions are shown above."}</p>
+        <Button variant="destructive" disabled>Request destructive recovery</Button>
+      </div>}
     </>}
     <p className="text-xs text-muted-foreground">Recovery may lose retained results and continuations. Accepted requests are immutable. This tab saves the exact request for manual retries; it never automatically submits or retries recovery.</p>
   </section>;

docs/runbooks/trino-pool-admin-recovery.md

@@ -30,6 +30,15 @@ Automatic inventory and preview polling stops when the API returns `404` or
 `503`. Use the corresponding refresh button after recovery becomes available.
 Authentication and permission failures also stop polling; restore access first.
 
+After selecting an instance, the recovery action remains visible when it cannot
+be used. The preview lists each failed prerequisite: a phase other than
+`DRAINING`, a frozen pool, missing or invalid admitted identity fields, or a
+stored serving count below the minimum. The capacity message shows both counts.
+A draining phase alone does not make an instance eligible. Do not bypass the
+minimum by editing lifecycle records. Restore sufficient serving capacity before
+requesting recovery. Existing requests and retired instances show their status
+with the new-request action disabled.
+
 Before submitting, complete the independent checks above, enter a short reason,
 type the exact instance ID, and acknowledge both those checks and the potential
 loss of results. The confirmation applies to the displayed process identity and