Merge pull request #2988 from trycompai/tofik/fix-onboarding-stale-manifest-fk

fix(frameworks): prune deleted editor templates during org initialization

Author: tofikwest

apps/api/src/frameworks/frameworks-source-loader.helper.spec.ts

@@ -0,0 +1,193 @@
+// loadFrameworkSources operates purely on the injected `tx`; its only @db
+// imports are `import type`, which are erased at runtime, so a no-op mock keeps
+// jest from initialising a real PrismaClient.
+jest.mock('@db', () => ({}));
+
+import { loadFrameworkSources } from './frameworks-source-loader.helper';
+import type { FrameworkManifest } from './framework-versioning/manifest.types';
+
+type LoaderTx = Parameters<typeof loadFrameworkSources>[0]['tx'];
+
+function manifest(overrides: Partial<FrameworkManifest> = {}): FrameworkManifest {
+  return {
+    framework: { id: 'frk_pci', name: 'PCI DSS', catalogVersion: '1', description: null },
+    requirements: [],
+    controls: [],
+    policies: [],
+    tasks: [],
+    ...overrides,
+  };
+}
+
+/**
+ * A manifest with one control wiring up one requirement, one policy and one
+ * task. Callers extend it to introduce ids that no longer exist live.
+ */
+function fullManifest(): FrameworkManifest {
+  return manifest({
+    requirements: [{ id: 'req_live', identifier: 'R1', name: 'Req', description: null }],
+    controls: [
+      {
+        id: 'ct_live',
+        name: 'Control',
+        description: 'd',
+        requirementIds: ['req_live'],
+        policyIds: ['pt_live'],
+        taskIds: ['tt_live'],
+        documentTypes: [],
+      },
+    ],
+    policies: [
+      { id: 'pt_live', name: 'Policy', description: null, content: [], frequency: null, department: null },
+    ],
+    tasks: [{ id: 'tt_live', name: 'Task', description: 'd', frequency: null, department: null }],
+  });
+}
+
+function mockTx({
+  versions,
+  liveControlIds,
+  livePolicyIds,
+  liveTasks,
+  liveRequirementIds,
+}: {
+  versions: Array<{ id: string; frameworkId: string; manifest: FrameworkManifest }>;
+  liveControlIds: string[];
+  livePolicyIds: string[];
+  liveTasks: Array<{ id: string; automationStatus: string }>;
+  liveRequirementIds: string[];
+}): LoaderTx {
+  return {
+    frameworkVersion: { findMany: jest.fn().mockResolvedValue(versions) },
+    frameworkEditorControlTemplate: {
+      findMany: jest.fn().mockResolvedValue(liveControlIds.map((id) => ({ id }))),
+    },
+    frameworkEditorPolicyTemplate: {
+      findMany: jest.fn().mockResolvedValue(livePolicyIds.map((id) => ({ id }))),
+    },
+    frameworkEditorTaskTemplate: {
+      findMany: jest.fn().mockResolvedValue(liveTasks),
+    },
+    frameworkEditorRequirement: {
+      findMany: jest.fn().mockResolvedValue(liveRequirementIds.map((id) => ({ id }))),
+    },
+  } as unknown as LoaderTx;
+}
+
+function ids<T extends { id: string }>(rows: T[]): string[] {
+  return rows.map((r) => r.id);
+}
+
+describe('loadFrameworkSources — stale-manifest reconciliation', () => {
+  const frameworkEditorIds = ['frk_pci'];
+
+  it('drops a manifest TASK whose live template was hard-deleted (the reported Task_taskTemplateId_fkey bug)', async () => {
+    const m = fullManifest();
+    m.controls[0].taskIds = ['tt_live', 'tt_dead'];
+    m.tasks.push({ id: 'tt_dead', name: 'Deleted Task', description: 'd', frequency: null, department: null });
+
+    const tx = mockTx({
+      versions: [{ id: 'fv_1', frameworkId: 'frk_pci', manifest: m }],
+      liveControlIds: ['ct_live'],
+      livePolicyIds: ['pt_live'],
+      liveTasks: [{ id: 'tt_live', automationStatus: 'AUTOMATED' }], // tt_dead absent
+      liveRequirementIds: ['req_live'],
+    });
+
+    const result = await loadFrameworkSources({ frameworkEditorIds, frameworkEditorFrameworks: [], tx });
+
+    // tt_dead must never reach task.createMany — it would FK-fail on insert.
+    expect(ids(result.taskTemplates)).toEqual(['tt_live']);
+  });
+
+  it('drops a manifest CONTROL whose live template was hard-deleted', async () => {
+    const m = fullManifest();
+    m.controls.push({
+      id: 'ct_dead',
+      name: 'Deleted Control',
+      description: 'd',
+      requirementIds: [],
+      policyIds: [],
+      taskIds: [],
+      documentTypes: [],
+    });
+
+    const tx = mockTx({
+      versions: [{ id: 'fv_1', frameworkId: 'frk_pci', manifest: m }],
+      liveControlIds: ['ct_live'], // ct_dead absent
+      livePolicyIds: ['pt_live'],
+      liveTasks: [{ id: 'tt_live', automationStatus: 'AUTOMATED' }],
+      liveRequirementIds: ['req_live'],
+    });
+
+    const result = await loadFrameworkSources({ frameworkEditorIds, frameworkEditorFrameworks: [], tx });
+
+    expect(ids(result.controlTemplates)).toEqual(['ct_live']);
+  });
+
+  it('drops a manifest POLICY whose live template was hard-deleted', async () => {
+    const m = fullManifest();
+    m.controls[0].policyIds = ['pt_live', 'pt_dead'];
+    m.policies.push({
+      id: 'pt_dead',
+      name: 'Deleted Policy',
+      description: null,
+      content: [],
+      frequency: null,
+      department: null,
+    });
+
+    const tx = mockTx({
+      versions: [{ id: 'fv_1', frameworkId: 'frk_pci', manifest: m }],
+      liveControlIds: ['ct_live'],
+      livePolicyIds: ['pt_live'], // pt_dead absent
+      liveTasks: [{ id: 'tt_live', automationStatus: 'AUTOMATED' }],
+      liveRequirementIds: ['req_live'],
+    });
+
+    const result = await loadFrameworkSources({ frameworkEditorIds, frameworkEditorFrameworks: [], tx });
+
+    expect(ids(result.policyTemplates)).toEqual(['pt_live']);
+  });
+
+  it('drops a dead REQUIREMENT from groupedRelations (RequirementMap.requirementId has no downstream guard)', async () => {
+    const m = fullManifest();
+    m.controls[0].requirementIds = ['req_live', 'req_dead'];
+    m.requirements.push({ id: 'req_dead', identifier: 'R2', name: 'Deleted Req', description: null });
+
+    const tx = mockTx({
+      versions: [{ id: 'fv_1', frameworkId: 'frk_pci', manifest: m }],
+      liveControlIds: ['ct_live'],
+      livePolicyIds: ['pt_live'],
+      liveTasks: [{ id: 'tt_live', automationStatus: 'AUTOMATED' }],
+      liveRequirementIds: ['req_live'], // req_dead absent
+    });
+
+    const result = await loadFrameworkSources({ frameworkEditorIds, frameworkEditorFrameworks: [], tx });
+
+    const rel = result.groupedRelations.find((r) => r.controlTemplateId === 'ct_live');
+    expect(rel?.requirementTemplateIds).toEqual(['req_live']);
+  });
+
+  it('passes everything through unchanged and resolves automationStatus when all templates are live', async () => {
+    const tx = mockTx({
+      versions: [{ id: 'fv_1', frameworkId: 'frk_pci', manifest: fullManifest() }],
+      liveControlIds: ['ct_live'],
+      livePolicyIds: ['pt_live'],
+      liveTasks: [{ id: 'tt_live', automationStatus: 'MANUAL' }],
+      liveRequirementIds: ['req_live'],
+    });
+
+    const result = await loadFrameworkSources({ frameworkEditorIds, frameworkEditorFrameworks: [], tx });
+
+    expect(ids(result.controlTemplates)).toEqual(['ct_live']);
+    expect(ids(result.policyTemplates)).toEqual(['pt_live']);
+    expect(ids(result.taskTemplates)).toEqual(['tt_live']);
+    // automationStatus is not in the manifest — it must come from the live row.
+    expect(result.taskTemplates[0].automationStatus).toBe('MANUAL');
+    const rel = result.groupedRelations.find((r) => r.controlTemplateId === 'ct_live');
+    expect(rel?.requirementTemplateIds).toEqual(['req_live']);
+    expect(rel?.policyTemplateIds).toEqual(['pt_live']);
+    expect(rel?.taskTemplateIds).toEqual(['tt_live']);
+  });
+});

apps/api/src/frameworks/frameworks-source-loader.helper.ts

@@ -89,6 +89,11 @@ export async function loadFrameworkSources({
   const controlsMap = new Map<string, LoadedFrameworkSources['controlTemplates'][number]>();
   const policiesMap = new Map<string, LoadedFrameworkSources['policyTemplates'][number]>();
   const tasksMap = new Map<string, LoadedFrameworkSources['taskTemplates'][number]>();
+  // Requirement ids referenced by manifest controls, validated against live
+  // FrameworkEditorRequirement rows below. Dead ones are pruned from relations
+  // so RequirementMap inserts never reference a deleted requirement.
+  const manifestRequirementIds = new Set<string>();
+  const deadRequirementIds = new Set<string>();
 
   // groupedRelations accumulates per-framework control edges. A reusable
   // control can carry different policy/task/document links in each framework.
@@ -139,7 +144,10 @@ export async function loadFrameworkSources({
         });
       }
       const rel = getOrCreateRelation(frameworkId, c.id);
-      for (const rid of c.requirementIds) rel.requirementTemplateIds.add(rid);
+      for (const rid of c.requirementIds) {
+        rel.requirementTemplateIds.add(rid);
+        manifestRequirementIds.add(rid);
+      }
       for (const pid of c.policyIds) rel.policyTemplateIds.add(pid);
       for (const tid of c.taskIds) rel.taskTemplateIds.add(tid);
       for (const formType of c.documentTypes ?? []) {
@@ -182,17 +190,74 @@ export async function loadFrameworkSources({
     void frameworkId; // keep loop structure tidy; id used above
   }
 
-  // Resolve automationStatus from live task templates for any manifest tasks
-  const manifestTaskIds = Array.from(tasksMap.keys());
-  if (manifestTaskIds.length > 0) {
-    const liveTasks = await tx.frameworkEditorTaskTemplate.findMany({
-      where: { id: { in: manifestTaskIds } },
-      select: { id: true, automationStatus: true },
-    });
+  // Manifests are frozen snapshots: a control/policy/task/requirement they
+  // reference may have been hard-deleted from the live framework-editor tables
+  // since the version was published. Org rows FK to those live tables
+  // (Control.controlTemplateId, Policy.policyTemplateId, Task.taskTemplateId,
+  // RequirementMap.requirementId), so creating one that points at a deleted
+  // template raises a P2003 FK violation and aborts onboarding. Reconcile
+  // against the live tables: resolve task automationStatus (only carried live)
+  // and drop any manifest reference whose live row is gone. Fallback-path ids
+  // are read straight from live tables below, so they are never pruned here.
+  if (manifestByFrameworkId.size > 0) {
+    const manifestControlIds = Array.from(controlsMap.keys());
+    const manifestPolicyIds = Array.from(policiesMap.keys());
+    const manifestTaskIds = Array.from(tasksMap.keys());
+    const manifestReqIds = Array.from(manifestRequirementIds);
+
+    const [liveControls, livePolicies, liveTasks, liveRequirements] = await Promise.all([
+      tx.frameworkEditorControlTemplate.findMany({
+        where: { id: { in: manifestControlIds } },
+        select: { id: true },
+      }),
+      tx.frameworkEditorPolicyTemplate.findMany({
+        where: { id: { in: manifestPolicyIds } },
+        select: { id: true },
+      }),
+      tx.frameworkEditorTaskTemplate.findMany({
+        where: { id: { in: manifestTaskIds } },
+        select: { id: true, automationStatus: true },
+      }),
+      tx.frameworkEditorRequirement.findMany({
+        where: { id: { in: manifestReqIds } },
+        select: { id: true },
+      }),
+    ]);
+
+    // automationStatus isn't carried in the manifest — copy it from the live row.
     for (const lt of liveTasks) {
       const existing = tasksMap.get(lt.id);
       if (existing) existing.automationStatus = lt.automationStatus;
     }
+
+    const liveControlIds = new Set(liveControls.map((c) => c.id));
+    const livePolicyIds = new Set(livePolicies.map((p) => p.id));
+    const liveTaskIds = new Set(liveTasks.map((t) => t.id));
+    const liveRequirementIds = new Set(liveRequirements.map((r) => r.id));
+
+    const droppedControls = manifestControlIds.filter((id) => !liveControlIds.has(id));
+    const droppedPolicies = manifestPolicyIds.filter((id) => !livePolicyIds.has(id));
+    const droppedTasks = manifestTaskIds.filter((id) => !liveTaskIds.has(id));
+    const droppedRequirements = manifestReqIds.filter((id) => !liveRequirementIds.has(id));
+
+    for (const id of droppedControls) controlsMap.delete(id);
+    for (const id of droppedPolicies) policiesMap.delete(id);
+    for (const id of droppedTasks) tasksMap.delete(id);
+    for (const id of droppedRequirements) deadRequirementIds.add(id);
+
+    if (
+      droppedControls.length ||
+      droppedPolicies.length ||
+      droppedTasks.length ||
+      droppedRequirements.length
+    ) {
+      console.warn(
+        `loadFrameworkSources: pruned manifest references with no live framework-editor template ` +
+          `(stale manifest — republish the affected framework version). ` +
+          `controls=[${droppedControls.join(', ')}] policies=[${droppedPolicies.join(', ')}] ` +
+          `tasks=[${droppedTasks.join(', ')}] requirements=[${droppedRequirements.join(', ')}]`,
+      );
+    }
   }
 
   // Fallback: frameworks without a published version load from live tables.
@@ -327,7 +392,12 @@ export async function loadFrameworkSources({
   const groupedRelations = Array.from(relationsByControl.values()).map((rel) => ({
     frameworkId: rel.frameworkId,
     controlTemplateId: rel.controlTemplateId,
-    requirementTemplateIds: Array.from(rel.requirementTemplateIds),
+    // Dead manifest requirements are pruned here: RequirementMap.requirementId
+    // has no downstream instance-map guard (unlike policy/task ids), so a stale
+    // id would otherwise FK-fail on RequirementMap_requirementId_fkey.
+    requirementTemplateIds: Array.from(rel.requirementTemplateIds).filter(
+      (id) => !deadRequirementIds.has(id),
+    ),
     policyTemplateIds: Array.from(rel.policyTemplateIds),
     taskTemplateIds: Array.from(rel.taskTemplateIds),
     documentTypes: Array.from(rel.documentTypes),