⚡ Bolt: Refactor dashboard KPIs to use database aggregations

.jules/bolt.md

@@ -0,0 +1,3 @@
+## 2024-08-25 - N+1 Memory Fetch Bottleneck in Astro DB/Drizzle
+**Learning:** In Astro API routes using Drizzle ORM, fetching full tables via `db.select().from(Table)` just to compute `.length` and perform in-memory filtering (e.g. `filter(t => t.createdAt >= today)`) creates a severe memory bottleneck as the dataset grows.
+**Action:** Always use Drizzle's `count()` aggregation (e.g., `db.select({ value: count() })`) combined with database-level filtering functions like `gte()` or `inArray()` directly inside `where()` clauses. Ensure proper `import` or destructuring of these functions from `loadAstroDb()` wrapper.

src/pages/api/dashboard-kpis.ts

@@ -1,27 +1,28 @@
-import type { APIRoute } from 'astro';
-import { FORGE_SWARM_AGENT_COUNT } from '../../lib/agent-instruction-defaults';
-import { loadAstroDb } from '../../lib/load-astro-db';
+import type { APIRoute } from "astro";
+import { FORGE_SWARM_AGENT_COUNT } from "../../lib/agent-instruction-defaults";
+import { loadAstroDb } from "../../lib/load-astro-db";
 import {
   fetchZimaOSSessionsPayload,
   normalizeZimaOSSessions,
-} from '../../lib/forge-gateway';
+} from "../../lib/forge-gateway";
 
 function countRunningSessions(sessions: unknown[]): number {
   return sessions.filter((s) => {
-    const row = s && typeof s === 'object' ? (s as Record<string, unknown>) : {};
-    const st = String(row.status || row.state || '').toLowerCase();
+    const row =
+      s && typeof s === "object" ? (s as Record<string, unknown>) : {};
+    const st = String(row.status || row.state || "").toLowerCase();
     return (
-      st === 'running' ||
-      st === 'active' ||
-      st === 'connected' ||
-      st === 'online'
+      st === "running" ||
+      st === "active" ||
+      st === "connected" ||
+      st === "online"
     );
   }).length;
 }
 
 function isOpenQueueStatus(st: string): boolean {
   const s = String(st).toLowerCase();
-  return s === 'open' || s === 'in_progress';
+  return s === "open" || s === "in_progress";
 }
 
 export const GET: APIRoute = async () => {
@@ -42,56 +43,94 @@ export const GET: APIRoute = async () => {
   };
 
   try {
-    const { db, Project, AgentTask, Request, AgentAppIssue, AgentDependencyRequest, eq } =
-      await loadAstroDb();
+    const {
+      db,
+      Project,
+      AgentTask,
+      Request,
+      AgentAppIssue,
+      AgentDependencyRequest,
+      eq,
+      count,
+      gte,
+      inArray,
+    } = await loadAstroDb();
     const today = new Date();
     today.setHours(0, 0, 0, 0);
 
-    const [projects, tasksAll, openRequests, issuesAll, depsAll] = await Promise.all([
-      db.select().from(Project),
-      db.select().from(AgentTask),
-      db.select().from(Request).where(eq(Request.status, 'pending')),
-      db.select().from(AgentAppIssue),
-      db.select().from(AgentDependencyRequest),
+    // ⚡ Bolt Optimization: Replaced N+1 full-table memory fetches with database-level `count()` aggregations
+    // and `where()` filters (like `gte` and `inArray`) to reduce memory consumption and execution time from O(N) to O(1).
+    const [
+      projectsCount,
+      tasksTotalCount,
+      tasksTodayCount,
+      openRequestsCount,
+      openAppIssuesCount,
+      openDependencyRequestsCount,
+    ] = await Promise.all([
+      db.select({ value: count() }).from(Project),
+      db.select({ value: count() }).from(AgentTask),
+      db
+        .select({ value: count() })
+        .from(AgentTask)
+        .where(gte(AgentTask.createdAt, today)),
+      db
+        .select({ value: count() })
+        .from(Request)
+        .where(eq(Request.status, "pending")),
+      db
+        .select({ value: count() })
+        .from(AgentAppIssue)
+        .where(inArray(AgentAppIssue.status, ["open", "in_progress"])),
+      db
+        .select({ value: count() })
+        .from(AgentDependencyRequest)
+        .where(inArray(AgentDependencyRequest.status, ["open", "in_progress"])),
     ]);
 
-    const tasksTodayCount = tasksAll.filter((t) => {
-      const d = t.createdAt instanceof Date ? t.createdAt : new Date(t.createdAt as Date);
-      return d >= today;
-    }).length;
-
-    base.projectCount = projects.length;
-    base.tasksTotal = tasksAll.length;
-    base.tasksToday = tasksTodayCount;
-    base.openRequests = openRequests.length;
-    base.openAppIssues = issuesAll.filter((r) => isOpenQueueStatus(String(r.status))).length;
-    base.openDependencyRequests = depsAll.filter((r) => isOpenQueueStatus(String(r.status))).length;
+    base.projectCount = projectsCount[0].value;
+    base.tasksTotal = tasksTotalCount[0].value;
+    base.tasksToday = tasksTodayCount[0].value;
+    base.openRequests = openRequestsCount[0].value;
+    base.openAppIssues = openAppIssuesCount[0].value;
+    base.openDependencyRequests = openDependencyRequestsCount[0].value;
   } catch (e) {
     const msg = e instanceof Error ? e.message : String(e);
     base.dbError = msg;
     if (import.meta.env.DEV) {
-      console.error('[dashboard-kpis] lecture base:', e);
+      console.error("[dashboard-kpis] lecture base:", e);
     }
-    return new Response(JSON.stringify(base), { status: 200, headers: { 'Content-Type': 'application/json' } });
+    return new Response(JSON.stringify(base), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    });
   }
 
   try {
     const zimaosResult = await fetchZimaOSSessionsPayload(undefined);
     const ocSessions = zimaosResult.ok
-      ? (normalizeZimaOSSessions(zimaosResult.data) as Record<string, unknown>[])
+      ? (normalizeZimaOSSessions(zimaosResult.data) as Record<
+          string,
+          unknown
+        >[])
       : [];
     base.zimaosOk = zimaosResult.ok;
     base.zimaosSessionCount = ocSessions.length;
-    base.zimaosRunningCount = zimaosResult.ok ? countRunningSessions(ocSessions) : 0;
+    base.zimaosRunningCount = zimaosResult.ok
+      ? countRunningSessions(ocSessions)
+      : 0;
     base.zimaosVia = zimaosResult.via ?? null;
-    base.zimaosError = zimaosResult.ok ? null : zimaosResult.error ?? null;
+    base.zimaosError = zimaosResult.ok ? null : (zimaosResult.error ?? null);
   } catch (e) {
     const msg = e instanceof Error ? e.message : String(e);
-    base.zimaosError = msg || 'ZimaOS : erreur inattendue';
+    base.zimaosError = msg || "ZimaOS : erreur inattendue";
     if (import.meta.env.DEV) {
-      console.error('[dashboard-kpis] ZimaOS:', e);
+      console.error("[dashboard-kpis] ZimaOS:", e);
     }
   }
 
-  return new Response(JSON.stringify(base), { status: 200, headers: { 'Content-Type': 'application/json' } });
+  return new Response(JSON.stringify(base), {
+    status: 200,
+    headers: { "Content-Type": "application/json" },
+  });
 };