diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..9658f114 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-08-25 - N+1 Memory Fetch Bottleneck in Astro DB/Drizzle +**Learning:** In Astro API routes using Drizzle ORM, fetching full tables via `db.select().from(Table)` just to compute `.length` and perform in-memory filtering (e.g. `filter(t => t.createdAt >= today)`) creates a severe memory bottleneck as the dataset grows. +**Action:** Always use Drizzle's `count()` aggregation (e.g., `db.select({ value: count() })`) combined with database-level filtering functions like `gte()` or `inArray()` directly inside `where()` clauses. Ensure proper `import` or destructuring of these functions from `loadAstroDb()` wrapper. diff --git a/src/pages/api/dashboard-kpis.ts b/src/pages/api/dashboard-kpis.ts index 5cf2dd56..ae26e9d9 100644 --- a/src/pages/api/dashboard-kpis.ts +++ b/src/pages/api/dashboard-kpis.ts @@ -1,27 +1,28 @@ -import type { APIRoute } from 'astro'; -import { FORGE_SWARM_AGENT_COUNT } from '../../lib/agent-instruction-defaults'; -import { loadAstroDb } from '../../lib/load-astro-db'; +import type { APIRoute } from "astro"; +import { FORGE_SWARM_AGENT_COUNT } from "../../lib/agent-instruction-defaults"; +import { loadAstroDb } from "../../lib/load-astro-db"; import { fetchZimaOSSessionsPayload, normalizeZimaOSSessions, -} from '../../lib/forge-gateway'; +} from "../../lib/forge-gateway"; function countRunningSessions(sessions: unknown[]): number { return sessions.filter((s) => { - const row = s && typeof s === 'object' ? (s as Record) : {}; - const st = String(row.status || row.state || '').toLowerCase(); + const row = + s && typeof s === "object" ? (s as Record) : {}; + const st = String(row.status || row.state || "").toLowerCase(); return ( - st === 'running' || - st === 'active' || - st === 'connected' || - st === 'online' + st === "running" || + st === "active" || + st === "connected" || + st === "online" ); }).length; } function isOpenQueueStatus(st: string): boolean { const s = String(st).toLowerCase(); - return s === 'open' || s === 'in_progress'; + return s === "open" || s === "in_progress"; } export const GET: APIRoute = async () => { @@ -42,56 +43,94 @@ export const GET: APIRoute = async () => { }; try { - const { db, Project, AgentTask, Request, AgentAppIssue, AgentDependencyRequest, eq } = - await loadAstroDb(); + const { + db, + Project, + AgentTask, + Request, + AgentAppIssue, + AgentDependencyRequest, + eq, + count, + gte, + inArray, + } = await loadAstroDb(); const today = new Date(); today.setHours(0, 0, 0, 0); - const [projects, tasksAll, openRequests, issuesAll, depsAll] = await Promise.all([ - db.select().from(Project), - db.select().from(AgentTask), - db.select().from(Request).where(eq(Request.status, 'pending')), - db.select().from(AgentAppIssue), - db.select().from(AgentDependencyRequest), + // ⚡ Bolt Optimization: Replaced N+1 full-table memory fetches with database-level `count()` aggregations + // and `where()` filters (like `gte` and `inArray`) to reduce memory consumption and execution time from O(N) to O(1). + const [ + projectsCount, + tasksTotalCount, + tasksTodayCount, + openRequestsCount, + openAppIssuesCount, + openDependencyRequestsCount, + ] = await Promise.all([ + db.select({ value: count() }).from(Project), + db.select({ value: count() }).from(AgentTask), + db + .select({ value: count() }) + .from(AgentTask) + .where(gte(AgentTask.createdAt, today)), + db + .select({ value: count() }) + .from(Request) + .where(eq(Request.status, "pending")), + db + .select({ value: count() }) + .from(AgentAppIssue) + .where(inArray(AgentAppIssue.status, ["open", "in_progress"])), + db + .select({ value: count() }) + .from(AgentDependencyRequest) + .where(inArray(AgentDependencyRequest.status, ["open", "in_progress"])), ]); - const tasksTodayCount = tasksAll.filter((t) => { - const d = t.createdAt instanceof Date ? t.createdAt : new Date(t.createdAt as Date); - return d >= today; - }).length; - - base.projectCount = projects.length; - base.tasksTotal = tasksAll.length; - base.tasksToday = tasksTodayCount; - base.openRequests = openRequests.length; - base.openAppIssues = issuesAll.filter((r) => isOpenQueueStatus(String(r.status))).length; - base.openDependencyRequests = depsAll.filter((r) => isOpenQueueStatus(String(r.status))).length; + base.projectCount = projectsCount[0].value; + base.tasksTotal = tasksTotalCount[0].value; + base.tasksToday = tasksTodayCount[0].value; + base.openRequests = openRequestsCount[0].value; + base.openAppIssues = openAppIssuesCount[0].value; + base.openDependencyRequests = openDependencyRequestsCount[0].value; } catch (e) { const msg = e instanceof Error ? e.message : String(e); base.dbError = msg; if (import.meta.env.DEV) { - console.error('[dashboard-kpis] lecture base:', e); + console.error("[dashboard-kpis] lecture base:", e); } - return new Response(JSON.stringify(base), { status: 200, headers: { 'Content-Type': 'application/json' } }); + return new Response(JSON.stringify(base), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); } try { const zimaosResult = await fetchZimaOSSessionsPayload(undefined); const ocSessions = zimaosResult.ok - ? (normalizeZimaOSSessions(zimaosResult.data) as Record[]) + ? (normalizeZimaOSSessions(zimaosResult.data) as Record< + string, + unknown + >[]) : []; base.zimaosOk = zimaosResult.ok; base.zimaosSessionCount = ocSessions.length; - base.zimaosRunningCount = zimaosResult.ok ? countRunningSessions(ocSessions) : 0; + base.zimaosRunningCount = zimaosResult.ok + ? countRunningSessions(ocSessions) + : 0; base.zimaosVia = zimaosResult.via ?? null; - base.zimaosError = zimaosResult.ok ? null : zimaosResult.error ?? null; + base.zimaosError = zimaosResult.ok ? null : (zimaosResult.error ?? null); } catch (e) { const msg = e instanceof Error ? e.message : String(e); - base.zimaosError = msg || 'ZimaOS : erreur inattendue'; + base.zimaosError = msg || "ZimaOS : erreur inattendue"; if (import.meta.env.DEV) { - console.error('[dashboard-kpis] ZimaOS:', e); + console.error("[dashboard-kpis] ZimaOS:", e); } } - return new Response(JSON.stringify(base), { status: 200, headers: { 'Content-Type': 'application/json' } }); + return new Response(JSON.stringify(base), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); };