Repository navigation
Expand file tree
/
Copy pathagent.ts
More file actions
268 lines (225 loc) · 10.2 KB
/
Copy pathagent.ts
File metadata and controls
268 lines (225 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
import 'dotenv/config';
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { join } from 'node:path';
import { parse as parseYaml } from 'yaml';
import { Hyperbrowser } from '@hyperbrowser/sdk';
import OpenAI from 'openai';
import fetch from 'node-fetch';
import cron from 'node-cron';
interface Config {
cadence_hours?: number;
notify?: { slack_webhook?: string };
urls: Array<{ id: string; url: string; group?: string; extract?: string }>;
rules?: { taggers?: Array<{ pattern: string; tag: string }> };
}
interface Snapshot {
text: string;
hash: string;
timestamp: string;
}
const hb = new Hyperbrowser({ apiKey: process.env.HYPERBROWSER_API_KEY! });
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY! });
function loadConfig(): Config {
return parseYaml(readFileSync('config.yaml', 'utf8'));
}
function sha(text: string): string {
return createHash('sha256').update(text).digest('hex');
}
function log(message: string): void {
const timestamp = new Date().toLocaleString('en-IN', { timeZone: 'Asia/Kolkata' });
console.log(`[${timestamp} IST] ${message}`);
}
function hoursToCronExpression(hours: number): string {
// Convert hours to cron expression: "0 */N * * *" (every N hours)
return `0 */${hours} * * *`;
}
function readSnap(id: string): Snapshot | null {
const path = join('.data/snapshots', `${id}.json`);
return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null;
}
function writeSnap(id: string, text: string): void {
const snapshot = { text, hash: sha(text), timestamp: new Date().toISOString() };
writeFileSync(join('.data/snapshots', `${id}.json`), JSON.stringify(snapshot, null, 2));
}
async function browse(url: string): Promise<string> {
const result = await hb.scrape.startAndWait({
url: url,
});
return result.data?.text || '';
}
async function analyzeContent(id: string, url: string, prevText: string | null, currText: string): Promise<string> {
const isInitial = !prevText;
const systemPrompt = isInitial
? 'You are a research intern briefing your founder. Analyze this content and provide insights in this format: Key Points: [3-4 clean bullets] Strategic Insights: [2-3 sentences] Business Impact: [1-2 sentences] Priority: P1/P2 Tags: [2-3 relevant tags]. Keep it clean and readable.'
: 'You are a research intern briefing your founder on changes. Format: Changes: [3-4 clean bullets] Strategic Impact: [2-3 sentences] Business Implications: [1-2 sentences] Priority: P0/P1/P2 Tags: [2-3 relevant tags]. Keep it clean and readable.';
const userContent = isInitial
? `Analyze ${url} (${id}) for competitive intelligence:\n\n${currText.slice(0, 3000)}`
: `Analyze changes for ${id}:\n\nPREVIOUS:\n${prevText?.slice(0, 2000)}\n\nCURRENT:\n${currText.slice(0, 2000)}`;
const response = await openai.chat.completions.create({
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
messages: [{ role: 'system', content: systemPrompt }, { role: 'user', content: userContent }]
});
return response.choices[0]?.message?.content || (isInitial ? 'Initial baseline established' : 'No analysis available');
}
async function generatePlan(urls: string[]): Promise<string> {
const response = await openai.chat.completions.create({
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
messages: [
{ role: 'system', content: 'You are a research intern. Create a brief monitoring plan (3-4 sentences max per URL) explaining what to watch and why it matters. Keep it clean and Slack-ready with minimal formatting.' },
{ role: 'user', content: `Brief monitoring plan for: ${urls.join(', ')}` }
]
});
return response.choices[0]?.message?.content || 'No plan available';
}
function diffChanged(prev: Snapshot | null, curr: string): boolean {
return !prev || prev.hash !== sha(curr);
}
async function notifySlack(webhook: string, content: string): Promise<void> {
try {
// Slack supports up to 40,000 characters, but we'll use 35,000 to be safe
let message = content;
if (content.length > 35000) {
// Smart truncation: keep the beginning and add truncation notice
message = content.slice(0, 34900) + '\n\n...(message truncated due to length)';
}
await fetch(webhook, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ text: message })
});
} catch (error) {
console.warn('Slack notification failed:', error);
}
}
async function writeReport(group: string, changes: Array<{ id: string; url: string; summary: string }>, plan: string): Promise<string> {
const timestamp = new Date().toLocaleString('en-IN', { timeZone: 'Asia/Kolkata' });
const filename = `${Date.now()}.md`;
let content = `COMPETITIVE INTELLIGENCE REPORT\nGroup: ${group} | ${timestamp} IST\n\nMONITORING STRATEGY:\n${plan}\n\nFINDINGS:\n\n`;
if (changes.length > 0) {
for (const change of changes) {
content += `${change.id.replace('_', ' ').toUpperCase()}\nSource: ${change.url}\n\n${change.summary}\n\n---\n\n`;
}
const p0Count = changes.filter(c => c.summary.includes('P0')).length;
const p1Count = changes.filter(c => c.summary.includes('P1')).length;
content += `SUMMARY:\n`;
if (p0Count > 0) content += `CRITICAL (P0): ${p0Count} urgent changes\n`;
if (p1Count > 0) content += `HIGH PRIORITY (P1): ${p1Count} important developments\n`;
content += `Total updates: ${changes.length} sources\n\n`;
content += `ACTIONS:\n- Immediate: Review P0 changes\n- This week: Analyze P1 implications\n- Strategic: Update positioning\n\n`;
} else {
content += `No changes detected. All sources stable.\n\n`;
}
content += `Generated by Hyperbrowser Research Agent`;
writeFileSync(join('.data/reports', filename), content);
return content;
}
async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
const timeout = new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)
);
return Promise.race([promise, timeout]);
}
async function processUrl(urlConfig: { id: string; url: string }): Promise<{ id: string; url: string; summary: string } | null> {
try {
log(`Processing ${urlConfig.id}...`);
const currText = await withTimeout(browse(urlConfig.url), 60000);
const prevSnap = readSnap(urlConfig.id);
if (diffChanged(prevSnap, currText)) {
const summary = await analyzeContent(urlConfig.id, urlConfig.url, prevSnap?.text || null, currText);
writeSnap(urlConfig.id, currText);
log(`✓ Changes detected in ${urlConfig.id}`);
return { id: urlConfig.id, url: urlConfig.url, summary };
}
log(`○ No changes in ${urlConfig.id}`);
return null;
} catch (error) {
log(`✗ Error processing ${urlConfig.id}: ${error instanceof Error ? error.message : error}`);
return null;
}
}
async function runBatch(urls: Array<{ id: string; url: string }>, concurrency = 4): Promise<Array<{ id: string; url: string; summary: string }>> {
const results: Array<{ id: string; url: string; summary: string }> = [];
for (let i = 0; i < urls.length; i += concurrency) {
const batch = urls.slice(i, i + concurrency);
const batchResults = await Promise.all(batch.map(processUrl));
results.push(...batchResults.filter(r => r !== null));
}
return results;
}
async function runMonitoring(): Promise<void> {
const groupFilter = process.argv.includes('--group') ? process.argv[process.argv.indexOf('--group') + 1] : null;
mkdirSync('.data/snapshots', { recursive: true });
mkdirSync('.data/reports', { recursive: true });
const config = loadConfig();
const filteredUrls = groupFilter ? config.urls.filter(u => u.group === groupFilter) : config.urls;
if (filteredUrls.length === 0) {
log('No URLs to process');
return;
}
log(`Starting monitoring run for ${filteredUrls.length} URLs${groupFilter ? ` (group: ${groupFilter})` : ''}`);
const plan = await generatePlan(filteredUrls.map(u => u.url));
const changes = await runBatch(filteredUrls);
if (changes.length > 0) {
const report = await writeReport(groupFilter || 'all', changes, plan);
log(`✓ Report generated with ${changes.length} changes`);
if (config.notify?.slack_webhook) {
const webhook = config.notify.slack_webhook.replace('${SLACK_WEBHOOK_URL}', process.env.SLACK_WEBHOOK_URL || '');
if (webhook && webhook !== '${SLACK_WEBHOOK_URL}') {
await notifySlack(webhook, report);
log('✓ Slack notification sent');
}
}
} else {
log('○ No changes detected across all sources');
}
log('Monitoring run completed\n');
}
async function startContinuousMode(): Promise<void> {
const config = loadConfig();
const cadenceHours = config.cadence_hours || 3;
const cronExpression = hoursToCronExpression(cadenceHours);
log(`🚀 Starting Research Bot in continuous mode`);
log(`📅 Schedule: Every ${cadenceHours} hours (${cronExpression})`);
log(`🎯 Monitoring ${config.urls.length} URLs`);
log(`Press Ctrl+C to stop\n`);
// Run immediately on startup
log('⚡ Running initial monitoring...');
await runMonitoring().catch(error => log(`Error in monitoring: ${error}`));
// Schedule recurring runs
const task = cron.schedule(cronExpression, async () => {
log('⏰ Scheduled monitoring triggered');
await runMonitoring().catch(error => log(`Error in monitoring: ${error}`));
}, {
scheduled: false
});
task.start();
log(`✅ Scheduler started. Next run in ${cadenceHours} hours.`);
// Keep the process alive
process.on('SIGINT', () => {
log('\n🛑 Shutting down gracefully...');
task.destroy();
process.exit(0);
});
process.on('SIGTERM', () => {
log('\n🛑 Received SIGTERM, shutting down...');
task.destroy();
process.exit(0);
});
}
async function main(): Promise<void> {
const args = process.argv.slice(2);
const isContinuous = args.includes('--continuous');
const isOnce = args.includes('--once');
if (isContinuous) {
await startContinuousMode();
} else if (isOnce || (!isContinuous && !isOnce)) {
// Default to one-shot mode for backward compatibility
log('🔄 Running in one-shot mode');
await runMonitoring();
}
}
main().catch(error => {
log(`Fatal error: ${error}`);
process.exit(1);
});