From 28002d5ffda63ddbc7fe5bd37fb2f7b0ba4727c6 Mon Sep 17 00:00:00 2001 From: Nick Trombley Date: Thu, 31 Jul 2025 05:54:03 -0400 Subject: [PATCH] Allow UI uploads for local history and twitter data --- src/app/routes/integrations/index.tsx | 134 +++++++++++++ src/server/api/routers/integrations.ts | 189 ++++++++++++++++++ src/server/db/connections/arc-sqlite.ts | 18 +- src/server/db/connections/dia-sqlite.ts | 18 +- src/server/db/connections/index.ts | 2 + .../integrations/browser-history/sync-all.ts | 38 +++- .../integrations/browser-history/sync.ts | 24 ++- src/server/integrations/twitter/sync.ts | 26 ++- 8 files changed, 426 insertions(+), 23 deletions(-) diff --git a/src/app/routes/integrations/index.tsx b/src/app/routes/integrations/index.tsx index a188c460..609c3665 100644 --- a/src/app/routes/integrations/index.tsx +++ b/src/app/routes/integrations/index.tsx @@ -7,6 +7,19 @@ import { Alert, AlertDescription } from '@/components/ui/alert'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; +const fileToBase64 = (file: File): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result as string; + const base64 = result.split(',')[1] ?? ''; + resolve(base64); + }; + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); +}; + export const Route = createFileRoute('/integrations/')({ component: RouteComponent, }); @@ -19,6 +32,9 @@ interface LogMessage { function RouteComponent() { const [messages, setMessages] = useState([]); + const [arcData, setArcData] = useState(undefined); + const [diaData, setDiaData] = useState(undefined); + const [twitterFiles, setTwitterFiles] = useState([]); const raindropMutation = trpc.integrations.runRaindrop.useMutation({ onSuccess: (data) => { @@ -146,6 +162,48 @@ function RouteComponent() { githubMutation.mutate(); }; + const browsingMutation = trpc.integrations.runBrowsing.useMutation({ + onSuccess: (data) => { + setMessages(data.messages); + }, + onError: (error) => { + setMessages((prev) => [ + ...prev, + { + type: 'error', + message: error.message, + timestamp: new Date(), + }, + ]); + }, + }); + + const handleRunBrowsing = () => { + setMessages([]); + browsingMutation.mutate({ arc: arcData, dia: diaData }); + }; + + const twitterMutation = trpc.integrations.runTwitter.useMutation({ + onSuccess: (data) => { + setMessages(data.messages); + }, + onError: (error) => { + setMessages((prev) => [ + ...prev, + { + type: 'error', + message: error.message, + timestamp: new Date(), + }, + ]); + }, + }); + + const handleRunTwitter = () => { + setMessages([]); + twitterMutation.mutate({ files: twitterFiles }); + }; + const getMessageIcon = (type: LogMessage['type']) => { switch (type) { case 'success': @@ -326,6 +384,82 @@ function RouteComponent() { + + + Browser History + Sync history from Arc and Dia + + +
+ { + const f = e.target.files?.[0]; + if (f) setArcData(await fileToBase64(f)); + }} + /> + { + const f = e.target.files?.[0]; + if (f) setDiaData(await fileToBase64(f)); + }} + /> +
+ + + + + + Twitter + Import local Twitter bookmarks + + + { + const files = e.target.files ? [...e.target.files] : []; + const bases = await Promise.all(files.map(fileToBase64)); + setTwitterFiles(bases); + }} + className="mb-2" + /> + + + GitHub diff --git a/src/server/api/routers/integrations.ts b/src/server/api/routers/integrations.ts index 97503c5d..b47701e2 100644 --- a/src/server/api/routers/integrations.ts +++ b/src/server/api/routers/integrations.ts @@ -1,10 +1,13 @@ import { TRPCError } from '@trpc/server'; +import { z } from 'zod/v4'; import { syncLightroomImages } from '../../integrations/adobe/sync'; import { syncAirtableData } from '../../integrations/airtable/sync'; +import { syncAllBrowserData } from '../../integrations/browser-history/sync-all'; import { syncFeedbin } from '../../integrations/feedbin/sync'; import { syncGitHubData } from '../../integrations/github/sync'; import { syncRaindropData } from '../../integrations/raindrop/sync'; import { syncReadwiseDocuments } from '../../integrations/readwise/sync'; +import { parseBookmarkDataStrings, syncTwitterData } from '../../integrations/twitter/sync'; import { createTRPCRouter, publicProcedure } from '../init'; interface LogMessage { @@ -555,4 +558,190 @@ export const integrationsRouter = createTRPCRouter({ console.warn = originalConsoleWarn; } }), + runBrowsing: publicProcedure + .input(z.object({ arc: z.string().optional(), dia: z.string().optional() })) + .mutation(async ({ input }): Promise => { + const messages: LogMessage[] = []; + let entriesCreated = 0; + + const originalConsoleLog = console.log; + const originalConsoleError = console.error; + const originalConsoleWarn = console.warn; + + console.log = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + const entryMatch = message.match(/Successfully created (\d+) entries/); + if (entryMatch && entryMatch[1]) { + entriesCreated = parseInt(entryMatch[1], 10); + } + + messages.push({ + type: 'info', + message, + timestamp: new Date(), + }); + + originalConsoleLog(...args); + }; + + console.error = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + messages.push({ + type: 'error', + message, + timestamp: new Date(), + }); + + originalConsoleError(...args); + }; + + console.warn = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + messages.push({ + type: 'warn', + message, + timestamp: new Date(), + }); + + originalConsoleWarn(...args); + }; + + try { + const files = { + arc: input.arc ? Buffer.from(input.arc, 'base64') : undefined, + dia: input.dia ? Buffer.from(input.dia, 'base64') : undefined, + }; + await syncAllBrowserData(async () => true, files); + + messages.push({ + type: 'success', + message: `Browser history sync completed successfully${entriesCreated > 0 ? `. Created ${entriesCreated} entries.` : '.'}`, + timestamp: new Date(), + }); + + return { + success: true, + messages, + entriesCreated, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; + + messages.push({ + type: 'error', + message: `Sync failed: ${errorMessage}`, + timestamp: new Date(), + }); + + throw new TRPCError({ + code: 'INTERNAL_SERVER_ERROR', + message: `Browser history sync failed: ${errorMessage}`, + }); + } finally { + console.log = originalConsoleLog; + console.error = originalConsoleError; + console.warn = originalConsoleWarn; + } + }), + runTwitter: publicProcedure + .input(z.object({ files: z.array(z.string()).optional() })) + .mutation(async ({ input }): Promise => { + const messages: LogMessage[] = []; + let entriesCreated = 0; + + const originalConsoleLog = console.log; + const originalConsoleError = console.error; + const originalConsoleWarn = console.warn; + + console.log = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + const entryMatch = message.match(/Successfully created (\d+) entries/); + if (entryMatch && entryMatch[1]) { + entriesCreated = parseInt(entryMatch[1], 10); + } + + messages.push({ + type: 'info', + message, + timestamp: new Date(), + }); + + originalConsoleLog(...args); + }; + + console.error = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + messages.push({ + type: 'error', + message, + timestamp: new Date(), + }); + + originalConsoleError(...args); + }; + + console.warn = (...args: unknown[]) => { + const message = args + .map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' '); + + messages.push({ + type: 'warn', + message, + timestamp: new Date(), + }); + + originalConsoleWarn(...args); + }; + + try { + const contents = input.files?.map((f) => Buffer.from(f, 'base64').toString('utf8')) ?? []; + const data = contents.length > 0 ? parseBookmarkDataStrings(contents) : undefined; + await syncTwitterData(data); + + messages.push({ + type: 'success', + message: `Twitter sync completed successfully${entriesCreated > 0 ? `. Created ${entriesCreated} entries.` : '.'}`, + timestamp: new Date(), + }); + + return { + success: true, + messages, + entriesCreated, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; + + messages.push({ + type: 'error', + message: `Sync failed: ${errorMessage}`, + timestamp: new Date(), + }); + + throw new TRPCError({ + code: 'INTERNAL_SERVER_ERROR', + message: `Twitter sync failed: ${errorMessage}`, + }); + } finally { + console.log = originalConsoleLog; + console.error = originalConsoleError; + console.warn = originalConsoleWarn; + } + }), }); diff --git a/src/server/db/connections/arc-sqlite.ts b/src/server/db/connections/arc-sqlite.ts index a46d3cfc..dc0efb7a 100644 --- a/src/server/db/connections/arc-sqlite.ts +++ b/src/server/db/connections/arc-sqlite.ts @@ -1,6 +1,5 @@ -import { copyFileSync } from 'fs'; -import { existsSync } from 'fs'; -import { homedir } from 'os'; +import { copyFileSync, existsSync, mkdtempSync, writeFileSync } from 'fs'; +import { homedir, tmpdir } from 'os'; import { join } from 'path'; import { createClient } from '@libsql/client'; import { drizzle } from 'drizzle-orm/libsql'; @@ -27,3 +26,16 @@ export const createArcConnection = () => { return drizzle(client, { schema }); }; + +export const createArcConnectionFromBuffer = (buffer: Buffer) => { + const tempDir = mkdtempSync(join(tmpdir(), 'arc-history-')); + const tempPath = join(tempDir, 'History'); + writeFileSync(tempPath, buffer); + + const client = createClient({ + url: `file:${tempPath}`, + intMode: 'bigint', + }); + + return drizzle(client, { schema }); +}; diff --git a/src/server/db/connections/dia-sqlite.ts b/src/server/db/connections/dia-sqlite.ts index 94394420..ee170591 100644 --- a/src/server/db/connections/dia-sqlite.ts +++ b/src/server/db/connections/dia-sqlite.ts @@ -1,6 +1,5 @@ -import { copyFileSync } from 'fs'; -import { existsSync } from 'fs'; -import { homedir } from 'os'; +import { copyFileSync, existsSync, mkdtempSync, writeFileSync } from 'fs'; +import { homedir, tmpdir } from 'os'; import { join } from 'path'; import { createClient } from '@libsql/client'; import { drizzle } from 'drizzle-orm/libsql'; @@ -27,3 +26,16 @@ export const createDiaConnection = () => { return drizzle(client, { schema }); }; + +export const createDiaConnectionFromBuffer = (buffer: Buffer) => { + const tempDir = mkdtempSync(join(tmpdir(), 'dia-history-')); + const tempPath = join(tempDir, 'History'); + writeFileSync(tempPath, buffer); + + const client = createClient({ + url: `file:${tempPath}`, + intMode: 'bigint', + }); + + return drizzle(client, { schema }); +}; diff --git a/src/server/db/connections/index.ts b/src/server/db/connections/index.ts index a2010850..096d3bf4 100644 --- a/src/server/db/connections/index.ts +++ b/src/server/db/connections/index.ts @@ -1 +1,3 @@ export * from './postgres'; +export { createArcConnection, createArcConnectionFromBuffer } from './arc-sqlite'; +export { createDiaConnection, createDiaConnectionFromBuffer } from './dia-sqlite'; diff --git a/src/server/integrations/browser-history/sync-all.ts b/src/server/integrations/browser-history/sync-all.ts index fb68f056..ae258f50 100644 --- a/src/server/integrations/browser-history/sync-all.ts +++ b/src/server/integrations/browser-history/sync-all.ts @@ -1,8 +1,16 @@ +import { + createArcConnectionFromBuffer, + createDiaConnectionFromBuffer, +} from '@/server/db/connections'; import { createIntegrationLogger } from '../common/logging'; import { runIntegration } from '../common/run-integration'; import { arcConfig } from './browsers/arc'; import { diaConfig } from './browsers/dia'; -import { syncBrowserHistory as syncSingleBrowser } from './sync'; +import { + askForConfirmation, + syncBrowserHistory as syncSingleBrowser, + type ConfirmFn, +} from './sync'; import { BrowserNotInstalledError } from './types'; const logger = createIntegrationLogger('browser-history', 'sync-all'); @@ -11,14 +19,26 @@ const logger = createIntegrationLogger('browser-history', 'sync-all'); * Synchronizes all browser history (Arc and Dia) with the database * This function orchestrates multiple browser syncs under a single integration run */ -async function syncAllBrowserData(integrationRunId: number): Promise { +interface BrowserHistoryFiles { + arc?: Buffer; + dia?: Buffer; +} + +async function syncAllBrowserData( + integrationRunId: number, + confirmFn: ConfirmFn, + files?: BrowserHistoryFiles +): Promise { logger.start('Starting all browser history synchronization'); let totalEntriesCreated = 0; // Run Arc browser sync try { logger.info('Starting Arc Browser Sync'); - const arcEntries = await syncSingleBrowser(arcConfig, integrationRunId); + const config = files?.arc + ? { ...arcConfig, createConnection: () => createArcConnectionFromBuffer(files.arc as Buffer) } + : arcConfig; + const arcEntries = await syncSingleBrowser(config, integrationRunId, confirmFn); totalEntriesCreated += arcEntries; logger.complete('Arc Browser Sync', arcEntries); } catch (error) { @@ -36,7 +56,10 @@ async function syncAllBrowserData(integrationRunId: number): Promise { // Run Dia browser sync try { logger.info('Starting Dia Browser Sync'); - const diaEntries = await syncSingleBrowser(diaConfig, integrationRunId); + const config = files?.dia + ? { ...diaConfig, createConnection: () => createDiaConnectionFromBuffer(files.dia as Buffer) } + : diaConfig; + const diaEntries = await syncSingleBrowser(config, integrationRunId, confirmFn); totalEntriesCreated += diaEntries; logger.complete('Dia Browser Sync', diaEntries); } catch (error) { @@ -54,10 +77,13 @@ async function syncAllBrowserData(integrationRunId: number): Promise { /** * Orchestrates all browser history synchronization */ -async function syncAllBrowserHistory(): Promise { +async function syncAllBrowserHistory( + confirmFn: ConfirmFn = askForConfirmation, + files?: BrowserHistoryFiles +): Promise { try { logger.start('Starting browser history synchronization'); - await runIntegration('browser_history', syncAllBrowserData); + await runIntegration('browser_history', (id) => syncAllBrowserData(id, confirmFn, files)); logger.complete('Browser history synchronization completed'); } catch (error) { logger.error('Error syncing browser history', error); diff --git a/src/server/integrations/browser-history/sync.ts b/src/server/integrations/browser-history/sync.ts index bdbc62a1..587654e3 100644 --- a/src/server/integrations/browser-history/sync.ts +++ b/src/server/integrations/browser-history/sync.ts @@ -80,7 +80,9 @@ const createPrompt = (): readline.Interface => { * @param message - The message to display to the user * @returns Promise that resolves to true if the user confirms, false otherwise */ -const askForConfirmation = async (message: string): Promise => { +export type ConfirmFn = (message: string) => Promise; + +export const askForConfirmation: ConfirmFn = async (message: string): Promise => { const rl = createPrompt(); return new Promise((resolve) => { @@ -144,14 +146,15 @@ function sanitizeUrl(url: string): string | null { */ async function syncBrowserHistory( browserConfig: BrowserConfig, - integrationRunId: number + integrationRunId: number, + confirmFn: ConfirmFn = askForConfirmation ): Promise { try { // Get current hostname const currentHostname = os.hostname(); // Step 1: Check if the current hostname is known - const shouldProceed = await checkHostname(currentHostname, browserConfig.name); + const shouldProceed = await checkHostname(currentHostname, browserConfig.name, confirmFn); if (!shouldProceed) { logger.info('Sync cancelled by user'); return 0; @@ -218,7 +221,11 @@ async function syncBrowserHistory( * @param browser - The browser type * @returns Promise that resolves to true if the sync should proceed, false otherwise */ -async function checkHostname(currentHostname: string, _browser: Browser): Promise { +async function checkHostname( + currentHostname: string, + _browser: Browser, + confirmFn: ConfirmFn +): Promise { // Get all unique hostnames from the database const uniqueHostnames = await db .select({ @@ -233,7 +240,7 @@ async function checkHostname(currentHostname: string, _browser: Browser): Promis // If current hostname is not in the database, ask for confirmation if (!knownHostnames.has(currentHostname)) { logger.info('Known hostnames: ' + Array.from(knownHostnames).join(', ')); - return askForConfirmation( + return confirmFn( `Current hostname "${currentHostname}" has not been seen before. Proceed with sync?` ); } @@ -510,11 +517,14 @@ async function insertHistoryEntries(processedHistory: BrowsingHistoryInsert[]): */ export { syncBrowserHistory }; -export function createBrowserSyncFunction(browserConfig: BrowserConfig) { +export function createBrowserSyncFunction( + browserConfig: BrowserConfig, + confirmFn: ConfirmFn = askForConfirmation +) { return async (): Promise => { try { await runIntegration('browser_history', (integrationRunId) => - syncBrowserHistory(browserConfig, integrationRunId) + syncBrowserHistory(browserConfig, integrationRunId, confirmFn) ); } catch (error) { logger.error(`Error syncing ${browserConfig.displayName} browser history`, error); diff --git a/src/server/integrations/twitter/sync.ts b/src/server/integrations/twitter/sync.ts index c1759ab2..0529816c 100644 --- a/src/server/integrations/twitter/sync.ts +++ b/src/server/integrations/twitter/sync.ts @@ -172,10 +172,15 @@ export async function loadBookmarksData(): Promise<{ * @returns The number of successfully processed tweets * @throws Error if processing fails */ -async function syncTwitterBookmarks(integrationRunId: number): Promise { +async function syncTwitterBookmarks( + integrationRunId: number, + uploadedData?: TwitterBookmarksArray +): Promise { try { // Step 1: Load bookmarks data - const { data: bookmarkResponses, processedFiles } = await loadBookmarksData(); + const { data: bookmarkResponses, processedFiles } = uploadedData + ? { data: uploadedData, processedFiles: [] } + : await loadBookmarksData(); if (bookmarkResponses.length === 0) { logger.info('No Twitter bookmarks data found'); return 0; @@ -394,10 +399,10 @@ async function createRelatedRecords(): Promise { /** * Orchestrates the Twitter data synchronization process */ -async function syncTwitterData(): Promise { +async function syncTwitterData(data?: TwitterBookmarksArray): Promise { try { logger.start('Starting Twitter data synchronization'); - await runIntegration('twitter', syncTwitterBookmarks); + await runIntegration('twitter', (id) => syncTwitterBookmarks(id, data)); logger.complete('Twitter data synchronization completed'); } catch (error) { logger.error('Error syncing Twitter data', error); @@ -405,4 +410,17 @@ async function syncTwitterData(): Promise { } } +export function parseBookmarkDataStrings(strings: string[]): TwitterBookmarksArray { + const combined: TwitterBookmarksArray = []; + for (const content of strings) { + const rawData = JSON.parse(content); + const result = TwitterBookmarksArraySchema.safeParse(rawData); + if (!result.success) { + throw new Error('Invalid Twitter bookmarks data'); + } + combined.push(...result.data); + } + return combined; +} + export { syncTwitterData };