Skip to content

Enable local file uploads for browser history and Twitter syncs #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Aias wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Enable local file uploads for browser history and Twitter syncs #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions src/app/routes/integrations/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ import { Alert, AlertDescription } from '@/components/ui/alert';
import { Button } from '@/components/ui/button';
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';

const fileToBase64 = (file: File): Promise<string> => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = reader.result as string;
const base64 = result.split(',')[1] ?? '';
resolve(base64);
};
reader.onerror = () => reject(reader.error);
reader.readAsDataURL(file);
});
};

export const Route = createFileRoute('/integrations/')({
component: RouteComponent,
});
Expand All @@ -19,6 +32,9 @@ interface LogMessage {

function RouteComponent() {
const [messages, setMessages] = useState<LogMessage[]>([]);
const [arcData, setArcData] = useState<string | undefined>(undefined);
const [diaData, setDiaData] = useState<string | undefined>(undefined);
const [twitterFiles, setTwitterFiles] = useState<string[]>([]);

const raindropMutation = trpc.integrations.runRaindrop.useMutation({
onSuccess: (data) => {
Expand Down Expand Up @@ -146,6 +162,48 @@ function RouteComponent() {
githubMutation.mutate();
};

const browsingMutation = trpc.integrations.runBrowsing.useMutation({
onSuccess: (data) => {
setMessages(data.messages);
},
onError: (error) => {
setMessages((prev) => [
...prev,
{
type: 'error',
message: error.message,
timestamp: new Date(),
},
]);
},
});

const handleRunBrowsing = () => {
setMessages([]);
browsingMutation.mutate({ arc: arcData, dia: diaData });
};

const twitterMutation = trpc.integrations.runTwitter.useMutation({
onSuccess: (data) => {
setMessages(data.messages);
},
onError: (error) => {
setMessages((prev) => [
...prev,
{
type: 'error',
message: error.message,
timestamp: new Date(),
},
]);
},
});

const handleRunTwitter = () => {
setMessages([]);
twitterMutation.mutate({ files: twitterFiles });
};

const getMessageIcon = (type: LogMessage['type']) => {
switch (type) {
case 'success':
Expand Down Expand Up @@ -326,6 +384,82 @@ function RouteComponent() {
</Button>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Browser History</CardTitle>
<CardDescription>Sync history from Arc and Dia</CardDescription>
</CardHeader>
<CardContent>
<div className="mb-2 flex flex-col gap-2">
<input
type="file"
onChange={async (e) => {
const f = e.target.files?.[0];
if (f) setArcData(await fileToBase64(f));
}}
/>
<input
type="file"
onChange={async (e) => {
const f = e.target.files?.[0];
if (f) setDiaData(await fileToBase64(f));
}}
/>
</div>
<Button
onClick={handleRunBrowsing}
disabled={browsingMutation.isPending}
className="flex items-center gap-2"
>
{browsingMutation.isPending ? (
<>
<Spinner className="size-4" />
Running...
</>
) : (
<>
<PlayIcon className="size-4" />
Run Sync
</>
)}
</Button>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Twitter</CardTitle>
<CardDescription>Import local Twitter bookmarks</CardDescription>
</CardHeader>
<CardContent>
<input
type="file"
multiple
onChange={async (e) => {
const files = e.target.files ? [...e.target.files] : [];
const bases = await Promise.all(files.map(fileToBase64));
setTwitterFiles(bases);
}}
className="mb-2"
/>
<Button
onClick={handleRunTwitter}
disabled={twitterMutation.isPending}
className="flex items-center gap-2"
>
{twitterMutation.isPending ? (
<>
<Spinner className="size-4" />
Running...
</>
) : (
<>
<PlayIcon className="size-4" />
Run Sync
</>
)}
</Button>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>GitHub</CardTitle>
Expand Down
189 changes: 189 additions & 0 deletions src/server/api/routers/integrations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import { TRPCError } from '@trpc/server';
import { z } from 'zod/v4';
import { syncLightroomImages } from '../../integrations/adobe/sync';
import { syncAirtableData } from '../../integrations/airtable/sync';
import { syncAllBrowserData } from '../../integrations/browser-history/sync-all';
import { syncFeedbin } from '../../integrations/feedbin/sync';
import { syncGitHubData } from '../../integrations/github/sync';
import { syncRaindropData } from '../../integrations/raindrop/sync';
import { syncReadwiseDocuments } from '../../integrations/readwise/sync';
import { parseBookmarkDataStrings, syncTwitterData } from '../../integrations/twitter/sync';
import { createTRPCRouter, publicProcedure } from '../init';

interface LogMessage {
Expand Down Expand Up @@ -555,4 +558,190 @@ export const integrationsRouter = createTRPCRouter({
console.warn = originalConsoleWarn;
}
}),
runBrowsing: publicProcedure
.input(z.object({ arc: z.string().optional(), dia: z.string().optional() }))
.mutation(async ({ input }): Promise<IntegrationResult> => {
const messages: LogMessage[] = [];
let entriesCreated = 0;

const originalConsoleLog = console.log;
const originalConsoleError = console.error;
const originalConsoleWarn = console.warn;

console.log = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

const entryMatch = message.match(/Successfully created (\d+) entries/);
if (entryMatch && entryMatch[1]) {
entriesCreated = parseInt(entryMatch[1], 10);
}

messages.push({
type: 'info',
message,
timestamp: new Date(),
});

originalConsoleLog(...args);
};

console.error = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

messages.push({
type: 'error',
message,
timestamp: new Date(),
});

originalConsoleError(...args);
};

console.warn = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

messages.push({
type: 'warn',
message,
timestamp: new Date(),
});

originalConsoleWarn(...args);
};

try {
const files = {
arc: input.arc ? Buffer.from(input.arc, 'base64') : undefined,
dia: input.dia ? Buffer.from(input.dia, 'base64') : undefined,
};
await syncAllBrowserData(async () => true, files);

messages.push({
type: 'success',
message: `Browser history sync completed successfully${entriesCreated > 0 ? `. Created ${entriesCreated} entries.` : '.'}`,
timestamp: new Date(),
});

return {
success: true,
messages,
entriesCreated,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';

messages.push({
type: 'error',
message: `Sync failed: ${errorMessage}`,
timestamp: new Date(),
});

throw new TRPCError({
code: 'INTERNAL_SERVER_ERROR',
message: `Browser history sync failed: ${errorMessage}`,
});
} finally {
console.log = originalConsoleLog;
console.error = originalConsoleError;
console.warn = originalConsoleWarn;
}
}),
runTwitter: publicProcedure
.input(z.object({ files: z.array(z.string()).optional() }))
.mutation(async ({ input }): Promise<IntegrationResult> => {
const messages: LogMessage[] = [];
let entriesCreated = 0;

const originalConsoleLog = console.log;
const originalConsoleError = console.error;
const originalConsoleWarn = console.warn;

console.log = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

const entryMatch = message.match(/Successfully created (\d+) entries/);
if (entryMatch && entryMatch[1]) {
entriesCreated = parseInt(entryMatch[1], 10);
}

messages.push({
type: 'info',
message,
timestamp: new Date(),
});

originalConsoleLog(...args);
};

console.error = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

messages.push({
type: 'error',
message,
timestamp: new Date(),
});

originalConsoleError(...args);
};

console.warn = (...args: unknown[]) => {
const message = args
.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)))
.join(' ');

messages.push({
type: 'warn',
message,
timestamp: new Date(),
});

originalConsoleWarn(...args);
};

try {
const contents = input.files?.map((f) => Buffer.from(f, 'base64').toString('utf8')) ?? [];
const data = contents.length > 0 ? parseBookmarkDataStrings(contents) : undefined;
await syncTwitterData(data);

messages.push({
type: 'success',
message: `Twitter sync completed successfully${entriesCreated > 0 ? `. Created ${entriesCreated} entries.` : '.'}`,
timestamp: new Date(),
});

return {
success: true,
messages,
entriesCreated,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';

messages.push({
type: 'error',
message: `Sync failed: ${errorMessage}`,
timestamp: new Date(),
});

throw new TRPCError({
code: 'INTERNAL_SERVER_ERROR',
message: `Twitter sync failed: ${errorMessage}`,
});
} finally {
console.log = originalConsoleLog;
console.error = originalConsoleError;
console.warn = originalConsoleWarn;
}
}),
});
18 changes: 15 additions & 3 deletions src/server/db/connections/arc-sqlite.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { copyFileSync } from 'fs';
import { existsSync } from 'fs';
import { homedir } from 'os';
import { copyFileSync, existsSync, mkdtempSync, writeFileSync } from 'fs';
import { homedir, tmpdir } from 'os';
import { join } from 'path';
import { createClient } from '@libsql/client';
import { drizzle } from 'drizzle-orm/libsql';
Expand All @@ -27,3 +26,16 @@ export const createArcConnection = () => {

return drizzle(client, { schema });
};

export const createArcConnectionFromBuffer = (buffer: Buffer) => {
const tempDir = mkdtempSync(join(tmpdir(), 'arc-history-'));
const tempPath = join(tempDir, 'History');
writeFileSync(tempPath, buffer);

const client = createClient({
url: `file:${tempPath}`,
intMode: 'bigint',
});

return drizzle(client, { schema });
};
Loading
Loading