This repository was archived by the owner on Jun 4, 2025. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathindex.ts
More file actions
147 lines (127 loc) · 4.1 KB
/
Copy pathindex.ts
File metadata and controls
147 lines (127 loc) · 4.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import { config } from "dotenv";
import Hyperbrowser from "@hyperbrowser/sdk";
import OpenAI from "openai";
import { zodResponseFormat } from "openai/helpers/zod";
import { z } from "zod";
import ora from "ora";
config();
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
const HYPERBROWSER_API_KEY = process.env.HYPERBROWSER_API_KEY || "";
const client = new Hyperbrowser({
apiKey: HYPERBROWSER_API_KEY,
});
const spinner = ora();
const Product = z.object({
name: z.string(),
price: z.number(),
description: z.string(),
image: z.string(),
});
const ProductSchema = z.object({ products: z.array(Product) });
const SYSTEM_PROMPT = `You are an expert data extractor. Your task is to extract product information from the provided scraped content.
Ensure the output adheres to the following structure:
- Name: Product name
- Description: A brief description of the product
- Price: The price in the provided currency (if available)
- Availability: Whether the product is in stock or out of stock (if available)
- Additional Details: Any other relevant information (e.g., SKU, category)
Provide the extracted data as a JSON object. Parse the Markdown content carefully to identify and categorize the product details accurately.`;
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const scrapeShopifySite = async (url: string) => {
spinner.start("Crawling Shopify site...");
spinner.start();
const crawlJob = await client.startCrawlJob({
url: url,
maxPages: 1000,
});
console.log("Crawl job started with jobId:", crawlJob.jobId);
let completed = false;
while (!completed) {
const job = await client.getCrawlJob(crawlJob.jobId);
if (job.status === "completed") {
completed = true;
spinner.succeed(`Crawl job completed with jobId: ${crawlJob.jobId}`);
} else if (job.status === "failed") {
spinner.fail(
`Crawl job failed with jobId: ${crawlJob.jobId}\nError: ${job.error}`
);
completed = true;
} else {
spinner.info(`Crawl job is still running: ${job.status}`);
await sleep(5_000);
}
}
};
const extractProductData = async (jobId: string) => {
let scrapedMarkdown = "";
let pageIndex = 1;
spinner.start();
let error = false;
while (true) {
spinner.text = `Getting markdown data from page batch ${pageIndex}`;
try {
const crawlJobResult = await client.getCrawlJob(jobId, {
page: pageIndex,
});
const pages = crawlJobResult.data;
if (pages) {
for (const page of pages) {
const pageData = page.markdown;
if (pageData) {
scrapedMarkdown += pageData;
}
}
if (pageIndex >= crawlJobResult.totalPageBatches) {
break;
}
console.log(crawlJobResult);
pageIndex++;
}
} catch (err) {
spinner.fail(`Error getting crawl job result: ${err}`);
error = true;
break;
}
}
if (error) {
return;
}
spinner.text = "Extracting product data...";
const completion = await openai.beta.chat.completions.parse({
model: "gpt-4o-mini",
messages: [
{
role: "system",
content: SYSTEM_PROMPT,
},
{ role: "user", content: scrapedMarkdown },
],
response_format: zodResponseFormat(ProductSchema, "product"),
});
const productInfo = completion.choices[0].message.parsed;
spinner.succeed("Product data extracted successfully");
console.log(JSON.stringify(productInfo?.products, null, 2));
};
const args = process.argv.slice(2);
const command = args[0];
const param = args[1];
if (!command) {
console.error("Please provide a command: scrape <url> or extract <jobId>");
process.exit(1);
}
if (command === "scrape") {
if (!param) {
console.error("Please provide a URL to scrape");
process.exit(1);
}
scrapeShopifySite(param);
} else if (command === "extract") {
if (!param) {
console.error("Please provide a job ID to extract data from");
process.exit(1);
}
extractProductData(param);
} else {
console.error("Invalid command. Use 'scrape <url>' or 'extract <jobId>'");
process.exit(1);
}