Add the String Web Access component

components/string_web_access/README.md

@@ -0,0 +1,41 @@
+# Overview
+
+The String Web Access API returns any page on the web as clean, LLM-ready Markdown, runs a
+structured web search, and maps a site's URLs.
+
+# Example Use Cases
+
+**Competitor price watch into a warehouse**
+
+- Fetch a product page on a schedule, extract fields with a JSON schema, and append the result to
+  Google Sheets, Postgres or BigQuery. No parser to maintain when the page changes shape.
+
+**Research agent with live sources**
+
+- Run Search Web on an incoming question, fetch the top results as Markdown, and pass them to
+  OpenAI or Claude in the same workflow. The model reads the page text, not raw HTML.
+
+**Inbound lead enrichment**
+
+- When a form submission arrives, fetch the company's site, pull the fields you need with a schema,
+  and write them back to HubSpot or Salesforce before the sales rep opens the record.
+
+# Getting Started
+
+1. Create a free account at [portal.usestring.ai](https://portal.usestring.ai/sign-up). The first
+   5,000 requests are free.
+2. Open [Settings](https://portal.usestring.ai/settings) and copy an API key.
+3. In Pipedream, connect the String Web Access account and paste the key. Pipedream sends it as a
+   bearer token, so the key never appears in a step's code.
+
+# Troubleshooting
+
+**402 Payment Required** — the key is valid but the account balance can't cover the request. Top up
+in the portal.
+
+**A fetch comes back empty or incomplete** — set **Render JavaScript** to true for pages that build
+their content in the browser, or set a **Country Code** for sites that serve different content by
+country. Failed requests are not billed.
+
+**Map Site URLs costs more than expected** — the crawl is quoted before it runs and stops at the
+**Budget (USD)** you set, so lower the budget or the page cap.

components/string_web_access/actions/extract-data/extract-data.mjs

@@ -0,0 +1,57 @@
+import app from "../../string_web_access.app.mjs";
+
+export default {
+  key: "string_web_access-extract-data",
+  name: "Extract Data From URL",
+  description: "Fetch a URL and return only the fields described by a JSON schema. [See the documentation](https://portal.usestring.ai/docs/fetch/structured-extraction)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  ai: "optimized",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+      description: "The URL to extract fields from, e.g. `https://example.com/products/widget`",
+    },
+    jsonSchema: {
+      type: "object",
+      label: "JSON Schema",
+      description: "The shape to extract, as a [JSON Schema](https://portal.usestring.ai/docs/fetch/structured-extraction) object. E.g. `{\"type\":\"object\",\"properties\":{\"price\":{\"type\":\"string\"}}}`",
+    },
+    executeJS: {
+      propDefinition: [
+        app,
+        "executeJS",
+      ],
+    },
+    countryCode: {
+      propDefinition: [
+        app,
+        "countryCode",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.app.fetchUrl({
+      $,
+      data: {
+        url: this.url,
+        format: "json",
+        jsonSchema: this.jsonSchema,
+        executeJS: this.executeJS,
+        countryCode: this.countryCode,
+      },
+    });
+
+    $.export("$summary", `Successfully extracted fields from ${this.url}`);
+    return response;
+  },
+};

components/string_web_access/actions/fetch-url/fetch-url.mjs

@@ -0,0 +1,77 @@
+import app from "../../string_web_access.app.mjs";
+
+export default {
+  key: "string_web_access-fetch-url",
+  name: "Fetch URL",
+  description: "Fetch a URL and return the page as Markdown, raw content, or a JSON envelope. [See the documentation](https://portal.usestring.ai/docs/api-reference/fetch)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  ai: "optimized",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+    },
+    format: {
+      propDefinition: [
+        app,
+        "format",
+      ],
+    },
+    mainContentOnly: {
+      type: "boolean",
+      label: "Main Content Only",
+      description: "Drop navigation, headers and footers, e.g. `true`. Only affects Markdown responses.",
+      optional: true,
+    },
+    executeJS: {
+      propDefinition: [
+        app,
+        "executeJS",
+      ],
+    },
+    countryCode: {
+      propDefinition: [
+        app,
+        "countryCode",
+      ],
+    },
+    solveCaptcha: {
+      propDefinition: [
+        app,
+        "solveCaptcha",
+      ],
+    },
+    headers: {
+      propDefinition: [
+        app,
+        "headers",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.app.fetchUrl({
+      $,
+      data: {
+        url: this.url,
+        format: this.format ?? "markdown",
+        mainContentOnly: this.mainContentOnly,
+        executeJS: this.executeJS,
+        countryCode: this.countryCode,
+        solveCaptcha: this.solveCaptcha,
+        headers: this.headers,
+      },
+    });
+
+    $.export("$summary", `Successfully fetched ${this.url}`);
+    return response;
+  },
+};

components/string_web_access/actions/get-mapped-site-urls/get-mapped-site-urls.mjs

@@ -0,0 +1,60 @@
+import app from "../../string_web_access.app.mjs";
+import { TERMINAL_STATUSES } from "../../common/constants.mjs";
+
+export default {
+  key: "string_web_access-get-mapped-site-urls",
+  name: "Get Mapped Site URLs",
+  description: "Get the status of a crawl started by **Map Site URLs** and, once it has finished, every URL it discovered. Use it when **Map Site URLs** returned a `jobId` before its crawl finished. While the crawl is still running, the job's status and progress are returned instead. If the crawl's approval was interrupted, it approves the job again so the crawl resumes within the budget set in **Map Site URLs**. [See the documentation](https://portal.usestring.ai/docs/api-reference/sitemap)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: false,
+  },
+  type: "action",
+  ai: "optimized",
+  props: {
+    app,
+    jobId: {
+      propDefinition: [
+        app,
+        "jobId",
+      ],
+    },
+  },
+  async run({ $ }) {
+    let state = await this.app.getSitemapStatus({
+      $,
+      jobId: this.jobId,
+    });
+    if (state.status === "partial_state") {
+      state = await this.app.approveSitemap({
+        $,
+        jobId: this.jobId,
+      });
+    }
+    const { status } = state;
+
+    if (status === "failed") {
+      throw new Error(`Crawl ${this.jobId} failed: ${state.errorMessage}`);
+    }
+
+    if (!TERMINAL_STATUSES.includes(status)) {
+      $.export("$summary", `Crawl ${this.jobId} is ${status}; run this action again later to collect its URLs`);
+      return state;
+    }
+
+    const results = await this.app.getAllSitemapUrls({
+      $,
+      jobId: this.jobId,
+    });
+
+    $.export("$summary", `Retrieved ${results.total} URL${results.total === 1
+      ? ""
+      : "s"} from crawl ${this.jobId}`);
+    return {
+      status,
+      ...results,
+    };
+  },
+};

components/string_web_access/actions/map-site-urls/map-site-urls.mjs

@@ -0,0 +1,141 @@
+import app from "../../string_web_access.app.mjs";
+import { TERMINAL_STATUSES } from "../../common/constants.mjs";
+import { sleep } from "../../common/utils.mjs";
+
+const POLLING_INTERVAL = 5000;
+
+export default {
+  key: "string_web_access-map-site-urls",
+  name: "Map Site URLs",
+  description: "Crawl a site within a spend cap and return the URLs it discovered. If the crawl is still running when `maxWaitSeconds` elapses, pass the returned `jobId` to **Get Mapped Site URLs** to collect the URLs later. [See the documentation](https://portal.usestring.ai/docs/api-reference/sitemap)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: false,
+  },
+  type: "action",
+  ai: "optimized",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+      label: "Start URL",
+      description: "Where the crawl starts, e.g. `https://example.com/docs`. It stays on this hostname.",
+    },
+    budgetUsd: {
+      type: "string",
+      label: "Budget (USD)",
+      description: "Hard spend cap for this crawl, e.g. `1.00`. The job stops rather than exceeding it, so a workflow can never approve an open-ended bill.",
+    },
+    maxPages: {
+      type: "integer",
+      label: "Max Pages",
+      description: "Maximum pages to crawl, 1-10000, e.g. `500`",
+      optional: true,
+      default: 100,
+    },
+    maxDepth: {
+      type: "integer",
+      label: "Max Depth",
+      description: "Maximum link depth from the start URL, 1-100, e.g. `3`",
+      optional: true,
+      default: 2,
+    },
+    pathPrefix: {
+      type: "string",
+      label: "Path Prefix",
+      description: "Only crawl URLs whose path starts with this, e.g. `/docs`",
+      optional: true,
+    },
+    useSitemap: {
+      type: "boolean",
+      label: "Seed From sitemap.xml",
+      description: "Also seed from the site's root `sitemap.xml`, e.g. `true`. Costs one extra page and finds pages that internal links miss.",
+      optional: true,
+    },
+    maxWaitSeconds: {
+      type: "integer",
+      label: "Max Wait (seconds)",
+      description: "How long to wait for the crawl, e.g. `300`. If it is still running, the job ID is returned so **Get Mapped Site URLs** can collect the URLs later.",
+      optional: true,
+      default: 300,
+    },
+  },
+  async run({ $ }) {
+    const quote = await this.app.submitSitemap({
+      $,
+      data: {
+        url: this.url,
+        budgetUsd: Number(this.budgetUsd),
+        maxPages: this.maxPages,
+        maxDepth: this.maxDepth,
+        pathPrefix: this.pathPrefix,
+        useSitemap: this.useSitemap,
+      },
+    });
+
+    await this.app.approveSitemap({
+      $,
+      jobId: quote.jobId,
+    });
+
+    const deadline = Date.now() + ((this.maxWaitSeconds ?? 300) * 1000);
+    let state = {
+      status: "running",
+    };
+
+    while (Date.now() < deadline) {
+      await sleep(POLLING_INTERVAL);
+      state = await this.app.getSitemapStatus({
+        $,
+        jobId: quote.jobId,
+      });
+
+      if (state.status === "partial_state") {
+        await this.app.approveSitemap({
+          $,
+          jobId: quote.jobId,
+        });
+        continue;
+      }
+      if (TERMINAL_STATUSES.includes(state.status)) {
+        break;
+      }
+    }
+
+    const { status } = state;
+
+    if (status === "failed") {
+      throw new Error(`Crawl ${quote.jobId} failed: ${state.errorMessage}`);
+    }
+
+    if (!TERMINAL_STATUSES.includes(status)) {
+      $.export("$summary", `Crawl ${quote.jobId} is still running; collect its URLs with Get Mapped Site URLs`);
+      return {
+        ...quote,
+        status,
+        urls: [],
+      };
+    }
+
+    const results = await this.app.getAllSitemapUrls({
+      $,
+      jobId: quote.jobId,
+    });
+
+    $.export("$summary", `Discovered ${results.total} URL${results.total === 1
+      ? ""
+      : "s"} on ${this.url}`);
+    return {
+      jobId: quote.jobId,
+      status,
+      estimatedCostUsd: quote.estimatedCostUsd,
+      estimatedPages: quote.estimatedPages,
+      ...results,
+    };
+  },
+};