Add the String Web Access component

components/string_web_access/README.md

@@ -0,0 +1,43 @@
+# Overview
+
+The String Web Access API returns any page on the web as clean, LLM-ready Markdown, runs a
+structured web search, and maps a site's URLs. Proxy rotation, anti-bot handling, CAPTCHA solving
+and JavaScript rendering happen server-side, so a workflow step gets the page instead of a block
+screen. Best for pages that rate-limit, geo-gate or refuse automated traffic — the ones where a
+plain HTTP step returns a 403 and your workflow quietly stops.
+
+# Example Use Cases
+
+**Competitor price watch into a warehouse**
+
+- Fetch a product page on a schedule, extract fields with a JSON schema, and append the result to
+  Google Sheets, Postgres or BigQuery. No parser to maintain when the page changes shape.
+
+**Research agent with live sources**
+
+- Run Search Web on an incoming question, fetch the top results as Markdown, and pass them to
+  OpenAI or Claude in the same workflow. The model reads pages, not block screens.
+
+**Inbound lead enrichment**
+
+- When a form submission arrives, fetch the company's site, pull the fields you need with a schema,
+  and write them back to HubSpot or Salesforce before the sales rep opens the record.
+
+# Getting Started
+
+1. Create a free account at [portal.usestring.ai](https://portal.usestring.ai/sign-up). The first
+   5,000 requests are free.
+2. Open [Settings](https://portal.usestring.ai/settings) and copy an API key.
+3. In Pipedream, connect the String Web Access account and paste the key. Pipedream sends it as a
+   bearer token, so the key never appears in a step's code.
+
+# Troubleshooting
+
+**402 Payment Required** — the key is valid but the account balance can't cover the request. Top up
+in the portal.
+
+**A fetch returns a block page** — set **Render JavaScript** to true, or set a **Country Code** to
+route through a proxy in the country the site serves. Failed requests are not billed.
+
+**Map Site URLs costs more than expected** — the crawl is quoted before it runs and stops at the
+**Budget (USD)** you set, so lower the budget or the page cap.

components/string_web_access/actions/extract-data/extract-data.mjs

@@ -0,0 +1,56 @@
+import app from "../../string_web_access.app.mjs";
+
+export default {
+  key: "string_web_access-extract-data",
+  name: "Extract Data From URL",
+  description: "Fetch a URL and return only the fields described by a JSON schema. [See the documentation](https://portal.usestring.ai/docs/fetch/structured-extraction)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+      description: "The URL to extract fields from",
+    },
+    jsonSchema: {
+      type: "object",
+      label: "JSON Schema",
+      description: "The shape to extract, as a [JSON Schema](https://portal.usestring.ai/docs/fetch/structured-extraction) object. E.g. `{\"type\":\"object\",\"properties\":{\"price\":{\"type\":\"string\"}}}`",
+    },
+    executeJS: {
+      propDefinition: [
+        app,
+        "executeJS",
+      ],
+    },
+    countryCode: {
+      propDefinition: [
+        app,
+        "countryCode",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.app.fetchUrl({
+      $,
+      data: {
+        url: this.url,
+        format: "json",
+        jsonSchema: this.jsonSchema,
+        executeJS: this.executeJS,
+        countryCode: this.countryCode,
+      },
+    });
+
+    $.export("$summary", `Successfully extracted fields from ${this.url}`);
+    return response;
+  },
+};

components/string_web_access/actions/fetch-url/fetch-url.mjs

@@ -0,0 +1,76 @@
+import app from "../../string_web_access.app.mjs";
+
+export default {
+  key: "string_web_access-fetch-url",
+  name: "Fetch URL",
+  description: "Fetch a URL and return the page as Markdown, raw content, or a JSON envelope. [See the documentation](https://portal.usestring.ai/docs/api-reference/fetch)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+    },
+    format: {
+      propDefinition: [
+        app,
+        "format",
+      ],
+    },
+    mainContentOnly: {
+      type: "boolean",
+      label: "Main Content Only",
+      description: "Drop navigation, headers and footers. Only affects Markdown responses.",
+      optional: true,
+    },
+    executeJS: {
+      propDefinition: [
+        app,
+        "executeJS",
+      ],
+    },
+    countryCode: {
+      propDefinition: [
+        app,
+        "countryCode",
+      ],
+    },
+    solveCaptcha: {
+      propDefinition: [
+        app,
+        "solveCaptcha",
+      ],
+    },
+    headers: {
+      propDefinition: [
+        app,
+        "headers",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.app.fetchUrl({
+      $,
+      data: {
+        url: this.url,
+        format: this.format ?? "markdown",
+        mainContentOnly: this.mainContentOnly,
+        executeJS: this.executeJS,
+        countryCode: this.countryCode,
+        solveCaptcha: this.solveCaptcha,
+        headers: this.headers,
+      },
+    });
+
+    $.export("$summary", `Successfully fetched ${this.url}`);
+    return response;
+  },
+};

components/string_web_access/actions/map-site-urls/map-site-urls.mjs

@@ -0,0 +1,146 @@
+import app from "../../string_web_access.app.mjs";
+
+const POLLING_INTERVAL = 5000;
+const TERMINAL_STATUSES = [
+  "completed",
+  "failed",
+  "canceled",
+  "token_cap_exceeded",
+];
+
+export default {
+  key: "string_web_access-map-site-urls",
+  name: "Map Site URLs",
+  description: "Crawl a site within a spend cap and return the URLs it discovered. [See the documentation](https://portal.usestring.ai/docs/api-reference/sitemap)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    app,
+    url: {
+      propDefinition: [
+        app,
+        "url",
+      ],
+      label: "Start URL",
+      description: "Where the crawl starts. It stays on this hostname.",
+    },
+    budgetUsd: {
+      type: "string",
+      label: "Budget (USD)",
+      description: "Hard spend cap for this crawl, e.g. `1.00`. The job stops rather than exceeding it, so a workflow can never approve an open-ended bill.",
+    },
+    maxPages: {
+      type: "integer",
+      label: "Max Pages",
+      description: "Maximum pages to crawl, 1-10000",
+      optional: true,
+      default: 100,
+    },
+    maxDepth: {
+      type: "integer",
+      label: "Max Depth",
+      description: "Maximum link depth from the start URL, 1-100",
+      optional: true,
+      default: 2,
+    },
+    pathPrefix: {
+      type: "string",
+      label: "Path Prefix",
+      description: "Only crawl URLs whose path starts with this, e.g. `/docs`",
+      optional: true,
+    },
+    useSitemap: {
+      type: "boolean",
+      label: "Seed From sitemap.xml",
+      description: "Also seed from the site's root `sitemap.xml`. Costs one extra page and finds pages that internal links miss.",
+      optional: true,
+    },
+    maxWaitSeconds: {
+      type: "integer",
+      label: "Max Wait (seconds)",
+      description: "How long to wait for the crawl. If it is still running, the job ID is returned so a later step can collect the URLs.",
+      optional: true,
+      default: 300,
+    },
+  },
+  methods: {
+    sleep(ms) {
+      return new Promise((resolve) => setTimeout(resolve, ms));
+    },
+  },
+  async run({ $ }) {
+    const quote = await this.app.submitSitemap({
+      $,
+      data: {
+        url: this.url,
+        budgetUsd: Number(this.budgetUsd),
+        maxPages: this.maxPages,
+        maxDepth: this.maxDepth,
+        pathPrefix: this.pathPrefix,
+        useSitemap: this.useSitemap,
+      },
+    });
+
+    await this.app.approveSitemap({
+      $,
+      jobId: quote.jobId,
+    });
+
+    const deadline = Date.now() + ((this.maxWaitSeconds ?? 300) * 1000);
+    let status = "running";
+
+    while (Date.now() < deadline) {
+      await this.sleep(POLLING_INTERVAL);
+      const state = await this.app.getSitemapStatus({
+        $,
+        jobId: quote.jobId,
+      });
+      status = state.status;
+
+      if (status === "partial_state") {
+        await this.app.approveSitemap({
+          $,
+          jobId: quote.jobId,
+        });
+        continue;
+      }
+      if (TERMINAL_STATUSES.includes(status)) {
+        break;
+      }
+    }
+
+    if (!TERMINAL_STATUSES.includes(status)) {
+      $.export("$summary", `Crawl ${quote.jobId} is still running; collect its URLs with the job ID`);
+      return {
+        ...quote,
+        status,
+        urls: [],
+      };
+    }
+
+    const results = await this.app.getSitemapUrls({
+      $,
+      jobId: quote.jobId,
+      params: {
+        limit: 1000,
+        offset: 0,
+      },
+    });
+
+    $.export("$summary", `Discovered ${results.total} URL${results.total === 1
+      ? ""
+      : "s"} on ${this.url}`);
+    return {
+      jobId: quote.jobId,
+      status,
+      estimatedCostUsd: quote.estimatedCostUsd,
+      estimatedPages: quote.estimatedPages,
+      ...results,
+    };
+  },
+};

components/string_web_access/actions/search-web/search-web.mjs

@@ -0,0 +1,64 @@
+import app from "../../string_web_access.app.mjs";
+
+export default {
+  key: "string_web_access-search-web",
+  name: "Search Web",
+  description: "Search the web and return the organic results. [See the documentation](https://portal.usestring.ai/docs/api-reference/search)",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    app,
+    query: {
+      type: "string",
+      label: "Query",
+      description: "What to search for",
+    },
+    engine: {
+      type: "string",
+      label: "Engine",
+      description: "Which search engine to query",
+      options: [
+        "google",
+        "duckduckgo",
+        "brave",
+        "mojeek",
+      ],
+      optional: true,
+      default: "google",
+    },
+    country: {
+      type: "string",
+      label: "Country",
+      description: "ISO 3166-1 alpha-2 country code used to localize results, e.g. `US` or `GB`",
+      optional: true,
+    },
+    language: {
+      type: "string",
+      label: "Language",
+      description: "Language tag for the results, e.g. `en` or `pt-br`",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.app.search({
+      $,
+      data: {
+        query: this.query,
+        engine: this.engine,
+        country: this.country,
+        language: this.language,
+      },
+    });
+
+    const count = response?.results?.length ?? 0;
+    $.export("$summary", `Found ${count} result${count === 1
+      ? ""
+      : "s"} for "${this.query}"`);
+    return response;
+  },
+};