From 200cc27c786938970676744555f45de1ab92aaf5 Mon Sep 17 00:00:00 2001 From: Syd Shields <86916890+syd-shields@users.noreply.github.com> Date: Wed, 13 May 2026 11:40:05 -0400 Subject: [PATCH 1/2] [DASH-1732] [templates] add build step to create javascript templates (#75) * init * add a build step that turns new ts templates into js in CI * add file filter to build script * add built js templates * add exclude flag * add comments to explain build filters * add build step to run on dev branch in CI * regen templats on each pr merge * include all readme, package json and env examples in build * get paths to build js files first so that the entire repo isn't deleted and rewritten with new build * remove built js files * only build js files to production branch * remove build javascript workflow * before recursing, skip directories whose basename is in DEFAULT_EXCLUDED_DIRS * before recursing, skip directories whose basename is in DEFAULT_EXCLUDED_DIRS * if there are zero playground compatible templates, do not call build-javascript.mjs * add test-production branch support * build js templates * remove built js files * add pkg.type = module to adaptPackageJsonForJavaScript so it's unconditionally set for all generated JS templates * consolidate workflows to one plauground.yml file to remove need to make updates to it twice * write contents to test-production and production branches * use hardcoded branch names instead of unused var * add --no-verify flag to build * add stripLeadingTscBuildCommand to handle when multiple commands exist alongside tsc * add parseBuildOptions to parse CLI flags at once instead of in multiple places to prevent drift --- .github/CODEOWNERS | 10 + .github/workflows/playground-production.yml | 20 + .../workflows/playground-test-production.yml | 20 + .github/workflows/playground.yml | 53 +++ README.md | 6 + package.json | 4 + pnpm-lock.yaml | 3 + scripts/build-javascript.mjs | 348 ++++++++++++++++++ scripts/fetch-playground-typescript-dirs.mjs | 111 ++++++ scripts/lib/playground-checks.mjs | 13 + scripts/playground-ci.mjs | 51 +++ scripts/validate-playground-templates.mjs | 76 ++++ 12 files changed, 715 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/playground-production.yml create mode 100644 .github/workflows/playground-test-production.yml create mode 100644 .github/workflows/playground.yml create mode 100644 scripts/build-javascript.mjs create mode 100644 scripts/fetch-playground-typescript-dirs.mjs create mode 100644 scripts/lib/playground-checks.mjs create mode 100644 scripts/playground-ci.mjs create mode 100644 scripts/validate-playground-templates.mjs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..4c28f42f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# When merging changes under these paths to `production`, require review from the +# Dashboard team. +/javascript/ @browserbase/dashboard +/scripts/fetch-playground-typescript-dirs.mjs @browserbase/dashboard +/scripts/validate-playground-templates.mjs @browserbase/dashboard +/scripts/playground-ci.mjs @browserbase/dashboard +/scripts/lib/playground-checks.mjs @browserbase/dashboard +/.github/workflows/playground.yml @browserbase/dashboard +/.github/workflows/playground-production.yml @browserbase/dashboard +/.github/workflows/playground-test-production.yml @browserbase/dashboard diff --git a/.github/workflows/playground-production.yml b/.github/workflows/playground-production.yml new file mode 100644 index 00000000..b6b77570 --- /dev/null +++ b/.github/workflows/playground-production.yml @@ -0,0 +1,20 @@ +# Triggers the shared playground workflow for the `production` branch. +# All CI steps live in playground.yml — edit there to change behavior. + +name: Playground templates (production) + +on: + pull_request: + branches: + - production + push: + branches: + - production + workflow_dispatch: + +permissions: + contents: write + +jobs: + playground-templates: + uses: ./.github/workflows/playground.yml diff --git a/.github/workflows/playground-test-production.yml b/.github/workflows/playground-test-production.yml new file mode 100644 index 00000000..be4fff5d --- /dev/null +++ b/.github/workflows/playground-test-production.yml @@ -0,0 +1,20 @@ +# Triggers the shared playground workflow for the `test-production` branch. +# All CI steps live in playground.yml — edit there to change behavior. + +name: Playground templates (test production) + +on: + pull_request: + branches: + - test-production + push: + branches: + - test-production + workflow_dispatch: + +permissions: + contents: write + +jobs: + playground-templates: + uses: ./.github/workflows/playground.yml diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml new file mode 100644 index 00000000..3ab2e1c3 --- /dev/null +++ b/.github/workflows/playground.yml @@ -0,0 +1,53 @@ +# Reusable workflow: validates TypeScript → JavaScript builds for playground- +# runnable templates and optionally commits the generated javascript/ tree. +# +# Called by playground-production.yml and playground-test-production.yml so that +# CI steps are defined in exactly one place. + +name: Playground templates + +on: + workflow_call: + +permissions: + contents: write + +jobs: + playground-templates: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clean javascript output directory + run: rm -rf javascript && mkdir -p javascript + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build and validate playground TypeScript templates + env: + TEMPLATES_API_URL: ${{ vars.TEMPLATES_API_URL }} + run: pnpm run ci:playground + + - name: Commit and push playground javascript (push only) + if: github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A javascript/ + if git diff --staged --quiet; then + echo "No javascript changes to commit." + else + git commit --no-verify -m "chore: regenerate playground javascript templates" + git push + fi diff --git a/README.md b/README.md index 51ff915f..35457c74 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,12 @@ Templates use the Model Gateway to route LLM requests -- you only need your `BRO Each template's README contains detailed installation steps, environment variable requirements, and troubleshooting guides. +### TypeScript and generated JavaScript + +- **Source of truth:** edit templates under `typescript/`. The `javascript/` tree is generated output, not authored by hand for day-to-day changes. +- **Local only:** run `pnpm run build:javascript` when you want a full mirror of `typescript/` into `javascript/` on your machine (for example to smoke-test the transpiler or compare JS output). There is **no** GitHub Actions workflow that builds the full tree into the repo anymore. +- **Playground releases:** the `production` branch is updated by CI (`.github/workflows/playground-production.yml`), which builds and commits **only** templates that are playground-runnable per the public templates API, then validates them. + ## Resources ### Documentation diff --git a/package.json b/package.json index 2be8b400..5b0cc55b 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,9 @@ "private": true, "type": "module", "scripts": { + "build:javascript": "node scripts/build-javascript.mjs", + "ci:playground": "node scripts/playground-ci.mjs", + "validate:playground": "node scripts/validate-playground-templates.mjs", "check:readme-template-index": "node scripts/check-readme-template-index.mjs", "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md,yml,yaml}\"", "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md,yml,yaml}\"", @@ -25,6 +28,7 @@ "husky": "^9.1.7", "lint-staged": "^16.2.7", "prettier": "^3.2.5", + "typescript": "^5.9.3", "typescript-eslint": "^8.50.1" }, "packageManager": "pnpm@9.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c2e1df4d..1a2db3b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: prettier: specifier: ^3.2.5 version: 3.7.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 typescript-eslint: specifier: ^8.50.1 version: 8.50.1(eslint@9.39.2)(typescript@5.9.3) diff --git a/scripts/build-javascript.mjs b/scripts/build-javascript.mjs new file mode 100644 index 00000000..67be0e82 --- /dev/null +++ b/scripts/build-javascript.mjs @@ -0,0 +1,348 @@ +/** + * Transpiles `typescript/` → `javascript/` for local development and tooling + * (e.g. `pnpm run build:javascript`). It is not run by CI on `main`/`dev`. + * Playground-facing JS on the `production` branch is produced by + * `scripts/playground-ci.mjs` via `.github/workflows/playground-production.yml`. + */ +import { copyFile, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import console from "node:console"; +import path from "node:path"; +import process from "node:process"; +import ts from "typescript"; + +const ROOT_DIR = process.cwd(); +const TYPESCRIPT_DIR = path.join(ROOT_DIR, "typescript"); +const JAVASCRIPT_DIR = path.join(ROOT_DIR, "javascript"); + +const DEFAULT_EXCLUDED_DIRS = new Set(["node_modules", ".next", "dist", "build", "coverage"]); + +const TYPESCRIPT_ONLY_DEV_DEPENDENCIES = new Set(["typescript", "tsx"]); + +/** Builds JavaScript from TypeScript (local development; see file header). */ +async function buildJavaScriptTemplates() { + // Get the command line arguments. + const argv = process.argv.slice(2); + const options = parseBuildOptions(argv); + // Filter function that determines which files to include in the build. + const fileFilter = createFileFilter(options); + const allFiles = await getAllFilteredFiles(TYPESCRIPT_DIR, fileFilter); + + // Avoid wiping unrelated templates when filters narrow the build (e.g. --include-template=foo). + if (options.includeTemplates.size > 0) { + const templatesToRefresh = new Set(); + for (const filePath of allFiles) { + const relativePath = path.relative(TYPESCRIPT_DIR, filePath); + const normalizedPath = normalizeRelativePath(relativePath); + const [templateName] = normalizedPath.split("/"); + if (templateName) templatesToRefresh.add(templateName); + } + await mkdir(JAVASCRIPT_DIR, { recursive: true }); + await Promise.all( + [...templatesToRefresh].map((name) => + rm(path.join(JAVASCRIPT_DIR, name), { recursive: true, force: true }), + ), + ); + } else if (options.includePaths.length > 0 || options.excludePathOnly.length > 0) { + await mkdir(JAVASCRIPT_DIR, { recursive: true }); + // Path-level filters can match a subset of files per template; only overwrites run. + } else { + await rm(JAVASCRIPT_DIR, { recursive: true, force: true }); + await mkdir(JAVASCRIPT_DIR, { recursive: true }); + } + const tsFiles = allFiles.filter(isTranspilableTypeScriptSource); + const assetFiles = allFiles.filter((filePath) => !isTranspilableTypeScriptSource(filePath)); + + await Promise.all(tsFiles.map((file) => transpileFile(file))); + + await Promise.all( + assetFiles.map(async (filePath) => { + if (path.basename(filePath) === "package.json") { + await writeAdaptedPackageJson(filePath); + } else { + await copyAssetFile(filePath); + } + }), + ); + + if (argv.length > 0) { + console.log(`Filters: ${argv.join(" ")}`); + } + console.log( + `Built ${tsFiles.length} TypeScript files and ${assetFiles.length} other files into javascript/`, + ); +} + +/** + * @typedef {object} BuildOptions + * @property {Set} includeTemplates + * @property {Set} excludeTemplates + * @property {string[]} includePaths + * @property {string[]} excludePaths + * @property {string[]} excludePathOnly + */ + +/** + * Parses build flags once so filtering and cleanup decisions stay in sync. + * Flags: + * --include-template= Include only matching top-level template folders. + * --exclude-template= Exclude matching top-level template folders. + * --include-path= Include files whose relative path contains any token. + * --exclude-path= Exclude files whose relative path contains any token. + * --exclude= Convenience alias: applies to both template-name and path excludes. + * Each flag supports both "--flag value" and "--flag=value" forms. + * @param {string[]} argv + * @returns {BuildOptions} + */ +function parseBuildOptions(argv) { + const genericExcludes = parseListFlag(argv, "--exclude"); + const excludePathOnly = parseListFlag(argv, "--exclude-path"); + + return { + includeTemplates: new Set(parseListFlag(argv, "--include-template")), + excludeTemplates: new Set([...parseListFlag(argv, "--exclude-template"), ...genericExcludes]), + includePaths: parseListFlag(argv, "--include-path"), + excludePaths: [...excludePathOnly, ...genericExcludes], + excludePathOnly, + }; +} + +/** + * Creates a file filter function from parsed command line options. + * @param {BuildOptions} options + * @returns {function(string): boolean} + */ +function createFileFilter(options) { + return (sourcePath) => { + const relativePath = path.relative(TYPESCRIPT_DIR, sourcePath); + const normalizedPath = normalizeRelativePath(relativePath); + const [templateName] = normalizedPath.split("/"); + + if (!templateName) return false; + if (normalizedPath.endsWith(".d.ts")) return false; + + const pathSegments = normalizedPath.split("/"); + if (pathSegments.some((segment) => DEFAULT_EXCLUDED_DIRS.has(segment))) return false; + + if (options.includeTemplates.size > 0 && !options.includeTemplates.has(templateName)) { + return false; + } + if (options.excludeTemplates.has(templateName)) return false; + + if ( + options.includePaths.length > 0 && + !options.includePaths.some((token) => normalizedPath.includes(token)) + ) { + return false; + } + + if (options.excludePaths.some((token) => normalizedPath.includes(token))) return false; + + return true; + }; +} + +function isTranspilableTypeScriptSource(filePath) { + const name = path.basename(filePath); + if (name.endsWith(".tsx")) return true; + if (name.endsWith(".ts") && !name.endsWith(".d.ts")) return true; + return false; +} + +/** Rewrite npm script strings from TypeScript runner / paths to plain Node. */ +function adaptScriptCommand(command) { + let value = command + .replaceAll(/\bnpx\s+tsx\s+watch\s+/g, "node --watch ") + .replaceAll(/\bnpx\s+tsx\s+/g, "node ") + .replaceAll(/\btsx\s+watch\s+/g, "node --watch ") + .replaceAll(/\btsx\s+/g, "node "); + return value + .replaceAll(/\.d\.ts\b/g, "__PRESERVE_D_TS__") + .replaceAll(/\.tsx\b/g, ".jsx") + .replaceAll(/\.ts\b/g, ".js") + .replaceAll(/__PRESERVE_D_TS__/g, ".d.ts"); +} + +function isTypesPackage(depName) { + return depName.startsWith("@types/"); +} + +/** + * `build` is rewritten for JS output: drop a compile-only `tsc` step. If the script + * is only `tsc` (plus flags), remove it entirely; if `tsc` is chained with `&&` / + * `||` / `;`, keep everything after the first separator so steps like `next build` + * are not lost. + * + * @param {string} buildScript + * @returns {string | null} `null` means delete the `build` script entry. + */ +function stripLeadingTscBuildCommand(buildScript) { + const trimmed = buildScript.trim(); + // Match `tsc` as a CLI token (not `tscheck` / `tsc.exe`). Allow `tsc&&…` without spaces. + if (!/^\s*tsc(?=[\s;&]|$)/u.test(trimmed)) { + return buildScript; + } + const chain = /\s*(?:&&|\|\||;)\s*/u.exec(trimmed); + if (!chain) { + return null; + } + const rest = trimmed.slice(chain.index + chain[0].length).trim(); + return rest.length > 0 ? rest : null; +} + +function adaptPackageJsonForJavaScript(packageJson) { + const pkg = JSON.parse(JSON.stringify(packageJson)); + + pkg.type = "module"; + + if (typeof pkg.main === "string") { + pkg.main = pkg.main.replace(/\.tsx$/u, ".jsx").replace(/\.ts$/u, ".js"); + } + + if (pkg.scripts !== null && typeof pkg.scripts === "object" && !Array.isArray(pkg.scripts)) { + const scripts = { ...pkg.scripts }; + for (const key of Object.keys(scripts)) { + const scriptValue = scripts[key]; + if (typeof scriptValue !== "string") continue; + scripts[key] = adaptScriptCommand(scriptValue); + } + if (typeof scripts.build === "string") { + const nextBuild = stripLeadingTscBuildCommand(scripts.build); + if (nextBuild === null) { + delete scripts.build; + } else if (nextBuild !== scripts.build) { + scripts.build = nextBuild; + } + } + pkg.scripts = scripts; + } + + if ( + pkg.devDependencies !== null && + typeof pkg.devDependencies === "object" && + !Array.isArray(pkg.devDependencies) + ) { + const devDeps = { ...pkg.devDependencies }; + for (const name of Object.keys(devDeps)) { + if (TYPESCRIPT_ONLY_DEV_DEPENDENCIES.has(name) || isTypesPackage(name)) { + delete devDeps[name]; + } + } + if (Object.keys(devDeps).length === 0) { + delete pkg.devDependencies; + } else { + pkg.devDependencies = devDeps; + } + } + + return pkg; +} + +function normalizeRelativePath(filePath) { + return filePath.split(path.sep).join("/"); +} + +function parseListFlag(argv, flagName) { + const values = []; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg !== flagName && !arg.startsWith(`${flagName}=`)) continue; + + const inlineValue = arg.startsWith(`${flagName}=`) ? arg.slice(flagName.length + 1) : null; + const nextValue = argv[index + 1]; + const value = inlineValue ?? nextValue; + if (!value || value.startsWith("--")) { + throw new Error( + `Missing value for ${flagName} (use "${flagName} value" or "${flagName}=value")`, + ); + } + + values.push( + ...value + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ); + if (inlineValue === null) { + index += 1; + } + } + + return values; +} + +async function getAllFilteredFiles(dir, fileFilter) { + const entries = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (DEFAULT_EXCLUDED_DIRS.has(entry.name)) { + return []; + } + return getAllFilteredFiles(fullPath, fileFilter); + } + + if (entry.isFile() && fileFilter(fullPath)) { + return [fullPath]; + } + + return []; + }), + ); + + return files.flat(); +} + +async function copyAssetFile(sourcePath) { + const relativePath = path.relative(TYPESCRIPT_DIR, sourcePath); + const outputPath = path.join(JAVASCRIPT_DIR, relativePath); + await mkdir(path.dirname(outputPath), { recursive: true }); + await copyFile(sourcePath, outputPath); +} + +async function writeAdaptedPackageJson(sourcePath) { + const relativePath = path.relative(TYPESCRIPT_DIR, sourcePath); + const outputPath = path.join(JAVASCRIPT_DIR, relativePath); + const raw = await readFile(sourcePath, "utf8"); + const pkg = JSON.parse(raw); + const adapted = adaptPackageJsonForJavaScript(pkg); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(adapted, null, 2)}\n`, "utf8"); +} + +async function transpileFile(sourcePath) { + const relativePath = path.relative(TYPESCRIPT_DIR, sourcePath); + const outputPath = path + .join(JAVASCRIPT_DIR, relativePath) + .replace(/\.tsx?$/, (ext) => (ext === ".tsx" ? ".jsx" : ".js")); + + const source = await readFile(sourcePath, "utf8"); + const transpiled = ts.transpileModule(source, { + compilerOptions: { + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + jsx: ts.JsxEmit.Preserve, + }, + reportDiagnostics: true, + fileName: sourcePath, + }); + + if (transpiled.diagnostics?.length) { + const message = ts.formatDiagnosticsWithColorAndContext(transpiled.diagnostics, { + getCurrentDirectory: () => ROOT_DIR, + getCanonicalFileName: (fileName) => fileName, + getNewLine: () => "\n", + }); + + throw new Error(`TypeScript transpile diagnostics in ${relativePath}\n${message}`); + } + + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, transpiled.outputText, "utf8"); +} + +buildJavaScriptTemplates().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/fetch-playground-typescript-dirs.mjs b/scripts/fetch-playground-typescript-dirs.mjs new file mode 100644 index 00000000..b3f0dfce --- /dev/null +++ b/scripts/fetch-playground-typescript-dirs.mjs @@ -0,0 +1,111 @@ +import { access, constants, stat } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; + +const ROOT = process.cwd(); +const TYPESCRIPT_ROOT = path.join(ROOT, "typescript"); + +/** + * create-browser-app --template name → folder under typescript/ when names differ. + * Keys are the `--template` flag value from website `commands`, not marketing slug. + */ +const TEMPLATE_FLAG_TO_TYPESCRIPT_DIR = new Map([ + ["google-trends-keywords", "google-trends"], + ["amazon-price-comparison", "amazon-global-price-comparison"], + ["real-estate-license-verification", "license-verification"], +]); + +/** + * @param {unknown} commands + * @returns {string | null} + */ +export function extractCreateBrowserAppTemplateName(commands) { + if (!Array.isArray(commands)) return null; + for (const cmd of commands) { + if (typeof cmd !== "string") continue; + if (!/\bcreate-browser-app\b/.test(cmd)) continue; + const match = /--template(?:=|\s+)(\S+)/.exec(cmd); + if (match) return match[1]; + } + return null; +} + +/** + * @param {string} dirName + */ +async function assertTypescriptTemplateDir(dirName) { + const abs = path.join(TYPESCRIPT_ROOT, dirName); + const dirStat = await stat(abs); + if (!dirStat.isDirectory()) { + throw new Error(`Not a directory: typescript/${dirName}`); + } + const tsEntry = path.join(abs, "index.ts"); + const tsxEntry = path.join(abs, "index.tsx"); + try { + await access(tsEntry, constants.F_OK); + return; + } catch { + await access(tsxEntry, constants.F_OK); + } +} + +/** + * Fetches the public templates list (already filtered to playgroundRunnable on the server). + * + * @param {string} [apiUrl] + * @returns {Promise<{ slug: string; templateFlag: string; typescriptDir: string }[]>} + */ +function resolveTemplatesApiUrl(explicit) { + if (explicit && String(explicit).trim()) return String(explicit).trim(); + const fromEnv = process.env.TEMPLATES_API_URL; + if (typeof fromEnv === "string" && fromEnv.trim()) return fromEnv.trim(); + return "https://www.browserbase.com/website-api/templates"; +} + +export async function fetchPlaygroundTypescriptTemplateEntries(apiUrl) { + const url = resolveTemplatesApiUrl(apiUrl); + + const response = await fetch(url, { + headers: { accept: "application/json" }, + }); + + if (!response.ok) { + throw new Error(`Templates API ${response.status} ${response.statusText} (${url})`); + } + + const body = await response.json(); + const templates = body?.templates; + if (!Array.isArray(templates)) { + throw new Error("Templates API response missing templates[]"); + } + + /** @type {{ slug: string; templateFlag: string; typescriptDir: string }[]} */ + const entries = []; + + for (const template of templates) { + const slug = template?.slug; + const flag = extractCreateBrowserAppTemplateName(template?.commands); + if (typeof slug !== "string" || !flag) { + throw new Error( + `Template missing slug or npx create-browser-app --template in commands: ${JSON.stringify(template?.slug)}`, + ); + } + + const typescriptDir = TEMPLATE_FLAG_TO_TYPESCRIPT_DIR.get(flag) ?? flag; + await assertTypescriptTemplateDir(typescriptDir); + entries.push({ slug, templateFlag: flag, typescriptDir }); + } + + return entries; +} + +/** + * Unique typescript folder names for build filters. + * + * @param {string} [apiUrl] + * @returns {Promise} + */ +export async function fetchPlaygroundTypescriptDirNames(apiUrl) { + const entries = await fetchPlaygroundTypescriptTemplateEntries(apiUrl); + return [...new Set(entries.map((e) => e.typescriptDir))]; +} diff --git a/scripts/lib/playground-checks.mjs b/scripts/lib/playground-checks.mjs new file mode 100644 index 00000000..8819cf26 --- /dev/null +++ b/scripts/lib/playground-checks.mjs @@ -0,0 +1,13 @@ +/** + * Mirrors dashboard `hasStagehandUsage` from core `src/utils/playground-stagehand.ts`. + */ + +/** + * @param {string} code + * @returns {boolean} + */ +export function hasStagehandUsage(code) { + const stagehandVariablePattern = /(?:let|const|var)\s+\w+\s*=\s*new\s+Stagehand\s*\(/; + const stagehandDirectPattern = /(?:^|\s|await\s+)(?:new\s+)?Stagehand\s*\(/; + return stagehandVariablePattern.test(code) || stagehandDirectPattern.test(code); +} diff --git a/scripts/playground-ci.mjs b/scripts/playground-ci.mjs new file mode 100644 index 00000000..292268c3 --- /dev/null +++ b/scripts/playground-ci.mjs @@ -0,0 +1,51 @@ +import { spawnSync } from "node:child_process"; +import console from "node:console"; +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; + +import { fetchPlaygroundTypescriptDirNames } from "./fetch-playground-typescript-dirs.mjs"; + +const JAVASCRIPT_DIR = path.join(process.cwd(), "javascript"); + +async function main() { + const dirs = await fetchPlaygroundTypescriptDirNames(); + dirs.sort(); + + // With zero dirs, do not invoke build-javascript with no filters: that path wipes + // javascript/ and rebuilds every template. Playground CI must produce an empty tree + // (or only future API-listed templates), not a full mirror of typescript/. + if (dirs.length === 0) { + process.stdout.write( + "No playground-runnable templates from API; skipping transpile and clearing javascript/.\n", + ); + await rm(JAVASCRIPT_DIR, { recursive: true, force: true }); + await mkdir(JAVASCRIPT_DIR, { recursive: true }); + } else { + const argv = ["scripts/build-javascript.mjs"]; + for (const name of dirs) { + argv.push("--include-template", name); + } + + process.stdout.write(`Building ${dirs.length} playground TypeScript templates…\n`); + const build = spawnSync(process.execPath, argv, { stdio: "inherit", encoding: "utf8" }); + if (build.status !== 0) { + process.exit(build.status ?? 1); + } + } + + const validate = spawnSync(process.execPath, ["scripts/validate-playground-templates.mjs"], { + stdio: "inherit", + encoding: "utf8", + }); + if (validate.status !== 0) { + process.exit(validate.status ?? 1); + } + + process.stdout.write("Playground CI: build + validation passed.\n"); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/validate-playground-templates.mjs b/scripts/validate-playground-templates.mjs new file mode 100644 index 00000000..19a1e67f --- /dev/null +++ b/scripts/validate-playground-templates.mjs @@ -0,0 +1,76 @@ +import console from "node:console"; +import { access, constants, readFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import ts from "typescript"; + +import { hasStagehandUsage } from "./lib/playground-checks.mjs"; +import { fetchPlaygroundTypescriptTemplateEntries } from "./fetch-playground-typescript-dirs.mjs"; + +const ROOT = process.cwd(); +const TYPESCRIPT_ROOT = path.join(ROOT, "typescript"); + +/** + * @param {string} filePath + */ +async function validateSourceFile(filePath) { + const sourceText = await readFile(filePath, "utf8"); + + const sf = ts.createSourceFile( + filePath, + sourceText, + ts.ScriptTarget.Latest, + true, + filePath.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ); + + const parseDiagnostics = sf.parseDiagnostics ?? []; + if (parseDiagnostics.length > 0) { + const message = ts.formatDiagnosticsWithColorAndContext(parseDiagnostics, { + getCurrentDirectory: () => ROOT, + getCanonicalFileName: (f) => f, + getNewLine: () => "\n", + }); + throw new Error(`Parse diagnostics in ${path.relative(ROOT, filePath)}\n${message}`); + } + + if (sourceText.includes("window.playwright.chromium.connectOverCDP")) { + throw new Error( + `${path.relative(ROOT, filePath)}: Playground injects the browser connection — remove window.playwright.chromium.connectOverCDP and use the provided globals.`, + ); + } + + if (hasStagehandUsage(sourceText) && !/new\s+Stagehand\s*\(/.test(sourceText)) { + throw new Error( + `${path.relative(ROOT, filePath)}: Stagehand usage detected but no \`new Stagehand({...})\` — playground config merge requires a constructor call.`, + ); + } +} + +async function main() { + const entries = await fetchPlaygroundTypescriptTemplateEntries(); + + for (const { slug, typescriptDir } of entries) { + const base = path.join(TYPESCRIPT_ROOT, typescriptDir); + const tsPath = path.join(base, "index.ts"); + const tsxPath = path.join(base, "index.tsx"); + + let entryPath = tsPath; + try { + await access(tsPath, constants.F_OK); + } catch { + await access(tsxPath, constants.F_OK); + entryPath = tsxPath; + } + + await validateSourceFile(entryPath); + process.stdout.write(`OK playground source: ${slug} → typescript/${typescriptDir}\n`); + } + + process.stdout.write(`Validated ${entries.length} playground template sources.\n`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); From d0fa10a82751411cd9f83f07863237d67c18386b Mon Sep 17 00:00:00 2001 From: bb Date: Wed, 20 May 2026 16:31:31 -0700 Subject: [PATCH 2/2] [DASH-2089] [templates] smart-fetch-scraper: comment about Fetch markdown/json formats (#98) --- python/smart-fetch-scraper/main.py | 8 ++++++++ typescript/smart-fetch-scraper/index.ts | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/python/smart-fetch-scraper/main.py b/python/smart-fetch-scraper/main.py index 302a5d45..bd8a6864 100644 --- a/python/smart-fetch-scraper/main.py +++ b/python/smart-fetch-scraper/main.py @@ -3,6 +3,11 @@ # Tries the Browserbase Fetch API first (fast, no browser session needed). # If the page is JS-rendered or the content is insufficient, falls back to # a full Stagehand browser session with AI-powered extraction. +# +# Note: the Fetch API can return content as `raw` HTML, clean `markdown`, +# or schema-extracted `json` — pass `format="markdown"` or `format="json"` +# (with a JSON schema) to `bb.fetch_api.create` to skip writing your own +# HTML parser. Docs: https://docs.browserbase.com/platform/fetch/overview import asyncio import json @@ -116,6 +121,9 @@ async def try_fetch_api(url: str) -> dict | None: print("[Fetch API] Attempting lightweight fetch...") try: + # Tip: pass `format="markdown"` or `format="json"` (with a `schema`) + # here to have Browserbase return cleaner content or structured data + # directly — see https://docs.browserbase.com/platform/fetch/overview # Use asyncio.to_thread for synchronous SDK calls data = await asyncio.to_thread(bb.fetch_api.create, url=url, allow_redirects=True) diff --git a/typescript/smart-fetch-scraper/index.ts b/typescript/smart-fetch-scraper/index.ts index a31baaaa..f20b799e 100644 --- a/typescript/smart-fetch-scraper/index.ts +++ b/typescript/smart-fetch-scraper/index.ts @@ -3,6 +3,11 @@ // Tries the Browserbase Fetch API first (fast, no browser session needed). // If the page is JS-rendered or the content is insufficient, falls back to // a full Stagehand browser session with AI-powered extraction. +// +// Note: the Fetch API can return content as `raw` HTML, clean `markdown`, +// or schema-extracted `json` — pass `format: "markdown"` or `format: "json"` +// (with a JSON schema) to `bb.fetchAPI.create` to skip writing your own +// HTML parser. Docs: https://docs.browserbase.com/platform/fetch/overview import "dotenv/config"; import Browserbase from "@browserbasehq/sdk"; @@ -91,6 +96,9 @@ async function tryFetchApi(url: string): Promise<{ content: string; statusCode: console.log("[Fetch API] Attempting lightweight fetch..."); try { + // Tip: pass `format: "markdown"` or `format: "json"` (with a `schema`) + // here to have Browserbase return cleaner content or structured data + // directly — see https://docs.browserbase.com/platform/fetch/overview const data = await bb.fetchAPI.create({ url, allowRedirects: true }); console.log(