diff --git a/.env.example b/.env.example index d15b3d7..9340f33 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,9 @@ BLOOM_PARSE_APP_ID_PROD= BLOOM_PARSE_APP_ID_DEV= BLOOM_PARSE_APP_ID_UNIT_TEST= + +# send-concern-email: Mailgun ("report-a-book" template). Optional - if either is +# unset, the function logs and succeeds as a no-op (matches the legacy Parse cloud +# function's test-environment behavior). +MAILGUN_API_KEY= +EMAIL_REPORT_BOOK_RECIPIENT= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa75371..2d0f229 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,4 +35,17 @@ jobs: env: BLOOM_PARSE_APP_ID_PROD: ${{ secrets.BLOOM_PARSE_APP_ID_PROD }} BLOOM_PARSE_APP_ID_DEV: ${{ secrets.BLOOM_PARSE_APP_ID_DEV }} - BLOOM_PARSE_APP_ID_UNIT_TEST: ${{ secrets.BLOOM_PARSE_APP_ID_UNIT_TEST }} \ No newline at end of file + BLOOM_PARSE_APP_ID_UNIT_TEST: ${{ secrets.BLOOM_PARSE_APP_ID_UNIT_TEST }} + + # Keep in sync with the supabase version pinned in package.json. + - uses: supabase/setup-cli@v1 + with: + version: 2.109.1 + + # Prove the SQL migrations apply cleanly from scratch (catches broken + # or order-dependent migrations before they reach staging). + - name: Validate migrations apply from scratch + run: | + supabase start -x logflare,vector,studio,imgproxy,realtime,mailpit,edge-runtime,storage-api + supabase db reset + supabase stop --no-backup \ No newline at end of file diff --git a/FUNCTIONS-MIGRATION-PLAN.md b/FUNCTIONS-MIGRATION-PLAN.md index 4a9036d..f021a41 100644 --- a/FUNCTIONS-MIGRATION-PLAN.md +++ b/FUNCTIONS-MIGRATION-PLAN.md @@ -48,6 +48,21 @@ All clients call `https://api.bloomlibrary.org/v1/`. Known consumers: - CI/CD: GitHub Actions. PRs run tests; push to `develop` deploys to the **staging** Supabase project; push to `main` deploys to **production** (`.github/workflows/`). - Secrets so far: `BLOOM_PARSE_APP_ID_{PROD,DEV,UNIT_TEST}` plus deploy credentials. +- **`send-concern-email` (`supabase/functions/send-concern-email/`)** — implemented on the + `send-concern-email` branch, **not yet merged or deployed**. This isn't an Azure Function; it + replaces the Parse Cloud Code function `sendConcernEmail` (bloom-parse-server, `cloud/emails.js`), + called by blorg's "report a concern about this book" form. Same shape as `fs`: `verify_jwt = + false` (blorg still authenticates via Parse, not Supabase Auth, so there's no Supabase JWT to + check yet — see the TODO in `config.toml`), book data comes from `BloomParseServer` (there is no + Supabase `books` table on `develop` yet — see the note below), and it sends via the Mailgun REST + API directly (no `mailgun-js`). New secrets: `MAILGUN_API_KEY`, `EMAIL_REPORT_BOOK_RECIPIENT` + (both optional — unset means log-and-no-op, matching the legacy cloud function's behavior on + the unit-test Parse server). + + Note on book data: a Postgres `books` table (with `title`/`copyright`/`license` etc.) exists on + the separate `parse-to-supabase-db-foundation` branch but has not merged to `develop` as of this + writing. Once it lands, this function's Parse lookup is a candidate to switch to a direct + Postgres query, consistent with wherever the rest of this repo's book-reading functions land. --- diff --git a/PAPERCUTS.md b/PAPERCUTS.md new file mode 100644 index 0000000..34535d8 --- /dev/null +++ b/PAPERCUTS.md @@ -0,0 +1,18 @@ +Papercuts for bloom-core-supabase — small dev/agent/tooling friction points, captured now and +fixed later. See the "papercut" skill for the procedure. + +Note: when resolving a git merge conflict here, keep both sides' entries unless they merge cleanly. + +--- + +## 2026-07-18 — `pnpm test` hard-fails when `.env.local` doesn't exist + +- **Cut:** `pnpm test` and `pnpm test:secrets-optional` both pass `--env-file=.env.local` to + deno, and deno hard-fails ("node: .env.local: not found", exit 126) when the file is + missing — the repo ships only `.env.example`. On a fresh checkout the only test script that + runs at all is `pnpm test:ci`. +- **Idea:** Make `.env.local` optional (deno supports `--env-file` gracefully via a wrapper + that checks existence, or document `cp .env.example .env.local` as a required setup step in + the README / setup hook). +- **Context:** Hit running unit tests on branch parse-to-supabase-db-foundation; agent had to + discover `test:ci` as the fallback by trial and error. diff --git a/README.md b/README.md index 3e86fd3..6eb5947 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ pnpm install cp .env.example .env.local # Edit .env.local with your Parse Server credentials -# Start local development (requires Docker) +# Start local development (requires a container runtime — see Prerequisites) pnpm dev # Run tests @@ -21,7 +21,7 @@ pnpm test Test the fs function: ```bash -curl "http://127.0.0.1:54321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" -o test.bloompub +curl "http://127.0.0.1:44321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" -o test.bloompub ``` ## 📋 Prerequisites @@ -29,9 +29,46 @@ curl "http://127.0.0.1:54321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" - [pnpm](https://pnpm.io/) v11+ (pins its own version via `packageManager` and downloads Node.js v22.20.0 via `devEngines.runtime` in `package.json` — no Volta or manual Node install needed) - [Deno](https://deno.com/) v2.9+ (runs and tests the edge functions) -- [Docker Desktop](https://docs.docker.com/desktop/) (local development only) +- A container runtime (local development only): [Podman](https://podman.io/) (see + [Windows + Podman setup](#-windows--podman-setup) below) or + [Docker Desktop](https://docs.docker.com/desktop/) - [Supabase account](https://supabase.com/) +## 🪟 Windows + Podman setup + +Podman is the supported non-Docker-Desktop way to run the local stack (verified 2026-07): + +```powershell +winget install RedHat.Podman +podman machine init +podman machine set --rootful # required: rootless port forwarding doesn't reach the Windows host +podman machine start +``` + +Then start the stack. If Docker Desktop is also installed, point the CLI at Podman's pipe +explicitly, and exclude the analytics services (on Windows they require a TCP-exposed +Docker daemon, which Podman doesn't provide): + +```powershell +$env:DOCKER_HOST = "npipe:////./pipe/podman-machine-default" +pnpm exec supabase start -x logflare,vector +``` + +Gotchas we hit so you don't have to: + +- **Local ports are 443xx, not Supabase's default 543xx** (API `44321`, DB `44322`, + Studio `44323`, Mailpit `44324`). Windows reserves semi-random "excluded port ranges" + for Hyper-V/WSL inside the dynamic range (49152+), and Supabase's defaults landed inside + one — every service unreachable from the host, with no error anywhere. Ports below 49152 + can't be dynamically excluded. Check yours with + `netsh interface ipv4 show excludedportrange protocol=tcp`. +- Podman (unlike Docker) doesn't auto-create missing bind-mount sources; the repo now + commits `supabase/snippets/` and `supabase/seed.sql` so `supabase start` has everything + it needs. +- `supabase stop`'s volume prune trips over a Podman/Docker API difference + ("all" is an invalid volume filter) — harmless; use `supabase db reset` to get a truly + fresh database. + ### Dependency policy Both package resolvers enforce a **7-day cooldown**: a version published less than 7 days @@ -48,17 +85,37 @@ lockfile changes. ## 📁 Project Structure ``` -supabase/functions/ -├── _shared/ # Shared utilities -│ ├── BloomParseServer.ts -│ └── utils.ts -├── fs/ # S3 proxy function (streams book files) -│ ├── index.ts -│ ├── BookData.ts -│ └── README.md -└── tests/ # Deno tests +supabase/ +├── migrations/ # SQL migrations (the database schema: books, languages, tags, ...) +├── seed.sql # Local-dev seed (real data comes from packages/sync-tool) +└── functions/ + ├── _shared/ # Shared utilities + │ ├── BloomParseServer.ts + │ └── utils.ts + ├── fs/ # S3 proxy function (streams book files) + │ ├── index.ts + │ ├── BookData.ts + │ └── README.md + └── tests/ # Deno tests +packages/ +└── sync-tool/ # Parse -> Supabase data import (v0: sample importer) +docs/ +└── db/ # Database migration docs (field mapping, plan review, roadmap) ``` +## 📚 Importing sample data (local dev) + +With the local stack running, pull ~100 real books (plus their languages, tags, uploaders, +and relatedBooks) from the production Parse server into your local database: + +```bash +pnpm --filter @bloom/sync-tool import-sample +``` + +Idempotent — re-run any time to refresh. It refuses to write to a non-localhost Supabase +unless you set `SYNC_ALLOW_REMOTE=1` (env vars are `SYNC_*`-prefixed on purpose; see +`packages/sync-tool/src/import-sample.mjs`). + ## 🔧 Scripts ```bash @@ -78,13 +135,13 @@ Streams book files from S3 without exposing bucket details. **Example**: ```bash # Get thumbnail -curl "http://localhost:54321/functions/v1/fs/dev-harvest/ZWI7FUQnDd/thumbnails/thumbnail-256.png" +curl "http://localhost:44321/functions/v1/fs/dev-harvest/ZWI7FUQnDd/thumbnails/thumbnail-256.png" # Download book (streams, no buffering) -curl "http://localhost:54321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" -o book.bloompub +curl "http://localhost:44321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" -o book.bloompub # Range request -curl -H "Range: bytes=0-1023" "http://localhost:54321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" +curl -H "Range: bytes=0-1023" "http://localhost:44321/functions/v1/fs/harvest/VuebFgcL0R/Ososi.bloompub" ``` See [`supabase/functions/fs/README.md`](supabase/functions/fs/README.md) for details. diff --git a/api-spec-next.yml b/api-spec-next.yml index 0fd10b0..d4a3a63 100644 --- a/api-spec-next.yml +++ b/api-spec-next.yml @@ -34,6 +34,52 @@ components: type: integer required: [tag] paths: + # Implemented (supabase/functions/send-concern-email/) on the send-concern-email branch, but not + # yet merged/deployed, so it stays in api-spec-next.yml until it goes live. Replaces the Parse + # Cloud Code function `sendConcernEmail` used by blorg's "report a concern about this book" form. + /send-concern-email: + post: + description: + Emails the Bloom team a concern reported about a book, via a Mailgun + template. No auth (blorg gates this behind login client-side; there is + no Supabase JWT yet since blorg still authenticates via Parse). + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + fromAddress: + description: the reporter's email address; used as the outgoing email's "from" + type: string + format: email + content: + description: the reporter's free-text description of the concern (max 5000 chars) + type: string + maxLength: 5000 + bookId: + description: the database ID of the book being reported + type: string + required: [fromAddress, content, bookId] + responses: + 200: + description: OK + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: [success] + 400: + description: Bad Request - missing/invalid fromAddress, content, or bookId + 404: + description: Not Found - no book with that ID + 500: + description: Internal Server Error - the book lookup or the Mailgun request failed + # not needed yet (since we have the expanded form from /books) /languages/{tag}: get: diff --git a/docs/db/README.md b/docs/db/README.md new file mode 100644 index 0000000..379d806 --- /dev/null +++ b/docs/db/README.md @@ -0,0 +1,131 @@ +# The database: Parse → Supabase migration notes + +Status (2026-07-17): the **"Local Supabase + Blorg" milestone** is implemented — the +read-scope schema exists as a migration, `packages/sync-tool` imports a ~100-book sample +from production Parse, and blorg (branch `SupabaseMigration`) can browse it anonymously +against a local stack. Parse remains the production system of record; nothing here is +serving production traffic yet. + +See `FUNCTIONS-MIGRATION-PLAN.md` at the repo root for the *functions* migration (Azure → Edge +Functions, phases F0–F8). This doc covers the *database* side. + +## Schema conventions + +- **IDs**: `id TEXT PRIMARY KEY` preserving legacy Parse objectIds. New rows get a + DB-generated legacy-style 10-char alphanumeric id (`generate_legacy_style_id()`). +- **Names**: Parse camelCase → snake_case (`bookInstanceId` → `book_instance_id`). + The conversion is capital-run-aware, so `bloomPUBVersion` → `bloom_pub_version` with no + special-casing. (The `analytics_*` fields already carry underscores in Parse.) +- **Types**: Parse String → `text`, Number → `integer`/`numeric`/`bigint` (timestamps), + Boolean → `boolean`, Date → `timestamptz`, Array-of-strings → `text[]`, + Object / Array-of-objects → `jsonb` (`show`, `internet_limits`, `tools`). +- **Field authority**: the private repo `BloomBooks/bloom-parser-server-schema` + (`schema/production.json`), NOT `setupTables` in bloom-parse-server's cloud code (its + header admits it's out of date). `publisher_book_id` postdates the dump (bloom-parse-server + PR #76). +- **Relations**: `books.lang_pointers text[]` keeps the raw language ids for sync fidelity, + AND `book_languages` (junction, FKs) exists so PostgREST/supabase-js can embed language + records with books — the equivalent of Parse's `include=langPointers`: + `select("*, languages(*)")` (PostgREST resolves the many-to-many through the junction + automatically). This works because Parse's legacy `languages` array field is deliberately + not ported (see correction 8) — the name is reserved for the embed. + `books.uploader_id` → `users(id)`. +- **RLS**: enabled everywhere; anonymous public read via `Public read` policies + table + grants; no client write policies (the sync tool writes with the service role, which + bypasses RLS). Two policies are row-gated: soft-deleted books (`is_deleted`, the future + sync's tombstones) are never served, and a `users` row is only readable while the user + has at least one visible book — uploader emails stay embeddable for book display without + the table being a listable email directory. +- **Deliberately absent (post-milestone work)**: derivation triggers replacing Parse's + `beforeSave` (search string, tag normalization, moderator-field preservation — these must + be gated off for sync connections when they arrive), `updated_at` triggers (same gating + problem — see correction 7), write policies, Firebase third-party + auth wiring, and the classes `apiAccount` (needed for the opds function), + `appSpecification`/`appDetailsInLanguage`/`booksInApp` (verify they're used at all before + porting), `downloadHistory`, `version`, `bookDeletion` tombstones. + +## Design goal: perform at 200k books + +Production is ~25k books today; **every read-path design decision should assume 200k** +(decided 2026-07-22). "Small table, seq scan is fine" reasoning is not acceptable for +`books` — features must be designed against the 200k target, not current row counts. + +What this implies for the query patterns blorg actually issues (see +`SupabaseBookQueryBuilder.ts` in blorg): + +- **Exact tag filters** use `tags @> ARRAY[...]` against the existing GIN index on + `tags` — already fine at 200k. +- **Fuzzy/wildcard matching on vocabularies (tags, topics, bookshelves, branding, …)**: + the guiding pattern is **resolve against the small vocabulary table first, then apply + exact matches to books**. These matches are rare, and the vocabularies are tiny (the + `tags` table is a few thousand rows — a seq scan there is microseconds), so pattern + matching belongs on the vocabulary, never on 200k book rows. `match_topic_tags` already + works this way (LIKE over `tags`, then array containment on books via the GIN index). + The wildcard-tag path in blorg (`LIKE` on the generated `tags_text` column) should + eventually migrate to the same resolve-then-contain shape, at which point `tags_text` + can be retired; until then it's an accepted (rare) seq scan, not something to index. +- **Free-text search** is the exception — it's per-word `ILIKE '%word%'` against the + per-book `search` column, which is book content, not a vocabulary, and it's the *common* + query (the search box). This is the one place a books-side index earns its keep: + a **`pg_trgm` GIN index on `books.search`**. Trigram GIN serves `ILIKE` directly, so no + query changes are needed. (Caveats: words shorter than 3 chars fall back to seq scan; + write amplification lands only on the batch importer.) +- **Rare facets** (`title:`, `branding:`, `phash:` ILIKE on books) stay accepted seq + scans — infrequent enough that ~200k-row scans are tolerable, revisit only if slow-query + logs disagree. +- **Operational**: on a populated hosted database new indexes must be added with + `CREATE INDEX CONCURRENTLY` (outside a transaction); and bulk imports should end with + `ANALYZE` so the planner has fresh statistics before the first real queries arrive. + +## Corrections to the earlier draft plans + +The `supabase/*.md` docs in bloom-parse-server were a useful starting point; things fixed +or superseded here: + +1. `uploader_id REFERENCES auth.users(id)` was wrong — under Firebase third-party auth, + `auth.users` stays empty. It references `public.users` (TEXT legacy ids). +2. The draft DDL was missing `edition`, `upload_pending_timestamp`, `banner_image_url` + (languages), and the whole `apiAccount` class; and misnamed `analytics_started_count`. +3. `language.isoCode` is NOT unique in production (multiple rows per code) — the draft's + UNIQUE constraint would have broken the import. +4. `tools` can hold objects — `jsonb`, not `text[]`. +5. The draft's legacy-id generator could produce <10 chars (base64 of 8 bytes minus + stripped symbols); ours draws 24 bytes. +6. The auth plan (custom login endpoint + bespoke session tokens) is superseded by + Supabase's native **third-party Firebase auth** (`[auth.third_party.firebase]`, + supabase-js `accessToken`, `auth.jwt()` in RLS; requires a `role: 'authenticated'` + custom claim on Firebase users — a backfill task when we get there). +7. The sync plan's derivation-trigger idea conflicts with sync writes: triggers re-deriving + `search`/tags would fight the Parse-computed values the sync delivers. When triggers + arrive they must be gated (e.g. a `bloom.sync` session flag) so sync writes pass through + verbatim — which also gives us a parity test (replay a synced row through the triggers, + diff against Parse's output). This bit immediately: the v0 schema shipped `updated_at` + triggers stamping `now()` on update, which meant every importer re-run (an upsert's + update path) silently replaced Parse's real timestamps with the import time. They're + removed until the gating mechanism exists. +8. Parse's legacy `books.languages` array is not ported. Production evidence (2026-07-18): + only 146 books have the field, none created after Jan 2015, and every value is `[]` — + it carries zero information. Keeping it would also have collided with the natural + PostgREST embed name: a `languages` *column* on `books` and an embedded `languages` + *relation* can't both appear in one response, forcing every consumer to alias the embed + forever. `lang_pointers` + `book_languages` carry the real language data. + +## Roadmap after this milestone (summary) + +1. **Full sync tool**: watermark-based incremental sync (order: users → languages → tags → + books), tombstones for book deletions (the ONE change bloom-parse-server needs: an + `afterDelete("books")` trigger), scheduled runs + validation harness (sampled deep-diffs, + count checks, lag alerts). +2. **Auth**: enable Firebase third-party auth; `role` claim backfill; RLS policies for + uploader/moderator writes (email-verified, username==email semantics from + `bloomFirebaseAuthAdapter`). +3. **Read-path flips**: edge functions (opds, stats, books-GET) switch from + `BloomParseServer.ts` to Postgres, one at a time behind env switches, staging first; + byte-diff outputs against the Parse-sourced versions. +4. **blorg**: contract tests green on both backends, production flip of reads+writes + together at cutover. +5. **Write cutover** (joint with F8 books/upload): freeze Parse writes → final sync → + flip → 1–2 week read-only rollback window → decommission Parse + MongoDB Atlas. + +Key risks: search-semantics parity (Mongo `$text` vs Postgres), the untested `beforeSave` +behaviors, Parse Pointer-shaped JSON in frozen API contracts, and Firebase claim coverage. diff --git a/packages/sync-tool/package.json b/packages/sync-tool/package.json new file mode 100644 index 0000000..f5cd72d --- /dev/null +++ b/packages/sync-tool/package.json @@ -0,0 +1,13 @@ +{ + "name": "@bloom/sync-tool", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Parse -> Supabase data import. v0: imports a diverse sample of production books into a local Supabase for the 'Local Supabase + Blorg' milestone. Grows into the full incremental sync tool.", + "scripts": { + "import-sample": "node src/import-sample.mjs" + }, + "dependencies": { + "@supabase/supabase-js": "2.110.2" + } +} diff --git a/packages/sync-tool/src/import-sample.mjs b/packages/sync-tool/src/import-sample.mjs new file mode 100644 index 0000000..c1b4df2 --- /dev/null +++ b/packages/sync-tool/src/import-sample.mjs @@ -0,0 +1,345 @@ +// Parse -> Supabase sample importer (sync tool v0). +// +// Fetches a diverse sample of in-circulation books from a Parse server's +// public read API and upserts them (plus their uploaders, languages, tags, +// and relatedBooks rows) into a Supabase database. Idempotent: re-running +// refreshes the same rows. +// +// Environment. Variables are SYNC_-prefixed on purpose: generic names like +// SUPABASE_URL are often set machine-wide (deploy credentials!) and must not +// silently redirect this tool. +// SYNC_PARSE_SERVER_URL default: production bloom-parse-server +// SYNC_PARSE_APP_ID default: production app id (public; it ships in +// the bloomlibrary.org client bundle) +// SYNC_SUPABASE_URL default: http://127.0.0.1:44321 (local stack) +// SYNC_SUPABASE_SERVICE_ROLE_KEY default: the local stack's demo key +// SYNC_ALLOW_REMOTE must be "1" to write to a non-localhost Supabase +// SAMPLE_TARGET approximate number of books to import, default 100 + +import { createClient } from "@supabase/supabase-js"; + +const PARSE_SERVER_URL = + process.env.SYNC_PARSE_SERVER_URL ?? + "https://bloom-parse-server-production.azurewebsites.net/parse"; +const PARSE_APP_ID = + process.env.SYNC_PARSE_APP_ID ?? "R6qNTeumQXjJCMutAJYAwPtip1qBulkFyLefkCE5"; +const SUPABASE_URL = process.env.SYNC_SUPABASE_URL ?? "http://127.0.0.1:44321"; +const SUPABASE_SERVICE_ROLE_KEY = + process.env.SYNC_SUPABASE_SERVICE_ROLE_KEY ?? + // Well-known local-dev service key printed by `supabase start`. Not a secret. + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU"; +const SAMPLE_TARGET = parseInt(process.env.SAMPLE_TARGET ?? "100", 10); + +const isLocalTarget = /^https?:\/\/(127\.0\.0\.1|localhost)([:/]|$)/.test( + SUPABASE_URL +); +if (!isLocalTarget && process.env.SYNC_ALLOW_REMOTE !== "1") { + console.error( + `Refusing to write to non-local Supabase (${SUPABASE_URL}).\n` + + `Set SYNC_ALLOW_REMOTE=1 if you really mean to.` + ); + process.exit(1); +} + +const supabase = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, { + auth: { persistSession: false }, +}); + +// --------------------------------------------------------------------------- +// Parse REST helpers +// --------------------------------------------------------------------------- + +async function parseQuery(className, params) { + // POST with _method:GET (Parse REST convention) so large `where` clauses + // don't hit query-string length limits (IIS rejects >2KB with a bare 404). + const res = await fetch(`${PARSE_SERVER_URL}/classes/${className}`, { + method: "POST", + headers: { + "X-Parse-Application-Id": PARSE_APP_ID, + "Content-Type": "application/json", + }, + body: JSON.stringify({ _method: "GET", ...params }), + }); + if (!res.ok) { + throw new Error( + `Parse query ${className} failed: ${res.status} ${await res.text()}` + ); + } + return (await res.json()).results; +} + +// --------------------------------------------------------------------------- +// The diverse sampler: several angles so search, facets, levels, features and +// multiple scripts all have something to show in the local library. +// --------------------------------------------------------------------------- + +// Books must be publicly visible and harvested (so S3 artifacts exist). +const BASE_WHERE = { + inCirculation: true, + draft: { $ne: true }, + rebrand: { $ne: true }, + harvestState: "Done", + baseUrl: { $exists: true }, +}; + +function langQuery(isoCode) { + return { + langPointers: { + $inQuery: { where: { isoCode }, className: "language" }, + }, + }; +} + +const SAMPLER = [ + { label: "newest", where: {}, order: "-createdAt", limit: 20 }, + { label: "talking books", where: { features: "talkingBook" }, limit: 10 }, + { label: "activities", where: { features: { $in: ["activity", "quiz"] } }, limit: 8 }, + { label: "sign language", where: { features: "signLanguage" }, limit: 6 }, + { label: "level 1", where: { tags: "computedLevel:1" }, limit: 5 }, + { label: "level 2", where: { tags: "computedLevel:2" }, limit: 5 }, + { label: "level 3", where: { tags: "computedLevel:3" }, limit: 5 }, + { label: "level 4", where: { tags: "computedLevel:4" }, limit: 5 }, + { label: "topic: animal stories", where: { tags: "topic:Animal Stories" }, limit: 6 }, + { label: "topic: science", where: { tags: "topic:Science" }, limit: 6 }, + { label: "topic: health", where: { tags: "topic:Health" }, limit: 6 }, + { label: "french", where: langQuery("fr"), limit: 5 }, + { label: "spanish", where: langQuery("es"), limit: 5 }, + { label: "swahili", where: langQuery("sw"), limit: 5 }, + { label: "thai", where: langQuery("th"), limit: 5 }, + { label: "arabic", where: langQuery("ar"), limit: 5 }, + { label: "chinese", where: langQuery("zh"), limit: 5 }, + { label: "bengali", where: langQuery("bn"), limit: 5 }, + { label: "derivatives", where: { bookLineage: { $exists: true, $ne: "" } }, limit: 6 }, +]; + +// --------------------------------------------------------------------------- +// Transform: Parse book JSON -> snake_case row + relations +// --------------------------------------------------------------------------- + +// Columns of public.books (see supabase/migrations/20260717120000_*.sql), +// minus id/created_at/updated_at/uploader_id which are handled specially. +const BOOK_COLUMNS = new Set([ + "all_titles", "analytics_bloompub_downloads", "analytics_epub_downloads", + "analytics_finished_count", "analytics_mean_questions_correct_pct", + "analytics_median_questions_correct_pct", "analytics_pdf_downloads", + "analytics_questions_in_book_count", "analytics_quizzes_taken_count", + "analytics_shell_downloads", "analytics_started_count", "authors", + "base_url", "bloom_pub_version", "book_hash_from_images", + "book_instance_id", "book_lineage", "book_lineage_array", "book_order", + "booklet_making_is_appropriate", "branding_project_name", "copyright", + "country", "credits", "current_tool", "district", "download_count", + "download_source", "draft", "edition", "experimental", "features", "folio", + "format_version", "harvest_log", "harvest_started_at", "harvest_state", + "harvester_id", "harvester_major_version", "harvester_minor_version", + "has_bloom_pub", "imported_book_source_url", "importer_major_version", + "importer_minor_version", "importer_name", "in_circulation", + "internet_limits", "isbn", "keyword_stems", "keywords", "lang_pointers", + "last_uploaded", "leveled_reader_level", "librarian_note", + "license", "license_notes", "original_publisher", "original_title", + "page_count", "phash_of_first_content_image", "province", "publisher", + "publisher_book_id", "reader_tools_available", "rebrand", "search", "show", + "suitable_for_making_shells", "suitable_for_vernacular_library", "summary", + "tags", "thumbnail", "title", "tools", "update_source", + "upload_pending_timestamp", +]); + +// camelCase -> snake_case, aware of capital runs: bloomPUBVersion -> +// bloom_pub_version (not bloom_pubversion). +function toSnake(name) { + return name + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") + .replace(/([a-z0-9])([A-Z])/g, "$1_$2") + .toLowerCase(); +} + +function plainValue(v) { + if (v && typeof v === "object" && v.__type === "Date") return v.iso; + return v; +} + +const droppedFields = new Set(); + +function transformBook(parseBook) { + const row = { + id: parseBook.objectId, + created_at: parseBook.createdAt, + updated_at: parseBook.updatedAt, + uploader_id: parseBook.uploader?.objectId ?? null, + }; + // `languages` is deliberately dropped: empty on every production row, and + // its column-name slot is reserved for the PostgREST languages(*) embed + // (see the schema migration). + const specialFields = [ + "objectId", "createdAt", "updatedAt", "ACL", "uploader", "languages", + ]; + for (const [key, value] of Object.entries(parseBook)) { + if (specialFields.includes(key)) continue; + if (key === "langPointers") { + row.lang_pointers = (value ?? []).map((p) => p.objectId); + continue; + } + const col = toSnake(key); + if (!BOOK_COLUMNS.has(col)) { + droppedFields.add(key); + continue; + } + row[col] = plainValue(value); + } + return row; +} + +// --------------------------------------------------------------------------- +// Import +// --------------------------------------------------------------------------- + +async function upsert(table, rows, options = {}) { + if (rows.length === 0) return; + for (let i = 0; i < rows.length; i += 200) { + const batch = rows.slice(i, i + 200); + const { error } = await supabase + .from(table) + .upsert(batch, { onConflict: options.onConflict ?? "id" }); + if (error) throw new Error(`upsert into ${table} failed: ${error.message}`); + } + console.log(` ${table}: upserted ${rows.length} rows`); +} + +async function main() { + console.log(`Parse: ${PARSE_SERVER_URL}`); + console.log(`Supabase: ${SUPABASE_URL}`); + + // 1. Gather a diverse sample of books (deduped by objectId). + const booksById = new Map(); + for (const q of SAMPLER) { + if (booksById.size >= SAMPLE_TARGET * 1.2) break; + const results = await parseQuery("books", { + where: { ...BASE_WHERE, ...q.where }, + // Per-category limits below assume SAMPLE_TARGET's ~100 default; scale + // them so a larger SAMPLE_TARGET actually grows the sample instead of + // just changing the early-exit threshold above. + limit: Math.ceil(q.limit * (SAMPLE_TARGET / 100)), + ...(q.order ? { order: q.order } : {}), + include: "uploader,langPointers", + }); + let fresh = 0; + for (const b of results) { + if (!booksById.has(b.objectId)) fresh++; + booksById.set(b.objectId, b); + } + console.log(`sampled ${q.label}: ${results.length} found, ${fresh} new`); + } + const parseBooks = [...booksById.values()]; + console.log(`total sample: ${parseBooks.length} books`); + + // 2. Collect referenced users and languages from the expanded pointers. + const usersById = new Map(); + const languagesById = new Map(); + for (const b of parseBooks) { + const u = b.uploader; + if (u?.objectId) { + usersById.set(u.objectId, { + id: u.objectId, + email: u.username ?? u.email ?? null, + created_at: u.createdAt, + updated_at: u.updatedAt, + }); + } + for (const lp of b.langPointers ?? []) { + if (lp?.objectId && lp.__type !== "Pointer") { + languagesById.set(lp.objectId, { + id: lp.objectId, + created_at: lp.createdAt, + updated_at: lp.updatedAt, + iso_code: lp.isoCode ?? null, + name: lp.name ?? null, + english_name: lp.englishName ?? null, + ethnologue_code: lp.ethnologueCode ?? null, + banner_image_url: lp.bannerImageUrl ?? null, + usage_count: 0, // recomputed below from the local sample + }); + } + } + } + + // Local usage counts so the language menu reflects what's actually here. + for (const b of parseBooks) { + for (const lp of b.langPointers ?? []) { + const lang = languagesById.get(lp?.objectId); + if (lang) lang.usage_count++; + } + } + + // 3. All tag rows (cheap, and the topic/search menus need the vocabulary). + const parseTags = await parseQuery("tag", { limit: 10000, order: "name" }); + const tagRows = parseTags.map((t) => ({ + id: t.objectId, + name: t.name, + created_at: t.createdAt, + updated_at: t.updatedAt, + })); + + // 4. relatedBooks rows that mention any sampled book. + const relatedRows = new Map(); + const ids = [...booksById.keys()]; + for (let i = 0; i < ids.length; i += 25) { + const pointers = ids.slice(i, i + 25).map((id) => ({ + __type: "Pointer", + className: "books", + objectId: id, + })); + const results = await parseQuery("relatedBooks", { + where: { books: { $in: pointers } }, + limit: 100, + }); + for (const r of results) { + relatedRows.set(r.objectId, { + id: r.objectId, + created_at: r.createdAt, + updated_at: r.updatedAt, + book_ids: (r.books ?? []).map((p) => p.objectId), + }); + } + } + + // 5. Transform books and build the junction rows. + const bookRows = parseBooks.map(transformBook); + // Dedupe per book: a duplicated langPointer would put the same + // (book_id, language_id) twice in one upsert statement, which Postgres + // rejects ("ON CONFLICT DO UPDATE command cannot affect row a second time"). + const bookLanguageRows = parseBooks.flatMap((b) => { + const langIds = (b.langPointers ?? []) + .filter((lp) => lp?.objectId && languagesById.has(lp.objectId)) + .map((lp) => lp.objectId); + return [...new Set(langIds)].map((id) => ({ + book_id: b.objectId, + language_id: id, + })); + }); + if (droppedFields.size > 0) { + console.warn( + `WARNING: Parse fields not in the books schema were dropped: ` + + [...droppedFields].join(", ") + ); + } + + // 6. Upsert in FK order. + console.log("importing into Supabase..."); + await upsert("users", [...usersById.values()]); + await upsert("languages", [...languagesById.values()]); + await upsert("tags", tagRows); + await upsert("books", bookRows); + await upsert("book_languages", bookLanguageRows, { + onConflict: "book_id,language_id", + }); + await upsert("related_books", [...relatedRows.values()]); + + const { count } = await supabase + .from("books") + .select("*", { count: "exact", head: true }); + console.log(`done. books in Supabase: ${count}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9216328..51fabdf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,12 @@ importers: specifier: 2.109.1 version: 2.109.1 + packages/sync-tool: + dependencies: + '@supabase/supabase-js': + specifier: 2.110.2 + version: 2.110.2 + packages: '@ecies/ciphers@0.2.6': @@ -35,6 +41,10 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} + '@supabase/auth-js@2.110.2': + resolution: {integrity: sha512-Qj7a6EDP+AMMQFWqGv+qFa8r6re//dk+qQI5bA0KK+PZmnI3JPu97TDeNt6SMiQ2FkklP79hP2yDFYSnA989OA==} + engines: {node: '>=22.0.0'} + '@supabase/cli-darwin-arm64@2.109.1': resolution: {integrity: sha512-tkn8tfunyqIL7RE+7DVjg6Ql2cJLPkGgh9cPafp2LbXI0qDgds0TaS+UOTHQEjci8JQXXe2wS00+122ko2QI8A==} cpu: [arm64] @@ -79,10 +89,37 @@ packages: cpu: [x64] os: [win32] + '@supabase/functions-js@2.110.2': + resolution: {integrity: sha512-ZjjqrXpxM9/rE+eAtZxiK45EWy9EBoJQ322Q5Y75LccYQNh212neHTgXP/o4MIzmH0LNXT8UzvTZtQOfOzyoeQ==} + engines: {node: '>=22.0.0'} + + '@supabase/phoenix@0.4.4': + resolution: {integrity: sha512-Gt0pqoXuIqX/8dvG0OKp/wMCobXNH3klNbUPBNyOfN0YA1IswrM3HyWFMOPk1Jy+BRaIyDPcFx4jLBwHNmlyfQ==} + + '@supabase/postgrest-js@2.110.2': + resolution: {integrity: sha512-++LBmcIMwCtgO4tISQUmo9+2xkRwHQqS8ZKMCnhXLe9P8k8YQRXuMoh/RiSzQSoev8gqet0W7yOboW0cUxnt0Q==} + engines: {node: '>=22.0.0'} + + '@supabase/realtime-js@2.110.2': + resolution: {integrity: sha512-z3jTOTPgyn6E3r6dVOOQ10He4yAMB2czjFw7xVdX3s16MHElna5rY1gVaePs0NIo6xvtMYbtmOXlFaFt/ePLpg==} + engines: {node: '>=22.0.0'} + + '@supabase/storage-js@2.110.2': + resolution: {integrity: sha512-EhsRSwSnmQefKJsAxoRUZ0hvHr92ECM8DDGAKR5z0HdoJx4heI60PjHUTruVNZxKX6XeobLGDyLud020Bw1iwg==} + engines: {node: '>=22.0.0'} + + '@supabase/supabase-js@2.110.2': + resolution: {integrity: sha512-r9q9w4ZQ6mOjh36aqUNFSisBF611vzpO8JphBESr2Q1SWvmGFQeI7Jq7Y+PaNMZ6Zszz+S2yTlJStCpnaMSnQg==} + engines: {node: '>=22.0.0'} + eciesjs@0.5.0: resolution: {integrity: sha512-s0J9SEVYAEPg7J63GFMApLYzPH9VNIQIyC6s15JpnqVc0TqcKWdbgFlnAweEBRyMmko2dcs2sfC83Hj4J43tuA==} engines: {bun: '>=1', deno: '>=2.7.10', node: '>=16'} + iceberg-js@0.8.1: + resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==} + engines: {node: '>=20.0.0'} + jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} @@ -221,6 +258,9 @@ packages: resolution: {integrity: sha512-N2yP2MHTxOxXBWhfn3poudpJn4pkPosAUo7J/46FTou/l7wOwFi9tox8NSN6HljWkfM0zhwPRimNNGC9XBMoxQ==} hasBin: true + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + snapshots: '@ecies/ciphers@0.2.6(@noble/ciphers@1.3.0)': @@ -235,6 +275,10 @@ snapshots: '@noble/hashes@1.8.0': {} + '@supabase/auth-js@2.110.2': + dependencies: + tslib: 2.8.1 + '@supabase/cli-darwin-arm64@2.109.1': optional: true @@ -259,6 +303,34 @@ snapshots: '@supabase/cli-windows-x64@2.109.1': optional: true + '@supabase/functions-js@2.110.2': + dependencies: + tslib: 2.8.1 + + '@supabase/phoenix@0.4.4': {} + + '@supabase/postgrest-js@2.110.2': + dependencies: + tslib: 2.8.1 + + '@supabase/realtime-js@2.110.2': + dependencies: + '@supabase/phoenix': 0.4.4 + tslib: 2.8.1 + + '@supabase/storage-js@2.110.2': + dependencies: + iceberg-js: 0.8.1 + tslib: 2.8.1 + + '@supabase/supabase-js@2.110.2': + dependencies: + '@supabase/auth-js': 2.110.2 + '@supabase/functions-js': 2.110.2 + '@supabase/postgrest-js': 2.110.2 + '@supabase/realtime-js': 2.110.2 + '@supabase/storage-js': 2.110.2 + eciesjs@0.5.0: dependencies: '@ecies/ciphers': 0.2.6(@noble/ciphers@1.3.0) @@ -266,6 +338,8 @@ snapshots: '@noble/curves': 1.9.7 '@noble/hashes': 1.8.0 + iceberg-js@0.8.1: {} + jose@6.2.3: {} node@runtime:22.20.0: {} @@ -283,3 +357,5 @@ snapshots: '@supabase/cli-linux-x64-musl': 2.109.1 '@supabase/cli-windows-arm64': 2.109.1 '@supabase/cli-windows-x64': 2.109.1 + + tslib@2.8.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e75d0ce..507b43c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,6 @@ +packages: + - "packages/*" + # Only the supabase CLI may run install scripts (its postinstall downloads # the platform binary); everything else is blocked by pnpm's default. allowBuilds: diff --git a/supabase/config.toml b/supabase/config.toml index 5e4a01a..f8e4f7b 100644 --- a/supabase/config.toml +++ b/supabase/config.toml @@ -7,7 +7,13 @@ project_id = "bloom-supabase-bloom-core" [api] enabled = true # Port to use for the API URL. -port = 54321 +# NOTE: we deliberately use ports below 49152 (Supabase defaults are 543xx). +# Windows reserves semi-random "excluded port ranges" for Hyper-V/WSL inside +# the dynamic range (49152+), and the Supabase defaults landed inside one on a +# dev machine, silently making the whole stack unreachable from the host +# (check yours with: netsh interface ipv4 show excludedportrange protocol=tcp). +# Ports below 49152 are never dynamically excluded. +port = 44321 # Schemas to expose in your API. Tables, views and stored procedures in this schema will get API # endpoints. `public` and `graphql_public` schemas are included by default. schemas = ["public", "graphql_public"] @@ -26,9 +32,9 @@ enabled = false [db] # Port to use for the local database URL. -port = 54322 +port = 44322 # Port used by db diff command to initialize the shadow database. -shadow_port = 54320 +shadow_port = 44320 # The database major version to use. This has to be the same as your remote database's. Run `SHOW # server_version;` on the remote database to check. major_version = 17 @@ -36,7 +42,7 @@ major_version = 17 [db.pooler] enabled = false # Port to use for the local connection pooler. -port = 54329 +port = 44329 # Specifies when a server connection can be reused by other clients. # Configure one of the supported pooler modes: `transaction`, `session`. pool_mode = "transaction" @@ -82,7 +88,7 @@ enabled = true [studio] enabled = true # Port to use for Supabase Studio. -port = 54323 +port = 44323 # External URL of the API server that frontend connects to. api_url = "http://127.0.0.1" # OpenAI API Key to use for Supabase AI in the Supabase Studio. @@ -93,7 +99,7 @@ openai_api_key = "env(OPENAI_API_KEY)" [inbucket] enabled = true # Port to use for the email testing server web interface. -port = 54324 +port = 44324 # Uncomment to expose additional ports for testing user applications that send emails. # smtp_port = 54325 # pop3_port = 54326 @@ -330,10 +336,13 @@ deno_version = 2 BLOOM_PARSE_APP_ID_PROD = "env(BLOOM_PARSE_APP_ID_PROD)" BLOOM_PARSE_APP_ID_DEV = "env(BLOOM_PARSE_APP_ID_DEV)" BLOOM_PARSE_APP_ID_UNIT_TEST = "env(BLOOM_PARSE_APP_ID_UNIT_TEST)" +# send-concern-email (optional locally; unset -> the function no-ops, see .env.example) +MAILGUN_API_KEY = "env(MAILGUN_API_KEY)" +EMAIL_REPORT_BOOK_RECIPIENT = "env(EMAIL_REPORT_BOOK_RECIPIENT)" [analytics] enabled = true -port = 54327 +port = 44327 # Configure one of the supported backends: `postgres`, `bigquery`. backend = "postgres" @@ -355,4 +364,11 @@ s3_secret_key = "env(S3_SECRET_KEY)" verify_jwt = false [functions.social] +verify_jwt = false + +# blorg users still authenticate via Parse during the transition, so there is no +# Supabase JWT to verify here. TODO: set this back to true (the default) once blorg +# moves to Supabase Auth, and have the function derive fromAddress from the JWT +# instead of trusting the request body. +[functions.send-concern-email] verify_jwt = false \ No newline at end of file diff --git a/supabase/functions/_shared/BloomParseServer.ts b/supabase/functions/_shared/BloomParseServer.ts index 9875ea6..44c1b7a 100644 --- a/supabase/functions/_shared/BloomParseServer.ts +++ b/supabase/functions/_shared/BloomParseServer.ts @@ -20,6 +20,11 @@ export type Book = { inCirculation: boolean; ACL: Record; harvestState: string; + // Optional because most callers don't request them; Parse returns whatever + // fields exist on the record regardless of this type, so any book that has + // them set will still populate these when read. + copyright?: string; + license?: string; }; export default class BloomParseServer { diff --git a/supabase/functions/send-concern-email/concernEmail.ts b/supabase/functions/send-concern-email/concernEmail.ts new file mode 100644 index 0000000..07a6c3f --- /dev/null +++ b/supabase/functions/send-concern-email/concernEmail.ts @@ -0,0 +1,223 @@ +import BloomParseServer, { Book } from "../_shared/BloomParseServer.ts"; +import { getCorsHeaders, getEnvironment } from "../_shared/utils.ts"; + +// Replaces the legacy Parse cloud function `sendConcernEmail` (bloom-parse-server, +// cloud/emails.js). A blorg visitor fills out a "report a concern about this book" +// form; we email the report to the Bloom team via a Mailgun template. +// +// Request: POST JSON { fromAddress, content, bookId } +// - fromAddress: the reporter's email (blorg passes the logged-in user's address). +// Used as the outgoing email's "from" - there is no server-side account to +// attribute this to since blorg users still authenticate via Parse, not Supabase. +// - content: the reporter's free-text description of the concern. +// - bookId: the Parse objectId of the book being reported. +// +// TODO: once blorg authenticates via Supabase Auth (see FUNCTIONS-MIGRATION-PLAN.md), +// require a Supabase JWT here and derive fromAddress from it instead of trusting the +// request body. Legacy had no server-side auth at all (only blorg's UI gated this +// behind login); this preserves that behavior for now rather than breaking blorg's +// existing flow ahead of the auth cutover. + +const MAX_CONTENT_LENGTH = 5000; + +// Parse's default objectId shape: 10 alphanumeric characters. +const BOOK_ID_PATTERN = /^[A-Za-z0-9]{10}$/; + +// Not full RFC 5322 - just enough to reject obvious garbage. Deliberately permissive; +// Mailgun/the recipient's mail server is the real authority on validity. +const EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +export interface ConcernEmailRequestBody { + fromAddress?: unknown; + content?: unknown; + bookId?: unknown; +} + +function jsonResponse( + body: unknown, + status: number, + extraHeaders: Record +): Response { + return new Response(JSON.stringify(body), { + status, + headers: { ...extraHeaders, "Content-Type": "application/json" }, + }); +} + +// Returns an error message if invalid, or null if the value is a usable book id. +function validateBookId(value: unknown): string | null { + if (typeof value !== "string" || !BOOK_ID_PATTERN.test(value)) { + return "bookId is required and must be a valid book id."; + } + return null; +} + +function validateContent(value: unknown): string | null { + if (typeof value !== "string" || value.trim().length === 0) { + return "content is required."; + } + if (value.length > MAX_CONTENT_LENGTH) { + return `content must be ${MAX_CONTENT_LENGTH} characters or fewer.`; + } + return null; +} + +function validateFromAddress(value: unknown): string | null { + if (typeof value !== "string" || !EMAIL_PATTERN.test(value)) { + return "fromAddress is required and must be a valid email address."; + } + return null; +} + +// The variables handed to the Mailgun "report-a-book" template (and to the plain-text +// fallback below). Field names/fallback text match the legacy template exactly. +function buildTemplateVariables( + book: Book, + reportContent: string +): Record { + return { + title: book.title || "unknown title", + copyright: book.copyright || "unknown copyright", + license: book.license || "unknown license", + uploader: book.uploader?.username || "unknown uploader", + url: `https://bloomlibrary.org/book/${book.objectId}`, + body: reportContent, + }; +} + +// Fallback plain-text body, for Mailgun environments where the "report-a-book" +// dashboard template hasn't been configured (e.g. some test/sandbox domains). +// Assembled from the same variables as the template so the two stay in sync. +function buildTextFallback(vars: Record): string { + return ( + `A concern was reported about a Bloom Library book.\n\n` + + `Title: ${vars.title}\n` + + `Copyright: ${vars.copyright}\n` + + `License: ${vars.license}\n` + + `Uploader: ${vars.uploader}\n` + + `URL: ${vars.url}\n\n` + + `Reported concern:\n${vars.body}\n` + ); +} + +// Sends the report via the Mailgun REST API directly (no mailgun-js dependency - +// it's deprecated). Returns without making a network call if either MAILGUN_API_KEY +// or EMAIL_REPORT_BOOK_RECIPIENT is unset, logging instead - this mirrors the legacy +// cloud function's no-op behavior on the unit-test Parse server, where those env +// vars are deliberately left unset. +export async function sendConcernEmail( + fromAddress: string, + book: Book, + reportContent: string +): Promise { + const apiKey = Deno.env.get("MAILGUN_API_KEY"); + const recipient = Deno.env.get("EMAIL_REPORT_BOOK_RECIPIENT"); + + if (!apiKey) { + console.log( + "MAILGUN_API_KEY is not set; sendConcernEmail will just log and no-op." + ); + return; + } + if (!recipient) { + console.log( + "EMAIL_REPORT_BOOK_RECIPIENT is not set; sendConcernEmail will just log and no-op." + ); + return; + } + + const vars = buildTemplateVariables(book, reportContent); + + const form = new URLSearchParams(); + form.set("from", fromAddress); + form.set("to", recipient); + form.set("subject", `[BloomLibrary] Book reported - ${vars.title}`); + form.set("template", "report-a-book"); + form.set("h:X-Mailgun-Variables", JSON.stringify(vars)); + form.set("text", buildTextFallback(vars)); + + const response = await fetch( + "https://api.mailgun.net/v3/bloomlibrary.org/messages", + { + method: "POST", + headers: { + Authorization: "Basic " + btoa(`api:${apiKey}`), + "Content-Type": "application/x-www-form-urlencoded", + }, + body: form.toString(), + } + ); + + if (!response.ok) { + const errorText = await response.text(); + console.error( + `Mailgun request failed: ${response.status} ${errorText}` + ); + throw new Error(`Mailgun request failed: ${response.status}`); + } +} + +// Exported separately from index.ts so tests can exercise it with plain Request objects. +export async function handleSendConcernEmailRequest( + req: Request +): Promise { + const corsHeaders = getCorsHeaders(req); + + if (req.method === "OPTIONS") { + return new Response(null, { status: 204, headers: corsHeaders }); + } + + if (req.method !== "POST") { + return jsonResponse( + { error: "Method not allowed" }, + 405, + { ...corsHeaders, Allow: "POST, OPTIONS" } + ); + } + + let body: ConcernEmailRequestBody; + try { + body = await req.json(); + } catch { + return jsonResponse({ error: "Request body must be valid JSON." }, 400, corsHeaders); + } + + const bookIdError = validateBookId(body.bookId); + if (bookIdError) { + return jsonResponse({ error: bookIdError }, 400, corsHeaders); + } + const contentError = validateContent(body.content); + if (contentError) { + return jsonResponse({ error: contentError }, 400, corsHeaders); + } + const fromAddressError = validateFromAddress(body.fromAddress); + if (fromAddressError) { + return jsonResponse({ error: fromAddressError }, 400, corsHeaders); + } + + const bookId = body.bookId as string; + const content = body.content as string; + const fromAddress = body.fromAddress as string; + + let book: Book | undefined; + try { + const parseServer = new BloomParseServer(getEnvironment(req)); + book = await parseServer.getBookByDatabaseId(bookId, ["uploader"]); + } catch (error) { + console.error("Error looking up book for concern email:", error); + return jsonResponse({ error: "Internal Server Error" }, 500, corsHeaders); + } + + if (!book) { + return jsonResponse({ error: "Book not found" }, 404, corsHeaders); + } + + try { + await sendConcernEmail(fromAddress, book, content); + } catch (error) { + console.error("Error sending concern email:", error); + return jsonResponse({ error: "Internal Server Error" }, 500, corsHeaders); + } + + return jsonResponse({ success: true }, 200, corsHeaders); +} diff --git a/supabase/functions/send-concern-email/index.ts b/supabase/functions/send-concern-email/index.ts new file mode 100644 index 0000000..831f878 --- /dev/null +++ b/supabase/functions/send-concern-email/index.ts @@ -0,0 +1,6 @@ +import { handleSendConcernEmailRequest } from "./concernEmail.ts"; + +// Replaces the legacy Parse cloud function `sendConcernEmail`: a blorg visitor +// reports a concern about a book, and we email the report to the Bloom team. +// The logic lives in concernEmail.ts so tests can call it directly. +Deno.serve(handleSendConcernEmailRequest); diff --git a/supabase/functions/tests/send-concern-email-test.ts b/supabase/functions/tests/send-concern-email-test.ts new file mode 100644 index 0000000..e9617a2 --- /dev/null +++ b/supabase/functions/tests/send-concern-email-test.ts @@ -0,0 +1,378 @@ +import { assert, assertEquals, assertStringIncludes } from "@std/assert"; + +import { handleSendConcernEmailRequest } from "../send-concern-email/concernEmail.ts"; +import BloomParseServer, { Book } from "../_shared/BloomParseServer.ts"; + +// --------------------------------------------------------------------------- +// Test helpers: stub the Parse lookup and global fetch (the Mailgun request) so +// no network is touched, and control the Mailgun env vars. Every test restores +// the originals so it can't leak into other test files. +// --------------------------------------------------------------------------- + +const kValidBookId = "validBookI"; // 10 chars, matches BOOK_ID_PATTERN + +const mockBook = (overrides: Partial = {}): Book => + ({ + objectId: kValidBookId, + title: "Flowers for Foobar", + copyright: "Copyright 2020 Jane Author", + license: "cc-by", + bookInstanceId: "", + baseUrl: "", + uploader: { + objectId: "testUserId", + email: "uploader@example.com", + username: "uploader@example.com", + sessionToken: "", + }, + tags: [], + brandingProjectName: "", + updateSource: "test", + uploadPendingTimestamp: 0, + inCirculation: true, + ACL: {}, + harvestState: "Done", + ...overrides, + }) as Book; + +// Runs `fn` with getBookByDatabaseId returning `book` for kValidBookId (and +// undefined for anything else), and with global fetch replaced by `mailgunResponse`. +// Captures what the handler sent to "Mailgun" for assertions. +async function withStubs( + book: Book | undefined, + mailgunResponse: (() => Response) | null, + fn: (captured: { url?: string; init?: RequestInit }) => Promise +) { + const originalGetBook = BloomParseServer.prototype.getBookByDatabaseId; + const originalFetch = globalThis.fetch; + const captured: { url?: string; init?: RequestInit } = {}; + + BloomParseServer.prototype.getBookByDatabaseId = ( + objectId: string + ): Promise => + Promise.resolve(objectId === kValidBookId ? book : undefined); + + globalThis.fetch = ((input: string | URL | Request, init?: RequestInit) => { + captured.url = input.toString(); + captured.init = init; + if (!mailgunResponse) { + throw new Error("Test made an unexpected network request: " + input); + } + return Promise.resolve(mailgunResponse()); + }) as typeof fetch; + + try { + await fn(captured); + } finally { + BloomParseServer.prototype.getBookByDatabaseId = originalGetBook; + globalThis.fetch = originalFetch; + } +} + +// Runs `fn` with the given Mailgun-related env vars set (or deleted, for undefined), +// restoring the previous values afterward so tests can't leak into each other or +// into whatever real .env.local a developer has locally. +async function withEnv( + vars: Record, + fn: () => Promise +) { + const previous: Record = {}; + for (const key of Object.keys(vars)) { + previous[key] = Deno.env.get(key); + } + try { + for (const [key, value] of Object.entries(vars)) { + if (value === undefined) { + Deno.env.delete(key); + } else { + Deno.env.set(key, value); + } + } + await fn(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) { + Deno.env.delete(key); + } else { + Deno.env.set(key, value); + } + } + } +} + +const request = (body: unknown, init?: RequestInit) => + new Request("http://localhost/send-concern-email", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + ...init, + }); + +const kValidBody = { + fromAddress: "reporter@example.com", + content: "This book has an offensive image on page 3.", + bookId: kValidBookId, +}; + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +Deno.test("send-concern-email - 400 when bookId is missing", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest( + request({ ...kValidBody, bookId: undefined }) + ); + assertEquals(response.status, 400); + const json = await response.json(); + assertStringIncludes(json.error, "bookId"); + }); +}); + +Deno.test("send-concern-email - 400 when bookId has the wrong shape", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest( + request({ ...kValidBody, bookId: "tooShort" }) + ); + assertEquals(response.status, 400); + }); +}); + +Deno.test("send-concern-email - 400 when content is missing", async () => { + await withStubs(mockBook(), null, async () => { + const response = await handleSendConcernEmailRequest( + request({ ...kValidBody, content: "" }) + ); + assertEquals(response.status, 400); + const json = await response.json(); + assertStringIncludes(json.error, "content"); + }); +}); + +Deno.test("send-concern-email - 400 when content exceeds the length cap", async () => { + await withStubs(mockBook(), null, async () => { + const response = await handleSendConcernEmailRequest( + request({ ...kValidBody, content: "x".repeat(5001) }) + ); + assertEquals(response.status, 400); + }); +}); + +Deno.test("send-concern-email - 400 when fromAddress is not email-shaped", async () => { + await withStubs(mockBook(), null, async () => { + const response = await handleSendConcernEmailRequest( + request({ ...kValidBody, fromAddress: "not-an-email" }) + ); + assertEquals(response.status, 400); + const json = await response.json(); + assertStringIncludes(json.error, "fromAddress"); + }); +}); + +Deno.test("send-concern-email - 400 for malformed JSON body", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest( + new Request("http://localhost/send-concern-email", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "not json", + }) + ); + assertEquals(response.status, 400); + }); +}); + +Deno.test("send-concern-email - 405 for GET", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest( + new Request("http://localhost/send-concern-email", { method: "GET" }) + ); + assertEquals(response.status, 405); + }); +}); + +Deno.test("send-concern-email - 204 with CORS headers for OPTIONS preflight", async () => { + const response = await handleSendConcernEmailRequest( + new Request("http://localhost/send-concern-email", { + method: "OPTIONS", + headers: { Origin: "https://bloomlibrary.org" }, + }) + ); + assertEquals(response.status, 204); + assertEquals( + response.headers.get("Access-Control-Allow-Origin"), + "https://bloomlibrary.org" + ); +}); + +// --------------------------------------------------------------------------- +// Book lookup +// --------------------------------------------------------------------------- + +Deno.test("send-concern-email - 404 when the book does not exist", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest(request(kValidBody)); + assertEquals(response.status, 404); + }); +}); + +// --------------------------------------------------------------------------- +// No-op parity with legacy test-environment behavior +// --------------------------------------------------------------------------- + +Deno.test("send-concern-email - succeeds as a no-op when MAILGUN_API_KEY is unset", async () => { + await withEnv( + { MAILGUN_API_KEY: undefined, EMAIL_REPORT_BOOK_RECIPIENT: "team@bloomlibrary.org" }, + async () => { + await withStubs(mockBook(), null, async () => { + const response = await handleSendConcernEmailRequest(request(kValidBody)); + assertEquals(response.status, 200); + const json = await response.json(); + assertEquals(json.success, true); + }); + } + ); +}); + +Deno.test("send-concern-email - succeeds as a no-op when EMAIL_REPORT_BOOK_RECIPIENT is unset", async () => { + await withEnv( + { MAILGUN_API_KEY: "test-key", EMAIL_REPORT_BOOK_RECIPIENT: undefined }, + async () => { + await withStubs(mockBook(), null, async () => { + const response = await handleSendConcernEmailRequest(request(kValidBody)); + assertEquals(response.status, 200); + const json = await response.json(); + assertEquals(json.success, true); + }); + } + ); +}); + +// --------------------------------------------------------------------------- +// Happy path: exact Mailgun request shape +// --------------------------------------------------------------------------- + +Deno.test("send-concern-email - happy path sends the expected Mailgun request", async () => { + await withEnv( + { MAILGUN_API_KEY: "test-key", EMAIL_REPORT_BOOK_RECIPIENT: "team@bloomlibrary.org" }, + async () => { + await withStubs( + mockBook(), + () => new Response("OK", { status: 200 }), + async (captured) => { + const response = await handleSendConcernEmailRequest(request(kValidBody)); + assertEquals(response.status, 200); + const json = await response.json(); + assertEquals(json.success, true); + + assertEquals( + captured.url, + "https://api.mailgun.net/v3/bloomlibrary.org/messages" + ); + assertEquals(captured.init?.method, "POST"); + assertEquals( + captured.init?.headers && + (captured.init.headers as Record)[ + "Authorization" + ], + "Basic " + btoa("api:test-key") + ); + + const sentBody = new URLSearchParams( + captured.init?.body as string + ); + assertEquals(sentBody.get("from"), kValidBody.fromAddress); + assertEquals(sentBody.get("to"), "team@bloomlibrary.org"); + assertEquals( + sentBody.get("subject"), + "[BloomLibrary] Book reported - Flowers for Foobar" + ); + assertEquals(sentBody.get("template"), "report-a-book"); + + const vars = JSON.parse(sentBody.get("h:X-Mailgun-Variables")!); + assertEquals(vars, { + title: "Flowers for Foobar", + copyright: "Copyright 2020 Jane Author", + license: "cc-by", + uploader: "uploader@example.com", + url: `https://bloomlibrary.org/book/${kValidBookId}`, + body: kValidBody.content, + }); + + const text = sentBody.get("text")!; + assertStringIncludes(text, "Flowers for Foobar"); + assertStringIncludes(text, kValidBody.content); + } + ); + } + ); +}); + +Deno.test("send-concern-email - falls back to 'unknown X' for missing book fields", async () => { + await withEnv( + { MAILGUN_API_KEY: "test-key", EMAIL_REPORT_BOOK_RECIPIENT: "team@bloomlibrary.org" }, + async () => { + const bareBook = mockBook({ + title: "", + copyright: undefined, + license: undefined, + uploader: { + objectId: "testUserId", + email: "", + username: "", + sessionToken: "", + }, + }); + await withStubs( + bareBook, + () => new Response("OK", { status: 200 }), + async (captured) => { + await handleSendConcernEmailRequest(request(kValidBody)); + const sentBody = new URLSearchParams(captured.init?.body as string); + const vars = JSON.parse(sentBody.get("h:X-Mailgun-Variables")!); + assertEquals(vars.title, "unknown title"); + assertEquals(vars.copyright, "unknown copyright"); + assertEquals(vars.license, "unknown license"); + assertEquals(vars.uploader, "unknown uploader"); + } + ); + } + ); +}); + +Deno.test("send-concern-email - 500 when Mailgun request fails", async () => { + await withEnv( + { MAILGUN_API_KEY: "test-key", EMAIL_REPORT_BOOK_RECIPIENT: "team@bloomlibrary.org" }, + async () => { + await withStubs( + mockBook(), + () => new Response("Bad Request", { status: 400 }), + async () => { + const response = await handleSendConcernEmailRequest(request(kValidBody)); + assertEquals(response.status, 500); + } + ); + } + ); +}); + +Deno.test("send-concern-email - CORS headers are present on error responses too", async () => { + await withStubs(undefined, null, async () => { + const response = await handleSendConcernEmailRequest( + new Request("http://localhost/send-concern-email", { + method: "POST", + headers: { + "Content-Type": "application/json", + Origin: "https://bloomlibrary.org", + }, + body: JSON.stringify({ ...kValidBody, bookId: "bad" }), + }) + ); + assertEquals(response.status, 400); + assertEquals( + response.headers.get("Access-Control-Allow-Origin"), + "https://bloomlibrary.org" + ); + assert(response.headers.get("Content-Type")?.includes("application/json")); + }); +}); diff --git a/supabase/migrations/20260717120000_initial_read_schema.sql b/supabase/migrations/20260717120000_initial_read_schema.sql new file mode 100644 index 0000000..01f1e47 --- /dev/null +++ b/supabase/migrations/20260717120000_initial_read_schema.sql @@ -0,0 +1,236 @@ +-- Initial read-scope schema for the "Local Supabase + Blorg" milestone. +-- Columns are generated from the authoritative Parse schema dump +-- (BloomBooks/bloom-parser-server-schema, schema/production.json), mapped +-- camelCase -> snake_case. IDs preserve legacy Parse objectIds (TEXT PKs); +-- rows created in Supabase get a legacy-style 10-char alphanumeric id. +-- +-- Deliberately NOT here yet (post-milestone): derivation triggers replacing +-- Parse beforeSave (search string, tag normalization, ...), updated_at +-- triggers (an update trigger stamping now() would overwrite the Parse +-- timestamps the importer/sync writes on every re-run; updated_at triggers +-- arrive together with the sync-gating mechanism when the write path lands), +-- write RLS policies, auth wiring, apiAccount and the app* / downloadHistory +-- classes. + +create extension if not exists pgcrypto; + +-- Legacy-style short id: 10 alphanumeric chars, like Parse objectIds. +-- 24 random bytes -> base64 (32 chars) always retains >= 10 alphanumerics +-- after stripping '+', '/', '='. +create or replace function public.generate_legacy_style_id() +returns text +language sql +volatile +as $$ + select substring( + regexp_replace(encode(gen_random_bytes(24), 'base64'), '[^0-9A-Za-z]', '', 'g') + from 1 for 10 + ); +$$; + +-- --------------------------------------------------------------------------- +-- users (minimal: what anonymous book display needs, i.e. uploader email). +-- Full auth design (Firebase third-party) comes later. +-- --------------------------------------------------------------------------- +create table public.users ( + id text primary key default public.generate_legacy_style_id(), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + email text unique -- in Parse, username == email +); + +create table public.languages ( + id text primary key default public.generate_legacy_style_id(), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + iso_code text, -- NOT unique: production has multiple rows per isoCode + name text, + english_name text, + ethnologue_code text, + usage_count integer, + banner_image_url text +); + +create table public.tags ( + id text primary key default public.generate_legacy_style_id(), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + name text unique -- Parse has uniqueNameIndex +); + +create table public.books ( + id text primary key default public.generate_legacy_style_id(), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + + uploader_id text references public.users(id), + + all_titles text, -- JSON-as-string, parsed client-side (matches Parse) + analytics_bloompub_downloads integer, + analytics_epub_downloads integer, + analytics_finished_count integer, + analytics_mean_questions_correct_pct numeric, + analytics_median_questions_correct_pct numeric, + analytics_pdf_downloads integer, + analytics_questions_in_book_count integer, + analytics_quizzes_taken_count integer, + analytics_shell_downloads integer, + analytics_started_count integer, + authors text[], + base_url text, + bloom_pub_version integer, + book_hash_from_images text, + book_instance_id text, + book_lineage text, + book_lineage_array text[], + book_order text, + booklet_making_is_appropriate boolean, + branding_project_name text, + copyright text, + country text, + credits text, + current_tool text, + district text, + download_count integer, + download_source text, + draft boolean, + edition text, + experimental boolean, + features text[], + folio boolean, + format_version text, + harvest_log text[], + harvest_started_at timestamptz, + harvest_state text, + harvester_id text, + harvester_major_version integer, + harvester_minor_version integer, + has_bloom_pub boolean, + imported_book_source_url text, + importer_major_version integer, + importer_minor_version integer, + importer_name text, + in_circulation boolean, + internet_limits jsonb, + isbn text, + keyword_stems text[], + keywords text[], + lang_pointers text[], -- language ids; kept alongside book_languages for sync fidelity + -- Parse's legacy `languages` array is deliberately NOT ported: every + -- production row that has it holds [] (nothing has written it since Jan + -- 2015), and the name must stay free so PostgREST can embed language + -- records as `languages(*)` through book_languages. + last_uploaded timestamptz, + leveled_reader_level integer, + librarian_note text, + license text, + license_notes text, + original_publisher text, + original_title text, + page_count integer, + phash_of_first_content_image text, + province text, + publisher text, + publisher_book_id text, -- added to Parse after the schema dump (bloom-parse-server PR #76) + reader_tools_available boolean, + rebrand boolean, + search text, + show jsonb, + suitable_for_making_shells boolean, + suitable_for_vernacular_library boolean, + summary text, + tags text[], + thumbnail text, + title text, + tools jsonb, -- array that can hold objects in Parse + update_source text, + upload_pending_timestamp bigint, + + -- soft-delete support for the future incremental sync (tombstones) + is_deleted boolean not null default false, + deleted_at timestamptz +); + +-- Junction table so PostgREST/supabase-js can embed language records with +-- books (Parse `include=langPointers` equivalent). +create table public.book_languages ( + book_id text not null references public.books(id) on delete cascade, + language_id text not null references public.languages(id) on delete cascade, + primary key (book_id, language_id) +); + +create table public.related_books ( + id text primary key default public.generate_legacy_style_id(), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + book_ids text[] -- Parse field `books`: array of Pointer, flattened to ids +); + +-- --------------------------------------------------------------------------- +-- Indexes for the anonymous read path (derived from blorg query patterns and +-- the hot Mongo indexes on the books class) +-- --------------------------------------------------------------------------- +create index idx_books_book_instance_id on public.books (book_instance_id); +create index idx_books_created_at on public.books (created_at desc); +create index idx_books_last_uploaded on public.books (last_uploaded desc); +create index idx_books_uploader_id on public.books (uploader_id); +create index idx_books_tags on public.books using gin (tags); +create index idx_books_features on public.books using gin (features); +create index idx_books_lang_pointers on public.books using gin (lang_pointers); +create index idx_books_book_lineage_array on public.books using gin (book_lineage_array); +create index idx_book_languages_language_id on public.book_languages (language_id); +create index idx_languages_iso_code on public.languages (iso_code); +create index idx_languages_usage_count on public.languages (usage_count desc); + +-- --------------------------------------------------------------------------- +-- RLS: anonymous public read; no client writes (service role bypasses RLS, +-- which is how the importer/sync writes). +-- --------------------------------------------------------------------------- +alter table public.users enable row level security; +alter table public.languages enable row level security; +alter table public.tags enable row level security; +alter table public.books enable row level security; +alter table public.book_languages enable row level security; +alter table public.related_books enable row level security; + +-- Uploader emails are shown publicly on bloomlibrary.org book pages, so the +-- uploaders of visible books are readable (blorg embeds uploader:users(email) +-- via books.uploader_id). But the users table must not be a listable email +-- directory: rows without at least one visible book stay hidden. +create policy "Public read" on public.users for select to anon, authenticated + using ( + exists ( + select 1 from public.books b + where b.uploader_id = users.id and not b.is_deleted + ) + ); +create policy "Public read" on public.languages for select to anon, authenticated using (true); +create policy "Public read" on public.tags for select to anon, authenticated using (true); +-- Soft-deleted books (tombstones from the future incremental sync) must never +-- be served to clients. +create policy "Public read" on public.books for select to anon, authenticated + using (not is_deleted); +-- Junction rows must not enumerate the ids of soft-deleted (invisible) books. +create policy "Public read" on public.book_languages for select to anon, authenticated + using ( + exists ( + select 1 from public.books b + where b.id = book_id and not b.is_deleted + ) + ); +-- Same principle: a related_books row is only served while at least one of +-- its books is visible. (A mixed row can still contain a deleted book's id +-- in book_ids — an opaque id whose book row stays unreadable.) +create policy "Public read" on public.related_books for select to anon, authenticated + using ( + exists ( + select 1 from public.books b + where b.id = any (book_ids) and not b.is_deleted + ) + ); + +-- Table-level grants (RLS policies filter rows; grants allow the operation). +grant select on public.users, public.languages, public.tags, public.books, + public.book_languages, public.related_books to anon, authenticated; +grant all on public.users, public.languages, public.tags, public.books, + public.book_languages, public.related_books to service_role; diff --git a/supabase/migrations/20260717230000_add_books_tags_text.sql b/supabase/migrations/20260717230000_add_books_tags_text.sql new file mode 100644 index 0000000..e96a447 --- /dev/null +++ b/supabase/migrations/20260717230000_add_books_tags_text.sql @@ -0,0 +1,23 @@ +-- Enables per-array-element wildcard tag matching (Parse supported e.g. +-- "bookshelf:Enabling Writers*" via Mongo regex on the tags array; PostgREST +-- has no per-element pattern operator). tags_text renders the tags array as +-- "|tag1|tag2|...|", so a LIKE against it can anchor on element boundaries: +-- prefix bookshelf:X* -> LIKE '%|bookshelf:X%' +-- suffix *X -> LIKE '%X|%' +-- contains *X* -> LIKE '%X%' (tags never contain '|') +-- +-- array_to_string() is only STABLE in Postgres, and generated columns demand +-- IMMUTABLE expressions; for text[] (no element rendering ambiguity) this +-- wrapper is genuinely immutable. +create or replace function public.immutable_text_array_to_string(text[], text) +returns text +language sql +immutable +as $$ + select array_to_string($1, $2); +$$; + +alter table public.books + add column tags_text text generated always as ( + '|' || public.immutable_text_array_to_string(tags, '|') || '|' + ) stored; diff --git a/supabase/migrations/20260718000000_add_match_topic_tags_rpc.sql b/supabase/migrations/20260718000000_add_match_topic_tags_rpc.sql new file mode 100644 index 0000000..38d21fd --- /dev/null +++ b/supabase/migrations/20260718000000_add_match_topic_tags_rpc.sql @@ -0,0 +1,76 @@ +-- Resolves non-canonical `topic:` filter values the way Parse/blorg did. +-- +-- Background: blorg's `topic` filter (src/connection/BookQueryBuilder.ts) splits +-- the value on commas. Each part that matches a canonical topic in kTopicList +-- (case-insensitively) becomes an exact `topic:` tag requirement. +-- Parts that DON'T match a canonical topic ("non-canonical" topics) were +-- resolved by Parse with a case-insensitive Mongo $regex over the tag array: +-- * exactly one non-canonical value -> anchored ^topic:$ (whole-tag +-- equality, case-insensitive) +-- * two or more non-canonical values -> the values are OR-joined as +-- `topic:|topic:|...` UNANCHORED, i.e. a case-insensitive +-- substring/contains match; a book matches if any of its tags contains +-- any of those substrings. +-- The value is regex-escaped (processRegExp) before being placed in the pattern, +-- so for every realistic input it behaves as a literal string, never a pattern. +-- +-- PostgREST has no per-array-element regex operator, so the Supabase query +-- builder previously DROPPED non-canonical topics (a console.warn + TODO), +-- yielding an empty shelf where Parse showed books. This function closes that +-- gap: given the array of non-canonical topic values, it returns the set of +-- matching tag names, which the client feeds into its existing tag-requirement +-- machinery as an "any of these tags" (overlaps) constraint. That composes +-- (AND) with the canonical `topic:` requirements exactly as Parse's $and did. +-- +-- Faithfulness / deliberate divergence: the value is treated as a LITERAL +-- string (exact equality for the single case via lower(), substring search via +-- strpos() for the multi case) rather than a regex. This mirrors processRegExp +-- for every normal input (it escapes all regex metacharacters) and, crucially, +-- does NOT honor processRegExp's obscure `/.../`-delimited raw-regex escape +-- hatch. That is intentional: this function is anon-executable, and evaluating +-- attacker-supplied regexes server-side is a ReDoS surface. The escape hatch is +-- not a reachable topic-filter input in practice. +-- +-- Matching is done against the `tags` vocabulary table (small, unique-indexed +-- on name) rather than unnesting every book's tags on each call. In this +-- dataset the vocabulary is complete (every distinct book tag has a tags row), +-- and the Bloom tag vocabulary is maintained to stay so. A tag returned here +-- that no visible book carries is harmless (overlaps simply matches nothing); +-- soft-deleted books are still excluded by the outer book query's RLS. +-- +-- Security posture: STABLE, read-only, SECURITY INVOKER (no RLS bypass -- it +-- only reads public.tags, which is public-read, and never returns book ids so +-- there is nothing soft-delete-related to leak). search_path is pinned. +create or replace function public.match_topic_tags(topic_names text[]) +returns text[] +language sql +stable +security invoker +set search_path = '' +as $$ + select coalesce(array_agg(distinct t.name order by t.name), '{}'::text[]) + from public.tags t + where case + -- Single non-canonical value: anchored, whole-tag, case-insensitive + -- equality (Parse's ^topic:value$ with the /i flag). + when coalesce(array_length(topic_names, 1), 0) = 1 then + lower(t.name) = lower('topic:' || topic_names[1]) + -- Two or more: unanchored, case-insensitive contains-match, OR-ed across + -- the values (Parse's `topic:v1|topic:v2` with the /i flag). strpos on + -- lower()ed operands is a pure literal substring test -- no pattern + -- metacharacters, so nothing to escape and no ReDoS surface. + when coalesce(array_length(topic_names, 1), 0) > 1 then + exists ( + select 1 + from unnest(topic_names) as tn + where strpos(lower(t.name), lower('topic:' || tn)) > 0 + ) + -- Empty/NULL input matches nothing. + else false + end; +$$; + +-- Functions default to EXECUTE for PUBLIC; make the grant explicit and scoped. +revoke all on function public.match_topic_tags(text[]) from public; +grant execute on function public.match_topic_tags(text[]) + to anon, authenticated, service_role; diff --git a/supabase/seed.sql b/supabase/seed.sql new file mode 100644 index 0000000..87d7a8f --- /dev/null +++ b/supabase/seed.sql @@ -0,0 +1 @@ +-- Local dev seed data. The real content comes from packages/sync-tool import. diff --git a/supabase/snippets/.gitkeep b/supabase/snippets/.gitkeep new file mode 100644 index 0000000..b113b05 --- /dev/null +++ b/supabase/snippets/.gitkeep @@ -0,0 +1,2 @@ +# Keeps this directory in git: the Supabase CLI bind-mounts it and Podman, +# unlike Docker, refuses to start containers whose bind-mount source is missing.