feat(cli): modular CLI with SQL query support over all published tables

.github/workflows/publish-pypi.yml

@@ -0,0 +1,85 @@
+# Publishes the package to PyPI when a GitHub release is published.
+#
+# Uses PyPI "trusted publishing" (OIDC) — no API token secrets. One-time setup
+# a maintainer must do before the first release:
+#   1. On pypi.org (logged in as the owning account): Account → Publishing →
+#      add a pending publisher with project name `spicy-regs`, owner
+#      `civictechdc`, repository `spicy-regs`, workflow `publish-pypi.yml`,
+#      environment `pypi`.
+#   2. In this repo's Settings → Environments: create an environment named
+#      `pypi` (optionally require reviewers so releases need an approval).
+#
+# Release flow: bump `version` in pyproject.toml, merge to main, then create a
+# GitHub release whose tag is `v<version>` (e.g. v0.2.0). The tag/version
+# guard below fails the build if the two drift.
+
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Check release tag matches the project version
+        run: |
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          TAG="${GITHUB_REF_NAME}"
+          if [ "v${VERSION}" != "${TAG}" ]; then
+            echo "::error::Release tag ${TAG} does not match pyproject.toml version ${VERSION} (expected v${VERSION})."
+            exit 1
+          fi
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Smoke-test the wheel in a clean environment
+        run: |
+          uv venv /tmp/smoke
+          VIRTUAL_ENV=/tmp/smoke uv pip install dist/*.whl
+          /tmp/smoke/bin/spicy-regs --help
+          /tmp/smoke/bin/spicy-regs tables --source r2
+          /tmp/smoke/bin/spicy-regs-dict --help
+          /tmp/smoke/bin/python -c "import spicy_regs.mcp_server"
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spicy-regs
+    permissions:
+      # Required for PyPI trusted publishing (OIDC token exchange).
+      id-token: write
+
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -24,4 +24,4 @@ recovery
 .env.local
 .env*.local
 __pycache__
-notebooks/.cache
+notebooks/.cachedist/

CONTRIBUTING.md

@@ -58,12 +58,17 @@ processed output from the public R2 bucket:
 
 ```bash
 uv run spicy-regs download                       # dockets + documents + comments
-uv run spicy-regs download --types comments      # just comments
+uv run spicy-regs download --tables comments     # just comments
 uv run spicy-regs stats                          # sanity-check what you got
 uv run spicy-regs sample comments -n 5           # peek at a few rows
+uv run spicy-regs query "SELECT count(*) FROM dockets"   # arbitrary SQL (works pre-download too)
 ```
 
-Files default to `./spicy-regs-data/`; override with `-o some/dir`.
+Files default to `./spicy-regs-data/`; override with `-o some/dir`. The query
+commands (`tables`, `describe`, `query`, and friends) default to `--source
+auto`: they read your local downloads when present and stream from the public
+R2 bucket otherwise, so you can explore the full dataset before downloading
+anything.
 
 **B. Run the ETL pipeline yourself (slower, but it's the real thing).**
 Reads JSON from the Mirrulations S3 mirror, flattens it, writes Parquet to
@@ -112,7 +117,10 @@ Some terms you'll see throughout the codebase:
 
 ```
 src/spicy_regs/
-├── cli.py                # main CLI entrypoint (`spicy-regs` script)
+├── cli/                  # main CLI (`spicy-regs` script) — one module per subcommand
+│   ├── _registry.py      #   the COMMANDS list; add your module here
+│   ├── engine.py         #   shared DuckDB engine (views over R2 or local parquet)
+│   └── download.py, query.py, …   # subcommands: register() + run()
 ├── schemas/              # RecordType definitions — one per data shape
 ├── sources/              # Reader and Writer subclasses (S3, R2, parquet, …)
 ├── transforms/           # Transform subclasses + bulk-transform helpers
@@ -175,6 +183,12 @@ not connectors — don't subclass them.
 
 ### Recipes
 
+- **Add a CLI command:** copy `src/spicy_regs/cli/tables.py` as a template,
+  implement `register(subparsers)` and `run(args) -> int` in a new module under
+  `src/spicy_regs/cli/`, then add the module to `COMMANDS` in
+  `src/spicy_regs/cli/_registry.py`. Use `spicy_regs.cli.engine` for anything
+  that reads the published tables (it handles local vs. R2 resolution for you),
+  and add tests next to `tests/test_cli_commands.py`.
 - **Add a record shape:** construct a new `RecordType` in `schemas/` (set
   `path_pattern` only if your source addresses files by path).
 - **Add a source:** subclass `Reader`, implement `iter_records()` to yield raw
@@ -197,6 +211,23 @@ not connectors — don't subclass them.
 - Include a test plan.
 - Ensure CI passes before requesting review.
 
+## Releasing to PyPI (maintainers)
+
+Publishing is automated by `.github/workflows/publish-pypi.yml` via PyPI
+[trusted publishing](https://docs.pypi.org/trusted-publishers/) — no API
+tokens. To cut a release:
+
+1. Bump `version` in `pyproject.toml` and merge to `main`.
+2. Create a GitHub release whose tag is `v<version>` (e.g. `v0.2.0`).
+3. The workflow builds the sdist + wheel, checks the tag matches the version,
+   smoke-tests the wheel in a clean environment, and publishes to PyPI.
+
+One-time setup (already-released projects skip this): add a pending trusted
+publisher on pypi.org (project `spicy-regs`, owner `civictechdc`, repo
+`spicy-regs`, workflow `publish-pypi.yml`, environment `pypi`) and create the
+`pypi` environment in the repo settings. Once published, users can run the CLI
+with just `uvx spicy-regs` — no git URL needed.
+
 ## Reporting issues
 
 Open a GitHub issue with steps to reproduce, expected vs. actual behavior, and environment details. Pick a template from [the issue chooser](https://github.com/civictechdc/spicy-regs/issues/new/choose).

README.md

@@ -21,27 +21,50 @@ upload your output to live Cloudflare R2 storage.
 
 ### Download the published data locally
 
-The processed dockets / documents / comments parquet files are published to a
-public Cloudflare R2 bucket. Grab them with the bundled CLI — no credentials
-needed:
+Every published table (see the [data dictionary](https://civictechdc.github.io/spicy-regs/))
+lives as a parquet file in a public Cloudflare R2 bucket. Grab them with the
+bundled CLI — no credentials needed:
 
 ```bash
-uv run spicy-regs download                        # all three (dockets, documents, comments)
-uv run spicy-regs download --types comments       # comments only
+uv run spicy-regs download                        # the core trio (dockets, documents, comments)
+uv run spicy-regs download --tables comments      # comments only
+uv run spicy-regs download --tables feed_summary agency_stats   # any published table
+uv run spicy-regs download --all                  # everything (comments alone is multiple GB)
 uv run spicy-regs download -o ./my-data           # custom output dir
 ```
 
-Files land in `./spicy-regs-data/` by default. Once downloaded, poke around:
+Files land in `./spicy-regs-data/` by default. Downloads are atomic and
+incremental — a file whose size still matches the bucket is skipped (`--force`
+re-downloads).
+
+### Query the data with SQL
+
+`spicy-regs query` runs DuckDB SQL over every published table. By default
+(`--source auto`) each table reads from your local download when present and
+streams from the public bucket otherwise — so it works with no download at all:
 
 ```bash
-uv run spicy-regs stats                # row counts + top agencies per file
-uv run spicy-regs sample comments -n 5 # 5 random rows from comments.parquet
-uv run spicy-regs search "climate"     # substring search across files
+uv run spicy-regs tables                             # list every table + where it resolves
+uv run spicy-regs describe dockets                   # column schema of one table
+uv run spicy-regs query "SELECT agency_code, count(*) FROM dockets GROUP BY 1 ORDER BY 2 DESC LIMIT 5"
+uv run spicy-regs query "SELECT * FROM feed_summary LIMIT 3" --format json   # or csv
+uv run spicy-regs query "SELECT * FROM agency_stats" --output stats.csv --format csv --max-rows 0
+```
+
+Or poke around without writing SQL:
+
+```bash
+uv run spicy-regs stats                # row counts + top agencies per core table
+uv run spicy-regs sample comments -n 5 # 5 random rows from any table
+uv run spicy-regs search "climate"     # substring search across the core tables
 uv run spicy-regs agencies             # list every agency code
 ```
 
+These accept `--source local` (only your downloads) or `--source r2` (only the
+bucket) when you want to pin where data comes from.
+
 > Don't have the repo cloned? You can also run it one-shot with
-> `uvx --from "spicy-regs @ git+https://github.com/civictechdc/spicy-regs" spicy-regs download --types comments`.
+> `uvx --from "spicy-regs @ git+https://github.com/civictechdc/spicy-regs" spicy-regs download --tables comments`.
 
 ### Run the ETL pipeline yourself
 

docs/cli.md

@@ -0,0 +1,112 @@
+# CLI
+
+The `spicy-regs` command-line tool downloads the published parquet files and
+runs SQL against every table in this data dictionary — no credentials, no
+setup beyond Python.
+
+## Install
+
+Run it one-shot with [uv](https://docs.astral.sh/uv/getting-started/installation/)
+(nothing to install):
+
+```bash
+uvx --from "spicy-regs @ git+https://github.com/civictechdc/spicy-regs" spicy-regs --help
+```
+
+Or from a clone of the repo:
+
+```bash
+git clone https://github.com/civictechdc/spicy-regs.git
+cd spicy-regs && uv sync
+uv run spicy-regs --help
+```
+
+The examples below use `uv run spicy-regs …`; substitute the `uvx` form if you
+haven't cloned the repo.
+
+## Where commands read data from
+
+Every command that reads tables takes a `--source` flag:
+
+| Source | Behavior |
+| --- | --- |
+| `auto` (default) | Per table: use your local download when present, otherwise stream from the public bucket |
+| `local` | Only files downloaded to your data directory (default `./spicy-regs-data/`, override with `-o`) |
+| `r2` | Only the public bucket (`https://r2.spicy-regs.dev`) |
+
+Because of `auto`, **you can query everything without downloading anything** —
+DuckDB reads the remote parquet over HTTPS and only fetches the row groups a
+query needs. Download the tables you use heavily to make repeated queries fast
+and offline.
+
+## Explore the tables
+
+```bash
+uv run spicy-regs tables                 # every table + whether it resolves locally or to R2
+uv run spicy-regs describe dockets       # column names and types (matches this data dictionary)
+uv run spicy-regs describe comments --format json
+```
+
+## Query with SQL
+
+`spicy-regs query` runs [DuckDB](https://duckdb.org/docs/stable/sql/introduction)
+SQL with one view per published table, so you can filter, aggregate, and join
+across all of them:
+
+```bash
+# Top agencies by docket volume
+uv run spicy-regs query "SELECT agency_code, count(*) AS n FROM dockets GROUP BY 1 ORDER BY n DESC LIMIT 10"
+
+# Join: most-commented dockets with their titles
+uv run spicy-regs query "
+  SELECT d.docket_id, d.title, f.comment_count
+  FROM feed_summary f JOIN dockets d USING (docket_id)
+  ORDER BY f.comment_count DESC LIMIT 10"
+
+# Machine-readable output for scripts
+uv run spicy-regs query "SELECT * FROM agency_stats LIMIT 5" --format json
+uv run spicy-regs query "SELECT * FROM agency_stats" --format csv --output agency_stats.csv --max-rows 0
+```
+
+Options:
+
+| Flag | Meaning |
+| --- | --- |
+| `--format table\|json\|csv` | Output format (default: aligned table) |
+| `--max-rows N` | Cap returned rows; `0` = unlimited (default 25) |
+| `--output FILE` | Write results to a file instead of stdout |
+| `--source`, `--r2-url`, `-o` | Data source controls (see above) |
+
+Keep a `LIMIT` on exploratory queries — `comments` in particular is tens of
+millions of rows.
+
+## Quick looks without SQL
+
+```bash
+uv run spicy-regs stats                  # row counts + top agencies for the core tables
+uv run spicy-regs sample comments -n 5   # random rows from any table (--agency EPA to filter)
+uv run spicy-regs search "climate"       # substring search across dockets/documents/comments
+uv run spicy-regs agencies               # every agency code in the dataset
+```
+
+## Download the parquet files
+
+```bash
+uv run spicy-regs download                                  # core trio: dockets, documents, comments
+uv run spicy-regs download --tables feed_summary agency_stats
+uv run spicy-regs download --all                            # every table (comments alone is multiple GB)
+uv run spicy-regs download -o ./my-data                     # custom directory
+```
+
+Downloads stream with a progress bar and are written atomically, so an
+interrupted download never leaves a truncated file. Files whose size still
+matches the bucket are skipped on re-runs; `--force` re-downloads
+unconditionally.
+
+## Extending the CLI
+
+Each subcommand is a small module in
+[`src/spicy_regs/cli/`](https://github.com/civictechdc/spicy-regs/tree/main/src/spicy_regs/cli)
+— adding one is a copy-a-template-and-register-it change. See the
+[contributing guide](https://github.com/civictechdc/spicy-regs/blob/main/CONTRIBUTING.md)
+for the recipe.

docs/index.md

@@ -127,11 +127,19 @@ of `dockets`; they join to the corpus (and to each other) on a few shared keys:
 
 === "CLI"
 
+    The bundled [`spicy-regs` CLI](cli.md) downloads the parquet files and runs
+    SQL over every table — by default it reads local downloads when present and
+    streams from the bucket otherwise, so no download is required:
+
     ```bash
-    uvx --from "spicy-regs @ git+https://github.com/civictechdc/spicy-regs" spicy-regs download
-    uv run spicy-regs stats
+    uvx --from "spicy-regs @ git+https://github.com/civictechdc/spicy-regs" \
+      spicy-regs query "SELECT agency_code, count(*) FROM dockets GROUP BY 1 ORDER BY 2 DESC LIMIT 5"
+    uv run spicy-regs tables              # list every table
+    uv run spicy-regs download --all      # take the dataset offline
     ```
 
+    See the [CLI page](cli.md) for the full command reference.
+
 === "DuckDB (SQL)"
 
     ```sql

mkdocs.yml

@@ -36,6 +36,7 @@ markdown_extensions:
 
 nav:
   - Overview: index.md
+  - CLI: cli.md
   - Querying with Python: querying-python.md
   - Tables:
       - dockets: tables/dockets.md

pyproject.toml

@@ -2,6 +2,17 @@
 name = "spicy-regs"
 version = "0.1.0"
 description = "Regulations Data Analysis Tools"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Civic Tech DC" }]
+keywords = ["regulations", "rulemaking", "regulations.gov", "civic-tech", "open-data", "duckdb", "parquet"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+]
 requires-python = ">=3.10"
 dependencies = [
     "boto3",
@@ -42,6 +53,12 @@ docs = [
     "mkdocs-material",
 ]
 
+[project.urls]
+Homepage = "https://github.com/civictechdc/spicy-regs"
+Documentation = "https://docs.spicy-regs.dev"
+Repository = "https://github.com/civictechdc/spicy-regs"
+Issues = "https://github.com/civictechdc/spicy-regs/issues"
+
 [project.scripts]
 run-pipeline = "spicy_regs.pipelines.regulations:app"
 run-rollup-feed-summary = "spicy_regs.pipelines.rollups.feed_summary:app"