diff --git a/AGENTS.md b/AGENTS.md index 96938261..6ce619d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Tina4 Python — Agent Instructions -v3.13.155. 140 cataloged features, zero dependencies. Python 3.12+. +v3.13.156. 140 cataloged features, zero dependencies. Python 3.12+. ## AI Skills diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ac21a86..16234f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,38 @@ https://tina4.com/python/36-releases This file records framework-specific changes. The release notes above remain the authority for shipped versions. +## 3.13.156 - 2026-11-01 + +### Added +- Kafka queue: `Queue.push(data, raw=True)` (and `produce(..., raw=True)`) now + produces the value with NO Tina4 envelope (`{payload, priority, attempts}`) and + no injected `id`, so a non-Tina4 service consuming the topic receives the exact + top-level schema it expects (tina4-python#264). It is a Kafka-only option; the + file/rabbitmq/mongodb backends refuse `raw=True` by name (the envelope is + internal bookkeeping that pop/retries/dead-lettering depend on, so there is + nothing to omit). The default `raw=False` path is unchanged. `pop()` already + unwraps both shapes (`result.get("payload", result)`). + +### Fixed +- `migrate:create` next steps named `tina4 migrate --rollback` and + `tina4 migrate --status`, flags `migrate` does not accept — a person or model + following the hint hit an error on the first command. They now name the + commands that exist: `tina4 migrate:rollback` and `tina4 migrate:status` + (tina4-python#268). Also closed two consistency gaps the report found: `migrate` + applies both `.sql` and `.py` migrations, but `migrate:status` listed only + `.sql` as pending and startup auto-migration only triggered on a `.sql` file — + so a pending `.py` migration was invisible to status and never auto-applied at + boot. `migrate()`, `get_files()`, `status()` and the startup gate now share one + discovery helper that globs `.sql` + `.py`. (Python-only: the hint and `.py` + migrations are python's.) +- Kafka queue: the confluent consumer hardcoded `auto.offset.reset: earliest`, so + every NEW consumer group replayed the entire retained backlog on first start + with no way to opt into `latest`. It now reads `TINA4_KAFKA_AUTO_OFFSET_RESET` + (then the bare `KAFKA_AUTO_OFFSET_RESET`), defaulting to `earliest` so current + behaviour is unchanged (tina4-python#263). Takes effect on the confluent-kafka + consume path; the zero-dependency raw fallback always reads from the earliest + retained offset. + ## 3.13.155 - 2026-10-29 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index b4d90918..1c916c34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ The full discipline lives in `.claude/skills/tina4-maintainer/SKILL.md`; this bl # Tina4 Python -Version 3.13.155 - Lightweight Python web framework. See https://tina4.com for full documentation. +Version 3.13.156 - Lightweight Python web framework. See https://tina4.com for full documentation. ## Build & Test @@ -605,7 +605,11 @@ queue = Queue(topic="tasks", max_retries=3) # consumer dies before complete()/fail() the next pop() reclaims it # (at-least-once delivery). Default 300; env TINA4_QUEUE_VISIBILITY_TIMEOUT; <= 0 disables. queue = Queue(topic="tasks", visibility_timeout=300) -queue.push(data: dict, priority=0, delay_seconds=0) -> int +queue.push(data: dict, priority=0, delay_seconds=0, raw=False) -> int +# raw=True is Kafka-only: produce the value with NO envelope and no injected id, +# for a non-Tina4 consumer (TINA4_KAFKA_AUTO_OFFSET_RESET controls where a new +# consumer group starts: earliest default, or latest). file/rabbitmq/mongodb +# refuse raw=True by name. queue.pop() -> Job | None queue.size(status="pending") -> int # size() status vocabulary: "pending" (waiting for pop -- includes retryable- @@ -944,7 +948,7 @@ uv run tina4python test # Discovers @tests in src/**/*.py - SSE/Streaming via `response.stream()` — Server-Sent Events support for real-time data push. Pass an async generator; framework handles chunked transfer encoding, `text/event-stream` content type, and connection keep-alive - MCP server (`tina4_python.mcp`): built-in dev tools auto-start when MCP is a capability of the deployment. Developer API: `McpServer`, `@mcp_tool`, `@mcp_resource`. JSON-RPC 2.0 over SSE. **Security is a two-layer gate (v3.13.40):** `is_enabled()` is a pure capability gate (explicit `TINA4_MCP` wins, else `TINA4_DEBUG`; host-independent), and `is_request_allowed(remote_ip, has_valid_token)` authorises each request on the RAW socket peer (`request.remote_ip`, never X-Forwarded-For): loopback always; a remote caller needs `TINA4_MCP_REMOTE=true` AND a token matching `TINA4_MCP_TOKEN` (never `TINA4_API_KEY`, ADR-0078; sent as Authorization Bearer / X-MCP-Token; no configured token means remote is always denied). Every `/__dev` request (reads too), the MCP endpoints and the `/__dev_reload` socket also require a loopback Host (`localhost`, `127.0.0.1`, `[::1]` or `TINA4_HOST`) and refuse `Sec-Fetch-Site: same-site`/`cross-site` (ADR-0078). `/health` omits `version` outside debug. All MCP surfaces (REST shim, JSON-RPC, SSE) 404 a disallowed caller. `database_query` is SELECT/WITH-only and rejects stacked statements; the file tools are sandboxed to the project root. `is_localhost()` is informational only, not the gate. **`TINA4_DEV_ALLOWED_PEERS`** (opt-in, default none) is a comma-separated IP/CIDR allow-list of RAW socket peers (v4/v6, never a forwarded header) admitted to `/__dev` - the documented way to reach the dashboard from a Docker dev box whose requests arrive from the container-network gateway, e.g. `172.16.0.0/12` (#279). `/__dev/toolbar.css` and `/__dev/toolbar.js` are static and always load; a viewer the gate would refuse gets no toolbar injected - Tests: 6,088 passing, 0 failures, **0 skipped** — measured 2026-09-17 on the lab (Ubuntu 24.04.4 LTS x86_64, Python 3.13.13, live services, `TINA4_REQUIRE_SERVICES=1`, 604s, run as root). **Firebird is NOT excluded.** The lab provisions a live Firebird 5 (`firebirdsql/firebird:5` on :3050, `TINA4_TEST_FIREBIRD_URL`) and those tests run. What IS deliberate is Firebird's absence from the `TINA4_REQUIRE_SERVICES` keyword gate in `tests/conftest.py`: GitHub CI does not provision Firebird, so a Firebird skip has to stay green *there*. Those are two different things and this line used to conflate them into "excluded by design", which read as "the Firebird tests do not run" — they do. **Nothing skips any more.** Reaching 0 skips now also needs the graph engines (Neo4j, Memgraph, ArangoDB, Ultipa, wired via `TINA4_TEST_NEO4J_URL`/`_MEMGRAPH_URL`/`_ARANGO_URL`/`_ULTIPA_URL` + the `graph` extra) and the lab Keycloak OIDC realm (`TINA4_REQUIRE_OIDC=1`), both added after the August baseline; without them `test_graph.py` (Feature 139) and the real OIDC contract test skip. The last skip was `tests/test_session_backend_failure.py` `[needs:no-dac-override]`: the suite runs as root, root holds `CAP_DAC_OVERRIDE`, so a write went straight through a 0400 file and no real `EACCES` was reachable — and as root the second `save()` returned `True`, so that skip was hiding a FAILING assertion, not merely an unrunnable one. The test now stops being root for the length of the failing write (`os.seteuid` to `nobody`; the saved uid stays 0, so a `finally` always restores it) and the kernel raises the genuine errno-13 the test asserts. Two parts of that are load-bearing: the log directory is handed to the same uid, because dropping the uid otherwise denies `Log.error` as well and the EACCES under test goes unrecorded (`_LogWriter.write` swallows `OSError` unless `TINA4_LOG_STRICT`); and the fixture root is its own 0755 `mkdtemp` rather than `tmp_path`, whose 0700 root-owned parents make every denial a directory traversal, which would pass for the wrong reason. Re-measure with `.venv/bin/python -m pytest tests/ -q` and quote the summary line, never the exit code -- Version: 3.13.155 +- Version: 3.13.156 ## Links diff --git a/pyproject.toml b/pyproject.toml index 4c30106b..773b0d0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tina4-python" -version = "3.13.155" +version = "3.13.156" description = "Tina4 Python v3 — Zero-dependency, lightweight web framework" authors = [ {name = "Andre van Zuydam", email = "andrevanzuydam@gmail.com"} diff --git a/tests/test_issue145_pool_transaction_isolation.py b/tests/test_issue145_pool_transaction_isolation.py index d161f532..790d300e 100644 --- a/tests/test_issue145_pool_transaction_isolation.py +++ b/tests/test_issue145_pool_transaction_isolation.py @@ -47,6 +47,16 @@ def _pg(): return url, os.environ.get("TINA4_TEST_PG_USERNAME", ""), os.environ.get("TINA4_TEST_PG_PASSWORD", "") +# This test drives real PostgreSQL across two threads; a pool/lock problem can +# leave a DB call blocked inside libpq (a C call). The suite's default +# `--timeout-method=signal` delivers SIGALRM to the main thread only and cannot +# interrupt a blocking C call, so such a block does not time out — it silently +# wedges the whole pytest-split group until the CI step is killed (observed: +# ~7 min of no output, no traceback). A per-test `method="thread"` override turns +# that into a bounded, diagnosable failure (the watchdog thread dumps every +# thread's stack and fails the test) instead of a hang. The marker overrides the +# CLI method for this test only; it never weakens what the test asserts. +@pytest.mark.timeout(120, method="thread") def test_issue145_open_transaction_does_not_leak_to_a_concurrent_pooled_thread(): url, user, password = _pg() db = Database(url, user, password, pool=4) diff --git a/tests/test_issue_268_migrate_hints_and_py_discovery.py b/tests/test_issue_268_migrate_hints_and_py_discovery.py new file mode 100644 index 00000000..190e24c0 --- /dev/null +++ b/tests/test_issue_268_migrate_hints_and_py_discovery.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026 Code Infinity +# SPDX-License-Identifier: MPL-2.0 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""#268: migrate:create's next steps named 'tina4 migrate --rollback'/'--status', +which `migrate` refuses — the real commands are `migrate:rollback`/`migrate:status`. + +And the two consistency gaps the reporter found: `migrate` applies both .sql AND +.py migrations, but `migrate:status` listed only .sql as pending and startup +auto-migration only triggered on a .sql file — so a pending .py migration was +invisible to status and never auto-applied at boot. Both now route through one +discovery helper that globs .sql + .py. + +Real SQLite throughout (a real dependency — no mocks). This is python-only: the +wrong hint is python's, and .py migrations are a python feature; the other three +frameworks carry neither the hint nor the .py discovery path. +""" +from tina4_python.cli import _next_steps_for +from tina4_python.core.server import _auto_migrate_on_startup +from tina4_python.database import Database +from tina4_python.migration import Migration + +PY_MIGRATION = """ +from tina4_python.migration import MigrationBase + + +class CreateWidgets(MigrationBase): + def up(self, db): + db.execute("CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)") + + def down(self, db): + db.execute("DROP TABLE IF EXISTS widgets") +""" + + +# ── the reported bug: the next-steps hint names commands that exist ────────── + +def test_migration_next_steps_name_commands_that_exist(): + steps = _next_steps_for("migration", "add_widgets", {}, {}) + joined = "\n".join(steps) + # The commands that actually exist: + assert "tina4 migrate:rollback" in joined, joined + assert "tina4 migrate:status" in joined, joined + # NOT the flags `migrate` refuses (what the hint used to print): + assert "migrate --rollback" not in joined, joined + assert "migrate --status" not in joined, joined + + +# ── .py migrations are visible to status() and get_files() (real SQLite) ───── + +def test_status_lists_a_pending_py_migration(tmp_path): + d = tmp_path / "migrations" + d.mkdir() + (d / "20260101000000_create_widgets.py").write_text(PY_MIGRATION, encoding="utf-8") + db = Database(f"sqlite:///{tmp_path / 'app.db'}") + try: + status = Migration(db, str(d)).status() + pending = [m["migration_name"] for m in status["pending"]] + assert "20260101000000_create_widgets" in pending, pending + finally: + db.close() + + +def test_get_files_includes_py_and_excludes_dunder(tmp_path): + d = tmp_path / "migrations" + d.mkdir() + (d / "20260101000000_create_widgets.py").write_text(PY_MIGRATION, encoding="utf-8") + (d / "__init__.py").write_text("", encoding="utf-8") + db = Database(f"sqlite:///{tmp_path / 'app.db'}") + try: + files = Migration(db, str(d)).get_files() + assert "20260101000000_create_widgets.py" in files, files + assert "__init__.py" not in files, files + finally: + db.close() + + +# ── startup auto-migration triggers for a .py-only migrations folder ───────── + +def test_startup_auto_migrate_runs_for_a_py_only_folder(tmp_path, monkeypatch): + d = tmp_path / "migrations" + d.mkdir() + (d / "20260101000000_create_widgets.py").write_text(PY_MIGRATION, encoding="utf-8") + db_path = tmp_path / "app.db" + monkeypatch.setenv("TINA4_DATABASE_URL", f"sqlite:///{db_path}") + monkeypatch.delenv("TINA4_AUTO_MIGRATE", raising=False) + monkeypatch.chdir(tmp_path) + + _auto_migrate_on_startup() # must NOT early-return on a .py-only folder + + db = Database(f"sqlite:///{db_path}") + try: + assert db.table_exists("widgets"), "startup auto-migrate skipped the .py-only folder" + finally: + db.close() diff --git a/tests/test_mcp_migration_create_envelope.py b/tests/test_mcp_migration_create_envelope.py index 354515b5..261fdc27 100644 --- a/tests/test_mcp_migration_create_envelope.py +++ b/tests/test_mcp_migration_create_envelope.py @@ -136,13 +136,17 @@ def test_migration_create_returns_generate_v1_1_envelope(tmp_path): assert isinstance(hint["line"], int) and hint["line"] > 0, hint assert isinstance(hint["label"], str) and hint["label"], hint - # Populated next[] — three curated steps for the migration verb. + # Populated next[] — three curated steps for the migration verb. The + # rollback/status steps name the commands that EXIST — `tina4 migrate:rollback` + # and `tina4 migrate:status` — not the flags `migrate` refuses (#268). steps = resolution["next"] assert isinstance(steps, list) and len(steps) == 3, steps assert any("tina4 migrate" in s and "rollback" not in s and "status" not in s for s in steps), steps - assert any("--rollback" in s for s in steps), steps - assert any("--status" in s for s in steps), steps + assert any("tina4 migrate:rollback" in s for s in steps), steps + assert any("tina4 migrate:status" in s for s in steps), steps + # The broken flags the hint used to print must not come back. + assert not any("migrate --rollback" in s or "migrate --status" in s for s in steps), steps # actions_taken reflects the real writes. actions = envelope["actions_taken"] diff --git a/tests/test_queue_kafka_offset_and_raw.py b/tests/test_queue_kafka_offset_and_raw.py new file mode 100644 index 00000000..fb79510d --- /dev/null +++ b/tests/test_queue_kafka_offset_and_raw.py @@ -0,0 +1,143 @@ +# Copyright (c) 2026 Code Infinity +# SPDX-License-Identifier: MPL-2.0 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Kafka consumer offset-reset override (#263) and raw producer option (#264). + +#263: the confluent consumer hardcoded ``auto.offset.reset: earliest``, so every +NEW consumer group replayed the whole retained backlog on first start with no way +to opt into 'latest'. It now reads TINA4_KAFKA_AUTO_OFFSET_RESET (then bare +KAFKA_AUTO_OFFSET_RESET), defaulting to earliest so current behaviour is unchanged. + +#264: the Kafka adapter always wrapped the value in the {payload, priority, +attempts} job envelope, which breaks a non-Tina4 consumer that expects a specific +top-level schema. push(raw=True) now emits the value exactly as given — no +envelope, no injected id. It is a Kafka-only option; the file/rabbitmq/mongodb +backends refuse raw=True by name rather than silently ignore it. + +The offset-reset resolution and the raw refusals are pure and run everywhere. The +live round trips need a real broker (TINA4_KAFKA_BROKERS) and confluent-kafka. +""" +import os +import secrets + +import pytest + +from tina4_python.queue import Queue +from tina4_python.queue_backends import KafkaConnector + +KAFKA_BROKERS = os.environ.get("TINA4_KAFKA_BROKERS", "") + + +# ── #263: auto.offset.reset resolution (pure, runs everywhere) ─────────────── + +def test_offset_reset_defaults_to_earliest(monkeypatch): + monkeypatch.delenv("TINA4_KAFKA_AUTO_OFFSET_RESET", raising=False) + monkeypatch.delenv("KAFKA_AUTO_OFFSET_RESET", raising=False) + assert KafkaConnector._offset_reset() == "earliest" + + +def test_offset_reset_honours_tina4_env(monkeypatch): + monkeypatch.setenv("TINA4_KAFKA_AUTO_OFFSET_RESET", "latest") + monkeypatch.delenv("KAFKA_AUTO_OFFSET_RESET", raising=False) + assert KafkaConnector._offset_reset() == "latest" + + +def test_offset_reset_falls_back_to_bare_kafka_env(monkeypatch): + monkeypatch.delenv("TINA4_KAFKA_AUTO_OFFSET_RESET", raising=False) + monkeypatch.setenv("KAFKA_AUTO_OFFSET_RESET", "latest") + assert KafkaConnector._offset_reset() == "latest" + + +def test_tina4_env_wins_over_bare_kafka_env(monkeypatch): + monkeypatch.setenv("TINA4_KAFKA_AUTO_OFFSET_RESET", "latest") + monkeypatch.setenv("KAFKA_AUTO_OFFSET_RESET", "earliest") + assert KafkaConnector._offset_reset() == "latest" + + +def test_blank_offset_reset_falls_through_to_default(monkeypatch): + monkeypatch.setenv("TINA4_KAFKA_AUTO_OFFSET_RESET", "") + monkeypatch.delenv("KAFKA_AUTO_OFFSET_RESET", raising=False) + assert KafkaConnector._offset_reset() == "earliest" + + +# ── #264: raw is Kafka-only — other backends refuse by name (real, offline) ── + +def test_push_raw_on_file_backend_is_refused_by_name(): + queue = Queue(topic="tina4_raw_refusal_" + secrets.token_hex(4), backend="file") + try: + with pytest.raises(NotImplementedError) as raised: + queue.push({"event": "x"}, raw=True) + message = str(raised.value) + assert "file queue backend" in message, message + assert "raw=True" in message, message + finally: + queue.close() + + +def test_push_without_raw_on_file_backend_is_unchanged(): + """Positive control: the default raw=False path is untouched.""" + queue = Queue(topic="tina4_raw_default_" + secrets.token_hex(4), backend="file") + try: + job_id = queue.push({"event": "x"}) + assert job_id + job = queue.pop() + assert job is not None and job.data == {"event": "x"} + job.complete() + finally: + queue.close() + + +# ── #264 live: a raw produce carries no envelope and no injected id ────────── + +def test_raw_push_emits_unenveloped_value(monkeypatch): + pytest.importorskip("confluent_kafka", reason="confluent-kafka not installed (uv sync --extra test)") + if not KAFKA_BROKERS: + pytest.skip("kafka not set: export TINA4_KAFKA_BROKERS (e.g. 127.0.0.1:9092)") + monkeypatch.delenv("TINA4_QUEUE_URL", raising=False) + monkeypatch.setenv("TINA4_KAFKA_GROUP_ID", "tina4_raw_" + secrets.token_hex(8)) + topic = "tina4_raw_" + secrets.token_hex(6) + external_schema = {"event_type": "order.created", "order_id": 42, "amount": 9.99} + + producer = Queue(topic=topic, backend="kafka") + try: + producer.push(external_schema, raw=True) + finally: + producer.close() + + # Read the raw value with the low-level connector — exactly what a non-Tina4 + # consumer would see off the wire. It must be the schema as given: no + # "payload"/"priority"/"attempts" envelope and no injected "id". + reader = KafkaConnector(brokers=KAFKA_BROKERS, group_id="tina4_raw_r_" + secrets.token_hex(8)) + try: + value = reader.dequeue(topic) + assert value == external_schema, value + assert "payload" not in value and "id" not in value, value + finally: + reader.close() + + +def test_non_raw_push_still_carries_the_envelope(monkeypatch): + """Positive control for #264: the default push is still enveloped + id'd.""" + pytest.importorskip("confluent_kafka", reason="confluent-kafka not installed (uv sync --extra test)") + if not KAFKA_BROKERS: + pytest.skip("kafka not set: export TINA4_KAFKA_BROKERS (e.g. 127.0.0.1:9092)") + monkeypatch.delenv("TINA4_QUEUE_URL", raising=False) + monkeypatch.setenv("TINA4_KAFKA_GROUP_ID", "tina4_env_" + secrets.token_hex(8)) + topic = "tina4_env_" + secrets.token_hex(6) + + producer = Queue(topic=topic, backend="kafka") + try: + producer.push({"event": "y"}) + finally: + producer.close() + + reader = KafkaConnector(brokers=KAFKA_BROKERS, group_id="tina4_env_r_" + secrets.token_hex(8)) + try: + value = reader.dequeue(topic) + assert value.get("payload") == {"event": "y"}, value + assert "id" in value, value + finally: + reader.close() diff --git a/tina4_python/.tina4-metrics.json b/tina4_python/.tina4-metrics.json index 8c31efa2..6ba07e5f 100644 --- a/tina4_python/.tina4-metrics.json +++ b/tina4_python/.tina4-metrics.json @@ -1868,14 +1868,14 @@ "duplicate_lines": 0 }, "queue_backends/kafka_backend.py": { - "offenders": 3, + "offenders": 4, "worst_cc": 13, - "maintainability": 12.9, - "loc": 499, - "complexity": 107, - "avg_complexity": 3.24, - "functions": 33, - "halstead_volume": 9510.68, + "maintainability": 12.4, + "loc": 523, + "complexity": 111, + "avg_complexity": 3.26, + "functions": 34, + "halstead_volume": 9657.57, "has_referencing_test": true, "dep_count": 8, "coupling_efferent": 0, @@ -35785,4 +35785,4 @@ } } ] -} \ No newline at end of file +} diff --git a/tina4_python/__init__.py b/tina4_python/__init__.py index e6209d20..2146323a 100644 --- a/tina4_python/__init__.py +++ b/tina4_python/__init__.py @@ -57,7 +57,7 @@ def _resolve_version() -> str: # # test_version_constant.py now asserts this literal equals the pyproject # version, so the release bump cannot leave it behind again. - return "3.13.155" + return "3.13.156" __version__ = _resolve_version() diff --git a/tina4_python/cli/__init__.py b/tina4_python/cli/__init__.py index 53590b51..8223edd3 100644 --- a/tina4_python/cli/__init__.py +++ b/tina4_python/cli/__init__.py @@ -1397,8 +1397,8 @@ def _next_steps_for(target: str, name: str, resolution: dict, flags: dict) -> li if target == "migration": return [ "Apply it: tina4 migrate", - "Roll back last batch: tina4 migrate --rollback", - "Inspect the pending: tina4 migrate --status", + "Roll back last batch: tina4 migrate:rollback", + "Inspect the pending: tina4 migrate:status", ] if target == "middleware": diff --git a/tina4_python/core/server.py b/tina4_python/core/server.py index 8ef8b199..6b2a6b72 100644 --- a/tina4_python/core/server.py +++ b/tina4_python/core/server.py @@ -4172,8 +4172,13 @@ def _auto_migrate_on_startup(migration_folder: str = "migrations") -> None: from pathlib import Path from tina4_python.dotenv import is_truthy + from tina4_python.migration.runner import _migration_source_files + folder = Path(migration_folder) - if not folder.is_dir() or not any(folder.glob("*.sql")): + # Both .sql AND .py migrations count — the runner applies both, so a .py-only + # migrations folder must still trigger startup auto-migration (#268). Reuse + # the one discovery helper so this gate and the runner never disagree. + if not folder.is_dir() or not _migration_source_files(folder): return # no migrations → nothing to do (silent) if not is_truthy(os.environ.get("TINA4_AUTO_MIGRATE", "true")): Log.debug("TINA4_AUTO_MIGRATE is off — skipping startup migrations") diff --git a/tina4_python/migration/runner.py b/tina4_python/migration/runner.py index 1d6dafa7..67c77118 100644 --- a/tina4_python/migration/runner.py +++ b/tina4_python/migration/runner.py @@ -676,6 +676,26 @@ def _migration_sort_key(name: str): return (0, int(m.group(1)), name) if m else (1, 0, name) +def _migration_source_files(folder: Path) -> list: + """Every migration file the runner APPLIES — ``.sql`` and ``.py`` — sorted by + the apply order (numeric/timestamp prefix). Excludes ``.down.sql`` rollback + files and dunder ``.py`` helpers (``__init__`` etc.). + + The single source of truth so migrate(), get_files() and status() can never + disagree on what is pending. They used to each glob ``*.sql`` separately while + only migrate() also ran ``*.py``, so a pending ``.py`` migration was applied by + migrate() yet never shown by status() and never triggered startup + auto-migration (tina4-python#268). + """ + if not folder.is_dir(): + return [] + return sorted( + [f for f in folder.glob("*.sql") if not f.name.endswith(".down.sql")] + + [f for f in folder.glob("*.py") if not f.name.startswith("__")], + key=lambda f: _migration_sort_key(f.name), + ) + + # CREATE TABLE — name may be quoted ("x"), bracketed ([x] MSSQL), or bare. _CREATE_TABLE_RE = re.compile( r"^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" @@ -900,15 +920,9 @@ def _run_pending(db, migration_folder: str = "migrations", delimiter: str = ";") batch = _get_next_batch(db) ran = [] - # Collect both .sql and .py migration files. Sort by a leading numeric / - # timestamp prefix (numeric-aware) so `9_*` applies before `10_*` — a plain - # lexical sort misorders unpadded prefixes ("10" < "9"). Files with no numeric - # prefix sort after the numbered ones, then lexically. - migration_files = sorted( - [f for f in folder.glob("*.sql") if not f.name.endswith(".down.sql")] - + [f for f in folder.glob("*.py") if not f.name.startswith("__")], - key=lambda f: _migration_sort_key(f.name), - ) + # Collect both .sql and .py migration files, in apply order — the one + # discovery helper that get_files() and status() also use (#268). + migration_files = _migration_source_files(folder) # Warn (once) about filenames without a recognized NNNNNN_/timestamp prefix — # their ordering relative to numbered migrations is undefined, a silent # out-of-order-apply footgun. @@ -1200,11 +1214,9 @@ def get_pending(self) -> list[dict]: return self.status()["pending"] def get_files(self) -> list[str]: - """Return sorted list of all migration filenames on disk (excludes .down.sql).""" - folder = Path(self._dir) - if not folder.is_dir(): - return [] - return sorted(f.name for f in folder.glob("*.sql") if not f.name.endswith(".down.sql")) + """Sorted migration filenames on disk — both .sql and .py (the two the + runner applies), excluding .down.sql and dunder .py (#268).""" + return [f.name for f in _migration_source_files(Path(self._dir))] def record_migration(self, name: str, batch: int, passed: int = 1) -> None: """Insert a record into the migration tracking table. @@ -1245,11 +1257,9 @@ def _status(db, migration_folder: str = "migrations") -> dict: executed = _get_executed(db) - # Get all .sql files sorted by name (excluding .down.sql) - sql_files = sorted( - f for f in folder.glob("*.sql") - if not f.name.endswith(".down.sql") - ) + # Every migration the runner would apply — .sql AND .py — so a pending .py + # migration shows up here instead of being invisible until it runs (#268). + migration_files = _migration_source_files(folder) # Build completed list from the database with execution metadata result = db.fetch( @@ -1268,7 +1278,7 @@ def _status(db, migration_folder: str = "migrations") -> dict: # Build pending list from files not yet executed pending = [] - for sql_file in sql_files: + for sql_file in migration_files: migration_name = sql_file.stem if migration_name not in executed: desc = re.sub(r"^\d+_", "", migration_name, count=1).replace("_", " ") diff --git a/tina4_python/queue/__init__.py b/tina4_python/queue/__init__.py index beb4553b..14f74f49 100644 --- a/tina4_python/queue/__init__.py +++ b/tina4_python/queue/__init__.py @@ -124,9 +124,19 @@ def _retarget(self, topic: str) -> None: self.retry_backoff, self.visibility_timeout, ) - def push(self, data: dict, priority: int = 0, delay_seconds: int = 0): - """Add a job to the queue. Returns job ID.""" - return self._backend.push(data, priority, delay_seconds) + def push(self, data: dict, priority: int = 0, delay_seconds: int = 0, + raw: bool = False): + """Add a job to the queue. Returns job ID. + + ``raw=True`` is a Kafka-only option (#264): it produces the value with + NO Tina4 envelope ({payload, priority, attempts}) and no injected id, so + a non-Tina4 service consuming the topic receives the exact schema it + expects. On the file/rabbitmq/mongodb backends the envelope is internal + bookkeeping that pop(), retries and dead-lettering depend on, so there is + nothing to omit — those backends refuse raw=True by name rather than + silently ignore it. + """ + return self._backend.push(data, priority, delay_seconds, raw) def pop(self) -> Job | None: """Atomically claim the next available job. Returns None if empty.""" @@ -303,9 +313,12 @@ def close(self) -> None: self._backend.close() def produce(self, topic: str, data: dict, priority: int = 0, - delay_seconds: int = 0, delay_until: datetime | None = None): + delay_seconds: int = 0, delay_until: datetime | None = None, + raw: bool = False): """Produce a message onto a topic. Convenience wrapper around push(). + ``raw`` behaves exactly as on push() — Kafka-only, un-enveloped (#264). + Either ``delay_seconds`` (integer offset) or ``delay_until`` (absolute ``datetime``) may be passed for scheduled jobs. ``delay_until`` takes precedence when both are set — naive datetimes are interpreted @@ -322,7 +335,7 @@ def produce(self, topic: str, data: dict, priority: int = 0, old_topic = self.topic self._retarget(topic) try: - return self.push(data, priority, delay_seconds) + return self.push(data, priority, delay_seconds, raw) finally: self._retarget(old_topic) diff --git a/tina4_python/queue/kafka_backend.py b/tina4_python/queue/kafka_backend.py index 09e1430a..d7cd69c2 100644 --- a/tina4_python/queue/kafka_backend.py +++ b/tina4_python/queue/kafka_backend.py @@ -32,7 +32,8 @@ def __init__(self, topic: str, max_retries: int): self._topic = topic self._max_retries = max_retries - def push(self, data: dict, priority: int = 0, delay_seconds: int = 0) -> str: + def push(self, data: dict, priority: int = 0, delay_seconds: int = 0, + raw: bool = False) -> str: if priority > 0: raise NotImplementedError( "The kafka queue backend cannot honour push(priority): Kafka has " @@ -48,6 +49,12 @@ def push(self, data: dict, priority: int = 0, delay_seconds: int = 0) -> str: "every record behind it. Use the file or mongodb backend for " "delayed jobs, or schedule the push itself." ) + if raw: + # Emit the value exactly as given — no {payload, priority, attempts} + # envelope and no injected id — so a non-Tina4 consumer reading the + # topic gets the schema it expects (#264). The symmetric unwrap is + # already on pop(): result.get("payload", result). + return self._backend.enqueue(self._topic, data, add_id=False) msg = {"payload": data, "priority": priority, "attempts": 0} return self._backend.enqueue(self._topic, msg) diff --git a/tina4_python/queue/lite_backend.py b/tina4_python/queue/lite_backend.py index 3d573a71..05caafa6 100644 --- a/tina4_python/queue/lite_backend.py +++ b/tina4_python/queue/lite_backend.py @@ -93,7 +93,18 @@ def _next_prefix(self) -> str: self._seq += 1 return f"{int(time.time() * 1000)}-{self._seq:06d}" - def push(self, data: dict, priority: int = 0, delay_seconds: int = 0) -> str: + def push(self, data: dict, priority: int = 0, delay_seconds: int = 0, + raw: bool = False) -> str: + if raw: + raise NotImplementedError( + "The file queue backend cannot honour push(raw=True): raw emits " + "the message with no Tina4 envelope, which only a message bus " + "consumed by non-Tina4 systems (Kafka) needs. This backend stores " + "an internal envelope (id/status/attempts/available_at) that " + "pop(), the retry lifecycle and dead-lettering depend on, so there " + "is nothing to omit. Use the kafka backend for raw external " + "messages." + ) job_id = str(uuid.uuid4()) available = _now() if delay_seconds == 0 else _future(delay_seconds) job = { diff --git a/tina4_python/queue/mongo_backend.py b/tina4_python/queue/mongo_backend.py index b308aa9c..5db07041 100644 --- a/tina4_python/queue/mongo_backend.py +++ b/tina4_python/queue/mongo_backend.py @@ -41,7 +41,18 @@ def __init__(self, topic: str, max_retries: int, visibility_timeout: float = 300 self._topic = topic self._max_retries = max_retries - def push(self, data: dict, priority: int = 0, delay_seconds: int = 0) -> str: + def push(self, data: dict, priority: int = 0, delay_seconds: int = 0, + raw: bool = False) -> str: + if raw: + raise NotImplementedError( + "The mongodb queue backend cannot honour push(raw=True): raw " + "emits the message with no Tina4 envelope, which only a message " + "bus consumed by non-Tina4 systems (Kafka) needs. This backend " + "stores an internal envelope (payload/priority/attempts) that " + "pop(), the retry lifecycle and dead-lettering depend on, so there " + "is nothing to omit. Use the kafka backend for raw external " + "messages." + ) msg = {"payload": data, "priority": priority, "attempts": 0} return self._backend.enqueue(self._topic, msg, delay_seconds) diff --git a/tina4_python/queue/rabbitmq_backend.py b/tina4_python/queue/rabbitmq_backend.py index b9002bae..c7abefce 100644 --- a/tina4_python/queue/rabbitmq_backend.py +++ b/tina4_python/queue/rabbitmq_backend.py @@ -31,7 +31,18 @@ def __init__(self, topic: str, max_retries: int): self._max_retries = max_retries self._jobs: dict = {} # track jobs by id for complete/fail/retry - def push(self, data: dict, priority: int = 0, delay_seconds: int = 0) -> str: + def push(self, data: dict, priority: int = 0, delay_seconds: int = 0, + raw: bool = False) -> str: + if raw: + raise NotImplementedError( + "The rabbitmq queue backend cannot honour push(raw=True): raw " + "emits the message with no Tina4 envelope, which only a message " + "bus consumed by non-Tina4 systems (Kafka) needs. This backend " + "stores an internal envelope (payload/priority/attempts) that " + "pop(), the retry lifecycle and dead-lettering depend on, so there " + "is nothing to omit. Use the kafka backend for raw external " + "messages." + ) if priority > 0: raise NotImplementedError( "The rabbitmq queue backend cannot honour push(priority): " diff --git a/tina4_python/queue_backends/kafka_backend.py b/tina4_python/queue_backends/kafka_backend.py index 294c9292..a22fc5e5 100644 --- a/tina4_python/queue_backends/kafka_backend.py +++ b/tina4_python/queue_backends/kafka_backend.py @@ -12,6 +12,13 @@ Environment variables: TINA4_KAFKA_BROKERS — comma-separated broker list (default: localhost:9092) TINA4_KAFKA_GROUP_ID — consumer group ID (default: tina4_consumer_group) + TINA4_KAFKA_AUTO_OFFSET_RESET — where a NEW consumer group starts when it has + no committed offset: 'earliest' (default, unchanged behaviour) or 'latest'. + Read as TINA4_KAFKA_AUTO_OFFSET_RESET first, then bare KAFKA_AUTO_OFFSET_RESET + (the librdkafka-convention name many deployments already set). Takes effect + on the confluent-kafka consume path; the zero-dependency raw fallback always + reads from the earliest retained offset (it tracks offsets from 0 and has no + offset-reset concept). TLS/SASL (each read as TINA4_KAFKA_ first, then bare KAFKA_): TINA4_KAFKA_SECURITY_PROTOCOL — e.g. SSL / SASL_SSL (default: PLAINTEXT) @@ -90,11 +97,23 @@ def connect(self): else: self._connect_raw() - def enqueue(self, topic: str, message: dict) -> str: - """Push a message onto a Kafka topic. Returns the message ID.""" + def enqueue(self, topic: str, message: dict, add_id: bool = True) -> str: + """Push a message onto a Kafka topic. Returns the message ID. + + ``add_id`` (default True) stamps a generated ``id`` into the message + value so a Tina4 consumer can acknowledge it. For a RAW produce + (``Queue.push(raw=True)``, #264) it is False: the value is serialized + exactly as given — no ``id`` injected and the caller's dict is not + mutated — so a non-Tina4 consumer receives the bytes it expects. A + record key is still generated for partitioning; the key is never part + of the value. + """ self._ensure_connected() - msg_id = message.get("id", secrets.token_hex(8)) - message["id"] = msg_id + if add_id: + msg_id = message.get("id", secrets.token_hex(8)) + message["id"] = msg_id + else: + msg_id = secrets.token_hex(8) # record key only; never written into the value body = json.dumps(message, default=str) if self._use_confluent: @@ -233,12 +252,19 @@ def _connect_confluent(self): "bootstrap.servers": self._brokers, "client.id": self._client_id, "group.id": self._group_id, - "auto.offset.reset": "earliest", + "auto.offset.reset": self._offset_reset(), "enable.auto.commit": False, **security, } self._consumer = self._confluent.Consumer(consumer_conf) + @staticmethod + def _offset_reset() -> str: + """Consumer auto.offset.reset from env: TINA4_KAFKA_AUTO_OFFSET_RESET, + then bare KAFKA_AUTO_OFFSET_RESET, default 'earliest' (#263).""" + return (os.environ.get("TINA4_KAFKA_AUTO_OFFSET_RESET") + or os.environ.get("KAFKA_AUTO_OFFSET_RESET") or "earliest") + @staticmethod def _security_config() -> dict: """Build SSL/SASL client config from env (for a TLS broker/proxy). diff --git a/uv.lock b/uv.lock index adcef017..a3d22495 100644 --- a/uv.lock +++ b/uv.lock @@ -1599,7 +1599,7 @@ wheels = [ [[package]] name = "tina4-python" -version = "3.13.155" +version = "3.13.156" source = { editable = "." } [package.optional-dependencies]