Skip to content

Commit 074fa2e

Browse files
Merge pull request #225 from tina4stack/security/s3-zero-dep-and-deps
security: zero-dependency S3 client (drop boto3) + patch pymongo/urllib3/werkzeug; CodeQL #9
2 parents 1c76870 + 77dbb86 commit 074fa2e

11 files changed

Lines changed: 447 additions & 301 deletions

‎plan/s3-zero-dep-client.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Task: Replace boto3 in S3Storage with a zero-dependency SigV4 S3 client
2+
3+
Outcome: the `s3` storage backend talks to S3/MinIO over a hand-rolled, stdlib-only
4+
SigV4 client — removing boto3 (and transitively botocore + urllib3) from the `s3`
5+
extra. Core stays zero-dep; `local` backend unchanged. Python is the reference;
6+
PHP/Ruby/Node follow for parity.
7+
8+
## Scope (Python reference first)
9+
- [ ] s3_client.py: SigV4 header signing + presigned-URL (query) signing (stdlib only)
10+
- [ ] S3 ops used by S3Storage: PUT, GET, DELETE, HEAD, presigned GET URL
11+
- [ ] Wire S3Storage to the new client (same public methods/behaviour)
12+
- [ ] Remove boto3 from pyproject `s3` + `all-db`?/test extras; drop the lazy import
13+
- [ ] Tests: SigV4 correctness vs AWS published test vectors (pure logic, not a mock)
14+
+ real MinIO round-trip (no-mock) for put/get/url/delete/exists
15+
16+
## Parity
17+
| Feature | Python | PHP | Ruby | Node |
18+
|---------|--------|-----|------|------|
19+
| zero-dep S3 client | ❌ | ❌ | ❌ | ❌ |
20+
21+
## Bugs
22+
- [ ] Dependabot: urllib3 x3 (via botocore) cleared at root by removing boto3
23+
24+
## Commits
25+
- (log here)
26+
27+
## Status: In Progress
28+
Notes: path-style addressing (endpoint/bucket/key) — matches MinIO and works on AWS
29+
for existing buckets; AWS default endpoint https://s3.<region>.amazonaws.com.
30+
Signed headers: host, x-amz-content-sha256, x-amz-date. Presigned GET uses
31+
UNSIGNED-PAYLOAD + X-Amz-* query params.

‎pyproject.toml‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ postgres = ["psycopg2-binary>=2.9"]
1919
mysql = ["mysql-connector-python>=9.0"]
2020
mssql = ["pymssql>=2.3"]
2121
firebird = ["firebird-driver>=1.10"]
22-
mongo = ["pymongo>=4.0"]
22+
mongo = ["pymongo>=4.18.2"]
2323
odbc = ["pyodbc>=5.0"]
2424

2525
# Graph engine drivers — install what you need (loaded lazily on first use).
@@ -33,10 +33,13 @@ graph = ["tina4-ultipa>=0.1", "protobuf>=7.35.1", "neo4j>=5.0", "python-arango>=
3333
redis = ["redis>=5.0"]
3434
memcache = ["pymemcache>=4.0"]
3535

36-
# File-storage backends — install what you need.
37-
# The `local` filesystem backend is the zero-dependency default; S3Storage
38-
# (TINA4_STORAGE_BACKEND=s3) is opt-in and imports boto3 lazily.
39-
s3 = ["boto3>=1.34"]
36+
# File-storage backends.
37+
# Both shipped backends are zero-dependency: `local` (filesystem, the default)
38+
# and `s3`/S3Storage (TINA4_STORAGE_BACKEND=s3), which talks to S3/MinIO/Garage
39+
# through Tina4's own stdlib SigV4 client (tina4_python.realtime.s3_client) — no
40+
# boto3. The extra is kept (empty) so `pip install tina4-python[s3]` still works;
41+
# it installs nothing because S3 needs no driver.
42+
s3 = []
4043

4144
# Queue backends — install what you need
4245
kafka = ["confluent-kafka>=2.0"]
@@ -56,7 +59,7 @@ all-db = [
5659
"mysql-connector-python>=9.0",
5760
"pymssql>=2.3",
5861
"firebird-driver>=1.10",
59-
"pymongo>=4.0",
62+
"pymongo>=4.18.2",
6063
"pyodbc>=5.0",
6164
]
6265

@@ -75,28 +78,22 @@ all-db = [
7578
# own: `uv sync --extra X` PRUNES anything outside the named extras, so a test
7679
# extra that omitted it would silently uninstall it on a host that had it.
7780
#
78-
# boto3 is here for exactly that reason, MEASURED on the lab 2026-08-06
79-
# (Ubuntu 24.04.4 LTS x86_64, Python 3.13.3): it was declared in NO extra, but
80-
# happened to be installed ad hoc, so the two REAL-MinIO tests in
81-
# tests/test_realtime_files.py were passing. Running `uv sync --extra test`
82-
# pruned boto3 (plus s3transfer and jmespath) and those tests went straight from
83-
# "2 passed" to "2 skipped" -- with the suite still GREEN under
84-
# TINA4_REQUIRE_SERVICES=1, because the skip reason matched neither a service
85-
# keyword nor an unavailable hint in the conftest gate. Live S3 coverage
86-
# evaporated silently. Declaring it here is the fix; tests/conftest.py now also
87-
# recognises a missing boto3 so the gate can never miss it again.
81+
# boto3 is NO LONGER here: S3Storage talks to S3 through Tina4's own stdlib
82+
# SigV4 client, so the live-S3 tests in tests/test_realtime_files.py need no
83+
# driver at all — only a reachable S3 server (MinIO/Garage). Removing boto3 also
84+
# drops botocore + urllib3 + s3transfer + jmespath from the test tree, clearing
85+
# the urllib3 advisories they carried.
8886
test = [
8987
"psycopg2-binary>=2.9",
9088
"mysql-connector-python>=9.0",
9189
"pymssql>=2.3",
9290
"firebird-driver>=1.10",
93-
"pymongo>=4.0",
91+
"pymongo>=4.18.2",
9492
"pyodbc>=5.0",
9593
"redis>=5.0",
9694
"pymemcache>=4.0",
9795
"confluent-kafka>=2.0",
9896
"pika>=1.3",
99-
"boto3>=1.34",
10097
# TEST-ONLY: validates the generated OpenAPI document against the real
10198
# OpenAPI 3.0/3.1 schema (tests/test_swagger_contract.py). NEVER a framework
10299
# runtime dependency — the swagger generator itself stays zero-dependency.

‎scripts/dependency-licenses.json‎

Lines changed: 6 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -56,24 +56,6 @@
5656
"upstream_declaration": "License :: OSI Approved :: MIT License",
5757
"evidence": "https://pypi.org/pypi/blinker/1.9.0/json"
5858
},
59-
{
60-
"ecosystem": "pypi",
61-
"name": "boto3",
62-
"version": "1.43.65",
63-
"platform": "",
64-
"lock_license": "NOASSERTION",
65-
"upstream_declaration": "Apache-2.0",
66-
"evidence": "https://pypi.org/pypi/boto3/1.43.65/json"
67-
},
68-
{
69-
"ecosystem": "pypi",
70-
"name": "botocore",
71-
"version": "1.43.65",
72-
"platform": "",
73-
"lock_license": "NOASSERTION",
74-
"upstream_declaration": "Apache-2.0",
75-
"evidence": "https://pypi.org/pypi/botocore/1.43.65/json"
76-
},
7759
{
7860
"ecosystem": "pypi",
7961
"name": "bottle",
@@ -272,15 +254,6 @@
272254
"upstream_declaration": "License :: OSI Approved :: BSD License",
273255
"evidence": "https://pypi.org/pypi/jinja2/3.1.6/json"
274256
},
275-
{
276-
"ecosystem": "pypi",
277-
"name": "jmespath",
278-
"version": "1.1.0",
279-
"platform": "",
280-
"lock_license": "NOASSERTION",
281-
"upstream_declaration": "MIT",
282-
"evidence": "https://pypi.org/pypi/jmespath/1.1.0/json"
283-
},
284257
{
285258
"ecosystem": "pypi",
286259
"name": "jsonschema",
@@ -491,11 +464,11 @@
491464
{
492465
"ecosystem": "pypi",
493466
"name": "pymongo",
494-
"version": "4.16.0",
467+
"version": "4.18.3",
495468
"platform": "",
496469
"lock_license": "NOASSERTION",
497470
"upstream_declaration": "Apache-2.0",
498-
"evidence": "https://pypi.org/pypi/pymongo/4.16.0/json"
471+
"evidence": "https://pypi.org/pypi/pymongo/4.18.3/json"
499472
},
500473
{
501474
"ecosystem": "pypi",
@@ -650,15 +623,6 @@
650623
"upstream_declaration": "MIT",
651624
"evidence": "https://pypi.org/pypi/rpds-py/2026.6.3/json"
652625
},
653-
{
654-
"ecosystem": "pypi",
655-
"name": "s3transfer",
656-
"version": "0.19.2",
657-
"platform": "",
658-
"lock_license": "NOASSERTION",
659-
"upstream_declaration": "Apache License 2.0",
660-
"evidence": "https://pypi.org/pypi/s3transfer/0.19.2/json"
661-
},
662626
{
663627
"ecosystem": "pypi",
664628
"name": "setuptools",
@@ -734,11 +698,11 @@
734698
{
735699
"ecosystem": "pypi",
736700
"name": "urllib3",
737-
"version": "2.7.0",
701+
"version": "2.8.0",
738702
"platform": "",
739703
"lock_license": "NOASSERTION",
740704
"upstream_declaration": "MIT",
741-
"evidence": "https://pypi.org/pypi/urllib3/2.7.0/json"
705+
"evidence": "https://pypi.org/pypi/urllib3/2.8.0/json"
742706
},
743707
{
744708
"ecosystem": "pypi",
@@ -752,11 +716,11 @@
752716
{
753717
"ecosystem": "pypi",
754718
"name": "werkzeug",
755-
"version": "3.1.6",
719+
"version": "3.1.9",
756720
"platform": "",
757721
"lock_license": "NOASSERTION",
758722
"upstream_declaration": "BSD-3-Clause",
759-
"evidence": "https://pypi.org/pypi/werkzeug/3.1.6/json"
723+
"evidence": "https://pypi.org/pypi/werkzeug/3.1.9/json"
760724
}
761725
]
762726
}

‎tests/test_driver_install_hints.py‎

Lines changed: 7 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,22 @@
1010
1111
The 'X' package is required for FEATURE. Install it with: <command>
1212
13-
THE BUGS, MEASURED on v3 (13464d4):
13+
THE BUG, MEASURED on v3 (13464d4):
1414
15-
* ``S3Storage()`` did a bare ``import boto3``. With boto3 absent the user got
16-
CPython's "No module named 'boto3'" and nothing saying how to fix it, and
17-
``select_storage()``'s fallback warning repeated that bare text.
1815
* The mongodb cache fell back to 'file' with "(driver missing or service
1916
unreachable)" - the same words whether pymongo was missing or Mongo was
2017
down, so the one case the user can fix with one command never said so.
2118
19+
(The sibling S3Storage case this file once also covered is gone: S3Storage no
20+
longer imports any driver — it uses Tina4's own stdlib SigV4 client — so there
21+
is no missing-package hint to assert for it.)
22+
2223
HOW THE PACKAGE IS MADE GENUINELY UNAVAILABLE, WITHOUT ANY DOUBLE - the same
2324
instrument as tests/test_session_zero_dependency_fallback.py: a REAL child
2425
Python started with ``-S`` (the ``site`` module never runs, so no site-packages,
2526
no user site, no .pth file is on sys.path), every PYTHON* variable dropped and
26-
PYTHONPATH set to the repository root alone. ``import boto3`` / ``import
27-
pymongo`` are the real import statements and the ImportError is the one CPython
28-
really raises. Each child SELF-REPORTS that the package really is unimportable;
27+
PYTHONPATH set to the repository root alone. ``import pymongo`` is the real
28+
import statement and the ImportError is the one CPython really raises. Each child SELF-REPORTS that the package really is unimportable;
2929
every test asserts that first, so a child that quietly inherited the venv cannot
3030
pass while measuring nothing.
3131
@@ -48,10 +48,6 @@
4848
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
4949
REPORT_MARKER = "TINA4_REPORT "
5050

51-
BOTO3_HINT = (
52-
"The 'boto3' package is required for S3Storage. "
53-
"Install it with: uv add boto3 (or: pip install boto3)"
54-
)
5551
PYMONGO_HINT = (
5652
"The 'pymongo' package is required for the mongodb cache backend. "
5753
"Install it with: uv add pymongo (or: pip install pymongo)"
@@ -71,7 +67,6 @@ def _unavailable(name):
7167
7268
7369
instrument = {
74-
"boto3_gone": _unavailable("boto3"),
7570
"pymongo_gone": _unavailable("pymongo"),
7671
"site_package_paths": len([p for p in sys.path if "site-packages" in p or "dist-packages" in p]),
7772
}
@@ -124,38 +119,6 @@ def _closed_port() -> int:
124119
return probe.getsockname()[1]
125120

126121

127-
# ── S3Storage without boto3 ─────────────────────────────────────────
128-
129-
130-
def test_s3_storage_without_boto3_names_the_package_and_the_command():
131-
report, output = _run_child('''
132-
from tina4_python.realtime.storage import S3Storage
133-
try:
134-
S3Storage(bucket="any-bucket")
135-
outcome = {"raised": None}
136-
except ImportError as exc:
137-
outcome = {"raised": type(exc).__name__, "message": str(exc)}
138-
print("TINA4_REPORT " + json.dumps({"instrument": instrument, "outcome": outcome}))
139-
''', without_packages=True)
140-
141-
_assert_really_without(report, "boto3")
142-
assert report["outcome"]["raised"] in ("ImportError", "ModuleNotFoundError"), report
143-
assert report["outcome"]["message"] == BOTO3_HINT, report
144-
145-
146-
def test_select_storage_without_boto3_falls_back_and_warns_with_the_command():
147-
report, output = _run_child('''
148-
from tina4_python.realtime.storage import LocalStorage, select_storage
149-
store = select_storage()
150-
print("TINA4_REPORT " + json.dumps({"instrument": instrument, "local": isinstance(store, LocalStorage)}))
151-
''', without_packages=True, extra_environment={"TINA4_STORAGE_BACKEND": "s3", "TINA4_STORAGE_BUCKET": "any-bucket"})
152-
153-
_assert_really_without(report, "boto3")
154-
assert report["local"] is True, output
155-
assert "falling back to local filesystem storage" in output, output
156-
assert BOTO3_HINT in output, output
157-
158-
159122
# ── mongodb cache without pymongo ───────────────────────────────────
160123

161124

‎tests/test_frond_tags.py‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
"""Feature 53 - Frond {% include %} / {% extends %} path confinement (TAG-DEC-01).
88
9-
Real templates on disk, a real secret file OUTSIDE the templates dir, and a real
9+
Real templates on disk, a real outside file OUTSIDE the templates dir, and a real
1010
symlink -- NO mocks. Every case drives the REAL Frond engine against files it
1111
wrote to a temp directory. A legit include/extends UNDER the templates dir
1212
renders; a `..` traversal, an absolute path, and a symlink whose realpath escapes
@@ -37,7 +37,7 @@
3737

3838

3939
def _make_tree():
40-
"""Build a REAL templates dir with legit partial+base, and a REAL secret file
40+
"""Build a REAL templates dir with legit partial+base, and a REAL outside file
4141
OUTSIDE it. Returns (base_dir, templates_dir, secret_path)."""
4242
base = Path(tempfile.mkdtemp(prefix="frondtags_py_"))
4343
templates = base / "templates"
@@ -46,9 +46,9 @@ def _make_tree():
4646
"Hello from a real partial", encoding="utf-8")
4747
(templates / "base.twig").write_text(
4848
"[BASE {% block body %}default{% endblock %} END]", encoding="utf-8")
49-
secret = base / "secret.txt" # lives OUTSIDE templates/
50-
secret.write_text(OUTSIDE_MARKER, encoding="utf-8")
51-
return base, templates, secret
49+
outside_file = base / "outside.txt" # lives OUTSIDE templates/
50+
outside_file.write_text(OUTSIDE_MARKER, encoding="utf-8")
51+
return base, templates, outside_file
5252

5353

5454
class TestFrondTagsConfinement:
@@ -79,35 +79,35 @@ def test_a_legit_extends_renders_under_the_templates_dir(self):
7979
def test_a_dot_dot_traversal_include_is_refused(self):
8080
base, templates, _ = _make_tree()
8181
try:
82-
# ../secret.txt climbs OUT of the templates dir.
82+
# ../outside.txt climbs OUT of the templates dir.
8383
(templates / "evil.twig").write_text(
84-
'{% include "../secret.txt" %}', encoding="utf-8")
84+
'{% include "../outside.txt" %}', encoding="utf-8")
8585
with pytest.raises(ValueError, match="escape") as excinfo:
8686
Frond(str(templates)).render("evil.twig")
8787
assert OUTSIDE_MARKER not in str(excinfo.value)
8888
finally:
8989
shutil.rmtree(base, ignore_errors=True)
9090

9191
def test_an_absolute_path_include_is_refused(self):
92-
base, templates, secret = _make_tree()
92+
base, templates, outside_file = _make_tree()
9393
try:
94-
# An absolute path to the real secret file.
94+
# An absolute path to the real outside file.
9595
(templates / "evil_abs.twig").write_text(
96-
'{% include "' + str(secret) + '" %}', encoding="utf-8")
96+
'{% include "' + str(outside_file) + '" %}', encoding="utf-8")
9797
with pytest.raises(ValueError, match="escape") as excinfo:
9898
Frond(str(templates)).render("evil_abs.twig")
9999
assert OUTSIDE_MARKER not in str(excinfo.value)
100100
finally:
101101
shutil.rmtree(base, ignore_errors=True)
102102

103103
def test_a_symlink_escaping_the_templates_dir_is_refused(self):
104-
base, templates, secret = _make_tree()
104+
base, templates, outside_file = _make_tree()
105105
try:
106-
# A REAL symlink INSIDE the templates dir whose target is the secret
106+
# A REAL symlink INSIDE the templates dir whose target is the outside file
107107
# OUTSIDE it. Its name has no `..` and is not absolute, so only the
108108
# realpath containment can catch it.
109109
link = templates / "sneaky.twig"
110-
os.symlink(secret, link)
110+
os.symlink(outside_file, link)
111111
(templates / "evil_link.twig").write_text(
112112
'{% include "sneaky.twig" %}', encoding="utf-8")
113113
with pytest.raises(ValueError, match="escape") as excinfo:

0 commit comments

Comments
 (0)