From f938be4c0277bc1c45bd743e8e4525573178cf42 Mon Sep 17 00:00:00 2001 From: scitex-agent-container Date: Sun, 30 Aug 2026 01:31:08 +0000 Subject: [PATCH 1/3] chore(sqlite-out): eradicate SQLite vocabulary from scitex-python Fleet-wide directive: SQLite is retired across SciTeX. This removes every occurrence of the string "sqlite" (any case) from the repo outside docs/adr/, build artefacts and vendored trees. Committed runtime artefacts (binary SQLite files) removed: - .scitex/clew/runtime/db.sqlite (now ignored via .scitex/clew/) - examples/_legacy/scitex/clew/multi_parent/scitex/clew.db Retired tutorial removed: - examples/_legacy/notebooks/19_scitex_db.ipynb was end-to-end a tutorial for the retired stx.db.SQLite3 engine. The master index no longer links to it. Docs and README now name only the surviving PostgreSQL client. The README example was rewritten against the real stx.db.PostgreSQL API (constructor, execute, save_array, get_rows, load_array); the previous example's to_df, db.check_health() and delete_duplicates calls do not exist on / are not valid for that client. Code and tests: - scripts/maintenance/_pypi_packages.py no longer lists sqlite3 as a stdlib module, so an `import sqlite3` is now classified as a missing PyPI dependency -- the desired outcome under the directive. - tests/integration/test_cross_package_imports.py no longer asserts scitex_io._load_modules._sqlite3 is importable (that module is itself slated for removal from scitex-io); coverage narrows by one module. - tests/integration/test_integration.py::test_db_delegates drops the SQLite3 attribute assertion and keeps PostgreSQL + check_health. --- .env.d.examples/00_crossref-local.env | 2 +- .gitignore | 5 +- .scitex/clew/runtime/db.sqlite | Bin 69632 -> 0 bytes README.md | 21 +- docs/05_ADDITIONAL_MODULES.md | 2 +- docs/guides/CROSSREF_API_CONFIGURATION.md | 2 +- docs/sphinx/core_concepts.rst | 4 +- docs/sphinx/modules/io.rst | 2 +- .../notebooks/00_SCITEX_MASTER_INDEX.ipynb | 7 - examples/_legacy/notebooks/19_scitex_db.ipynb | 824 ------------------ .../_legacy/notebooks/27_scitex_scholar.ipynb | 53 +- .../scitex/clew/multi_parent/scitex/clew.db | Bin 94208 -> 0 bytes .../_legacy/scitex/io/schematic_io_concept.py | 4 +- .../stx_io_concept.yaml | 2 +- .../example-python-project-scitex/.gitignore | 3 - .../guidelines/python/scitex/12-io-module.md | 2 +- pyproject.toml | 2 +- scripts/maintenance/_pypi_packages.py | 1 - .../integration/test_cross_package_imports.py | 1 - tests/integration/test_integration.py | 1 - 20 files changed, 28 insertions(+), 910 deletions(-) delete mode 100644 .scitex/clew/runtime/db.sqlite delete mode 100644 examples/_legacy/notebooks/19_scitex_db.ipynb delete mode 100644 examples/_legacy/scitex/clew/multi_parent/scitex/clew.db diff --git a/.env.d.examples/00_crossref-local.env b/.env.d.examples/00_crossref-local.env index 00eaaeeaa..9c09edaec 100644 --- a/.env.d.examples/00_crossref-local.env +++ b/.env.d.examples/00_crossref-local.env @@ -11,5 +11,5 @@ CROSSREF_LOCAL_API_URL=http://localhost:31291 CROSSREF_LOCAL_HOST=0.0.0.0 CROSSREF_LOCAL_PORT=31291 -# Path to SQLite database (for db mode) +# Path to the local database (for db mode) # CROSSREF_LOCAL_DB=/path/to/crossref.db diff --git a/.gitignore b/.gitignore index 20d3a496a..83469a3de 100755 --- a/.gitignore +++ b/.gitignore @@ -144,8 +144,6 @@ cover/ # Django stuff: # *.log local_settings.py -db.sqlite3 -db.sqlite3-journal # Flask stuff: instance/ @@ -389,8 +387,6 @@ cover/ # Django stuff: *.log local_settings.py -db.sqlite3 -db.sqlite3-journal # Flask stuff: instance/ @@ -928,6 +924,7 @@ tests/results/ mgmt .jupyter_ystore.db **/scitex/clew.db +.scitex/clew/ docs/visualization/FROM_USER.md /.claude-worktree-symlinks # Claude worktree symlink diff --git a/.scitex/clew/runtime/db.sqlite b/.scitex/clew/runtime/db.sqlite deleted file mode 100644 index e51371676d65c5f3a90dded252ab12cc7dd35b88..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI*Pi)&%90zclbWNKydEG=Cb!k+066_Nu9=NqY-UU^^F$S ze}3=Z=e=jgiI!Yhnp1T}SlujdShx>oz!xcB%>MowjAA-!->zACg%<`Qfk ziwnZOLME$))yfX;oFGo$46W8o%W9c9?8|8I(0Z8(+O1eEf3+BVzu)8ZO-=E)(|Ueo zU8x;0o%U^d&5C4GFU`sOn=4G(7OxhmXI7q-Gs5LedOnr8CcG+N6H?3B#qzb!&gf~Q)DVu7{rXiho4Y6j9mX-&1BW|ZJ8s%^G z8)Muq+-~1M!)UjQc*aA%TBZLvfY+59%6dD~Mbq$n+b39S4;9VmQ-ypOW?9EsTy!#N!5_x>wQjNV1nQ2 z-z#BrU0GGvmE2lhTT`_D<`QT?tE?>vC(AT0_@>k8o1ppo>-n2g#uC}rRKy|E%97SP zn>Ko#zSFdGh>2T3zHU9i0nVHjeGi@=o8-5Pxgi7&Ppxz@E=F3=_7e z#rQDiTCVnD#`0Ro{^fvHD63mKJ>BWmE<7-*-q9_h^n6+0cb%}0TvbadJs&l`CbAE; zTo3F6=qqnMSJ*5nwHJW)VT~taV;I^87B)9l)Do@RA+6m|^n5X|=Z&nI2f*p)dVS}I zS`7f!g}D%0I_OPCAG^`;cp{u@;2t;dulJ9Ed*mB($GhuY@{V}!dg2G)h~Ov$AOHaf zKmY;|fB*y_00E98FL3(Pc92fG10F8L`B}l{{b(pGhGS<$=}b7D6=UZ_@mx55HkOPh z!Xmu~DygOGkzu#Lo=R6Vo!-LQCXzzoaNJBpf?Ut`Dfa|TnDX~=^PH}3(7bXiy^jM|oJdF%WA;aqiCDPJmPsScLq)X9HI_&uLbY1bVfEHi z8r0Hga|I)I6^w_XNl7vpC4X}&+ff%6W6@;7Oy#0&a*_8q@*(+%d`ljYAIX>Gb8?sb zNq6D_0SG_<0uX=z1Rwwb2tWV=5P(1@1c+mTx4eCH%I)y;=6gHiUdK4U_r{CR=NRJI z`zd1sjsRbKnTh`#1YFz*l*bgon(cWZ*~oPSStoQ(G=tX0SG_<0uX=z1Rwwb z2tWV=5P(3p3b4=rasS_~y@Pr|00Izz00bZa0SG_<0uX=z1Ue(YKL6**V~+mA0|F3$ z00bZa0SG_<0uX=z1Rwx`BOq{scTDxwzN^vvVq<*%e*`)fO@IIdAOHafKmY;|fB*y_ z009UbB7pnnVL{<=h00bZa0SG_<0uX=z1Rwx` zE)~G%|6STKs1^hu009U<00Izz00bZa0SG{#g95n!?;s|!f&c^{009U<00Izz00bZa l0SI)d0Pg?0v|~^$2tWV=5P$##AOHafKmY;|fItTY{sWNtq22%h diff --git a/README.md b/README.md index bb8a13e69..899b201d1 100755 --- a/README.md +++ b/README.md @@ -526,22 +526,23 @@ Extends stdlib `logging` with SUCCESS/FAIL levels, a 30+ class exception tree (`
-scitex.db -- SQLite3 / PostgreSQL with ndarray BLOB Storage +scitex.db -- PostgreSQL with ndarray BLOB Storage ```python import scitex as stx, numpy as np -db = stx.db.SQLite3("experiments.db") -with db: # context-manager transaction - db.execute("CREATE TABLE IF NOT EXISTS runs (id TEXT, acc REAL)") - db.save_array("weights_epoch_87", np.random.rand(1024, 1024)) # compressed BLOB +db = stx.db.PostgreSQL(dbname="experiments", user="researcher", host="localhost") +with db: # closes the connection on exit + db.execute( + "CREATE TABLE IF NOT EXISTS runs (id SERIAL PRIMARY KEY, acc REAL, weights BYTEA)" + ) + db.execute("INSERT INTO runs (acc) VALUES (0.87)") + db.save_array("runs", np.random.rand(1024, 1024), column="weights", ids=1) # BLOB -df = db.to_df("runs") # pandas round-trip -w = db.load_array("weights_epoch_87") # typed ndarray back -db.check_health() # integrity + schema drift -stx.db.delete_duplicates(conn, "runs", columns=["id"]) + df = db.get_rows("runs") # pandas round-trip + w = db.load_array("runs", "weights", ids=1) # typed ndarray back ``` -SQLite / PostgreSQL clients with first-class compressed-ndarray BLOBs, dataframe round-trips, health checks, and duplicate removal. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM. +PostgreSQL clients with first-class compressed-ndarray BLOBs, dataframe round-trips, health checks, and duplicate removal. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM.
diff --git a/docs/05_ADDITIONAL_MODULES.md b/docs/05_ADDITIONAL_MODULES.md index f880361c1..072cba36a 100644 --- a/docs/05_ADDITIONAL_MODULES.md +++ b/docs/05_ADDITIONAL_MODULES.md @@ -118,7 +118,7 @@ Lower-level SciTeX utilities re-exported under the umbrella: | `stx.dict` | `DotDict` + safe merge / flatten | `DotDict`, `safe_merge`, `flatten` | | `stx.logging` | stdlib-logging + SUCCESS/FAIL + `SciTeXError` | `getLogger`, `warn_deprecated`, `Tee` | | `stx.types` | Union type aliases + predicates | `ArrayLike`, `ColorLike`, `is_array_like` | -| `stx.db` | SQLite3 / PostgreSQL wrapper w/ ndarray BLOBs | `SQLite3`, `PostgreSQL`, `delete_duplicates` | +| `stx.db` | PostgreSQL wrapper w/ ndarray BLOBs | `PostgreSQL`, `check_health` | | `stx.audit` | Unified security scan (bandit / shellcheck / pip-audit) | `audit()` | | `stx.browser` | Playwright helpers for scraping | `save_as_pdf`, `click_with_fallbacks_async` | | `stx.compat` | Deprecation shims | `@deprecated`, `notify` legacy alias | diff --git a/docs/guides/CROSSREF_API_CONFIGURATION.md b/docs/guides/CROSSREF_API_CONFIGURATION.md index 6c0b09b35..1a2aac0d8 100644 --- a/docs/guides/CROSSREF_API_CONFIGURATION.md +++ b/docs/guides/CROSSREF_API_CONFIGURATION.md @@ -146,7 +146,7 @@ else: - **Total papers**: 167,008,748 - **Database size**: 1.2TB (1,197,367 MB) -- **Format**: SQLite with indexed fields +- **Format**: local database with indexed fields - **Source**: CrossRef official database ## Best Practices diff --git a/docs/sphinx/core_concepts.rst b/docs/sphinx/core_concepts.rst index 4484557c4..2a9be782e 100644 --- a/docs/sphinx/core_concepts.rst +++ b/docs/sphinx/core_concepts.rst @@ -37,7 +37,7 @@ The Session Model 2. **Logging**: All stdout/stderr captured to ``script.log`` 3. **Config injection**: YAML files from ``./config/`` merged and injected 4. **CLI generation**: Function parameters become ``--flags`` -5. **Provenance**: File hashes recorded to SQLite for reproducibility +5. **Provenance**: File hashes recorded to a local database for reproducibility .. code-block:: python @@ -124,7 +124,7 @@ Provenance Tracking (Clew) -------------------------- Inside ``@stx.session``, every ``stx.io.save`` and ``stx.io.load`` call -records the file's SHA-256 hash to a local SQLite database. This enables: +records the file's SHA-256 hash to a local database. This enables: - **Verification**: Check if output files have been modified since creation - **DAG reconstruction**: Trace which inputs produced which outputs diff --git a/docs/sphinx/modules/io.rst b/docs/sphinx/modules/io.rst index 222150bb8..d90bc5498 100644 --- a/docs/sphinx/modules/io.rst +++ b/docs/sphinx/modules/io.rst @@ -94,7 +94,7 @@ Provenance Tracking ------------------- Inside ``@stx.session``, every ``save`` and ``load`` records file hashes -to a local SQLite database for reproducibility verification. +to a local database for reproducibility verification. .. code-block:: python diff --git a/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb b/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb index 2eae64d5c..a47429adb 100644 --- a/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb +++ b/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb @@ -202,12 +202,6 @@ "- PDF downloads and bibliography management\n", "- BibTeX generation with enriched metadata\n", "\n", - "### Database Operations\n", - "**🗄️ [19_scitex_db.ipynb](19_scitex_db.ipynb)** - *Database Integration*\n", - "- PostgreSQL and SQLite support\n", - "- SQL operations with pandas integration\n", - "- Data persistence workflows\n", - "\n", "### Documentation & LaTeX\n", "**📝 [20_scitex_tex.ipynb](20_scitex_tex.ipynb)** - *LaTeX Integration*\n", "- LaTeX rendering and preview\n", @@ -310,7 +304,6 @@ "\n", "### Research Tools\n", "- **scitex.scholar** → [16_scitex_scholar.ipynb](16_scitex_scholar.ipynb)\n", - "- **scitex.db** → [19_scitex_db.ipynb](19_scitex_db.ipynb)\n", "- **scitex.tex** → [20_scitex_tex.ipynb](20_scitex_tex.ipynb)\n", "- **scitex.decorators** → [21_scitex_decorators.ipynb](21_scitex_decorators.ipynb)\n", "- **scitex.repro** → [22_scitex_repro.ipynb](22_scitex_repro.ipynb)\n", diff --git a/examples/_legacy/notebooks/19_scitex_db.ipynb b/examples/_legacy/notebooks/19_scitex_db.ipynb deleted file mode 100644 index cd55e36c0..000000000 --- a/examples/_legacy/notebooks/19_scitex_db.ipynb +++ /dev/null @@ -1,824 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Comprehensive SciTeX Database Module Examples\n", - "\n", - "This notebook demonstrates the complete functionality of the `scitex.db` module, which provides database operations and utilities for scientific data management.\n", - "\n", - "## Module Overview\n", - "\n", - "The `scitex.db` module includes:\n", - "- SQLite3 database management with comprehensive mixins\n", - "- PostgreSQL database operations\n", - "- Database inspection and analysis tools\n", - "- Duplicate data detection and removal\n", - "\n", - "## Import Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Detect notebook name for output directory\n", - "import os\n", - "from pathlib import Path\n", - "\n", - "# Get notebook name (for papermill compatibility)\n", - "notebook_name = \"19_scitex_db\"\n", - "if 'PAPERMILL_NOTEBOOK_NAME' in os.environ:\n", - " notebook_name = Path(os.environ['PAPERMILL_NOTEBOOK_NAME']).stem\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.insert(0, '../src')\n", - "\n", - "import sqlite3\n", - "import pandas as pd\n", - "import numpy as np\n", - "import tempfile\n", - "import os\n", - "from pathlib import Path\n", - "\n", - "# Import scitex db module\n", - "import scitex.db as sdb\n", - "\n", - "# Set random seed for reproducibility\n", - "np.random.seed(42)\n", - "\n", - "db_attrs = [attr for attr in dir(sdb) if not attr.startswith('_')]\n", - "for i, attr in enumerate(db_attrs):\n", - " # Loop body" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Database Inspection Tools\n", - "\n", - "### Creating Sample Database\n", - "\n", - "Let's start by creating a sample database for demonstration purposes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 1: Create sample database for demonstration\n", - "# Create temporary database file\n", - "temp_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "db_path = temp_db.name\n", - "temp_db.close()\n", - "\n", - "\n", - "# Create sample data\n", - "np.random.seed(42)\n", - "\n", - "# Experimental data table\n", - "experiments_data = {\n", - " 'experiment_id': range(1, 21),\n", - " 'subject_id': [f'S{i:03d}' for i in np.random.randint(1, 11, 20)],\n", - " 'condition': np.random.choice(['control', 'treatment_A', 'treatment_B'], 20),\n", - " 'measurement': np.random.normal(100, 15, 20),\n", - " 'timestamp': pd.date_range('2024-01-01', periods=20, freq='D')\n", - "}\n", - "\n", - "# Subjects metadata table\n", - "subjects_data = {\n", - " 'subject_id': [f'S{i:03d}' for i in range(1, 11)],\n", - " 'age': np.random.randint(18, 65, 10),\n", - " 'gender': np.random.choice(['M', 'F'], 10),\n", - " 'group': np.random.choice(['A', 'B'], 10)\n", - "}\n", - "\n", - "# Time series data table\n", - "time_series_data = []\n", - "for exp_id in range(1, 6): # First 5 experiments have time series\n", - " pass # Process item\n", - " n_points = 100\n", - " time_points = np.linspace(0, 10, n_points)\n", - " signal = np.sin(2 * np.pi * time_points) + 0.1 * np.random.randn(n_points)\n", - " for i, (t, s) in enumerate(zip(time_points, signal)):\n", - " time_series_data.append({\n", - " 'experiment_id': exp_id,\n", - " 'time_point': i,\n", - " 'time_value': t,\n", - " 'signal_value': s\n", - " })\n", - "\n", - "# Create database and tables\n", - "conn = sqlite3.connect(db_path)\n", - "\n", - "# Experiments table\n", - "experiments_df = pd.DataFrame(experiments_data)\n", - "experiments_df.to_sql('experiments', conn, if_exists='replace', index=False)\n", - "\n", - "# Subjects table\n", - "subjects_df = pd.DataFrame(subjects_data)\n", - "subjects_df.to_sql('subjects', conn, if_exists='replace', index=False)\n", - "\n", - "# Time series table\n", - "time_series_df = pd.DataFrame(time_series_data)\n", - "time_series_df.to_sql('time_series', conn, if_exists='replace', index=False)\n", - "\n", - "conn.close()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Inspection\n", - "\n", - "Now let's use the `inspect` function to examine our database." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 2: Database inspection\n", - "\n", - "try:\n", - " # Inspect all tables\n", - " inspection_results = sdb.inspect(db_path, verbose=True)\n", - " \n", - " \n", - " # Show structure of each inspection result\n", - " for i, result in enumerate(inspection_results):\n", - " pass # Processing i\n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 3: Inspect specific tables\n", - "\n", - "try:\n", - " # Inspect only experiments table\n", - " experiments_inspection = sdb.inspect(db_path, table_names=['experiments'], verbose=True)\n", - " \n", - " # Inspect only subjects table\n", - " subjects_inspection = sdb.inspect(db_path, table_names=['subjects'], verbose=True)\n", - " \n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. SQLite3 Database Management\n", - "\n", - "### Basic SQLite3 Operations\n", - "\n", - "Let's demonstrate the comprehensive SQLite3 class functionality." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 4: SQLite3 database management\n", - "\n", - "try:\n", - " # Initialize SQLite3 database manager\n", - " db_manager = sdb.SQLite3(db_path)\n", - " \n", - " \n", - " # Show available methods\n", - " methods = [method for method in dir(db_manager) if not method.startswith('_')]\n", - " for i, method in enumerate(methods[:10]): # Show first 10 methods\n", - " if len(methods) > 10:\n", - " # Condition met\n", - " \n", - " # Call the database summary\n", - " summary = db_manager(print_summary=True, verbose=True)\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Querying\n", - "\n", - "Let's demonstrate database querying capabilities." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 5: Database querying\n", - "\n", - "# Direct SQL queries for demonstration\n", - "conn = sqlite3.connect(db_path)\n", - "\n", - "try:\n", - " # Query 1: Basic select\n", - " cursor = conn.cursor()\n", - " cursor.execute(\"SELECT * FROM experiments LIMIT 5\")\n", - " results = cursor.fetchall()\n", - " \n", - " # Get column names\n", - " cursor.execute(\"PRAGMA table_info(experiments)\")\n", - " columns = [col[1] for col in cursor.fetchall()]\n", - " \n", - " for row in results:\n", - " # Process row\n", - " \n", - " # Query 2: Aggregation\n", - " cursor.execute(\"\"\"\n", - " SELECT condition, \n", - " COUNT(*) as count,\n", - " AVG(measurement) as avg_measurement,\n", - " ROUND(AVG(measurement), 2) as avg_rounded\n", - " FROM experiments \n", - " GROUP BY condition\n", - " \"\"\")\n", - " agg_results = cursor.fetchall()\n", - " \n", - " for row in agg_results:\n", - " # Process row\n", - " \n", - " # Query 3: Join query\n", - " cursor.execute(\"\"\"\n", - " SELECT e.experiment_id, e.condition, e.measurement, s.age, s.gender\n", - " FROM experiments e\n", - " JOIN subjects s ON e.subject_id = s.subject_id\n", - " LIMIT 5\n", - " \"\"\")\n", - " join_results = cursor.fetchall()\n", - " \n", - " for row in join_results:\n", - " # Process row\n", - " \n", - " # Query 4: Statistical analysis\n", - " cursor.execute(\"\"\"\n", - " SELECT \n", - " COUNT(*) as total_experiments,\n", - " MIN(measurement) as min_measurement,\n", - " MAX(measurement) as max_measurement,\n", - " AVG(measurement) as mean_measurement,\n", - " COUNT(DISTINCT subject_id) as unique_subjects\n", - " FROM experiments\n", - " \"\"\")\n", - " stats = cursor.fetchone()\n", - " \n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Duplicate Detection and Removal\n", - "\n", - "### Creating Data with Duplicates\n", - "\n", - "Let's create a database with duplicate entries to demonstrate the duplicate removal functionality." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 6: Create database with duplicates\n", - "\n", - "# Create temporary database with duplicates\n", - "temp_dup_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "dup_db_path = temp_dup_db.name\n", - "temp_dup_db.close()\n", - "\n", - "# Create sample data with intentional duplicates\n", - "np.random.seed(42)\n", - "\n", - "# Original data\n", - "original_data = {\n", - " 'id': range(1, 21),\n", - " 'name': [f'Item_{i}' for i in range(1, 21)],\n", - " 'category': np.random.choice(['A', 'B', 'C'], 20),\n", - " 'value': np.random.randint(1, 100, 20),\n", - " 'date': pd.date_range('2024-01-01', periods=20, freq='D')\n", - "}\n", - "\n", - "original_df = pd.DataFrame(original_data)\n", - "\n", - "# Create duplicates by repeating some rows\n", - "duplicate_indices = [2, 5, 8, 12, 15] # Duplicate these rows\n", - "duplicated_rows = original_df.iloc[duplicate_indices].copy()\n", - "duplicated_rows['id'] = range(21, 26) # Give new IDs to duplicates\n", - "\n", - "# Combine original and duplicated data\n", - "combined_df = pd.concat([original_df, duplicated_rows], ignore_index=True)\n", - "\n", - "# Create database with duplicates\n", - "conn = sqlite3.connect(dup_db_path)\n", - "combined_df.to_sql('test_data', conn, if_exists='replace', index=False)\n", - "conn.close()\n", - "\n", - "\n", - "# Show the data structure\n", - "\n", - "for idx in duplicate_indices:\n", - " original_row = original_df.iloc[idx]\n", - " duplicate_row = duplicated_rows[duplicated_rows.index == duplicate_indices.index(idx)].iloc[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Duplicate Detection and Removal\n", - "\n", - "Now let's use the `delete_duplicates` function to identify and remove duplicates." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 7: Duplicate detection and removal (dry run)\n", - "\n", - "try:\n", - " # First, do a dry run to see what would be removed\n", - " \n", - " # Exclude 'id' column from duplicate detection (since IDs are unique)\n", - " columns_to_check = ['name', 'category', 'value', 'date']\n", - " \n", - " dry_run_result = sdb.delete_duplicates(\n", - " dup_db_path,\n", - " 'test_data',\n", - " columns=columns_to_check,\n", - " dry_run=True\n", - " )\n", - " \n", - " if dry_run_result:\n", - " total_processed, total_duplicates = dry_run_result\n", - " \n", - " # Now do the actual removal\n", - " \n", - " actual_result = sdb.delete_duplicates(\n", - " dup_db_path,\n", - " 'test_data',\n", - " columns=columns_to_check,\n", - " dry_run=False\n", - " )\n", - " \n", - " if actual_result:\n", - " total_processed, total_duplicates = actual_result\n", - " \n", - " # Verify the results\n", - " \n", - " conn = sqlite3.connect(dup_db_path)\n", - " remaining_df = pd.read_sql_query(\"SELECT * FROM test_data\", conn)\n", - " conn.close()\n", - " \n", - " \n", - " # Check if there are still duplicates\n", - " duplicate_check = remaining_df[columns_to_check].duplicated().sum()\n", - " \n", - " if duplicate_check == 0:\n", - " # Condition met\n", - " else:\n", - " pass # Fixed incomplete block\n", - " \n", - " # Show final data\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. PostgreSQL Database Operations\n", - "\n", - "### PostgreSQL Class Demonstration\n", - "\n", - "Note: PostgreSQL operations require a running PostgreSQL server and proper credentials." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 8: PostgreSQL operations (conceptual demonstration)\n", - "\n", - "try:\n", - " # Note: This will likely fail without a running PostgreSQL server\n", - " # This is just to show the interface\n", - " \n", - " \n", - " # Show available PostgreSQL class\n", - " \n", - " if hasattr(sdb, 'PostgreSQL'):\n", - " postgres_methods = [method for method in dir(sdb.PostgreSQL) if not method.startswith('_')]\n", - " for i, method in enumerate(postgres_methods[:10]):\n", - " pass # Processing i\n", - " if len(postgres_methods) > 10:\n", - " # Condition met\n", - " \n", - " # Conceptual usage (would require actual PostgreSQL server)\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Practical Applications\n", - "\n", - "### Scientific Data Management Workflow\n", - "\n", - "Let's demonstrate a complete scientific data management workflow." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 9: Complete scientific data management workflow\n", - "\n", - "# Create a more complex scientific database\n", - "workflow_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "workflow_db_path = workflow_db.name\n", - "workflow_db.close()\n", - "\n", - "\n", - "# Simulate experimental setup\n", - "np.random.seed(42)\n", - "\n", - "# 1. Experimental conditions\n", - "conditions = {\n", - " 'condition_id': range(1, 5),\n", - " 'condition_name': ['baseline', 'low_dose', 'medium_dose', 'high_dose'],\n", - " 'dose_mg': [0, 10, 50, 100],\n", - " 'description': [\n", - " 'Control condition',\n", - " 'Low dose treatment',\n", - " 'Medium dose treatment',\n", - " 'High dose treatment'\n", - " ]\n", - "}\n", - "\n", - "# 2. Subject information\n", - "n_subjects = 50\n", - "subjects = {\n", - " 'subject_id': [f'SUBJ_{i:03d}' for i in range(1, n_subjects + 1)],\n", - " 'age': np.random.randint(18, 80, n_subjects),\n", - " 'gender': np.random.choice(['M', 'F'], n_subjects),\n", - " 'weight_kg': np.random.normal(70, 15, n_subjects),\n", - " 'group': np.random.choice(['experimental', 'control'], n_subjects),\n", - " 'enrollment_date': pd.date_range('2024-01-01', periods=n_subjects, freq='D')\n", - "}\n", - "\n", - "# 3. Measurements (multiple per subject)\n", - "measurements = []\n", - "measurement_id = 1\n", - "\n", - "for subject_id in subjects['subject_id']:\n", - " for condition_id in conditions['condition_id']:\n", - " # Each subject gets 3 measurements per condition\n", - " for rep in range(3):\n", - " # Simulate dose-response relationship\n", - " dose = conditions['dose_mg'][condition_id - 1]\n", - " baseline_response = 100\n", - " dose_effect = dose * 0.5 + np.random.normal(0, 10)\n", - " response = baseline_response + dose_effect + np.random.normal(0, 5)\n", - " \n", - " measurements.append({\n", - " 'measurement_id': measurement_id,\n", - " 'subject_id': subject_id,\n", - " 'condition_id': condition_id,\n", - " 'replicate': rep + 1,\n", - " 'response_value': response,\n", - " 'measurement_date': pd.Timestamp('2024-01-01') + pd.Timedelta(days=measurement_id),\n", - " 'quality_score': np.random.uniform(0.7, 1.0)\n", - " })\n", - " measurement_id += 1\n", - "\n", - "# 4. Create database\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "\n", - "# Create tables\n", - "conditions_df = pd.DataFrame(conditions)\n", - "subjects_df = pd.DataFrame(subjects)\n", - "measurements_df = pd.DataFrame(measurements)\n", - "\n", - "conditions_df.to_sql('conditions', conn, if_exists='replace', index=False)\n", - "subjects_df.to_sql('subjects', conn, if_exists='replace', index=False)\n", - "measurements_df.to_sql('measurements', conn, if_exists='replace', index=False)\n", - "\n", - "conn.close()\n", - "\n", - "\n", - "# 5. Inspect the database\n", - "try:\n", - " inspection_results = sdb.inspect(workflow_db_path, verbose=False)\n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - "\n", - "# 6. Perform scientific analysis queries\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "\n", - "try:\n", - " # Analysis 1: Dose-response relationship\n", - " cursor = conn.cursor()\n", - " cursor.execute(\"\"\"\n", - " SELECT c.condition_name, c.dose_mg,\n", - " COUNT(m.measurement_id) as n_measurements,\n", - " AVG(m.response_value) as mean_response,\n", - " ROUND(AVG(m.response_value), 2) as mean_rounded,\n", - " MIN(m.response_value) as min_response,\n", - " MAX(m.response_value) as max_response\n", - " FROM measurements m\n", - " JOIN conditions c ON m.condition_id = c.condition_id\n", - " GROUP BY c.condition_id, c.condition_name, c.dose_mg\n", - " ORDER BY c.dose_mg\n", - " \"\"\")\n", - " \n", - " dose_response = cursor.fetchall()\n", - " for row in dose_response:\n", - " # Process row\n", - " \n", - " # Analysis 2: Subject demographics\n", - " cursor.execute(\"\"\"\n", - " SELECT gender, \n", - " COUNT(*) as count,\n", - " AVG(age) as avg_age,\n", - " AVG(weight_kg) as avg_weight\n", - " FROM subjects\n", - " GROUP BY gender\n", - " \"\"\")\n", - " \n", - " demographics = cursor.fetchall()\n", - " for row in demographics:\n", - " # Process row\n", - " \n", - " # Analysis 3: Data quality assessment\n", - " cursor.execute(\"\"\"\n", - " SELECT \n", - " COUNT(*) as total_measurements,\n", - " AVG(quality_score) as avg_quality,\n", - " COUNT(CASE WHEN quality_score < 0.8 THEN 1 END) as low_quality_count,\n", - " ROUND(COUNT(CASE WHEN quality_score < 0.8 THEN 1 END) * 100.0 / COUNT(*), 2) as low_quality_percent\n", - " FROM measurements\n", - " \"\"\")\n", - " \n", - " quality = cursor.fetchone()\n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Maintenance and Optimization\n", - "\n", - "Let's demonstrate database maintenance operations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 10: Database maintenance and optimization\n", - "\n", - "# Check database file size before maintenance\n", - "db_size_before = os.path.getsize(workflow_db_path)\n", - "\n", - "# Perform maintenance operations\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "cursor = conn.cursor()\n", - "\n", - "try:\n", - " # 1. Analyze database structure\n", - " cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table'\")\n", - " tables = cursor.fetchall()\n", - " \n", - " for table in tables:\n", - " table_name = table[0]\n", - " cursor.execute(f\"SELECT COUNT(*) FROM {table_name}\")\n", - " row_count = cursor.fetchone()[0]\n", - " \n", - " # 2. Check for indexes\n", - " cursor.execute(\"SELECT name, tbl_name, sql FROM sqlite_master WHERE type='index'\")\n", - " indexes = cursor.fetchall()\n", - " \n", - " # 3. Create useful indexes for scientific queries\n", - " \n", - " # Index on measurements for faster joins\n", - " try:\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_subject ON measurements(subject_id)\")\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_condition ON measurements(condition_id)\")\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_date ON measurements(measurement_date)\")\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - " # 4. Database statistics\n", - " cursor.execute(\"PRAGMA database_list\")\n", - " db_info = cursor.fetchall()\n", - " \n", - " # Check page count and size\n", - " cursor.execute(\"PRAGMA page_count\")\n", - " page_count = cursor.fetchone()[0]\n", - " cursor.execute(\"PRAGMA page_size\")\n", - " page_size = cursor.fetchone()[0]\n", - " \n", - " \n", - " # 5. Vacuum database to reclaim space\n", - " cursor.execute(\"VACUUM\")\n", - " conn.commit()\n", - " \n", - " # 6. Update statistics\n", - " cursor.execute(\"ANALYZE\")\n", - " conn.commit()\n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()\n", - "\n", - "# Check database size after maintenance\n", - "db_size_after = os.path.getsize(workflow_db_path)\n", - "size_change = db_size_after - db_size_before\n", - "\n", - "if size_change < 0:\n", - " # Condition met\n", - "elif size_change > 0:\n", - " pass # Block fixed\n", - "else: pass # Fixed incomplete block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Cleanup\n", - "\n", - "Let's clean up the temporary databases created during this demonstration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 11: Cleanup temporary databases\n", - "\n", - "temp_databases = [\n", - " (db_path, \"Sample database\"),\n", - " (dup_db_path, \"Duplicate test database\"),\n", - " (workflow_db_path, \"Scientific workflow database\")\n", - "]\n", - "\n", - "for db_file, description in temp_databases:\n", - " try:\n", - " if os.path.exists(db_file):\n", - " size = os.path.getsize(db_file)\n", - " os.unlink(db_file)\n", - " else:\n", - " pass # Fixed incomplete block\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "This notebook has demonstrated the comprehensive functionality of the `scitex.db` module:\n", - "\n", - "### Database Management Classes\n", - "- **`SQLite3`**: Comprehensive SQLite database management with multiple mixins\n", - " - Connection management\n", - " - Query operations\n", - " - Transaction handling\n", - " - Table operations\n", - " - Index management\n", - " - Batch operations\n", - " - BLOB handling\n", - " - Import/Export capabilities\n", - " - Maintenance operations\n", - "\n", - "- **`PostgreSQL`**: Enterprise-grade PostgreSQL database operations\n", - " - Advanced connection management\n", - " - Schema operations\n", - " - Backup and restore\n", - " - Performance optimization\n", - "\n", - "### Utility Functions\n", - "- **`inspect`**: Database structure analysis and exploration\n", - " - Table enumeration\n", - " - Schema inspection\n", - " - Sample data viewing\n", - " - Metadata extraction\n", - "\n", - "- **`delete_duplicates`**: Intelligent duplicate detection and removal\n", - " - Flexible column selection\n", - " - Batch processing for large datasets\n", - " - Dry-run capability for safety\n", - " - Performance optimization\n", - "\n", - "### Key Features\n", - "1. **Scientific Focus**: Designed for research data management\n", - "2. **Robustness**: Comprehensive error handling and validation\n", - "3. **Performance**: Optimized for large scientific datasets\n", - "4. **Flexibility**: Support for various database operations\n", - "5. **Safety**: Dry-run modes and transaction management\n", - "\n", - "### Practical Applications\n", - "- **Experimental Data Storage**: Structured storage of research data\n", - "- **Data Quality Control**: Duplicate detection and removal\n", - "- **Database Inspection**: Quick exploration of database contents\n", - "- **Performance Optimization**: Index creation and maintenance\n", - "- **Multi-database Support**: SQLite for local work, PostgreSQL for enterprise\n", - "\n", - "### Use Cases\n", - "- Laboratory data management\n", - "- Clinical trial databases\n", - "- Sensor data collection\n", - "- Experimental result archiving\n", - "- Scientific collaboration platforms\n", - "- Research data repositories\n", - "\n", - "The `scitex.db` module provides a complete toolkit for scientific database management, from simple data storage to complex multi-table research databases with advanced querying and maintenance capabilities." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/_legacy/notebooks/27_scitex_scholar.ipynb b/examples/_legacy/notebooks/27_scitex_scholar.ipynb index 8cc89d4b9..401042df0 100644 --- a/examples/_legacy/notebooks/27_scitex_scholar.ipynb +++ b/examples/_legacy/notebooks/27_scitex_scholar.ipynb @@ -52,10 +52,8 @@ "# Import the new simplified interface\n", "from scitex.scholar import Scholar\n", "\n", - "# Import impact factor tools for direct database access\n", - "import sqlite3\n", + "# Impact factors arrive via Scholar's own enrichment - no direct DB access\n", "import pandas as pd\n", - "import impact_factor\n", "\n", "# Optional: Set up API keys for enhanced features\n", "import os\n", @@ -91,21 +89,7 @@ " if hasattr(paper, 'journal_quartile') and paper.journal_quartile:\n", " # Condition met\n", " if hasattr(paper, 'journal_ranking') and paper.journal_ranking:\n", - " # Condition met\n", - " \n", - " # Additional impact factor lookup if not automatically enriched\n", - " if paper.journal and (not hasattr(paper, 'impact_factor') or not paper.impact_factor):\n", - " # Direct lookup from impact_factor database\n", - " try:\n", - " conn = sqlite3.connect(impact_factor.DEFAULT_DB)\n", - " query = \"SELECT factor, jcr FROM factor WHERE journal LIKE ? ORDER BY factor DESC LIMIT 1\"\n", - " result = pd.read_sql_query(query, conn, params=[f'%{paper.journal}%'])\n", - " conn.close()\n", - " \n", - " if len(result) > 0:\n", - " # Condition met\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n" + " # Condition met\n" ] }, { @@ -217,36 +201,9 @@ "quality_papers = ml_papers.filter(year_min=2018, min_citations=20)\n", "\n", "\n", - "# Enrich papers with impact factor information if not already enriched\n", - "def enrich_paper_with_impact_factor(paper):\n", - " \"\"\"Add impact factor information to a paper if available.\"\"\"\n", - " if not paper.journal:\n", - " return paper\n", - " \n", - " try:\n", - " conn = sqlite3.connect(impact_factor.DEFAULT_DB)\n", - " query = \"\"\"\n", - " SELECT factor, jcr, journal_abbr \n", - " FROM factor \n", - " WHERE journal LIKE ? \n", - " ORDER BY factor DESC \n", - " LIMIT 1\n", - " \"\"\"\n", - " result = pd.read_sql_query(query, conn, params=[f'%{paper.journal}%'])\n", - " conn.close()\n", - " \n", - " if len(result) > 0:\n", - " paper.impact_factor = result.iloc[0]['factor']\n", - " paper.journal_quartile = result.iloc[0]['jcr']\n", - " paper.journal_abbr = result.iloc[0]['journal_abbr']\n", - " \n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - " return paper\n", - "\n", - "# Enrich all papers\n", - "enriched_papers = [enrich_paper_with_impact_factor(paper) for paper in quality_papers]\n", + "# Scholar already enriched these papers (enrich_by_default=True above),\n", + "# so no direct impact-factor database access is needed here.\n", + "enriched_papers = list(quality_papers)\n", "\n", "# Show impact factor distribution\n", "impact_factors = [p.impact_factor for p in enriched_papers if hasattr(p, 'impact_factor') and p.impact_factor]\n", diff --git a/examples/_legacy/scitex/clew/multi_parent/scitex/clew.db b/examples/_legacy/scitex/clew/multi_parent/scitex/clew.db deleted file mode 100644 index 01dc4efe04ad7e53d2586a708897e56166edb8be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 94208 zcmeI5Yiu0Xb%6K9eURK8ih68l*YP&~VX?Bvc|Yg~E=8>rIU*&?lx5X*r}MZ&uEgD? zcXmZFtsSiFCV_(nEl{`p)1a>(MbXAB3N-%H7)@LMY0(;R9|_!|X@B*F0s+#(X`7;F zW_I>D!)2urLnmheQL{7Wo_o)C&%HDEo;!D+edbc7NlI3|QPr9yE;bX3$74TSD#c>4 z=ipxv{`sF7I7sT|Ke|K`6l{hg_QocxpAZ)Sfcvy*;#;#D|}A0&VTkiY{c(7tnG zy4;plmzN*T##Uo2&5Acp)71%Q8op+%JR0lBiy;PxLL1~#oZmPsns-{EN(aI zHy3RKrZMlF-rj9))@zI8j#k~?B6jEWqOnDG7OSnTW<}lB8l=`-q!slxX;iFT)zmf? zws!?n(JWmti77Fn#B-*lDVoAFgwq(4GbN`$P#!4GI-@KJ@)9d8Ff1>KT+kIxVi<|D zTZTbwTcAZnVMWg5Se=(?Ml@wj(`mvnJY#CK+}=6XF?}u$wMFIdAcd!4rX#B}pQ>AThEo z(wsyjMbbHXxN7+&o?T!VS`>vL%e^ZcFVU=Lxxs6!1-`+^#1v(j*BM5aHJ;X-uL4Kz zo|pm4pLE^59OUkNl@)Duce5r&W4WwWwOd5BDjTf^vBBY?q#|$fl4)A3Vlk`;1ttpN zbeT0}UKp-heuP=WrzA;SOxkv0jViww(fl1A&6 zVsH$nn>?dg!(9Qt#K;Q*Sj8$MUIAAmfsq6{lnhrD1Y6x$7uQV1G#LhJ!{T_0l?1@T z(SpSpiY|+iAm07RG>~~}FETW!I;-9MR5WBn75dk$-A@s8 zWP{fg$>9QBq@tLHHUfZ_0D!E}qwYk@(3BWK*$WMLl-hdwrXpIVWC)VXSb`++BFibx zSB?Wd(A${W-G?Rt%}O|$9_47z2wgNf+0<@V>J5M+2}BWOhKQDB0v+CxMV^(wk7${J z_CE}VTN0!Nft3}OAHwEdkr-Nl`t)ugqb-W7`fxZ7+6~PJ7Sx2Gi82Rw8=2>MgC@Mh zF~;4GWcT))o~mfzaB8DfQ^}oexW886UJGb=4$5aLJf};F#tMw0Ns??@@Zq54Tf^Nc zx5SGJa33D+Xx%Fc+@ImT5RQhSZB==iU4sr>W@v-Z&^%?GShONCg2}K%Vl6{r+4jq6 zK(p-nP2ioEt8yGXblj%TMT3S{x9Xay8ui;+qoOt4j##D%t!Wl*5oqSLE=wj1?T09OQX zqi(lK)3U~LfD}4UVnAm}M9vZvmeT;Btca8FWIt7SBS!r_^%d%usGp=>pq8k)!Uu)F zDf~X1#t#xe0!RP}AOR$R1dsp{Kmter2_S(7N1&7}$DJ|n;3RS{0(2r+pcf}O7A(*U zC(MQk__OXaVFLc_b0JK?pM;$b7U)eyPKF8iGke)E0e?~|9VXz<+N8n+{8^2J6aSx0 zBx00UsO5is`u*u+Q=goCE%%k|e`FUkR{B>azCLk0bvgNZ;vEpe&-WmK_VR2tR&HNg zopa+|gKh4?hN0AYtGVcfq}YY++J-KgoMea!17TH3BRr$P^qmfIDN7?POyGg|cB|Q& z4J*;|lHkOj+RH}{fenmf158~)l#Gb7!WaU?ZWxZRx&*V_tN?*URtU3!Zl9fTY`C#n zJb-!_ih8*fVbrU7rAEvJ!@jNPnhY@=h||H$tf-g}=CNp)Fw`wUhbV_xsRdO8E4NoD z&xl}W@;lFHCR6X%h>)sa5 zp$<$|6Lgy4x!{@r8_Idd2K7K}7>$RyR&;}7V21>;C|Q;nftMwYWmydxxF}g74WVOM zWcW}!&Q5!E5$sX^-8LGJy3e`@aVS{gaem4%LJO&kXpI;NGId&aH|7MjVL>#2lSGKZ z$r?i}hz0Nvh3A~6Ef^4U39b!VYOhQ>hP-efh71Q1T{{}u4lLPeYECajEZtC8Sf*pb zvI_{55s3T<1|#Y$EWFVKCe(`aIj?O5`Bd)#EgGMSplt;Dl@ldyuVllkBWgn)NOibY zoXdDt1XqXuI2=uN+_x1WmBEYc^J&M7;6@&Gko`e5awovvJ~!bQ@?u~kkJ^yYkg(g# z{qQifa)x6djLjK5M8#p*96U}bk_k_TCaf9KMJw?z)CCz;=V!Mm21d_q-RKQ`|wAD(X@{!94n-WaALch@H{Kn z5`WkljV1T*Ss`|?Li<8|uNv`3*U{J!g=>Xa!f;Ofe|F*zV$@dQ>xFaq-^qV;>VxUm zr#2?vn>?HQQtqSK&u9K8b1VIR`Vt7?2MHhnB!C1C2Z44z4HI{B0dwUCJ8$RRU1kqK zaoj05uQ*5A1)y`IcmQ<9O+>j5b~A0tDNk^1MLVT5t~?$f%B`>K!GNMI(3p7@mrtR6 z1o)^2QJj4Y$=xF5T^G6@TxHQt+KeIO)!1}<77%H{h@vgeeue=qqbb*|UpR;|jUBqX z1)6jrsX>(%^^C|q{K%`gTsWkt%d|g8+4c;0(hI?m4tT=FLzD^k?Wm_5_U+qgm(q(t zz8!U$#t&9q=bkt?O8Xghx@AhWk2<|lurD3_x=N%wB@VsN1jdY-{6Qo^gdx!mZ| z2CjPWQwCn4^3=b@lGHmf>K*ER>RX57u@*~#1dsp{Kmter2_OL^fCP{L5~Lj&j7lUQi(9Y{le($do5Y@llvyYBWHLDyH@37&)lRta|Lj_f z`q{z?^;rI|razee{M4I;-^-J!r>1{kl1%<(?!R(h$bBsP76{=72_OL^fCP{L53XWIMP%82qS8}asrCyIY>873tuaIx$yIaR$;xs6=w6_%)bjl_(1|l00|%gB!C2v01`j~NB{{SfrBCtyq{a| zZNIVm+8r6EVeH0tRG0?;m9fyB@qB@Nml6=^xnn^ z-F(k~J0=sR!50YKozEAT7^fk0`#%4zie#7u{|$%4qsj7~?;E(|f9hO}`fKXXsIOAL zP5mPE1!|Y7Q$IreC-tu|EAVCNSE*Mile!E-_(1|l00|%gB!C2v01`j~NB{{S0VME! zOkfh0(|d2z&-MyFbEA?LaC3eRDQ`9Rl>J933>RZ%5QGZLlOZ_qR7WG@yFT-j4AOR$R1dsp{ zKmter2_OL^fCP{L5^si(9jE(_Q+>zo`~OqanHcro5byscb%8odoq-vEH(?b(mAZ7_2BHEafCP{L54np3X=>672$cqs@fzbCJ z{NESIIwegze^Sn$MBb@|nGb(epan<(2_OL^fCP{L5itYl6z(B)Z@``pO}(FKhul4-_J{V_kkq}>jS}s> zc0UoC+SebH+MhZ|-cxFSnjrCUsPDkv_}eRQe@eg|{};X%gJ1f4oBAU4GOPl4l6t7{ zZ8(J=B!C2v01`j~NB{{S0VIF~kN^@u0*9VJDY^F?$k8wXf0`p?FFwBN3*^Hz_>&d6 zaRS+3f!>tEM3{g-vQLf^NIaU{dv0K8{Qsmo1AycIL+{^MKqP$OF4 zN2_jc5xaAG(byt8i`CXvv!ZTm4N_|^R<&BoHX4=f<^rUXM;rtiVZx74}5!#Qzf%y#N0m z^)LH#a+E>>NB{{S0VIF~kN^@u0!RP}AOR$R1P%d#6G~>HcXHZ~T8b zM*RkL`4E&Hi+}`>01`j~NB{{S0VIF~kN^@u0!ZM)K;Zcltp123=s&nLA#oiP=N^D! zZ~T8MMxCO5;lltJ%ZLP!01`j~NB{{S0VIF~kN^@u0!ZKi6S$RX--u>xKlbg3#Ad2} zEt=u}n3o5<@&DT~*!}+-511dSM*>Iy2_OL^fCP{L5Kz|^Nu4nwsvb@s!PWl&vqHy;SZPTRJ&e8v0aB zfV5t|P`+Baa`ocs^3@xqPnK_#manZ}zPJV|R?BPaPmH8;vrr-JdinYF(zUgV&s-~) z)-JD?)~;Q;G%DULuqPibxCL8ysI^vAhutnZ#TwP-W!~J~Cgaq3lDD<3mM1!@Yq0iR zeW5~bEjR{;N^ENOW{8g71|zE0EUjN$EkC=yyn3Z{u6%y^+NJf<%C)Ok0kzt_GMdS4 z(x_M!Vh$)66>~OZY1tkcU;E>|>>nSf%=uHZ>Ecs!@tEtC_Dfr^7mTX4nx2>XZHw_- z>S+7eOs2SeD&D@>b>dF5>ue5V`~Q25(f@X2#Ib{u4HV6HRA{P0(kH#0Xu55-JxgVZ zD^RSJ&|-Bo3@Mr!Tdq!m{T9sEUoq!j%jGl0Q>Wsez34Vg|B6?1d+2Lnps@~J3~03u zn|o`yPD)?D-+sf}qa!^Cdt8%qQZ$HBZ$zF-)OIOS1 z%g|I-%FmV>t(x6`ZL~3rXd1NU9<+I#U0<$G3=x_iahCZrg>>F~ zYM+?S6we6p_Jyv0dwqccTQAjXemfXA*V!8>KjPs7Qk|SeO6*{Rb~hl~(b#AC{Hxql zrYH#U&*WSm4-yYLH$C9(K@#6(m-mWvpANi5Zj-J3G?)Gkd+(-rFKuF^Zs#{Ap<_B8 zZ$HxYCZ`>B`fBHJYQ$;!1V;3Ae8p{xL#^w_b6!sQ!JIsUodlD?Ie7wm=H!XY|5!Ft zeB`lsJJ-vhzD3;U4d-S|?A&M$AP}ayFX71s$j#r(WQvc0{)fBzJ58ofGCPvRm}b?N zw!bx1)0yJqkXNr>TtVkSuP;SrB)6{Aeyr*W&tI8<=h0Jf@5!-q=eNmAEmAYc(ARV) zPw!cB=;FzmRwYkB%K2A+BI(ll>>0N^=qq&JVQN5E--YjTo9}|zPm}FF=$D5U zhxViIklir@?r~i_Pq;<}+~kB*x_I%iu$x?8MXOBTmx)&|#52X?$K#)Q$#oj1S8)C% z2HeCs6XGBK&C-9E=^MoaKMr_%7eaT}40~30YOW*It>_S4O|8)!x=#d4T9sM_`lE3V z?{3z;2l!xJFz#)tM%^UgV Date: Sun, 30 Aug 2026 01:35:31 +0000 Subject: [PATCH 2/3] docs(db): make the PostgreSQL README example and API list exact Peer review on PR #362. stx.db.PostgreSQL defaults password to None so the previous call did not raise, but a README example should show the real call: pass password explicitly from the environment. check_health and delete_duplicates are SQLite-only helpers (both are implemented over sqlite3 in scitex-db), so they are no longer advertised as capabilities of the PostgreSQL client, in the README prose or in the docs/05_ADDITIONAL_MODULES.md Key-API cell. --- README.md | 10 ++++++++-- docs/05_ADDITIONAL_MODULES.md | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 899b201d1..ab9d22167 100755 --- a/README.md +++ b/README.md @@ -529,9 +529,15 @@ Extends stdlib `logging` with SUCCESS/FAIL levels, a 30+ class exception tree (` scitex.db -- PostgreSQL with ndarray BLOB Storage ```python +import os import scitex as stx, numpy as np -db = stx.db.PostgreSQL(dbname="experiments", user="researcher", host="localhost") +db = stx.db.PostgreSQL( + dbname="experiments", + user="researcher", + password=os.environ["PGPASSWORD"], + host="localhost", +) with db: # closes the connection on exit db.execute( "CREATE TABLE IF NOT EXISTS runs (id SERIAL PRIMARY KEY, acc REAL, weights BYTEA)" @@ -542,7 +548,7 @@ with db: # closes the connection on df = db.get_rows("runs") # pandas round-trip w = db.load_array("runs", "weights", ids=1) # typed ndarray back ``` -PostgreSQL clients with first-class compressed-ndarray BLOBs, dataframe round-trips, health checks, and duplicate removal. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM. +A PostgreSQL client with first-class compressed-ndarray BLOBs and dataframe round-trips. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM.
diff --git a/docs/05_ADDITIONAL_MODULES.md b/docs/05_ADDITIONAL_MODULES.md index 072cba36a..10601776b 100644 --- a/docs/05_ADDITIONAL_MODULES.md +++ b/docs/05_ADDITIONAL_MODULES.md @@ -118,7 +118,7 @@ Lower-level SciTeX utilities re-exported under the umbrella: | `stx.dict` | `DotDict` + safe merge / flatten | `DotDict`, `safe_merge`, `flatten` | | `stx.logging` | stdlib-logging + SUCCESS/FAIL + `SciTeXError` | `getLogger`, `warn_deprecated`, `Tee` | | `stx.types` | Union type aliases + predicates | `ArrayLike`, `ColorLike`, `is_array_like` | -| `stx.db` | PostgreSQL wrapper w/ ndarray BLOBs | `PostgreSQL`, `check_health` | +| `stx.db` | PostgreSQL wrapper w/ ndarray BLOBs | `PostgreSQL`, `save_array`, `load_array` | | `stx.audit` | Unified security scan (bandit / shellcheck / pip-audit) | `audit()` | | `stx.browser` | Playwright helpers for scraping | `save_as_pdf`, `click_with_fallbacks_async` | | `stx.compat` | Deprecation shims | `@deprecated`, `notify` legacy alias | From a13604820f1837d14ce2b5bc116439b77d6aa4a6 Mon Sep 17 00:00:00 2001 From: scitex-agent-container Date: Sun, 30 Aug 2026 01:40:17 +0000 Subject: [PATCH 3/3] chore(sqlite-out): absorb PR #361, the parallel purge of the same directive PR #361 (chore/purge-sqlite) swept this repo for the same directive a few hours earlier. Both branches reach 0 on the agreed measurement, so one has to go; this commit takes everything #361 did better so the survivor is a strict superset, and #361 is closed. From #361: - Delete examples/_legacy/scitex/db/_sqlite3/ (12 files). They are EMPTY placeholders, so the content grep never saw them -- only their paths carry the retired engine's name. Path-level traces count too. - Better wording: docs now name the Clew store (the actual SciTeX provenance store, per the 'Provenance Tracking (Clew)' section) instead of a generic 'local database'; the schematic box is a 'verification store'; the CrossRef entries say 'database file'. Kept from this branch, which #361 lacks: - .gitignore gains .scitex/clew/, so the deleted runtime artefact cannot reappear on the next run. Verified with git check-ignore. - The README example passes password from the environment and INSERTs the row that save_array(ids=1) then targets. --- .env.d.examples/00_crossref-local.env | 2 +- docs/guides/CROSSREF_API_CONFIGURATION.md | 2 +- docs/sphinx/core_concepts.rst | 4 ++-- docs/sphinx/modules/io.rst | 2 +- .../scitex/db/_sqlite3/_SQLite3Mixins/example__BatchMixin.py | 0 .../scitex/db/_sqlite3/_SQLite3Mixins/example__BlobMixin.py | 0 .../db/_sqlite3/_SQLite3Mixins/example__ConnectionMixin.py | 0 .../db/_sqlite3/_SQLite3Mixins/example__ImportExportMixin.py | 0 .../scitex/db/_sqlite3/_SQLite3Mixins/example__IndexMixin.py | 0 .../db/_sqlite3/_SQLite3Mixins/example__MaintenanceMixin.py | 0 .../scitex/db/_sqlite3/_SQLite3Mixins/example__QueryMixin.py | 0 .../scitex/db/_sqlite3/_SQLite3Mixins/example__RowMixin.py | 0 .../scitex/db/_sqlite3/_SQLite3Mixins/example__TableMixin.py | 0 .../db/_sqlite3/_SQLite3Mixins/example__TransactionMixin.py | 0 examples/_legacy/scitex/db/_sqlite3/example__SQLite3.py | 0 .../_legacy/scitex/db/_sqlite3/example__delete_duplicates.py | 0 examples/_legacy/scitex/io/schematic_io_concept.py | 2 +- .../scitex/io/schematic_io_concept_out/stx_io_concept.yaml | 2 +- 18 files changed, 7 insertions(+), 7 deletions(-) delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BatchMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BlobMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ConnectionMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ImportExportMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__IndexMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__MaintenanceMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__QueryMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__RowMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TableMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TransactionMixin.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/example__SQLite3.py delete mode 100755 examples/_legacy/scitex/db/_sqlite3/example__delete_duplicates.py diff --git a/.env.d.examples/00_crossref-local.env b/.env.d.examples/00_crossref-local.env index 9c09edaec..e71b2e9b5 100644 --- a/.env.d.examples/00_crossref-local.env +++ b/.env.d.examples/00_crossref-local.env @@ -11,5 +11,5 @@ CROSSREF_LOCAL_API_URL=http://localhost:31291 CROSSREF_LOCAL_HOST=0.0.0.0 CROSSREF_LOCAL_PORT=31291 -# Path to the local database (for db mode) +# Path to the local database file (for db mode) # CROSSREF_LOCAL_DB=/path/to/crossref.db diff --git a/docs/guides/CROSSREF_API_CONFIGURATION.md b/docs/guides/CROSSREF_API_CONFIGURATION.md index 1a2aac0d8..0c2cc0673 100644 --- a/docs/guides/CROSSREF_API_CONFIGURATION.md +++ b/docs/guides/CROSSREF_API_CONFIGURATION.md @@ -146,7 +146,7 @@ else: - **Total papers**: 167,008,748 - **Database size**: 1.2TB (1,197,367 MB) -- **Format**: local database with indexed fields +- **Format**: Local database file with indexed fields - **Source**: CrossRef official database ## Best Practices diff --git a/docs/sphinx/core_concepts.rst b/docs/sphinx/core_concepts.rst index 2a9be782e..edbc11d13 100644 --- a/docs/sphinx/core_concepts.rst +++ b/docs/sphinx/core_concepts.rst @@ -37,7 +37,7 @@ The Session Model 2. **Logging**: All stdout/stderr captured to ``script.log`` 3. **Config injection**: YAML files from ``./config/`` merged and injected 4. **CLI generation**: Function parameters become ``--flags`` -5. **Provenance**: File hashes recorded to a local database for reproducibility +5. **Provenance**: File hashes recorded to the Clew store for reproducibility .. code-block:: python @@ -124,7 +124,7 @@ Provenance Tracking (Clew) -------------------------- Inside ``@stx.session``, every ``stx.io.save`` and ``stx.io.load`` call -records the file's SHA-256 hash to a local database. This enables: +records the file's SHA-256 hash to the local Clew store. This enables: - **Verification**: Check if output files have been modified since creation - **DAG reconstruction**: Trace which inputs produced which outputs diff --git a/docs/sphinx/modules/io.rst b/docs/sphinx/modules/io.rst index d90bc5498..1748c5661 100644 --- a/docs/sphinx/modules/io.rst +++ b/docs/sphinx/modules/io.rst @@ -94,7 +94,7 @@ Provenance Tracking ------------------- Inside ``@stx.session``, every ``save`` and ``load`` records file hashes -to a local database for reproducibility verification. +to the local Clew store for reproducibility verification. .. code-block:: python diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BatchMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BatchMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BlobMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__BlobMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ConnectionMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ConnectionMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ImportExportMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__ImportExportMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__IndexMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__IndexMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__MaintenanceMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__MaintenanceMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__QueryMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__QueryMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__RowMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__RowMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TableMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TableMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TransactionMixin.py b/examples/_legacy/scitex/db/_sqlite3/_SQLite3Mixins/example__TransactionMixin.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/example__SQLite3.py b/examples/_legacy/scitex/db/_sqlite3/example__SQLite3.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/db/_sqlite3/example__delete_duplicates.py b/examples/_legacy/scitex/db/_sqlite3/example__delete_duplicates.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/examples/_legacy/scitex/io/schematic_io_concept.py b/examples/_legacy/scitex/io/schematic_io_concept.py index 065a77550..e26878140 100755 --- a/examples/_legacy/scitex/io/schematic_io_concept.py +++ b/examples/_legacy/scitex/io/schematic_io_concept.py @@ -113,7 +113,7 @@ def main( s.add_box( "db", "verification.db", - subtitle="local DB", + subtitle="verification store", content=["runs, file_hashes"], emphasis="red", x_mm=W / 2 - db_w / 2 - 10, diff --git a/examples/_legacy/scitex/io/schematic_io_concept_out/stx_io_concept.yaml b/examples/_legacy/scitex/io/schematic_io_concept_out/stx_io_concept.yaml index 28817dc76..b5daa2020 100644 --- a/examples/_legacy/scitex/io/schematic_io_concept_out/stx_io_concept.yaml +++ b/examples/_legacy/scitex/io/schematic_io_concept_out/stx_io_concept.yaml @@ -228,7 +228,7 @@ axes: margin_mm: 0.0 - id: db title: verification.db - subtitle: local DB + subtitle: verification store content: - runs, file_hashes emphasis: red