diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md deleted file mode 100644 index 8ce2136..0000000 --- a/.github/WORKFLOWS.md +++ /dev/null @@ -1,107 +0,0 @@ -# GitHub Actions Workflows - -This repository includes several GitHub Actions workflows for automated testing, building, and releasing the GEO MCP Server package. - -## Workflows - -### 1. Test and Build (`test.yml`) -- **Triggers**: Push to `main`/`develop` branches, Pull Requests to `main` -- **Purpose**: Runs tests across multiple Python versions and builds the package -- **Actions**: - - Tests the package on Python 3.9, 3.10, 3.11, and 3.12 - - Builds the package (only on pushes to main) - - Uploads build artifacts - -### 2. Release Package (`release.yml`) -- **Triggers**: When a GitHub release is published -- **Purpose**: Publishes stable releases to PyPI -- **Actions**: - - Builds the package - - Publishes to PyPI - - Creates GitHub release assets - -### 3. Development Release (`dev-release.yml`) -- **Triggers**: When a GitHub pre-release is published (tags containing `dev`, `alpha`, `beta`, or `rc`) -- **Purpose**: Publishes development releases to TestPyPI -- **Actions**: - - Builds the package - - Publishes to TestPyPI - - Creates GitHub pre-release assets - -## Setup Instructions - -### 1. PyPI API Token -To publish to PyPI, you need to create an API token: - -1. Go to [PyPI Account Settings](https://pypi.org/manage/account/) -2. Create a new API token with "Entire account" scope -3. Add the token to your GitHub repository secrets: - - Go to your repository → Settings → Secrets and variables → Actions - - Create a new secret named `PYPI_API_TOKEN` - - Paste your PyPI API token - -### 2. TestPyPI API Token (Optional) -For development releases, you can also set up TestPyPI: - -1. Go to [TestPyPI Account Settings](https://test.pypi.org/manage/account/) -2. Create a new API token -3. Add the token to your GitHub repository secrets as `TEST_PYPI_API_TOKEN` - -### 3. GitHub Token -The `GITHUB_TOKEN` is automatically provided by GitHub Actions, so no setup is needed. - -## Usage - -### Creating a Release -1. Create a new release on GitHub -2. Tag it with a version number (e.g., `v1.0.0`) -3. Publish the release -4. The workflow will automatically build and publish to PyPI - -### Creating a Development Release -1. Create a new release on GitHub -2. Tag it with a development version (e.g., `v1.0.0-dev.1`, `v1.0.0-alpha.1`) -3. Mark it as a pre-release -4. Publish the release -5. The workflow will automatically build and publish to TestPyPI - -### Testing -- Push to `main` or `develop` branches to trigger tests -- Create pull requests to `main` to trigger tests -- Tests run on multiple Python versions to ensure compatibility - -## Version Management - -The package uses [hatch-vcs](https://github.com/ofek/hatch-vcs) for automatic version detection from git tags. The version format follows PEP 440: - -- `0.1.dev1+g91ddaa4.d20250623` - Development version -- `0.1.0` - Stable release -- `0.1.0-dev.1` - Development release -- `0.1.0-alpha.1` - Alpha release -- `0.1.0-beta.1` - Beta release -- `0.1.0-rc.1` - Release candidate - -## Troubleshooting - -### Common Issues - -1. **Build fails**: Check that all dependencies are properly specified in `pyproject.toml` -2. **PyPI upload fails**: Verify your `PYPI_API_TOKEN` is correct and has proper permissions -3. **Tests fail**: Ensure all test dependencies are installed and tests are properly configured - -### Manual Release -If you need to release manually: - -```bash -# Build the package -python -m build - -# Check the package -twine check dist/* - -# Upload to PyPI -twine upload dist/* - -# Upload to TestPyPI (for development releases) -twine upload --repository testpypi dist/* -``` \ No newline at end of file diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml deleted file mode 100644 index e77d7cc..0000000 --- a/.github/workflows/dev-release.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Development Release - -on: - release: - types: [published] - -jobs: - dev-release: - runs-on: ubuntu-latest - if: contains(github.event.release.tag_name, 'dev') || contains(github.event.release.tag_name, 'alpha') || contains(github.event.release.tag_name, 'beta') || contains(github.event.release.tag_name, 'rc') - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Build package - run: | - python -m build - - - name: Check package - run: | - twine check dist/* - - - name: Publish to TestPyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - run: | - twine upload --repository testpypi dist/* - - - name: Create Release Assets - uses: softprops/action-gh-release@v1 - with: - files: | - dist/*.whl - dist/*.tar.gz - draft: false - prerelease: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml deleted file mode 100644 index 1e12b6f..0000000 --- a/.github/workflows/pypi-publish.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Publish to PyPI - -on: - release: - types: [published] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index 8983ea2..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Publish Python Package - -on: - release: - types: [published] - -jobs: - build-and-publish: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for version detection - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Build package - run: | - python -m build - - - name: Check package - run: | - twine check dist/* - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - twine upload dist/* - - - name: Create Release Assets - uses: softprops/action-gh-release@v1 - with: - files: | - dist/*.whl - dist/*.tar.gz - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 6c8f907..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Test and Build - -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main ] - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.10', '3.11', '3.12'] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest geo_mcp_server/test/ -v --tb=short - - build: - runs-on: ubuntu-latest - needs: test - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Build package - run: | - python -m build - - - name: Check package - run: | - twine check dist/* - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: dist-files - path: dist/ \ No newline at end of file diff --git a/.gitignore b/.gitignore deleted file mode 100644 index cc3c6e8..0000000 --- a/.gitignore +++ /dev/null @@ -1,159 +0,0 @@ -# Configuration files -geo_mcp_server/config.json -claude_desktop_config.json -mcp_config.json -*.json -*.txt -*.csv -*.tsv -*.fasta -*.fastq -*.fq -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Virtual environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# macOS -.DS_Store -.AppleDouble -.LSOverride - -# Windows -Thumbs.db -ehthumbs.db -Desktop.ini - -# Logs -*.log -logs/ - -# Temporary files -*.tmp -*.temp -.cursor* - -# Mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Ruff -.ruff_cache/ - -# Black -.black_cache/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -Pipfile.lock - -# poetry -poetry.lock - -# uv -uv.lock - -# pdm -.pdm.toml - -# PEP 582 -__pypackages__/ - -# Celery -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 93e1f2d..0000000 --- a/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2025, Matthias Flotho - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 81f946e..647a67a 100644 --- a/README.md +++ b/README.md @@ -1,247 +1,411 @@ -# Gene Expression Omnibus (GEO) MCP - -
- GEO Logo -
- Gene Expression Omnibus (GEO) - A public functional genomics data repository -
- -[![PyPI version](https://badge.fury.io/py/geo-mcp.svg)](https://pypi.org/project/geo-mcp/) -[![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/downloads/) -[![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![MCP](https://img.shields.io/badge/MCP-Protocol-blue.svg)](https://modelcontextprotocol.io/) -[![GEO](https://img.shields.io/badge/GEO-NCBI-orange.svg)](https://www.ncbi.nlm.nih.gov/geo/) - -> ⚠️ **Development Warning** ⚠️ -> -> **This project is currently in active development and is not yet production-ready.** -> -> - The MCP stdio server is functional but may have bugs or incomplete features -> - API endpoints and functionality may change without notice -> - Use at your own risk and report any issues you encounter -> -> We recommend testing thoroughly in a development environment before using in production. - -A Model Context Protocol (MCP) server for accessing [GEO (Gene Expression Omnibus)](https://www.ncbi.nlm.nih.gov/geo/) data through NCBI E-Utils API. -The tool will enable you to search for GEO datasets, series, samples, platforms, and profiles for your LLM. -Tested with Claude Desktop, chatGPT has no out of the box support for this tool yet. -Claude will automatically use the tools if it fits the context. - -## Quick Install (pip) - -install from pip -```bash -pip install geo-mcp -``` -install from source +# GEO MCP Server with SRA Support + +An enhanced [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server for accessing **GEO (Gene Expression Omnibus)** data through NCBI E-Utils API, with comprehensive support for **SRA (Sequence Read Archive)** raw sequencing data downloads and conversion. + +## Features + +### GEO Data Access +- 🔍 **Search GEO databases**: Series (GSE), Samples (GSM), Platforms (GPL), Datasets (GDS), and Profiles +- 📥 **Download metadata**: SOFT format, Series Matrix, MINiML XML +- 📦 **Download supplementary files**: Processed data tables, raw array data + +### SRA Raw Sequencing Data (NEW) +- 🔗 **Query SRA from GEO**: Map GEO Samples (GSM) to SRA Runs (SRR) +- 📊 **Size estimation**: Dry-run mode to check file sizes before downloading +- 💾 **Download & convert**: Integrated prefetch + fastq-dump workflow +- 🛡️ **Safety constraints**: Size warnings and confirmation requirements +- ✅ **Check sra-toolkit**: Verify installation and get setup instructions + +## Safety Features + +Following [maintainer recommendations](https://github.com/MCPmed/GEOmcp/issues/1), this server implements safety constraints for large file downloads: + +| Constraint | Implementation | +|------------|----------------| +| **Safe by Default** | `dry_run=True` by default - only estimates sizes | +| **Dry-Run Mode** | `sra_estimate_size()` tool for size checking | +| **Explicit Output** | Required `output_dir` for files >1GB | +| **Confirmation** | `confirm_large=True` required for files >5GB | + +## Installation + +### Prerequisites +- Python 3.10 or higher +- (Optional) [SRA Toolkit](https://github.com/ncbi/sra-tools) for downloading raw FASTQ files + +### Install from Source ```bash -# Clone the repo (if not already) -git clone https://github.com/MCPmed/GEOmcp -cd GEO_MCP +# Clone the repository +git clone https://github.com/yourusername/geo-mcp-server.git +cd geo-mcp-server + +# Install dependencies pip install -e . ``` -## Configuration +### Configuration -Run init to create a config file +1. **Initialize configuration:** ```bash -geo-mcp --init +python server.py --init ``` -This will create a config file at `~/.geo-mcp/config.json` (auto-created on first run if missing). -This file will contain the following: +2. **Edit the config file** at `~/.geo-mcp/config.json`: ```json { - "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", "email": "your_email@example.com", - "api_key": "YOUR_API_KEY" + "api_key": "YOUR_NCBI_API_KEY (optional)", + "download_dir": "~/geo_downloads", + "sra_toolkit_path": "/path/to/sra-toolkit/bin (optional)" +} +``` + +> **Note:** NCBI requires an email address for E-Utils access. An API key is optional but recommended for higher rate limits (10 req/s vs 3 req/s). Get one at [NCBI](https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/). + +## Usage + +### Running the Server + +**MCP stdio mode** (for Claude Desktop): +```bash +python server.py +``` + +**HTTP mode**: +```bash +python server.py --http --port 8000 +``` + +### Claude Desktop Integration + +Add to your Claude Desktop configuration (`~/.config/claude-desktop/config.json`): + +```json +{ + "mcpServers": { + "geo_mcp": { + "command": "python", + "args": ["/path/to/geo_mcp_server/server.py"], + "env": { + "CONFIG_PATH": "/home/yourusername/.geo-mcp/config.json" + } + } + } } ``` -It will also print out a configuration template for your claude desktop configuration file. +## Available Tools (18 Total) -- `email` is required by NCBI. -- `api_key` is optional but recommended for higher rate limits ([get one here](https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/)). +### GEO Search Tools -## Running the Server +| Tool | Description | +|------|-------------| +| `geo_search` | Search all GEO record types with natural language (e.g., "Human RNA-seq") | +| `geo_search_series` | Search GEO Series (GSE) - complete experiments | +| `geo_search_samples` | Search GEO Samples (GSM) - individual samples | +| `geo_search_platforms` | Search GEO Platforms (GPL) - array/sequencing platforms | +| `geo_search_datasets` | Search GEO Datasets (GDS) - curated gene expression | +| `geo_search_profiles` | Search GEO Profiles - gene expression profiles | -- **MCP stdio mode:** - ```bash - geo-mcp - ``` -- **HTTP mode:** - ```bash - geo-mcp --http --port 8001 - ``` +### GEO Download Tools -## Claude Desktop Integration +| Tool | Description | +|------|-------------| +| `geo_download_series` | Download GSE data files (SOFT, matrix, supplementary) | +| `geo_download_sample` | Download GSM supplementary files | +| `geo_get_download_status` | Check if a GEO dataset has been downloaded | +| `geo_list_downloads` | List all downloaded datasets | +| `geo_cleanup_downloads` | Clean up downloaded files | -### Common Issue: `spawn geo-mcp ENOENT` +### SRA Tools -This error means Claude Desktop cannot find the `geo-mcp` command. This is usually a PATH issue. +| Tool | Description | +|------|-------------| +| `sra_query_from_geo` | Query SRA Run information from a GEO Series | +| `sra_get_metadata` | Get SRA run metadata | +| `sra_estimate_size` | **NEW** Estimate download sizes before downloading (dry-run) | +| `sra_generate_download_commands` | Generate download commands for various methods | +| `sra_check_toolkit` | Check sra-toolkit installation | +| `sra_download` | Download SRA directly via HTTP (small files only) | +| `sra_download_and_convert` | **NEW** Download with prefetch and convert to FASTQ | -### Solution -1. **Find the full path to the executable:** - ```bash - which geo-mcp - ``` - Example output: `/Users/youruser/miniforge3/bin/geo-mcp` +## Example Workflows -2. **Update your Claude config:** - Instead of just `"geo-bio-mcp"`, use the full path: - ```json - { - "mcpServers": { - "geo-mcp": { - "command": "/Users/youruser/miniforge3/bin/geo-mcp", - "env": { - "CONFIG_PATH": "/Users/youruser/.geo-mcp/config.json" - } - } - } - } - ``` +### 1. Search with Natural Language -3. **(Optional) Use a Conda Environment:** - - Activate your conda env and run `which geo-mcp` to get the correct path. - - Use that path in your Claude config as above. +``` +User: Find Human RNA-seq datasets -4. **Restart Claude Desktop** after updating the config. +AI: I'll search for Human RNA-seq datasets in GEO. +[Uses geo_search_series with term="Human RNA-seq"] ---- +Results: +- GSE319287: "TPD52 Inhibits Aldosterone Synthesis..." (6 samples) +- GSE213095: "Positive feedback between cDC1s and CD8+ T cells..." (46 samples) +- GSE320065: "Transcriptomic and Functional Responses of Human Airway Cells..." (56 samples) +``` -## Troubleshooting -- If you see `command not found: geo-mcp`, make sure you installed with the correct Python/conda environment and that its `bin` directory is in your PATH. -- If the config file is missing, it will be auto-created on first run, or you can copy the template from `geo_mcp_server/config_template.json`. +### 2. Download Processed Data ---- +``` +User: Download the metadata for GSE12345 -## Usage +AI: I'll download the SOFT file and series matrix for GSE12345. +[Uses geo_download_series with geo_id="GSE12345", file_types=["soft", "matrix"]] + +Results: +✓ Downloaded GSE12345_family.soft.gz (1.2 MB) +✓ Downloaded GSE12345_series_matrix.txt.gz (5.6 MB) +Location: ~/geo_downloads/gse/GSE12345/ +``` -### MCP Server (stdio mode) -For use with MCP clients like Claude Desktop: +### 3. Find and Download Raw Sequencing Data (Safe Workflow) -```bash -cd geo_mcp_server -python main.py --mode stdio +**Step 1: Get SRA accessions** ``` +User: Get the SRA run accessions for GSE272467 -### HTTP Server (localhost:8000) -For HTTP API access: +AI: I'll query the SRA information from this GEO Series. +[Uses sra_query_from_geo with gse_id="GSE272467"] -```bash -# Option 1: Using the main script -cd geo_mcp_server -python main.py --mode http --host localhost --port 8000 +Results: +- Total samples: 4 +- Samples with SRA data: 4 +- Total SRA runs: 4 +- All SRA accessions: ["SRR29864801", "SRR29864802", "SRR29864803", "SRR29864804"] +``` -# Option 2: Using the convenience script -python run_server.py +**Step 2: Estimate size (recommended)** ``` +User: How large are these files? -### HTTP API Endpoints +AI: Let me estimate the download sizes. +[Uses sra_estimate_size with sra_ids=["SRR29864801"]] -Once the HTTP server is running on localhost:8000, you can access: +Results: +- SRA size: ~76 MB +- Estimated FASTQ size: ~190 MB +- Read count: 7,178,576 +- Warning: Small file (< 100 MB) +``` -- **Root**: `GET /` - Server status -- **Health Check**: `GET /health` - Health check endpoint -- **List Tools**: `GET /tools` - List all available tools -- **Call Tool**: `POST /tools/call` - Execute a tool with arguments -- **API Documentation**: `GET /docs` - Interactive API documentation (Swagger UI) +**Step 3: Download and convert** +``` +User: Download and convert SRR29864801 to FASTQ -### Example HTTP API Usage +AI: I'll download the SRA file and convert it to FASTQ format. +[Uses sra_download_and_convert with sra_id="SRR29864801", dry_run=False] + +Results: +✓ Downloaded SRR29864801.sra (76 MB) +✓ Converted to FASTQ + - SRR29864801_1.fastq (660 KB, 5000 reads) + - SRR29864801_2.fastq (660 KB, 5000 reads) +Location: ~/geo_downloads/sra/SRR29864801/ +``` +### 4. Safety Constraints in Action + +``` +User: Download SRR1234567 + +AI: First, let me estimate the size before downloading. +[Uses sra_estimate_size with sra_ids=["SRR1234567"]] + +Results: +- SRA size: ~2.5 GB +- Estimated FASTQ size: ~6.25 GB +- Warning: LARGE FILE (1-5 GB) + +⚠️ This is a large download that requires explicit output directory. +Please provide output_dir parameter to proceed. + +User: Download to /data/sra + +AI: +[Uses sra_download_and_convert with + sra_id="SRR1234567", + dry_run=False, + output_dir="/data/sra"] + +✓ Download complete! +``` + +### 5. Check SRA Toolkit + +``` +User: Do I have sra-toolkit installed? + +AI: Let me check your sra-toolkit installation. +[Uses sra_check_toolkit] + +Results: +✓ prefetch: available (version 3.3.0) +✓ fastq-dump: available (version 3.3.0) +✓ fasterq-dump: available (version 3.3.0) +✓ vdb-validate: available (version 3.3.0) + +All tools are installed and ready to use! +``` + +## SRA Download Methods + +### Method 1: Using sra_download_and_convert (Recommended) + +Integrated workflow with safety features: +```python +# Step 1: Estimate size (dry-run) +sra_estimate_size(sra_ids=["SRR1234567"]) + +# Step 2: Download and convert based on size +# Small file (<1GB) +sra_download_and_convert(sra_id="SRR1234567", dry_run=False) + +# Medium file (1-5GB) - requires output_dir +sra_download_and_convert( + sra_id="SRR1234567", + dry_run=False, + output_dir="/path/to/output" +) + +# Large file (>5GB) - requires confirmation +sra_download_and_convert( + sra_id="SRR1234567", + dry_run=False, + output_dir="/path/to/output", + confirm_large=True +) +``` + +### Method 2: Using SRA Toolkit Manually + +1. **Install sra-toolkit**: Follow instructions at https://github.com/ncbi/sra-tools + +2. **Download and convert**: ```bash -# List available tools -curl http://localhost:8000/tools - -# Search GEO Profiles -curl -X POST http://localhost:8000/tools/call \ - -H "Content-Type: application/json" \ - -d '{ - "name": "search_geo_profiles", - "arguments": { - "term": "cancer", - "retmax": 5 - } - }' - -# Search GEO Datasets -curl -X POST http://localhost:8000/tools/call \ - -H "Content-Type: application/json" \ - -d '{ - "name": "search_geo_datasets", - "arguments": { - "term": "breast cancer", - "retmax": 10 - } - }' +# Download SRA file +prefetch SRR1234567 + +# Convert to FASTQ (3-way split for paired-end) +fastq-dump --split-3 SRR1234567 ``` -## Available Tools +### Method 3: Direct HTTP Download (Small Files Only) + +For small files or when sra-toolkit is not available: +```bash +# Using wget +wget https://sra-downloadb.be-md.ncbi.nlm.nih.gov/sos2/sra-pub-run-11/SRR1234567/SRR1234567.1 -This MCP server provides access to all major GEO databases through the following tools: +# Using curl +curl -o SRR1234567.sra https://sra-downloadb.be-md.ncbi.nlm.nih.gov/sos2/sra-pub-run-11/SRR1234567/SRR1234567.1 +``` -### Search Tools +## Project Structure -- **`search_geo_profiles`** - Search gene expression profiles across different biological contexts - - *Example searches*: "cancer", "breast cancer", "p53", "apoptosis" - -- **`search_geo_datasets`** - Search curated gene expression datasets - - *Example searches*: "diabetes", "Alzheimer's disease", "drug response", "tissue specific" - -- **`search_geo_series`** - Search original submitter-supplied gene expression series - - *Example searches*: "GSE12345", "microarray", "RNA-seq", "time course" - -- **`search_geo_samples`** - Search individual gene expression samples - - *Example searches*: "GSM123456", "human", "mouse", "tumor", "normal" - -- **`search_geo_platforms`** - Search microarray platform definitions - - *Example searches*: "Affymetrix", "Illumina", "Agilent", "GPL96" +``` +geo_mcp_server/ +├── geomcp_sra/ +│ ├── __init__.py +│ ├── config.py # Configuration management +│ ├── geo_search.py # GEO search functionality (E-Utilities) +│ ├── geo_download.py # GEO data download +│ └── sra_handler.py # SRA query, size estimation, download & convert +├── server.py # Main MCP server with 18 tools +├── config.json # Configuration template +├── pyproject.toml # Project dependencies +├── requirements.txt # Python dependencies +├── README.md # This file +└── SKILL.md # Skill documentation +``` -### Search Parameters +## Comparison with Original GEOmcp -Each tool accepts: -- **`term`** (required): Search term or query string -- **`retmax`** (optional, default: 20): Maximum number of results to return +| Feature | GEOmcp (Original) | This Project (geo-mcp-server) | +|---------|-------------------|---------------------------| +| GEO Search | ✓ | ✓ (Enhanced with natural language) | +| SOFT Download | ✓ | ✓ | +| Matrix Download | ✓ | ✓ | +| Supplementary Files | ✓ | ✓ | +| **SRA Query** | ✗ | **✓** | +| **SRR Mapping** | ✗ | **✓** | +| **Size Estimation** | ✗ | **✓ (Dry-run mode)** | +| **Download & Convert** | ✗ | **✓ (Integrated workflow)** | +| **Safety Constraints** | ✗ | **✓ (>1GB, >5GB checks)** | +| Tool Count | 11 | **18** | -### Example usage -For now just chat with claude desktop and it will use the tools automatically if it fits the context. +## Configuration Options -**User:** find a small brain dataset +### Full Config File Reference + +>>>>>>> ad562a4 (initialize files, commit to branch Test) +```json +{ + "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", + "email": "your_email@example.com", + "api_key": "YOUR_NCBI_API_KEY (optional but recommended)", + "download_dir": "~/geo_downloads", + "max_file_size_mb": 5000, + "max_total_downloads_mb": 10000, + "max_concurrent_downloads": 3, + "download_timeout_seconds": 300, + "allowed_download_paths": ["~/geo_downloads", "/tmp/geo_downloads"], + "sra_toolkit_path": "/path/to/sra-toolkit/bin (optional)" +} +``` -**Assistant:** I'll help you find a small brain dataset from the GEO database. Let me search for brain-related datasets with a maximum of one result. +## Troubleshooting -**Assistant (system):** *search_geo_datasets* +### Email not configured +``` +Error: Email is required for NCBI E-utilities. +``` +**Solution**: Edit `~/.geo-mcp/config.json` and add your email address. -**Assistant:** I found a brain-related dataset for you: +### SRA toolkit not found +``` +fasterq-dump not found. Please install sra-toolkit. +``` +**Solution**: Install sra-toolkit from https://github.com/ncbi/sra-tools and set `sra_toolkit_path` in config. -- **Dataset:** GSE290848 -- **Title:** Maternal Stress Triggers Early-Life Eczema via Fetal Mast Cell Reprogramming [10x Genomics Multiome ATAC + Gene Expression] -- **Species:** Mouse (*Mus musculus*) -- **Samples:** 8 -- **Data type:** ATAC-seq & gene expression (10x Genomics Multiome) -- **Publication date:** 21 Jun 2025 +### Rate limiting +``` +Error: Rate limit exceeded +``` +**Solution**: Add an NCBI API key to your config for higher rate limits (10 req/s vs 3 req/s). -This small dataset explores how prenatal stress affects neuro-immune circuits and mast-cell development. +### Large file safety error +``` +SAFETY CHECK: File size is ~2.5 GB. Large downloads require an explicit output directory. +``` +**Solution**: Provide `output_dir` parameter for files >1GB, or both `output_dir` and `confirm_large=True` for files >5GB. -Would you like me to download this dataset or search for others? +## References ---- +- [GEO Home](https://www.ncbi.nlm.nih.gov/geo/) +- [SRA Home](https://www.ncbi.nlm.nih.gov/sra) +- [NCBI E-Utilities](https://www.ncbi.nlm.nih.gov/books/NBK25501/) +- [SRA Toolkit Documentation](https://github.com/ncbi/sra-tools/wiki) +- [MCP Documentation](https://modelcontextprotocol.io/) -**User:** download GSE290848 +## License -**Assistant:** I'll download the GSE290848 dataset for you. +MIT License - See LICENSE file for details. -**Assistant (system):** *download_geo_data* +## Contributing -**Assistant:** Download complete. +Contributions are welcome! Please feel free to submit issues or pull requests. -**Downloaded files** +### Contributing SRA Support Back to Original GEOmcp -- `GSE290848_family.soft.gz` — series, platform & sample info -- `GSE290848_metadata.xml` — detailed study metadata +This project implements the [enhancement request](https://github.com/MCPmed/GEOmcp/issues/1) for SRA raw data support with the following safety features as recommended by maintainers: +- Safe by default (dry-run mode) +- Size estimation before download +- Explicit output directory requirement for large files +- Confirmation for very large files (>5GB) -**Location:** `/Users/matthiasflo/Documents/2025/GEO_MCP/geo_mcp_server/downloads/gse/GSE290848` -**Total size:** 0.01 MB +## Acknowledgments -The dataset is ready for analysis. Let me know if you’d like to explore it further. +- Original [GEOmcp](https://github.com/MCPmed/GEOmcp) project for the foundation +- NCBI for providing the GEO and SRA databases +- MCP team for the Model Context Protocol diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..8c1de1a --- /dev/null +++ b/SKILL.md @@ -0,0 +1,245 @@ +--- +name: geo-mcp +description: MCP server for accessing GEO (Gene Expression Omnibus) data with comprehensive SRA (Sequence Read Archive) raw sequencing support. Enables natural language search, metadata download, size estimation with dry-run mode, and safe FASTQ downloads with prefetch/fastq-dump integration. +--- + +# GEO MCP Server with SRA Support + +This skill provides a Model Context Protocol (MCP) server for accessing NCBI's GEO and SRA databases programmatically with enhanced safety features for large file downloads. + +## Capabilities + +### GEO Data Access +- Search GEO databases (GSE, GSM, GPL, GDS, Profiles) using natural language +- Download SOFT format metadata files +- Download series matrix files +- Download supplementary processed data + +### SRA Raw Sequencing Data (with Safety Features) +- Query SRA Run accessions from GEO Series +- Map GSM samples to SRR run accessions +- **Estimate download sizes** (dry-run mode) +- **Download & convert** with integrated prefetch + fastq-dump workflow +- **Safety constraints**: Size warnings, explicit output directory, confirmation for large files + +## Installation + +```bash +cd /path/to/geo_mcp_server +pip install -e . +``` + +## Configuration + +1. Create config file: +```bash +python3 server.py --init +``` + +2. Edit `~/.geo-mcp/config.json`: +```json +{ + "email": "your_email@example.com", + "api_key": "YOUR_NCBI_API_KEY (optional but recommended)", + "download_dir": "~/geo_downloads", + "sra_toolkit_path": "/path/to/sra-toolkit/bin (optional)" +} +``` + +> **Note:** NCBI requires an email address. API key provides higher rate limits (10 req/s vs 3 req/s). + +## Usage with Claude Desktop + +Add to `~/.config/claude-desktop/config.json`: + +```json +{ + "mcpServers": { + "geo_mcp": { + "command": "python3", + "args": ["/path/to/geo_mcp_server/server.py"], + "env": { + "CONFIG_PATH": "/home/username/.geo-mcp/config.json" + } + } + } +} +``` + +## Available Tools (18 Total) + +### Search Tools (Natural Language Support) + +All search tools support natural language queries like "Human RNA-seq", "mouse brain single cell", "breast cancer transcriptome". + +| Tool | Description | Example Query | +|------|-------------|---------------| +| `geo_search` | Universal GEO search | "cancer RNA-seq" | +| `geo_search_series` | Search GSE records | "Human RNA-seq" | +| `geo_search_samples` | Search GSM records | "HeLa cell line" | +| `geo_search_platforms` | Search GPL records | "Illumina HiSeq" | +| `geo_search_datasets` | Search GDS records | "breast cancer" | +| `geo_search_profiles` | Search GEO Profiles | "p53 expression" | + +### GEO Download Tools + +| Tool | Description | +|------|-------------| +| `geo_download_series` | Download GSE data (SOFT, matrix, supplementary) | +| `geo_download_sample` | Download GSM supplementary files | +| `geo_get_download_status` | Check download status | +| `geo_list_downloads` | List downloaded datasets | +| `geo_cleanup_downloads` | Clean up files | + +### SRA Tools + +| Tool | Description | Safety Features | +|------|-------------|-----------------| +| `sra_query_from_geo` | Get SRA accessions from GEO Series | - | +| `sra_get_metadata` | Get SRA run metadata | - | +| `sra_estimate_size` | **Estimate sizes (dry-run)** | Shows warnings for >1GB, >5GB | +| `sra_generate_download_commands` | Generate download commands | - | +| `sra_check_toolkit` | Check sra-toolkit installation | - | +| `sra_download` | Direct HTTP download | Small files only | +| `sra_download_and_convert` | **Download & convert to FASTQ** | dry_run=True default, size checks | + +## Safety Features + +### Default Safe Behavior + +All SRA downloads default to **dry-run mode** (`dry_run=True`): + +```python +# This only estimates size, does NOT download +sra_download_and_convert(sra_id="SRR1234567") +``` + +### Size-Based Safety Constraints + +| File Size | Required Parameters | +|-----------|---------------------| +| < 1 GB | `dry_run=False` | +| 1-5 GB | `dry_run=False` + `output_dir="/path"` | +| > 5 GB | `dry_run=False` + `output_dir="/path"` + `confirm_large=True` | + +### Safety Check Examples + +**Error for >1GB without output_dir:** +``` +SAFETY CHECK: File size is ~2.5 GB. Large downloads require an explicit +output directory. Please provide output_dir parameter. +Tip: Run with dry_run=True first to see size estimates. +``` + +**Error for >5GB without confirmation:** +``` +SAFETY CHECK: File size is ~6.2 GB (>5GB). This is a VERY LARGE download +that will consume significant disk space and time. +To proceed, set confirm_large=True. +``` + +## Example Workflows + +### Workflow 1: Search with Natural Language + +```python +# Search for Human RNA-seq datasets +geo_search_series(term="Human RNA-seq", retmax=10) + +# Search for specific tissue + disease +geo_search_series(term="mouse brain Alzheimer's", retmax=5) +``` + +### Workflow 2: Safe SRA Download + +**Step 1: Always estimate first** +```python +sra_estimate_size(sra_ids=["SRR1234567"]) +# Returns: SRA size, FASTQ estimate, read count, safety warnings +``` + +**Step 2: Download based on size** + +Small file (<1GB): +```python +sra_download_and_convert( + sra_id="SRR1234567", + dry_run=False, + split_3=True, # Properly handle paired-end + check_refseq=False # Skip refseq to save space +) +``` + +Medium file (1-5GB): +```python +sra_download_and_convert( + sra_id="SRR1234567", + dry_run=False, + output_dir="/data/sra", # Required! + split_3=True +) +``` + +Large file (>5GB): +```python +sra_download_and_convert( + sra_id="SRR1234567", + dry_run=False, + output_dir="/data/sra", # Required + confirm_large=True, # Required + split_3=True +) +``` + +### Workflow 3: Complete Analysis Pipeline + +```python +# 1. Search for datasets +results = geo_search_series(term="GLOR2 m6A", retmax=5) + +# 2. Get SRA accessions for a dataset +sra_info = sra_query_from_geo(gse_id="GSE272467") +# Returns: 4 SRR accessions + +# 3. Estimate sizes +sizes = sra_estimate_size(sra_ids=sra_info["all_sra_accessions"]) +# Shows: ~76 MB each, total ~304 MB + +# 4. Download and convert +for sra_id in sra_info["all_sra_accessions"]: + sra_download_and_convert( + sra_id=sra_id, + dry_run=False, + split_3=True, + check_refseq=False + ) +``` + +## Architecture + +``` +geo_mcp_server/ +├── geomcp_sra/ +│ ├── config.py # Configuration management +│ ├── geo_search.py # NCBI E-Utilities search +│ ├── geo_download.py # FTP/HTTP downloads +│ └── sra_handler.py # SRA query, size estimation, download & convert +├── server.py # MCP server with 18 tools (FastMCP) +├── config.json # Config template +└── pyproject.toml # Project metadata +``` + +## Dependencies + +- `mcp>=1.9.0` - MCP Python SDK +- `httpx>=0.27.0` - Async HTTP client +- `aiofiles>=23.0.0` - Async file operations +- `pydantic>=2.0.0` - Input validation + +## References + +- [GEO Documentation](https://www.ncbi.nlm.nih.gov/geo/info/) +- [SRA Documentation](https://www.ncbi.nlm.nih.gov/sra/docs/) +- [SRA Toolkit](https://github.com/ncbi/sra-tools) +- [MCP Specification](https://modelcontextprotocol.io/) +- [Original GEOmcp Issue #1 - SRA Support](https://github.com/MCPmed/GEOmcp/issues/1) diff --git a/claude_desktop_config_example.json b/claude_desktop_config_example.json deleted file mode 100644 index 2b9b50c..0000000 --- a/claude_desktop_config_example.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "mcpServers": { - "geo-bio-mcp": { - "command": "/path_to_envs/geo-mcp-server/bin/geo-bio-mcp", - "env": { - "CONFIG_PATH": "/path_to_user_home/.geo-bio-mcp/config.json" - } - } - } -} diff --git a/config.json b/config.json new file mode 100644 index 0000000..3251bd4 --- /dev/null +++ b/config.json @@ -0,0 +1,17 @@ +{ + "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", + "sra_base_url": "https://trace.ncbi.nlm.nih.gov/Traces/sra", + "email": "your_email@example.com", + "api_key": "", + "retmax": 20, + "download_dir": "./downloads", + "max_file_size_mb": 5000, + "max_total_downloads_mb": 50000, + "max_concurrent_downloads": 3, + "download_timeout_seconds": 300, + "sra_toolkit_path": "", + "allowed_download_paths": [ + "./downloads", + "/tmp/geo_downloads" + ] +} diff --git a/geomcp/__init__.py b/geomcp/__init__.py deleted file mode 100644 index 77a9d12..0000000 --- a/geomcp/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -GEO MCP Server - A Model Context Protocol (MCP) server for accessing GEO data. - -This package provides tools to search and download data from the Gene Expression Omnibus (GEO) -through NCBI E-Utils API using the Model Context Protocol. -""" - -__version__ = "0.1.1" -__author__ = "MCPmed Contributors" -__email__ = "matthias.flotho@ccb.uni-saarland.de" - -from .main import main -from .geo_profiles import search_geo, search_geo_profiles, search_geo_datasets -from .geo_downloader import download_geo - -__all__ = [ - "main", - "search_geo", - "search_geo_profiles", - "search_geo_datasets", - "download_geo" -] \ No newline at end of file diff --git a/geomcp/geo_downloader.py b/geomcp/geo_downloader.py deleted file mode 100644 index 9dd7073..0000000 --- a/geomcp/geo_downloader.py +++ /dev/null @@ -1,375 +0,0 @@ -import json -import os -import sys -import re -import requests -import asyncio -import aiohttp -import aiofiles -import shutil -from pathlib import Path -from typing import Dict, Any, Optional, List - -CONFIG_PATH = os.getenv("CONFIG_PATH") # optional override - -# ------------------------------------------------------------------ -# Configuration loading -# ------------------------------------------------------------------ - -def _load_config() -> Dict[str, Any]: - """Return the first config.json we can find. - - Search order: - 1. $CONFIG_PATH (env‑var) if it points to a real file - 2. ./config.json next to this script - 3. ~/.geo-mcp/config.json (user default) - """ - candidates: List[Path] = [] - if CONFIG_PATH: - candidates.append(Path(os.path.expanduser(CONFIG_PATH))) - candidates.append(Path(__file__).parent / "config.json") - candidates.append(Path.home() / ".geo-mcp" / "config.json") - - for p in candidates: - if p.exists(): - with open(p) as f: - return json.load(f) - - raise FileNotFoundError("No config.json found. Looked in: " + ", ".join(str(p) for p in candidates)) - - -config = _load_config() - -# ------------------------------------------------------------------ -# Global constants (with sane defaults) -# ------------------------------------------------------------------ -BASE_URL = config.get("base_url", "https://eutils.ncbi.nlm.nih.gov/entrez/eutils") -EMAIL = config.get("email") # this *must* be set – NCBI requirement -API_KEY = config.get("api_key", "") -DOWNLOAD_DIR = config.get("download_dir", "./downloads") -MAX_FILE_MB = config.get("max_file_size_mb", 5000) -MAX_TOTAL_MB = config.get("max_total_downloads_mb", 10000) -MAX_CONCURRENT = config.get("max_concurrent_downloads", 3) -TIMEOUT = config.get("download_timeout_seconds", 300) -ALLOWED_PATHS = config.get("allowed_download_paths", ["./downloads", "/tmp/geo_downloads"]) - -if not EMAIL: - print("✘ 'email' is missing in config.json (required by NCBI)", file=sys.stderr) - sys.exit(1) - -# byte helpers -BYTES_IN_MB = 1024 * 1024 -MAX_FILE_BYTES = MAX_FILE_MB * BYTES_IN_MB -MAX_TOTAL_BYTES = MAX_TOTAL_MB * BYTES_IN_MB - -root_path = Path(DOWNLOAD_DIR) -if not root_path.is_absolute(): - root_path = Path(__file__).parent / root_path - -# ------------------------------------------------------------------ -# Utility functions -# ------------------------------------------------------------------ - -def _is_child(path: Path, parent: Path) -> bool: - try: - path.relative_to(parent) - return True - except ValueError: - return False - - -def _allowed(path: Path) -> bool: - path = path.resolve() - for ap in ALLOWED_PATHS: - p = Path(ap) - if not p.is_absolute(): - p = Path(__file__).parent / p - p = p.resolve() - if path == p or _is_child(path, p): - return True - return False - - -def _dir_size(p: Path) -> int: - return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) - - -def _disk_free(p: Path) -> int: - try: - return shutil.disk_usage(p).free - except Exception: - return 0 - -# ------------------------------------------------------------------ -# E‑utilities wrappers -# ------------------------------------------------------------------ - -def _request(path: str, params: Dict[str, str]) -> requests.Response: - prm = params.copy() - prm["email"] = EMAIL - if API_KEY: - prm["api_key"] = API_KEY - resp = requests.get(f"{BASE_URL}/{path}", params=prm) - resp.raise_for_status() - return resp - - -def _esearch_uid(acc: str) -> Optional[str]: - r = _request("esearch.fcgi", { - "db": "gds", - "term": f"{acc}[ACCN]", - "retmode": "json", - "retmax": "1", - }) - ids = r.json().get("esearchresult", {}).get("idlist", []) - return ids[0] if ids else None - - -def _efetch_gds(uid: str) -> str: - return _request("efetch.fcgi", {"db": "gds", "id": uid, "retmode": "xml"}).text - - -# ------------------------------------------------------------------ -# FTP/HTTP link extraction -# ------------------------------------------------------------------ - -def _extract_ftp_links(xml_text: str) -> List[str]: - """Return *https* direct links to SOFT archives. - - The XML from GDS often includes only an FTP directory. This helper: - • converts any `ftp://` prefix to `https://` (to avoid FTP handling) - • if the link ends with a GEO accession directory, appends the - standard SOFT location (`soft/[_family].soft.gz`). - """ - raw_links = re.findall(r"ftp://[\w./-]+", xml_text, re.I) - cleaned: List[str] = [] - - for link in raw_links: - base = re.sub(r"^ftp://", "https://", link.rstrip("/")) - - # if it's already a file we can use it straight away - if base.endswith(".soft.gz"): - cleaned.append(base) - continue - - # otherwise build the conventional SOFT file path - m = re.search(r"/(GSE\d+|GSM\d+|GPL\d+|GDS\d+)$", base, re.I) - if not m: - continue - acc = m.group(1) - soft = f"{base}/soft/{acc}_family.soft.gz" if acc.startswith("GSE") else f"{base}/soft/{acc}.soft.gz" - cleaned.append(soft) - - return cleaned - -# ------------------------------------------------------------------ -# Download logic -# ------------------------------------------------------------------ -sem = asyncio.Semaphore(MAX_CONCURRENT) - - -async def download_geo(acc: str, db_type: str, out_dir: Optional[str] = None) -> Dict[str, Any]: - async with sem: - dest = Path(out_dir or root_path / db_type / acc) - if not _allowed(dest): - raise ValueError("output dir violates ALLOWED_DOWNLOAD_PATHS") - dest.mkdir(parents=True, exist_ok=True) - - if _dir_size(root_path) >= MAX_TOTAL_BYTES: - raise ValueError("total download limit reached") - - uid = _esearch_uid(acc) - if not uid: - raise ValueError(f"{acc} not found in GDS database") - xml = _efetch_gds(uid) - urls = _extract_ftp_links(xml) - if not urls: - raise ValueError("no downloadable SOFT file exposed by E‑utilities") - - downloaded: List[str] = [] - total_bytes = 0 - timeout = aiohttp.ClientTimeout(total=TIMEOUT) - - async with aiohttp.ClientSession(timeout=timeout) as session: - for url in urls: - filename = url.split("/")[-1] - filepath = dest / filename - - if _disk_free(dest) < MAX_FILE_BYTES * 2: - raise ValueError("insufficient disk space to continue") - - async with session.get(url) as resp: - if resp.status != 200: - raise ValueError(f"download failed with HTTP {resp.status}: {url}") - clen = int(resp.headers.get("content-length", "0")) - if clen and clen > MAX_FILE_BYTES: - raise ValueError("SOFT archive exceeds MAX_FILE_SIZE_MB") - - sz = 0 - async with aiofiles.open(filepath, "wb") as fh: - async for chunk in resp.content.iter_chunked(8192): - sz += len(chunk) - if sz > MAX_FILE_BYTES: - raise ValueError("file size grew beyond limit during transfer") - await fh.write(chunk) - total_bytes += sz - downloaded.append(str(filepath)) - - # save XML metadata next to archives - meta_path = dest / f"{acc}_metadata.xml" - meta_path.write_text(xml) - downloaded.append(str(meta_path)) - - return { - "acc": acc, - "db_type": db_type, - "output_dir": str(dest), - "files": downloaded, - "total_size_mb": round(total_bytes / BYTES_IN_MB, 2), - } - -# ------------------------------------------------------------------ -# Status and management functions -# ------------------------------------------------------------------ - -def get_download_status(geo_id: str, db_type: str) -> Dict[str, Any]: - """Check if a GEO dataset has been downloaded.""" - try: - dataset_path = root_path / db_type / geo_id - if dataset_path.exists(): - files = list(dataset_path.glob("*")) - total_size = sum(f.stat().st_size for f in files if f.is_file()) - return { - "geo_id": geo_id, - "db_type": db_type, - "downloaded": True, - "path": str(dataset_path), - "files": [f.name for f in files], - "total_size_mb": round(total_size / BYTES_IN_MB, 2) - } - else: - return { - "geo_id": geo_id, - "db_type": db_type, - "downloaded": False, - "path": str(dataset_path) - } - except Exception as e: - return { - "geo_id": geo_id, - "db_type": db_type, - "downloaded": False, - "error": str(e) - } - -def list_downloaded_datasets(db_type: str = None) -> Dict[str, Any]: - """List all downloaded datasets, optionally filtered by database type.""" - try: - datasets = [] - if db_type: - db_path = root_path / db_type - if db_path.exists(): - for dataset_dir in db_path.iterdir(): - if dataset_dir.is_dir(): - datasets.append({ - "geo_id": dataset_dir.name, - "db_type": db_type, - "path": str(dataset_dir) - }) - else: - for db_dir in root_path.iterdir(): - if db_dir.is_dir(): - for dataset_dir in db_dir.iterdir(): - if dataset_dir.is_dir(): - datasets.append({ - "geo_id": dataset_dir.name, - "db_type": db_dir.name, - "path": str(dataset_dir) - }) - - return { - "datasets": datasets, - "count": len(datasets) - } - except Exception as e: - return { - "error": str(e), - "datasets": [], - "count": 0 - } - -def get_download_stats() -> Dict[str, Any]: - """Get overall download statistics and limits.""" - try: - total_size = _dir_size(root_path) - total_size_mb = round(total_size / BYTES_IN_MB, 2) - - return { - "download_dir": str(root_path), - "total_downloaded_mb": total_size_mb, - "max_total_mb": MAX_TOTAL_MB, - "max_file_mb": MAX_FILE_MB, - "max_concurrent": MAX_CONCURRENT, - "timeout_seconds": TIMEOUT, - "allowed_paths": ALLOWED_PATHS, - "disk_free_mb": round(_disk_free(root_path) / BYTES_IN_MB, 2) - } - except Exception as e: - return { - "error": str(e), - "download_dir": str(root_path) - } - -def cleanup_downloads(geo_id: str = None, db_type: str = None) -> Dict[str, Any]: - """Clean up downloaded files.""" - try: - removed = [] - - if geo_id and db_type: - # Remove specific dataset - dataset_path = root_path / db_type / geo_id - if dataset_path.exists(): - shutil.rmtree(dataset_path) - removed.append(str(dataset_path)) - elif db_type: - # Remove all datasets of a specific type - db_path = root_path / db_type - if db_path.exists(): - for dataset_dir in db_path.iterdir(): - if dataset_dir.is_dir(): - shutil.rmtree(dataset_dir) - removed.append(str(dataset_dir)) - else: - # Remove all downloads - if root_path.exists(): - shutil.rmtree(root_path) - removed.append(str(root_path)) - - return { - "removed": removed, - "count": len(removed) - } - except Exception as e: - return { - "error": str(e), - "removed": [], - "count": 0 - } - -# ------------------------------------------------------------------ -# Minimal CLI example -# ------------------------------------------------------------------ -if __name__ == "__main__": - import argparse, json as _json - - parser = argparse.ArgumentParser(description="Download GEO SOFT archives using E‑utilities only") - parser.add_argument("acc", nargs="?", default="GSE10072", help="GEO accession (e.g. GSE10072)") - parser.add_argument("--db", dest="db", default="gse", help="Database type: gse/gsm/gpl/gds") - args = parser.parse_args() - - try: - result = asyncio.run(download_geo(args.acc, args.db)) - print(_json.dumps(result, indent=2)) - except Exception as exc: - print(f"✘ {exc}", file=sys.stderr) - sys.exit(1) diff --git a/geomcp/geo_profiles.py b/geomcp/geo_profiles.py deleted file mode 100644 index a7a6a76..0000000 --- a/geomcp/geo_profiles.py +++ /dev/null @@ -1,431 +0,0 @@ -import json -import os -import requests -from pathlib import Path -import sys -import time - -# Load configuration from JSON file -CONFIG_PATH = os.getenv("CONFIG_PATH", "config.json") - -def load_config(): - """Load configuration from JSON file with fallback to defaults.""" - try: - # Try to load config from the specified path - config_file = Path(CONFIG_PATH) - if not config_file.is_absolute(): - # If relative path, make it relative to the directory containing this script - script_dir = Path(__file__).parent - config_file = script_dir / config_file - - if not config_file.exists(): - print(f"Config file not found: {config_file}", file=sys.stderr) - raise FileNotFoundError(f"Config file not found: {config_file}") - - with open(config_file, 'r') as cfg_file: - return json.load(cfg_file) - except Exception as e: - print(f"Error loading config from {CONFIG_PATH}: {e}", file=sys.stderr) - print("Please run `geo-mcp --init` to create a config file.", file=sys.stderr) - raise e - -def _get_config(): - """Get configuration, loading it when needed.""" - try: - return load_config() - except Exception: - # Return default config for basic functionality - return { - "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", - "email": None, - "api_key": None - } - -def _esearch(db: str, term: str, retmax: int = 20) -> dict: - """Perform an ESearch query and return JSON results.""" - config = _get_config() - email = config.get("email") - - # Make email optional with warning - if not email: - print("Warning: No email configured for NCBI E-Utils. Consider adding one for better compliance.", file=sys.stderr) - - params = { - 'db': db, - 'term': term, - 'retmax': retmax, - 'retmode': 'json', - } - - # Only add email if configured - if email: - params['email'] = email - - api_key = config.get("api_key") - if api_key: - params['api_key'] = api_key - - # Add rate limiting to be respectful to NCBI servers - time.sleep(0.1) - - try: - resp = requests.get(f"{config.get('base_url', 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils')}/esearch.fcgi", params=params, timeout=30) - resp.raise_for_status() - return resp.json() - except requests.exceptions.RequestException as e: - print(f"Error during esearch: {e}", file=sys.stderr) - raise - -def _esummary(db: str, ids: list) -> dict: - """Fetch summaries for a list of IDs.""" - if not ids: - return {"result": {}} - - config = _get_config() - email = config.get("email") - - params = { - 'db': db, - 'id': ','.join(map(str, ids)), - 'retmode': 'json', - } - - # Only add email if configured - if email: - params['email'] = email - - api_key = config.get("api_key") - if api_key: - params['api_key'] = api_key - - # Add rate limiting - time.sleep(0.1) - - try: - resp = requests.get(f"{config.get('base_url', 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils')}/esummary.fcgi", params=params, timeout=30) - resp.raise_for_status() - return resp.json() - except requests.exceptions.RequestException as e: - print(f"Error during esummary: {e}", file=sys.stderr) - raise - -def search_geo(term: str, retmax: int = 20, record_types: list = None) -> dict: - """ - Universal GEO search function that finds all relevant records. - - Args: - term: Search term (e.g., "breast cancer", "GSE12345", "RNA-seq") - retmax: Maximum number of results to return - record_types: Optional filter for specific types ["GSE", "GSM", "GPL", "GDS"] - - Returns: - Dict with categorized results by record type - """ - try: - # Search the gds database which contains most GEO records - data = _esearch('gds', term, retmax) - ids = data.get('esearchresult', {}).get('idlist', []) - - if not ids: - return { - "total_count": 0, - "results": [], - "series": [], - "samples": [], - "platforms": [], - "datasets": [] - } - - # Get detailed summaries - summaries = _esummary('gds', ids) - results = summaries.get('result', {}) - - # Categorize results by accession type - categorized = { - "total_count": len(ids), - "results": [], - "series": [], # GSE records - "samples": [], # GSM records - "platforms": [], # GPL records - "datasets": [] # GDS records - } - - for uid in ids: - if uid in results: - record = results[uid] - accession = record.get('accession', '') - - # Add to main results - categorized["results"].append(record) - - # Categorize by type - if accession.startswith('GSE'): - categorized["series"].append(record) - elif accession.startswith('GSM'): - categorized["samples"].append(record) - elif accession.startswith('GPL'): - categorized["platforms"].append(record) - elif accession.startswith('GDS'): - categorized["datasets"].append(record) - - # Filter by record types if specified - if record_types: - record_types = [rt.upper() for rt in record_types] - filtered_results = [] - - if "GSE" in record_types: - filtered_results.extend(categorized["series"]) - if "GSM" in record_types: - filtered_results.extend(categorized["samples"]) - if "GPL" in record_types: - filtered_results.extend(categorized["platforms"]) - if "GDS" in record_types: - filtered_results.extend(categorized["datasets"]) - - categorized["results"] = filtered_results - categorized["total_count"] = len(filtered_results) - - return categorized - - except Exception as e: - print(f"Error in search_geo: {e}", file=sys.stderr) - return { - "total_count": 0, - "results": [], - "series": [], - "samples": [], - "platforms": [], - "datasets": [], - "error": str(e) - } - -def search_geo_profiles(term: str, retmax: int = 20) -> dict: - """Search GEO Profiles - keeping original functionality.""" - try: - data = _esearch('geoprofiles', term, retmax) - ids = data.get('esearchresult', {}).get('idlist', []) - if not ids: - return {"esummaryresult": ["Empty id list - nothing todo"]} - summary = _esummary('geoprofiles', ids) - return summary - except Exception as e: - print(f"Error in search_geo_profiles: {e}", file=sys.stderr) - return {"esummaryresult": ["Empty id list - nothing todo"], "error": str(e)} - -def search_geo_datasets(term: str, retmax: int = 20) -> dict: - """Search for GEO Dataset (GDS) records only.""" - try: - result = search_geo(term, retmax, record_types=["GDS"]) - - # Format to match expected output structure - if result["datasets"]: - formatted_result = { - "header": {"type": "esummary", "version": "0.3"}, - "result": { - "uids": [r.get("uid") for r in result["datasets"]] - } - } - # Add each record to the result dict - for record in result["datasets"]: - uid = record.get("uid") - if uid: - formatted_result["result"][uid] = record - - return formatted_result - else: - return {"esummaryresult": ["Empty id list - nothing todo"]} - - except Exception as e: - print(f"Error in search_geo_datasets: {e}", file=sys.stderr) - return {"esummaryresult": ["Empty id list - nothing todo"], "error": str(e)} - -def search_geo_series(term: str, retmax: int = 20) -> dict: - """Search for GEO Series (GSE) records only.""" - try: - result = search_geo(term, retmax, record_types=["GSE"]) - - # Format to match expected output structure - if result["series"]: - formatted_result = { - "header": {"type": "esummary", "version": "0.3"}, - "result": { - "uids": [r.get("uid") for r in result["series"]] - } - } - # Add each record to the result dict - for record in result["series"]: - uid = record.get("uid") - if uid: - formatted_result["result"][uid] = record - - return formatted_result - else: - return {"esummaryresult": ["Empty id list - nothing todo"]} - - except Exception as e: - print(f"Error in search_geo_series: {e}", file=sys.stderr) - return {"esummaryresult": ["Empty id list - nothing todo"], "error": str(e)} - -def search_geo_samples(term: str, retmax: int = 20) -> dict: - """Search for GEO Sample (GSM) records only.""" - try: - result = search_geo(term, retmax, record_types=["GSM"]) - - # Format to match expected output structure - if result["samples"]: - formatted_result = { - "header": {"type": "esummary", "version": "0.3"}, - "result": { - "uids": [r.get("uid") for r in result["samples"]] - } - } - # Add each record to the result dict - for record in result["samples"]: - uid = record.get("uid") - if uid: - formatted_result["result"][uid] = record - - return formatted_result - else: - return {"esummaryresult": ["Empty id list - nothing todo"]} - - except Exception as e: - print(f"Error in search_geo_samples: {e}", file=sys.stderr) - return {"esummaryresult": ["Empty id list - nothing todo"], "error": str(e)} - -def search_geo_platforms(term: str, retmax: int = 20) -> dict: - """Search for GEO Platform (GPL) records only.""" - try: - result = search_geo(term, retmax, record_types=["GPL"]) - - # Format to match expected output structure - if result["platforms"]: - formatted_result = { - "header": {"type": "esummary", "version": "0.3"}, - "result": { - "uids": [r.get("uid") for r in result["platforms"]] - } - } - # Add each record to the result dict - for record in result["platforms"]: - uid = record.get("uid") - if uid: - formatted_result["result"][uid] = record - - return formatted_result - else: - return {"esummaryresult": ["Empty id list - nothing todo"]} - - except Exception as e: - print(f"Error in search_geo_platforms: {e}", file=sys.stderr) - return {"esummaryresult": ["Empty id list - nothing todo"], "error": str(e)} - -def download_geo_data(geo_id: str, output_dir: str = "downloads") -> str: - """Download GEO data file.""" - try: - # Create output directory if it doesn't exist - Path(output_dir).mkdir(parents=True, exist_ok=True) - - # Determine the correct URL based on GEO ID type - if geo_id.startswith('GSE'): - # Series data - url = f"https://ftp.ncbi.nlm.nih.gov/geo/series/{geo_id[:-3]}nnn/{geo_id}/matrix/{geo_id}_series_matrix.txt.gz" - elif geo_id.startswith('GDS'): - # Dataset data - url = f"https://ftp.ncbi.nlm.nih.gov/geo/datasets/{geo_id[:-3]}nnn/{geo_id}/soft/{geo_id}.soft.gz" - elif geo_id.startswith('GPL'): - # Platform data - url = f"https://ftp.ncbi.nlm.nih.gov/geo/platforms/{geo_id[:-3]}nnn/{geo_id}/annot/{geo_id}.annot.gz" - else: - raise ValueError(f"Unsupported GEO ID format: {geo_id}") - - filename = os.path.join(output_dir, url.split('/')[-1]) - - print(f"Downloading {geo_id} from {url}...", file=sys.stderr) - - response = requests.get(url, stream=True, timeout=300) - response.raise_for_status() - - with open(filename, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - print(f"Downloaded: {filename}", file=sys.stderr) - return filename - - except Exception as e: - print(f"Error downloading {geo_id}: {e}", file=sys.stderr) - raise - -def list_downloaded_datasets(output_dir: str = "downloads") -> list: - """List all downloaded datasets.""" - try: - download_path = Path(output_dir) - if not download_path.exists(): - return [] - - files = list(download_path.glob("*")) - return [str(f) for f in files if f.is_file()] - except Exception as e: - print(f"Error listing downloads: {e}", file=sys.stderr) - return [] - -def get_download_stats(output_dir: str = "downloads") -> dict: - """Get statistics about downloaded files.""" - try: - files = list_downloaded_datasets(output_dir) - total_size = 0 - - for file_path in files: - try: - total_size += Path(file_path).stat().st_size - except OSError: - continue - - return { - "total_files": len(files), - "total_size_bytes": total_size, - "total_size_mb": round(total_size / (1024 * 1024), 2), - "files": files - } - except Exception as e: - print(f"Error getting download stats: {e}", file=sys.stderr) - return {"total_files": 0, "total_size_bytes": 0, "total_size_mb": 0, "files": []} - -if __name__ == '__main__': - # Example usage - try: - term = 'cancer' - print(f"Searching for: {term}") - - # Test the new universal search function - print("\n=== Testing universal search function ===") - all_results = search_geo(term, 10) - print(f"Total found: {all_results['total_count']}") - print(f"- Series (GSE): {len(all_results['series'])}") - print(f"- Samples (GSM): {len(all_results['samples'])}") - print(f"- Platforms (GPL): {len(all_results['platforms'])}") - print(f"- Datasets (GDS): {len(all_results['datasets'])}") - - # Test individual functions - print("\n=== Testing individual search functions ===") - - datasets = search_geo_datasets(term, 5) - dataset_count = len(datasets.get('result', {}).get('uids', [])) - print(f"DataSets found: {dataset_count}") - - series = search_geo_series(term, 5) - series_count = len(series.get('result', {}).get('uids', [])) - print(f"Series found: {series_count}") - - samples = search_geo_samples(term, 5) - samples_count = len(samples.get('result', {}).get('uids', [])) - print(f"Samples found: {samples_count}") - - platforms = search_geo_platforms("Illumina", 5) - platforms_count = len(platforms.get('result', {}).get('uids', [])) - print(f"Illumina platforms found: {platforms_count}") - - except Exception as e: - print(f"Error in main: {e}", file=sys.stderr) - sys.exit(1) \ No newline at end of file diff --git a/geomcp/main.py b/geomcp/main.py deleted file mode 100644 index e65d03d..0000000 --- a/geomcp/main.py +++ /dev/null @@ -1,300 +0,0 @@ -import asyncio -import argparse -import os -import sys -import json -import shutil -from pathlib import Path - -def setup_environment(): - """Set up the environment for the MCP server.""" - # Set the working directory to the script's directory - script_dir = Path(__file__).parent.absolute() - os.chdir(script_dir) - - # Add the script directory to Python path to ensure local imports work - if str(script_dir) not in sys.path: - sys.path.insert(0, str(script_dir)) - - # Set CONFIG_PATH environment variable if not already set - if not os.getenv("CONFIG_PATH"): - config_dir = Path.home() / ".geo-mcp" - config_path = config_dir / "config.json" - os.environ["CONFIG_PATH"] = str(config_path) - - # Get the config path - config_path = Path(os.getenv("CONFIG_PATH", str(Path.home() / ".geo-mcp" / "config.json"))) - - # If config file doesn't exist, create it from template - if not config_path.exists(): - template_path = script_dir / "config_template.json" - if template_path.exists(): - # Create parent directories if they don't exist - config_path.parent.mkdir(parents=True, exist_ok=True) - # Copy template to config location - shutil.copy2(template_path, config_path) - print(f"Created default configuration file at: {config_path}", file=sys.stderr) - else: - print(f"Config file not found: {config_path}", file=sys.stderr) - print(f"Template file not found: {template_path}", file=sys.stderr) - print(f"Current working directory: {os.getcwd()}", file=sys.stderr) - print(f"Available files in current directory: {list(Path('.').glob('*'))}", file=sys.stderr) - sys.exit(1) - - # point any child-spawns at the venv python - venv_bin = os.path.join(os.path.dirname(__file__), ".venv", "bin") - os.environ["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "") - -def run_http_server(host: str = "localhost", port: int = 8001): - """Run the HTTP server.""" - import uvicorn - - # Check if we're running as a package or as a script - try: - from .mcp_http_server import app - except ImportError: - # Running as script, use absolute import - from mcp_http_server import app - - print(f"Starting HTTP server on http://{host}:{port}") - uvicorn.run(app, host=host, port=port) - -async def run_mcp_server(): - """Run the MCP stdio server.""" - import mcp.server.stdio - - # Check if we're running as a package or as a script - try: - from .mcp_server import server - except ImportError: - # Running as script, use absolute import - from mcp_server import server - - try: - async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): - await server.run( - read_stream, - write_stream, - server.create_initialization_options() - ) - except Exception as e: - print(f"Error starting MCP server: {e}", file=sys.stderr) - sys.exit(1) - -def init_config(config_path: Path = None): - """Initialize a new configuration file with user input.""" - if config_path is None: - # Use the same logic as setup_environment for consistency - if not os.getenv("CONFIG_PATH"): - config_dir = Path.home() / ".geo-mcp" - config_path = config_dir / "config.json" - else: - config_path = Path(os.getenv("CONFIG_PATH")) - - # Create parent directories if they don't exist - config_path.parent.mkdir(parents=True, exist_ok=True) - - print("GEO MCP Server Configuration Initialization") - print("=" * 50) - - # Get user input - email = input("Enter your email address (required for NCBI E-utilities): ").strip() - if not email: - print("Error: Email address is required!") - sys.exit(1) - - api_key = input("Enter your NCBI API key (optional, press Enter to skip): ").strip() - if not api_key: - api_key = "" - print("Note: Without an API key, you'll be limited to 3 requests/second") - else: - print("Note: With an API key, you'll have 10 requests/second limit") - - # Create config with user input - config = { - "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", - "email": email, - "api_key": api_key, - "retmax": 20, - "download_dir": "./downloads", - "max_file_size_mb": 5000, - "max_total_downloads_mb": 10000, - "max_concurrent_downloads": 3, - "download_timeout_seconds": 300, - "allowed_download_paths": ["./downloads", "/tmp/geo_downloads"] - } - - # Find absolute path to geo-mcp executable - geo_mcp_path = shutil.which("geo-mcp") or "geo-mcp" - if geo_mcp_path == "geo-mcp": - print("WARNING: Could not find absolute path to geo-mcp executable. Falling back to 'geo-mcp'.", file=sys.stderr) - - # Write config file - try: - print(f"Creating config file at: {config_path}") - print(f"config: {config}") - with open(config_path, 'w') as f: - json.dump(config, f, indent=4) - print(f"""\ - - Configuration file created successfully at: {config_path} - - You can now run the server with: - {geo_mcp_path} --http - {geo_mcp_path} - - ================================================== - CLAUDE DESKTOP CONFIGURATION - ================================================== - Add the following to your Claude Desktop configuration file: - (Usually located at ~/.config/claude-desktop/config.json) - - WARNING: INSERT CORRECT PATH TO CONFIG FILE BELOW - - {{ - "mcpServers": {{ - "geo-mcp": {{ - "command": "{geo_mcp_path}", - "env": {{ - "CONFIG_PATH": "{config_path}" - }} - }} - }} - }} - - After adding this configuration: - 1. Restart Claude Desktop - 2. You should see GEO tools available in Claude - """) - return True - except Exception as e: - print(f"Error creating config file: {e}") - return False - -def main(): - """Main entry point for the GEO MCP server.""" - parser = argparse.ArgumentParser( - description="GEO MCP Server - Access GEO data through Model Context Protocol", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - geo-mcp --init # Initialize configuration file - geo-mcp # Run MCP stdio server - geo-mcp --http # Run HTTP server on localhost:8001 - geo-mcp --http --port 8080 # Run HTTP server on port 8080 - geo-mcp --http --host 0.0.0.0 --port 8080 # Run HTTP server on all interfaces - """ - ) - - parser.add_argument( - "--init", - action="store_true", - help="Initialize configuration file with interactive prompts" - ) - - parser.add_argument( - "--http", - action="store_true", - help="Run HTTP server instead of MCP stdio server" - ) - - parser.add_argument( - "--host", - default="localhost", - help="Host for HTTP server (default: localhost)" - ) - - parser.add_argument( - "--port", - type=int, - default=8001, - help="Port for HTTP server (default: 8001)" - ) - - # Configuration options - parser.add_argument( - "--email", - help="Email address for NCBI E-utilities (required)" - ) - - parser.add_argument( - "--api-key", - help="NCBI API key for higher rate limits (optional)" - ) - - parser.add_argument( - "--retmax", - type=int, - default=20, - help="Maximum number of search results to return (default: 20)" - ) - - parser.add_argument( - "--download-dir", - default="./downloads", - help="Directory where downloads will be stored (default: ./downloads)" - ) - - parser.add_argument( - "--max-file-size-mb", - type=int, - default=5000, - help="Maximum size of individual files to download in MB (default: 5000)" - ) - - parser.add_argument( - "--max-total-downloads-mb", - type=int, - default=10000, - help="Maximum total size of all downloads in MB (default: 10000)" - ) - - parser.add_argument( - "--max-concurrent-downloads", - type=int, - default=3, - help="Maximum number of concurrent downloads (default: 3)" - ) - - parser.add_argument( - "--download-timeout-seconds", - type=int, - default=300, - help="Timeout for download requests in seconds (default: 300)" - ) - - parser.add_argument( - "--allowed-download-paths", - nargs="+", - default=["./downloads", "/tmp/geo_downloads"], - help="List of allowed download paths for security (default: ./downloads /tmp/geo_downloads)" - ) - - parser.add_argument( - "--version", - action="version", - version="geo-mcp 0.1.1" - ) - - args = parser.parse_args() - - # Handle init command first, before any environment setup - if args.init: - success = init_config() - if success: - sys.exit(0) - else: - sys.exit(1) - - # Set up environment only for non-init commands - setup_environment() - - if args.http: - # Run HTTP server - run_http_server(args.host, args.port) - else: - # Run MCP stdio server - asyncio.run(run_mcp_server()) - -if __name__ == "__main__": - main() diff --git a/geomcp/mcp_http_server.py b/geomcp/mcp_http_server.py deleted file mode 100755 index 7b9129b..0000000 --- a/geomcp/mcp_http_server.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -"""HTTP gateway for GEO-MCP on http://localhost:8001""" -import asyncio, os, sys -from pathlib import Path -from typing import Dict, Any, List -import json -from collections import deque - -from fastapi import FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse -from pydantic import BaseModel -import uvicorn -from .mcp_server import server - -# Global event queue for SSE -event_queue = deque(maxlen=100) # Keep last 100 events -connected_clients = set() - -def publish_event(event_type: str, data: Any): - """Publish an event to all connected SSE clients.""" - event = { - "type": event_type, - "data": data, - "timestamp": asyncio.get_event_loop().time() - } - event_queue.append(event) - # Notify all connected clients - for client in connected_clients.copy(): - if not client.is_disconnected(): - client.put_nowait(event) - - -async def _get_tools() -> List[Any]: - """Return the MCP tool-model list or raise.""" - try: - # Import the handle_list_tools function directly - from .mcp_server import handle_list_tools - - # Call the async function directly - tools = await handle_list_tools() - if isinstance(tools, list): - return tools - except Exception as e: - print(f"Error getting tools: {e}", file=sys.stderr) - - # Fallback: try to access tools from server object - for attr in ("tools", "_tools"): - if hasattr(server, attr): - tools = getattr(server, attr) - if isinstance(tools, list): - return tools - - try: - maybe = server.list_tools() - if isinstance(maybe, list): - return maybe - except TypeError: - pass - - raise RuntimeError("Could not locate tool registry in mcp_server.server") - - -# Create the FastAPI app at module level -app = FastAPI(title="GEO MCP Server", version="1.0.0") -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], allow_methods=["*"], allow_headers=["*"] -) - -class ToolCallRequest(BaseModel): - name: str - arguments: Dict[str, Any] - -class ToolCallResponse(BaseModel): - result: List[Dict[str, Any]] - -@app.get("/") -async def root(): - return {"status": "healthy"} - -@app.get("/tools", response_model=List[Dict[str, Any]]) -async def list_tools(): - try: - return [t.model_dump() for t in await _get_tools()] - except Exception as e: - raise HTTPException(500, f"Error listing tools: {e}") - -@app.post("/tools/call", response_model=ToolCallResponse) -async def call_tool(req: ToolCallRequest): - try: - # Publish tool call start event - publish_event("tool_call_start", { - "tool": req.name, - "arguments": req.arguments - }) - - # Import the handle_call_tool function directly - from .mcp_server import handle_call_tool - - # Call the async function - out = await handle_call_tool(req.name, req.arguments) - result = [o.model_dump() for o in out] - - # Publish tool call completion event - publish_event("tool_call_complete", { - "tool": req.name, - "arguments": req.arguments, - "result": result - }) - - return ToolCallResponse(result=result) - except ValueError as e: - # Publish error event - publish_event("tool_call_error", { - "tool": req.name, - "arguments": req.arguments, - "error": str(e) - }) - raise HTTPException(400, f"Invalid tool call: {e}") - except Exception as e: - # Publish error event - publish_event("tool_call_error", { - "tool": req.name, - "arguments": req.arguments, - "error": str(e) - }) - raise HTTPException(500, f"Error calling tool: {e}") - -@app.get("/health") -async def health(): - try: - return {"status": "healthy", "tools_available": len(await _get_tools())} - except Exception as e: - return {"status": "error", "error": str(e)} - -@app.get("/events") -async def events(request: Request): - # Create a queue for this client - client_queue = asyncio.Queue() - connected_clients.add(client_queue) - - try: - # Send initial connection event - await client_queue.put({ - "type": "connection_established", - "data": {"message": "Connected to GEO-MCP server"}, - "timestamp": asyncio.get_event_loop().time() - }) - - # Send recent events (last 10) - recent_events = list(event_queue)[-10:] - for event in recent_events: - await client_queue.put(event) - - async def event_generator(): - while True: - if await request.is_disconnected(): - break - - try: - # Wait for new events with timeout - event = await asyncio.wait_for(client_queue.get(), timeout=30.0) - yield f"data: {json.dumps(event)}\n\n" - except asyncio.TimeoutError: - # Send heartbeat to keep connection alive - yield f"data: {json.dumps({'type': 'heartbeat', 'data': {}, 'timestamp': asyncio.get_event_loop().time()})}\n\n" - except Exception as e: - # Send error event - yield f"data: {json.dumps({'type': 'error', 'data': {'error': str(e)}, 'timestamp': asyncio.get_event_loop().time()})}\n\n" - break - - return StreamingResponse(event_generator(), media_type="text/event-stream") - - finally: - # Clean up when client disconnects - connected_clients.discard(client_queue) - - -async def main() -> None: - os.chdir(Path(__file__).parent) - os.environ.setdefault("CONFIG_PATH", str(Path("config.json"))) - - print("Starting GEO-MCP HTTP server on http://localhost:8001") - await uvicorn.Server(uvicorn.Config(app, host="localhost", port=8001, log_level="info")).serve() - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/geomcp/mcp_server.py b/geomcp/mcp_server.py deleted file mode 100644 index 89c8c51..0000000 --- a/geomcp/mcp_server.py +++ /dev/null @@ -1,544 +0,0 @@ -#!/usr/bin/env python3 -""" -MCP Server for GEO (Gene Expression Omnibus) Data -Handles MCP protocol and tool definitions for accessing GEO data through NCBI E-Utils -""" - -import json -import logging -import asyncio -from typing import Any, Dict, List -from mcp.server import Server -import mcp.types as types -from . import geo_profiles, geo_downloader - -logger = logging.getLogger("geo-mcp-server") - - -class GEOMCPServer: - """ - MCP Server implementation for GEO Data Access - Handles MCP protocol and tool definitions for Gene Expression Omnibus - """ - - def __init__(self): - self.server = Server("geo-mcp") - self._setup_tools() - - def _setup_tools(self): - """Register all available GEO tools""" - - @self.server.call_tool() - async def search_geo(arguments: dict) -> list[types.TextContent]: - """Search GEO for all types of records (GSE, GSM, GPL, GDS)""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - record_types = arguments.get("record_types") - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo(term, retmax, record_types) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # GEO Profiles search - @self.server.call_tool() - async def search_geo_profiles(arguments: dict) -> list[types.TextContent]: - """Search GEO Profiles database""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo_profiles(term, retmax) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # GEO Datasets search - @self.server.call_tool() - async def search_geo_datasets(arguments: dict) -> list[types.TextContent]: - """Search GEO Datasets (GDS) specifically""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo_datasets(term, retmax) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # GEO Series search - @self.server.call_tool() - async def search_geo_series(arguments: dict) -> list[types.TextContent]: - """Search GEO Series (GSE) specifically""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo_series(term, retmax) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # GEO Samples search - @self.server.call_tool() - async def search_geo_samples(arguments: dict) -> list[types.TextContent]: - """Search GEO Samples (GSM) specifically""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo_samples(term, retmax) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # GEO Platforms search - @self.server.call_tool() - async def search_geo_platforms(arguments: dict) -> list[types.TextContent]: - """Search GEO Platforms (GPL) specifically""" - term = arguments.get("term", "") - retmax = arguments.get("retmax", 20) - - if not term: - raise ValueError("term parameter is required") - - result = geo_profiles.search_geo_platforms(term, retmax) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # Download GEO data - @self.server.call_tool() - async def download_geo_data(arguments: dict) -> list[types.TextContent]: - """Download GEO data files""" - geo_id = arguments.get("geo_id", "") - db_type = arguments.get("db_type", "gse") - output_dir = arguments.get("output_dir") - - if not geo_id: - raise ValueError("geo_id parameter is required") - - result = await geo_downloader.download_geo(geo_id, db_type, output_dir) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # Get download status - @self.server.call_tool() - async def get_download_status(arguments: dict) -> list[types.TextContent]: - """Check download status of a GEO dataset""" - geo_id = arguments.get("geo_id", "") - db_type = arguments.get("db_type", "gse") - - if not geo_id: - raise ValueError("geo_id parameter is required") - - result = geo_downloader.get_download_status(geo_id, db_type) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # List downloaded datasets - @self.server.call_tool() - async def list_downloaded_datasets(arguments: dict) -> list[types.TextContent]: - """List all downloaded GEO datasets""" - db_type = arguments.get("db_type") - - result = geo_downloader.list_downloaded_datasets(db_type) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # Get download statistics - @self.server.call_tool() - async def get_download_stats(arguments: dict) -> list[types.TextContent]: - """Get download statistics and limits""" - result = geo_downloader.get_download_stats() - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - # Cleanup downloads - @self.server.call_tool() - async def cleanup_downloads_tool(arguments: dict) -> list[types.TextContent]: - """Clean up downloaded files""" - geo_id = arguments.get("geo_id") - db_type = arguments.get("db_type") - - result = geo_downloader.cleanup_downloads(geo_id, db_type) - - return [types.TextContent( - type="text", - text=json.dumps(result, indent=2) - )] - - def get_server(self) -> Server: - """Get the configured MCP server""" - return self.server - - def get_tool_definitions(self) -> List[types.Tool]: - """Get all tool definitions for the GEO MCP server""" - return [ - # Universal GEO search - types.Tool( - name="search_geo", - description="Search GEO for all types of records (GSE, GSM, GPL, GDS)", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term (e.g., 'breast cancer', 'GSE12345', 'RNA-seq')" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - }, - "record_types": { - "type": "array", - "items": {"type": "string"}, - "description": "Filter for specific record types: GSE, GSM, GPL, GDS" - } - }, - "required": ["term"] - } - ), - - # GEO Profiles search - types.Tool( - name="search_geo_profiles", - description="Search GEO Profiles database for gene expression profiles", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term for GEO Profiles" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - } - }, - "required": ["term"] - } - ), - - # GEO Datasets search - types.Tool( - name="search_geo_datasets", - description="Search GEO Datasets (GDS) - curated gene expression datasets", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term for GEO Datasets" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - } - }, - "required": ["term"] - } - ), - - # GEO Series search - types.Tool( - name="search_geo_series", - description="Search GEO Series (GSE) - complete experiments", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term for GEO Series" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - } - }, - "required": ["term"] - } - ), - - # GEO Samples search - types.Tool( - name="search_geo_samples", - description="Search GEO Samples (GSM) - individual samples", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term for GEO Samples" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - } - }, - "required": ["term"] - } - ), - - # GEO Platforms search - types.Tool( - name="search_geo_platforms", - description="Search GEO Platforms (GPL) - array/sequencing platforms", - inputSchema={ - "type": "object", - "properties": { - "term": { - "type": "string", - "description": "Search term for GEO Platforms" - }, - "retmax": { - "type": "integer", - "description": "Maximum number of results to return (default: 20)", - "default": 20 - } - }, - "required": ["term"] - } - ), - - # Download tool - types.Tool( - name="download_geo_data", - description="Download GEO data files (SOFT format)", - inputSchema={ - "type": "object", - "properties": { - "geo_id": { - "type": "string", - "description": "GEO accession ID (e.g., GSE12345, GSM789, GPL456, GDS123)" - }, - "db_type": { - "type": "string", - "description": "Database type: gse, gsm, gpl, or gds (default: gse)", - "default": "gse" - }, - "output_dir": { - "type": "string", - "description": "Optional custom output directory" - } - }, - "required": ["geo_id"] - } - ), - - # Download status - types.Tool( - name="get_download_status", - description="Check if a GEO dataset has been downloaded", - inputSchema={ - "type": "object", - "properties": { - "geo_id": { - "type": "string", - "description": "GEO accession ID" - }, - "db_type": { - "type": "string", - "description": "Database type: gse, gsm, gpl, or gds (default: gse)", - "default": "gse" - } - }, - "required": ["geo_id"] - } - ), - - # List downloads - types.Tool( - name="list_downloaded_datasets", - description="List all downloaded GEO datasets", - inputSchema={ - "type": "object", - "properties": { - "db_type": { - "type": "string", - "description": "Optional filter by database type: gse, gsm, gpl, or gds" - } - } - } - ), - - # Download stats - types.Tool( - name="get_download_stats", - description="Get download statistics and limits", - inputSchema={ - "type": "object", - "properties": {} - } - ), - - # Cleanup downloads - types.Tool( - name="cleanup_downloads_tool", - description="Clean up downloaded files", - inputSchema={ - "type": "object", - "properties": { - "geo_id": { - "type": "string", - "description": "Optional specific GEO ID to remove" - }, - "db_type": { - "type": "string", - "description": "Optional database type filter for cleanup" - } - } - } - ) - ] - - -# Create the server instance -mcp_server = GEOMCPServer() -server = mcp_server.get_server() - -# List tools for Claude Desktop integration -@server.list_tools() -async def handle_list_tools() -> list[types.Tool]: - """Return the list of available tools""" - return mcp_server.get_tool_definitions() - -# Handle tool calls for HTTP server -async def handle_call_tool(name: str, arguments: dict): - """Handle tool calls from HTTP server""" - # Get the server's tool handlers - server_instance = mcp_server.get_server() - - # Call the tool through the server - from mcp.types import CallToolRequest - request = CallToolRequest( - method="tools/call", - params={ - "name": name, - "arguments": arguments - } - ) - - # Simulate calling the tool - if hasattr(server_instance, '_call_tool_handlers') and name in server_instance._call_tool_handlers: - handler = server_instance._call_tool_handlers[name] - return await handler(arguments) - else: - # Fallback: call functions directly - if name == "search_geo": - from . import geo_profiles - result = geo_profiles.search_geo( - arguments.get("term", ""), - arguments.get("retmax", 20), - arguments.get("record_types") - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "search_geo_profiles": - from . import geo_profiles - result = geo_profiles.search_geo_profiles( - arguments.get("term", ""), - arguments.get("retmax", 20) - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "search_geo_datasets": - from . import geo_profiles - result = geo_profiles.search_geo_datasets( - arguments.get("term", ""), - arguments.get("retmax", 20) - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "search_geo_series": - from . import geo_profiles - result = geo_profiles.search_geo_series( - arguments.get("term", ""), - arguments.get("retmax", 20) - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "search_geo_samples": - from . import geo_profiles - result = geo_profiles.search_geo_samples( - arguments.get("term", ""), - arguments.get("retmax", 20) - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "search_geo_platforms": - from . import geo_profiles - result = geo_profiles.search_geo_platforms( - arguments.get("term", ""), - arguments.get("retmax", 20) - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "download_geo_data": - from . import geo_downloader - result = await geo_downloader.download_geo( - arguments.get("geo_id", ""), - arguments.get("db_type", "gse"), - arguments.get("output_dir") - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "get_download_status": - from . import geo_downloader - result = geo_downloader.get_download_status( - arguments.get("geo_id", ""), - arguments.get("db_type", "gse") - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "list_downloaded_datasets": - from . import geo_downloader - result = geo_downloader.list_downloaded_datasets( - arguments.get("db_type") - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "get_download_stats": - from . import geo_downloader - result = geo_downloader.get_download_stats() - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - elif name == "cleanup_downloads_tool": - from . import geo_downloader - result = geo_downloader.cleanup_downloads( - arguments.get("geo_id"), - arguments.get("db_type") - ) - return [types.TextContent(type="text", text=json.dumps(result, indent=2))] - else: - raise ValueError(f"Unknown tool: {name}") \ No newline at end of file diff --git a/geomcp_sra/__init__.py b/geomcp_sra/__init__.py new file mode 100644 index 0000000..1597189 --- /dev/null +++ b/geomcp_sra/__init__.py @@ -0,0 +1,10 @@ +""" +GEO MCP Server with SRA Support + +An enhanced MCP server for accessing GEO (Gene Expression Omnibus) data +through NCBI E-Utils API, with additional support for SRA (Sequence Read Archive) +raw data download capabilities. +""" + +__version__ = "0.2.0" +__author__ = "GEO MCP Contributors" diff --git a/geomcp_sra/config.py b/geomcp_sra/config.py new file mode 100644 index 0000000..498fba8 --- /dev/null +++ b/geomcp_sra/config.py @@ -0,0 +1,159 @@ +"""Configuration management for GEO MCP Server.""" + +import json +import os +import sys +from pathlib import Path +from typing import Dict, Any, Optional + + +# Default configuration values +DEFAULT_CONFIG = { + "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", + "sra_base_url": "https://trace.ncbi.nlm.nih.gov/Traces/sra", + "email": None, + "api_key": None, + "retmax": 20, + "download_dir": "./downloads", + "max_file_size_mb": 5000, + "max_total_downloads_mb": 50000, + "max_concurrent_downloads": 3, + "download_timeout_seconds": 300, + "sra_toolkit_path": None, # Path to sra-toolkit (prefetch/fasterq-dump) + "allowed_download_paths": ["./downloads", "/tmp/geo_downloads"], +} + + +def find_config_file() -> Optional[Path]: + """Find configuration file in standard locations.""" + # Check environment variable first + env_config = os.getenv("CONFIG_PATH") + if env_config: + path = Path(env_config).expanduser() + if path.exists(): + return path + + # Check current directory + current_dir = Path.cwd() / "config.json" + if current_dir.exists(): + return current_dir + + # Check module directory + module_dir = Path(__file__).parent / "config.json" + if module_dir.exists(): + return module_dir + + # Check user home directory + home_config = Path.home() / ".geo-mcp" / "config.json" + if home_config.exists(): + return home_config + + return None + + +def load_config(config_path: Optional[Path] = None) -> Dict[str, Any]: + """Load configuration from file or use defaults. + + Args: + config_path: Optional explicit path to config file + + Returns: + Configuration dictionary + """ + config = DEFAULT_CONFIG.copy() + + # Find config file + if config_path is None: + config_path = find_config_file() + + if config_path and config_path.exists(): + try: + with open(config_path, 'r') as f: + user_config = json.load(f) + config.update(user_config) + except (json.JSONDecodeError, IOError) as e: + print(f"Warning: Error loading config from {config_path}: {e}", file=sys.stderr) + + # Override with environment variables + if os.getenv("NCBI_EMAIL"): + config["email"] = os.getenv("NCBI_EMAIL") + if os.getenv("NCBI_API_KEY"): + config["api_key"] = os.getenv("NCBI_API_KEY") + if os.getenv("GEO_DOWNLOAD_DIR"): + config["download_dir"] = os.getenv("GEO_DOWNLOAD_DIR") + if os.getenv("SRA_TOOLKIT_PATH"): + config["sra_toolkit_path"] = os.getenv("SRA_TOOLKIT_PATH") + + return config + + +def validate_config(config: Dict[str, Any]) -> bool: + """Validate configuration values. + + Args: + config: Configuration dictionary + + Returns: + True if valid, raises ValueError otherwise + """ + if not config.get("email"): + raise ValueError( + "Email is required for NCBI E-utilities. " + "Set it in config.json or via NCBI_EMAIL environment variable." + ) + + # Validate download directory + download_dir = Path(config.get("download_dir", "./downloads")) + try: + download_dir.mkdir(parents=True, exist_ok=True) + except OSError as e: + raise ValueError(f"Cannot create download directory: {e}") + + return True + + +def create_config_template(path: Path) -> None: + """Create a configuration file template. + + Args: + path: Path where to create the config file + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + template = { + "base_url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", + "sra_base_url": "https://trace.ncbi.nlm.nih.gov/Traces/sra", + "email": "your_email@example.com", + "api_key": "YOUR_NCBI_API_KEY (optional)", + "retmax": 20, + "download_dir": "./downloads", + "max_file_size_mb": 5000, + "max_total_downloads_mb": 50000, + "max_concurrent_downloads": 3, + "download_timeout_seconds": 300, + "sra_toolkit_path": "/path/to/sra-toolkit/bin (optional, for fasterq-dump)", + "allowed_download_paths": ["./downloads", "/tmp/geo_downloads"] + } + + with open(path, 'w') as f: + json.dump(template, f, indent=4) + + print(f"Configuration template created at: {path}") + + +# Global config instance (lazy loading) +_config: Optional[Dict[str, Any]] = None + + +def get_config() -> Dict[str, Any]: + """Get the global configuration instance. + + Returns: + Configuration dictionary + """ + global _config + if _config is None: + _config = load_config() + validate_config(_config) + return _config diff --git a/geomcp_sra/geo_download.py b/geomcp_sra/geo_download.py new file mode 100644 index 0000000..8db3f8a --- /dev/null +++ b/geomcp_sra/geo_download.py @@ -0,0 +1,463 @@ +"""GEO data download functionality.""" + +import asyncio +import gzip +import json +import os +import re +import shutil +import tarfile +from pathlib import Path +from typing import Dict, Any, List, Optional +import httpx +import aiofiles + +from .config import get_config + + +class GEODownloadError(Exception): + """Exception raised for GEO download errors.""" + pass + + +class GEODownloadClient: + """Client for downloading GEO data files.""" + + # GEO FTP base URL (using HTTPS) + GEO_FTP_BASE = "https://ftp.ncbi.nlm.nih.gov/geo" + + def __init__(self): + self.config = get_config() + self.download_dir = Path(self.config["download_dir"]).resolve() + self.max_file_bytes = self.config.get("max_file_size_mb", 5000) * 1024 * 1024 + self.max_total_bytes = self.config.get("max_total_downloads_mb", 50000) * 1024 * 1024 + self.timeout = self.config.get("download_timeout_seconds", 300) + self.allowed_paths = self.config.get("allowed_download_paths", ["./downloads"]) + + # Create download directory + self.download_dir.mkdir(parents=True, exist_ok=True) + + def _is_allowed_path(self, path: Path) -> bool: + """Check if path is within allowed download directories.""" + path = path.resolve() + for allowed in self.allowed_paths: + allowed_path = Path(allowed).resolve() + try: + path.relative_to(allowed_path) + return True + except ValueError: + continue + return False + + def _get_dir_size(self, path: Path) -> int: + """Calculate total size of files in directory.""" + total = 0 + for f in path.rglob("*"): + if f.is_file(): + total += f.stat().st_size + return total + + def _get_range_dir(self, accession: str) -> str: + """Get range directory for GEO accession. + + GEO uses range directories to avoid too many files in one directory. + E.g., GSE15701 -> GSE15nnn + """ + match = re.match(r'(GSE|GSM|GPL|GDS)(\d+)', accession, re.IGNORECASE) + if match: + prefix = match.group(1).upper() + number = match.group(2) + return f"{prefix}{number[:-3]}nnn" + return accession + + def _build_geo_urls(self, accession: str) -> Dict[str, str]: + """Build download URLs for a GEO accession. + + Args: + accession: GEO accession ID + + Returns: + Dictionary of file types to URLs + """ + urls = {} + prefix = accession[:3].upper() + range_dir = self._get_range_dir(accession) + base_url = f"{self.GEO_FTP_BASE}" + + if prefix == "GSE": + # Series files + base = f"{base_url}/series/{range_dir}/{accession}" + urls["series_matrix"] = f"{base}/matrix/{accession}_series_matrix.txt.gz" + urls["soft"] = f"{base}/soft/{accession}_family.soft.gz" + urls["miniml"] = f"{base}/miniml/{accession}_family.xml.tgz" + urls["supplementary"] = f"{base}/suppl/{accession}_RAW.tar" + elif prefix == "GDS": + # Dataset files + base = f"{base_url}/datasets/{range_dir}/{accession}" + urls["soft"] = f"{base}/soft/{accession}.soft.gz" + urls["soft_full"] = f"{base}/soft/{accession}_full.soft.gz" + elif prefix == "GPL": + # Platform files + base = f"{base_url}/platforms/{range_dir}/{accession}" + urls["annot"] = f"{base}/annot/{accession}.annot.gz" + urls["soft"] = f"{base}/soft/{accession}_family.soft.gz" + urls["supplementary"] = f"{base}/suppl/" + elif prefix == "GSM": + # Sample files + base = f"{base_url}/samples/{range_dir}/{accession}" + urls["supplementary"] = f"{base}/suppl/" + + return urls + + async def download_file( + self, + url: str, + dest_path: Path, + progress_callback: Optional[callable] = None + ) -> Path: + """Download a single file. + + Args: + url: URL to download + dest_path: Destination path + progress_callback: Optional callback for progress updates + + Returns: + Path to downloaded file + """ + if not self._is_allowed_path(dest_path): + raise GEODownloadError(f"Destination path not allowed: {dest_path}") + + # Check total download limit + current_total = self._get_dir_size(self.download_dir) + if current_total >= self.max_total_bytes: + raise GEODownloadError("Total download limit reached") + + # Check disk space + free_space = shutil.disk_usage(dest_path.parent).free + if free_space < self.max_file_bytes: + raise GEODownloadError("Insufficient disk space") + + async with httpx.AsyncClient() as client: + async with client.stream( + "GET", + url, + timeout=self.timeout, + follow_redirects=True + ) as response: + response.raise_for_status() + + # Check content length + content_length = response.headers.get('content-length') + if content_length: + size = int(content_length) + if size > self.max_file_bytes: + raise GEODownloadError( + f"File size ({size} bytes) exceeds maximum allowed" + ) + + # Download file + downloaded = 0 + async with aiofiles.open(dest_path, 'wb') as f: + async for chunk in response.aiter_bytes(chunk_size=8192): + downloaded += len(chunk) + if downloaded > self.max_file_bytes: + dest_path.unlink() + raise GEODownloadError("File size exceeded limit during download") + await f.write(chunk) + + if progress_callback: + progress_callback(downloaded) + + return dest_path + + async def download_geo_series( + self, + gse_id: str, + file_types: Optional[List[str]] = None, + output_dir: Optional[Path] = None + ) -> Dict[str, Any]: + """Download files for a GEO Series. + + Args: + gse_id: GSE accession ID + file_types: List of file types to download (soft, matrix, miniml, supplementary) + output_dir: Optional custom output directory + + Returns: + Download results dictionary + """ + if not gse_id.upper().startswith("GSE"): + raise GEODownloadError(f"Invalid GSE ID: {gse_id}") + + file_types = file_types or ["soft"] + output_dir = output_dir or self.download_dir / "series" / gse_id + output_dir.mkdir(parents=True, exist_ok=True) + + urls = self._build_geo_urls(gse_id) + downloaded = [] + errors = [] + + for file_type in file_types: + if file_type not in urls: + errors.append(f"Unknown file type: {file_type}") + continue + + url = urls[file_type] + filename = url.split("/")[-1] + dest_path = output_dir / filename + + # Skip if already exists + if dest_path.exists(): + downloaded.append({ + "type": file_type, + "path": str(dest_path), + "status": "already_exists", + "size_mb": round(dest_path.stat().st_size / (1024*1024), 2) + }) + continue + + try: + await self.download_file(url, dest_path) + downloaded.append({ + "type": file_type, + "path": str(dest_path), + "status": "downloaded", + "size_mb": round(dest_path.stat().st_size / (1024*1024), 2) + }) + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + errors.append(f"{file_type}: File not found on server") + else: + errors.append(f"{file_type}: HTTP {e.response.status_code}") + except Exception as e: + errors.append(f"{file_type}: {str(e)}") + + return { + "accession": gse_id, + "output_dir": str(output_dir), + "downloaded": downloaded, + "errors": errors, + "total_downloaded": len([d for d in downloaded if d["status"] == "downloaded"]) + } + + async def download_geo_sample( + self, + gsm_id: str, + output_dir: Optional[Path] = None + ) -> Dict[str, Any]: + """Download supplementary files for a GEO Sample. + + Args: + gsm_id: GSM accession ID + output_dir: Optional custom output directory + + Returns: + Download results dictionary + """ + if not gsm_id.upper().startswith("GSM"): + raise GEODownloadError(f"Invalid GSM ID: {gsm_id}") + + output_dir = output_dir or self.download_dir / "samples" / gsm_id + output_dir.mkdir(parents=True, exist_ok=True) + + # Get the supplementary file listing + urls = self._build_geo_urls(gsm_id) + suppl_url = urls.get("supplementary") + + if not suppl_url: + return { + "accession": gsm_id, + "output_dir": str(output_dir), + "downloaded": [], + "errors": ["No supplementary files URL available"] + } + + # For samples, we need to list and download files + # This requires parsing the directory listing + downloaded = [] + errors = [] + + try: + # Try to get directory listing + async with httpx.AsyncClient() as client: + response = await client.get(suppl_url, timeout=30.0) + if response.status_code == 200: + # Parse HTML directory listing + files = self._parse_directory_listing(response.text) + + for filename in files: + if filename.endswith('/'): + continue + + file_url = suppl_url + filename + dest_path = output_dir / filename + + try: + await self.download_file(file_url, dest_path) + downloaded.append({ + "filename": filename, + "path": str(dest_path), + "size_mb": round(dest_path.stat().st_size / (1024*1024), 2) + }) + except Exception as e: + errors.append(f"{filename}: {str(e)}") + else: + errors.append(f"Could not list supplementary files: HTTP {response.status_code}") + except Exception as e: + errors.append(f"Error accessing supplementary files: {str(e)}") + + return { + "accession": gsm_id, + "output_dir": str(output_dir), + "downloaded": downloaded, + "errors": errors, + "total_downloaded": len(downloaded) + } + + def _parse_directory_listing(self, html: str) -> List[str]: + """Parse HTML directory listing for file names.""" + # Simple regex to extract href values + files = [] + for match in re.finditer(r'href=["\']([^"\']+)["\']', html): + filename = match.group(1) + if filename not in ['../', './']: + files.append(filename) + return files + + def get_download_status(self, accession: str, db_type: str = "gse") -> Dict[str, Any]: + """Check download status of a GEO dataset. + + Args: + accession: GEO accession ID + db_type: Database type (gse, gsm, gpl, gds) + + Returns: + Status dictionary + """ + dataset_path = self.download_dir / db_type / accession + + if not dataset_path.exists(): + return { + "accession": accession, + "db_type": db_type, + "downloaded": False, + "path": str(dataset_path) + } + + files = [] + total_size = 0 + for f in dataset_path.rglob("*"): + if f.is_file(): + size = f.stat().st_size + files.append({ + "name": f.name, + "path": str(f), + "size_mb": round(size / (1024*1024), 2) + }) + total_size += size + + return { + "accession": accession, + "db_type": db_type, + "downloaded": True, + "path": str(dataset_path), + "files": files, + "total_size_mb": round(total_size / (1024*1024), 2), + "file_count": len(files) + } + + def list_downloaded_datasets(self, db_type: Optional[str] = None) -> Dict[str, Any]: + """List all downloaded datasets. + + Args: + db_type: Optional filter by database type + + Returns: + List of downloaded datasets + """ + datasets = [] + + if db_type: + db_path = self.download_dir / db_type + if db_path.exists(): + for dataset_dir in db_path.iterdir(): + if dataset_dir.is_dir(): + total_size = sum( + f.stat().st_size for f in dataset_dir.rglob("*") if f.is_file() + ) + datasets.append({ + "accession": dataset_dir.name, + "db_type": db_type, + "path": str(dataset_dir), + "size_mb": round(total_size / (1024*1024), 2) + }) + else: + for db_dir in self.download_dir.iterdir(): + if db_dir.is_dir(): + for dataset_dir in db_dir.iterdir(): + if dataset_dir.is_dir(): + total_size = sum( + f.stat().st_size for f in dataset_dir.rglob("*") if f.is_file() + ) + datasets.append({ + "accession": dataset_dir.name, + "db_type": db_dir.name, + "path": str(dataset_dir), + "size_mb": round(total_size / (1024*1024), 2) + }) + + return { + "datasets": datasets, + "count": len(datasets) + } + + def cleanup_downloads( + self, + accession: Optional[str] = None, + db_type: Optional[str] = None + ) -> Dict[str, Any]: + """Clean up downloaded files. + + Args: + accession: Optional specific accession to remove + db_type: Optional database type filter + + Returns: + Cleanup results + """ + removed = [] + + if accession and db_type: + # Remove specific dataset + dataset_path = self.download_dir / db_type / accession + if dataset_path.exists(): + shutil.rmtree(dataset_path) + removed.append(str(dataset_path)) + elif db_type: + # Remove all datasets of a specific type + db_path = self.download_dir / db_type + if db_path.exists(): + for dataset_dir in db_path.iterdir(): + if dataset_dir.is_dir(): + shutil.rmtree(dataset_dir) + removed.append(str(dataset_dir)) + elif accession: + # Remove all matching accessions across types + for db_dir in self.download_dir.iterdir(): + if db_dir.is_dir(): + dataset_path = db_dir / accession + if dataset_path.exists(): + shutil.rmtree(dataset_path) + removed.append(str(dataset_path)) + else: + # Remove all downloads + for db_dir in self.download_dir.iterdir(): + if db_dir.is_dir(): + shutil.rmtree(db_dir) + removed.append(str(db_dir)) + + return { + "removed": removed, + "count": len(removed) + } diff --git a/geomcp_sra/geo_search.py b/geomcp_sra/geo_search.py new file mode 100644 index 0000000..a9be1aa --- /dev/null +++ b/geomcp_sra/geo_search.py @@ -0,0 +1,384 @@ +"""GEO search functionality using NCBI E-Utilities.""" + +import json +import time +from typing import Dict, Any, List, Optional +import httpx + +from .config import get_config + + +class GEOSearchError(Exception): + """Exception raised for GEO search errors.""" + pass + + +class GEOSearchClient: + """Client for searching GEO database using NCBI E-Utilities.""" + + def __init__(self): + self.config = get_config() + self.base_url = self.config["base_url"] + self.email = self.config["email"] + self.api_key = self.config.get("api_key") + self.retmax = self.config.get("retmax", 20) + self._last_request_time = 0 + + def _rate_limit(self): + """Apply rate limiting to be respectful to NCBI servers.""" + # Without API key: 3 requests per second + # With API key: 10 requests per second + min_interval = 0.1 if self.api_key else 0.34 + + elapsed = time.time() - self._last_request_time + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + self._last_request_time = time.time() + + def _build_params(self, extra_params: Dict[str, Any]) -> Dict[str, str]: + """Build request parameters with authentication.""" + params = {"email": self.email, **extra_params} + if self.api_key: + params["api_key"] = self.api_key + return params + + async def _esearch(self, db: str, term: str, retmax: int = 20) -> Dict[str, Any]: + """Perform ESearch query. + + Args: + db: Database to search (gds, geoprofiles, etc.) + term: Search term + retmax: Maximum results to return + + Returns: + ESearch response dictionary + """ + self._rate_limit() + + params = self._build_params({ + "db": db, + "term": term, + "retmax": retmax, + "retmode": "json", + }) + + async with httpx.AsyncClient() as client: + response = await client.get( + f"{self.base_url}/esearch.fcgi", + params=params, + timeout=30.0 + ) + response.raise_for_status() + return response.json() + + async def _esummary(self, db: str, ids: List[str]) -> Dict[str, Any]: + """Fetch summaries for a list of IDs. + + Args: + db: Database + ids: List of IDs to summarize + + Returns: + ESummary response dictionary + """ + if not ids: + return {"result": {}} + + self._rate_limit() + + params = self._build_params({ + "db": db, + "id": ",".join(map(str, ids)), + "retmode": "json", + }) + + async with httpx.AsyncClient() as client: + response = await client.get( + f"{self.base_url}/esummary.fcgi", + params=params, + timeout=30.0 + ) + response.raise_for_status() + return response.json() + + async def _efetch(self, db: str, id: str, retmode: str = "xml") -> str: + """Fetch full records. + + Args: + db: Database + id: Record ID + retmode: Return mode (xml, json, etc.) + + Returns: + EFetch response text + """ + self._rate_limit() + + params = self._build_params({ + "db": db, + "id": id, + "retmode": retmode, + }) + + async with httpx.AsyncClient() as client: + response = await client.get( + f"{self.base_url}/efetch.fcgi", + params=params, + timeout=30.0 + ) + response.raise_for_status() + return response.text + + async def search_geo( + self, + term: str, + retmax: Optional[int] = None, + record_types: Optional[List[str]] = None + ) -> Dict[str, Any]: + """Search GEO for all types of records. + + Args: + term: Search term (e.g., 'breast cancer', 'GSE12345', 'RNA-seq') + retmax: Maximum number of results to return + record_types: Optional filter for specific types ["GSE", "GSM", "GPL", "GDS"] + + Returns: + Dictionary with categorized results by record type + """ + try: + retmax = retmax or self.retmax + + # Search the gds database + data = await self._esearch('gds', term, retmax) + ids = data.get('esearchresult', {}).get('idlist', []) + + if not ids: + return { + "total_count": 0, + "results": [], + "series": [], + "samples": [], + "platforms": [], + "datasets": [] + } + + # Get detailed summaries + summaries = await self._esummary('gds', ids) + results = summaries.get('result', {}) + + # Categorize results by accession type + categorized = { + "total_count": len(ids), + "results": [], + "series": [], # GSE records + "samples": [], # GSM records + "platforms": [], # GPL records + "datasets": [] # GDS records + } + + for uid in ids: + if uid in results: + record = results[uid] + accession = record.get('accession', '') + + # Add to main results + categorized["results"].append(record) + + # Categorize by type + if accession.startswith('GSE'): + categorized["series"].append(record) + elif accession.startswith('GSM'): + categorized["samples"].append(record) + elif accession.startswith('GPL'): + categorized["platforms"].append(record) + elif accession.startswith('GDS'): + categorized["datasets"].append(record) + + # Filter by record types if specified + if record_types: + record_types = [rt.upper() for rt in record_types] + filtered_results = [] + + if "GSE" in record_types: + filtered_results.extend(categorized["series"]) + if "GSM" in record_types: + filtered_results.extend(categorized["samples"]) + if "GPL" in record_types: + filtered_results.extend(categorized["platforms"]) + if "GDS" in record_types: + filtered_results.extend(categorized["datasets"]) + + categorized["results"] = filtered_results + categorized["total_count"] = len(filtered_results) + + return categorized + + except httpx.HTTPStatusError as e: + raise GEOSearchError(f"HTTP error: {e.response.status_code} - {e.response.text}") + except Exception as e: + raise GEOSearchError(f"Search error: {str(e)}") + + async def search_geo_profiles(self, term: str, retmax: Optional[int] = None) -> Dict[str, Any]: + """Search GEO Profiles database. + + Args: + term: Search term + retmax: Maximum results to return + + Returns: + GEO Profiles search results + """ + try: + retmax = retmax or self.retmax + data = await self._esearch('geoprofiles', term, retmax) + ids = data.get('esearchresult', {}).get('idlist', []) + + if not ids: + return {"esummaryresult": ["Empty id list - nothing to do"]} + + summary = await self._esummary('geoprofiles', ids) + return summary + + except Exception as e: + raise GEOSearchError(f"GEO Profiles search error: {str(e)}") + + async def search_geo_datasets(self, term: str, retmax: Optional[int] = None) -> Dict[str, Any]: + """Search GEO Datasets (GDS) specifically. + + Args: + term: Search term + retmax: Maximum results to return + + Returns: + GDS search results + """ + result = await self.search_geo(term, retmax, record_types=["GDS"]) + + if result["datasets"]: + formatted_result = { + "header": {"type": "esummary", "version": "0.3"}, + "result": { + "uids": [r.get("uid") for r in result["datasets"]] + } + } + for record in result["datasets"]: + uid = record.get("uid") + if uid: + formatted_result["result"][uid] = record + + return formatted_result + else: + return {"esummaryresult": ["Empty id list - nothing to do"]} + + async def search_geo_series(self, term: str, retmax: Optional[int] = None) -> Dict[str, Any]: + """Search GEO Series (GSE) specifically. + + Args: + term: Search term + retmax: Maximum results to return + + Returns: + GSE search results + """ + result = await self.search_geo(term, retmax, record_types=["GSE"]) + + if result["series"]: + formatted_result = { + "header": {"type": "esummary", "version": "0.3"}, + "result": { + "uids": [r.get("uid") for r in result["series"]] + } + } + for record in result["series"]: + uid = record.get("uid") + if uid: + formatted_result["result"][uid] = record + + return formatted_result + else: + return {"esummaryresult": ["Empty id list - nothing to do"]} + + async def search_geo_samples(self, term: str, retmax: Optional[int] = None) -> Dict[str, Any]: + """Search GEO Samples (GSM) specifically. + + Args: + term: Search term + retmax: Maximum results to return + + Returns: + GSM search results + """ + result = await self.search_geo(term, retmax, record_types=["GSM"]) + + if result["samples"]: + formatted_result = { + "header": {"type": "esummary", "version": "0.3"}, + "result": { + "uids": [r.get("uid") for r in result["samples"]] + } + } + for record in result["samples"]: + uid = record.get("uid") + if uid: + formatted_result["result"][uid] = record + + return formatted_result + else: + return {"esummaryresult": ["Empty id list - nothing to do"]} + + async def search_geo_platforms(self, term: str, retmax: Optional[int] = None) -> Dict[str, Any]: + """Search GEO Platforms (GPL) specifically. + + Args: + term: Search term + retmax: Maximum results to return + + Returns: + GPL search results + """ + result = await self.search_geo(term, retmax, record_types=["GPL"]) + + if result["platforms"]: + formatted_result = { + "header": {"type": "esummary", "version": "0.3"}, + "result": { + "uids": [r.get("uid") for r in result["platforms"]] + } + } + for record in result["platforms"]: + uid = record.get("uid") + if uid: + formatted_result["result"][uid] = record + + return formatted_result + else: + return {"esummaryresult": ["Empty id list - nothing to do"]} + + async def get_series_info(self, gse_id: str) -> Dict[str, Any]: + """Get detailed information about a GEO Series. + + Args: + gse_id: GSE accession ID (e.g., 'GSE12345') + + Returns: + Series information dictionary + """ + try: + # Search for the specific GSE + data = await self._esearch('gds', f"{gse_id}[ACCN]", 1) + ids = data.get('esearchresult', {}).get('idlist', []) + + if not ids: + raise GEOSearchError(f"Series {gse_id} not found") + + # Get summary + summaries = await self._esummary('gds', ids) + result = summaries.get('result', {}) + + if ids[0] in result: + return result[ids[0]] + else: + raise GEOSearchError(f"No summary available for {gse_id}") + + except Exception as e: + raise GEOSearchError(f"Error getting series info: {str(e)}") diff --git a/geomcp_sra/sra_handler.py b/geomcp_sra/sra_handler.py new file mode 100644 index 0000000..d059f94 --- /dev/null +++ b/geomcp_sra/sra_handler.py @@ -0,0 +1,1040 @@ +"""SRA (Sequence Read Archive) handling functionality. + +This module provides capabilities to: +1. Query SRA Run information from GEO Series (map GSM to SRR accessions) +2. Get SRA accession lists for datasets +3. Generate download commands for SRA data +4. Optionally download FASTQ files using sra-toolkit +""" + +import asyncio +import json +import os +import re +import subprocess +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Dict, Any, List, Optional, Tuple +import httpx + +from .config import get_config + + +class SRAError(Exception): + """Exception raised for SRA-related errors.""" + pass + + +class SRAHandler: + """Handler for SRA data queries and downloads.""" + + # SRA endpoints + SRA_TRACE_URL = "https://trace.ncbi.nlm.nih.gov/Traces/sra" + SRA_EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils" + SRA_FETCH_URL = "https://sra-downloadb.be-md.ncbi.nlm.nih.gov/sos2/sra-pub-run-11" + + def __init__(self): + self.config = get_config() + self.email = self.config["email"] + self.api_key = self.config.get("api_key") + self.sra_toolkit_path = self.config.get("sra_toolkit_path") + self.download_dir = Path(self.config["download_dir"]).resolve() / "sra" + self.download_dir.mkdir(parents=True, exist_ok=True) + + def _build_params(self, extra_params: Dict[str, Any]) -> Dict[str, str]: + """Build request parameters with authentication.""" + params = {"email": self.email, **extra_params} + if self.api_key: + params["api_key"] = self.api_key + return params + + async def _fetch_geo_soft(self, gse_id: str) -> str: + """Fetch GEO Series SOFT file to extract SRA information. + + Args: + gse_id: GSE accession ID + + Returns: + SOFT file content as string + """ + range_dir = f"{gse_id[:-3]}nnn" + url = f"https://ftp.ncbi.nlm.nih.gov/geo/series/{range_dir}/{gse_id}/soft/{gse_id}_family.soft.gz" + + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=60.0) + + if response.status_code == 404: + # Try without _family suffix + url = f"https://ftp.ncbi.nlm.nih.gov/geo/series/{range_dir}/{gse_id}/soft/{gse_id}.soft.gz" + response = await client.get(url, timeout=60.0) + + response.raise_for_status() + + # Decompress gzip content + import gzip + content = gzip.decompress(response.content) + return content.decode('utf-8', errors='replace') + + def _parse_sra_accessions_from_soft(self, soft_content: str) -> Dict[str, List[str]]: + """Parse SRA accession numbers from SOFT file content. + + Args: + soft_content: SOFT file content + + Returns: + Dictionary mapping GSM IDs to lists of SRA accessions (SRX experiments) + """ + gsm_to_sra = {} + current_gsm = None + + for line in soft_content.split('\n'): + line = line.strip() + + # Track current sample + if line.startswith('^SAMPLE = '): + current_gsm = line.split('=')[1].strip() + gsm_to_sra[current_gsm] = [] + + # Look for SRA relation links (contain SRX experiment IDs) + elif current_gsm and line.startswith('!Sample_relation = SRA:'): + # Extract SRX ID from URL like "https://www.ncbi.nlm.nih.gov/sra?term=SRX25362135" + srx_pattern = r'([SED]RX\d+)' + matches = re.findall(srx_pattern, line) + gsm_to_sra[current_gsm].extend(matches) + + # Remove empty entries + gsm_to_sra = {k: v for k, v in gsm_to_sra.items() if v} + + return gsm_to_sra + + async def _get_srr_from_srx(self, srx_id: str) -> List[str]: + """Get SRA Run (SRR) IDs from Experiment (SRX) ID. + + Args: + srx_id: SRX accession (e.g., 'SRX25362135') + + Returns: + List of SRR accessions + """ + srr_list = [] + + try: + async with httpx.AsyncClient() as client: + # Use E-Utilities to search for runs linked to this experiment + search_params = self._build_params({ + "db": "sra", + "term": f"{srx_id}[Experiment]", + "retmode": "json", + "retmax": 100 + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esearch.fcgi", + params=search_params, + timeout=30.0 + ) + response.raise_for_status() + + search_data = response.json() + sra_ids = search_data.get('esearchresult', {}).get('idlist', []) + + if not sra_ids: + return srr_list + + # Get summary for each SRA entry to find SRR IDs + for sra_id in sra_ids: + summary_params = self._build_params({ + "db": "sra", + "id": sra_id, + "retmode": "json" + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esummary.fcgi", + params=summary_params, + timeout=30.0 + ) + response.raise_for_status() + + summary_data = response.json() + + # Parse the result to find SRR IDs + result = summary_data.get('result', {}) + for uid in result.get('uids', []): + item = result.get(uid, {}) + # Look for run accessions in the summary + runs = item.get('runs', '') + if runs: + # Parse SRR from runs field + srr_matches = re.findall(r'([SED]RR\d+)', runs) + srr_list.extend(srr_matches) + + # Also check other fields + for key in ['accession', 'runlist', 'experiment']: + val = item.get(key, '') + if val: + srr_matches = re.findall(r'([SED]RR\d+)', str(val)) + srr_list.extend(srr_matches) + + except Exception as e: + # Don't fail if we can't get SRR info + pass + + return list(set(srr_list)) # Remove duplicates + + async def query_sra_from_geo(self, gse_id: str) -> Dict[str, Any]: + """Query SRA Run information from a GEO Series. + + This method extracts the mapping between GSM samples and SRR runs + from the GEO Series SOFT file. + + Args: + gse_id: GSE accession ID (e.g., 'GSE12345') + + Returns: + Dictionary with SRA run information + + Example: + { + "gse_id": "GSE12345", + "total_samples": 10, + "samples_with_sra": 8, + "sra_runs": [ + { + "gsm_id": "GSM123456", + "sra_accessions": ["SRR1234567", "SRR1234568"] + } + ] + } + """ + if not gse_id.upper().startswith("GSE"): + raise SRAError(f"Invalid GSE ID: {gse_id}") + + try: + # Fetch and parse SOFT file + soft_content = await self._fetch_geo_soft(gse_id) + gsm_to_sra = self._parse_sra_accessions_from_soft(soft_content) + + # Convert SRX (Experiment) IDs to SRR (Run) IDs + gsm_to_srr = {} + for gsm_id, srx_list in gsm_to_sra.items(): + srr_list = [] + for srx_id in srx_list: + srrs = await self._get_srr_from_srx(srx_id) + srr_list.extend(srrs) + if srr_list: + gsm_to_srr[gsm_id] = list(set(srr_list)) # Remove duplicates + + # Also try to get from E-Utilities link + additional_sra = await self._get_sra_from_eutils(gse_id) + + # Merge results + for gsm_id, sra_list in additional_sra.items(): + if gsm_id in gsm_to_srr: + # Merge without duplicates + existing = set(gsm_to_srr[gsm_id]) + for sra in sra_list: + if sra not in existing: + gsm_to_srr[gsm_id].append(sra) + else: + gsm_to_srr[gsm_id] = sra_list + + # Build response + samples_with_sra = [ + { + "gsm_id": gsm_id, + "sra_accessions": sra_list + } + for gsm_id, sra_list in gsm_to_srr.items() + ] + + # Get all unique SRR accessions + all_srr = set() + for sra_list in gsm_to_srr.values(): + all_srr.update(sra_list) + + return { + "gse_id": gse_id.upper(), + "total_samples": len(samples_with_sra), + "samples_with_sra": len(samples_with_sra), + "total_sra_runs": len(all_srr), + "all_sra_accessions": sorted(list(all_srr)), + "samples": samples_with_sra + } + + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + raise SRAError(f"GEO Series {gse_id} not found or SOFT file unavailable") + raise SRAError(f"HTTP error querying SRA info: {e.response.status_code}") + except Exception as e: + raise SRAError(f"Error querying SRA information: {str(e)}") + + async def _get_sra_from_eutils(self, gse_id: str) -> Dict[str, List[str]]: + """Get SRA accessions using E-Utilities. + + Args: + gse_id: GSE accession ID + + Returns: + Dictionary mapping GSM IDs to SRA accessions + """ + result = {} + + try: + # Search for samples in this series + async with httpx.AsyncClient() as client: + # First, get series UID + search_params = self._build_params({ + "db": "gds", + "term": f"{gse_id}[ACCN]", + "retmode": "json" + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esearch.fcgi", + params=search_params, + timeout=30.0 + ) + response.raise_for_status() + + search_data = response.json() + gds_ids = search_data.get('esearchresult', {}).get('idlist', []) + + if not gds_ids: + return result + + # Fetch summary to get sample information + summary_params = self._build_params({ + "db": "gds", + "id": gds_ids[0], + "retmode": "json" + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esummary.fcgi", + params=summary_params, + timeout=30.0 + ) + response.raise_for_status() + + # Note: E-utilities doesn't always have SRA links + # This is a fallback method + + except Exception: + # Don't fail if eutils doesn't return data + pass + + return result + + async def estimate_sra_size(self, sra_ids: List[str]) -> Dict[str, Any]: + """Estimate the size of SRA files before downloading. + + This is a "dry-run" mode that queries the SRA database to estimate + download sizes without actually downloading any data. + + Args: + sra_ids: List of SRA accession IDs + + Returns: + Dictionary with size estimates and warnings + """ + if not sra_ids: + raise SRAError("No SRA IDs provided") + + results = [] + total_size_bytes = 0 + + try: + async with httpx.AsyncClient() as client: + for sra_id in sra_ids: + sra_id = sra_id.upper() + + # Use EBI ENA API to get file size (more reliable than NCBI for size info) + ena_url = f"https://www.ebi.ac.uk/ena/portal/api/filereport?accession={sra_id}&result=read_run&fields=run_accession,fastq_bytes,sra_bytes,read_count,base_count" + + try: + response = await client.get(ena_url, timeout=30.0) + response.raise_for_status() + + lines = response.text.strip().split('\n') + if len(lines) >= 2: + # Parse TSV response + headers = lines[0].split('\t') + values = lines[1].split('\t') + + data = dict(zip(headers, values)) + + # Get SRA size if available, otherwise estimate from FASTQ + sra_bytes = data.get('sra_bytes', '') + fastq_bytes = data.get('fastq_bytes', '') + read_count = data.get('read_count', '0') + base_count = data.get('base_count', '0') + + # Calculate sizes + sra_size_mb = 0 + if sra_bytes and sra_bytes.isdigit(): + sra_size_mb = int(sra_bytes) / (1024 * 1024) + elif fastq_bytes: + # Estimate SRA size as ~40% of FASTQ (compressed) + fastq_sizes = fastq_bytes.split(';') + total_fastq = sum(int(x) for x in fastq_sizes if x.isdigit()) + sra_size_mb = (total_fastq * 0.4) / (1024 * 1024) + + # Estimate FASTQ size (SRA * 2.5 for decompressed) + estimated_fastq_mb = sra_size_mb * 2.5 + + total_size_bytes += sra_size_mb * 1024 * 1024 + + results.append({ + "sra_id": sra_id, + "sra_size_mb": round(sra_size_mb, 2), + "estimated_fastq_size_mb": round(estimated_fastq_mb, 2), + "read_count": int(read_count) if read_count.isdigit() else 0, + "base_count": int(base_count) if base_count.isdigit() else 0, + "warning": self._get_size_warning(sra_size_mb) + }) + else: + results.append({ + "sra_id": sra_id, + "sra_size_mb": "unknown", + "estimated_fastq_size_mb": "unknown", + "warning": "Could not retrieve size information from ENA" + }) + + except Exception as e: + results.append({ + "sra_id": sra_id, + "sra_size_mb": "unknown", + "estimated_fastq_size_mb": "unknown", + "warning": f"Error querying ENA: {str(e)}" + }) + + except Exception as e: + raise SRAError(f"Error estimating sizes: {str(e)}") + + # Calculate total + total_mb = total_size_bytes / (1024 * 1024) + total_gb = total_mb / 1024 + + return { + "dry_run": True, + "sra_count": len(sra_ids), + "individual_estimates": results, + "total_sra_size_mb": round(total_mb, 2), + "total_sra_size_gb": round(total_gb, 2), + "estimated_total_fastq_size_gb": round(total_gb * 2.5, 2), + "safety_warnings": self._get_safety_warnings(total_mb) + } + + def _get_size_warning(self, size_mb: float) -> str: + """Get warning message based on file size.""" + if size_mb == 0: + return "Size unknown" + elif size_mb < 100: + return "Small file (< 100 MB)" + elif size_mb < 1024: + return "Medium file (100 MB - 1 GB)" + elif size_mb < 5120: # 5 GB + return "⚠️ LARGE FILE (1-5 GB) - Ensure sufficient disk space" + else: + return "🚨 VERY LARGE FILE (> 5 GB) - Requires explicit confirmation" + + def _get_safety_warnings(self, total_mb: float) -> List[str]: + """Get safety warnings for the total download size.""" + warnings = [] + + if total_mb > 1024: # > 1 GB + warnings.append("Total download exceeds 1 GB. Ensure you have sufficient disk space.") + if total_mb > 5120: # > 5 GB + warnings.append("Total download exceeds 5 GB. This will take significant time and space.") + if total_mb > 10240: # > 10 GB + warnings.append("🚨 WARNING: Total download exceeds 10 GB! Consider downloading individual files.") + + return warnings + + async def get_sra_metadata(self, sra_id: str) -> Dict[str, Any]: + """Get metadata for an SRA run. + + Args: + sra_id: SRA accession (e.g., 'SRR1234567') + + Returns: + SRA run metadata + """ + if not re.match(r'^[SED]RR\d+$', sra_id, re.IGNORECASE): + raise SRAError(f"Invalid SRA ID: {sra_id}") + + sra_id = sra_id.upper() + + try: + # Use E-Utilities to get SRA metadata + async with httpx.AsyncClient() as client: + # Search in SRA database + search_params = self._build_params({ + "db": "sra", + "term": sra_id, + "retmode": "json" + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esearch.fcgi", + params=search_params, + timeout=30.0 + ) + response.raise_for_status() + + search_data = response.json() + sra_ids = search_data.get('esearchresult', {}).get('idlist', []) + + if not sra_ids: + return { + "sra_id": sra_id, + "found": False, + "error": "SRA accession not found in database" + } + + # Get summary + summary_params = self._build_params({ + "db": "sra", + "id": sra_ids[0], + "retmode": "json" + }) + + response = await client.get( + f"{self.SRA_EUTILS_URL}/esummary.fcgi", + params=summary_params, + timeout=30.0 + ) + response.raise_for_status() + + summary_data = response.json() + + return { + "sra_id": sra_id, + "found": True, + "metadata": summary_data + } + + except Exception as e: + raise SRAError(f"Error getting SRA metadata: {str(e)}") + + def get_sra_download_url(self, sra_id: str) -> str: + """Get direct download URL for an SRA run. + + Args: + sra_id: SRA accession (e.g., 'SRR1234567') + + Returns: + Direct download URL + """ + sra_id = sra_id.upper() + + # Construct SRA download URL + # Format: https://sra-downloadb.be-md.ncbi.nlm.nih.gov/sos2/sra-pub-run-11/{SRRxxxxxxx}/{SRRxxxxxxx}.1 + base_url = "https://sra-downloadb.be-md.ncbi.nlm.nih.gov/sos2/sra-pub-run-11" + return f"{base_url}/{sra_id}/{sra_id}.1" + + def generate_download_commands( + self, + sra_ids: List[str], + method: str = "prefetch", + output_dir: Optional[str] = None + ) -> Dict[str, Any]: + """Generate commands to download SRA data. + + Args: + sra_ids: List of SRA accessions + method: Download method ('prefetch', 'wget', 'curl', or 'aspera') + output_dir: Optional output directory + + Returns: + Dictionary with download commands and instructions + """ + if not sra_ids: + raise SRAError("No SRA IDs provided") + + output_dir = output_dir or str(self.download_dir) + commands = [] + + if method == "prefetch": + # sra-toolkit prefetch command + toolkit_path = self.sra_toolkit_path or "" + prefetch = f"{toolkit_path}/prefetch" if toolkit_path else "prefetch" + + for sra_id in sra_ids: + cmd = f"{prefetch} -O {output_dir} {sra_id}" + commands.append({ + "sra_id": sra_id, + "command": cmd, + "description": f"Download {sra_id} using sra-toolkit prefetch" + }) + + elif method == "fasterq-dump": + # Directly download and convert to FASTQ + toolkit_path = self.sra_toolkit_path or "" + fasterq_dump = f"{toolkit_path}/fasterq-dump" if toolkit_path else "fasterq-dump" + + for sra_id in sra_ids: + cmd = f"{fasterq_dump} --outdir {output_dir} {sra_id}" + commands.append({ + "sra_id": sra_id, + "command": cmd, + "description": f"Download and convert {sra_id} to FASTQ" + }) + + elif method == "wget": + # Direct HTTP download + for sra_id in sra_ids: + url = self.get_sra_download_url(sra_id) + cmd = f"wget -P {output_dir} {url}" + commands.append({ + "sra_id": sra_id, + "command": cmd, + "description": f"Download {sra_id} using wget" + }) + + elif method == "curl": + # Direct HTTP download with curl + for sra_id in sra_ids: + url = self.get_sra_download_url(sra_id) + output_file = f"{output_dir}/{sra_id}.sra" + cmd = f"curl -o {output_file} {url}" + commands.append({ + "sra_id": sra_id, + "command": cmd, + "description": f"Download {sra_id} using curl" + }) + + elif method == "aspera": + # Aspera high-speed download + # Requires aspera-cli to be installed + for sra_id in sra_ids: + aspera_url = f"anonftp@ftp.ncbi.nlm.nih.gov:/sra/sra-instant/reads/ByRun/sra/{sra_id[:3]}/{sra_id[:6]}/{sra_id}/{sra_id}.sra" + cmd = f"ascp -QT -l 300m -P33001 -i $HOME/.aspera/connect/etc/asperaweb_id_dsa.openssh {aspera_url} {output_dir}" + commands.append({ + "sra_id": sra_id, + "command": cmd, + "description": f"Download {sra_id} using Aspera (high-speed)" + }) + + else: + raise SRAError(f"Unknown download method: {method}") + + return { + "sra_ids": sra_ids, + "method": method, + "output_dir": output_dir, + "commands": commands, + "notes": self._get_method_notes(method) + } + + def _get_method_notes(self, method: str) -> str: + """Get notes for a download method.""" + notes = { + "prefetch": ( + "Requires sra-toolkit (https://github.com/ncbi/sra-tools). " + "Downloads SRA files which can then be converted to FASTQ using fasterq-dump." + ), + "fasterq-dump": ( + "Requires sra-toolkit. Downloads and converts to FASTQ in one step. " + "May take longer but produces immediately usable files." + ), + "wget": ( + "Direct HTTP download. Works without sra-toolkit but downloads SRA format files " + "which need to be converted using fasterq-dump." + ), + "curl": ( + "Direct HTTP download using curl. Similar to wget but more portable." + ), + "aspera": ( + "High-speed download using Aspera protocol. Requires aspera-cli. " + "Fastest method for large files." + ) + } + return notes.get(method, "") + + async def download_sra( + self, + sra_id: str, + convert_to_fastq: bool = False, + output_dir: Optional[Path] = None + ) -> Dict[str, Any]: + """Download an SRA file. + + Note: This method downloads SRA files directly. For production use, + it's recommended to use sra-toolkit prefetch/fasterq-dump instead. + + Args: + sra_id: SRA accession + convert_to_fastq: Whether to convert to FASTQ (requires sra-toolkit) + output_dir: Optional output directory + + Returns: + Download results + """ + sra_id = sra_id.upper() + output_dir = output_dir or self.download_dir + output_dir.mkdir(parents=True, exist_ok=True) + + url = self.get_sra_download_url(sra_id) + sra_file = output_dir / f"{sra_id}.sra" + + try: + # Download the SRA file + async with httpx.AsyncClient() as client: + async with client.stream("GET", url, timeout=300.0) as response: + response.raise_for_status() + + with open(sra_file, 'wb') as f: + async for chunk in response.aiter_bytes(): + f.write(chunk) + + result = { + "sra_id": sra_id, + "sra_file": str(sra_file), + "size_mb": round(sra_file.stat().st_size / (1024*1024), 2), + "converted_to_fastq": False + } + + # Convert to FASTQ if requested + if convert_to_fastq: + fastq_result = await self._convert_to_fastq(sra_file, output_dir) + result["converted_to_fastq"] = True + result["fastq_files"] = fastq_result + + return result + + except Exception as e: + # Clean up partial download + if sra_file.exists(): + sra_file.unlink() + raise SRAError(f"Download failed: {str(e)}") + + async def _convert_to_fastq( + self, + sra_file: Path, + output_dir: Path + ) -> List[str]: + """Convert SRA file to FASTQ using fasterq-dump. + + Args: + sra_file: Path to SRA file + output_dir: Output directory for FASTQ files + + Returns: + List of generated FASTQ files + """ + toolkit_path = self.sra_toolkit_path or "" + fasterq_dump = f"{toolkit_path}/fasterq-dump" if toolkit_path else "fasterq-dump" + + # Check if fasterq-dump is available + try: + result = subprocess.run( + [fasterq_dump, "--version"], + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode != 0: + raise SRAError("fasterq-dump not found. Please install sra-toolkit.") + except FileNotFoundError: + raise SRAError( + "fasterq-dump not found. Please install sra-toolkit: " + "https://github.com/ncbi/sra-tools" + ) + + # Run fasterq-dump + cmd = [ + fasterq_dump, + "--outdir", str(output_dir), + "--threads", "4", + str(sra_file) + ] + + # Run in thread pool to avoid blocking + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, + lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=3600) + ) + + if result.returncode != 0: + raise SRAError(f"fasterq-dump failed: {result.stderr}") + + # Find generated FASTQ files + sra_id = sra_file.stem + fastq_files = [] + for pattern in [f"{sra_id}*.fastq", f"{sra_id}*.fastq.gz"]: + fastq_files.extend(output_dir.glob(pattern)) + + return [str(f) for f in fastq_files] + + async def download_and_convert( + self, + sra_id: str, + output_dir: Optional[Path] = None, + split_3: bool = True, + check_refseq: bool = True, + dry_run: bool = False, + confirm_large: bool = False + ) -> Dict[str, Any]: + """Download SRA data using prefetch and convert to FASTQ using fastq-dump. + + SAFETY FEATURES (as per maintainer recommendations): + - Dry-run mode: Estimate sizes before downloading + - Size warnings: Alerts for large files (>1GB, >5GB) + - Explicit confirmation: Required for very large downloads + - Explicit output directory: Must be provided for large files + + This is the recommended workflow for downloading and converting SRA data: + 1. Uses prefetch to download SRA file (handles large files better) + 2. Uses fastq-dump --split-3 to convert to FASTQ (handles paired-end properly) + + Args: + sra_id: SRA accession ID (e.g., 'SRR1234567') + output_dir: Optional output directory (default: download_dir/sra_id) + split_3: Use --split-3 for 3-way splitting (recommended for mate-pairs) + check_refseq: Whether to check/download reference sequences + dry_run: If True, only estimate sizes without downloading (default: False) + confirm_large: Must be True to download files >5GB (safety check) + + Returns: + Dictionary with download and conversion results, or dry-run estimates + + Raises: + SRAError: If output_dir not provided for large files, or if confirm_large=False for >5GB files + """ + sra_id = sra_id.upper() + + # SAFETY CHECK 1: Estimate size before downloading + size_estimate = await self.estimate_sra_size([sra_id]) + total_mb = size_estimate.get("total_sra_size_mb", 0) + + # SAFETY CHECK 2: Dry-run mode - return estimates without downloading + if dry_run: + return { + "mode": "dry_run", + "sra_id": sra_id, + "size_estimate": size_estimate, + "note": "To proceed with download, call with dry_run=False" + } + + # SAFETY CHECK 3: Explicit output directory required for large files + if total_mb > 1024 and output_dir is None: # > 1GB + raise SRAError( + f"SAFETY CHECK: File size is ~{total_mb/1024:.1f} GB. " + f"Large downloads require an explicit output directory. " + f"Please provide output_dir parameter. " + f"Tip: Run with dry_run=True first to see size estimates." + ) + + # SAFETY CHECK 4: Confirmation required for very large files + if total_mb > 5120 and not confirm_large: # > 5GB + raise SRAError( + f"SAFETY CHECK: File size is ~{total_mb/1024:.1f} GB (>5GB). " + f"This is a VERY LARGE download that will consume significant " + f"disk space and time. To proceed, set confirm_large=True. " + f"Tip: Run with dry_run=True first to see detailed estimates." + ) + + output_dir = output_dir or (self.download_dir / sra_id) + output_dir.mkdir(parents=True, exist_ok=True) + + # Check for sra-toolkit + toolkit_check = self.check_sra_toolkit() + if not toolkit_check["all_available"]: + raise SRAError( + "sra-toolkit not found. Please install from: " + "https://github.com/ncbi/sra-tools" + ) + + toolkit_path = self.sra_toolkit_path or "" + prefetch = f"{toolkit_path}/prefetch" if toolkit_path else "prefetch" + fastq_dump = f"{toolkit_path}/fastq-dump" if toolkit_path else "fastq-dump" + + result = { + "sra_id": sra_id, + "output_dir": str(output_dir), + "size_estimate_mb": total_mb, + "safety_warnings": size_estimate.get("safety_warnings", []), + "steps": [] + } + + try: + # Step 1: Download using prefetch + import logging + logger = logging.getLogger(__name__) + + cmd = [prefetch, "--progress", "--output-directory", str(output_dir)] + + # Handle refseq checking + if not check_refseq: + cmd.extend(["--check-rs", "no"]) + + cmd.append(sra_id) + + loop = asyncio.get_event_loop() + prefetch_result = await loop.run_in_executor( + None, + lambda: subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=3600 # 1 hour timeout for large files + ) + ) + + if prefetch_result.returncode != 0: + raise SRAError(f"prefetch failed: {prefetch_result.stderr}") + + # Find the downloaded SRA file + sra_file = output_dir / sra_id / f"{sra_id}.sra" + if not sra_file.exists(): + # Try alternative locations + alt_paths = [ + output_dir / f"{sra_id}.sra", + self.download_dir / sra_id / f"{sra_id}.sra", + Path(f"{sra_id}/{sra_id}.sra"), + ] + for alt_path in alt_paths: + if alt_path.exists(): + sra_file = alt_path + break + + if not sra_file.exists(): + raise SRAError(f"SRA file not found after download: {sra_file}") + + sra_size_mb = round(sra_file.stat().st_size / (1024*1024), 2) + + result["steps"].append({ + "step": "download", + "status": "success", + "sra_file": str(sra_file), + "sra_size_mb": sra_size_mb + }) + + # Step 2: Convert to FASTQ using fastq-dump + print(f"Step 2: Converting {sra_id} to FASTQ using fastq-dump...") + + cmd = [fastq_dump, "--outdir", str(output_dir)] + + if split_3: + cmd.append("--split-3") + else: + cmd.append("--split-files") + + cmd.append(str(sra_file)) + + fastq_result = await loop.run_in_executor( + None, + lambda: subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=3600 # 1 hour timeout + ) + ) + + if fastq_result.returncode != 0: + raise SRAError(f"fastq-dump failed: {fastq_result.stderr}") + + # Find generated FASTQ files + fastq_files = [] + for pattern in [f"{sra_id}*.fastq", f"{sra_id}*.fastq.gz"]: + fastq_files.extend(output_dir.glob(pattern)) + + # Also check in sra_id subdirectory + if (output_dir / sra_id).exists(): + for pattern in [f"{sra_id}*.fastq", f"{sra_id}*.fastq.gz"]: + fastq_files.extend((output_dir / sra_id).glob(pattern)) + + fastq_info = [] + total_fastq_size_mb = 0 + for f in fastq_files: + size_mb = round(f.stat().st_size / (1024*1024), 2) + total_fastq_size_mb += size_mb + + # Count reads (each read = 4 lines) + line_count = 0 + try: + with open(f, 'r') as fp: + for _ in fp: + line_count += 1 + if line_count >= 4: + break + # Get total lines + result_count = subprocess.run( + ["wc", "-l", str(f)], + capture_output=True, + text=True + ) + total_lines = int(result_count.stdout.split()[0]) + read_count = total_lines // 4 + except: + read_count = "unknown" + + fastq_info.append({ + "file": str(f.name), + "path": str(f), + "size_mb": size_mb, + "reads": read_count + }) + + result["steps"].append({ + "step": "convert", + "status": "success", + "fastq_files": fastq_info, + "total_fastq_size_mb": total_fastq_size_mb + }) + + result["status"] = "success" + result["total_size_mb"] = sra_size_mb + total_fastq_size_mb + + return result + + except Exception as e: + result["status"] = "failed" + result["error"] = str(e) + raise SRAError(f"Download and convert failed: {str(e)}") + + def check_sra_toolkit(self) -> Dict[str, Any]: + """Check if sra-toolkit is installed and available. + + Returns: + Status information about sra-toolkit + """ + toolkit_path = self.sra_toolkit_path or "" + tools = ["prefetch", "fastq-dump", "fasterq-dump", "vdb-validate"] + + results = {} + all_found = True + + for tool in tools: + cmd = f"{toolkit_path}/{tool}" if toolkit_path else tool + try: + result = subprocess.run( + [cmd, "--version"], + capture_output=True, + text=True, + timeout=10 + ) + results[tool] = { + "available": result.returncode == 0, + "version": result.stdout.strip() if result.returncode == 0 else None, + "path": cmd + } + if result.returncode != 0: + all_found = False + except FileNotFoundError: + results[tool] = {"available": False, "path": cmd} + all_found = False + except Exception as e: + results[tool] = {"available": False, "error": str(e)} + all_found = False + + return { + "all_available": all_found, + "toolkit_path": toolkit_path or "System PATH", + "tools": results, + "installation_url": "https://github.com/ncbi/sra-tools/wiki/02.-Installing-SRA-Toolkit" + } diff --git a/pyproject.toml b/pyproject.toml index a2608b1..0fe5662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,18 +3,18 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" [project] -name = "geo-mcp" +name = "geo-mcp-sra" dynamic = ["version"] -description = "A Model Context Protocol (MCP) server for accessing GEO (Gene Expression Omnibus) data through NCBI E-Utils API" +description = "An enhanced MCP server for accessing GEO data with SRA raw sequencing support" readme = "README.md" license = {text = "MIT"} requires-python = ">=3.10" -keywords = ["mcp", "geo", "bioinformatics", "gene-expression", "ncbi", "e-utils"] +keywords = ["mcp", "geo", "sra", "bioinformatics", "gene-expression", "ncbi", "rna-seq", "fastq"] authors = [ - {name ="MCPmed Contributors", email = "matthias.flotho@ccb.uni-saarland.de"} + {name = "GEO MCP Contributors"} ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", @@ -22,26 +22,25 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Bio-Informatics", "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "mcp[cli]>=1.9.4", - "requests>=2.31.0", - "aiohttp>=3.8.0", + "mcp>=1.9.0", + "httpx>=0.27.0", "aiofiles>=23.0.0", - "fastapi>=0.100.0", - "uvicorn>=0.20.0", + "pydantic>=2.0.0", ] [project.urls] -Homepage = "https://github.com/MCPmed/geomcp" -Repository = "https://github.com/MCPmed/geomcp" -Documentation = "https://github.com/MCPmed/geomcp#readme" -"Bug Tracker" = "https://github.com/MCPmed/geomcp/issues" +Homepage = "https://github.com/yourusername/geo-mcp-sra" +Repository = "https://github.com/yourusername/geo-mcp-sra" +Documentation = "https://github.com/yourusername/geo-mcp-sra#readme" +"Bug Tracker" = "https://github.com/yourusername/geo-mcp-sra/issues" [project.scripts] -geo-mcp = "geomcp.main:main" +geo-mcp-sra = "server:main" [project.optional-dependencies] dev = [ @@ -51,23 +50,19 @@ dev = [ "ruff>=0.1.0", "mypy>=1.0.0", ] -test = [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", -] [tool.hatch.version] source = "vcs" [tool.hatch.build.targets.wheel] -packages = ["geomcp"] +packages = ["geomcp_sra"] [tool.black] -line-length = 88 +line-length = 100 target-version = ['py310'] [tool.ruff] -line-length = 88 +line-length = 100 target-version = "py310" [tool.mypy] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index d69da20..0000000 --- a/pytest.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tool:pytest] -testpaths = geo_mcp_server/test -python_files = test_*.py -python_classes = Test* -python_functions = test_* -addopts = -v --tb=short -asyncio_mode = auto -filterwarnings = - ignore::DeprecationWarning - ignore::PendingDeprecationWarning \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ad97d61 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +mcp>=1.9.0 +httpx>=0.27.0 +aiofiles>=23.0.0 +pydantic>=2.0.0 diff --git a/server.py b/server.py new file mode 100644 index 0000000..f921c08 --- /dev/null +++ b/server.py @@ -0,0 +1,1031 @@ +#!/usr/bin/env python3 +""" +GEO MCP Server with SRA Support + +An enhanced Model Context Protocol (MCP) server for accessing +GEO (Gene Expression Omnibus) data through NCBI E-Utils API, +with additional support for SRA (Sequence Read Archive) raw data queries. + +Usage: + python server.py # Run MCP stdio server + python server.py --http # Run HTTP server + python server.py --http --port 8080 # Run HTTP server on custom port + python server.py --init # Initialize configuration +""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path +from typing import Optional, List +from enum import Enum + +from pydantic import BaseModel, Field, field_validator, ConfigDict +from mcp.server.fastmcp import FastMCP + +# Import our modules +from geomcp_sra.config import load_config, validate_config, create_config_template, get_config +from geomcp_sra.geo_search import GEOSearchClient, GEOSearchError +from geomcp_sra.geo_download import GEODownloadClient, GEODownloadError +from geomcp_sra.sra_handler import SRAHandler, SRAError + + +# Initialize MCP server +mcp = FastMCP("geo_mcp") + + +# ============================================================================ +# Enums and Response Formats +# ============================================================================ + +class ResponseFormat(str, Enum): + """Output format for tool responses.""" + MARKDOWN = "markdown" + JSON = "json" + + +class DownloadMethod(str, Enum): + """SRA download methods.""" + PREFETCH = "prefetch" + FASTERQ_DUMP = "fasterq-dump" + WGET = "wget" + CURL = "curl" + ASPERA = "aspera" + + +# ============================================================================ +# Pydantic Models for Input Validation +# ============================================================================ + +class SearchInput(BaseModel): + """Base input for search operations.""" + model_config = ConfigDict(str_strip_whitespace=True) + + term: str = Field( + ..., + description="Search term (e.g., 'breast cancer', 'GSE12345', 'RNA-seq')", + min_length=1, + max_length=500 + ) + retmax: int = Field( + default=20, + description="Maximum number of results to return", + ge=1, + le=1000 + ) + response_format: ResponseFormat = Field( + default=ResponseFormat.JSON, + description="Output format: 'json' for structured data or 'markdown' for readable text" + ) + + +class SearchWithTypesInput(SearchInput): + """Input for search with record type filtering.""" + record_types: Optional[List[str]] = Field( + default=None, + description="Filter for specific record types: GSE, GSM, GPL, GDS" + ) + + +class GeoIdInput(BaseModel): + """Input for GEO ID operations.""" + model_config = ConfigDict(str_strip_whitespace=True) + + geo_id: str = Field( + ..., + description="GEO accession ID (e.g., GSE12345, GSM789, GPL456, GDS123)", + pattern=r'^(GSE|GSM|GPL|GDS)\d+$' + ) + + +class DownloadInput(GeoIdInput): + """Input for download operations.""" + db_type: str = Field( + default="gse", + description="Database type: gse, gsm, gpl, or gds" + ) + output_dir: Optional[str] = Field( + default=None, + description="Optional custom output directory" + ) + file_types: Optional[List[str]] = Field( + default=None, + description="File types to download (for series: soft, matrix, miniml, supplementary)" + ) + + +class SRASearchInput(BaseModel): + """Input for SRA search operations.""" + model_config = ConfigDict(str_strip_whitespace=True) + + gse_id: str = Field( + ..., + description="GEO Series accession ID (e.g., GSE12345)", + pattern=r'^GSE\d+$' + ) + + +class SRADownloadInput(BaseModel): + """Input for SRA download command generation.""" + model_config = ConfigDict(str_strip_whitespace=True) + + sra_ids: List[str] = Field( + ..., + description="List of SRA accession IDs (e.g., ['SRR1234567', 'SRR1234568'])", + min_length=1 + ) + method: DownloadMethod = Field( + default=DownloadMethod.PREFETCH, + description="Download method: prefetch, fasterq-dump, wget, curl, or aspera" + ) + output_dir: Optional[str] = Field( + default=None, + description="Optional output directory for downloads" + ) + + +class SRADirectDownloadInput(BaseModel): + """Input for direct SRA download.""" + model_config = ConfigDict(str_strip_whitespace=True) + + sra_id: str = Field( + ..., + description="SRA accession ID (e.g., SRR1234567)", + pattern=r'^[SED]RR\d+$' + ) + convert_to_fastq: bool = Field( + default=False, + description="Whether to convert SRA to FASTQ format (requires sra-toolkit)" + ) + output_dir: Optional[str] = Field( + default=None, + description="Optional output directory" + ) + + +class CleanupInput(BaseModel): + """Input for cleanup operations.""" + model_config = ConfigDict(str_strip_whitespace=True) + + geo_id: Optional[str] = Field( + default=None, + description="Optional specific GEO ID to remove" + ) + db_type: Optional[str] = Field( + default=None, + description="Optional database type filter for cleanup (gse, gsm, gpl, gds, sra)" + ) + + +# ============================================================================ +# Helper Functions +# ============================================================================ + +def format_as_markdown(data: dict, title: str = "Results") -> str: + """Format results as markdown for human readability.""" + lines = [f"# {title}", ""] + + def format_value(value, indent=0): + prefix = " " * indent + if isinstance(value, dict): + for k, v in value.items(): + if isinstance(v, (dict, list)): + lines.append(f"{prefix}- **{k}:**") + format_value(v, indent + 1) + else: + lines.append(f"{prefix}- **{k}:** {v}") + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + lines.append(f"{prefix}-") + format_value(item, indent + 1) + else: + lines.append(f"{prefix}- {item}") + else: + lines.append(f"{prefix}{value}") + + format_value(data) + return "\n".join(lines) + + +def handle_error(error: Exception) -> str: + """Format error messages consistently.""" + if isinstance(error, GEOSearchError): + return f"GEO Search Error: {str(error)}" + elif isinstance(error, GEODownloadError): + return f"GEO Download Error: {str(error)}" + elif isinstance(error, SRAError): + return f"SRA Error: {str(error)}" + elif isinstance(error, ValueError): + return f"Validation Error: {str(error)}" + else: + return f"Error: {type(error).__name__}: {str(error)}" + + +# ============================================================================ +# GEO Search Tools +# ============================================================================ + +@mcp.tool( + name="geo_search", + annotations={ + "title": "Search GEO Database", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search(params: SearchWithTypesInput) -> str: + '''Search GEO for all types of records (GSE, GSM, GPL, GDS). + + This tool searches across all GEO databases and returns categorized results + by record type (Series, Samples, Platforms, Datasets). + + Args: + params: Search parameters including term, retmax, record_types, and response_format + + Returns: + JSON or Markdown formatted search results + + Examples: + - Search for cancer studies: term="breast cancer" + - Find specific series: term="GSE12345" + - Search for RNA-seq data: term="RNA-seq" + - Filter to only series: record_types=["GSE"] + ''' + try: + client = GEOSearchClient() + result = await client.search_geo( + term=params.term, + retmax=params.retmax, + record_types=params.record_types + ) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Search Results: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_search_series", + annotations={ + "title": "Search GEO Series", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search_series(params: SearchInput) -> str: + '''Search for GEO Series (GSE) - complete experiments. + + Args: + params: Search parameters including term and retmax + + Returns: + JSON or Markdown formatted GSE search results + ''' + try: + client = GEOSearchClient() + result = await client.search_geo_series(params.term, params.retmax) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Series Search: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_search_samples", + annotations={ + "title": "Search GEO Samples", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search_samples(params: SearchInput) -> str: + '''Search for GEO Samples (GSM) - individual samples. + + Args: + params: Search parameters including term and retmax + + Returns: + JSON or Markdown formatted GSM search results + ''' + try: + client = GEOSearchClient() + result = await client.search_geo_samples(params.term, params.retmax) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Sample Search: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_search_platforms", + annotations={ + "title": "Search GEO Platforms", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search_platforms(params: SearchInput) -> str: + '''Search for GEO Platforms (GPL) - array/sequencing platforms. + + Args: + params: Search parameters including term and retmax + + Returns: + JSON or Markdown formatted GPL search results + ''' + try: + client = GEOSearchClient() + result = await client.search_geo_platforms(params.term, params.retmax) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Platform Search: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_search_datasets", + annotations={ + "title": "Search GEO Datasets", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search_datasets(params: SearchInput) -> str: + '''Search for GEO Datasets (GDS) - curated gene expression datasets. + + Args: + params: Search parameters including term and retmax + + Returns: + JSON or Markdown formatted GDS search results + ''' + try: + client = GEOSearchClient() + result = await client.search_geo_datasets(params.term, params.retmax) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Dataset Search: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_search_profiles", + annotations={ + "title": "Search GEO Profiles", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_search_profiles(params: SearchInput) -> str: + '''Search GEO Profiles database for gene expression profiles. + + Args: + params: Search parameters including term and retmax + + Returns: + JSON or Markdown formatted GEO Profiles search results + ''' + try: + client = GEOSearchClient() + result = await client.search_geo_profiles(params.term, params.retmax) + + if params.response_format == ResponseFormat.MARKDOWN: + return format_as_markdown(result, f"GEO Profiles Search: '{params.term}'") + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +# ============================================================================ +# GEO Download Tools +# ============================================================================ + +@mcp.tool( + name="geo_download_series", + annotations={ + "title": "Download GEO Series Data", + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_download_series(params: DownloadInput) -> str: + '''Download GEO Series data files (SOFT, matrix, supplementary). + + Downloads data files for a GEO Series including: + - SOFT format metadata (soft) + - Series matrix file (matrix) + - MINiML XML (miniml) + - Supplementary files (supplementary) + + Args: + params: Download parameters including geo_id, file_types, and output_dir + + Returns: + JSON formatted download results + + Examples: + - Download SOFT file: geo_id="GSE12345", file_types=["soft"] + - Download all: geo_id="GSE12345", file_types=["soft", "matrix", "supplementary"] + ''' + try: + client = GEODownloadClient() + output_dir = Path(params.output_dir) if params.output_dir else None + + result = await client.download_geo_series( + gse_id=params.geo_id.upper(), + file_types=params.file_types, + output_dir=output_dir + ) + + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_download_sample", + annotations={ + "title": "Download GEO Sample Data", + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def geo_download_sample(params: DownloadInput) -> str: + '''Download GEO Sample supplementary files. + + Downloads supplementary data files for a specific GEO Sample (GSM). + + Args: + params: Download parameters including geo_id (GSM) and output_dir + + Returns: + JSON formatted download results + ''' + try: + client = GEODownloadClient() + output_dir = Path(params.output_dir) if params.output_dir else None + + result = await client.download_geo_sample( + gsm_id=params.geo_id.upper(), + output_dir=output_dir + ) + + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_get_download_status", + annotations={ + "title": "Get GEO Download Status", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False + } +) +async def geo_get_download_status(params: GeoIdInput) -> str: + '''Check if a GEO dataset has been downloaded. + + Args: + params: Parameters including geo_id and optional db_type + + Returns: + JSON formatted status information + ''' + try: + client = GEODownloadClient() + + # Determine db_type from geo_id prefix + db_type = params.geo_id[:3].lower() + + result = client.get_download_status( + accession=params.geo_id.upper(), + db_type=db_type + ) + + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_list_downloads", + annotations={ + "title": "List Downloaded GEO Datasets", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False + } +) +async def geo_list_downloads(db_type: Optional[str] = None) -> str: + '''List all downloaded GEO datasets. + + Args: + db_type: Optional filter by database type (gse, gsm, gpl, gds, sra) + + Returns: + JSON formatted list of downloaded datasets + ''' + try: + client = GEODownloadClient() + result = client.list_downloaded_datasets(db_type) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="geo_cleanup_downloads", + annotations={ + "title": "Clean Up GEO Downloads", + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": False + } +) +async def geo_cleanup_downloads(params: CleanupInput) -> str: + '''Clean up downloaded GEO files. + + Args: + params: Parameters including optional geo_id and db_type to remove + + Returns: + JSON formatted cleanup results + + Warning: + This is a destructive operation that deletes downloaded files. + ''' + try: + client = GEODownloadClient() + result = client.cleanup_downloads( + accession=params.geo_id, + db_type=params.db_type + ) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +# ============================================================================ +# SRA Tools (NEW - for raw sequencing data) +# ============================================================================ + +@mcp.tool( + name="sra_query_from_geo", + annotations={ + "title": "Query SRA Accessions from GEO Series", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def sra_query_from_geo(params: SRASearchInput) -> str: + '''Query SRA Run information from a GEO Series. + + This tool extracts the mapping between GEO Samples (GSM) and + SRA Run accessions (SRR) from a GEO Series. This allows you to + identify which SRA files contain the raw sequencing data for a dataset. + + Args: + params: Parameters including gse_id + + Returns: + JSON formatted SRA accession information + + Example: + Input: gse_id="GSE12345" + Output: { + "gse_id": "GSE12345", + "total_samples": 10, + "samples_with_sra": 8, + "all_sra_accessions": ["SRR1234567", "SRR1234568", ...], + "samples": [ + { + "gsm_id": "GSM123456", + "sra_accessions": ["SRR1234567"] + } + ] + } + ''' + try: + handler = SRAHandler() + result = await handler.query_sra_from_geo(params.gse_id.upper()) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="sra_get_metadata", + annotations={ + "title": "Get SRA Run Metadata", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def sra_get_metadata(sra_id: str) -> str: + '''Get metadata for an SRA Run accession. + + Args: + sra_id: SRA accession ID (e.g., SRR1234567) + + Returns: + JSON formatted SRA metadata + ''' + try: + handler = SRAHandler() + result = await handler.get_sra_metadata(sra_id) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="sra_generate_download_commands", + annotations={ + "title": "Generate SRA Download Commands", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False + } +) +async def sra_generate_download_commands(params: SRADownloadInput) -> str: + '''Generate commands to download SRA data. + + Generates download commands for SRA accessions using various methods: + - prefetch: Using sra-toolkit prefetch (recommended) + - fasterq-dump: Download and convert to FASTQ in one step + - wget: Direct HTTP download + - curl: Direct HTTP download using curl + - aspera: High-speed Aspera download + + Args: + params: Parameters including sra_ids list, method, and output_dir + + Returns: + JSON formatted commands and instructions + + Note: + This tool generates commands but does not execute them. + Run the commands in your terminal or use the shell tool. + ''' + try: + handler = SRAHandler() + result = handler.generate_download_commands( + sra_ids=[s.upper() for s in params.sra_ids], + method=params.method.value, + output_dir=params.output_dir + ) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="sra_check_toolkit", + annotations={ + "title": "Check SRA Toolkit Installation", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False + } +) +async def sra_check_toolkit() -> str: + '''Check if SRA Toolkit is installed and available. + + Returns: + JSON formatted toolkit status and installation information + ''' + try: + handler = SRAHandler() + result = handler.check_sra_toolkit() + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="sra_download", + annotations={ + "title": "Download SRA Data Directly", + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def sra_download(params: SRADirectDownloadInput) -> str: + '''Download SRA data directly (without sra-toolkit). + + Note: For large files or multiple downloads, using sra-toolkit + prefetch is recommended instead. This method is suitable for + small files or when sra-toolkit is not available. + + Args: + params: Parameters including sra_id, convert_to_fastq, and output_dir + + Returns: + JSON formatted download results + ''' + try: + handler = SRAHandler() + output_dir = Path(params.output_dir) if params.output_dir else None + + result = await handler.download_sra( + sra_id=params.sra_id.upper(), + convert_to_fastq=params.convert_to_fastq, + output_dir=output_dir + ) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +class SRADownloadAndConvertInput(BaseModel): + """Input for SRA download and convert operations.""" + model_config = ConfigDict(str_strip_whitespace=True) + + sra_id: str = Field( + ..., + description="SRA accession ID (e.g., SRR1234567)", + pattern=r'^[SED]RR\d+$' + ) + output_dir: Optional[str] = Field( + default=None, + description="Optional output directory (default: ~/geo_downloads/sra/). REQUIRED for files >1GB" + ) + split_3: bool = Field( + default=True, + description="Use --split-3 for 3-way splitting (recommended for mate-pairs)" + ) + check_refseq: bool = Field( + default=True, + description="Whether to check/download reference sequences (disable to save space)" + ) + dry_run: bool = Field( + default=True, + description="SAFETY: If True (default), only estimates size without downloading. Set to False to actually download." + ) + confirm_large: bool = Field( + default=False, + description="SAFETY: Must be True to download files >5GB. Use dry_run=True first to check size." + ) + + +class SRASizeEstimateInput(BaseModel): + """Input for SRA size estimation (dry-run).""" + model_config = ConfigDict(str_strip_whitespace=True) + + sra_ids: List[str] = Field( + ..., + description="List of SRA accession IDs to estimate (e.g., ['SRR1234567', 'SRR1234568'])", + min_length=1 + ) + + +@mcp.tool( + name="sra_download_and_convert", + annotations={ + "title": "Download SRA and Convert to FASTQ", + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def sra_download_and_convert(params: SRADownloadAndConvertInput) -> str: + '''Download SRA data using prefetch and convert to FASTQ using fastq-dump. + + SAFETY FEATURES (following maintainer recommendations): + - DRY-RUN BY DEFAULT: dry_run=True (default) only estimates size without downloading + - SIZE WARNINGS: Alerts for files >1GB, requires confirmation for >5GB + - EXPLICIT OUTPUT: Required output_dir for files >1GB + - CONFIRMATION: confirm_large=True required for files >5GB + + RECOMMENDED WORKFLOW: + 1. First, run with dry_run=True to see size estimates + 2. If size is acceptable, run with dry_run=False and output_dir="/path" + 3. For large files (>5GB), also set confirm_large=True + + Args: + params: Parameters including sra_id, output_dir, split_3, check_refseq, dry_run, confirm_large + + Returns: + JSON formatted results with download/conversion details OR dry-run estimates + + Examples: + - Step 1: Check size first + sra_id="SRR1234567", dry_run=true + + - Step 2a: Download small file (<1GB) + sra_id="SRR1234567", dry_run=false + + - Step 2b: Download medium file (1-5GB) + sra_id="SRR1234567", dry_run=false, output_dir="/path/to/output" + + - Step 2c: Download large file (>5GB) + sra_id="SRR1234567", dry_run=false, output_dir="/path/to/output", confirm_large=true + ''' + try: + handler = SRAHandler() + output_dir = Path(params.output_dir) if params.output_dir else None + + result = await handler.download_and_convert( + sra_id=params.sra_id.upper(), + output_dir=output_dir, + split_3=params.split_3, + check_refseq=params.check_refseq, + dry_run=params.dry_run, + confirm_large=params.confirm_large + ) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +@mcp.tool( + name="sra_estimate_size", + annotations={ + "title": "Estimate SRA Download Size (Dry-Run)", + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": True + } +) +async def sra_estimate_size(params: SRASizeEstimateInput) -> str: + '''Estimate the size of SRA files before downloading (dry-run mode). + + This tool queries the EBI ENA database to get file size estimates without + actually downloading any data. Use this before sra_download_and_convert + to check if you have sufficient disk space. + + Args: + params: Parameters including list of sra_ids to estimate + + Returns: + JSON formatted size estimates and safety warnings: + - Individual file sizes (SRA and estimated FASTQ) + - Total download size + - Safety warnings for large files (>1GB, >5GB) + + Examples: + - Estimate single file: sra_ids=["SRR1234567"] + - Estimate multiple: sra_ids=["SRR1234567", "SRR1234568"] + ''' + try: + handler = SRAHandler() + + result = await handler.estimate_sra_size( + sra_ids=[s.upper() for s in params.sra_ids] + ) + return json.dumps(result, indent=2) + + except Exception as e: + return handle_error(e) + + +# ============================================================================ +# Main Entry Point +# ============================================================================ + +def init_config(): + """Initialize configuration file.""" + config_path = Path.home() / ".geo-mcp" / "config.json" + create_config_template(config_path) + print(f"\nConfiguration template created at: {config_path}") + print("\nPlease edit the file and add your email address (required by NCBI).") + print("Optionally, add your NCBI API key for higher rate limits.") + print(f"\nTo use with Claude Desktop, add this to your config:") + print(json.dumps({ + "mcpServers": { + "geo_mcp": { + "command": "python", + "args": [str(Path(__file__).resolve())], + "env": { + "CONFIG_PATH": str(config_path) + } + } + } + }, indent=2)) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="GEO MCP Server with SRA Support", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python server.py --init # Initialize configuration + python server.py # Run MCP stdio server + python server.py --http # Run HTTP server on localhost:8000 + python server.py --http --port 8080 # Run HTTP server on custom port + """ + ) + + parser.add_argument( + "--init", + action="store_true", + help="Initialize configuration file" + ) + + parser.add_argument( + "--http", + action="store_true", + help="Run HTTP server instead of MCP stdio" + ) + + parser.add_argument( + "--host", + default="localhost", + help="Host for HTTP server (default: localhost)" + ) + + parser.add_argument( + "--port", + type=int, + default=8000, + help="Port for HTTP server (default: 8000)" + ) + + args = parser.parse_args() + + if args.init: + init_config() + return + + # Validate config before starting + try: + config = get_config() + validate_config(config) + except Exception as e: + print(f"Configuration error: {e}", file=sys.stderr) + print("Run with --init to create a configuration template.", file=sys.stderr) + sys.exit(1) + + if args.http: + # Run HTTP server + print(f"Starting HTTP server on http://{args.host}:{args.port}") + mcp.run(transport="streamable_http", host=args.host, port=args.port) + else: + # Run MCP stdio server + mcp.run() + + +if __name__ == "__main__": + main()