Add bed_snapshots table for the bulk export index

README.md

@@ -48,9 +48,9 @@ from bbconf import BedBaseAgent
 agent = BedBaseAgent(config="config.yaml")
 
 # Access submodules
-agent.bed        # BED file operations
-agent.bedset     # BED set operations
-agent.objects    # Generic object/file operations
+agent.bed  # BED file operations
+agent.bedset  # BED set operations
+agent.objects  # Generic object/file operations
 
 # Get platform statistics
 stats = agent.get_stats()

bbconf/bbagent.py

@@ -1,9 +1,11 @@
 import logging
 import statistics
+import threading
 from functools import cached_property
 from pathlib import Path
 
 import numpy as np
+from cachetools import TTLCache
 from sqlalchemy.engine import ScalarResult
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import and_, distinct, func, or_, select
@@ -36,6 +38,7 @@
 from bbconf.modules.bedfiles import BedAgentBedFile
 from bbconf.modules.bedsets import BedAgentBedSet
 from bbconf.modules.objects import BBObjects
+from bbconf.modules.snapshots import BedAgentSnapshot
 
 from .const import PKG_NAME
 
@@ -62,6 +65,15 @@ def __init__(
         self._bed = BedAgentBedFile(self.config, self)
         self._bedset = BedAgentBedSet(self.config)
         self._objects = BBObjects(self.config)
+        self._snapshot = BedAgentSnapshot(self.config)
+
+        # get_stats() runs three uncached COUNT queries on the multi-hundred-
+        # thousand-row bed table and is called on hot paths (the stats endpoint
+        # plus the neighbours/list/search result builders). Cache the result
+        # with a TTL so those paths do not hit the database on every request.
+        # The lock guards the cache dict only, never the DB query itself.
+        self._stats_cache = TTLCache(maxsize=1, ttl=3600)
+        self._stats_lock = threading.Lock()
 
     @property
     def bed(self) -> BedAgentBedFile:
@@ -75,6 +87,10 @@ def bedset(self) -> BedAgentBedSet:
     def objects(self) -> BBObjects:
         return self._objects
 
+    @property
+    def snapshot(self) -> BedAgentSnapshot:
+        return self._snapshot
+
     def __repr__(self) -> str:
         repr = f"BedBaseAgent(config={self.config})"
         repr += f"\n{self.bed}"
@@ -86,9 +102,17 @@ def get_stats(self) -> StatsReturn:
         """
         Get statistics for a bed file.
 
+        The result is cached with a TTL because this runs three COUNT queries
+        against the large bed table and is called on hot API paths.
+
         Returns:
             Statistics.
         """
+        with self._stats_lock:
+            cached = self._stats_cache.get("stats")
+        if cached is not None:
+            return cached
+
         with Session(self.config.db_engine.engine) as session:
             number_of_bed = session.execute(select(func.count(Bed.id))).one()[0]
             number_of_bedset = session.execute(select(func.count(BedSets.id))).one()[0]
@@ -97,12 +121,17 @@ def get_stats(self) -> StatsReturn:
                 select(func.count(distinct(Bed.genome_alias)))
             ).one()[0]
 
-        return StatsReturn(
+        stats = StatsReturn(
             bedfiles_number=number_of_bed,
             bedsets_number=number_of_bedset,
             genomes_number=number_of_genomes,
         )
 
+        with self._stats_lock:
+            self._stats_cache["stats"] = stats
+
+        return stats
+
     def get_detailed_stats(self, concise: bool = False) -> FileStats:
         """
         Get comprehensive statistics for all bed files.
@@ -116,11 +145,29 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
 
         _LOGGER.info("Getting detailed statistics for all bed files")
 
+        numeric_stats_statement = (
+            select(
+                BedStats.number_of_regions,
+                BedStats.mean_region_width,
+                Files.size,
+            )
+            .select_from(Bed)
+            .join(BedStats, BedStats.id == Bed.id)
+            .join(Files, Files.bedfile_id == Bed.id)
+            .where(
+                Files.name == "bed_file",
+                BedStats.number_of_regions.is_not(None),
+                BedStats.mean_region_width.is_not(None),
+                Files.size.is_not(None),
+            )
+        )
+
         with Session(self.config.db_engine.engine) as session:
             bed_compliance = {
                 f[0]: f[1]
                 for f in session.execute(
                     select(Bed.bed_compliance, func.count(Bed.bed_compliance))
+                    .where(Bed.bed_compliance.is_not(None))
                     .group_by(Bed.bed_compliance)
                     .order_by(func.count(Bed.bed_compliance).desc())
                 ).all()
@@ -129,6 +176,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
                 f[0]: f[1]
                 for f in session.execute(
                     select(Bed.data_format, func.count(Bed.data_format))
+                    .where(Bed.data_format.is_not(None))
                     .group_by(Bed.data_format)
                     .order_by(func.count(Bed.data_format).desc())
                 ).all()
@@ -137,6 +185,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
                 f[0]: f[1]
                 for f in session.execute(
                     select(Bed.genome_alias, func.count(Bed.genome_alias))
+                    .where(Bed.genome_alias.is_not(None))
                     .group_by(Bed.genome_alias)
                     .order_by(func.count(Bed.genome_alias).desc())
                 ).all()
@@ -147,6 +196,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
                     select(
                         BedMetadata.species_name, func.count(BedMetadata.species_name)
                     )
+                    .where(BedMetadata.species_name.is_not(None))
                     .group_by(BedMetadata.species_name)
                     .order_by(func.count(BedMetadata.species_name).desc())
                 ).all()
@@ -155,6 +205,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
                 f[0]: f[1]
                 for f in session.execute(
                     select(BedMetadata.assay, func.count(BedMetadata.assay))
+                    .where(BedMetadata.assay.is_not(None))
                     .group_by(BedMetadata.assay)
                     .order_by(func.count(BedMetadata.assay).desc())
                 ).all()
@@ -163,28 +214,29 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats:
                 f[0]: f[1]
                 for f in session.execute(
                     select(BedMetadata.cell_line, func.count(BedMetadata.cell_line))
+                    .where(BedMetadata.cell_line.is_not(None))
                     .group_by(BedMetadata.cell_line)
                     .order_by(func.count(BedMetadata.cell_line).desc())
                 ).all()
             }
 
-        slice_value = 20
+            bed_comments = self._stats_comments(session)
+            geo_status = self._stats_geo_status(session)
+
+            numeric_rows = session.execute(numeric_stats_statement).all()
 
-        bed_comments = self._stats_comments(session)
-        geo_status = self._stats_geo_status(session)
+            geo_stats = self._get_geo_stats(session)
 
-        bedfiles_info = self.bed_files_info()
+        slice_value = 20
 
-        number_of_regions = [bed.number_of_regions for bed in bedfiles_info.files]
-        list_mean_width = [bed.mean_region_width for bed in bedfiles_info.files]
-        list_file_size = [bed.file_size for bed in bedfiles_info.files]
+        number_of_regions = [row[0] for row in numeric_rows]
+        list_mean_width = [row[1] for row in numeric_rows]
+        list_file_size = [row[2] for row in numeric_rows]
 
         number_of_regions_bins = self._bin_number_of_regions(number_of_regions)
         list_mean_width_bins = self._bin_mean_region_width(list_mean_width)
         list_file_size_bins = self._bin_file_size(list_file_size)
 
-        geo_stats = self._get_geo_stats(session)
-
         if concise:
             bed_compliance_concise = dict(list(bed_compliance.items())[0:slice_value])
             bed_compliance_concise["other"] = sum(

bbconf/db_utils.py

@@ -338,6 +338,10 @@ class BedSets(Base):
         JSON, comment="Median values of the bedset"
     )
 
+    bedfile_count: Mapped[int] = mapped_column(
+        default=0, comment="Number of bedfiles in the bedset (denormalized count)"
+    )
+
     bedfiles: Mapped[list["BedFileBedSetRelation"]] = relationship(
         "BedFileBedSetRelation", back_populates="bedset", cascade="all, delete-orphan"
     )
@@ -597,6 +601,42 @@ class UsageSearch(Base):
     date_to: Mapped[datetime.datetime] = mapped_column(comment="Date to")
 
 
+class BedSnapshot(Base):
+    """
+    Index of bulk metadata exports published to S3.
+
+    One row per published artifact (metadata / bedsets / membership / manifest).
+    The exporter writes a row after a successful upload; the /v1/bed/exports
+    endpoint reads them newest-first. This is a new table, so
+    Base.metadata.create_all() creates it on the next connection.
+    """
+
+    __tablename__ = "bed_snapshots"
+
+    id: Mapped[int] = mapped_column(primary_key=True, index=True, autoincrement=True)
+    file_path: Mapped[str] = mapped_column(
+        nullable=False, comment="S3 object key, relative to the bucket root"
+    )
+    file_type: Mapped[str] = mapped_column(
+        nullable=False, comment="metadata | bedsets | bedset_membership | manifest"
+    )
+    creation_date: Mapped[datetime.datetime] = mapped_column(
+        default=deliver_update_date, comment="Build date of the export"
+    )
+    record_count: Mapped[Optional[int]] = mapped_column(
+        nullable=True, comment="Rows actually written to the file"
+    )
+    file_size: Mapped[Optional[int]] = mapped_column(
+        nullable=True, comment="Size of the file in bytes"
+    )
+    checksum: Mapped[Optional[str]] = mapped_column(
+        nullable=True, comment="SHA256 of the file"
+    )
+    schema_version: Mapped[Optional[int]] = mapped_column(
+        nullable=True, comment="Export schema version"
+    )
+
+
 class BaseEngine:
     """
     A class with base methods, that are used in several classes.

bbconf/exceptions.py

@@ -70,6 +70,13 @@ class BedSetExistsError(BedBaseConfError):
     pass
 
 
+class SnapshotNotFoundError(BedBaseConfError):
+    """
+    Error type for missing snapshot"""
+
+    pass
+
+
 class UniverseNotFoundError(BedBaseConfError):
     """
     Error type for missing universe"""

bbconf/models/base_models.py

@@ -103,3 +103,31 @@ class FileStats(BaseModel):
     file_size: BinValues
     number_of_regions: BinValues
     geo: GEOStatistics
+
+
+class BedSnapshotArtifact(BaseModel):
+    """A built snapshot file to publish (upload to S3 + record in the database)."""
+
+    path: str  # local file path to upload
+    file_type: str
+    record_count: int | None = None
+    file_size: int | None = None
+    checksum: str | None = None
+    schema_version: int | None = None
+
+
+class BedSnapshotResult(BaseModel):
+    """One published bulk-export artifact."""
+
+    file_path: str
+    file_type: str
+    creation_date: datetime.datetime
+    record_count: int | None = None
+    file_size: int | None = None
+    checksum: str | None = None
+    schema_version: int | None = None
+
+
+class BedSnapshotListResult(BaseModel):
+    count: int
+    results: list[BedSnapshotResult]

bbconf/models/bed_models.py

@@ -190,6 +190,7 @@ class BedSetMinimal(BaseModel):
     id: str
     name: str | None = None
     description: str | None = None
+    bedfile_count: int = 0
 
 
 class BedMetadataAll(BedMetadataBasic):

bbconf/models/bedset_models.py

@@ -28,6 +28,7 @@ class BedSetMetadata(BaseModel):
     description: str = None
     summary: str = None
     bed_ids: list[str] = None
+    bedfile_count: int = 0
     author: str | None = None
     source: str | None = None