diff --git a/README.md b/README.md index 227e40f..df71db0 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,9 @@ from bbconf import BedBaseAgent agent = BedBaseAgent(config="config.yaml") # Access submodules -agent.bed # BED file operations -agent.bedset # BED set operations -agent.objects # Generic object/file operations +agent.bed # BED file operations +agent.bedset # BED set operations +agent.objects # Generic object/file operations # Get platform statistics stats = agent.get_stats() diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index 6248f6d..41acaf4 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -1,9 +1,11 @@ import logging import statistics +import threading from functools import cached_property from pathlib import Path import numpy as np +from cachetools import TTLCache from sqlalchemy.engine import ScalarResult from sqlalchemy.orm import Session from sqlalchemy.sql import and_, distinct, func, or_, select @@ -36,6 +38,7 @@ from bbconf.modules.bedfiles import BedAgentBedFile from bbconf.modules.bedsets import BedAgentBedSet from bbconf.modules.objects import BBObjects +from bbconf.modules.snapshots import BedAgentSnapshot from .const import PKG_NAME @@ -62,6 +65,15 @@ def __init__( self._bed = BedAgentBedFile(self.config, self) self._bedset = BedAgentBedSet(self.config) self._objects = BBObjects(self.config) + self._snapshot = BedAgentSnapshot(self.config) + + # get_stats() runs three uncached COUNT queries on the multi-hundred- + # thousand-row bed table and is called on hot paths (the stats endpoint + # plus the neighbours/list/search result builders). Cache the result + # with a TTL so those paths do not hit the database on every request. + # The lock guards the cache dict only, never the DB query itself. + self._stats_cache = TTLCache(maxsize=1, ttl=3600) + self._stats_lock = threading.Lock() @property def bed(self) -> BedAgentBedFile: @@ -75,6 +87,10 @@ def bedset(self) -> BedAgentBedSet: def objects(self) -> BBObjects: return self._objects + @property + def snapshot(self) -> BedAgentSnapshot: + return self._snapshot + def __repr__(self) -> str: repr = f"BedBaseAgent(config={self.config})" repr += f"\n{self.bed}" @@ -86,9 +102,17 @@ def get_stats(self) -> StatsReturn: """ Get statistics for a bed file. + The result is cached with a TTL because this runs three COUNT queries + against the large bed table and is called on hot API paths. + Returns: Statistics. """ + with self._stats_lock: + cached = self._stats_cache.get("stats") + if cached is not None: + return cached + with Session(self.config.db_engine.engine) as session: number_of_bed = session.execute(select(func.count(Bed.id))).one()[0] number_of_bedset = session.execute(select(func.count(BedSets.id))).one()[0] @@ -97,12 +121,17 @@ def get_stats(self) -> StatsReturn: select(func.count(distinct(Bed.genome_alias))) ).one()[0] - return StatsReturn( + stats = StatsReturn( bedfiles_number=number_of_bed, bedsets_number=number_of_bedset, genomes_number=number_of_genomes, ) + with self._stats_lock: + self._stats_cache["stats"] = stats + + return stats + def get_detailed_stats(self, concise: bool = False) -> FileStats: """ Get comprehensive statistics for all bed files. @@ -116,11 +145,29 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: _LOGGER.info("Getting detailed statistics for all bed files") + numeric_stats_statement = ( + select( + BedStats.number_of_regions, + BedStats.mean_region_width, + Files.size, + ) + .select_from(Bed) + .join(BedStats, BedStats.id == Bed.id) + .join(Files, Files.bedfile_id == Bed.id) + .where( + Files.name == "bed_file", + BedStats.number_of_regions.is_not(None), + BedStats.mean_region_width.is_not(None), + Files.size.is_not(None), + ) + ) + with Session(self.config.db_engine.engine) as session: bed_compliance = { f[0]: f[1] for f in session.execute( select(Bed.bed_compliance, func.count(Bed.bed_compliance)) + .where(Bed.bed_compliance.is_not(None)) .group_by(Bed.bed_compliance) .order_by(func.count(Bed.bed_compliance).desc()) ).all() @@ -129,6 +176,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: f[0]: f[1] for f in session.execute( select(Bed.data_format, func.count(Bed.data_format)) + .where(Bed.data_format.is_not(None)) .group_by(Bed.data_format) .order_by(func.count(Bed.data_format).desc()) ).all() @@ -137,6 +185,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: f[0]: f[1] for f in session.execute( select(Bed.genome_alias, func.count(Bed.genome_alias)) + .where(Bed.genome_alias.is_not(None)) .group_by(Bed.genome_alias) .order_by(func.count(Bed.genome_alias).desc()) ).all() @@ -147,6 +196,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: select( BedMetadata.species_name, func.count(BedMetadata.species_name) ) + .where(BedMetadata.species_name.is_not(None)) .group_by(BedMetadata.species_name) .order_by(func.count(BedMetadata.species_name).desc()) ).all() @@ -155,6 +205,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: f[0]: f[1] for f in session.execute( select(BedMetadata.assay, func.count(BedMetadata.assay)) + .where(BedMetadata.assay.is_not(None)) .group_by(BedMetadata.assay) .order_by(func.count(BedMetadata.assay).desc()) ).all() @@ -163,28 +214,29 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: f[0]: f[1] for f in session.execute( select(BedMetadata.cell_line, func.count(BedMetadata.cell_line)) + .where(BedMetadata.cell_line.is_not(None)) .group_by(BedMetadata.cell_line) .order_by(func.count(BedMetadata.cell_line).desc()) ).all() } - slice_value = 20 + bed_comments = self._stats_comments(session) + geo_status = self._stats_geo_status(session) + + numeric_rows = session.execute(numeric_stats_statement).all() - bed_comments = self._stats_comments(session) - geo_status = self._stats_geo_status(session) + geo_stats = self._get_geo_stats(session) - bedfiles_info = self.bed_files_info() + slice_value = 20 - number_of_regions = [bed.number_of_regions for bed in bedfiles_info.files] - list_mean_width = [bed.mean_region_width for bed in bedfiles_info.files] - list_file_size = [bed.file_size for bed in bedfiles_info.files] + number_of_regions = [row[0] for row in numeric_rows] + list_mean_width = [row[1] for row in numeric_rows] + list_file_size = [row[2] for row in numeric_rows] number_of_regions_bins = self._bin_number_of_regions(number_of_regions) list_mean_width_bins = self._bin_mean_region_width(list_mean_width) list_file_size_bins = self._bin_file_size(list_file_size) - geo_stats = self._get_geo_stats(session) - if concise: bed_compliance_concise = dict(list(bed_compliance.items())[0:slice_value]) bed_compliance_concise["other"] = sum( diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 050c7e6..0bc9f1e 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -338,6 +338,10 @@ class BedSets(Base): JSON, comment="Median values of the bedset" ) + bedfile_count: Mapped[int] = mapped_column( + default=0, comment="Number of bedfiles in the bedset (denormalized count)" + ) + bedfiles: Mapped[list["BedFileBedSetRelation"]] = relationship( "BedFileBedSetRelation", back_populates="bedset", cascade="all, delete-orphan" ) @@ -597,6 +601,42 @@ class UsageSearch(Base): date_to: Mapped[datetime.datetime] = mapped_column(comment="Date to") +class BedSnapshot(Base): + """ + Index of bulk metadata exports published to S3. + + One row per published artifact (metadata / bedsets / membership / manifest). + The exporter writes a row after a successful upload; the /v1/bed/exports + endpoint reads them newest-first. This is a new table, so + Base.metadata.create_all() creates it on the next connection. + """ + + __tablename__ = "bed_snapshots" + + id: Mapped[int] = mapped_column(primary_key=True, index=True, autoincrement=True) + file_path: Mapped[str] = mapped_column( + nullable=False, comment="S3 object key, relative to the bucket root" + ) + file_type: Mapped[str] = mapped_column( + nullable=False, comment="metadata | bedsets | bedset_membership | manifest" + ) + creation_date: Mapped[datetime.datetime] = mapped_column( + default=deliver_update_date, comment="Build date of the export" + ) + record_count: Mapped[Optional[int]] = mapped_column( + nullable=True, comment="Rows actually written to the file" + ) + file_size: Mapped[Optional[int]] = mapped_column( + nullable=True, comment="Size of the file in bytes" + ) + checksum: Mapped[Optional[str]] = mapped_column( + nullable=True, comment="SHA256 of the file" + ) + schema_version: Mapped[Optional[int]] = mapped_column( + nullable=True, comment="Export schema version" + ) + + class BaseEngine: """ A class with base methods, that are used in several classes. diff --git a/bbconf/exceptions.py b/bbconf/exceptions.py index 3ad393c..6991c99 100644 --- a/bbconf/exceptions.py +++ b/bbconf/exceptions.py @@ -70,6 +70,13 @@ class BedSetExistsError(BedBaseConfError): pass +class SnapshotNotFoundError(BedBaseConfError): + """ + Error type for missing snapshot""" + + pass + + class UniverseNotFoundError(BedBaseConfError): """ Error type for missing universe""" diff --git a/bbconf/models/base_models.py b/bbconf/models/base_models.py index ef3efd5..a465c81 100644 --- a/bbconf/models/base_models.py +++ b/bbconf/models/base_models.py @@ -103,3 +103,31 @@ class FileStats(BaseModel): file_size: BinValues number_of_regions: BinValues geo: GEOStatistics + + +class BedSnapshotArtifact(BaseModel): + """A built snapshot file to publish (upload to S3 + record in the database).""" + + path: str # local file path to upload + file_type: str + record_count: int | None = None + file_size: int | None = None + checksum: str | None = None + schema_version: int | None = None + + +class BedSnapshotResult(BaseModel): + """One published bulk-export artifact.""" + + file_path: str + file_type: str + creation_date: datetime.datetime + record_count: int | None = None + file_size: int | None = None + checksum: str | None = None + schema_version: int | None = None + + +class BedSnapshotListResult(BaseModel): + count: int + results: list[BedSnapshotResult] \ No newline at end of file diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index c705649..4d66b7f 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -190,6 +190,7 @@ class BedSetMinimal(BaseModel): id: str name: str | None = None description: str | None = None + bedfile_count: int = 0 class BedMetadataAll(BedMetadataBasic): diff --git a/bbconf/models/bedset_models.py b/bbconf/models/bedset_models.py index ca074a9..3f3d852 100644 --- a/bbconf/models/bedset_models.py +++ b/bbconf/models/bedset_models.py @@ -28,6 +28,7 @@ class BedSetMetadata(BaseModel): description: str = None summary: str = None bed_ids: list[str] = None + bedfile_count: int = 0 author: str | None = None source: str | None = None diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index c45d0be..37e4036 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -15,7 +15,7 @@ from sqlalchemy import and_, cast, delete, func, or_, select from sqlalchemy.dialects import postgresql from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session, aliased +from sqlalchemy.orm import Session, aliased, selectinload from sqlalchemy.orm.attributes import flag_modified from tqdm import tqdm @@ -107,20 +107,56 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll: """ statement = select(Bed).where(and_(Bed.id == identifier)) - bed_plots = BedPlots() - bed_files = BedFiles() - with Session(self._sa_engine) as session: bed_object = session.scalar(statement) if not bed_object: raise BEDFileNotFoundError(f"Bed file with id: {identifier} not found.") - if full: - for result in bed_object.files: - # PLOTS - if result.name in BedPlots.model_fields: + return self._build_metadata(bed_object, full=full) + + def _build_metadata(self, bed_object: Bed, full: bool = False) -> BedMetadataAll: + """ + Build a BedMetadataAll model from a Bed ORM object. + + For ``full=True`` this assembles plots, files, stats, bedsets, and + universe metadata (which lazy-load relationships, so the caller must + keep the SQLAlchemy session open). For ``full=False`` only scalar + columns and the (joined-loaded) annotations are accessed, so the + Bed object may be detached from its session. + + Args: + bed_object: Bed ORM object to build metadata from. + full: If True, return full metadata, including statistics, files, + and raw metadata from pephub. + + Returns: + BED file metadata. + """ + identifier = bed_object.id + + bed_plots = BedPlots() + bed_files = BedFiles() + + if full: + for result in bed_object.files: + # PLOTS + if result.name in BedPlots.model_fields: + setattr( + bed_plots, + result.name, + FileModel( + **result.__dict__, + object_id=f"bed.{identifier}.{result.name}", + access_methods=self.config.construct_access_method_list( + result.path + ), + ), + ) + # FILES + elif result.name in BedFiles.model_fields: + ( setattr( - bed_plots, + bed_files, result.name, FileModel( **result.__dict__, @@ -129,48 +165,35 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll: result.path ), ), - ) - # FILES - elif result.name in BedFiles.model_fields: - ( - setattr( - bed_files, - result.name, - FileModel( - **result.__dict__, - object_id=f"bed.{identifier}.{result.name}", - access_methods=self.config.construct_access_method_list( - result.path - ), - ), - ), - ) - - else: - _LOGGER.error( - f"Unknown file type: {result.name}. And is not in the model fields. Skipping.." - ) - bed_stats = BedStatsModel(**bed_object.stats.__dict__) - bed_bedsets = [] - for relation in bed_object.bedsets: - bed_bedsets.append( - BedSetMinimal( - id=relation.bedset.id, - description=relation.bedset.description, - name=relation.bedset.name, - ) + ), ) - if bed_object.universe: - universe_meta = UniverseMetadata(**bed_object.universe.__dict__) else: - universe_meta = UniverseMetadata() + _LOGGER.error( + f"Unknown file type: {result.name}. And is not in the model fields. Skipping.." + ) + bed_stats = BedStatsModel(**bed_object.stats.__dict__) + bed_bedsets = [] + for relation in bed_object.bedsets: + bed_bedsets.append( + BedSetMinimal( + id=relation.bedset.id, + description=relation.bedset.description, + name=relation.bedset.name, + bedfile_count=relation.bedset.bedfile_count, + ) + ) + + if bed_object.universe: + universe_meta = UniverseMetadata(**bed_object.universe.__dict__) else: - bed_plots = None - bed_files = None - bed_stats = None - universe_meta = None - bed_bedsets = [] + universe_meta = UniverseMetadata() + else: + bed_plots = None + bed_files = None + bed_stats = None + universe_meta = None + bed_bedsets = [] try: if full: @@ -290,17 +313,32 @@ def get_neighbours( limit=limit, offset=offset, ) - result_list = [] - for result in results.points: - result_id = result.id.replace("-", "") - result_list.append( - QdrantSearchResult( - id=result_id, - payload=result.payload, - score=result.score, - metadata=self.get(result_id, full=False), - ) + # Hydrate all neighbours with a single batched query instead of one + # SELECT per neighbour (was an N+1). annotations is joined-loaded, + # but selectinload keeps that explicit for this detached-object path. + ids = [result.id.replace("-", "") for result in results.points] + with Session(self._sa_engine) as session: + beds = { + bed.id: bed + for bed in session.scalars( + select(Bed) + .where(Bed.id.in_(ids)) + .options(selectinload(Bed.annotations)) + ).all() + } + result_list = [ + QdrantSearchResult( + id=result.id.replace("-", ""), + payload=result.payload, + score=result.score, + metadata=self._build_metadata( + beds[result.id.replace("-", "")], full=False + ), ) + for result in results.points + # skip stale Qdrant points that no longer exist in the database + if result.id.replace("-", "") in beds + ] except UnexpectedResponse as err: _LOGGER.error( f"Qdrant request failed. Error: {err}. Returning empty result set." @@ -470,7 +508,7 @@ def get_ids_list( and_(Bed.bed_compliance == bed_compliance) ) - statement = statement.limit(limit).offset(offset) + statement = statement.order_by(Bed.id).limit(limit).offset(offset) result_list = [] with Session(self._sa_engine) as session: diff --git a/bbconf/modules/bedsets.py b/bbconf/modules/bedsets.py index 83566ac..0713c0a 100644 --- a/bbconf/modules/bedsets.py +++ b/bbconf/modules/bedsets.py @@ -89,6 +89,7 @@ def get(self, identifier: str, full: bool = False) -> BedSetMetadata: statistics=stats, plots=plots, bed_ids=list_of_bedfiles, + bedfile_count=bedset_obj.bedfile_count, submission_date=bedset_obj.submission_date, last_update_date=bedset_obj.last_update_date, author=bedset_obj.author, @@ -364,6 +365,9 @@ def create( if not no_fail: raise e + if no_fail: + bedid_list = list(set(bedid_list)) + new_bedset = BedSets( id=identifier, name=name, @@ -375,6 +379,7 @@ def create( author=annotation.get("author"), source=annotation.get("source"), processed=processed, + bedfile_count=len(bedid_list), ) if upload_s3: @@ -387,8 +392,6 @@ def create( with Session(self._db_engine.engine) as session: session.add(new_bedset) - if no_fail: - bedid_list = list(set(bedid_list)) for bedfile in bedid_list: session.add( BedFileBedSetRelation(bedset_id=identifier, bedfile_id=bedfile) @@ -459,47 +462,51 @@ def _calculate_statistics(self, bed_ids: list[str]) -> BedSetStats: _LOGGER.info("Bedset statistics were calculated successfully") return bedset_stats - def _create_pephub_view( - self, - bedset_id: str, - description: str = None, - bed_ids: list = None, - nofail: bool = False, - ) -> None: - """ - Create view in pephub for bedset. - - Args: - bedset_id: Bedset identifier. - description: Bedset description. - bed_ids: List of bed file identifiers. - nofail: Do not raise an error if sample not found. - - Returns: - None. - """ - - _LOGGER.info(f"Creating view in pephub for bedset '{bedset_id}'") - try: - self.config.phc.view.create( - namespace=self.config.config.phc.namespace, - name=self.config.config.phc.name, - tag=self.config.config.phc.tag, - view_name=bedset_id, - # description=description, - sample_list=bed_ids, - ) - except Exception as e: - _LOGGER.error(f"Failed to create view in pephub: {e}") - if not nofail: - raise e - return None + # def _create_pephub_view( + # self, + # bedset_id: str, + # description: str = None, + # bed_ids: list = None, + # nofail: bool = False, + # ) -> None: + # """ + # Create view in pephub for bedset. + # + # Args: + # bedset_id: Bedset identifier. + # description: Bedset description. + # bed_ids: List of bed file identifiers. + # nofail: Do not raise an error if sample not found. + # + # Returns: + # None. + # """ + # + # _LOGGER.info(f"Creating view in pephub for bedset '{bedset_id}'") + # try: + # self.config.phc.view.create( + # namespace=self.config.config.phc.namespace, + # name=self.config.config.phc.name, + # tag=self.config.config.phc.tag, + # view_name=bedset_id, + # # description=description, + # sample_list=bed_ids, + # ) + # except Exception as e: + # _LOGGER.error(f"Failed to create view in pephub: {e}") + # if not nofail: + # raise e + # return None def get_ids_list( - self, query: str = None, limit: int = 10, offset: int = 0 + self, query: str | None = None, limit: int = 10, offset: int = 0 ) -> BedSetListResult: """ - Get list of bedsets from the database. + Find (search) bedsets from the database. + + Use `get(identifier)` to + fetch a single bedset's member ids. `bedfile_count` is populated + directly from the denormalized column, so it's free. Args: query: Search query. @@ -509,7 +516,7 @@ def get_ids_list( Returns: List of bedsets. """ - statement = select(BedSets.id) + statement = select(BedSets) count_statement = select(func.count(BedSets.id)) if query: query = query.strip() @@ -528,12 +535,24 @@ def get_ids_list( ) with Session(self._db_engine.engine) as session: - bedset_list = session.execute(statement.limit(limit).offset(offset)) + bedset_list = session.scalars(statement.limit(limit).offset(offset)) bedset_count = session.execute(count_statement).one() - result_list = [] - for bedset_id in bedset_list: - result_list.append(self.get(bedset_id[0])) + result_list = [ + BedSetMetadata( + id=bedset_obj.id, + name=bedset_obj.name, + description=bedset_obj.description, + md5sum=bedset_obj.md5sum, + bedfile_count=bedset_obj.bedfile_count, + submission_date=bedset_obj.submission_date, + last_update_date=bedset_obj.last_update_date, + author=bedset_obj.author, + source=bedset_obj.source, + ) + for bedset_obj in bedset_list + ] + return BedSetListResult( count=bedset_count[0], limit=limit, @@ -601,34 +620,33 @@ def delete(self, identifier: str) -> None: session.delete(bedset_obj) session.commit() - self.delete_phc_view(identifier, nofail=True) if files: self.config.delete_files_s3(files) - def delete_phc_view(self, identifier: str, nofail: bool = False) -> None: - """ - Delete view in pephub. - - Args: - identifier: Bedset identifier. - nofail: Do not raise an error if view not found. - - Returns: - None. - """ - _LOGGER.info(f"Deleting view in pephub for bedset '{identifier}'") - try: - self.config.phc.view.delete( - namespace=self.config.config.phc.namespace, - name=self.config.config.phc.name, - tag=self.config.config.phc.tag, - view_name=identifier, - ) - except Exception as e: - _LOGGER.error(f"Failed to delete view in pephub: {e}") - if not nofail: - raise e - return None + # def delete_phc_view(self, identifier: str, nofail: bool = False) -> None: + # """ + # Delete view in pephub. + # + # Args: + # identifier: Bedset identifier. + # nofail: Do not raise an error if view not found. + # + # Returns: + # None. + # """ + # _LOGGER.info(f"Deleting view in pephub for bedset '{identifier}'") + # try: + # self.config.phc.view.delete( + # namespace=self.config.config.phc.namespace, + # name=self.config.config.phc.name, + # tag=self.config.config.phc.tag, + # view_name=identifier, + # ) + # except Exception as e: + # _LOGGER.error(f"Failed to delete view in pephub: {e}") + # if not nofail: + # raise e + # return None def exists(self, identifier: str) -> bool: """ @@ -688,6 +706,7 @@ def get_unprocessed(self, limit: int = 100, offset: int = 0) -> BedSetListResult statistics=None, plots=None, bed_ids=list_of_bedfiles, + bedfile_count=bedset_obj.bedfile_count, submission_date=bedset_obj.submission_date, last_update_date=bedset_obj.last_update_date, author=bedset_obj.author, diff --git a/bbconf/modules/snapshots.py b/bbconf/modules/snapshots.py new file mode 100644 index 0000000..ba488d1 --- /dev/null +++ b/bbconf/modules/snapshots.py @@ -0,0 +1,264 @@ +import logging +import os +from datetime import datetime, timezone + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from bbconf.config_parser import BedBaseConfig +from bbconf.const import PKG_NAME +from bbconf.db_utils import BedSnapshot +from bbconf.exceptions import SnapshotNotFoundError +from bbconf.models.base_models import ( + BedSnapshotArtifact, + BedSnapshotListResult, + BedSnapshotResult, +) + +_LOGGER = logging.getLogger(PKG_NAME) + +# All snapshots live under this single S3 prefix. Not configurable. +SNAPSHOT_S3_PREFIX = "snapshot" + + +class BedAgentSnapshot: + """ + Class that manages bulk-export snapshots (the ``bed_snapshots`` index). + + One row per published artifact (metadata / bedsets / bedset_membership / + manifest). Adding a snapshot always uploads the file to S3 *and* records it + in the database; both writes live here in bbconf. This class also exposes + read (``list`` / ``get``) and ``delete`` helpers. + """ + + def __init__(self, config: BedBaseConfig): + """ + Initialize BedAgentSnapshot. + + Args: + config: Config object. + """ + self.config = config + self._db_engine = self.config.db_engine + + def add( + self, + artifacts: BedSnapshotArtifact | list[BedSnapshotArtifact], + creation_date: datetime | None = None, + ) -> BedSnapshotListResult: + """ + Add snapshot artifacts: upload each to S3 and record it in the database. + + Every artifact is uploaded under the fixed ``snapshot/`` prefix and then + recorded in ``bed_snapshots``. The index rows are written only after all + uploads succeed, so a partial upload never leaves dangling rows. + + Args: + artifacts: One artifact or a list of them. Each carries the local + ``path`` to upload plus its ``file_type`` and file metadata. + creation_date: Build date recorded on every row + (defaults to now, UTC). + + Returns: + The created snapshot rows. + """ + if isinstance(artifacts, BedSnapshotArtifact): + artifacts = [artifacts] + if creation_date is None: + creation_date = datetime.now(timezone.utc) + + # Upload everything first; only record rows once all uploads succeed. + results: list[BedSnapshotResult] = [] + for artifact in artifacts: + key = f"{SNAPSHOT_S3_PREFIX}/{os.path.basename(artifact.path)}" + self.config.upload_s3(artifact.path, s3_path=key) + results.append( + BedSnapshotResult( + file_path=key, + file_type=artifact.file_type, + creation_date=creation_date, + record_count=artifact.record_count, + file_size=artifact.file_size, + checksum=artifact.checksum, + schema_version=artifact.schema_version, + ) + ) + + with Session(self._db_engine.engine) as session: + for result in results: + session.add( + BedSnapshot( + file_path=result.file_path, + file_type=result.file_type, + creation_date=result.creation_date, + record_count=result.record_count, + file_size=result.file_size, + checksum=result.checksum, + schema_version=result.schema_version, + ) + ) + session.commit() + + _LOGGER.info(f"Recorded {len(results)} rows in bed_snapshots") + return BedSnapshotListResult(count=len(results), results=results) + + def delete(self, id: int, remove_s3: bool = True) -> None: + """ + Delete a snapshot index row. + + Args: + id: Primary key of the snapshot row. + remove_s3: Also delete the underlying S3 object. + + Returns: + None. + + Raises: + SnapshotNotFoundError: If no row with this id exists. + """ + with Session(self._db_engine.engine) as session: + row = session.scalar(select(BedSnapshot).where(BedSnapshot.id == id)) + if row is None: + raise SnapshotNotFoundError(f"Snapshot with id '{id}' not found.") + file_path = row.file_path + session.delete(row) + session.commit() + + if remove_s3: + self.config.delete_s3(file_path) + + def list( + self, + file_type: str | None = None, + limit: int | None = 100, + offset: int = 0, + ) -> BedSnapshotListResult: + """ + List all snapshot index rows in the database, newest first. + + Args: + file_type: Optional filter on file type. + limit: Maximum number of rows to return. ``None`` returns all rows. + offset: Number of rows to skip. + + Returns: + List of snapshots and the total matching count. + """ + statement = select(BedSnapshot) + count_statement = select(func.count()).select_from(BedSnapshot) + if file_type is not None: + statement = statement.where(BedSnapshot.file_type == file_type) + count_statement = count_statement.where( + BedSnapshot.file_type == file_type + ) + statement = statement.order_by( + BedSnapshot.creation_date.desc(), BedSnapshot.id.desc() + ) + if limit is not None: + statement = statement.limit(limit).offset(offset) + elif offset: + statement = statement.offset(offset) + + with Session(self._db_engine.engine) as session: + total = session.execute(count_statement).scalar_one() + rows = session.scalars(statement).all() + results = [self._to_result(row) for row in rows] + + return BedSnapshotListResult(count=total, results=results) + + def get_by_filename(self, filename: str) -> BedSnapshotResult: + """ + Resolve a snapshot by its file name (the basename of its S3 key). + + Returns the newest row whose ``file_path`` basename equals ``filename``. + Used to round-trip an export's DRS object-id back to its row. + + Args: + filename: The bare file name, e.g. + ``bedbase_metadata_2026_08_03.parquet``. + + Returns: + The matching snapshot row. + + Raises: + SnapshotNotFoundError: If no row matches. + """ + filename = os.path.basename(filename) + with Session(self._db_engine.engine) as session: + rows = session.scalars( + select(BedSnapshot) + .where(BedSnapshot.file_path.like(f"%{filename}")) + .order_by( + BedSnapshot.creation_date.desc(), BedSnapshot.id.desc() + ) + ).all() + for row in rows: + if os.path.basename(row.file_path) == filename: + return self._to_result(row) + raise SnapshotNotFoundError(f"Snapshot '{filename}' not found.") + + def delete_by_checksum(self, checksum: str, remove_s3: bool = True) -> None: + """ + Delete snapshot index rows by their checksum. + + Deletes every ``bed_snapshots`` row whose ``checksum`` matches (a checksum + identifies one file's content) and optionally removes the underlying S3 + objects. + + Args: + checksum: SHA256 checksum of the snapshot file. + remove_s3: Also delete the underlying S3 object(s). + + Returns: + None. + + Raises: + SnapshotNotFoundError: If no row matches the checksum. + """ + with Session(self._db_engine.engine) as session: + rows = session.scalars( + select(BedSnapshot).where(BedSnapshot.checksum == checksum) + ).all() + if not rows: + raise SnapshotNotFoundError( + f"Snapshot with checksum '{checksum}' not found." + ) + file_paths = {row.file_path for row in rows} + for row in rows: + session.delete(row) + session.commit() + + if remove_s3: + for file_path in file_paths: + self.config.delete_s3(file_path) + + def get(self, id: int) -> BedSnapshotResult: + """ + Get a single snapshot index row by id. + + Args: + id: Primary key of the snapshot row. + + Returns: + The snapshot row. + + Raises: + SnapshotNotFoundError: If no row with this id exists. + """ + with Session(self._db_engine.engine) as session: + row = session.scalar(select(BedSnapshot).where(BedSnapshot.id == id)) + if row is None: + raise SnapshotNotFoundError(f"Snapshot with id '{id}' not found.") + return self._to_result(row) + + @staticmethod + def _to_result(row: BedSnapshot) -> BedSnapshotResult: + return BedSnapshotResult( + file_path=row.file_path, + file_type=row.file_type, + creation_date=row.creation_date, + record_count=row.record_count, + file_size=row.file_size, + checksum=row.checksum, + schema_version=row.schema_version, + ) diff --git a/docs/changelog.md b/docs/changelog.md index a5fe3fc..33a0af1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,21 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +### [0.14.13] - 2026-07-13 +### Fixed: +- Cache `get_stats()` with a TTL to avoid running uncached COUNT queries on the bed table on every request to hot API paths (stats, neighbours, list, search) +- Eliminated an N+1 query in `get_neighbours()` by fetching all neighbour metadata in a single batched query (with annotations eager-loaded) instead of one query per neighbour; stale Qdrant points are now skipped rather than raising +- Eliminated an N+1 in `BedAgentBedSet.get_ids_list()`: it was refetching each bedset by id and lazy-loading its full bedfile membership just to build the list page. Now builds results directly from the paginated query; `bed_ids` is left unpopulated on list results (use `get(identifier)` for a single bedset's member ids) +- `get_detailed_stats()` no longer reuses a `Session` after its `with` block has closed (was forcing 3 extra connection checkouts for `_stats_comments`/`_stats_geo_status`/`_get_geo_stats`); all queries now share one session/transaction +- Replaced the `bed_files_info()` call inside `get_detailed_stats()` with a targeted 3-column query, avoiding a full-table `FileInfo` Pydantic construction (with per-row try/except) for every bed record just to extract `number_of_regions`/`mean_region_width`/`file_size` for histogram binning +- `BedAgentBedFile.get_ids_list()` (backs `/bed/list`) had no `order_by()` on its paginated query, so row order across pages was undefined -- rows could be duplicated or skipped between requests. Now orders by `Bed.id`. +- `get_detailed_stats()` crashed with a pydantic `ValidationError` whenever `bed_compliance`, `data_format`, `genome_alias`, `species_name`, `assay`, or `cell_line` had NULL rows: the `GROUP BY` queries included the NULL group, producing a `None` dict key, which `FileStats`'s `dict[str, int]` fields reject. All six queries now filter out NULLs before grouping. + + +### Added: +- Added a denormalized `bedfile_count` column to `bedsets`, exposed as `BedSetMetadata.bedfile_count`. Set once at bedset creation time (membership is write-once; `add_bedfile`/`delete_bedfile` are unimplemented), so reads never need to touch `bedfile_bedset_relation` to know a bedset's size. Requires a DB migration -- see `scripts/migrations/2026_07_31_add_bedset_bedfile_count.sql` + + ### [0.14.12] - 2026-04-22 ### Changed: - Updated yacman version to 2.0.0 diff --git a/pyproject.toml b/pyproject.toml index fba05bd..ac1e468 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bbconf" -version = "0.14.12" +version = "0.14.13" description = "Configuration and data management tool for BEDbase" readme = "README.md" license = "BSD-2-Clause" @@ -37,6 +37,7 @@ dependencies = [ "umap-learn >= 0.5.8", "qdrant_client >= 1.16.1", "setuptools < 70.0.0", + "cachetools >= 4.2.4", ] [project.urls] diff --git a/tests/test_bedfile.py b/tests/test_bedfile.py index c668757..daf5f4e 100644 --- a/tests/test_bedfile.py +++ b/tests/test_bedfile.py @@ -8,7 +8,7 @@ from bbconf.exceptions import BedFIleExistsError, BEDFileNotFoundError from .conftest import SERVICE_UNAVAILABLE, get_bbagent -from .utils import BED_TEST_ID, ContextManagerDBTesting +from .utils import BED_TEST_ID, BEDSET_TEST_ID, ContextManagerDBTesting @pytest.mark.skipif(SERVICE_UNAVAILABLE, reason="Database is not available") @@ -69,6 +69,16 @@ def test_get_all(self, bbagent_obj, mocked_phc): assert return_result.plots.chrombins is not None assert return_result.license_id == DEFAULT_LICENSE + def test_get_all_bedsets_bedfile_count(self, bbagent_obj, mocked_phc): + with ContextManagerDBTesting( + config=bbagent_obj.config, add_data=True, bedset=True + ): + return_result = bbagent_obj.bed.get(BED_TEST_ID, full=True) + + assert len(return_result.bedsets) == 1 + assert return_result.bedsets[0].id == BEDSET_TEST_ID + assert return_result.bedsets[0].bedfile_count == 1 + def test_get_all_not_found(self, bbagent_obj): with ContextManagerDBTesting(config=bbagent_obj.config, add_data=True): return_result = bbagent_obj.bed.get(BED_TEST_ID, full=False) diff --git a/tests/test_bedset.py b/tests/test_bedset.py index ecc8948..71bde67 100644 --- a/tests/test_bedset.py +++ b/tests/test_bedset.py @@ -61,6 +61,7 @@ def test_crate_bedset_all(self, bbagent_obj, mocker): assert result is not None assert result.name == "test_name" assert len([k for k in result.files]) == 1 + assert result.bedfile_count == 1 def test_get_metadata_full(self, bbagent_obj): with ContextManagerDBTesting( @@ -73,6 +74,7 @@ def test_get_metadata_full(self, bbagent_obj): assert result.statistics.sd is not None assert result.statistics.mean is not None assert result.plots is not None + assert result.bedfile_count == 1 def test_get_metadata_not_full(self, bbagent_obj): with ContextManagerDBTesting( @@ -84,6 +86,7 @@ def test_get_metadata_not_full(self, bbagent_obj): assert result.md5sum == "bbad0000000000000000000000000000" assert result.statistics is None assert result.plots is None + assert result.bedfile_count == 1 def test_get_not_found(self, bbagent_obj): with ContextManagerDBTesting( @@ -128,6 +131,10 @@ def test_get_bedset_list(self, bbagent_obj): assert result.offset == 0 assert len(result.results) == 1 assert result.results[0].id == BEDSET_TEST_ID + # bed_ids is intentionally left unpopulated in list results to + # avoid lazy-loading full bedfile membership for every row + assert result.results[0].bed_ids is None + assert result.results[0].bedfile_count == 1 def test_get_bedset_list_offset(self, bbagent_obj): with ContextManagerDBTesting( diff --git a/tests/test_common.py b/tests/test_common.py index 76d6a7e..6baf756 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -20,6 +20,16 @@ def test_get_stats(bbagent_obj): assert return_result.genomes_number == 1 +@pytest.mark.skipif(SERVICE_UNAVAILABLE, reason="Database is not available") +def test_get_detailed_stats(bbagent_obj): + with ContextManagerDBTesting(config=bbagent_obj.config, add_data=True, bedset=True): + return_result = bbagent_obj.get_detailed_stats() + + assert return_result + assert return_result.number_of_regions.mean == 1 + assert return_result.mean_region_width.mean == 3 + + @pytest.mark.skipif(SERVICE_UNAVAILABLE, reason="Database is not available") def test_get_licenses(bbagent_obj): return_result = bbagent_obj.list_of_licenses diff --git a/tests/utils.py b/tests/utils.py index fd586ef..2714488 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -138,6 +138,7 @@ def _add_bedset_data(self): bedset_standard_deviation=stats, md5sum="bbad0000000000000000000000000000", processed=False, + bedfile_count=1, ) new_bed_bedset = BedFileBedSetRelation( bedfile_id=BED_TEST_ID,