From d211bce50b27f155e771236ef3d53baf52cc5055 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 7 Jul 2026 17:54:41 +0300 Subject: [PATCH 01/32] Add TemporalSpan model --- .../src/lightly_studio/api/db_tables.py | 3 + ...83435200_c3d4e5f6a7b8_add_temporal_span.py | 36 ++++ .../models/annotation/annotation_base.py | 21 ++ .../lightly_studio/models/temporal_span.py | 41 ++++ .../annotation_resolver/annotation_helper.py | 15 ++ .../annotation_resolver/create_many.py | 40 ++++ .../annotation_resolver/delete_annotation.py | 4 + .../annotation_resolver/delete_annotations.py | 2 + ..._by_collection_id_and_parent_sample_ids.py | 4 +- .../resolvers/dataset_resolver/deep_copy.py | 2 + .../dataset_resolver/delete_dataset.py | 12 ++ .../dataset_resolver/table_coverage_utils.py | 2 +- .../test_create_many_temporal_span.py | 193 ++++++++++++++++++ 13 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 lightly_studio/src/lightly_studio/migrations/versions/1783435200_c3d4e5f6a7b8_add_temporal_span.py create mode 100644 lightly_studio/src/lightly_studio/models/temporal_span.py create mode 100644 lightly_studio/tests/resolvers/test_create_many_temporal_span.py diff --git a/lightly_studio/src/lightly_studio/api/db_tables.py b/lightly_studio/src/lightly_studio/api/db_tables.py index 14ab950e95..0bc2f406e9 100644 --- a/lightly_studio/src/lightly_studio/api/db_tables.py +++ b/lightly_studio/src/lightly_studio/api/db_tables.py @@ -49,3 +49,6 @@ from lightly_studio.models.evaluation_sample_metric import ( EvaluationSampleMetricTable, # noqa: F401, required for SQLModel to work properly ) +from lightly_studio.models.temporal_span import ( + TemporalSpanTable, # noqa: F401, required for SQLModel to work properly +) diff --git a/lightly_studio/src/lightly_studio/migrations/versions/1783435200_c3d4e5f6a7b8_add_temporal_span.py b/lightly_studio/src/lightly_studio/migrations/versions/1783435200_c3d4e5f6a7b8_add_temporal_span.py new file mode 100644 index 0000000000..6c43e0852d --- /dev/null +++ b/lightly_studio/src/lightly_studio/migrations/versions/1783435200_c3d4e5f6a7b8_add_temporal_span.py @@ -0,0 +1,36 @@ +"""add-temporal-span. + +Revision ID: c3d4e5f6a7b8 +Revises: a1b2c3d4e5f6 +Create Date: 2026-07-07 10:00:00.000000 + +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c3d4e5f6a7b8" +down_revision: Union[str, Sequence[str], None] = "a1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "temporal_span", + sa.Column("sample_id", sa.Uuid(), nullable=False), + sa.Column("start_time_s", sa.Float(), nullable=False), + sa.Column("end_time_s", sa.Float(), nullable=False), + sa.ForeignKeyConstraint(["sample_id"], ["sample.sample_id"]), + sa.PrimaryKeyConstraint("sample_id"), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table("temporal_span") diff --git a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py index dabf066896..e9568f292b 100644 --- a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py +++ b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py @@ -22,6 +22,7 @@ ) from lightly_studio.models.collection import SampleType from lightly_studio.models.sample import SampleTable +from lightly_studio.models.temporal_span import TemporalSpanTable, TemporalSpanView from lightly_studio.models.video import VideoFrameTable if TYPE_CHECKING: @@ -94,6 +95,15 @@ class AnnotationBaseTable(SQLModel, table=True): sa_relationship_kwargs={"lazy": "select"}, ) + # Optional temporal bounds for this annotation's sample. + temporal_span_details: Mapped[Optional["TemporalSpanTable"]] = Relationship( + sa_relationship_kwargs={ + "primaryjoin": "AnnotationBaseTable.sample_id == foreign(TemporalSpanTable.sample_id)", + "lazy": "select", + "uselist": False, + }, + ) + # The track this annotation belongs to, if any. object_track: Mapped[Optional["ObjectTrackTable"]] = Relationship( sa_relationship_kwargs={"lazy": "joined"}, @@ -126,6 +136,10 @@ class AnnotationCreate(ABC, SQLModel): # Optional properties for segmentation. segmentation_mask: Optional[list[int]] = None + # Optional properties for temporal annotations. + start_time_s: Optional[float] = None + end_time_s: Optional[float] = None + class AnnotationView(BaseModel): """Response model for bounding box annotation.""" @@ -153,6 +167,7 @@ class AnnotationViewTag(SQLModel): object_detection_details: Optional[ObjectDetectionAnnotationView] = None segmentation_details: Optional[SegmentationAnnotationView] = None + temporal_span_details: Optional[TemporalSpanView] = None object_track_id: Optional[UUID] = None object_track_number: Optional[int] = None @@ -172,6 +187,12 @@ def from_annotation_table(cls, annotation: "AnnotationBaseTable") -> "Annotation object_track_number=annotation.object_track.object_track_number if annotation.object_track else None, + temporal_span_details=TemporalSpanView( + start_time_s=annotation.temporal_span_details.start_time_s, + end_time_s=annotation.temporal_span_details.end_time_s, + ) + if annotation.temporal_span_details + else None, annotation_label=cls.AnnotationLabel( annotation_label_name=annotation.annotation_label.annotation_label_name ), diff --git a/lightly_studio/src/lightly_studio/models/temporal_span.py b/lightly_studio/src/lightly_studio/models/temporal_span.py new file mode 100644 index 0000000000..04d0e0d8d7 --- /dev/null +++ b/lightly_studio/src/lightly_studio/models/temporal_span.py @@ -0,0 +1,41 @@ +"""Temporal span model. + +A temporal span stores start and end times in seconds for sample-backed +entities, such as an classification annotation or caption. +""" + +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy.orm import Mapped +from sqlmodel import Field, Relationship, SQLModel + +if TYPE_CHECKING: + from lightly_studio.models.sample import SampleTable +else: + SampleTable = object + + +class TemporalSpanTable(SQLModel, table=True): + """Database table model for temporal spans.""" + + __tablename__ = "temporal_span" + + sample_id: UUID = Field(foreign_key="sample.sample_id", primary_key=True) + + start_time_s: float + end_time_s: float + + sample: Mapped["SampleTable"] = Relationship( + sa_relationship_kwargs={ + "lazy": "select", + "foreign_keys": "[TemporalSpanTable.sample_id]", + }, + ) + + +class TemporalSpanView(SQLModel): + """API response model for temporal spans.""" + + start_time_s: float + end_time_s: float diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/annotation_helper.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/annotation_helper.py index ff55cff7b0..1f48b507da 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/annotation_helper.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/annotation_helper.py @@ -20,6 +20,7 @@ ) from lightly_studio.models.annotation.object_detection import ObjectDetectionAnnotationTable from lightly_studio.models.annotation.segmentation import SegmentationAnnotationTable +from lightly_studio.models.temporal_span import TemporalSpanTable from lightly_studio.resolvers import annotation_resolver @@ -70,6 +71,17 @@ def update_annotation_object( else None ) + temporal_span_row = annotation_copy.temporal_span_details + temporal_span = ( + TemporalSpanTable( + sample_id=annotation_copy.sample_id, + start_time_s=temporal_span_row.start_time_s, + end_time_s=temporal_span_row.end_time_s, + ) + if temporal_span_row + else None + ) + annotation_resolver.delete_annotation(session, annotation.sample_id, delete_sample=False) new_annotation = AnnotationBaseTable( @@ -89,6 +101,9 @@ def update_annotation_object( if object_detection: session.add(object_detection) + if temporal_span: + session.add(temporal_span) + session.commit() session.flush() diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py index 4de499ca97..96619286c4 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py @@ -19,6 +19,7 @@ ) from lightly_studio.models.collection import SampleType from lightly_studio.models.sample import SampleCreate +from lightly_studio.models.temporal_span import TemporalSpanTable from lightly_studio.resolvers import ( annotation_collection_coverage_resolver, collection_resolver, @@ -55,6 +56,7 @@ def create_many( base_annotations = [] object_detection_annotations = [] segmentation_annotations = [] + temporal_spans = [] annotation_collection_id = collection_resolver.get_or_create_child_collection( session=session, collection_id=parent_collection_id, @@ -80,6 +82,7 @@ def create_many( # Set other relationship details to None db_base_annotation.segmentation_details = None db_base_annotation.object_detection_details = None + db_base_annotation.temporal_span_details = None base_annotations.append(db_base_annotation) @@ -116,9 +119,22 @@ def create_many( ) segmentation_annotations.append(db_segmentation_mask) + start_time_s, end_time_s = _validate_optional_temporal_span( + annotation=annotation_create, annotation_type=annotation_type + ) + if start_time_s is not None and end_time_s is not None: + temporal_spans.append( + TemporalSpanTable( + sample_id=base_annotations[i].sample_id, + start_time_s=start_time_s, + end_time_s=end_time_s, + ) + ) + # Bulk save object detection annotations session.bulk_save_objects(object_detection_annotations) session.bulk_save_objects(segmentation_annotations) + session.bulk_save_objects(temporal_spans) # Bulk add annotation collection coverage entries. annotation_collection_coverage_resolver.add_many( @@ -142,3 +158,27 @@ def _validate_bbox(annotation: AnnotationCreate, kind: str) -> tuple[int, int, i raise ValueError(f"Missing height property for {kind}.") return (annotation.x, annotation.y, annotation.width, annotation.height) + + +def _validate_optional_temporal_span( + annotation: AnnotationCreate, annotation_type: AnnotationType +) -> tuple[float | None, float | None]: + start_time_s = annotation.start_time_s + end_time_s = annotation.end_time_s + if start_time_s is None and end_time_s is None: + return (None, None) + + if annotation_type != AnnotationType.CLASSIFICATION: + raise ValueError( + "start_time_s and end_time_s are only supported for CLASSIFICATION annotations." + ) + + kind = annotation_type.value + if start_time_s is None or end_time_s is None: + raise ValueError(f"Missing start_time_s or end_time_s properties for {kind}.") + if start_time_s < 0: + raise ValueError(f"start_time_s must be non-negative for {kind}.") + if start_time_s >= end_time_s: + raise ValueError(f"start_time_s must be less than end_time_s for {kind}.") + + return (start_time_s, end_time_s) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotation.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotation.py index 367bb05e23..ccc9bb0221 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotation.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotation.py @@ -17,6 +17,7 @@ from lightly_studio.models.evaluation_annotation_metric import EvaluationAnnotationMetricTable from lightly_studio.models.evaluation_sample_metric import EvaluationSampleMetricTable from lightly_studio.models.sample import SampleTable, SampleTagLinkTable +from lightly_studio.models.temporal_span import TemporalSpanTable from lightly_studio.resolvers import annotation_resolver from lightly_studio.utils import batching @@ -62,6 +63,9 @@ def delete_annotation( col(SegmentationAnnotationTable.sample_id) == annotation.sample_id ) ) + session.exec( + delete(TemporalSpanTable).where(col(TemporalSpanTable.sample_id) == annotation.sample_id) + ) session.commit() # Delete the annotation using explicit DELETE to avoid relationship cascade issues diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotations.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotations.py index 90912ca1a9..29f13916c8 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotations.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/delete_annotations.py @@ -42,6 +42,8 @@ def delete_annotations( session.delete(annotation.object_detection_details) if annotation.segmentation_details: session.delete(annotation.segmentation_details) + if annotation.temporal_span_details: + session.delete(annotation.temporal_span_details) session.commit() # Now delete the annotations themselves diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py index b9bca7227b..8796d371ce 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py @@ -25,7 +25,7 @@ def get_all_by_collection_id_and_parent_sample_ids( without triggering N+1 queries: - OBJECT_DETECTION: object_detection_details - SEGMENTATION_MASK: segmentation_details - - CLASSIFICATION: no extra load (label_id is on AnnotationBaseTable itself) + - CLASSIFICATION: temporal_span_details (when present) Args: session: Database session. @@ -54,5 +54,7 @@ def get_all_by_collection_id_and_parent_sample_ids( statement = statement.options(joinedload(AnnotationBaseTable.object_detection_details)) elif annotation_type == AnnotationType.SEGMENTATION_MASK: statement = statement.options(joinedload(AnnotationBaseTable.segmentation_details)) + elif annotation_type == AnnotationType.CLASSIFICATION: + statement = statement.options(joinedload(AnnotationBaseTable.temporal_span_details)) return list(session.exec(statement).all()) diff --git a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/deep_copy.py b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/deep_copy.py index af315673cd..25a1a6e117 100644 --- a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/deep_copy.py +++ b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/deep_copy.py @@ -56,6 +56,7 @@ from lightly_studio.models.sample import SampleTable, SampleTagLinkTable from lightly_studio.models.sample_embedding import SampleEmbeddingTable from lightly_studio.models.tag import TagTable +from lightly_studio.models.temporal_span import TemporalSpanTable from lightly_studio.models.video import VideoFrameTable, VideoTable from lightly_studio.resolvers import dataset_resolver from lightly_studio.resolvers.dataset_resolver import table_coverage_utils @@ -128,6 +129,7 @@ def deep_copy( _copy_annotations(session=session, now=now) _copy_annotation_details(session=session, detail_table=ObjectDetectionAnnotationTable) _copy_annotation_details(session=session, detail_table=SegmentationAnnotationTable) + _copy_annotation_details(session=session, detail_table=TemporalSpanTable) _copy_sample_embeddings(session=session) _copy_metadata(session=session, now=now) diff --git a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/delete_dataset.py b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/delete_dataset.py index c020f1762d..00f074e657 100644 --- a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/delete_dataset.py +++ b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/delete_dataset.py @@ -40,6 +40,7 @@ from lightly_studio.models.sample import SampleTable, SampleTagLinkTable from lightly_studio.models.sample_embedding import SampleEmbeddingTable from lightly_studio.models.tag import TagTable +from lightly_studio.models.temporal_span import TemporalSpanTable from lightly_studio.models.video import VideoFrameTable, VideoTable from lightly_studio.resolvers import dataset_resolver from lightly_studio.resolvers.dataset_resolver import table_coverage_utils @@ -81,6 +82,7 @@ def delete_dataset( # 1. Tables that reference annotation_base. _delete_object_detection_annotations(session=session, dataset_id=dataset_id) _delete_segmentation_annotations(session=session, dataset_id=dataset_id) + _delete_temporal_spans(session=session, dataset_id=dataset_id) _delete_evaluation_sample_metrics(session=session, dataset_id=dataset_id) _delete_evaluation_annotation_metrics(session=session, dataset_id=dataset_id) @@ -194,6 +196,16 @@ def _delete_segmentation_annotations(session: Session, dataset_id: UUID) -> None ) +def _delete_temporal_spans(session: Session, dataset_id: UUID) -> None: + """Delete temporal spans for the dataset's samples.""" + session.exec( + delete(TemporalSpanTable).where( + col(TemporalSpanTable.sample_id).in_(_sample_ids_subquery(dataset_id)) + ), + execution_options=_DELETE_EXECUTION_OPTIONS, + ) + + def _delete_annotation_base(session: Session, dataset_id: UUID) -> None: """Delete annotation base records for the dataset's samples.""" session.exec( diff --git a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/table_coverage_utils.py b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/table_coverage_utils.py index 20e6f7bbff..a36b5a7123 100644 --- a/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/table_coverage_utils.py +++ b/lightly_studio/src/lightly_studio/resolvers/dataset_resolver/table_coverage_utils.py @@ -8,7 +8,7 @@ from sqlmodel import SQLModel # Tables handled by deep_copy and delete_dataset. -_HANDLED_TABLES_COUNT = 23 +_HANDLED_TABLES_COUNT = 24 # Tables not relevant for collection operations: # - setting (application-level, not collection-specific) diff --git a/lightly_studio/tests/resolvers/test_create_many_temporal_span.py b/lightly_studio/tests/resolvers/test_create_many_temporal_span.py new file mode 100644 index 0000000000..f63322357f --- /dev/null +++ b/lightly_studio/tests/resolvers/test_create_many_temporal_span.py @@ -0,0 +1,193 @@ +"""Tests for creating annotations with optional temporal spans.""" + +from __future__ import annotations + +import pytest +from sqlmodel import Session + +from lightly_studio.models.annotation.annotation_base import ( + AnnotationCreate, + AnnotationType, + AnnotationView, +) +from lightly_studio.models.temporal_span import TemporalSpanTable +from lightly_studio.resolvers import annotation_resolver +from tests.helpers_resolvers import ( + create_annotation_label, + create_collection, + create_image, +) + + +def test_create_many__classification_with_temporal_span(db_session: Session) -> None: + """Creating a classification annotation with times stores a temporal span row.""" + collection = create_collection(session=db_session) + image = create_image(session=db_session, collection_id=collection.collection_id) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="action", + ) + + annotation_ids = annotation_resolver.create_many( + session=db_session, + parent_collection_id=collection.collection_id, + annotations=[ + AnnotationCreate( + parent_sample_id=image.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + start_time_s=1.5, + end_time_s=4.0, + ) + ], + ) + + annotations = annotation_resolver.get_all_by_parent_sample_ids( + session=db_session, + parent_sample_ids=[image.sample_id], + ) + assert len(annotations) == 1 + annotation = annotations[0] + assert annotation.sample_id == annotation_ids[0] + assert annotation.annotation_type == AnnotationType.CLASSIFICATION + assert annotation.temporal_span_details is not None + assert annotation.temporal_span_details.start_time_s == 1.5 + assert annotation.temporal_span_details.end_time_s == 4.0 + + view = AnnotationView.from_annotation_table(annotation) + assert view.temporal_span_details is not None + assert view.temporal_span_details.start_time_s == 1.5 + assert view.temporal_span_details.end_time_s == 4.0 + + +def test_create_many__classification_without_temporal_span(db_session: Session) -> None: + """Classification annotations without times do not create a temporal span row.""" + collection = create_collection(session=db_session) + image = create_image(session=db_session, collection_id=collection.collection_id) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="action", + ) + + annotation_resolver.create_many( + session=db_session, + parent_collection_id=collection.collection_id, + annotations=[ + AnnotationCreate( + parent_sample_id=image.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ) + ], + ) + + annotations = annotation_resolver.get_all_by_parent_sample_ids( + session=db_session, + parent_sample_ids=[image.sample_id], + ) + assert len(annotations) == 1 + assert annotations[0].temporal_span_details is None + + +def test_create_many__temporal_span_rejected_for_non_classification(db_session: Session) -> None: + """Temporal spans are only allowed on CLASSIFICATION annotations.""" + collection = create_collection(session=db_session) + image = create_image(session=db_session, collection_id=collection.collection_id) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="object", + ) + + with pytest.raises(ValueError, match="only supported for CLASSIFICATION"): + annotation_resolver.create_many( + session=db_session, + parent_collection_id=collection.collection_id, + annotations=[ + AnnotationCreate( + parent_sample_id=image.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.OBJECT_DETECTION, + x=0, + y=0, + width=10, + height=10, + start_time_s=0.0, + end_time_s=1.0, + ) + ], + ) + + +@pytest.mark.parametrize( + ("start_time_s", "end_time_s", "error_match"), + [ + (1.0, None, "Missing start_time_s or end_time_s"), + (None, 2.0, "Missing start_time_s or end_time_s"), + (-1.0, 2.0, "start_time_s must be non-negative"), + (3.0, 2.0, "start_time_s must be less than end_time_s"), + ], +) +def test_create_many__invalid_temporal_span( + db_session: Session, + start_time_s: float | None, + end_time_s: float | None, + error_match: str, +) -> None: + """Invalid temporal span input is rejected.""" + collection = create_collection(session=db_session) + image = create_image(session=db_session, collection_id=collection.collection_id) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="action", + ) + + with pytest.raises(ValueError, match=error_match): + annotation_resolver.create_many( + session=db_session, + parent_collection_id=collection.collection_id, + annotations=[ + AnnotationCreate( + parent_sample_id=image.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + start_time_s=start_time_s, + end_time_s=end_time_s, + ) + ], + ) + + +def test_delete_annotation__removes_temporal_span(db_session: Session) -> None: + """Deleting an annotation also deletes its temporal span row.""" + collection = create_collection(session=db_session) + image = create_image(session=db_session, collection_id=collection.collection_id) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="action", + ) + + annotation_ids = annotation_resolver.create_many( + session=db_session, + parent_collection_id=collection.collection_id, + annotations=[ + AnnotationCreate( + parent_sample_id=image.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + start_time_s=0.0, + end_time_s=1.0, + ) + ], + ) + annotation_id = annotation_ids[0] + + annotation_resolver.delete_annotation(session=db_session, annotation_id=annotation_id) + + assert db_session.get(TemporalSpanTable, annotation_id) is None, ( + "Temporal span row should be deleted with the annotation." + ) From 0c66a3811119369792ff1c5507bce55a806ec352 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 7 Jul 2026 18:37:40 +0300 Subject: [PATCH 02/32] update --- .../test_create_many_temporal_span.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/lightly_studio/tests/resolvers/test_create_many_temporal_span.py b/lightly_studio/tests/resolvers/test_create_many_temporal_span.py index f63322357f..8ce8a6733b 100644 --- a/lightly_studio/tests/resolvers/test_create_many_temporal_span.py +++ b/lightly_studio/tests/resolvers/test_create_many_temporal_span.py @@ -61,36 +61,6 @@ def test_create_many__classification_with_temporal_span(db_session: Session) -> assert view.temporal_span_details.end_time_s == 4.0 -def test_create_many__classification_without_temporal_span(db_session: Session) -> None: - """Classification annotations without times do not create a temporal span row.""" - collection = create_collection(session=db_session) - image = create_image(session=db_session, collection_id=collection.collection_id) - label = create_annotation_label( - session=db_session, - root_collection_id=collection.collection_id, - label_name="action", - ) - - annotation_resolver.create_many( - session=db_session, - parent_collection_id=collection.collection_id, - annotations=[ - AnnotationCreate( - parent_sample_id=image.sample_id, - annotation_label_id=label.annotation_label_id, - annotation_type=AnnotationType.CLASSIFICATION, - ) - ], - ) - - annotations = annotation_resolver.get_all_by_parent_sample_ids( - session=db_session, - parent_sample_ids=[image.sample_id], - ) - assert len(annotations) == 1 - assert annotations[0].temporal_span_details is None - - def test_create_many__temporal_span_rejected_for_non_classification(db_session: Session) -> None: """Temporal spans are only allowed on CLASSIFICATION annotations.""" collection = create_collection(session=db_session) From 3e04899d8f10b739aaba462b5e3b88a7248fcbaf Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 7 Jul 2026 19:07:08 +0300 Subject: [PATCH 03/32] fix --- .../resolvers/annotation_resolver/get_all_with_payload.py | 2 ++ .../resolvers/annotation_resolver/get_by_id_with_payload.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py index c12e98aa4c..1488cc7584 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py @@ -150,6 +150,7 @@ def _build_base_query( joinedload(ImageTable.sample).load_only( SampleTable.collection_id, # type: ignore[arg-type] ), + joinedload(AnnotationBaseTable.temporal_span_details), ) ) @@ -168,6 +169,7 @@ def _build_base_query( VideoTable.width, # type: ignore[arg-type] VideoTable.file_path_abs, # type: ignore[arg-type] ), + joinedload(AnnotationBaseTable.temporal_span_details), ) ) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py index db3af8ba24..dc7d066c2a 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py @@ -74,6 +74,7 @@ def _get_image_annotation_by_id( ImageTable.height, # type: ignore[arg-type] ImageTable.width, # type: ignore[arg-type] ), + joinedload(AnnotationBaseTable.temporal_span_details), ) .where(col(AnnotationBaseTable.sample_id) == sample_id) ) @@ -113,6 +114,7 @@ def _get_video_frame_annotation_by_id( VideoTable.width, # type: ignore[arg-type] VideoTable.file_path_abs, # type: ignore[arg-type] ), + joinedload(AnnotationBaseTable.temporal_span_details), ) .where(col(AnnotationBaseTable.sample_id) == sample_id) ) From bd48016eb07d8af3c06e75730d6cba2569aa0f08 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 7 Jul 2026 19:38:33 +0300 Subject: [PATCH 04/32] try eager load --- .../src/lightly_studio/models/annotation/annotation_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py index e9568f292b..01fc9cfcad 100644 --- a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py +++ b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py @@ -96,10 +96,13 @@ class AnnotationBaseTable(SQLModel, table=True): ) # Optional temporal bounds for this annotation's sample. + # Eager (selectin) so temporal spans are batch-loaded for all annotations in one extra + # query, avoiding an N+1 per annotation when serializing lists (grid, details). selectin + # (vs joined) issues a separate query and does not alter the annotation query's row order. temporal_span_details: Mapped[Optional["TemporalSpanTable"]] = Relationship( sa_relationship_kwargs={ "primaryjoin": "AnnotationBaseTable.sample_id == foreign(TemporalSpanTable.sample_id)", - "lazy": "select", + "lazy": "selectin", "uselist": False, }, ) From f668678dae9726ad7a7ea77580728f75b20b651c Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 7 Jul 2026 22:35:05 +0300 Subject: [PATCH 05/32] fix warning --- .../src/lightly_studio/models/annotation/annotation_base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py index 01fc9cfcad..1bde5a5d68 100644 --- a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py +++ b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py @@ -95,15 +95,14 @@ class AnnotationBaseTable(SQLModel, table=True): sa_relationship_kwargs={"lazy": "select"}, ) + # Optional temporal bounds for this annotation's sample. - # Eager (selectin) so temporal spans are batch-loaded for all annotations in one extra - # query, avoiding an N+1 per annotation when serializing lists (grid, details). selectin - # (vs joined) issues a separate query and does not alter the annotation query's row order. temporal_span_details: Mapped[Optional["TemporalSpanTable"]] = Relationship( sa_relationship_kwargs={ "primaryjoin": "AnnotationBaseTable.sample_id == foreign(TemporalSpanTable.sample_id)", "lazy": "selectin", "uselist": False, + "viewonly": True, }, ) From f3956d184c705d18dd01d27c0a58070ff64f7e63 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 8 Jul 2026 10:33:21 +0300 Subject: [PATCH 06/32] Add video annotation from activitynet --- lightly_studio/pyproject.toml | 2 +- .../core/video/add_annotations.py | 130 ++++++++++++++++++ .../core/video/video_dataset.py | 29 +++- .../resolvers/video_resolver/__init__.py | 2 + .../video_resolver/get_sample_ids_by_stems.py | 46 +++++++ .../video/test_add_annotations_activitynet.py | 83 +++++++++++ lightly_studio/uv.lock | 8 +- 7 files changed, 292 insertions(+), 8 deletions(-) create mode 100644 lightly_studio/src/lightly_studio/core/video/add_annotations.py create mode 100644 lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py create mode 100644 lightly_studio/tests/core/video/test_add_annotations_activitynet.py diff --git a/lightly_studio/pyproject.toml b/lightly_studio/pyproject.toml index 34cf63b7c1..fa0fb229b6 100644 --- a/lightly_studio/pyproject.toml +++ b/lightly_studio/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ # Unreleased and released labelformat versions can be referenced with this syntax: # - "labelformat @ git+https://github.com/lightly-ai/labelformat.git@325a20b" # - "labelformat>=0.1.9" - "labelformat>=0.1.16", + "labelformat @ git+https://github.com/lightly-ai/labelformat.git@adeaf09cd10ec1b6f105da6886c00370fd894103", "tqdm>=4.65.0", "eval-type-backport>=0.2.2", "xxhash>=3.5.0", diff --git a/lightly_studio/src/lightly_studio/core/video/add_annotations.py b/lightly_studio/src/lightly_studio/core/video/add_annotations.py new file mode 100644 index 0000000000..37e01c4b18 --- /dev/null +++ b/lightly_studio/src/lightly_studio/core/video/add_annotations.py @@ -0,0 +1,130 @@ +"""Functions to add annotations to videos already present in a dataset.""" + +from __future__ import annotations + +import logging +from pathlib import Path +from uuid import UUID + +from labelformat.formats import ActivityNetTemporalClassificationInput +from labelformat.model.temporal_classification import TemporalClassificationInput +from sqlmodel import Session +from tqdm import tqdm + +from lightly_studio.models.annotation.annotation_base import AnnotationCreate, AnnotationType +from lightly_studio.models.annotation_label import AnnotationLabelCreate +from lightly_studio.resolvers import ( + annotation_label_resolver, + annotation_resolver, + collection_resolver, + video_resolver, +) +from lightly_studio.type_definitions import PathLike + +logger = logging.getLogger(__name__) + +SAMPLE_BATCH_SIZE = 32 + + +def add_annotations_from_activitynet( + session: Session, + root_collection_id: UUID, + annotations_json: PathLike, + collection_name: str | None = None, + restrict_to_sample_ids: set[UUID] | None = None, +) -> list[str]: + """Add ActivityNet-style event annotations to videos already in a collection. + + Args: + session: The database session. + root_collection_id: The ID of the root video collection. + annotations_json: Path to an ActivityNet-style JSON file. + collection_name: Optional name for the annotation source. If ``None``, a default + name is used. + restrict_to_sample_ids: When provided, only annotate videos whose resolved + sample ID is in this set. + + Returns: + A list of video IDs from the JSON that had no matching video in the collection. + """ + input_labels = _load_activitynet_input(annotations_json=annotations_json) + stem_to_sample_id = video_resolver.get_sample_ids_by_stems( + session=session, + collection_id=root_collection_id, + ) + collection = collection_resolver.get_by_id(session=session, collection_id=root_collection_id) + if collection is None: + raise ValueError(f"Collection {root_collection_id} doesn't exist") + dataset_id = collection.dataset_id + + missing_video_ids: list[str] = [] + annotations_to_create: list[AnnotationCreate] = [] + + for video_label in tqdm( + input_labels.get_labels(), desc="Processing ActivityNet annotations", unit=" videos" + ): + video_sample_id = stem_to_sample_id.get(video_label.video_id) + if video_sample_id is None: + missing_video_ids.append(video_label.video_id) + continue + if restrict_to_sample_ids is not None and video_sample_id not in restrict_to_sample_ids: + continue + + for event in video_label.events: + label_name = event.category.name + label = annotation_label_resolver.get_by_label_name( + session=session, + dataset_id=dataset_id, + label_name=label_name, + ) + if label is None: + label = annotation_label_resolver.create( + session=session, + label=AnnotationLabelCreate( + dataset_id=dataset_id, + annotation_label_name=label_name, + ), + ) + + annotations_to_create.append( + AnnotationCreate( + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + confidence=event.confidence, + parent_sample_id=video_sample_id, + start_time_s=event.start_time_s, + end_time_s=event.end_time_s, + ) + ) + + if len(annotations_to_create) >= SAMPLE_BATCH_SIZE: + annotation_resolver.create_many( + session=session, + parent_collection_id=root_collection_id, + annotations=annotations_to_create, + collection_name=collection_name, + ) + annotations_to_create.clear() + + if annotations_to_create: + annotation_resolver.create_many( + session=session, + parent_collection_id=root_collection_id, + annotations=annotations_to_create, + collection_name=collection_name, + ) + + if missing_video_ids: + logger.warning( + "Skipped %d ActivityNet video IDs with no matching video in the collection.", + len(missing_video_ids), + ) + + return missing_video_ids + + +def _load_activitynet_input(annotations_json: PathLike) -> TemporalClassificationInput: + if isinstance(annotations_json, (str, Path)): + return ActivityNetTemporalClassificationInput(input_file=Path(annotations_json)) + + raise TypeError("annotations_json must be a path to an ActivityNet JSON file.") diff --git a/lightly_studio/src/lightly_studio/core/video/video_dataset.py b/lightly_studio/src/lightly_studio/core/video/video_dataset.py index 5460745edc..7731e5093b 100644 --- a/lightly_studio/src/lightly_studio/core/video/video_dataset.py +++ b/lightly_studio/src/lightly_studio/core/video/video_dataset.py @@ -15,7 +15,7 @@ from lightly_studio.core.dataset import BaseSampleDataset from lightly_studio.core.dataset_query.dataset_query import DatasetQuery -from lightly_studio.core.video import add_videos +from lightly_studio.core.video import add_annotations, add_videos from lightly_studio.core.video.add_videos import VIDEO_EXTENSIONS from lightly_studio.core.video.video_frame_dataset import VideoFrameDataset from lightly_studio.core.video.video_sample import VideoSample @@ -228,6 +228,33 @@ def add_videos_from_youtube_vis( # noqa: PLR0913 sample_ids=created_sample_ids, ) + def add_annotations_from_activitynet( + self, + annotations_json: PathLike, + annotation_source: str | None = None, + ) -> list[str]: + """Attach ActivityNet-style temporal annotations to videos already in the dataset. + + Videos are matched by ``file_name`` stem or ``file_path_abs`` stem against the + ActivityNet video identifier (e.g. ``v_o6wtvlVs3E8`` for ``v_o6wtvlVs3E8.mp4``). + + Args: + annotations_json: Path to an ActivityNet-style JSON file with a ``database`` + or ``results`` key. + annotation_source: Name of the annotation source to add the annotations to. + Reusing the same source name appends to that source. If ``None``, a + default source is used. + + Returns: + Video IDs from the JSON that had no matching video in the dataset. + """ + return add_annotations.add_annotations_from_activitynet( + session=self.session, + root_collection_id=self.collection_id, + annotations_json=annotations_json, + collection_name=annotation_source, + ) + def _generate_embeddings_video( session: Session, diff --git a/lightly_studio/src/lightly_studio/resolvers/video_resolver/__init__.py b/lightly_studio/src/lightly_studio/resolvers/video_resolver/__init__.py index 4304b93497..3fc89a9daa 100644 --- a/lightly_studio/src/lightly_studio/resolvers/video_resolver/__init__.py +++ b/lightly_studio/src/lightly_studio/resolvers/video_resolver/__init__.py @@ -15,6 +15,7 @@ build_sample_ids_query, get_sample_ids, ) +from lightly_studio.resolvers.video_resolver.get_sample_ids_by_stems import get_sample_ids_by_stems from lightly_studio.resolvers.video_resolver.get_table_fields_bounds import ( get_table_fields_bounds, ) @@ -30,6 +31,7 @@ "get_by_id", "get_many_by_id", "get_sample_ids", + "get_sample_ids_by_stems", "get_table_fields_bounds", "get_view_by_id", ] diff --git a/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py b/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py new file mode 100644 index 0000000000..b91e584638 --- /dev/null +++ b/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py @@ -0,0 +1,46 @@ +"""Implementation of get_sample_ids_by_stems function for videos.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +from sqlmodel import Session, col, select + +from lightly_studio.models.sample import SampleTable +from lightly_studio.models.video import VideoTable + + +def get_sample_ids_by_stems( + session: Session, + collection_id: UUID, +) -> dict[str, UUID]: + """Build a mapping from video identifier stems to sample IDs in a collection. + + Each video is indexed by ``Path(file_name).stem`` and ``Path(file_path_abs).stem``. + If the same stem maps to multiple videos, a ``ValueError`` is raised. + + Args: + session: The database session. + collection_id: The ID of the video collection to scope results to. + + Returns: + A mapping from stem to sample_id. + """ + query = ( + select(VideoTable).join(SampleTable).where(col(SampleTable.collection_id) == collection_id) + ) + videos = session.exec(query).all() + + stem_to_sample_id: dict[str, UUID] = {} + for video in videos: + for stem in {Path(video.file_name).stem, Path(video.file_path_abs).stem}: + existing = stem_to_sample_id.get(stem) + if existing is not None and existing != video.sample_id: + raise ValueError( + f"Duplicate video stem '{stem}' in collection {collection_id}. " + "Video identifiers must be unique." + ) + stem_to_sample_id[stem] = video.sample_id + + return stem_to_sample_id diff --git a/lightly_studio/tests/core/video/test_add_annotations_activitynet.py b/lightly_studio/tests/core/video/test_add_annotations_activitynet.py new file mode 100644 index 0000000000..2d7bdf9e77 --- /dev/null +++ b/lightly_studio/tests/core/video/test_add_annotations_activitynet.py @@ -0,0 +1,83 @@ +"""Tests for ActivityNet annotation import.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from sqlmodel import Session + +from lightly_studio.core.video import add_annotations +from lightly_studio.models.annotation.annotation_base import AnnotationType +from lightly_studio.models.collection import SampleType +from lightly_studio.resolvers import annotation_resolver, video_resolver +from tests.helpers_resolvers import create_collection +from tests.resolvers.video.helpers import VideoStub, create_video + + +def test_add_annotations_from_activitynet__imports_events( + db_session: Session, tmp_path: Path +) -> None: + collection = create_collection(session=db_session, sample_type=SampleType.VIDEO) + video = create_video( + session=db_session, + collection_id=collection.collection_id, + video=VideoStub(path="/data/videos/v_action1.mp4", duration_s=30.0), + ) + + annotations_json = tmp_path / "activitynet.json" + annotations_json.write_text( + json.dumps( + { + "database": { + "v_action1": { + "annotations": [ + {"label": "Running", "segment": [1.0, 5.5]}, + {"label": "Jumping", "segment": [6.0, 12.0], "score": 0.8}, + ] + }, + "v_missing": {"annotations": [{"label": "Missing", "segment": [0.0, 1.0]}]}, + } + } + ), + encoding="utf-8", + ) + + missing = add_annotations.add_annotations_from_activitynet( + session=db_session, + root_collection_id=collection.collection_id, + annotations_json=annotations_json, + collection_name="ground_truth", + ) + + assert missing == ["v_missing"] + + annotations = annotation_resolver.get_all_by_parent_sample_ids( + session=db_session, + parent_sample_ids=[video.sample_id], + ) + assert len(annotations) == 2 + assert all( + annotation.annotation_type == AnnotationType.CLASSIFICATION for annotation in annotations + ) + assert all(annotation.parent_sample_id == video.sample_id for annotation in annotations) + assert { + annotation.temporal_span_details.start_time_s + for annotation in annotations + if annotation.temporal_span_details is not None + } == {1.0, 6.0} + assert { + annotation.temporal_span_details.end_time_s + for annotation in annotations + if annotation.temporal_span_details is not None + } == {5.5, 12.0} + + labels = { + annotation.annotation_label.annotation_label_name + for annotation in annotations + if annotation.annotation_label is not None + } + assert labels == {"Running", "Jumping"} + + loaded_video = video_resolver.get_by_id(session=db_session, sample_id=video.sample_id) + assert loaded_video is not None diff --git a/lightly_studio/uv.lock b/lightly_studio/uv.lock index 4e725804e2..38c0fc0529 100644 --- a/lightly_studio/uv.lock +++ b/lightly_studio/uv.lock @@ -2697,7 +2697,7 @@ wheels = [ [[package]] name = "labelformat" version = "0.1.16" -source = { registry = "https://pypi.org/simple" } +source = { git = "https://github.com/lightly-ai/labelformat.git?rev=adeaf09cd10ec1b6f105da6886c00370fd894103#adeaf09cd10ec1b6f105da6886c00370fd894103" } dependencies = [ { name = "fsspec", version = "2025.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -2711,10 +2711,6 @@ dependencies = [ { name = "pyyaml" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d4/c6/9e5ecea46d0bc9d52cffccaeacad656f36c1082b315a30cb1b9c735815fe/labelformat-0.1.16.tar.gz", hash = "sha256:eee410ed77c94e37726c71b6092e651f1e831043e8759ff4a7ec76dd7d16357f", size = 40089, upload-time = "2026-06-05T09:42:46.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/67/829de439f50a6fdd0c69b58f23469191a0a78f489e7b52ac6e0201d543cd/labelformat-0.1.16-py3-none-any.whl", hash = "sha256:628c875858b1b3c5b88fac362901ab25488ab58197b310feee789ffdc6463e8d", size = 57447, upload-time = "2026-06-05T09:42:48.077Z" }, -] [[package]] name = "lazy-object-proxy" @@ -2990,7 +2986,7 @@ requires-dist = [ { name = "fsspec", specifier = ">=2023.1.0" }, { name = "gcsfs", marker = "extra == 'cloud-storage'", specifier = ">=2023.1.0" }, { name = "jinja2", specifier = ">=3.1.0" }, - { name = "labelformat", specifier = ">=0.1.16" }, + { name = "labelformat", git = "https://github.com/lightly-ai/labelformat.git?rev=adeaf09cd10ec1b6f105da6886c00370fd894103" }, { name = "lightly-mundig", specifier = "==0.1.13" }, { name = "open-clip-torch", specifier = ">=2.20.0" }, { name = "opencv-python-headless", specifier = ">=4.11.0.86" }, From 07ddf00fd65be20692488839f6671f1faddc0061 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 8 Jul 2026 10:47:38 +0300 Subject: [PATCH 07/32] remove not needed joins --- .../src/lightly_studio/models/annotation/annotation_base.py | 1 - .../get_all_by_collection_id_and_parent_sample_ids.py | 4 +--- .../resolvers/annotation_resolver/get_all_with_payload.py | 2 -- .../resolvers/annotation_resolver/get_by_id_with_payload.py | 2 -- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py index 1bde5a5d68..f9704b44b6 100644 --- a/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py +++ b/lightly_studio/src/lightly_studio/models/annotation/annotation_base.py @@ -95,7 +95,6 @@ class AnnotationBaseTable(SQLModel, table=True): sa_relationship_kwargs={"lazy": "select"}, ) - # Optional temporal bounds for this annotation's sample. temporal_span_details: Mapped[Optional["TemporalSpanTable"]] = Relationship( sa_relationship_kwargs={ diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py index 8796d371ce..b9bca7227b 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_by_collection_id_and_parent_sample_ids.py @@ -25,7 +25,7 @@ def get_all_by_collection_id_and_parent_sample_ids( without triggering N+1 queries: - OBJECT_DETECTION: object_detection_details - SEGMENTATION_MASK: segmentation_details - - CLASSIFICATION: temporal_span_details (when present) + - CLASSIFICATION: no extra load (label_id is on AnnotationBaseTable itself) Args: session: Database session. @@ -54,7 +54,5 @@ def get_all_by_collection_id_and_parent_sample_ids( statement = statement.options(joinedload(AnnotationBaseTable.object_detection_details)) elif annotation_type == AnnotationType.SEGMENTATION_MASK: statement = statement.options(joinedload(AnnotationBaseTable.segmentation_details)) - elif annotation_type == AnnotationType.CLASSIFICATION: - statement = statement.options(joinedload(AnnotationBaseTable.temporal_span_details)) return list(session.exec(statement).all()) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py index 1488cc7584..c12e98aa4c 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_all_with_payload.py @@ -150,7 +150,6 @@ def _build_base_query( joinedload(ImageTable.sample).load_only( SampleTable.collection_id, # type: ignore[arg-type] ), - joinedload(AnnotationBaseTable.temporal_span_details), ) ) @@ -169,7 +168,6 @@ def _build_base_query( VideoTable.width, # type: ignore[arg-type] VideoTable.file_path_abs, # type: ignore[arg-type] ), - joinedload(AnnotationBaseTable.temporal_span_details), ) ) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py index dc7d066c2a..db3af8ba24 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/get_by_id_with_payload.py @@ -74,7 +74,6 @@ def _get_image_annotation_by_id( ImageTable.height, # type: ignore[arg-type] ImageTable.width, # type: ignore[arg-type] ), - joinedload(AnnotationBaseTable.temporal_span_details), ) .where(col(AnnotationBaseTable.sample_id) == sample_id) ) @@ -114,7 +113,6 @@ def _get_video_frame_annotation_by_id( VideoTable.width, # type: ignore[arg-type] VideoTable.file_path_abs, # type: ignore[arg-type] ), - joinedload(AnnotationBaseTable.temporal_span_details), ) .where(col(AnnotationBaseTable.sample_id) == sample_id) ) From ebe86db8079f8ca06407e1eef7edabbf505b0198 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 8 Jul 2026 16:04:47 +0300 Subject: [PATCH 08/32] simplify --- .../core/video/add_annotations.py | 45 ++++--------------- 1 file changed, 8 insertions(+), 37 deletions(-) diff --git a/lightly_studio/src/lightly_studio/core/video/add_annotations.py b/lightly_studio/src/lightly_studio/core/video/add_annotations.py index 37e01c4b18..ada6ec290b 100644 --- a/lightly_studio/src/lightly_studio/core/video/add_annotations.py +++ b/lightly_studio/src/lightly_studio/core/video/add_annotations.py @@ -7,16 +7,13 @@ from uuid import UUID from labelformat.formats import ActivityNetTemporalClassificationInput -from labelformat.model.temporal_classification import TemporalClassificationInput from sqlmodel import Session from tqdm import tqdm +from lightly_studio.core import labelformat_helpers from lightly_studio.models.annotation.annotation_base import AnnotationCreate, AnnotationType -from lightly_studio.models.annotation_label import AnnotationLabelCreate from lightly_studio.resolvers import ( - annotation_label_resolver, annotation_resolver, - collection_resolver, video_resolver, ) from lightly_studio.type_definitions import PathLike @@ -31,7 +28,6 @@ def add_annotations_from_activitynet( root_collection_id: UUID, annotations_json: PathLike, collection_name: str | None = None, - restrict_to_sample_ids: set[UUID] | None = None, ) -> list[str]: """Add ActivityNet-style event annotations to videos already in a collection. @@ -41,21 +37,20 @@ def add_annotations_from_activitynet( annotations_json: Path to an ActivityNet-style JSON file. collection_name: Optional name for the annotation source. If ``None``, a default name is used. - restrict_to_sample_ids: When provided, only annotate videos whose resolved - sample ID is in this set. Returns: A list of video IDs from the JSON that had no matching video in the collection. """ - input_labels = _load_activitynet_input(annotations_json=annotations_json) + input_labels = ActivityNetTemporalClassificationInput(input_file=Path(annotations_json)) stem_to_sample_id = video_resolver.get_sample_ids_by_stems( session=session, collection_id=root_collection_id, ) - collection = collection_resolver.get_by_id(session=session, collection_id=root_collection_id) - if collection is None: - raise ValueError(f"Collection {root_collection_id} doesn't exist") - dataset_id = collection.dataset_id + label_map = labelformat_helpers.create_label_map( + session=session, + root_collection_id=root_collection_id, + input_labels=input_labels, + ) missing_video_ids: list[str] = [] annotations_to_create: list[AnnotationCreate] = [] @@ -67,28 +62,11 @@ def add_annotations_from_activitynet( if video_sample_id is None: missing_video_ids.append(video_label.video_id) continue - if restrict_to_sample_ids is not None and video_sample_id not in restrict_to_sample_ids: - continue for event in video_label.events: - label_name = event.category.name - label = annotation_label_resolver.get_by_label_name( - session=session, - dataset_id=dataset_id, - label_name=label_name, - ) - if label is None: - label = annotation_label_resolver.create( - session=session, - label=AnnotationLabelCreate( - dataset_id=dataset_id, - annotation_label_name=label_name, - ), - ) - annotations_to_create.append( AnnotationCreate( - annotation_label_id=label.annotation_label_id, + annotation_label_id=label_map[event.category.id], annotation_type=AnnotationType.CLASSIFICATION, confidence=event.confidence, parent_sample_id=video_sample_id, @@ -121,10 +99,3 @@ def add_annotations_from_activitynet( ) return missing_video_ids - - -def _load_activitynet_input(annotations_json: PathLike) -> TemporalClassificationInput: - if isinstance(annotations_json, (str, Path)): - return ActivityNetTemporalClassificationInput(input_file=Path(annotations_json)) - - raise TypeError("annotations_json must be a path to an ActivityNet JSON file.") From 47ec651c518219f8c8a40a1444824ff32c1a3b4f Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 15 Jul 2026 09:12:33 +0300 Subject: [PATCH 09/32] fix merge --- .../test_create_many_temporal_span.py | 163 ------------------ 1 file changed, 163 deletions(-) delete mode 100644 lightly_studio/tests/resolvers/test_create_many_temporal_span.py diff --git a/lightly_studio/tests/resolvers/test_create_many_temporal_span.py b/lightly_studio/tests/resolvers/test_create_many_temporal_span.py deleted file mode 100644 index 8ce8a6733b..0000000000 --- a/lightly_studio/tests/resolvers/test_create_many_temporal_span.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Tests for creating annotations with optional temporal spans.""" - -from __future__ import annotations - -import pytest -from sqlmodel import Session - -from lightly_studio.models.annotation.annotation_base import ( - AnnotationCreate, - AnnotationType, - AnnotationView, -) -from lightly_studio.models.temporal_span import TemporalSpanTable -from lightly_studio.resolvers import annotation_resolver -from tests.helpers_resolvers import ( - create_annotation_label, - create_collection, - create_image, -) - - -def test_create_many__classification_with_temporal_span(db_session: Session) -> None: - """Creating a classification annotation with times stores a temporal span row.""" - collection = create_collection(session=db_session) - image = create_image(session=db_session, collection_id=collection.collection_id) - label = create_annotation_label( - session=db_session, - root_collection_id=collection.collection_id, - label_name="action", - ) - - annotation_ids = annotation_resolver.create_many( - session=db_session, - parent_collection_id=collection.collection_id, - annotations=[ - AnnotationCreate( - parent_sample_id=image.sample_id, - annotation_label_id=label.annotation_label_id, - annotation_type=AnnotationType.CLASSIFICATION, - start_time_s=1.5, - end_time_s=4.0, - ) - ], - ) - - annotations = annotation_resolver.get_all_by_parent_sample_ids( - session=db_session, - parent_sample_ids=[image.sample_id], - ) - assert len(annotations) == 1 - annotation = annotations[0] - assert annotation.sample_id == annotation_ids[0] - assert annotation.annotation_type == AnnotationType.CLASSIFICATION - assert annotation.temporal_span_details is not None - assert annotation.temporal_span_details.start_time_s == 1.5 - assert annotation.temporal_span_details.end_time_s == 4.0 - - view = AnnotationView.from_annotation_table(annotation) - assert view.temporal_span_details is not None - assert view.temporal_span_details.start_time_s == 1.5 - assert view.temporal_span_details.end_time_s == 4.0 - - -def test_create_many__temporal_span_rejected_for_non_classification(db_session: Session) -> None: - """Temporal spans are only allowed on CLASSIFICATION annotations.""" - collection = create_collection(session=db_session) - image = create_image(session=db_session, collection_id=collection.collection_id) - label = create_annotation_label( - session=db_session, - root_collection_id=collection.collection_id, - label_name="object", - ) - - with pytest.raises(ValueError, match="only supported for CLASSIFICATION"): - annotation_resolver.create_many( - session=db_session, - parent_collection_id=collection.collection_id, - annotations=[ - AnnotationCreate( - parent_sample_id=image.sample_id, - annotation_label_id=label.annotation_label_id, - annotation_type=AnnotationType.OBJECT_DETECTION, - x=0, - y=0, - width=10, - height=10, - start_time_s=0.0, - end_time_s=1.0, - ) - ], - ) - - -@pytest.mark.parametrize( - ("start_time_s", "end_time_s", "error_match"), - [ - (1.0, None, "Missing start_time_s or end_time_s"), - (None, 2.0, "Missing start_time_s or end_time_s"), - (-1.0, 2.0, "start_time_s must be non-negative"), - (3.0, 2.0, "start_time_s must be less than end_time_s"), - ], -) -def test_create_many__invalid_temporal_span( - db_session: Session, - start_time_s: float | None, - end_time_s: float | None, - error_match: str, -) -> None: - """Invalid temporal span input is rejected.""" - collection = create_collection(session=db_session) - image = create_image(session=db_session, collection_id=collection.collection_id) - label = create_annotation_label( - session=db_session, - root_collection_id=collection.collection_id, - label_name="action", - ) - - with pytest.raises(ValueError, match=error_match): - annotation_resolver.create_many( - session=db_session, - parent_collection_id=collection.collection_id, - annotations=[ - AnnotationCreate( - parent_sample_id=image.sample_id, - annotation_label_id=label.annotation_label_id, - annotation_type=AnnotationType.CLASSIFICATION, - start_time_s=start_time_s, - end_time_s=end_time_s, - ) - ], - ) - - -def test_delete_annotation__removes_temporal_span(db_session: Session) -> None: - """Deleting an annotation also deletes its temporal span row.""" - collection = create_collection(session=db_session) - image = create_image(session=db_session, collection_id=collection.collection_id) - label = create_annotation_label( - session=db_session, - root_collection_id=collection.collection_id, - label_name="action", - ) - - annotation_ids = annotation_resolver.create_many( - session=db_session, - parent_collection_id=collection.collection_id, - annotations=[ - AnnotationCreate( - parent_sample_id=image.sample_id, - annotation_label_id=label.annotation_label_id, - annotation_type=AnnotationType.CLASSIFICATION, - start_time_s=0.0, - end_time_s=1.0, - ) - ], - ) - annotation_id = annotation_ids[0] - - annotation_resolver.delete_annotation(session=db_session, annotation_id=annotation_id) - - assert db_session.get(TemporalSpanTable, annotation_id) is None, ( - "Temporal span row should be deleted with the annotation." - ) From 9f97a845a3c33526d7fea3a6a997093c5dfe36c5 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 15 Jul 2026 09:35:12 +0300 Subject: [PATCH 10/32] add example sctipt --- lightly_studio/.env.example | 2 + .../examples/example_activitynet.py | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 lightly_studio/src/lightly_studio/examples/example_activitynet.py diff --git a/lightly_studio/.env.example b/lightly_studio/.env.example index a3669b15af..24b1322354 100644 --- a/lightly_studio/.env.example +++ b/lightly_studio/.env.example @@ -31,6 +31,8 @@ EXAMPLES_PASCALVOC_MASKS_PATH="dataset_examples/voc2012_10_images/SegmentationCl EXAMPLES_PASCALVOC_PRED_MASKS_PATH="dataset_examples/voc2012_10_images/SegmentationClassPredictions" EXAMPLES_PASCALVOC_CATEGORIES_JSON_PATH="dataset_examples/voc2012_10_images/class_id_to_name.json" EXAMPLES_MIDV_PATH="dataset_examples/midv_2020_10_samples" +EXAMPLES_ACTIVITYNET_VIDEOS_PATH="dataset_examples/activitynet_10_videos/videos" +EXAMPLES_ACTIVITYNET_JSON_PATH="dataset_examples/activitynet_10_videos/activity_net.v1-3.min.json" # Request timing for endpoint functionality. # diff --git a/lightly_studio/src/lightly_studio/examples/example_activitynet.py b/lightly_studio/src/lightly_studio/examples/example_activitynet.py new file mode 100644 index 0000000000..19f7993fbe --- /dev/null +++ b/lightly_studio/src/lightly_studio/examples/example_activitynet.py @@ -0,0 +1,44 @@ +"""Example of importing ActivityNet-style temporal annotations.""" + +from __future__ import annotations + +from environs import Env + +import lightly_studio as ls +from lightly_studio.core.video.video_dataset import VideoDataset +from lightly_studio.database import db_manager + +# Read environment variables +env = Env() +env.read_env() + +# Cleanup an existing database +db_manager.connect(cleanup_existing=True) + +videos_path = env.path( + "EXAMPLES_ACTIVITYNET_VIDEOS_PATH", + "dataset_examples/activitynet_10_videos/videos", +) +annotations_path = env.path( + "EXAMPLES_ACTIVITYNET_JSON_PATH", + "dataset_examples/activitynet_10_videos/activity_net.v1-3.min.json", +) + +dataset = VideoDataset.create() +dataset.add_videos_from_path(path=videos_path, embed=False, embed_frames=False, target_fps=1) +dataset.add_annotations_from_activitynet( + annotations_json=annotations_path, + annotation_source="activitynet", +) + +# Print the loaded annotations per video. +for video in dataset: + print(f"{video.file_name}: {len(video.annotations)} annotation(s)") + for annotation in video.annotations: + span = annotation.annotation_base.temporal_span_details + time_range = f"{span.start_time_s:.2f}s - {span.end_time_s:.2f}s" if span else "no span" + print( + f" - {annotation.class_name} [{time_range}] (confidence: {annotation.confidence})" + ) + +ls.start_gui() From d95200c9e3687bed60a4f19e48ea4e2390f61ac6 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 15 Jul 2026 09:44:21 +0300 Subject: [PATCH 11/32] format --- .../src/lightly_studio/examples/example_activitynet.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lightly_studio/src/lightly_studio/examples/example_activitynet.py b/lightly_studio/src/lightly_studio/examples/example_activitynet.py index 19f7993fbe..675105dbf0 100644 --- a/lightly_studio/src/lightly_studio/examples/example_activitynet.py +++ b/lightly_studio/src/lightly_studio/examples/example_activitynet.py @@ -37,8 +37,6 @@ for annotation in video.annotations: span = annotation.annotation_base.temporal_span_details time_range = f"{span.start_time_s:.2f}s - {span.end_time_s:.2f}s" if span else "no span" - print( - f" - {annotation.class_name} [{time_range}] (confidence: {annotation.confidence})" - ) + print(f" - {annotation.class_name} [{time_range}] (confidence: {annotation.confidence})") ls.start_gui() From 6a243bd231fd431bc018142ffc5204032b7a9d59 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 15 Jul 2026 09:57:25 +0300 Subject: [PATCH 12/32] update --- .../src/lightly_studio/core/video/add_annotations.py | 12 +++++++++--- .../video_resolver/get_sample_ids_by_stems.py | 12 +++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lightly_studio/src/lightly_studio/core/video/add_annotations.py b/lightly_studio/src/lightly_studio/core/video/add_annotations.py index ada6ec290b..e3128dd7ec 100644 --- a/lightly_studio/src/lightly_studio/core/video/add_annotations.py +++ b/lightly_studio/src/lightly_studio/core/video/add_annotations.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -SAMPLE_BATCH_SIZE = 32 +ANNOTATION_BATCH_SIZE = 1024 def add_annotations_from_activitynet( @@ -41,7 +41,13 @@ def add_annotations_from_activitynet( Returns: A list of video IDs from the JSON that had no matching video in the collection. """ - input_labels = ActivityNetTemporalClassificationInput(input_file=Path(annotations_json)) + annotations_json = Path(annotations_json).absolute() + if not annotations_json.is_file() or annotations_json.suffix != ".json": + raise FileNotFoundError( + f"ActivityNet annotations json file not found: '{annotations_json}'" + ) + + input_labels = ActivityNetTemporalClassificationInput(input_file=annotations_json) stem_to_sample_id = video_resolver.get_sample_ids_by_stems( session=session, collection_id=root_collection_id, @@ -75,7 +81,7 @@ def add_annotations_from_activitynet( ) ) - if len(annotations_to_create) >= SAMPLE_BATCH_SIZE: + if len(annotations_to_create) >= ANNOTATION_BATCH_SIZE: annotation_resolver.create_many( session=session, parent_collection_id=root_collection_id, diff --git a/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py b/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py index b91e584638..5a7a4c0d3b 100644 --- a/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py +++ b/lightly_studio/src/lightly_studio/resolvers/video_resolver/get_sample_ids_by_stems.py @@ -28,19 +28,21 @@ def get_sample_ids_by_stems( A mapping from stem to sample_id. """ query = ( - select(VideoTable).join(SampleTable).where(col(SampleTable.collection_id) == collection_id) + select(VideoTable.file_name, VideoTable.file_path_abs, VideoTable.sample_id) + .join(SampleTable) + .where(col(SampleTable.collection_id) == collection_id) ) videos = session.exec(query).all() stem_to_sample_id: dict[str, UUID] = {} - for video in videos: - for stem in {Path(video.file_name).stem, Path(video.file_path_abs).stem}: + for file_name, file_path_abs, sample_id in videos: + for stem in {Path(file_name).stem, Path(file_path_abs).stem}: existing = stem_to_sample_id.get(stem) - if existing is not None and existing != video.sample_id: + if existing is not None and existing != sample_id: raise ValueError( f"Duplicate video stem '{stem}' in collection {collection_id}. " "Video identifiers must be unique." ) - stem_to_sample_id[stem] = video.sample_id + stem_to_sample_id[stem] = sample_id return stem_to_sample_id From d32cbe4b8c10d818d3d9b1e2d49cf927e48f86d4 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 16 Jul 2026 15:57:05 +0300 Subject: [PATCH 13/32] Consider also own annotations for video filter Previously, filtering and counting video annotations only considered annotations attached to a video's frames. Videos can also carry annotations directly (e.g. ActivityNet-style event/classification labels on the whole video). This makes both the video filter and the annotation counter consider these direct video annotations in addition to frame annotations. --- ...t_video_frame_annotations_by_collection.py | 84 +++++++---- .../resolvers/video_resolver/video_filter.py | 18 ++- ...ideo_frame_annotations_by_video_dataset.py | 131 +++++++++++++++++- .../video/video_resolver/test_video_filter.py | 45 +++++- 4 files changed, 247 insertions(+), 31 deletions(-) diff --git a/lightly_studio/src/lightly_studio/resolvers/video_resolver/count_video_frame_annotations_by_collection.py b/lightly_studio/src/lightly_studio/resolvers/video_resolver/count_video_frame_annotations_by_collection.py index 62c917f693..556377b797 100644 --- a/lightly_studio/src/lightly_studio/resolvers/video_resolver/count_video_frame_annotations_by_collection.py +++ b/lightly_studio/src/lightly_studio/resolvers/video_resolver/count_video_frame_annotations_by_collection.py @@ -1,9 +1,10 @@ -"""Count video frame annotations by video collection.""" +"""Count video annotations by video collection.""" from typing import Any, Optional from uuid import UUID from pydantic import BaseModel +from sqlalchemy import union_all from sqlmodel import Session, asc, col, func, select from sqlmodel.sql.expression import Select @@ -31,33 +32,48 @@ def count_video_frame_annotations_by_video_collection( filters: Optional[VideoFilter] = None, annotation_type: Optional[AnnotationType] = None, ) -> list[CountAnnotationsView]: - """Count the annotations by video frames. + """Count annotations attached to videos, including frame and direct video labels. When ``annotation_type`` is provided, both the total and filtered counts are restricted to annotations of that type (e.g. only CLASSIFICATION or only OBJECT_DETECTION). """ + label_video_pairs = _build_label_video_pairs_subquery( + collection_id=collection_id, annotation_type=annotation_type + ) + unfiltered_query = ( - _build_base_query( - collection_id=collection_id, - count_column_name="total", - annotation_type=annotation_type, + select( + label_video_pairs.c.label_id, + func.count(func.distinct(label_video_pairs.c.video_id)).label("total"), ) - .group_by(col(AnnotationBaseTable.annotation_label_id)) + .select_from(label_video_pairs) + .group_by(label_video_pairs.c.label_id) .subquery("unfiltered") ) - filtered_query = _build_base_query( - collection_id=collection_id, - count_column_name="filtered_count", - annotation_type=annotation_type, - ) + filtered_pairs_query: Select[tuple[Any, Any]] = ( + select( + label_video_pairs.c.label_id, + label_video_pairs.c.video_id, + ) + .select_from(label_video_pairs) + .join(VideoTable, col(VideoTable.sample_id) == label_video_pairs.c.video_id) + .join(SampleTable, col(SampleTable.sample_id) == col(VideoTable.sample_id)) + ) if filters is not None: - filtered_query = filters.apply(filtered_query) + filtered_pairs_query = filters.apply(filtered_pairs_query) - filtered_subquery = filtered_query.group_by( - col(AnnotationBaseTable.annotation_label_id) - ).subquery("filtered") + filtered_pairs_subquery = filtered_pairs_query.subquery("filtered_pairs") + filtered_subquery = ( + select( + filtered_pairs_subquery.c.label_id, + func.count(func.distinct(filtered_pairs_subquery.c.video_id)).label("filtered_count"), + ) + .select_from(filtered_pairs_subquery) + .group_by(filtered_pairs_subquery.c.label_id) + .subquery("filtered") + ) final_query: Select[Any] = ( select( @@ -86,15 +102,17 @@ def count_video_frame_annotations_by_video_collection( ] -def _build_base_query( - collection_id: UUID, - count_column_name: str, - annotation_type: Optional[AnnotationType] = None, -) -> Select[tuple[Any, int]]: - query: Select[tuple[Any, int]] = ( +def _build_label_video_pairs_subquery( + collection_id: UUID, annotation_type: Optional[AnnotationType] = None +) -> Any: + """Return distinct (label_id, video_id) pairs from frame and direct video annotations. + + When ``annotation_type`` is provided, only annotations of that type are considered. + """ + frame_pairs: Select[tuple[Any, Any]] = ( select( col(AnnotationBaseTable.annotation_label_id).label("label_id"), - func.count(func.distinct(VideoTable.sample_id)).label(count_column_name), + col(VideoTable.sample_id).label("video_id"), ) .select_from(AnnotationBaseTable) .join( @@ -105,8 +123,20 @@ def _build_base_query( .join(VideoTable, col(VideoTable.sample_id) == col(SampleTable.sample_id)) .where(col(SampleTable.collection_id) == collection_id) ) - + video_pairs: Select[tuple[Any, Any]] = ( + select( + col(AnnotationBaseTable.annotation_label_id).label("label_id"), + col(VideoTable.sample_id).label("video_id"), + ) + .select_from(AnnotationBaseTable) + .join( + SampleTable, + col(SampleTable.sample_id) == col(AnnotationBaseTable.parent_sample_id), + ) + .join(VideoTable, col(VideoTable.sample_id) == col(SampleTable.sample_id)) + .where(col(SampleTable.collection_id) == collection_id) + ) if annotation_type is not None: - query = query.where(col(AnnotationBaseTable.annotation_type) == annotation_type) - - return query + frame_pairs = frame_pairs.where(col(AnnotationBaseTable.annotation_type) == annotation_type) + video_pairs = video_pairs.where(col(AnnotationBaseTable.annotation_type) == annotation_type) + return union_all(frame_pairs, video_pairs).subquery("label_video_pairs") diff --git a/lightly_studio/src/lightly_studio/resolvers/video_resolver/video_filter.py b/lightly_studio/src/lightly_studio/resolvers/video_resolver/video_filter.py index 6e8692f27f..7a1732846c 100644 --- a/lightly_studio/src/lightly_studio/resolvers/video_resolver/video_filter.py +++ b/lightly_studio/src/lightly_studio/resolvers/video_resolver/video_filter.py @@ -3,6 +3,7 @@ from typing import Literal, Optional from uuid import UUID +from sqlalchemy import or_ from sqlmodel import col, select from sqlmodel.sql.expression import SelectOfScalar @@ -82,7 +83,7 @@ def _apply_duration_filters(self, query: QueryType) -> QueryType: return query def _apply_annotation_filter(self, query: QueryType) -> QueryType: - """For videos, annotation filters are applied to the frames.""" + """For videos, annotation filters match frame or direct video annotations.""" assert self.frame_annotation_filter is not None frame_filtered_video_ids_subquery = select(VideoFrameTable.parent_sample_id) @@ -93,7 +94,20 @@ def _apply_annotation_filter(self, query: QueryType) -> QueryType: ) ) - return query.where(col(VideoTable.sample_id).in_(frame_filtered_video_ids_subquery)) + video_filtered_video_ids_subquery = select(VideoTable.sample_id) + video_filtered_video_ids_subquery = ( + self.frame_annotation_filter.apply_to_parent_sample_query( + query=video_filtered_video_ids_subquery, + sample_id_column=col(VideoTable.sample_id), + ) + ) + + return query.where( + or_( + col(VideoTable.sample_id).in_(frame_filtered_video_ids_subquery), + col(VideoTable.sample_id).in_(video_filtered_video_ids_subquery), + ) + ) def _select_sample_ids(self) -> SelectOfScalar[UUID]: return select(VideoTable.sample_id).join(VideoTable.sample) diff --git a/lightly_studio/tests/resolvers/video/video_resolver/test_count_video_frame_annotations_by_video_dataset.py b/lightly_studio/tests/resolvers/video/video_resolver/test_count_video_frame_annotations_by_video_dataset.py index 8755f7b02e..de9a13526c 100644 --- a/lightly_studio/tests/resolvers/video/video_resolver/test_count_video_frame_annotations_by_video_dataset.py +++ b/lightly_studio/tests/resolvers/video/video_resolver/test_count_video_frame_annotations_by_video_dataset.py @@ -17,7 +17,7 @@ create_annotations, create_collection, ) -from tests.resolvers.video.helpers import VideoStub, create_video_with_frames +from tests.resolvers.video.helpers import VideoStub, create_video, create_video_with_frames def test_count_video_frame_annotations_by_video_collection_without_filter( @@ -200,6 +200,135 @@ def test_count_video_frame_annotations_by_video_collection_with_annotation_filte assert annotations[1].current_count == 0 +def test_count_video_annotations_by_video_collection__direct_video_annotations( + db_session: Session, +) -> None: + collection = create_collection(session=db_session, sample_type=SampleType.VIDEO) + collection_id = collection.collection_id + + video_with_running = create_video( + session=db_session, + collection_id=collection_id, + video=VideoStub(path="/path/to/running.mp4"), + ) + video_with_jumping = create_video( + session=db_session, + collection_id=collection_id, + video=VideoStub(path="/path/to/jumping.mp4"), + ) + create_video( + session=db_session, + collection_id=collection_id, + video=VideoStub(path="/path/to/unlabeled.mp4"), + ) + + running_label = create_annotation_label( + session=db_session, + root_collection_id=collection_id, + label_name="Running", + ) + jumping_label = create_annotation_label( + session=db_session, + root_collection_id=collection_id, + label_name="Jumping", + ) + + create_annotations( + session=db_session, + collection_id=collection_id, + annotations=[ + AnnotationDetails( + sample_id=video_with_running.sample_id, + annotation_label_id=running_label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ), + AnnotationDetails( + sample_id=video_with_jumping.sample_id, + annotation_label_id=jumping_label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ), + ], + ) + + annotations = video_resolver.count_video_frame_annotations_by_video_collection( + session=db_session, + collection_id=collection_id, + ) + + assert len(annotations) == 2 + assert annotations[0].label_name == "Jumping" + assert annotations[0].total_count == 1 + assert annotations[0].current_count == 1 + assert annotations[1].label_name == "Running" + assert annotations[1].total_count == 1 + assert annotations[1].current_count == 1 + + +def test_count_video_annotations_by_video_collection__direct_video_annotations_with_filter( + db_session: Session, +) -> None: + collection = create_collection(session=db_session, sample_type=SampleType.VIDEO) + collection_id = collection.collection_id + + video_with_running = create_video( + session=db_session, + collection_id=collection_id, + video=VideoStub(path="/path/to/running.mp4"), + ) + video_with_jumping = create_video( + session=db_session, + collection_id=collection_id, + video=VideoStub(path="/path/to/jumping.mp4"), + ) + + running_label = create_annotation_label( + session=db_session, + root_collection_id=collection_id, + label_name="Running", + ) + jumping_label = create_annotation_label( + session=db_session, + root_collection_id=collection_id, + label_name="Jumping", + ) + + create_annotations( + session=db_session, + collection_id=collection_id, + annotations=[ + AnnotationDetails( + sample_id=video_with_running.sample_id, + annotation_label_id=running_label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ), + AnnotationDetails( + sample_id=video_with_jumping.sample_id, + annotation_label_id=jumping_label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ), + ], + ) + + annotations = video_resolver.count_video_frame_annotations_by_video_collection( + session=db_session, + collection_id=collection_id, + filters=VideoFilter( + frame_annotation_filter=AnnotationsFilter( + annotation_label_ids=[running_label.annotation_label_id] + ), + sample_filter=SampleFilter(), + ), + ) + + assert len(annotations) == 2 + assert annotations[0].label_name == "Jumping" + assert annotations[0].total_count == 1 + assert annotations[0].current_count == 0 + assert annotations[1].label_name == "Running" + assert annotations[1].total_count == 1 + assert annotations[1].current_count == 1 + + @pytest.fixture def typed_collection_id(db_session: Session) -> UUID: collection = create_collection(session=db_session, sample_type=SampleType.VIDEO) diff --git a/lightly_studio/tests/resolvers/video/video_resolver/test_video_filter.py b/lightly_studio/tests/resolvers/video/video_resolver/test_video_filter.py index ec0a99bfff..df56d83de1 100644 --- a/lightly_studio/tests/resolvers/video/video_resolver/test_video_filter.py +++ b/lightly_studio/tests/resolvers/video/video_resolver/test_video_filter.py @@ -4,6 +4,7 @@ from sqlmodel import Session, select +from lightly_studio.models.annotation.annotation_base import AnnotationType from lightly_studio.models.collection import SampleType from lightly_studio.models.video import VideoTable from lightly_studio.resolvers.annotations.annotations_filter import AnnotationsFilter @@ -14,7 +15,7 @@ create_annotations, create_collection, ) -from tests.resolvers.video.helpers import VideoStub, create_video_with_frames +from tests.resolvers.video.helpers import VideoStub, create_video, create_video_with_frames def test_query__annotation_filter(db_session: Session) -> None: @@ -56,3 +57,45 @@ def test_query__annotation_filter(db_session: Session) -> None: result = db_session.exec(filtered_query).all() assert len(result) == 1 assert result[0].sample_id == video_with_annotation.video_sample_id + + +def test_query__annotation_filter__direct_video_annotations(db_session: Session) -> None: + collection = create_collection(session=db_session, sample_type=SampleType.VIDEO) + video_with_annotation = create_video( + session=db_session, + collection_id=collection.collection_id, + video=VideoStub(path="with_annotation.mp4"), + ) + create_video( + session=db_session, + collection_id=collection.collection_id, + video=VideoStub(path="without_annotation.mp4"), + ) + label = create_annotation_label( + session=db_session, + root_collection_id=collection.collection_id, + label_name="running", + ) + create_annotations( + session=db_session, + collection_id=collection.collection_id, + annotations=[ + AnnotationDetails( + sample_id=video_with_annotation.sample_id, + annotation_label_id=label.annotation_label_id, + annotation_type=AnnotationType.CLASSIFICATION, + ) + ], + ) + + query = select(VideoTable).join(VideoTable.sample) + video_filter = VideoFilter( + frame_annotation_filter=AnnotationsFilter( + annotation_label_ids=[label.annotation_label_id], + ) + ) + + filtered_query = video_filter.apply(query=query) + result = db_session.exec(filtered_query).all() + assert len(result) == 1 + assert result[0].sample_id == video_with_annotation.sample_id From bdc02d2124a353607ebc47e036aa07e02ed0b51e Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 16 Jul 2026 16:09:32 +0300 Subject: [PATCH 14/32] fix merge error --- .../lightly_studio/resolvers/annotation_resolver/create_many.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py index ad7e8334ee..0b53f80e35 100644 --- a/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py +++ b/lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py @@ -82,7 +82,6 @@ def create_many( # Set other relationship details to None db_base_annotation.segmentation_details = None db_base_annotation.object_detection_details = None - db_base_annotation.temporal_span_details = None base_annotations.append(db_base_annotation) From a9d31fd363ffae01e111fa313668d8c92b7f76a3 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 16 Jul 2026 14:09:51 +0300 Subject: [PATCH 15/32] Add useVideoPlayback hook with tests Mirrors a