[do not merge]Add new player controls

lightly_studio/.env.example

@@ -31,6 +31,8 @@ EXAMPLES_PASCALVOC_MASKS_PATH="dataset_examples/voc2012_10_images/SegmentationCl
 EXAMPLES_PASCALVOC_PRED_MASKS_PATH="dataset_examples/voc2012_10_images/SegmentationClassPredictions"
 EXAMPLES_PASCALVOC_CATEGORIES_JSON_PATH="dataset_examples/voc2012_10_images/class_id_to_name.json"
 EXAMPLES_MIDV_PATH="dataset_examples/midv_2020_10_samples"
+EXAMPLES_ACTIVITYNET_VIDEOS_PATH="dataset_examples/activitynet_10_videos/videos"
+EXAMPLES_ACTIVITYNET_JSON_PATH="dataset_examples/activitynet_10_videos/activity_net.v1-3.min.json"
 
 # Request timing for endpoint functionality.
 # 

lightly_studio/pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     # Unreleased and released labelformat versions can be referenced with this syntax:
     # - "labelformat @ git+https://github.com/lightly-ai/labelformat.git@325a20b"
     # - "labelformat>=0.1.9"
-    "labelformat>=0.1.16",
+    "labelformat @ git+https://github.com/lightly-ai/labelformat.git@adeaf09cd10ec1b6f105da6886c00370fd894103",
     "tqdm>=4.65.0",
     "eval-type-backport>=0.2.2",
     "xxhash>=3.5.0",

lightly_studio/src/lightly_studio/core/video/add_annotations.py

@@ -0,0 +1,107 @@
+"""Functions to add annotations to videos already present in a dataset."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from uuid import UUID
+
+from labelformat.formats import ActivityNetTemporalClassificationInput
+from sqlmodel import Session
+from tqdm import tqdm
+
+from lightly_studio.core import labelformat_helpers
+from lightly_studio.models.annotation.annotation_base import AnnotationCreate, AnnotationType
+from lightly_studio.resolvers import (
+    annotation_resolver,
+    video_resolver,
+)
+from lightly_studio.type_definitions import PathLike
+
+logger = logging.getLogger(__name__)
+
+ANNOTATION_BATCH_SIZE = 1024
+
+
+def add_annotations_from_activitynet(
+    session: Session,
+    root_collection_id: UUID,
+    annotations_json: PathLike,
+    collection_name: str | None = None,
+) -> list[str]:
+    """Add ActivityNet-style event annotations to videos already in a collection.
+
+    Args:
+        session: The database session.
+        root_collection_id: The ID of the root video collection.
+        annotations_json: Path to an ActivityNet-style JSON file.
+        collection_name: Optional name for the annotation source. If ``None``, a default
+            name is used.
+
+    Returns:
+        A list of video IDs from the JSON that had no matching video in the collection.
+    """
+    annotations_json = Path(annotations_json).absolute()
+    if not annotations_json.is_file() or annotations_json.suffix != ".json":
+        raise FileNotFoundError(
+            f"ActivityNet annotations json file not found: '{annotations_json}'"
+        )
+
+    input_labels = ActivityNetTemporalClassificationInput(input_file=annotations_json)
+    stem_to_sample_id = video_resolver.get_sample_ids_by_stems(
+        session=session,
+        collection_id=root_collection_id,
+    )
+    label_map = labelformat_helpers.create_label_map(
+        session=session,
+        root_collection_id=root_collection_id,
+        input_labels=input_labels,
+    )
+
+    missing_video_ids: list[str] = []
+    annotations_to_create: list[AnnotationCreate] = []
+
+    for video_label in tqdm(
+        input_labels.get_labels(), desc="Processing ActivityNet annotations", unit=" videos"
+    ):
+        video_sample_id = stem_to_sample_id.get(video_label.video_id)
+        if video_sample_id is None:
+            missing_video_ids.append(video_label.video_id)
+            continue
+
+        for event in video_label.events:
+            annotations_to_create.append(
+                AnnotationCreate(
+                    annotation_label_id=label_map[event.category.id],
+                    annotation_type=AnnotationType.CLASSIFICATION,
+                    confidence=event.confidence,
+                    parent_sample_id=video_sample_id,
+                    start_time_s=event.start_time_s,
+                    end_time_s=event.end_time_s,
+                )
+            )
+
+            if len(annotations_to_create) >= ANNOTATION_BATCH_SIZE:
+                annotation_resolver.create_many(
+                    session=session,
+                    parent_collection_id=root_collection_id,
+                    annotations=annotations_to_create,
+                    collection_name=collection_name,
+                )
+                annotations_to_create.clear()
+
+    if annotations_to_create:
+        annotation_resolver.create_many(
+            session=session,
+            parent_collection_id=root_collection_id,
+            annotations=annotations_to_create,
+            collection_name=collection_name,
+        )
+
+    if missing_video_ids:
+        logger.warning(
+            "Skipped %d ActivityNet video IDs with no matching video in the collection.",
+            len(missing_video_ids),
+        )
+
+    return missing_video_ids

lightly_studio/src/lightly_studio/core/video/video_dataset.py

@@ -15,7 +15,7 @@
 
 from lightly_studio.core.dataset import BaseSampleDataset
 from lightly_studio.core.dataset_query.dataset_query import DatasetQuery
-from lightly_studio.core.video import add_videos
+from lightly_studio.core.video import add_annotations, add_videos
 from lightly_studio.core.video.add_videos import VIDEO_EXTENSIONS
 from lightly_studio.core.video.video_frame_dataset import VideoFrameDataset
 from lightly_studio.core.video.video_sample import VideoSample
@@ -236,6 +236,33 @@ def add_videos_from_youtube_vis(  # noqa: PLR0913
                 sample_ids=created_sample_ids,
             )
 
+    def add_annotations_from_activitynet(
+        self,
+        annotations_json: PathLike,
+        annotation_source: str | None = None,
+    ) -> list[str]:
+        """Attach ActivityNet-style temporal annotations to videos already in the dataset.
+
+        Videos are matched by ``file_name`` stem or ``file_path_abs`` stem against the
+        ActivityNet video identifier (e.g. ``v_o6wtvlVs3E8`` for ``v_o6wtvlVs3E8.mp4``).
+
+        Args:
+            annotations_json: Path to an ActivityNet-style JSON file with a ``database``
+                or ``results`` key.
+            annotation_source: Name of the annotation source to add the annotations to.
+                Reusing the same source name appends to that source. If ``None``, a
+                default source is used.
+
+        Returns:
+            Video IDs from the JSON that had no matching video in the dataset.
+        """
+        return add_annotations.add_annotations_from_activitynet(
+            session=self.session,
+            root_collection_id=self.collection_id,
+            annotations_json=annotations_json,
+            collection_name=annotation_source,
+        )
+
 
 def _generate_embeddings_video(
     session: Session,

lightly_studio/src/lightly_studio/examples/example_activitynet.py

@@ -0,0 +1,42 @@
+"""Example of importing ActivityNet-style temporal annotations."""
+
+from __future__ import annotations
+
+from environs import Env
+
+import lightly_studio as ls
+from lightly_studio.core.video.video_dataset import VideoDataset
+from lightly_studio.database import db_manager
+
+# Read environment variables
+env = Env()
+env.read_env()
+
+# Cleanup an existing database
+db_manager.connect(cleanup_existing=True)
+
+videos_path = env.path(
+    "EXAMPLES_ACTIVITYNET_VIDEOS_PATH",
+    "dataset_examples/activitynet_10_videos/videos",
+)
+annotations_path = env.path(
+    "EXAMPLES_ACTIVITYNET_JSON_PATH",
+    "dataset_examples/activitynet_10_videos/activity_net.v1-3.min.json",
+)
+
+dataset = VideoDataset.create()
+dataset.add_videos_from_path(path=videos_path, embed=False, embed_frames=False, target_fps=1)
+dataset.add_annotations_from_activitynet(
+    annotations_json=annotations_path,
+    annotation_source="activitynet",
+)
+
+# Print the loaded annotations per video.
+for video in dataset:
+    print(f"{video.file_name}: {len(video.annotations)} annotation(s)")
+    for annotation in video.annotations:
+        span = annotation.annotation_base.temporal_span_details
+        time_range = f"{span.start_time_s:.2f}s - {span.end_time_s:.2f}s" if span else "no span"
+        print(f"  - {annotation.class_name} [{time_range}] (confidence: {annotation.confidence})")
+
+ls.start_gui()

lightly_studio/src/lightly_studio/resolvers/annotation_resolver/create_many.py

@@ -82,6 +82,7 @@ def create_many(
         # Set other relationship details to None
         db_base_annotation.segmentation_details = None
         db_base_annotation.object_detection_details = None
+        db_base_annotation.temporal_span_details = None
 
         base_annotations.append(db_base_annotation)
 

lightly_studio/src/lightly_studio/resolvers/video_resolver/__init__.py

@@ -15,6 +15,7 @@
     build_sample_ids_query,
     get_sample_ids,
 )
+from lightly_studio.resolvers.video_resolver.get_sample_ids_by_stems import get_sample_ids_by_stems
 from lightly_studio.resolvers.video_resolver.get_table_fields_bounds import (
     get_table_fields_bounds,
 )
@@ -30,6 +31,7 @@
     "get_by_id",
     "get_many_by_id",
     "get_sample_ids",
+    "get_sample_ids_by_stems",
     "get_table_fields_bounds",
     "get_view_by_id",
 ]

lightly_studio/src/lightly_studio/resolvers/video_resolver/count_video_frame_annotations_by_collection.py

@@ -1,9 +1,10 @@
-"""Count video frame annotations by video collection."""
+"""Count video annotations by video collection."""
 
 from typing import Any, Optional
 from uuid import UUID
 
 from pydantic import BaseModel
+from sqlalchemy import union_all
 from sqlmodel import Session, asc, col, func, select
 from sqlmodel.sql.expression import Select
 
@@ -31,33 +32,48 @@ def count_video_frame_annotations_by_video_collection(
     filters: Optional[VideoFilter] = None,
     annotation_type: Optional[AnnotationType] = None,
 ) -> list[CountAnnotationsView]:
-    """Count the annotations by video frames.
+    """Count annotations attached to videos, including frame and direct video labels.
 
     When ``annotation_type`` is provided, both the total and filtered counts are
     restricted to annotations of that type (e.g. only CLASSIFICATION or only
     OBJECT_DETECTION).
     """
+    label_video_pairs = _build_label_video_pairs_subquery(
+        collection_id=collection_id, annotation_type=annotation_type
+    )
+
     unfiltered_query = (
-        _build_base_query(
-            collection_id=collection_id,
-            count_column_name="total",
-            annotation_type=annotation_type,
+        select(
+            label_video_pairs.c.label_id,
+            func.count(func.distinct(label_video_pairs.c.video_id)).label("total"),
         )
-        .group_by(col(AnnotationBaseTable.annotation_label_id))
+        .select_from(label_video_pairs)
+        .group_by(label_video_pairs.c.label_id)
         .subquery("unfiltered")
     )
-    filtered_query = _build_base_query(
-        collection_id=collection_id,
-        count_column_name="filtered_count",
-        annotation_type=annotation_type,
-    )
 
+    filtered_pairs_query: Select[tuple[Any, Any]] = (
+        select(
+            label_video_pairs.c.label_id,
+            label_video_pairs.c.video_id,
+        )
+        .select_from(label_video_pairs)
+        .join(VideoTable, col(VideoTable.sample_id) == label_video_pairs.c.video_id)
+        .join(SampleTable, col(SampleTable.sample_id) == col(VideoTable.sample_id))
+    )
     if filters is not None:
-        filtered_query = filters.apply(filtered_query)
+        filtered_pairs_query = filters.apply(filtered_pairs_query)
 
-    filtered_subquery = filtered_query.group_by(
-        col(AnnotationBaseTable.annotation_label_id)
-    ).subquery("filtered")
+    filtered_pairs_subquery = filtered_pairs_query.subquery("filtered_pairs")
+    filtered_subquery = (
+        select(
+            filtered_pairs_subquery.c.label_id,
+            func.count(func.distinct(filtered_pairs_subquery.c.video_id)).label("filtered_count"),
+        )
+        .select_from(filtered_pairs_subquery)
+        .group_by(filtered_pairs_subquery.c.label_id)
+        .subquery("filtered")
+    )
 
     final_query: Select[Any] = (
         select(
@@ -86,15 +102,17 @@ def count_video_frame_annotations_by_video_collection(
     ]
 
 
-def _build_base_query(
-    collection_id: UUID,
-    count_column_name: str,
-    annotation_type: Optional[AnnotationType] = None,
-) -> Select[tuple[Any, int]]:
-    query: Select[tuple[Any, int]] = (
+def _build_label_video_pairs_subquery(
+    collection_id: UUID, annotation_type: Optional[AnnotationType] = None
+) -> Any:
+    """Return distinct (label_id, video_id) pairs from frame and direct video annotations.
+
+    When ``annotation_type`` is provided, only annotations of that type are considered.
+    """
+    frame_pairs: Select[tuple[Any, Any]] = (
         select(
             col(AnnotationBaseTable.annotation_label_id).label("label_id"),
-            func.count(func.distinct(VideoTable.sample_id)).label(count_column_name),
+            col(VideoTable.sample_id).label("video_id"),
         )
         .select_from(AnnotationBaseTable)
         .join(
@@ -105,8 +123,20 @@ def _build_base_query(
         .join(VideoTable, col(VideoTable.sample_id) == col(SampleTable.sample_id))
         .where(col(SampleTable.collection_id) == collection_id)
     )
-
+    video_pairs: Select[tuple[Any, Any]] = (
+        select(
+            col(AnnotationBaseTable.annotation_label_id).label("label_id"),
+            col(VideoTable.sample_id).label("video_id"),
+        )
+        .select_from(AnnotationBaseTable)
+        .join(
+            SampleTable,
+            col(SampleTable.sample_id) == col(AnnotationBaseTable.parent_sample_id),
+        )
+        .join(VideoTable, col(VideoTable.sample_id) == col(SampleTable.sample_id))
+        .where(col(SampleTable.collection_id) == collection_id)
+    )
     if annotation_type is not None:
-        query = query.where(col(AnnotationBaseTable.annotation_type) == annotation_type)
-
-    return query
+        frame_pairs = frame_pairs.where(col(AnnotationBaseTable.annotation_type) == annotation_type)
+        video_pairs = video_pairs.where(col(AnnotationBaseTable.annotation_type) == annotation_type)
+    return union_all(frame_pairs, video_pairs).subquery("label_video_pairs")