Add new video events

lightly_studio/src/lightly_studio/api/routes/api/annotations/create_annotation.py

@@ -36,6 +36,8 @@ class AnnotationCreateInput(BaseModel):
     width: int | None = None
     height: int | None = None
     segmentation_mask: list[int] | None = None
+    start_time_s: float | None = None
+    end_time_s: float | None = None
 
 
 @create_annotation_router.post(

lightly_studio/src/lightly_studio/services/annotations_service/create_annotation.py

@@ -34,6 +34,9 @@ class AnnotationCreateParams(BaseModel):
 
     segmentation_mask: list[int] | None = None
 
+    start_time_s: float | None = None
+    end_time_s: float | None = None
+
 
 def create_annotation(session: Session, annotation: AnnotationCreateParams) -> AnnotationBaseTable:
     """Create a new annotation.

lightly_studio/tests/services/annotations_service/test_create_annotation.py

@@ -164,3 +164,26 @@ def test_create_annotation_with_collection_name(
     assert created_collection is not None
     assert created_collection.name == collection_name
     assert created_collection.parent_collection_id == collection.collection_id
+
+
+def test_create_annotation_classification_with_temporal_span(
+    db_session: Session,
+    collection: CollectionTable,
+    samples: list[ImageTable],
+    annotation_labels: list[AnnotationLabelTable],
+) -> None:
+    """Test to create a classification annotation carrying a temporal span."""
+    annotation = AnnotationCreateParams(
+        annotation_label_id=annotation_labels[0].annotation_label_id,
+        annotation_type=AnnotationType.CLASSIFICATION,
+        collection_id=collection.collection_id,
+        parent_sample_id=samples[0].sample_id,
+        start_time_s=2.5,
+        end_time_s=8.0,
+    )
+    result = create_annotation(session=db_session, annotation=annotation)
+
+    assert isinstance(result, AnnotationBaseTable)
+    assert result.temporal_span_details is not None
+    assert result.temporal_span_details.start_time_s == 2.5
+    assert result.temporal_span_details.end_time_s == 8.0

lightly_studio_view/src/lib/components/VideoControls/VideoControls.stories.svelte

@@ -7,9 +7,7 @@
         component: VideoControls,
         tags: ['autodocs']
     });
-</script>
 
-<script>
     // The bar owns no state; the Playground wires local state so the scrubber
     // and transport buttons actually respond.
     let currentTimeS = $state(15);

lightly_studio_view/src/lib/components/VideoDetails/VideoDetails.svelte

@@ -8,14 +8,24 @@
         VideoFrameDetails,
         VideoPlayer
     } from '$lib/components';
-    import { type FrameView, type SampleView, type VideoView } from '$lib/api/lightly_studio_local';
+    import {
+        AnnotationType,
+        type FrameView,
+        type SampleView,
+        type VideoView
+    } from '$lib/api/lightly_studio_local';
     import { getVideoURLById, toVideoEvents, type VideoEvent } from '$lib/utils';
     import VideoSampleMetadata from '../VideoSampleMetadata/VideoSampleMetadata.svelte';
     import SampleDetailsCaptionSegment from '../SampleDetails/SampleDetailsCaptionsSegment/SampleDetailsCaptionSegment.svelte';
+    import SelectClassDialog from '$lib/components/SelectClassDialog/SelectClassDialog.svelte';
     import { useVideoFrames } from '$lib/hooks/useVideoFrames/useVideoFrames';
     import { useVideoFrameAnnotations } from '$lib/hooks/useVideoFrameAnnotations/useVideoFrameAnnotations';
     import { useGlobalStorage } from '$lib/hooks/useGlobalStorage';
     import { useUpdateAnnotationsMutation } from '$lib/hooks/useUpdateAnnotationsMutation/useUpdateAnnotationsMutation';
+    import { useAnnotationLabels } from '$lib/hooks/useAnnotationLabels/useAnnotationLabels';
+    import { useCreateAnnotation } from '$lib/hooks/useCreateAnnotation/useCreateAnnotation';
+    import { useCreateLabel } from '$lib/hooks/useCreateLabel/useCreateLabel';
+    import { useSelectClassDialog } from '$lib/hooks/useSelectClassDialog/useSelectClassDialog';
     import { onMount } from 'svelte';
     import { toast } from 'svelte-sonner';
     import { routeHelpers } from '$lib/routes';
@@ -40,9 +50,23 @@
     // Reuse the global "Edit annotations" toggle to enable event editing.
     const { isEditingMode } = useGlobalStorage();
 
-    // Events live on the video's own collection.
+    // Events live on the video's own collection; labels are shared per dataset.
     const eventCollectionId = (video.sample as SampleView)?.collection_id ?? datasetId;
     const { updateAnnotations } = useUpdateAnnotationsMutation({ collectionId: eventCollectionId });
+    const { createAnnotation } = useCreateAnnotation({ collectionId: eventCollectionId });
+    const { createLabel } = useCreateLabel({ collectionId: eventCollectionId });
+    const annotationLabels = useAnnotationLabels(() => ({ collectionId: eventCollectionId }));
+
+    const {
+        open: selectClassOpen,
+        requestLabel,
+        handleConfirm: handleClassSelected,
+        handleCancel: handleClassDialogCancel
+    } = useSelectClassDialog();
+    const labelNames = $derived(
+        annotationLabels.data?.map((label) => label.annotation_label_name ?? '').filter(Boolean) ??
+            []
+    );
 
     async function handleEventResize(event: VideoEvent, startTimeS: number, endTimeS: number) {
         try {
@@ -64,6 +88,36 @@
         }
     }
 
+    async function handleEventAdd(startTimeS: number, endTimeS: number) {
+        const result = await requestLabel();
+        if (!result?.label) return;
+
+        try {
+            let label = annotationLabels.data?.find(
+                (item) => item.annotation_label_name === result.label
+            );
+            if (!label) {
+                label = await createLabel({
+                    dataset_id: datasetId,
+                    annotation_label_name: result.label
+                });
+            }
+
+            await createAnnotation({
+                parent_sample_id: video.sample_id,
+                annotation_type: AnnotationType.CLASSIFICATION,
+                annotation_label_id: label.annotation_label_id!,
+                start_time_s: startTimeS,
+                end_time_s: endTimeS
+            });
+            onVideoUpdate();
+            toast.success('Event created successfully');
+        } catch (error) {
+            toast.error('Failed to create event. Please try again.');
+            console.error('Error creating event:', error);
+        }
+    }
+
     const {
         currentFrame,
         frames: videoFrames,
@@ -200,6 +254,7 @@
                         durationS={video.duration_s ?? undefined}
                         editableEvents={$isEditingMode}
                         onEventResize={handleEventResize}
+                        onEventAdd={handleEventAdd}
                         videoProps={{
                             muted: true,
                             class: 'object-contain',
@@ -267,6 +322,13 @@
     </div>
 </div>
 
+<SelectClassDialog
+    bind:open={$selectClassOpen}
+    labels={labelNames}
+    onConfirm={handleClassSelected}
+    onCancel={handleClassDialogCancel}
+/>
+
 <style>
     .video-frame-container {
         width: 100%;

lightly_studio_view/src/lib/components/VideoDetails/VideoDetails.test.ts

@@ -43,11 +43,20 @@ vi.mock('$lib/hooks/useVideoFrames/useVideoFrames', () => ({ useVideoFrames: vi.
 vi.mock('$lib/hooks/useVideoFrameAnnotations/useVideoFrameAnnotations', () => ({
     useVideoFrameAnnotations: vi.fn(() => new Map())
 }));
-// Event editing needs a QueryClient; stub the mutation hook so the test runs
-// without a QueryClientProvider.
+// Event editing/adding needs a QueryClient; stub the annotation hooks so the
+// test runs without a QueryClientProvider.
 vi.mock('$lib/hooks/useUpdateAnnotationsMutation/useUpdateAnnotationsMutation', () => ({
     useUpdateAnnotationsMutation: vi.fn(() => ({ updateAnnotations: vi.fn() }))
 }));
+vi.mock('$lib/hooks/useCreateAnnotation/useCreateAnnotation', () => ({
+    useCreateAnnotation: vi.fn(() => ({ createAnnotation: vi.fn() }))
+}));
+vi.mock('$lib/hooks/useCreateLabel/useCreateLabel', () => ({
+    useCreateLabel: vi.fn(() => ({ createLabel: vi.fn() }))
+}));
+vi.mock('$lib/hooks/useAnnotationLabels/useAnnotationLabels', () => ({
+    useAnnotationLabels: vi.fn(() => ({ data: [] }))
+}));
 
 import VideoDetails from './VideoDetails.svelte';
 

lightly_studio_view/src/lib/components/VideoEventTimeline/VideoEventTimeline.svelte

@@ -26,6 +26,7 @@
      * />
      * ```
      */
+    import { Plus } from '@lucide/svelte';
     import { cn } from '$lib/utils/shadcn.js';
     import { assignEventLanes, type VideoEvent } from '$lib/utils';
 
@@ -42,6 +43,8 @@
         editable?: boolean;
         /** Called with the new span when an event edge finishes being edited. */
         onResize?: (event: VideoEvent, startTimeS: number, endTimeS: number) => void;
+        /** Called with a default span when the user requests a new event. */
+        onAddEvent?: (startTimeS: number, endTimeS: number) => void;
         /** Heading shown above the timeline. */
         title?: string;
         /** Whether to render the title/count header above the track. */
@@ -57,6 +60,7 @@
         onSeek,
         editable = false,
         onResize,
+        onAddEvent,
         title = 'Events',
         showHeader = true,
         class: className
@@ -68,6 +72,8 @@
     const MIN_BAR_WIDTH_PERCENT = 0.75;
     // Smallest span an edit may collapse an event to.
     const MIN_EVENT_DURATION_S = 0.1;
+    // Default length of an event created via the add button.
+    const DEFAULT_NEW_EVENT_DURATION_S = 1;
 
     type Span = { startTimeS: number; endTimeS: number };
     type Edge = 'start' | 'end';
@@ -159,13 +165,47 @@
         const span = pendingEdits[event.id];
         if (span) onResize?.(event, span.startTimeS, span.endTimeS);
     }
+
+    function addEvent() {
+        if (durationS <= 0) return;
+        const span = Math.min(DEFAULT_NEW_EVENT_DURATION_S, durationS);
+        let startTimeS = Math.min(Math.max(0, currentTimeS), durationS);
+        let endTimeS = startTimeS + span;
+        if (endTimeS > durationS) {
+            endTimeS = durationS;
+            startTimeS = Math.max(0, endTimeS - span);
+        }
+        onAddEvent?.(startTimeS, endTimeS);
+    }
+
+    const showAddButton = $derived(editable && onAddEvent != null && durationS > 0);
 </script>
 
 <div class={cn('flex flex-col gap-2', className)} data-testid="video-event-timeline">
-    {#if showHeader}
+    {#if showHeader || showAddButton}
         <div class="flex items-center justify-between text-xs text-muted-foreground">
-            <span class="font-medium">{title}</span>
-            <span>{events.length}</span>
+            {#if showHeader}
+                <span class="font-medium">{title}</span>
+            {:else}
+                <span></span>
+            {/if}
+            <div class="flex items-center gap-2">
+                {#if showHeader}
+                    <span>{events.length}</span>
+                {/if}
+                {#if showAddButton}
+                    <button
+                        type="button"
+                        class="flex items-center gap-1 rounded px-1.5 py-0.5 transition-colors hover:text-foreground focus:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+                        onclick={addEvent}
+                        aria-label="Add event"
+                        data-testid="add-event-button"
+                    >
+                        <Plus class="size-3.5" />
+                        Add event
+                    </button>
+                {/if}
+            </div>
         </div>
     {/if}
 

lightly_studio_view/src/lib/components/VideoEventTimeline/VideoEventTimeline.test.ts

@@ -126,4 +126,36 @@ describe('VideoEventTimeline', () => {
         // Playback follows the dragged edge so the frame at 2s is visible.
         expect(onSeek).toHaveBeenLastCalledWith(2);
     });
+
+    it('shows the add button only when editable with an onAddEvent handler', () => {
+        const { queryByTestId, rerender } = render(VideoEventTimeline, {
+            props: { events: [], durationS: 20, editable: false, onAddEvent: vi.fn() }
+        });
+        expect(queryByTestId('add-event-button')).toBeFalsy();
+
+        rerender({ events: [], durationS: 20, editable: true, onAddEvent: vi.fn() });
+        expect(queryByTestId('add-event-button')).toBeTruthy();
+    });
+
+    it('adds an event spanning from the current time with a default length', async () => {
+        const onAddEvent = vi.fn();
+        const { getByTestId } = render(VideoEventTimeline, {
+            props: { events: [], durationS: 20, currentTimeS: 3, editable: true, onAddEvent }
+        });
+
+        await fireEvent.click(getByTestId('add-event-button'));
+        // Default 1s span starting at the playhead (3s).
+        expect(onAddEvent).toHaveBeenCalledWith(3, 4);
+    });
+
+    it('clamps a new event that would overflow the end of the video', async () => {
+        const onAddEvent = vi.fn();
+        const { getByTestId } = render(VideoEventTimeline, {
+            props: { events: [], durationS: 10, currentTimeS: 10, editable: true, onAddEvent }
+        });
+
+        await fireEvent.click(getByTestId('add-event-button'));
+        // A 1s span can't start at 10s in a 10s video, so it is shifted back to end at 10s.
+        expect(onAddEvent).toHaveBeenCalledWith(9, 10);
+    });
 });

lightly_studio_view/src/lib/components/VideoPlayer/VideoPlayer.svelte

@@ -75,6 +75,9 @@
 
         /** Called with the new span when an event edge finishes being edited. */
         onEventResize?: (event: VideoEvent, startTimeS: number, endTimeS: number) => void;
+
+        /** Called with a default span when the user adds a new event. */
+        onEventAdd?: (startTimeS: number, endTimeS: number) => void;
     }
 
     let {
@@ -86,7 +89,8 @@
         events = [],
         durationS,
         editableEvents = false,
-        onEventResize
+        onEventResize,
+        onEventAdd
     }: VideoPlayerProps = $props();
 
     const defaultVideoProps: HTMLVideoAttributes = {
@@ -210,7 +214,7 @@
         onToggleMute={playback.toggleMute}
         onToggleFullscreen={playback.toggleFullscreen}
     >
-        {#if showEvents}
+        {#if showEvents || editableEvents}
             <VideoEventTimeline
                 class="w-full"
                 {events}
@@ -219,6 +223,7 @@
                 onSeek={playback.seekTo}
                 editable={editableEvents}
                 onResize={onEventResize}
+                onAddEvent={onEventAdd}
                 showHeader={false}
             />
         {/if}

lightly_studio_view/src/lib/components/VideoPlayer/VideoPlayer.test.ts

@@ -153,4 +153,18 @@ describe('VideoPlayer', () => {
         expect(getByTestId('video-event-timeline')).toBeTruthy();
         expect(getByText('Jump')).toBeTruthy();
     });
+
+    it('shows the event bar with an add button in edit mode even without events', () => {
+        const { getByTestId } = render(VideoPlayer, {
+            props: {
+                src: 'test-video.mp4',
+                durationS: 10,
+                editableEvents: true,
+                onEventAdd: () => {}
+            }
+        });
+
+        expect(getByTestId('video-event-timeline')).toBeTruthy();
+        expect(getByTestId('add-event-button')).toBeTruthy();
+    });
 });