From 158294c4771de7b625b92489d4f17787213aa34e Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 30 Apr 2026 18:28:46 +0200 Subject: [PATCH 01/34] save idea --- src/common/liveUpdateInterceptor.ts | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/common/liveUpdateInterceptor.ts diff --git a/src/common/liveUpdateInterceptor.ts b/src/common/liveUpdateInterceptor.ts new file mode 100644 index 000000000..354ee18c8 --- /dev/null +++ b/src/common/liveUpdateInterceptor.ts @@ -0,0 +1,56 @@ +// dataset-changed.interceptor.ts +import { Injectable, NestInterceptor, ExecutionContext, CallHandler, Controller, Delete, Patch, Post, Sse, UseInterceptors } from "@nestjs/common"; +import { Observable, Subject } from "rxjs"; +import { tap } from "rxjs/operators"; + +@Injectable() +export class DatasetChangedInterceptor implements NestInterceptor { + constructor(private readonly sseService: SseService) {} + + intercept(context: ExecutionContext, next: CallHandler): Observable { + return next.handle().pipe( + tap(() => this.sseService.emit()), + ); + } +} + +// sse.service.ts +@Injectable() +export class SseService { + private subject = new Subject(); + + emit() { + this.subject.next({ data: "dataset.changed" }); + } + + getEvents(): Observable { + return this.subject.asObservable(); + } +} +// datasets.controller.ts +@Controller("datasets") +export class DatasetsController { + + @Sse("events") + datasetEvents(): Observable { + return this.sseService.getEvents(); + } + + @UseInterceptors(DatasetChangedInterceptor) + @Post() + async create(...) { ... } + + @UseInterceptors(DatasetChangedInterceptor) + @Patch(":id") + async update(...) { ... } + + @UseInterceptors(DatasetChangedInterceptor) + @Delete(":id") + async remove(...) { ... } +} +// frontend +const eventSource = new EventSource("/api/v3/datasets/events"); + +eventSource.onmessage = () => { + this.store.dispatch(fetchDatasets()); +}; \ No newline at end of file From 2de2aaf28ff70b346fc09009cfb303c06f7a4665 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 2 Jun 2026 15:38:06 +0200 Subject: [PATCH 02/34] merge --- .../interceptors/liveUpdate.Interceptor.ts | 29 +++++++++++++++++++ .../serverSideEvent.controller.ts | 26 +++++++++++++++++ src/serverSideEvent/serverSideEvent.module.ts | 11 +++++++ .../serverSideEvent.service.ts | 15 ++++++++++ 4 files changed, 81 insertions(+) create mode 100644 src/common/interceptors/liveUpdate.Interceptor.ts create mode 100644 src/serverSideEvent/serverSideEvent.controller.ts create mode 100644 src/serverSideEvent/serverSideEvent.module.ts create mode 100644 src/serverSideEvent/serverSideEvent.service.ts diff --git a/src/common/interceptors/liveUpdate.Interceptor.ts b/src/common/interceptors/liveUpdate.Interceptor.ts new file mode 100644 index 000000000..3b305d9e2 --- /dev/null +++ b/src/common/interceptors/liveUpdate.Interceptor.ts @@ -0,0 +1,29 @@ +import { + Injectable, + NestInterceptor, + ExecutionContext, + CallHandler, +} from "@nestjs/common"; +import { Observable, tap } from "rxjs"; +import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; + +@Injectable() +export class DatasetEventsInterceptor implements NestInterceptor { + constructor(private readonly eventsService: EventsService) {} + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const request = context.switchToHttp().getRequest(); + const method = request.method; + + return next.handle().pipe( + tap(() => { + if (["POST", "PATCH", "PUT", "DELETE"].includes(method)) { + this.eventsService.emit({ + type: "event.dataset.updated", + message: "Dataset updated", + }); + } + }), + ); + } +} diff --git a/src/serverSideEvent/serverSideEvent.controller.ts b/src/serverSideEvent/serverSideEvent.controller.ts new file mode 100644 index 000000000..a6ef8b38a --- /dev/null +++ b/src/serverSideEvent/serverSideEvent.controller.ts @@ -0,0 +1,26 @@ +import { Controller, Post, Sse, MessageEvent } from "@nestjs/common"; +import { Observable } from "rxjs"; +import { map } from "rxjs/operators"; +import { EventsService } from "./serverSideEvent.service"; + +@Controller("events") +export class EventsController { + constructor(private readonly eventsService: EventsService) {} + + // SSE stream endpoint + @Sse("stream") + stream(): Observable { + return this.eventsService + .getEvents() + .pipe(map((payload) => ({ data: payload }))); + } + + @Post("test") + triggerTest() { + this.eventsService.emit({ + type: "event.dataset.updated", + message: "hello from server", + }); + return { ok: true }; + } +} diff --git a/src/serverSideEvent/serverSideEvent.module.ts b/src/serverSideEvent/serverSideEvent.module.ts new file mode 100644 index 000000000..5a1128cc7 --- /dev/null +++ b/src/serverSideEvent/serverSideEvent.module.ts @@ -0,0 +1,11 @@ +import { Global, Module } from "@nestjs/common"; +import { EventsController } from "./serverSideEvent.controller"; +import { EventsService } from "./serverSideEvent.service"; + +@Global() +@Module({ + controllers: [EventsController], + providers: [EventsService], + exports: [EventsService], +}) +export class EventsModule {} diff --git a/src/serverSideEvent/serverSideEvent.service.ts b/src/serverSideEvent/serverSideEvent.service.ts new file mode 100644 index 000000000..fbe795a65 --- /dev/null +++ b/src/serverSideEvent/serverSideEvent.service.ts @@ -0,0 +1,15 @@ +import { Injectable } from "@nestjs/common"; +import { Subject } from "rxjs"; + +@Injectable() +export class EventsService { + private eventSubject = new Subject<{ type: string; message: string }>(); + + emit({ type, message }: { type: string; message: string }) { + this.eventSubject.next({ type, message }); + } + + getEvents() { + return this.eventSubject.asObservable(); + } +} From e0c032dba5aa7f8a1142eaba15cafef85ff2af32 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 2 Jun 2026 15:47:20 +0200 Subject: [PATCH 03/34] remove draft --- src/common/liveUpdateInterceptor.ts | 56 ----------------------------- 1 file changed, 56 deletions(-) delete mode 100644 src/common/liveUpdateInterceptor.ts diff --git a/src/common/liveUpdateInterceptor.ts b/src/common/liveUpdateInterceptor.ts deleted file mode 100644 index 354ee18c8..000000000 --- a/src/common/liveUpdateInterceptor.ts +++ /dev/null @@ -1,56 +0,0 @@ -// dataset-changed.interceptor.ts -import { Injectable, NestInterceptor, ExecutionContext, CallHandler, Controller, Delete, Patch, Post, Sse, UseInterceptors } from "@nestjs/common"; -import { Observable, Subject } from "rxjs"; -import { tap } from "rxjs/operators"; - -@Injectable() -export class DatasetChangedInterceptor implements NestInterceptor { - constructor(private readonly sseService: SseService) {} - - intercept(context: ExecutionContext, next: CallHandler): Observable { - return next.handle().pipe( - tap(() => this.sseService.emit()), - ); - } -} - -// sse.service.ts -@Injectable() -export class SseService { - private subject = new Subject(); - - emit() { - this.subject.next({ data: "dataset.changed" }); - } - - getEvents(): Observable { - return this.subject.asObservable(); - } -} -// datasets.controller.ts -@Controller("datasets") -export class DatasetsController { - - @Sse("events") - datasetEvents(): Observable { - return this.sseService.getEvents(); - } - - @UseInterceptors(DatasetChangedInterceptor) - @Post() - async create(...) { ... } - - @UseInterceptors(DatasetChangedInterceptor) - @Patch(":id") - async update(...) { ... } - - @UseInterceptors(DatasetChangedInterceptor) - @Delete(":id") - async remove(...) { ... } -} -// frontend -const eventSource = new EventSource("/api/v3/datasets/events"); - -eventSource.onmessage = () => { - this.store.dispatch(fetchDatasets()); -}; \ No newline at end of file From f9ca821ba2caf95768f3b177cd2221fe146bfaa0 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 2 Jun 2026 15:47:25 +0200 Subject: [PATCH 04/34] fix mailer type --- src/app.module.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/app.module.ts b/src/app.module.ts index 1295cbfae..a8d4b6f54 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -34,7 +34,6 @@ import { JobConfigModule } from "./config/job-config/jobconfig.module"; import { CoreJobActionCreators } from "./config/job-config/actions/corejobactioncreators.module"; import { HttpModule, HttpService } from "@nestjs/axios"; import { MSGraphMailTransport } from "./common/graph-mail"; -import { TransportType } from "@nestjs-modules/mailer/dist/interfaces/mailer-options.interface"; import { MetricsModule } from "./metrics/metrics.module"; import { GenericHistory, @@ -46,6 +45,8 @@ import { RuntimeConfigModule } from "./config/runtime-config/runtime-config.modu import { MetadataKeysModule } from "./metadata-keys/metadatakeys.module"; import { OidcClientModule } from "./common/openid-client/openid-client.module"; import { ThrottlerModule } from "@nestjs/throttler"; +import { EventsModule } from "./serverSideEvent/serverSideEvent.module"; +import type { MailerOptions } from "@nestjs-modules/mailer"; @Module({ imports: [ @@ -55,6 +56,7 @@ import { ThrottlerModule } from "@nestjs/throttler"; cache: true, }), AuthModule, + EventsModule, OidcClientModule, CaslModule, AttachmentsModule, @@ -88,7 +90,7 @@ import { ThrottlerModule } from "@nestjs/throttler"; configService: ConfigService, httpService: HttpService, ) => { - let transport: TransportType; + let transport: MailerOptions["transport"]; const transportType = configService .get("email.type") ?.toLowerCase(); From f00381fc7e943e572d61439d801d4e5847f4f945 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 3 Jun 2026 15:29:51 +0200 Subject: [PATCH 05/34] first implementation improved --- src/auth/strategies/jwt.strategy.ts | 5 ++- .../interceptors/liveUpdate.Interceptor.ts | 9 +++-- src/datasets/datasets.controller.ts | 2 ++ src/datasets/datasets.v4.controller.ts | 2 ++ .../serverSideEvent.controller.ts | 30 +++++++++------- src/serverSideEvent/serverSideEvent.module.ts | 2 ++ .../serverSideEvent.service.ts | 34 +++++++++++++++---- 7 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 996267869..5dcdca4b0 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -15,7 +15,10 @@ export class JwtStrategy extends PassportStrategy(Strategy) { private usersService: UsersService, ) { super({ - jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(), + jwtFromRequest: ExtractJwt.fromExtractors([ + ExtractJwt.fromAuthHeaderAsBearerToken(), + (req) => req?.query?.token || null, + ]), ignoreExpiration: false, secretOrKey: configService.get("jwt.secret") || "defaultSecret", }); diff --git a/src/common/interceptors/liveUpdate.Interceptor.ts b/src/common/interceptors/liveUpdate.Interceptor.ts index 3b305d9e2..d73b5bc89 100644 --- a/src/common/interceptors/liveUpdate.Interceptor.ts +++ b/src/common/interceptors/liveUpdate.Interceptor.ts @@ -5,6 +5,7 @@ import { CallHandler, } from "@nestjs/common"; import { Observable, tap } from "rxjs"; +import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; @Injectable() @@ -16,11 +17,13 @@ export class DatasetEventsInterceptor implements NestInterceptor { const method = request.method; return next.handle().pipe( - tap(() => { + tap((responseData: OutputDatasetDto) => { if (["POST", "PATCH", "PUT", "DELETE"].includes(method)) { this.eventsService.emit({ - type: "event.dataset.updated", - message: "Dataset updated", + ownerGroup: responseData?.ownerGroup, + accessGroups: responseData?.accessGroups ?? [], + message: "dataset.updated", + type: "dataset.updated", }); } }), diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 47029504c..d3a145d1c 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,6 +121,7 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; +import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -131,6 +132,7 @@ import { Filter } from "./decorators/filter.decorator"; TechniqueClass, RelationshipClass, ) +@UseInterceptors(DatasetEventsInterceptor) @ApiTags("datasets") @Controller({ path: "datasets", version: "3" }) export class DatasetsController { diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index a8574be7a..d23445965 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,6 +90,7 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; +import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -98,6 +99,7 @@ import { TechniqueClass } from "./schemas/technique.schema"; TechniqueClass, RelationshipClass, ) +@UseInterceptors(DatasetEventsInterceptor) @ApiTags("datasets v4") /* NOTE: Generated SDK method names include "V4" twice: * - From the controller class name (DatasetsV4Controller) diff --git a/src/serverSideEvent/serverSideEvent.controller.ts b/src/serverSideEvent/serverSideEvent.controller.ts index a6ef8b38a..000471f39 100644 --- a/src/serverSideEvent/serverSideEvent.controller.ts +++ b/src/serverSideEvent/serverSideEvent.controller.ts @@ -1,26 +1,32 @@ -import { Controller, Post, Sse, MessageEvent } from "@nestjs/common"; +import { Controller, Sse, MessageEvent, UseGuards, Req } from "@nestjs/common"; import { Observable } from "rxjs"; + +import { Request } from "express"; import { map } from "rxjs/operators"; import { EventsService } from "./serverSideEvent.service"; +import { ApiBearerAuth } from "@nestjs/swagger"; +import { PoliciesGuard } from "src/casl/guards/policies.guard"; +import { Action } from "src/casl/action.enum"; +import { AppAbility } from "src/casl/casl-ability.factory"; +import { CheckPolicies } from "src/casl/decorators/check-policies.decorator"; +import { DatasetClass } from "src/datasets/schemas/dataset.schema"; +import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; @Controller("events") +@ApiBearerAuth() export class EventsController { constructor(private readonly eventsService: EventsService) {} // SSE stream endpoint @Sse("stream") - stream(): Observable { + @UseGuards(PoliciesGuard) + @CheckPolicies("datasets", (ability: AppAbility) => + ability.can(Action.DatasetRead, DatasetClass), + ) + stream(@Req() request: Request): Observable { + const user = request.user as JWTUser; return this.eventsService - .getEvents() + .getEvents(user) .pipe(map((payload) => ({ data: payload }))); } - - @Post("test") - triggerTest() { - this.eventsService.emit({ - type: "event.dataset.updated", - message: "hello from server", - }); - return { ok: true }; - } } diff --git a/src/serverSideEvent/serverSideEvent.module.ts b/src/serverSideEvent/serverSideEvent.module.ts index 5a1128cc7..8c2c3d1ef 100644 --- a/src/serverSideEvent/serverSideEvent.module.ts +++ b/src/serverSideEvent/serverSideEvent.module.ts @@ -1,9 +1,11 @@ import { Global, Module } from "@nestjs/common"; import { EventsController } from "./serverSideEvent.controller"; import { EventsService } from "./serverSideEvent.service"; +import { CaslModule } from "src/casl/casl.module"; @Global() @Module({ + imports: [CaslModule], controllers: [EventsController], providers: [EventsService], exports: [EventsService], diff --git a/src/serverSideEvent/serverSideEvent.service.ts b/src/serverSideEvent/serverSideEvent.service.ts index fbe795a65..e72e6f8e7 100644 --- a/src/serverSideEvent/serverSideEvent.service.ts +++ b/src/serverSideEvent/serverSideEvent.service.ts @@ -1,15 +1,37 @@ import { Injectable } from "@nestjs/common"; -import { Subject } from "rxjs"; +import { Subject, Observable } from "rxjs"; +import { MessageEvent } from "@nestjs/common"; +import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; @Injectable() export class EventsService { - private eventSubject = new Subject<{ type: string; message: string }>(); + private clients = new Map }>(); - emit({ type, message }: { type: string; message: string }) { - this.eventSubject.next({ type, message }); + getEvents(user: JWTUser): Observable { + const subject = new Subject(); + + this.clients.set(user, { + subject, + }); + return subject.asObservable(); } - getEvents() { - return this.eventSubject.asObservable(); + emit(event: { + ownerGroup: string; + accessGroups: string[]; + message: string; + type: string; + }) { + for (const [user, { subject }] of this.clients) { + const userGroups = user.currentGroups ?? []; + + const canSee = + userGroups.includes(event.ownerGroup) || + event.accessGroups.some((g) => userGroups.includes(g)) || + userGroups.includes("admin"); + if (canSee) { + subject.next({ type: event.type, data: event.message }); + } + } } } From 76e7669e85967b2be0078100bdad45bbb460d729 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 8 Jun 2026 10:43:16 +0200 Subject: [PATCH 06/34] some code improvements: - add connection removal - add connections check --- .../interceptors/eventEmit.Interceptor.ts | 40 +++++++++++ .../interceptors/liveUpdate.Interceptor.ts | 32 --------- src/datasets/datasets.controller.ts | 4 +- src/datasets/datasets.v4.controller.ts | 4 +- .../serverSideEvent.controller.ts | 21 +++++- .../serverSideEvent.service.ts | 69 ++++++++++++++----- 6 files changed, 116 insertions(+), 54 deletions(-) create mode 100644 src/common/interceptors/eventEmit.Interceptor.ts delete mode 100644 src/common/interceptors/liveUpdate.Interceptor.ts diff --git a/src/common/interceptors/eventEmit.Interceptor.ts b/src/common/interceptors/eventEmit.Interceptor.ts new file mode 100644 index 000000000..5aef93965 --- /dev/null +++ b/src/common/interceptors/eventEmit.Interceptor.ts @@ -0,0 +1,40 @@ +import { + Injectable, + NestInterceptor, + ExecutionContext, + CallHandler, + Type, + mixin, +} from "@nestjs/common"; +import { Observable, tap } from "rxjs"; +import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; +import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; + +export const EventEmitInterceptor = ( + eventType: string, +): Type => { + @Injectable() + class MixinEventEmitInterceptor implements NestInterceptor { + constructor(public readonly eventsService: EventsService) {} + + intercept( + context: ExecutionContext, + next: CallHandler, + ): Observable { + const method = context.switchToHttp().getRequest().method; + + return next.handle().pipe( + tap((responseData: OutputDatasetDto) => { + if (["POST", "PATCH", "PUT", "DELETE"].includes(method)) { + this.eventsService.emit({ + message: responseData, + type: eventType, + }); + } + }), + ); + } + } + + return mixin(MixinEventEmitInterceptor); +}; diff --git a/src/common/interceptors/liveUpdate.Interceptor.ts b/src/common/interceptors/liveUpdate.Interceptor.ts deleted file mode 100644 index d73b5bc89..000000000 --- a/src/common/interceptors/liveUpdate.Interceptor.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - Injectable, - NestInterceptor, - ExecutionContext, - CallHandler, -} from "@nestjs/common"; -import { Observable, tap } from "rxjs"; -import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; -import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; - -@Injectable() -export class DatasetEventsInterceptor implements NestInterceptor { - constructor(private readonly eventsService: EventsService) {} - - intercept(context: ExecutionContext, next: CallHandler): Observable { - const request = context.switchToHttp().getRequest(); - const method = request.method; - - return next.handle().pipe( - tap((responseData: OutputDatasetDto) => { - if (["POST", "PATCH", "PUT", "DELETE"].includes(method)) { - this.eventsService.emit({ - ownerGroup: responseData?.ownerGroup, - accessGroups: responseData?.accessGroups ?? [], - message: "dataset.updated", - type: "dataset.updated", - }); - } - }), - ); - } -} diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index d3a145d1c..24b83b21d 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,7 +121,7 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; -import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Interceptor"; +import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -132,7 +132,7 @@ import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Int TechniqueClass, RelationshipClass, ) -@UseInterceptors(DatasetEventsInterceptor) +@UseInterceptors(EventEmitInterceptor("dataset.updated")) @ApiTags("datasets") @Controller({ path: "datasets", version: "3" }) export class DatasetsController { diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index d23445965..34ec234fb 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,7 +90,7 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; -import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Interceptor"; +import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -99,7 +99,7 @@ import { DatasetEventsInterceptor } from "src/common/interceptors/liveUpdate.Int TechniqueClass, RelationshipClass, ) -@UseInterceptors(DatasetEventsInterceptor) +@UseInterceptors(EventEmitInterceptor("dataset.updated")) @ApiTags("datasets v4") /* NOTE: Generated SDK method names include "V4" twice: * - From the controller class name (DatasetsV4Controller) diff --git a/src/serverSideEvent/serverSideEvent.controller.ts b/src/serverSideEvent/serverSideEvent.controller.ts index 000471f39..231448da8 100644 --- a/src/serverSideEvent/serverSideEvent.controller.ts +++ b/src/serverSideEvent/serverSideEvent.controller.ts @@ -1,4 +1,11 @@ -import { Controller, Sse, MessageEvent, UseGuards, Req } from "@nestjs/common"; +import { + Controller, + Sse, + MessageEvent, + UseGuards, + Req, + Get, +} from "@nestjs/common"; import { Observable } from "rxjs"; import { Request } from "express"; @@ -11,6 +18,7 @@ import { AppAbility } from "src/casl/casl-ability.factory"; import { CheckPolicies } from "src/casl/decorators/check-policies.decorator"; import { DatasetClass } from "src/datasets/schemas/dataset.schema"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; +import { RuntimeConfig } from "src/config/runtime-config/schemas/runtime-config.schema"; @Controller("events") @ApiBearerAuth() @@ -29,4 +37,15 @@ export class EventsController { .getEvents(user) .pipe(map((payload) => ({ data: payload }))); } + + @Get("connections") + @UseGuards(PoliciesGuard) + @CheckPolicies( + "runtimeconfig", + (ability: AppAbility) => + ability.can(Action.RuntimeConfigUpdateEndpoint, RuntimeConfig), //TODO: define a correct policy for monitoring connections + ) + connections() { + return this.eventsService.getAllConnections(); + } } diff --git a/src/serverSideEvent/serverSideEvent.service.ts b/src/serverSideEvent/serverSideEvent.service.ts index e72e6f8e7..0acbfe7b1 100644 --- a/src/serverSideEvent/serverSideEvent.service.ts +++ b/src/serverSideEvent/serverSideEvent.service.ts @@ -1,37 +1,72 @@ -import { Injectable } from "@nestjs/common"; -import { Subject, Observable } from "rxjs"; +import { ForbiddenException, Injectable } from "@nestjs/common"; +import { Subject, Observable, finalize } from "rxjs"; import { MessageEvent } from "@nestjs/common"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; +import { randomUUID } from "crypto"; + +interface HasAccessGroups { + ownerGroup?: string; + accessGroups?: string[]; +} @Injectable() export class EventsService { - private clients = new Map }>(); + private readonly MAX_CONNECTIONS_PER_USER = 5; + private clients = new Map< + string, + { user: JWTUser; subject: Subject } + >(); getEvents(user: JWTUser): Observable { - const subject = new Subject(); + const userConnectionCount = [...this.clients.values()].filter( + (c) => c.user._id === user._id, + ).length; - this.clients.set(user, { + if (userConnectionCount >= this.MAX_CONNECTIONS_PER_USER) { + throw new ForbiddenException( + "Maximum number of open connections reached", + ); + } + + const subject = new Subject(); + const connectionId = `${user._id}-${randomUUID()}`; + this.clients.set(connectionId, { + user, subject, }); - return subject.asObservable(); + return subject.asObservable().pipe( + finalize(() => { + this.clients.delete(connectionId); + }), + ); } - emit(event: { - ownerGroup: string; - accessGroups: string[]; - message: string; - type: string; - }) { - for (const [user, { subject }] of this.clients) { + emit(event: { message: HasAccessGroups; type: string }) { + for (const [, { user, subject }] of this.clients) { const userGroups = user.currentGroups ?? []; + const instanceOnwerGroup = event.message.ownerGroup ?? ""; + const instanceAccessGroups = event.message.accessGroups ?? []; - const canSee = - userGroups.includes(event.ownerGroup) || - event.accessGroups.some((g) => userGroups.includes(g)) || + const canAccess = + userGroups.includes(instanceOnwerGroup) || + instanceAccessGroups.some((g) => userGroups.includes(g)) || userGroups.includes("admin"); - if (canSee) { + + if (canAccess) { subject.next({ type: event.type, data: event.message }); } } } + + getAllConnections() { + const counts = new Map(); + + for (const { user } of this.clients.values()) { + counts.set(user.username, (counts.get(user.username) ?? 0) + 1); + } + return { + connections: this.clients.size, + users: Object.fromEntries(counts), + }; + } } From 5c88e8e1428b5eeb474ce81f33a99a2a76244e97 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 15 Jun 2026 16:49:58 +0200 Subject: [PATCH 07/34] move event interceptor from controller layer down to each endpoint --- src/common/interceptors/eventEmit.Interceptor.ts | 15 ++++++++++++--- src/datasets/datasets.controller.ts | 7 +++++-- src/datasets/datasets.v4.controller.ts | 7 +++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/common/interceptors/eventEmit.Interceptor.ts b/src/common/interceptors/eventEmit.Interceptor.ts index 5aef93965..b2c3f3917 100644 --- a/src/common/interceptors/eventEmit.Interceptor.ts +++ b/src/common/interceptors/eventEmit.Interceptor.ts @@ -10,6 +10,13 @@ import { Observable, tap } from "rxjs"; import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; +export const EVENT_METHODS: Record = { + POST: "dataset.created", + // PATCH: "dataset.updated", + // PUT: "dataset.updated", + // DELETE: "dataset.deleted", +}; + export const EventEmitInterceptor = ( eventType: string, ): Type => { @@ -21,13 +28,15 @@ export const EventEmitInterceptor = ( context: ExecutionContext, next: CallHandler, ): Observable { - const method = context.switchToHttp().getRequest().method; - return next.handle().pipe( tap((responseData: OutputDatasetDto) => { - if (["POST", "PATCH", "PUT", "DELETE"].includes(method)) { + // "POST", "PATCH", "PUT", "DELETE" are the methods that trigger events for + // dataset creation, update and deletion. We can extend this list in the future if needed. + if (eventType) { this.eventsService.emit({ message: responseData, + // TODO: should have pre-defined event types instead of passing as parameter, + // to avoid typos and make it easier to maintain type: eventType, }); } diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 24b83b21d..b89a194c1 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,7 +121,10 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; -import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Interceptor"; +import { + EVENT_METHODS, + EventEmitInterceptor, +} from "src/common/interceptors/eventEmit.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -132,7 +135,6 @@ import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Intercep TechniqueClass, RelationshipClass, ) -@UseInterceptors(EventEmitInterceptor("dataset.updated")) @ApiTags("datasets") @Controller({ path: "datasets", version: "3" }) export class DatasetsController { @@ -547,6 +549,7 @@ export class DatasetsController { new UTCTimeInterceptor(["creationTime"]), new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), + EventEmitInterceptor(EVENT_METHODS.POST), ) @UsePipes(ScientificMetadataValidationPipe) @Post() diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index 34ec234fb..c2d74e456 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,7 +90,10 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; -import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Interceptor"; +import { + EVENT_METHODS, + EventEmitInterceptor, +} from "src/common/interceptors/eventEmit.Interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -99,7 +102,6 @@ import { EventEmitInterceptor } from "src/common/interceptors/eventEmit.Intercep TechniqueClass, RelationshipClass, ) -@UseInterceptors(EventEmitInterceptor("dataset.updated")) @ApiTags("datasets v4") /* NOTE: Generated SDK method names include "V4" twice: * - From the controller class name (DatasetsV4Controller) @@ -255,6 +257,7 @@ export class DatasetsV4Controller { new UTCTimeInterceptor(["creationTime"]), new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), + EventEmitInterceptor(EVENT_METHODS.POST), ) @UsePipes(ScientificMetadataValidationPipe) @Post() From 6c3f0a12443c04409c08ab661256ab38bc391843 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 16 Jun 2026 13:25:17 +0200 Subject: [PATCH 08/34] added decorator and some minor refactor --- src/app.module.ts | 4 +- .../interceptors/eventEmit.Interceptor.ts | 49 ------------------- src/datasets/datasets.controller.ts | 8 ++- src/datasets/datasets.v4.controller.ts | 8 ++- .../decorators/sse.decorator.ts | 6 +++ .../interceptors/sse.interceptor.ts | 44 +++++++++++++++++ .../sse.controller.ts} | 14 +++--- src/serverSentEvent/sse.module.ts | 13 +++++ .../sse.service.ts} | 6 +-- src/serverSideEvent/serverSideEvent.module.ts | 13 ----- 10 files changed, 81 insertions(+), 84 deletions(-) delete mode 100644 src/common/interceptors/eventEmit.Interceptor.ts create mode 100644 src/serverSentEvent/decorators/sse.decorator.ts create mode 100644 src/serverSentEvent/interceptors/sse.interceptor.ts rename src/{serverSideEvent/serverSideEvent.controller.ts => serverSentEvent/sse.controller.ts} (81%) create mode 100644 src/serverSentEvent/sse.module.ts rename src/{serverSideEvent/serverSideEvent.service.ts => serverSentEvent/sse.service.ts} (93%) delete mode 100644 src/serverSideEvent/serverSideEvent.module.ts diff --git a/src/app.module.ts b/src/app.module.ts index a8d4b6f54..8accc110e 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -45,7 +45,7 @@ import { RuntimeConfigModule } from "./config/runtime-config/runtime-config.modu import { MetadataKeysModule } from "./metadata-keys/metadatakeys.module"; import { OidcClientModule } from "./common/openid-client/openid-client.module"; import { ThrottlerModule } from "@nestjs/throttler"; -import { EventsModule } from "./serverSideEvent/serverSideEvent.module"; +import { SseModule } from "./serverSentEvent/sse.module"; import type { MailerOptions } from "@nestjs-modules/mailer"; @Module({ @@ -56,7 +56,7 @@ import type { MailerOptions } from "@nestjs-modules/mailer"; cache: true, }), AuthModule, - EventsModule, + SseModule, OidcClientModule, CaslModule, AttachmentsModule, diff --git a/src/common/interceptors/eventEmit.Interceptor.ts b/src/common/interceptors/eventEmit.Interceptor.ts deleted file mode 100644 index b2c3f3917..000000000 --- a/src/common/interceptors/eventEmit.Interceptor.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - Injectable, - NestInterceptor, - ExecutionContext, - CallHandler, - Type, - mixin, -} from "@nestjs/common"; -import { Observable, tap } from "rxjs"; -import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; -import { EventsService } from "src/serverSideEvent/serverSideEvent.service"; - -export const EVENT_METHODS: Record = { - POST: "dataset.created", - // PATCH: "dataset.updated", - // PUT: "dataset.updated", - // DELETE: "dataset.deleted", -}; - -export const EventEmitInterceptor = ( - eventType: string, -): Type => { - @Injectable() - class MixinEventEmitInterceptor implements NestInterceptor { - constructor(public readonly eventsService: EventsService) {} - - intercept( - context: ExecutionContext, - next: CallHandler, - ): Observable { - return next.handle().pipe( - tap((responseData: OutputDatasetDto) => { - // "POST", "PATCH", "PUT", "DELETE" are the methods that trigger events for - // dataset creation, update and deletion. We can extend this list in the future if needed. - if (eventType) { - this.eventsService.emit({ - message: responseData, - // TODO: should have pre-defined event types instead of passing as parameter, - // to avoid typos and make it easier to maintain - type: eventType, - }); - } - }), - ); - } - } - - return mixin(MixinEventEmitInterceptor); -}; diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index b89a194c1..c24918bdc 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,10 +121,8 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; -import { - EVENT_METHODS, - EventEmitInterceptor, -} from "src/common/interceptors/eventEmit.Interceptor"; +import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; +import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -549,8 +547,8 @@ export class DatasetsController { new UTCTimeInterceptor(["creationTime"]), new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), - EventEmitInterceptor(EVENT_METHODS.POST), ) + @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @ApiOperation({ diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index c2d74e456..ce0d11659 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,10 +90,8 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; -import { - EVENT_METHODS, - EventEmitInterceptor, -} from "src/common/interceptors/eventEmit.Interceptor"; +import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; +import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; @ApiBearerAuth() @ApiExtraModels( @@ -257,8 +255,8 @@ export class DatasetsV4Controller { new UTCTimeInterceptor(["creationTime"]), new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), - EventEmitInterceptor(EVENT_METHODS.POST), ) + @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @UseInterceptors(ClassSerializerInterceptor) diff --git a/src/serverSentEvent/decorators/sse.decorator.ts b/src/serverSentEvent/decorators/sse.decorator.ts new file mode 100644 index 000000000..68704e465 --- /dev/null +++ b/src/serverSentEvent/decorators/sse.decorator.ts @@ -0,0 +1,6 @@ +import { applyDecorators, UseInterceptors } from "@nestjs/common"; +import { SseEventType, SseInterceptor } from "../interceptors/sse.interceptor"; + +export const EmitSse = (eventType: SseEventType) => { + return applyDecorators(UseInterceptors(SseInterceptor(eventType))); +}; diff --git a/src/serverSentEvent/interceptors/sse.interceptor.ts b/src/serverSentEvent/interceptors/sse.interceptor.ts new file mode 100644 index 000000000..9fbca66fc --- /dev/null +++ b/src/serverSentEvent/interceptors/sse.interceptor.ts @@ -0,0 +1,44 @@ +import { + Injectable, + NestInterceptor, + ExecutionContext, + CallHandler, + Type, + mixin, +} from "@nestjs/common"; +import { Observable, tap } from "rxjs"; +import { HasAccessGroups, SseService } from "src/serverSentEvent/sse.service"; + +export const EVENT_METHODS: Record = { + DATASET_CREATED: "dataset.created", + // PROPOSAL_CREATED: "proposal.created", + // SAMPLE_CREATED: "sample.created", + // INSTRUMENT_CREATED: "instrument.created", +} as const; + +export type SseEventType = (typeof EVENT_METHODS)[keyof typeof EVENT_METHODS]; + +export const SseInterceptor = ( + eventType: SseEventType, +): Type => { + @Injectable() + class MixinSseInterceptor implements NestInterceptor { + constructor(public readonly sseService: SseService) {} + + intercept( + context: ExecutionContext, + next: CallHandler, + ): Observable { + return next.handle().pipe( + tap((responseData: HasAccessGroups) => { + this.sseService.emit({ + message: responseData, + type: eventType, + }); + }), + ); + } + } + + return mixin(MixinSseInterceptor); +}; diff --git a/src/serverSideEvent/serverSideEvent.controller.ts b/src/serverSentEvent/sse.controller.ts similarity index 81% rename from src/serverSideEvent/serverSideEvent.controller.ts rename to src/serverSentEvent/sse.controller.ts index 231448da8..bba5edb50 100644 --- a/src/serverSideEvent/serverSideEvent.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -10,8 +10,7 @@ import { Observable } from "rxjs"; import { Request } from "express"; import { map } from "rxjs/operators"; -import { EventsService } from "./serverSideEvent.service"; -import { ApiBearerAuth } from "@nestjs/swagger"; +import { ApiBearerAuth, ApiTags } from "@nestjs/swagger"; import { PoliciesGuard } from "src/casl/guards/policies.guard"; import { Action } from "src/casl/action.enum"; import { AppAbility } from "src/casl/casl-ability.factory"; @@ -19,13 +18,14 @@ import { CheckPolicies } from "src/casl/decorators/check-policies.decorator"; import { DatasetClass } from "src/datasets/schemas/dataset.schema"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { RuntimeConfig } from "src/config/runtime-config/schemas/runtime-config.schema"; +import { SseService } from "./sse.service"; +@ApiTags("events") @Controller("events") @ApiBearerAuth() -export class EventsController { - constructor(private readonly eventsService: EventsService) {} +export class SseController { + constructor(private readonly sseService: SseService) {} - // SSE stream endpoint @Sse("stream") @UseGuards(PoliciesGuard) @CheckPolicies("datasets", (ability: AppAbility) => @@ -33,7 +33,7 @@ export class EventsController { ) stream(@Req() request: Request): Observable { const user = request.user as JWTUser; - return this.eventsService + return this.sseService .getEvents(user) .pipe(map((payload) => ({ data: payload }))); } @@ -46,6 +46,6 @@ export class EventsController { ability.can(Action.RuntimeConfigUpdateEndpoint, RuntimeConfig), //TODO: define a correct policy for monitoring connections ) connections() { - return this.eventsService.getAllConnections(); + return this.sseService.getAllConnections(); } } diff --git a/src/serverSentEvent/sse.module.ts b/src/serverSentEvent/sse.module.ts new file mode 100644 index 000000000..c982a3afd --- /dev/null +++ b/src/serverSentEvent/sse.module.ts @@ -0,0 +1,13 @@ +import { Global, Module } from "@nestjs/common"; +import { SseService } from "./sse.service"; +import { CaslModule } from "src/casl/casl.module"; +import { SseController } from "./sse.controller"; + +@Global() +@Module({ + imports: [CaslModule], + controllers: [SseController], + providers: [SseService], + exports: [SseService], +}) +export class SseModule {} diff --git a/src/serverSideEvent/serverSideEvent.service.ts b/src/serverSentEvent/sse.service.ts similarity index 93% rename from src/serverSideEvent/serverSideEvent.service.ts rename to src/serverSentEvent/sse.service.ts index 0acbfe7b1..dead144a4 100644 --- a/src/serverSideEvent/serverSideEvent.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -4,13 +4,13 @@ import { MessageEvent } from "@nestjs/common"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { randomUUID } from "crypto"; -interface HasAccessGroups { +export interface HasAccessGroups { ownerGroup?: string; accessGroups?: string[]; } @Injectable() -export class EventsService { +export class SseService { private readonly MAX_CONNECTIONS_PER_USER = 5; private clients = new Map< string, @@ -24,7 +24,7 @@ export class EventsService { if (userConnectionCount >= this.MAX_CONNECTIONS_PER_USER) { throw new ForbiddenException( - "Maximum number of open connections reached", + `Maximum number of ${this.MAX_CONNECTIONS_PER_USER} open connections reached`, ); } diff --git a/src/serverSideEvent/serverSideEvent.module.ts b/src/serverSideEvent/serverSideEvent.module.ts deleted file mode 100644 index 8c2c3d1ef..000000000 --- a/src/serverSideEvent/serverSideEvent.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Global, Module } from "@nestjs/common"; -import { EventsController } from "./serverSideEvent.controller"; -import { EventsService } from "./serverSideEvent.service"; -import { CaslModule } from "src/casl/casl.module"; - -@Global() -@Module({ - imports: [CaslModule], - controllers: [EventsController], - providers: [EventsService], - exports: [EventsService], -}) -export class EventsModule {} From 217797376f83225ecffbf471caff4ccbbfa8a086 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Fri, 19 Jun 2026 10:05:08 +0200 Subject: [PATCH 09/34] minor typo --- src/serverSentEvent/sse.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index dead144a4..eec6edd21 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -44,11 +44,11 @@ export class SseService { emit(event: { message: HasAccessGroups; type: string }) { for (const [, { user, subject }] of this.clients) { const userGroups = user.currentGroups ?? []; - const instanceOnwerGroup = event.message.ownerGroup ?? ""; + const instanceOwnerGroup = event.message.ownerGroup ?? ""; const instanceAccessGroups = event.message.accessGroups ?? []; const canAccess = - userGroups.includes(instanceOnwerGroup) || + userGroups.includes(instanceOwnerGroup) || instanceAccessGroups.some((g) => userGroups.includes(g)) || userGroups.includes("admin"); From f72fb8ceae92893ee72ceb8a3c919d225b10b256 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Fri, 19 Jun 2026 13:29:53 +0200 Subject: [PATCH 10/34] change MAX_CONNECTION_PER_USER to MAX_CONNECTIONS_PER_USER_PER_INSTANCE avoid confusion --- src/serverSentEvent/sse.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index eec6edd21..d6e067f2d 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -11,7 +11,7 @@ export interface HasAccessGroups { @Injectable() export class SseService { - private readonly MAX_CONNECTIONS_PER_USER = 5; + private readonly MAX_CONNECTIONS_PER_USER_PER_INSTANCE = 5; private clients = new Map< string, { user: JWTUser; subject: Subject } @@ -22,9 +22,9 @@ export class SseService { (c) => c.user._id === user._id, ).length; - if (userConnectionCount >= this.MAX_CONNECTIONS_PER_USER) { + if (userConnectionCount >= this.MAX_CONNECTIONS_PER_USER_PER_INSTANCE) { throw new ForbiddenException( - `Maximum number of ${this.MAX_CONNECTIONS_PER_USER} open connections reached`, + `Maximum number of ${this.MAX_CONNECTIONS_PER_USER_PER_INSTANCE} open connections reached`, ); } From e1de928caa2c723a9b23cf5a9cbf746619c6f257 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 22 Jun 2026 15:53:46 +0200 Subject: [PATCH 11/34] added sse listener to watch mongodb changestream --- CI/E2E/docker-compose-local.yaml | 12 +++ CI/E2E/docker-compose.yaml | 18 +++-- src/datasets/datasets.controller.ts | 3 - src/datasets/datasets.v4.controller.ts | 3 - .../decorators/sse.decorator.ts | 6 -- .../interceptors/sse.interceptor.ts | 44 ---------- .../interfaces/sse-registry.interface.ts | 13 +++ src/serverSentEvent/sse.listener.ts | 81 +++++++++++++++++++ src/serverSentEvent/sse.module.ts | 3 +- src/serverSentEvent/sse.service.ts | 19 ++++- 10 files changed, 136 insertions(+), 66 deletions(-) delete mode 100644 src/serverSentEvent/decorators/sse.decorator.ts delete mode 100644 src/serverSentEvent/interceptors/sse.interceptor.ts create mode 100644 src/serverSentEvent/interfaces/sse-registry.interface.ts create mode 100644 src/serverSentEvent/sse.listener.ts diff --git a/CI/E2E/docker-compose-local.yaml b/CI/E2E/docker-compose-local.yaml index e8ce2a091..323bf6c05 100644 --- a/CI/E2E/docker-compose-local.yaml +++ b/CI/E2E/docker-compose-local.yaml @@ -2,10 +2,22 @@ version: "3.2" services: mongodb: image: mongo:latest + command: ["--replSet", "rs0", "--bind_ip_all"] + environment: + MONGO_RS_HOST: "localhost:27017" volumes: - "mongodb_data:/data/db" ports: - "27017:27017" + healthcheck: + test: > + mongosh --quiet --eval " + try { rs.status().ok } + catch { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: '$$MONGO_RS_HOST' }] }) } + " + interval: 10s + start_period: 40s + retries: 5 scichat-loopback: image: dacat/scichat-loopback:e2e diff --git a/CI/E2E/docker-compose.yaml b/CI/E2E/docker-compose.yaml index f5d10923d..fc0c4a4ea 100644 --- a/CI/E2E/docker-compose.yaml +++ b/CI/E2E/docker-compose.yaml @@ -8,17 +8,23 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock mongodb: - image: "mongo:latest" - ports: - - "27017:27017" + image: mongo:latest + command: ["--replSet", "rs0", "--bind_ip_all"] + environment: + MONGO_RS_HOST: "localhost:27017" volumes: - "mongodb_data:/data/db" + ports: + - "27017:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh mongodb:27017/test --quiet + test: > + mongosh --quiet --eval " + try { rs.status().ok } + catch { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: '$$MONGO_RS_HOST' }] }) } + " interval: 10s - timeout: 10s - retries: 5 start_period: 40s + retries: 5 scichat-loopback: image: dacat/scichat-loopback:e2e command: diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index c24918bdc..47029504c 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,8 +121,6 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; -import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; -import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -548,7 +546,6 @@ export class DatasetsController { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) - @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @ApiOperation({ diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index ce0d11659..a8574be7a 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,8 +90,6 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; -import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; -import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; @ApiBearerAuth() @ApiExtraModels( @@ -256,7 +254,6 @@ export class DatasetsV4Controller { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) - @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @UseInterceptors(ClassSerializerInterceptor) diff --git a/src/serverSentEvent/decorators/sse.decorator.ts b/src/serverSentEvent/decorators/sse.decorator.ts deleted file mode 100644 index 68704e465..000000000 --- a/src/serverSentEvent/decorators/sse.decorator.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { applyDecorators, UseInterceptors } from "@nestjs/common"; -import { SseEventType, SseInterceptor } from "../interceptors/sse.interceptor"; - -export const EmitSse = (eventType: SseEventType) => { - return applyDecorators(UseInterceptors(SseInterceptor(eventType))); -}; diff --git a/src/serverSentEvent/interceptors/sse.interceptor.ts b/src/serverSentEvent/interceptors/sse.interceptor.ts deleted file mode 100644 index 9fbca66fc..000000000 --- a/src/serverSentEvent/interceptors/sse.interceptor.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { - Injectable, - NestInterceptor, - ExecutionContext, - CallHandler, - Type, - mixin, -} from "@nestjs/common"; -import { Observable, tap } from "rxjs"; -import { HasAccessGroups, SseService } from "src/serverSentEvent/sse.service"; - -export const EVENT_METHODS: Record = { - DATASET_CREATED: "dataset.created", - // PROPOSAL_CREATED: "proposal.created", - // SAMPLE_CREATED: "sample.created", - // INSTRUMENT_CREATED: "instrument.created", -} as const; - -export type SseEventType = (typeof EVENT_METHODS)[keyof typeof EVENT_METHODS]; - -export const SseInterceptor = ( - eventType: SseEventType, -): Type => { - @Injectable() - class MixinSseInterceptor implements NestInterceptor { - constructor(public readonly sseService: SseService) {} - - intercept( - context: ExecutionContext, - next: CallHandler, - ): Observable { - return next.handle().pipe( - tap((responseData: HasAccessGroups) => { - this.sseService.emit({ - message: responseData, - type: eventType, - }); - }), - ); - } - } - - return mixin(MixinSseInterceptor); -}; diff --git a/src/serverSentEvent/interfaces/sse-registry.interface.ts b/src/serverSentEvent/interfaces/sse-registry.interface.ts new file mode 100644 index 000000000..3e15f7e68 --- /dev/null +++ b/src/serverSentEvent/interfaces/sse-registry.interface.ts @@ -0,0 +1,13 @@ +export type Collection = + | "Attachment" + | "RuntimeConfig" + | "Dataset" + | "Proposal" + | "Sample" + | "PublishedData" + | "MetadataKeys" + | "Datablock" + | "Instrument" + | "Job" + | "OrigDatablock" + | "History"; diff --git a/src/serverSentEvent/sse.listener.ts b/src/serverSentEvent/sse.listener.ts new file mode 100644 index 000000000..c277828fb --- /dev/null +++ b/src/serverSentEvent/sse.listener.ts @@ -0,0 +1,81 @@ +// src/serverSentEvent/sse.listener.ts +import { + Injectable, + Logger, + OnModuleInit, + OnModuleDestroy, +} from "@nestjs/common"; +import { InjectConnection } from "@nestjs/mongoose"; +import { Connection } from "mongoose"; +import { ChangeStream, ChangeStreamDocument } from "mongodb"; +import { ClassConstructor, plainToInstance } from "class-transformer"; +import { SseService } from "./sse.service"; + +import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; +import { Collection } from "./interfaces/sse-registry.interface"; + +// Registry to define which collections and operations should be listened to, +// and how to transform the raw MongoDB documents into DTOs for SSE messages. +const REGISTRY: Partial< + Record }> +> = { + Dataset: { entity: "Dataset", dto: OutputDatasetDto }, +}; + +// Map MongoDB operations to event action names. Only listed operations are processed. +const ACTION_BY_OPERATION: Record = { + insert: "created", + // update: "updated", + // replace: "updated", + // delete: "deleted", +}; + +@Injectable() +export class SseListener implements OnModuleInit, OnModuleDestroy { + private changeStream: ChangeStream; + + constructor( + @InjectConnection() private readonly connection: Connection, + private readonly sseService: SseService, + ) {} + + onModuleInit() { + this.changeStream = this.connection.watch([ + { + $match: { + "ns.coll": { $in: Object.keys(REGISTRY) }, + operationType: { $in: Object.keys(ACTION_BY_OPERATION) }, + }, + }, + ]); + + this.changeStream.on("change", (change) => this.onStream(change)); + this.changeStream.on("error", (err) => + Logger.error(`SSE change stream is closed due to error: ${err}`), + ); + } + + private onStream(change: ChangeStreamDocument) { + if (!("ns" in change) || !("coll" in change.ns)) { + return; + } + const config = REGISTRY[change.ns.coll as Collection]; + const action = ACTION_BY_OPERATION[change.operationType]; + + if (!config || !action) return; + + const rawDoc = "fullDocument" in change ? change.fullDocument : null; + + if (!rawDoc) return; + + this.sseService.emit({ + message: plainToInstance(config.dto, rawDoc), + action: action, + entity: config.entity, + }); + } + + onModuleDestroy() { + return this.changeStream?.close(); + } +} diff --git a/src/serverSentEvent/sse.module.ts b/src/serverSentEvent/sse.module.ts index c982a3afd..c1849b360 100644 --- a/src/serverSentEvent/sse.module.ts +++ b/src/serverSentEvent/sse.module.ts @@ -2,12 +2,13 @@ import { Global, Module } from "@nestjs/common"; import { SseService } from "./sse.service"; import { CaslModule } from "src/casl/casl.module"; import { SseController } from "./sse.controller"; +import { SseListener } from "./sse.listener"; @Global() @Module({ imports: [CaslModule], controllers: [SseController], - providers: [SseService], + providers: [SseService, SseListener], exports: [SseService], }) export class SseModule {} diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index d6e067f2d..ad9bad01a 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -3,6 +3,8 @@ import { Subject, Observable, finalize } from "rxjs"; import { MessageEvent } from "@nestjs/common"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { randomUUID } from "crypto"; +import { ConfigService } from "@nestjs/config"; +import { AccessGroupsType } from "src/config/configuration"; export interface HasAccessGroups { ownerGroup?: string; @@ -16,6 +18,12 @@ export class SseService { string, { user: JWTUser; subject: Subject } >(); + private accessGroups; + + constructor(private configService: ConfigService) { + this.accessGroups = + this.configService.get("accessGroups"); + } getEvents(user: JWTUser): Observable { const userConnectionCount = [...this.clients.values()].filter( @@ -41,19 +49,24 @@ export class SseService { ); } - emit(event: { message: HasAccessGroups; type: string }) { + emit(event: { message: HasAccessGroups; action: string; entity: string }) { for (const [, { user, subject }] of this.clients) { const userGroups = user.currentGroups ?? []; const instanceOwnerGroup = event.message.ownerGroup ?? ""; const instanceAccessGroups = event.message.accessGroups ?? []; + // TODO: this logic is duplicated from CaslAbilityFactory, consider centralizing it + // after the refacor of CaslAbilityFactory is done const canAccess = userGroups.includes(instanceOwnerGroup) || instanceAccessGroups.some((g) => userGroups.includes(g)) || - userGroups.includes("admin"); + userGroups.some((g) => this.accessGroups?.admin?.includes(g)); if (canAccess) { - subject.next({ type: event.type, data: event.message }); + subject.next({ + type: `${event.entity}.${event.action}`, + data: event.message, + }); } } } From 6a64f6935e38ff8201c16424feacb126af8d32cd Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 23 Jun 2026 18:21:20 +0200 Subject: [PATCH 12/34] disable sse service if db is not replica set --- src/serverSentEvent/sse.listener.ts | 22 +++++++++++++++++++++- src/serverSentEvent/sse.service.ts | 13 ++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/serverSentEvent/sse.listener.ts b/src/serverSentEvent/sse.listener.ts index c277828fb..635f3a4e8 100644 --- a/src/serverSentEvent/sse.listener.ts +++ b/src/serverSentEvent/sse.listener.ts @@ -39,7 +39,18 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { private readonly sseService: SseService, ) {} - onModuleInit() { + async onModuleInit() { + const isReplicaSet = await this.isDbReplicaSet(); + + if (!isReplicaSet) { + Logger.debug( + "MongoDB is not running as a replica set. SSE change streams are disabled.", + ); + return; + } + + this.sseService.sseEnabled = isReplicaSet; + this.changeStream = this.connection.watch([ { $match: { @@ -75,6 +86,15 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { }); } + private async isDbReplicaSet(): Promise { + try { + const info = await this.connection.db?.admin().command({ check: 1 }); + return Boolean(info?.setName); + } catch { + return false; + } + } + onModuleDestroy() { return this.changeStream?.close(); } diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index ad9bad01a..a97dbd7a9 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -1,4 +1,8 @@ -import { ForbiddenException, Injectable } from "@nestjs/common"; +import { + ForbiddenException, + Injectable, + ServiceUnavailableException, +} from "@nestjs/common"; import { Subject, Observable, finalize } from "rxjs"; import { MessageEvent } from "@nestjs/common"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; @@ -19,6 +23,7 @@ export class SseService { { user: JWTUser; subject: Subject } >(); private accessGroups; + public sseEnabled = false; constructor(private configService: ConfigService) { this.accessGroups = @@ -26,6 +31,9 @@ export class SseService { } getEvents(user: JWTUser): Observable { + if (!this.sseEnabled) { + throw new ServiceUnavailableException("SSE are not enabled."); + } const userConnectionCount = [...this.clients.values()].filter( (c) => c.user._id === user._id, ).length; @@ -72,6 +80,9 @@ export class SseService { } getAllConnections() { + if (!this.sseEnabled) { + throw new ServiceUnavailableException("SSE are not enabled."); + } const counts = new Map(); for (const { user } of this.clients.values()) { From fe6eb0946515f3422aef0ac5504adcc73d29a86e Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 24 Jun 2026 14:18:21 +0200 Subject: [PATCH 13/34] added enableRealTimeUpdates frontend config --- src/config/frontend.config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/frontend.config.json b/src/config/frontend.config.json index 44e714f8c..ad69e891f 100644 --- a/src/config/frontend.config.json +++ b/src/config/frontend.config.json @@ -5,6 +5,7 @@ }, "statusBannerMessage": "", "statusBannerCode": "INFO", + "enableRealTimeUpdates": true, "autoApplyFilters": false, "accessTokenPrefix": "Bearer ", "addDatasetEnabled": false, From 0b35844e8c81d2d302a2bbd09cdc0166460c7f1c Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 24 Jun 2026 14:26:37 +0200 Subject: [PATCH 14/34] frontend config name change for realTimeUpdatesEnabled --- src/config/frontend.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/frontend.config.json b/src/config/frontend.config.json index ad69e891f..5934b9814 100644 --- a/src/config/frontend.config.json +++ b/src/config/frontend.config.json @@ -5,7 +5,7 @@ }, "statusBannerMessage": "", "statusBannerCode": "INFO", - "enableRealTimeUpdates": true, + "realTimeUpdatesEnabled": true, "autoApplyFilters": false, "accessTokenPrefix": "Bearer ", "addDatasetEnabled": false, From af2ffbb2c0795699da7da1c0bd06705fa53646be Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 24 Jun 2026 14:28:53 +0200 Subject: [PATCH 15/34] correct param for isDbReplicaSet check --- src/serverSentEvent/sse.listener.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/serverSentEvent/sse.listener.ts b/src/serverSentEvent/sse.listener.ts index 635f3a4e8..a0eedb523 100644 --- a/src/serverSentEvent/sse.listener.ts +++ b/src/serverSentEvent/sse.listener.ts @@ -42,6 +42,7 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { async onModuleInit() { const isReplicaSet = await this.isDbReplicaSet(); + console.log("isReplicaSet", isReplicaSet); if (!isReplicaSet) { Logger.debug( "MongoDB is not running as a replica set. SSE change streams are disabled.", @@ -88,7 +89,7 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { private async isDbReplicaSet(): Promise { try { - const info = await this.connection.db?.admin().command({ check: 1 }); + const info = await this.connection.db?.admin().command({ hello: 1 }); return Boolean(info?.setName); } catch { return false; From 403738c139f751dea09b60e2c807a4f20336acfa Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 6 Jul 2026 12:07:10 +0200 Subject: [PATCH 16/34] JwtStrategy should accept query token only when it is events/stream endpoint --- src/auth/strategies/jwt.strategy.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 5dcdca4b0..08154e878 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -1,12 +1,21 @@ import { ExtractJwt, Strategy } from "passport-jwt"; import { PassportStrategy } from "@nestjs/passport"; import { Injectable } from "@nestjs/common"; +import { Request } from "express"; import { RolesService } from "src/users/roles.service"; import { UsersService } from "src/users/users.service"; import { JWTUser } from "../interfaces/jwt-user.interface"; import { User } from "src/users/schemas/user.schema"; import { ConfigService } from "@nestjs/config"; +const fromSseQueryAsBearerToken = (req: Request): string | null => { + if (req.path.endsWith("events/stream")) { + const token = req.query?.token; + return typeof token === "string" ? token : null; + } + return null; +}; + @Injectable() export class JwtStrategy extends PassportStrategy(Strategy) { constructor( @@ -17,7 +26,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) { super({ jwtFromRequest: ExtractJwt.fromExtractors([ ExtractJwt.fromAuthHeaderAsBearerToken(), - (req) => req?.query?.token || null, + fromSseQueryAsBearerToken, ]), ignoreExpiration: false, secretOrKey: configService.get("jwt.secret") || "defaultSecret", From d15322604131cab6d0e0bf1cb81dcb67c3d04a5f Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 6 Jul 2026 12:12:40 +0200 Subject: [PATCH 17/34] throw error when jwt.secret is missing on production instead of using defaut secret --- src/auth/strategies/jwt.strategy.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 08154e878..7c15eb5d9 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -29,7 +29,13 @@ export class JwtStrategy extends PassportStrategy(Strategy) { fromSseQueryAsBearerToken, ]), ignoreExpiration: false, - secretOrKey: configService.get("jwt.secret") || "defaultSecret", + secretOrKey: + configService.get("jwt.secret") ?? + (process.env.NODE_ENV === "production" + ? (() => { + throw new Error("jwt.secret is required"); + })() + : "defaultSecret"), }); } From d274665612da7d44c4fe0607e6bd8d12dc699113 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 6 Jul 2026 12:19:03 +0200 Subject: [PATCH 18/34] minor refactor for jwt secret is required in production logic --- src/auth/strategies/jwt.strategy.ts | 8 +------- src/config/configuration.ts | 6 +++++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 7c15eb5d9..08154e878 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -29,13 +29,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) { fromSseQueryAsBearerToken, ]), ignoreExpiration: false, - secretOrKey: - configService.get("jwt.secret") ?? - (process.env.NODE_ENV === "production" - ? (() => { - throw new Error("jwt.secret is required"); - })() - : "defaultSecret"), + secretOrKey: configService.get("jwt.secret") || "defaultSecret", }); } diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 6e1bb03c3..d8a6fc97c 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -5,6 +5,10 @@ import { DEFAULT_PROPOSAL_TYPE } from "src/proposals/schemas/proposal.schema"; import localconfiguration from "./localconfiguration"; const configuration = () => { + const jwtSecret = process.env.JWT_SECRET; + if (!jwtSecret && process.env.NODE_ENV === "production") { + throw new Error("JWT_SECRET is required in production"); + } const accessGroupsStaticValues = process.env.ACCESS_GROUPS_STATIC_VALUES || ""; const adminGroups = process.env.ADMIN_GROUPS || ""; @@ -326,7 +330,7 @@ const configuration = () => { httpMaxRedirects: process.env.HTTP_MAX_REDIRECTS ?? 5, httpTimeOut: process.env.HTTP_TIMEOUT ?? 5000, jwt: { - secret: process.env.JWT_SECRET, + secret: jwtSecret, expiresIn: parseInt(process.env.JWT_EXPIRES_IN ?? "3600", 10), neverExpires: process.env.JWT_NEVER_EXPIRES ?? "100y", }, From 463557b87c95ebdb75152888c0773b111c9a5294 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 6 Jul 2026 14:16:49 +0200 Subject: [PATCH 19/34] refactor: enhance SSE event handling and registry structure - Introduced WatchableCollection type for MongoDB collections in SSE. - Updated SseRegistryEntry interface to include entity and DTO mapping. - Improved SseListener to handle change streams with reconnection logic. - Enhanced SseService to manage client connections and access control. - Added SseClient and SseEvent interfaces for better type safety. --- .../interfaces/sse-event.interface.ts | 30 +++++ .../interfaces/sse-registry.interface.ts | 20 ++- src/serverSentEvent/sse.controller.ts | 26 +++- src/serverSentEvent/sse.listener.ts | 115 +++++++++++------ src/serverSentEvent/sse.service.ts | 121 +++++++++++------- 5 files changed, 223 insertions(+), 89 deletions(-) create mode 100644 src/serverSentEvent/interfaces/sse-event.interface.ts diff --git a/src/serverSentEvent/interfaces/sse-event.interface.ts b/src/serverSentEvent/interfaces/sse-event.interface.ts new file mode 100644 index 000000000..8cb146311 --- /dev/null +++ b/src/serverSentEvent/interfaces/sse-event.interface.ts @@ -0,0 +1,30 @@ +import { Subject } from "rxjs"; +import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; +import { MessageEvent } from "@nestjs/common"; + +export interface SseClient { + user: JWTUser; + subject: Subject; +} + +/** + * Event action names emitted over SSE. Only "created" is produced today + * (insert operations); "updated"/"deleted" are reserved for future use. + */ +export type SseAction = "created" | "updated" | "deleted"; + +export interface HasAccessGroups { + ownerGroup?: string; + accessGroups?: string[]; +} + +export interface SseEvent { + entity: string; + action: SseAction; + message: T; +} + +export interface SseConnectionsReport { + connections: number; + users: Record; +} diff --git a/src/serverSentEvent/interfaces/sse-registry.interface.ts b/src/serverSentEvent/interfaces/sse-registry.interface.ts index 3e15f7e68..59844d92f 100644 --- a/src/serverSentEvent/interfaces/sse-registry.interface.ts +++ b/src/serverSentEvent/interfaces/sse-registry.interface.ts @@ -1,4 +1,9 @@ -export type Collection = +import { ClassConstructor } from "class-transformer"; + +/** + * MongoDB collection names that the SSE listener may watch. + */ +export type WatchableCollection = | "Attachment" | "RuntimeConfig" | "Dataset" @@ -11,3 +16,16 @@ export type Collection = | "Job" | "OrigDatablock" | "History"; + +/** + * How a watched collection maps to an SSE entity name and the DTO used to + * serialize raw MongoDB documents before they are pushed to clients. + */ +export interface SseRegistryEntry { + entity: string; + dto: ClassConstructor; +} + +export type SseRegistry = Partial< + Record +>; diff --git a/src/serverSentEvent/sse.controller.ts b/src/serverSentEvent/sse.controller.ts index bba5edb50..2da5e89b7 100644 --- a/src/serverSentEvent/sse.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -9,8 +9,12 @@ import { import { Observable } from "rxjs"; import { Request } from "express"; -import { map } from "rxjs/operators"; -import { ApiBearerAuth, ApiTags } from "@nestjs/swagger"; +import { + ApiBearerAuth, + ApiOperation, + ApiQuery, + ApiTags, +} from "@nestjs/swagger"; import { PoliciesGuard } from "src/casl/guards/policies.guard"; import { Action } from "src/casl/action.enum"; import { AppAbility } from "src/casl/casl-ability.factory"; @@ -31,11 +35,20 @@ export class SseController { @CheckPolicies("datasets", (ability: AppAbility) => ability.can(Action.DatasetRead, DatasetClass), ) + @ApiOperation({ + summary: "Subscribe to server-sent events.", + description: + "Opens a text/event-stream connection pushing events for documents the authenticated user is allowed to read.", + }) + @ApiQuery({ + name: "token", + required: false, + description: + "JWT access token. Alternative to the Authorization header for EventSource clients that cannot set request headers.", + }) stream(@Req() request: Request): Observable { const user = request.user as JWTUser; - return this.sseService - .getEvents(user) - .pipe(map((payload) => ({ data: payload }))); + return this.sseService.getEvents(user); } @Get("connections") @@ -45,6 +58,9 @@ export class SseController { (ability: AppAbility) => ability.can(Action.RuntimeConfigUpdateEndpoint, RuntimeConfig), //TODO: define a correct policy for monitoring connections ) + @ApiOperation({ + summary: "List active SSE connections on this instance.", + }) connections() { return this.sseService.getAllConnections(); } diff --git a/src/serverSentEvent/sse.listener.ts b/src/serverSentEvent/sse.listener.ts index a0eedb523..8cffe9b2c 100644 --- a/src/serverSentEvent/sse.listener.ts +++ b/src/serverSentEvent/sse.listener.ts @@ -1,4 +1,3 @@ -// src/serverSentEvent/sse.listener.ts import { Injectable, Logger, @@ -8,31 +7,32 @@ import { import { InjectConnection } from "@nestjs/mongoose"; import { Connection } from "mongoose"; import { ChangeStream, ChangeStreamDocument } from "mongodb"; -import { ClassConstructor, plainToInstance } from "class-transformer"; +import { plainToInstance } from "class-transformer"; import { SseService } from "./sse.service"; import { OutputDatasetDto } from "src/datasets/dto/output-dataset.dto"; -import { Collection } from "./interfaces/sse-registry.interface"; +import { + SseRegistry, + WatchableCollection, +} from "./interfaces/sse-registry.interface"; +import { SseAction } from "./interfaces/sse-event.interface"; -// Registry to define which collections and operations should be listened to, -// and how to transform the raw MongoDB documents into DTOs for SSE messages. -const REGISTRY: Partial< - Record }> -> = { +const REGISTRY: SseRegistry = { Dataset: { entity: "Dataset", dto: OutputDatasetDto }, }; -// Map MongoDB operations to event action names. Only listed operations are processed. -const ACTION_BY_OPERATION: Record = { +const ACTION_BY_OPERATION: Record = { insert: "created", - // update: "updated", - // replace: "updated", - // delete: "deleted", }; @Injectable() export class SseListener implements OnModuleInit, OnModuleDestroy { - private changeStream: ChangeStream; + private static readonly MAX_RECONNECT_ATTEMPTS = 5; + private static readonly BASE_RECONNECT_DELAY_MS = 2000; + private changeStream: ChangeStream | null = null; + private reconnectAttempts = 0; + private reconnectTimer: NodeJS.Timeout | null = null; + private isShuttingDown = false; constructor( @InjectConnection() private readonly connection: Connection, @@ -40,17 +40,31 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { ) {} async onModuleInit() { - const isReplicaSet = await this.isDbReplicaSet(); - - console.log("isReplicaSet", isReplicaSet); - if (!isReplicaSet) { + if (!(await this.isDbReplicaSet())) { Logger.debug( "MongoDB is not running as a replica set. SSE change streams are disabled.", ); return; } - this.sseService.sseEnabled = isReplicaSet; + this.sseService.enable(); + this.startChangeStream(); + } + + async onModuleDestroy(): Promise { + this.isShuttingDown = true; + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + await this.changeStream?.close(); + this.changeStream = null; + } + + private startChangeStream() { + if (this.changeStream || this.isShuttingDown) { + return; + } this.changeStream = this.connection.watch([ { @@ -61,31 +75,62 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { }, ]); - this.changeStream.on("change", (change) => this.onStream(change)); - this.changeStream.on("error", (err) => - Logger.error(`SSE change stream is closed due to error: ${err}`), - ); + this.changeStream.on("change", (change) => this.onChange(change)); + this.changeStream.on("error", (err) => this.onStreamError(err)); + + this.reconnectAttempts = 0; + Logger.log("SSE change stream started"); } - private onStream(change: ChangeStreamDocument) { + private onChange(change: ChangeStreamDocument): void { if (!("ns" in change) || !("coll" in change.ns)) { return; } - const config = REGISTRY[change.ns.coll as Collection]; + const registryEntry = REGISTRY[change.ns.coll as WatchableCollection]; const action = ACTION_BY_OPERATION[change.operationType]; - - if (!config || !action) return; - const rawDoc = "fullDocument" in change ? change.fullDocument : null; - - if (!rawDoc) return; + if (!registryEntry || !action || !rawDoc) { + return; + } this.sseService.emit({ - message: plainToInstance(config.dto, rawDoc), - action: action, - entity: config.entity, + entity: registryEntry.entity, + action, + message: plainToInstance(registryEntry.dto, rawDoc), }); } + private async onStreamError(error: unknown): Promise { + Logger.error(`SSE change stream closed due to error: ${error}`); + + try { + await this.changeStream?.close(); + } catch (closeError) { + Logger.error( + `Failed to close SSE change stream after error: ${closeError}`, + ); + } finally { + this.changeStream = null; + } + + if (this.isShuttingDown) { + return; + } + + if (this.reconnectAttempts >= SseListener.MAX_RECONNECT_ATTEMPTS) { + Logger.error( + `Max reconnect attempts (${SseListener.MAX_RECONNECT_ATTEMPTS}) reached. SSE change stream will not restart automatically.`, + ); + return; + } + + this.reconnectAttempts += 1; + const delay = + SseListener.BASE_RECONNECT_DELAY_MS * 2 ** (this.reconnectAttempts - 1); + Logger.warn( + `Restarting SSE change stream in ${delay}ms (attempt ${this.reconnectAttempts}/${SseListener.MAX_RECONNECT_ATTEMPTS})`, + ); + this.reconnectTimer = setTimeout(() => this.startChangeStream(), delay); + } private async isDbReplicaSet(): Promise { try { @@ -95,8 +140,4 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { return false; } } - - onModuleDestroy() { - return this.changeStream?.close(); - } } diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index a97dbd7a9..170dd5073 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -9,38 +9,36 @@ import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { randomUUID } from "crypto"; import { ConfigService } from "@nestjs/config"; import { AccessGroupsType } from "src/config/configuration"; - -export interface HasAccessGroups { - ownerGroup?: string; - accessGroups?: string[]; -} +import { + SseClient, + HasAccessGroups, + SseEvent, + SseConnectionsReport, +} from "./interfaces/sse-event.interface"; @Injectable() export class SseService { - private readonly MAX_CONNECTIONS_PER_USER_PER_INSTANCE = 5; - private clients = new Map< - string, - { user: JWTUser; subject: Subject } - >(); - private accessGroups; + private static readonly MAX_CONNECTIONS_PER_USER_PER_INSTANCE = 5; + private readonly clients = new Map(); + private accessGroups?: AccessGroupsType; public sseEnabled = false; constructor(private configService: ConfigService) { this.accessGroups = this.configService.get("accessGroups"); } - + enable(): void { + this.sseEnabled = true; + } getEvents(user: JWTUser): Observable { - if (!this.sseEnabled) { - throw new ServiceUnavailableException("SSE are not enabled."); - } - const userConnectionCount = [...this.clients.values()].filter( - (c) => c.user._id === user._id, - ).length; + this.checkEnabled(); - if (userConnectionCount >= this.MAX_CONNECTIONS_PER_USER_PER_INSTANCE) { + if ( + this.getCurrentUserConnectionsCount(user) >= + SseService.MAX_CONNECTIONS_PER_USER_PER_INSTANCE + ) { throw new ForbiddenException( - `Maximum number of ${this.MAX_CONNECTIONS_PER_USER_PER_INSTANCE} open connections reached`, + `Maximum number of ${SseService.MAX_CONNECTIONS_PER_USER_PER_INSTANCE} open connections reached for user: ${user.username}`, ); } @@ -57,40 +55,71 @@ export class SseService { ); } - emit(event: { message: HasAccessGroups; action: string; entity: string }) { - for (const [, { user, subject }] of this.clients) { - const userGroups = user.currentGroups ?? []; - const instanceOwnerGroup = event.message.ownerGroup ?? ""; - const instanceAccessGroups = event.message.accessGroups ?? []; - - // TODO: this logic is duplicated from CaslAbilityFactory, consider centralizing it - // after the refacor of CaslAbilityFactory is done - const canAccess = - userGroups.includes(instanceOwnerGroup) || - instanceAccessGroups.some((g) => userGroups.includes(g)) || - userGroups.some((g) => this.accessGroups?.admin?.includes(g)); + emit(event: SseEvent): void { + const message: MessageEvent = { + data: { type: `${event.entity}.${event.action}`, data: event.message }, + }; - if (canAccess) { - subject.next({ - type: `${event.entity}.${event.action}`, - data: event.message, - }); + for (const { user, subject } of this.clients.values()) { + if (this.canUserAccess(user, event.message)) { + console.log(this.canUserAccess(user, event.message)); + subject.next(message); } } } - getAllConnections() { + getAllConnections(): SseConnectionsReport { + this.checkEnabled(); + + const users: Record = {}; + for (const { user } of this.clients.values()) { + users[user.username] = (users[user.username] ?? 0) + 1; + } + + return { connections: this.clients.size, users }; + } + + onModuleDestroy(): void { + for (const { subject } of this.clients.values()) { + subject.complete(); + } + this.clients.clear(); + } + + private checkEnabled(): void { if (!this.sseEnabled) { - throw new ServiceUnavailableException("SSE are not enabled."); + throw new ServiceUnavailableException("SSE is not enabled."); } - const counts = new Map(); + } - for (const { user } of this.clients.values()) { - counts.set(user.username, (counts.get(user.username) ?? 0) + 1); + private getCurrentUserConnectionsCount(user: JWTUser): number { + let count = 0; + for (const client of this.clients.values()) { + if (client.user._id === user._id) count++; } - return { - connections: this.clients.size, - users: Object.fromEntries(counts), - }; + return count; + } + + private canUserAccess(user: JWTUser, document: HasAccessGroups): boolean { + return this.canAccessByGroups( + user.currentGroups, + document, + this.accessGroups?.admin, + ); + } + + private canAccessByGroups( + userGroups: readonly string[] = [], + document: HasAccessGroups, + adminGroups: readonly string[] = [], + ): boolean { + const isOwner = + !!document.ownerGroup && userGroups.includes(document.ownerGroup); + const hasAccessGroup = (document.accessGroups ?? []).some((group) => + userGroups.includes(group), + ); + const isAdmin = userGroups.some((group) => adminGroups.includes(group)); + + return isOwner || hasAccessGroup || isAdmin; } } From 591cfa58307735cd04b371a972bf96d8a31f19de Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 6 Jul 2026 14:29:00 +0200 Subject: [PATCH 20/34] fix mongodb container health check --- CI/E2E/docker-compose-local.yaml | 2 +- CI/E2E/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/E2E/docker-compose-local.yaml b/CI/E2E/docker-compose-local.yaml index 323bf6c05..0cac38d70 100644 --- a/CI/E2E/docker-compose-local.yaml +++ b/CI/E2E/docker-compose-local.yaml @@ -12,7 +12,7 @@ services: healthcheck: test: > mongosh --quiet --eval " - try { rs.status().ok } + try { if (rs.status().ok !== 1) quit(1) } catch { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: '$$MONGO_RS_HOST' }] }) } " interval: 10s diff --git a/CI/E2E/docker-compose.yaml b/CI/E2E/docker-compose.yaml index fc0c4a4ea..94a7f07ed 100644 --- a/CI/E2E/docker-compose.yaml +++ b/CI/E2E/docker-compose.yaml @@ -19,7 +19,7 @@ services: healthcheck: test: > mongosh --quiet --eval " - try { rs.status().ok } + try { if (rs.status().ok !== 1) quit(1) } catch { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: '$$MONGO_RS_HOST' }] }) } " interval: 10s From 495ca08112ff2951a94fac98bc9ac28be14be3e8 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 7 Jul 2026 19:09:53 +0200 Subject: [PATCH 21/34] Disable real-time updates in frontend config by default Changed realTimeUpdatesEnabled from true to false. --- src/config/frontend.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/frontend.config.json b/src/config/frontend.config.json index 5934b9814..deeab429a 100644 --- a/src/config/frontend.config.json +++ b/src/config/frontend.config.json @@ -5,7 +5,7 @@ }, "statusBannerMessage": "", "statusBannerCode": "INFO", - "realTimeUpdatesEnabled": true, + "realTimeUpdatesEnabled": false, "autoApplyFilters": false, "accessTokenPrefix": "Bearer ", "addDatasetEnabled": false, From 4f81827619c27c557cd308f8da5e380288d53d65 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Tue, 16 Jun 2026 13:25:17 +0200 Subject: [PATCH 22/34] added decorator and some minor refactor --- src/datasets/datasets.controller.ts | 3 ++ src/datasets/datasets.v4.controller.ts | 3 ++ .../decorators/sse.decorator.ts | 6 +++ .../interceptors/sse.interceptor.ts | 44 +++++++++++++++++++ src/serverSentEvent/sse.controller.ts | 2 +- 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/serverSentEvent/decorators/sse.decorator.ts create mode 100644 src/serverSentEvent/interceptors/sse.interceptor.ts diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 47029504c..c24918bdc 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,6 +121,8 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; +import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; +import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -546,6 +548,7 @@ export class DatasetsController { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) + @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @ApiOperation({ diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index a8574be7a..ce0d11659 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,6 +90,8 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; +import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; +import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; @ApiBearerAuth() @ApiExtraModels( @@ -254,6 +256,7 @@ export class DatasetsV4Controller { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) + @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @UseInterceptors(ClassSerializerInterceptor) diff --git a/src/serverSentEvent/decorators/sse.decorator.ts b/src/serverSentEvent/decorators/sse.decorator.ts new file mode 100644 index 000000000..68704e465 --- /dev/null +++ b/src/serverSentEvent/decorators/sse.decorator.ts @@ -0,0 +1,6 @@ +import { applyDecorators, UseInterceptors } from "@nestjs/common"; +import { SseEventType, SseInterceptor } from "../interceptors/sse.interceptor"; + +export const EmitSse = (eventType: SseEventType) => { + return applyDecorators(UseInterceptors(SseInterceptor(eventType))); +}; diff --git a/src/serverSentEvent/interceptors/sse.interceptor.ts b/src/serverSentEvent/interceptors/sse.interceptor.ts new file mode 100644 index 000000000..9fbca66fc --- /dev/null +++ b/src/serverSentEvent/interceptors/sse.interceptor.ts @@ -0,0 +1,44 @@ +import { + Injectable, + NestInterceptor, + ExecutionContext, + CallHandler, + Type, + mixin, +} from "@nestjs/common"; +import { Observable, tap } from "rxjs"; +import { HasAccessGroups, SseService } from "src/serverSentEvent/sse.service"; + +export const EVENT_METHODS: Record = { + DATASET_CREATED: "dataset.created", + // PROPOSAL_CREATED: "proposal.created", + // SAMPLE_CREATED: "sample.created", + // INSTRUMENT_CREATED: "instrument.created", +} as const; + +export type SseEventType = (typeof EVENT_METHODS)[keyof typeof EVENT_METHODS]; + +export const SseInterceptor = ( + eventType: SseEventType, +): Type => { + @Injectable() + class MixinSseInterceptor implements NestInterceptor { + constructor(public readonly sseService: SseService) {} + + intercept( + context: ExecutionContext, + next: CallHandler, + ): Observable { + return next.handle().pipe( + tap((responseData: HasAccessGroups) => { + this.sseService.emit({ + message: responseData, + type: eventType, + }); + }), + ); + } + } + + return mixin(MixinSseInterceptor); +}; diff --git a/src/serverSentEvent/sse.controller.ts b/src/serverSentEvent/sse.controller.ts index 2da5e89b7..3580b60d7 100644 --- a/src/serverSentEvent/sse.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -56,7 +56,7 @@ export class SseController { @CheckPolicies( "runtimeconfig", (ability: AppAbility) => - ability.can(Action.RuntimeConfigUpdateEndpoint, RuntimeConfig), //TODO: define a correct policy for monitoring connections + ability.can(Action.RuntimeConfigUpdate, RuntimeConfig), //TODO: define a correct policy for monitoring connections ) @ApiOperation({ summary: "List active SSE connections on this instance.", From 3e364105cbf1bbbd23a5afd81394aea60f2c58e9 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 22 Jun 2026 15:53:46 +0200 Subject: [PATCH 23/34] added sse listener to watch mongodb changestream --- src/datasets/datasets.controller.ts | 3 -- src/datasets/datasets.v4.controller.ts | 3 -- .../decorators/sse.decorator.ts | 6 --- .../interceptors/sse.interceptor.ts | 44 ------------------- 4 files changed, 56 deletions(-) delete mode 100644 src/serverSentEvent/decorators/sse.decorator.ts delete mode 100644 src/serverSentEvent/interceptors/sse.interceptor.ts diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index c24918bdc..47029504c 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -121,8 +121,6 @@ import { IncludeValidationPipe } from "src/common/pipes/include-validation.pipe" import { DATASET_LOOKUP_FIELDS } from "./types/dataset-lookup"; import { getSwaggerDatasetFilterContentV3 } from "./types/dataset-filter-content.v3"; import { Filter } from "./decorators/filter.decorator"; -import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; -import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; @ApiBearerAuth() @ApiExtraModels( @@ -548,7 +546,6 @@ export class DatasetsController { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) - @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @ApiOperation({ diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index ce0d11659..a8574be7a 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -90,8 +90,6 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; -import { EVENT_METHODS } from "src/serverSentEvent/interceptors/sse.interceptor"; -import { EmitSse } from "src/serverSentEvent/decorators/sse.decorator"; @ApiBearerAuth() @ApiExtraModels( @@ -256,7 +254,6 @@ export class DatasetsV4Controller { new UTCTimeInterceptor(["endTime"]), new FormatPhysicalQuantitiesInterceptor("scientificMetadata"), ) - @EmitSse(EVENT_METHODS.DATASET_CREATED) @UsePipes(ScientificMetadataValidationPipe) @Post() @UseInterceptors(ClassSerializerInterceptor) diff --git a/src/serverSentEvent/decorators/sse.decorator.ts b/src/serverSentEvent/decorators/sse.decorator.ts deleted file mode 100644 index 68704e465..000000000 --- a/src/serverSentEvent/decorators/sse.decorator.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { applyDecorators, UseInterceptors } from "@nestjs/common"; -import { SseEventType, SseInterceptor } from "../interceptors/sse.interceptor"; - -export const EmitSse = (eventType: SseEventType) => { - return applyDecorators(UseInterceptors(SseInterceptor(eventType))); -}; diff --git a/src/serverSentEvent/interceptors/sse.interceptor.ts b/src/serverSentEvent/interceptors/sse.interceptor.ts deleted file mode 100644 index 9fbca66fc..000000000 --- a/src/serverSentEvent/interceptors/sse.interceptor.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { - Injectable, - NestInterceptor, - ExecutionContext, - CallHandler, - Type, - mixin, -} from "@nestjs/common"; -import { Observable, tap } from "rxjs"; -import { HasAccessGroups, SseService } from "src/serverSentEvent/sse.service"; - -export const EVENT_METHODS: Record = { - DATASET_CREATED: "dataset.created", - // PROPOSAL_CREATED: "proposal.created", - // SAMPLE_CREATED: "sample.created", - // INSTRUMENT_CREATED: "instrument.created", -} as const; - -export type SseEventType = (typeof EVENT_METHODS)[keyof typeof EVENT_METHODS]; - -export const SseInterceptor = ( - eventType: SseEventType, -): Type => { - @Injectable() - class MixinSseInterceptor implements NestInterceptor { - constructor(public readonly sseService: SseService) {} - - intercept( - context: ExecutionContext, - next: CallHandler, - ): Observable { - return next.handle().pipe( - tap((responseData: HasAccessGroups) => { - this.sseService.emit({ - message: responseData, - type: eventType, - }); - }), - ); - } - } - - return mixin(MixinSseInterceptor); -}; From e8b43d664c6517aca96bb6e22cf5ac185cf9b8c9 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 20 Aug 2026 18:49:27 +0200 Subject: [PATCH 24/34] Add ticket endpoint for SSE authentication EventSource cannot send auth headers, so mint a short-lived ticket here and pass it on the stream URL instead. --- src/auth/auth.service.ts | 8 +++++++ src/auth/strategies/jwt.strategy.ts | 31 ++++++++++++++++----------- src/auth/utils/jwt.util.ts | 21 ++++++++++++++++++ src/serverSentEvent/sse.controller.ts | 18 +++++++++++++++- src/serverSentEvent/sse.module.ts | 3 ++- 5 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 src/auth/utils/jwt.util.ts diff --git a/src/auth/auth.service.ts b/src/auth/auth.service.ts index d403fab7f..1e9445955 100644 --- a/src/auth/auth.service.ts +++ b/src/auth/auth.service.ts @@ -19,6 +19,7 @@ import { ReturnedUserDto } from "src/users/dto/returned-user.dto"; import { CreateUserSettingsDto } from "src/users/dto/create-user-settings.dto"; import { OidcClientService } from "../common/openid-client/openid-client.service"; import { OidcAuthService } from "src/common/openid-client/openid-auth.service"; +import { JWTUser } from "./interfaces/jwt-user.interface"; @Injectable() export class AuthService { @@ -94,6 +95,13 @@ export class AuthService { }; } + createSseTicket(user: JWTUser): string { + return this.jwtService.sign( + { ...user, purpose: "sse" }, + { expiresIn: "60s" }, + ); + } + async logout(req: Request) { const logoutURL = this.configService.get("logoutURL") || ""; const expressSessionSecret = this.configService.get( diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 08154e878..5889e2907 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -1,20 +1,17 @@ import { ExtractJwt, Strategy } from "passport-jwt"; import { PassportStrategy } from "@nestjs/passport"; -import { Injectable } from "@nestjs/common"; +import { Injectable, UnauthorizedException } from "@nestjs/common"; import { Request } from "express"; import { RolesService } from "src/users/roles.service"; import { UsersService } from "src/users/users.service"; import { JWTUser } from "../interfaces/jwt-user.interface"; import { User } from "src/users/schemas/user.schema"; import { ConfigService } from "@nestjs/config"; - -const fromSseQueryAsBearerToken = (req: Request): string | null => { - if (req.path.endsWith("events/stream")) { - const token = req.query?.token; - return typeof token === "string" ? token : null; - } - return null; -}; +import { + SSE_STREAM_PATH, + fromSseTicket, + requireJwtSecret, +} from "src/auth/utils/jwt.util"; @Injectable() export class JwtStrategy extends PassportStrategy(Strategy) { @@ -26,14 +23,24 @@ export class JwtStrategy extends PassportStrategy(Strategy) { super({ jwtFromRequest: ExtractJwt.fromExtractors([ ExtractJwt.fromAuthHeaderAsBearerToken(), - fromSseQueryAsBearerToken, + fromSseTicket, ]), ignoreExpiration: false, - secretOrKey: configService.get("jwt.secret") || "defaultSecret", + secretOrKey: requireJwtSecret(configService), + passReqToCallback: true, }); } - async validate(payload: Omit) { + async validate( + request: Request, + payload: Omit & { purpose?: string }, + ) { + const isSsePath = request.path === SSE_STREAM_PATH; + const isSseTicket = payload.purpose === "sse"; + if (isSsePath !== isSseTicket) { + throw new UnauthorizedException(); + } + const roles = await this.rolesService.find({ userId: payload._id }); const userIdentity = await this.usersService.findByIdUserIdentity( diff --git a/src/auth/utils/jwt.util.ts b/src/auth/utils/jwt.util.ts new file mode 100644 index 000000000..fc9a6678a --- /dev/null +++ b/src/auth/utils/jwt.util.ts @@ -0,0 +1,21 @@ +import { Request } from "express"; +import { ConfigService } from "@nestjs/config"; + +export const SSE_STREAM_PATH = "/api/v3/events/stream"; + +export const fromSseTicket = (req: Request): string | null => { + if (req.path !== SSE_STREAM_PATH) return null; + const ticket = req.query?.ticket; + return typeof ticket === "string" ? ticket : null; +}; + +export const requireJwtSecret = (configService: ConfigService): string => { + const secret = configService.get("jwt.secret"); + if (secret) return secret; + + if (process.env.NODE_ENV === "production") { + throw new Error("jwt.secret must be configured in production"); + } + console.warn("jwt.secret is not set, falling back to an insecure default"); + return "defaultSecret"; +}; diff --git a/src/serverSentEvent/sse.controller.ts b/src/serverSentEvent/sse.controller.ts index 3580b60d7..1b07c52b3 100644 --- a/src/serverSentEvent/sse.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -5,6 +5,7 @@ import { UseGuards, Req, Get, + Post, } from "@nestjs/common"; import { Observable } from "rxjs"; @@ -23,12 +24,16 @@ import { DatasetClass } from "src/datasets/schemas/dataset.schema"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { RuntimeConfig } from "src/config/runtime-config/schemas/runtime-config.schema"; import { SseService } from "./sse.service"; +import { AuthService } from "src/auth/auth.service"; @ApiTags("events") @Controller("events") @ApiBearerAuth() export class SseController { - constructor(private readonly sseService: SseService) {} + constructor( + private readonly sseService: SseService, + private readonly authService: AuthService, + ) {} @Sse("stream") @UseGuards(PoliciesGuard) @@ -64,4 +69,15 @@ export class SseController { connections() { return this.sseService.getAllConnections(); } + + @UseGuards(PoliciesGuard) + @CheckPolicies("datasets", (ability: AppAbility) => + ability.can(Action.DatasetRead, DatasetClass), + ) + @Post("ticket") + async createTicket(@Req() request: Request): Promise<{ ticket: string }> { + return { + ticket: this.authService.createSseTicket(request.user as JWTUser), + }; + } } diff --git a/src/serverSentEvent/sse.module.ts b/src/serverSentEvent/sse.module.ts index c1849b360..bcb56c41e 100644 --- a/src/serverSentEvent/sse.module.ts +++ b/src/serverSentEvent/sse.module.ts @@ -3,10 +3,11 @@ import { SseService } from "./sse.service"; import { CaslModule } from "src/casl/casl.module"; import { SseController } from "./sse.controller"; import { SseListener } from "./sse.listener"; +import { AuthModule } from "src/auth/auth.module"; @Global() @Module({ - imports: [CaslModule], + imports: [CaslModule, AuthModule], controllers: [SseController], providers: [SseService, SseListener], exports: [SseService], From 781997ff533d6cd7ad335014b86fcb0ca69b3af8 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 20 Aug 2026 18:49:42 +0200 Subject: [PATCH 25/34] cleanup --- src/config/configuration.ts | 3 --- src/serverSentEvent/sse.service.ts | 1 - 2 files changed, 4 deletions(-) diff --git a/src/config/configuration.ts b/src/config/configuration.ts index d8a6fc97c..229eba923 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -6,9 +6,6 @@ import localconfiguration from "./localconfiguration"; const configuration = () => { const jwtSecret = process.env.JWT_SECRET; - if (!jwtSecret && process.env.NODE_ENV === "production") { - throw new Error("JWT_SECRET is required in production"); - } const accessGroupsStaticValues = process.env.ACCESS_GROUPS_STATIC_VALUES || ""; const adminGroups = process.env.ADMIN_GROUPS || ""; diff --git a/src/serverSentEvent/sse.service.ts b/src/serverSentEvent/sse.service.ts index 170dd5073..abc40887e 100644 --- a/src/serverSentEvent/sse.service.ts +++ b/src/serverSentEvent/sse.service.ts @@ -62,7 +62,6 @@ export class SseService { for (const { user, subject } of this.clients.values()) { if (this.canUserAccess(user, event.message)) { - console.log(this.canUserAccess(user, event.message)); subject.next(message); } } From b05a0903131c6f4dc83d3ec620a13847a90f6e41 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 12:58:13 +0200 Subject: [PATCH 26/34] sse ticket should come first if available --- src/auth/strategies/jwt.strategy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auth/strategies/jwt.strategy.ts b/src/auth/strategies/jwt.strategy.ts index 5889e2907..661507e01 100644 --- a/src/auth/strategies/jwt.strategy.ts +++ b/src/auth/strategies/jwt.strategy.ts @@ -22,8 +22,8 @@ export class JwtStrategy extends PassportStrategy(Strategy) { ) { super({ jwtFromRequest: ExtractJwt.fromExtractors([ - ExtractJwt.fromAuthHeaderAsBearerToken(), fromSseTicket, + ExtractJwt.fromAuthHeaderAsBearerToken(), ]), ignoreExpiration: false, secretOrKey: requireJwtSecret(configService), From ab5e989d0bcd42e60ac9528ce7e486184df6f725 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:19:30 +0200 Subject: [PATCH 27/34] throw error if jwt secret env variable is not provided --- src/auth/utils/jwt.util.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/auth/utils/jwt.util.ts b/src/auth/utils/jwt.util.ts index fc9a6678a..3b0cb7e59 100644 --- a/src/auth/utils/jwt.util.ts +++ b/src/auth/utils/jwt.util.ts @@ -11,11 +11,9 @@ export const fromSseTicket = (req: Request): string | null => { export const requireJwtSecret = (configService: ConfigService): string => { const secret = configService.get("jwt.secret"); - if (secret) return secret; - if (process.env.NODE_ENV === "production") { - throw new Error("jwt.secret must be configured in production"); + if (!secret) { + throw new Error("JWT_SECRET is not defined in the environment variables."); } - console.warn("jwt.secret is not set, falling back to an insecure default"); - return "defaultSecret"; + return secret; }; From 3aacc3bc70fb2e8f72a8127c1312d13a1ea95d10 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:19:57 +0200 Subject: [PATCH 28/34] added SSE_TICKET_EXPIRES_IN env vairable --- src/auth/auth.service.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/auth/auth.service.ts b/src/auth/auth.service.ts index 1e9445955..331be0e70 100644 --- a/src/auth/auth.service.ts +++ b/src/auth/auth.service.ts @@ -96,10 +96,8 @@ export class AuthService { } createSseTicket(user: JWTUser): string { - return this.jwtService.sign( - { ...user, purpose: "sse" }, - { expiresIn: "60s" }, - ); + const expiresIn = this.configService.get("sseTicketExpiresIn"); + return this.jwtService.sign({ ...user, purpose: "sse" }, { expiresIn }); } async logout(req: Request) { From d75490073b1ea92a417ca3fdd58cb2cbbc2b870a Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:20:23 +0200 Subject: [PATCH 29/34] sseTicketExpiresIn added in config --- src/config/configuration.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 229eba923..4a831c540 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -458,6 +458,7 @@ const configuration = () => { ajvCustomDefinitions: ajvCustomDefinitions, opensearchConfig: jsonConfigMap.opensearchConfig, datafilesMetadataSchema: jsonConfigMap.datafilesMetadataSchema, + sseTicketExpiresIn: parseInt(process.env.SSE_TICKET_EXPIRES_IN || "60", 10), }; return merge(config, localconfiguration); }; From fea1b68f8cd3e461a2ff66cfb0a5f5f0d3eb55fe Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:21:09 +0200 Subject: [PATCH 30/34] add sse permission --- src/casl/abilities/sse.ability.ts | 58 +++++++++++++++++++ src/casl/action.enum.ts | 3 + src/casl/casl-ability.factory.ts | 7 +++ src/casl/casl.module.ts | 2 + src/casl/types/casl-subjects.ts | 2 + .../interfaces/sse-event.interface.ts | 1 + src/serverSentEvent/sse.controller.ts | 23 ++++---- 7 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 src/casl/abilities/sse.ability.ts diff --git a/src/casl/abilities/sse.ability.ts b/src/casl/abilities/sse.ability.ts new file mode 100644 index 000000000..045b597cf --- /dev/null +++ b/src/casl/abilities/sse.ability.ts @@ -0,0 +1,58 @@ +import { + AbilityBuilder, + ExtractSubjectType, + MongoAbility, + createMongoAbility, +} from "@casl/ability"; +import { SseClass } from "src/serverSentEvent/interfaces/sse-event.interface"; +import { Injectable } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AccessGroupsType } from "src/config/configuration"; +import { Action } from "../action.enum"; +import { + Subjects, + PossibleAbilities, + Conditions, +} from "../types/casl-subjects"; +import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; + +@Injectable() +export class SseAbility { + private accessGroups?: AccessGroupsType; + constructor(private configService: ConfigService) { + this.accessGroups = + this.configService.get("accessGroups") ?? + ({} as AccessGroupsType); + } + + buildAbility( + user: JWTUser | null, + ): MongoAbility { + const { can, build } = new AbilityBuilder( + createMongoAbility, + ); + + if (user) { + /** + * Authenticated user + */ + can(Action.SseRead, SseClass); + } + + if ( + user && + user.currentGroups.some((g) => this.accessGroups?.admin?.includes(g)) + ) { + /** + * User belonging to ADMIN_GROUPS + */ + can(Action.SseRead, SseClass); + can(Action.AccessAny, SseClass); + } + + return build({ + detectSubjectType: (item) => + item.constructor as ExtractSubjectType, + }); + } +} diff --git a/src/casl/action.enum.ts b/src/casl/action.enum.ts index 657bdea22..69251447f 100644 --- a/src/casl/action.enum.ts +++ b/src/casl/action.enum.ts @@ -92,6 +92,9 @@ export enum Action { RuntimeConfigRead = "runtimeconfig_read", RuntimeConfigUpdate = "runtimeconfig_update", + // Server-Sent Events + SseRead = "sse_read", + // ------------------------------------- // Samples // ------------------------------------- diff --git a/src/casl/casl-ability.factory.ts b/src/casl/casl-ability.factory.ts index 1a4135a34..2210bb6f6 100644 --- a/src/casl/casl-ability.factory.ts +++ b/src/casl/casl-ability.factory.ts @@ -26,6 +26,7 @@ import { OrigDatablockAbility } from "./abilities/origdatablocks.ability"; import { PolicyAbility } from "./abilities/policies.ability"; import { ProposalAbility } from "./abilities/proposals.ability"; import { RuntimeConfigAbility } from "./abilities/runtime-config.ability"; +import { SseAbility } from "./abilities/sse.ability"; export type AppAbility = MongoAbility; @@ -46,6 +47,7 @@ export class CaslAbilityFactory { private policyAbility: PolicyAbility, private proposalAbility: ProposalAbility, private runtimeConfigAbility: RuntimeConfigAbility, + private sseAbility: SseAbility, ) { this.accessGroups = this.configService.get("accessGroups"); @@ -71,6 +73,7 @@ export class CaslAbilityFactory { runtimeconfig: this.runtimeConfigAccess, samples: this.samplesEndpointAccess, users: this.userEndpointAccess, + sse: this.sseAccess, }; endpointAccess(endpoint: string, user: JWTUser) { @@ -135,6 +138,10 @@ export class CaslAbilityFactory { return this.runtimeConfigAbility.buildAbility(user); } + sseAccess(user: JWTUser | null) { + return this.sseAbility.buildAbility(user); + } + publishedDataEndpointAccess(user: JWTUser) { const { can, build } = new AbilityBuilder( createMongoAbility, diff --git a/src/casl/casl.module.ts b/src/casl/casl.module.ts index cf36bdf45..32ed6f35e 100644 --- a/src/casl/casl.module.ts +++ b/src/casl/casl.module.ts @@ -15,6 +15,7 @@ import { OrigDatablockAbility } from "./abilities/origdatablocks.ability"; import { PolicyAbility } from "./abilities/policies.ability"; import { ProposalAbility } from "./abilities/proposals.ability"; import { RuntimeConfigAbility } from "./abilities/runtime-config.ability"; +import { SseAbility } from "./abilities/sse.ability"; @Module({ imports: [JobConfigModule, ConfigModule], @@ -33,6 +34,7 @@ import { RuntimeConfigAbility } from "./abilities/runtime-config.ability"; PolicyAbility, ProposalAbility, RuntimeConfigAbility, + SseAbility, ], exports: [CaslAbilityFactory], }) diff --git a/src/casl/types/casl-subjects.ts b/src/casl/types/casl-subjects.ts index c2a2a03ca..4edaca913 100644 --- a/src/casl/types/casl-subjects.ts +++ b/src/casl/types/casl-subjects.ts @@ -18,6 +18,7 @@ import { RuntimeConfig } from "src/config/runtime-config/schemas/runtime-config. import { MetadataKeyClass } from "src/metadata-keys/schemas/metadatakey.schema"; import { Opensearch } from "src/opensearch/opensearch.subject"; import { GenericHistory } from "src/common/schemas/generic-history.schema"; +import { SseClass } from "src/serverSentEvent/interfaces/sse-event.interface"; export type Subjects = | string @@ -40,6 +41,7 @@ export type Subjects = | typeof User | typeof UserIdentity | typeof UserSettings + | typeof SseClass > | "all"; diff --git a/src/serverSentEvent/interfaces/sse-event.interface.ts b/src/serverSentEvent/interfaces/sse-event.interface.ts index 8cb146311..7f73a37a9 100644 --- a/src/serverSentEvent/interfaces/sse-event.interface.ts +++ b/src/serverSentEvent/interfaces/sse-event.interface.ts @@ -2,6 +2,7 @@ import { Subject } from "rxjs"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { MessageEvent } from "@nestjs/common"; +export class SseClass {} export interface SseClient { user: JWTUser; subject: Subject; diff --git a/src/serverSentEvent/sse.controller.ts b/src/serverSentEvent/sse.controller.ts index 1b07c52b3..400bfbf78 100644 --- a/src/serverSentEvent/sse.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -20,11 +20,10 @@ import { PoliciesGuard } from "src/casl/guards/policies.guard"; import { Action } from "src/casl/action.enum"; import { AppAbility } from "src/casl/casl-ability.factory"; import { CheckPolicies } from "src/casl/decorators/check-policies.decorator"; -import { DatasetClass } from "src/datasets/schemas/dataset.schema"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; -import { RuntimeConfig } from "src/config/runtime-config/schemas/runtime-config.schema"; import { SseService } from "./sse.service"; import { AuthService } from "src/auth/auth.service"; +import { SseClass } from "./interfaces/sse-event.interface"; @ApiTags("events") @Controller("events") @@ -37,19 +36,19 @@ export class SseController { @Sse("stream") @UseGuards(PoliciesGuard) - @CheckPolicies("datasets", (ability: AppAbility) => - ability.can(Action.DatasetRead, DatasetClass), + @CheckPolicies("sse", (ability: AppAbility) => + ability.can(Action.SseRead, SseClass), ) @ApiOperation({ summary: "Subscribe to server-sent events.", description: - "Opens a text/event-stream connection pushing events for documents the authenticated user is allowed to read.", + "Opens a text/event-stream connection pushing events for documents the authenticated user is allowed to read. Authenticate with a ticket from POST /events/ticket.", }) @ApiQuery({ - name: "token", + name: "ticket", required: false, description: - "JWT access token. Alternative to the Authorization header for EventSource clients that cannot set request headers.", + "Short-lived ticket from POST /events/ticket. Expires after 60 seconds and is only accepted on this route.", }) stream(@Req() request: Request): Observable { const user = request.user as JWTUser; @@ -58,10 +57,8 @@ export class SseController { @Get("connections") @UseGuards(PoliciesGuard) - @CheckPolicies( - "runtimeconfig", - (ability: AppAbility) => - ability.can(Action.RuntimeConfigUpdate, RuntimeConfig), //TODO: define a correct policy for monitoring connections + @CheckPolicies("sse", (ability: AppAbility) => + ability.can(Action.AccessAny, SseClass), ) @ApiOperation({ summary: "List active SSE connections on this instance.", @@ -71,8 +68,8 @@ export class SseController { } @UseGuards(PoliciesGuard) - @CheckPolicies("datasets", (ability: AppAbility) => - ability.can(Action.DatasetRead, DatasetClass), + @CheckPolicies("sse", (ability: AppAbility) => + ability.can(Action.SseRead, SseClass), ) @Post("ticket") async createTicket(@Req() request: Request): Promise<{ ticket: string }> { From 64bc81b3ad0bfa71275e1958286e8d73fecfab21 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:27:04 +0200 Subject: [PATCH 31/34] update README with SSE_TICKET_EXPIRES_IN variable --- README.md | 254 +++++++++++++------------- src/serverSentEvent/sse.controller.ts | 2 +- 2 files changed, 129 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 8c091288b..a0c3e163e 100644 --- a/README.md +++ b/README.md @@ -148,132 +148,134 @@ The `publishedDataConfig.json.example` file in the root directory showcases the ## Environment variables Valid environment variables for the .env file. See [.env.example](/.env.example) for examples value formats. -| Environment Variable | Type | Optional | Description | Default Value | -|---------------------------------------------|---------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------| -| `ADMIN_GROUPS` | string | Yes | Comma-separated list of admin groups with admin permission assigned to the listed users. Example: "admin, ingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `CREATE_DATASET_GROUPS` | string | Yes | Comma-separated list of create dataset groups. Users belong to the listed groups can create dataset with/without PID. Example: "group1, group2". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `CREATE_DATASET_WITH_PID_GROUPS` | string | Yes | Comma-separated list of create dataset with pid groups. Users belong to the listed groups can create dataset with PID. Example: "group1, group2". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `CREATE_DATASET_PRIVILEGED_GROUPS` | string | Yes | | | -| `CREATE_JOB_PRIVILEGED_GROUPS` | string | Yes | | | -| `UPDATE_JOB_PRIVILEGED_GROUP` | string | Yes | | | -| `DELETE_GROUPS` | string | Yes | Comma-separated list of delete groups. Users belong to the listed groups can delete any dataset, origDatablocks, datablocks, etc. For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `DATASET_CREATION_VALIDATION_ENABLED` | boolean | | Flag to enable/disable dataset validation to validate if requested new dataset is valid with given regular expression. Preconfigure **DATASET_CREATION_VALIDATION_REGEX** variable is required. | false | -| `DATASET_CREATION_VALIDATION_REGEX` | string | | Regular expression validation for new dataset request. | "" | -| `POLICY_GROUPS` | string | Yes | Comma-separated list of policy groups with permission to create any policy. Example: "policyadmin". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `PROPOSAL_GROUPS` | string | Yes | Comma-separated list of proposal groups with permission to create any proposals. Example: "proposaladmin, proposalingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `SAMPLE_GROUPS` | string | Yes | Comma-separated list of sample groups with permission to create any samples. Example: "sampleadmin, sampleingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | -| `SAMPLE_PRIVILEGED_GROUPS` | string | Yes | |"sampleingestor"| -| `ACCESS_GROUPS_GRAPHQL_ENABLED` | string | Yes | Flag to enable/disable the GraphQL service to get access groups. Requires configuration of `ACCESS_GROUP_SERVICE_TOKEN`, `ACCESS_GROUP_SERVICE_API_URL`, and `ACCESS_GROUP_SERVICE_HANDLER`. | true | -| `ACCESS_GROUPS_SERVICE_TOKEN` | string | Yes | Authentication token used if access groups are obtained from a third-party service. Not used by the vanilla installation, but only if the instance is customized to use an external service to provide user groups, like the ESS example. | | -| `ACCESS_GROUP_SERVICE_API_URL` | string | Yes | URL of the service providing the users' access groups. Not used by the vanilla installation, but only if the instance is customized to use an external service to provide user groups, like the ESS example. | | -| `ACCESS_GROUP_SERVICE_HANDLER` | string | Yes | Configuration property that points to the source of a module. This module provides a specific responseProcessor function for handling GraphQL responses and a query template for making GraphQL requests. | | -| `ACCESS_GROUPS_STATIC_ENABLED` | string | Yes | Flag to enable/disable automatic assignment of predefined access groups to all users. | true | -| `ACCESS_GROUPS_STATIC_VALUES` | string | Yes | Comma-separated list of access groups automatically assigned to all users. Example: "scicat, user". | | -| `ACCESS_GROUPS_OIDCPAYLOAD_ENABLED` | string | Yes | Flag to enable/disable fetching access groups directly from OIDC response. Requires specifying a field via `OIDC_ACCESS_GROUPS_PROPERTY` to extract access groups. | false | -| `ACCESS_GROUPS_LDAPPAYLOAD_ENABLED` | string | Yes | Flag to enable/disable fetching access groups directly from Ldap response. Requires specifying a field via `LDAP_ACCESS_GROUPS_PROPERTY` to extract access groups. | false | -| `DOI_PREFIX` | string | | The facility DOI prefix, with trailing slash. | | -| `DOI_SHORT_SUFFIX` | string | | By default `uuidv4` is used to generate the DOI suffix but if this flag is `true` the shorter version of 10 random characters is used as DOI suffix. | | -| `DOI_USERNAME` | string | | The facility DOI DataCite username. | | -| `DOI_PASSWORD` | string | | The facility DOI DataCite password. | | -| `PUBLISHED_DATA_CONFIG_FILE` | string | | Path to the file containing the metadata and UI schemas for PublishedData. | `"publishedDataConfig.json"` | -| `AJV_CUSTOM_DEFINITIONS_FILE` | string | | The path to a JavaScript module that defines custom AJV keywords or dynamic defaults functions. | | -| `EXPRESS_SESSION_SECRET` | string | No | Secret used to set up express session. Required if using OIDC authentication | | -| `EXPRESS_SESSION_STORE` | string | Yes | Where to store the express session. When "mongo" on mongo else in memory | | -| `HTTP_MAX_REDIRECTS` | number | Yes | Max redirects for HTTP requests. | 5 | -| `HTTP_TIMEOUT` | number | Yes | Timeout for HTTP requests in ms. | 5000 | -| `JWT_SECRET` | string | | The secret for your JWT token, used for authorization. | | -| `JWT_EXPIRES_IN` | number | Yes | How long, in seconds, the JWT token is valid. | 3600 | -| `LDAP_URL` | string | Yes | The URL to your LDAP server. | | -| `LDAP_BIND_DN` | string | Yes | Bind*DN for your LDAP server. | | -| `LDAP_BIND_CREDENTIALS` | string | Yes | Credentials for your LDAP server. | | -| `LDAP_SEARCH_BASE` | string | Yes | Search base for your LDAP server. | | -| `LDAP_SEARCH_FILTER` | string | Yes | Search filter for your LDAP server. | | -| `LDAP_GROUP_SEARCH_BASE` | string | Yes | Search base for the user groups. | | -| `LDAP_GROUP_SEARCH_FILTER` | string | Yes | Search filter for the user groups. | | -| `LDAP_ACCESS_GROUPS_PROPERTY`| string | Yes | Target field to get the access groups value from Ldap response. | | -| `LDAP_USERNAME_ATTR`| string | Yes | Target field to get the username from the Ldap response. Defaults to displayName. | | -| `OIDC_ISSUER` | string | Yes | URL of the OIDC server providing the authentication service. Example: https://identity.esss.dk/realm/ess. | | -| `OIDC_CLIENT_ID` | string | Yes | Identity of the client used to obtain the user token. Example: scicat. | | -| `OIDC_ADDITIONAL_AUTHORIZED_PARTIES` | string | No | Comma-separated list of additional OIDC client IDs allowed to present tokens to this backend. Used for token exchange scenarios where a third-party client obtains a token on behalf of a user. The client ID must appear as the `azp` claim in the token. Example: `additional-client1, additional-client2`. | | -| `OIDC_CLIENT_SECRET` | string | Yes | Secret to provide to the OIDC service to obtain the user token. Example: Aa1JIw3kv3mQlGFWhRrE3gOdkH6xreAwro. | | -| `OIDC_CALLBACK_URL` | string | Yes | SciCat callback URL to redirect to after a successful login. Example: http://myscicat/api/v3/oidc/callback. | | -| `OIDC_SCOPE` | string | Yes | Space-separated list of info returned by the OIDC service. Example: "openid profile email". | | -| `OIDC_SUCCESS_URL` | string | Yes | SciCat Frontend auth-callback URL. Required to pass user credentials to SciCat Frontend after OIDC login. Example: https://myscicatfrontend/auth-callback. Must be `frontend-base-url/auth-callback` or `frontend-base-url/login` for the official SciCat frontend. | | -| `OIDC_RETURN_URL` | string | Yes | The path segment within the SciCat Frontend to redirect to, passed as query param in `OIDC_SUCCESS_URL` and handled by frontend. Example: /datasets. | | -| `OIDC_FRONTEND_CLIENTS` | string | Yes | Comma separated list of additional frontend OIDC clients for this backend. Example: scilog,maxiv. Their success and return URLs can be configured by setting `OIDC*${CLIENT}_SUCCESS_URL` (E.g. `OIDC_SCILOG_SUCCESS_URL`) and `OIDC_${CLIENT}\_RETURN_URL`| | -|`OIDC_ACCESS_GROUPS`| string | Yes | Functionality is still unclear. | | -|`OIDC_ACCESS_GROUPS_PROPERTY`| string | Yes | Target field to get the access groups value from OIDC response. | | -|`OIDC_USERINFO_MAPPING_FIELD_USERNAME`| string | Yes | Comma-separated list of fields from the OIDC response to use as the user's profile username. Example:`OIDC_USERINFO_MAPPING_FIELD_USERNAME="iss, sub"`. | "preferred_username" \|\| "name" | -| `OIDC_USERINFO_MAPPING_FIELD_DISPLAYNAME`| string | Yes | Field from the OIDC response to use as the user's profile display name. Example:`OIDC_USERINFO_MAPPING_FIELD_DISPLAYNAME="preferred_username"`. | "name" | -| `OIDC_USERINFO_MAPPING_FIELD_EMAIL`| string | Yes | Field from the OIDC response to use as the user's profile email. | "email" | -|`OIDC_USERINFO_MAPPING_FIELD_FAMILYNAME`| string | Yes | Field from the OIDC response to use as the user's profile family name. | "family_name" | -|`OIDC_USERINFO_MAPPING_FIELD_ID`| string | Yes | Field from the OIDC response to use as the user's profile ID. | "sub" \|\| "user_id" | -|`OIDC_USERINFO_MAPPING_FIELD_THUMBNAILPHOTO`| string | Yes | Field from the OIDC response to use as the user's profile thumbnail photo. | "thumbnailPhoto" | -| `OIDC_USERINFO_MAPPING_FIELD_PROVIDER`| string | Yes | Field from the OIDC response to use as the user's profile provider. | "iss" | -|`OIDC_USERINFO_MAPPING_FIELD_GROUP`| string | Yes | Field from the OIDC response to use as the user's profile group. | "groups" | -|`OIDC_USERQUERY_OPERATOR`| string | Yes | Specifies the operator ("or" or "and") for UserModel.findOne queries, determining the logic used to match fields like "username" or "email". Example:`UserModel.findOne({$or: {"username":"testUser", "email":"test@test.com"}})`. | "or" | -| `OIDC_USERQUERY_FILTER`| string | Yes | Defines key-value pairs for UserModel.findOne queries, using a "key:value" format. Example:`OIDC_USERQUERY_FILTER="username:sub, email:email"`. | "username:username, email:email" | -| `LOGBOOK_ENABLED`| string | Yes | Flag to enable/disable the Logbook endpoints. Values "yes" or "no". | "no" | -|`LOGBOOK_BASE_URL`| string | Yes | The base URL to the Logbook API. Only required if Logbook is enabled. | | -|`METADATA_KEYS_RETURN_LIMIT`| number | Yes | The return limit for the`/Datasets/metadataKeys`endpoint. | | -|`METADATA_PARENT_INSTANCES_RETURN_LIMIT`| number | Yes | The return limit of Datasets to extract metadata keys from for the`/Datasets/metadataKeys`endpoint. | | -|`MONGODB_URI`| string | | The URI for your MongoDB instance. | | -|`OAI_PROVIDER_ROUTE`| string | Yes | URI to OAI provider, used for the`/publisheddata/:id/resync`endpoint. | | -|`PID_PREFIX`| string | | The facility PID prefix, with trailing slash. | | -|`PUBLIC_URL_PREFIX`| string | | The base URL to the facility Landing Page. | | -|`PORT`| number | Yes | The port on which you want to access the app. | 3000 | -|`RABBITMQ_ENABLED`| string | Yes | Flag to enable/disable RabbitMQ consumer. Values "yes" or "no". | "no" | -|`RABBITMQ_HOSTNAME`| string | Yes | The hostname of the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | -|`RABBITMQ_USERNAME`| string | Yes | The username used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | -|`RABBITMQ_PASSWORD`| string | Yes | The password used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | -|`RABBITMQ_PORT`| number | Yes | The port of the RabbitMQ message broker. Only required if RabbitMQ is enabled. |5672| -|`REGISTER_DOI_URI`| string | | URI to the organization that registers the facility's DOIs using the new DataCite API. |"https://api.test.datacite.org/dois"| -|`REGISTER_DOI_URI_V3`| string | | URI to the organization that registers the facility's DOIs using the old DataCite API. |"https://mds.test.datacite.org/doi"| -|`REGISTER_METADATA_URI`| string | | URI to the organization that registers the facility's published data metadata. |"https://mds.test.datacite.org/metadata"| -|`DOI_USERNAME`| string | Yes | DOI Username. |"username"| -|`DOI_PASSWORD`| string | Yes | DOI Password. |"password"| -|`SITE`| string | | The name of your site. | | -|`EMAIL_TYPE`| string | Yes | The type of your email provider. Options are "smtp" or "ms365". | "smtp" | -|`EMAIL_FROM`| string | Yes | Email address that emails should be sent from. | | -|`EMAIL_REPLYTO` | string | Yes | Email 'Reply-To' field. | | -|`SMTP_HOST`| string | Yes | Host of SMTP server. | | -|`SMTP_MESSAGE_FROM`| string | Yes | (Deprecated) Alternate spelling of EMAIL_FROM.| | -|`SMTP_PORT`| number | Yes | Port of SMTP server. | 587 | -|`SMTP_SECURE`| bool | Yes | Use encrypted SMTPS. | "no" | -|`MS365_TENANT_ID`| string | Yes | Tenant ID for sending emails over Microsoft Graph API. | | -|`MS365_CLIENT_ID`| string | Yes | Client ID for sending emails over Microsoft Graph API | | -|`MS365_CLIENT_SECRET`| string | Yes | Client Secret for sending emails over Microsoft Graph API | | -|`POLICY_PUBLICATION_SHIFT`| integer | Yes | Embargo period expressed in years. | 3 years | -|`POLICY_RETENTION_SHIFT`| integer | Yes | Retention period (how long the facility will hold on to data) expressed in years. | -1 (indefinitely) | -|`OPENSEARCH_ENABLED`| string | | Controls whether OpenSearch is enabled on application startup. If not provided or set to `no`, OpenSearch will not be instantiated.| "no" | -|`OPENSEARCH_DEFAULT_INDEX`| string | | Specifies the default index. If not provided, a default index named `dataset` will be created automatically. | "dataset" | -|`OPENSEARCH_HOST`| string | | Host of Opensearch server instance. | | -|`OPENSEARCH_USERNAME`| string | Yes | Username for OpenSearch authentication. Defaults to `admin` in standard deployments but can be configured to use a custom user with appropriate permissions. | "admin" | -|`OPENSEARCH_PASSWORD`| string | | Password used for OpenSearch authentication. Must match `OPENSEARCH_INITIAL_ADMIN_PASSWORD` used when creating the OpenSearch container. | | -|`OPENSEARCH_REFRESH`| string | | Controls index refresh behavior. `wait_for`waits for the next refresh cycle before returning, which is useful for development and testing.`false`skips waiting and is recommended for production. Defaults to false. | false | -|`OPENSEARCH_DATA_SYNC_BATCH_SIZE`| number | | Number of documents fetched from MongoDB per batch during OpenSearch data sync. | 1000 | -|`FRONTEND_CONFIG_FILE`| string | | The file name for frontend configuration, located in the`/src/config`directory by default. | "./src/config/frontend.config.json" | -|`FRONTEND_THEME_FILE`| string | | The file name for frontend theme, located in the`/src/config`directory by default. | "./src/config/frontend.theme.json" | -|`LOGGERS_CONFIG_FILE`| string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" | -|`PROPOSAL_TYPES_FILE`| string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" | -|`DATASET_TYPES_FILE`| string | | | "datasetTypes.json" | -|`SWAGGER_PATH`| string | Yes | swaggerPath is the path where the swagger UI will be available. | "explorer"| -|`MAX_FILE_UPLOAD_SIZE`| string | Yes | Maximum allowed file upload size. | "16mb"| -|`FUNCTIONAL_ACCOUNTS_FILE`| string | Yes | The file name for functional accounts, relative to the project root directory | "functionalAccounts.json"| -|`JOB_CONFIGURATION_FILE`| string | Yes | Path of a job configuration file (conventionally`"jobConfig.yaml"`). If unset, jobs are disabled | | -|`JOB_DEFAULT_STATUS_CODE`| string | Yes | Default statusCode for new jobs | "jobSubmitted" | -|`JOB_DEFAULT_STATUS_MESSAGE` | string | Yes | Default statusMessage for new jobs | "Job submitted." | -|`TRACKABLE_STRATEGY` | string | Yes | "document" or "delta". Document strategy (default): Stores full document copies in the history collection for both before and after states. Delta strategy: Only stores the fields that changed, saving database space. | "document" | -|`TRACKABLES` | string | Yes | The TRACKABLES environment variable configures which data models are tracked by the history system. When specified, only models listed in this variable will have their changes recorded in the History collection. | Dataset\[,Proposal\]\[,Sample\]\[,Instrument\]\[,PublishedData\] | -|`HISTORY_ACCESS_DATASET_GROUPS` | string | Yes | Roles in this list will be able to access history dataset records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_PROPOSAL_GROUPS`| string | Yes | Roles in this list will be able to access history proposal records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_SAMPLE_GROUPS` | string | Yes | Roles in this list will be able to access history sample records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_INSTRUMENT_GROUPS` | string | Yes | Roles in this list will be able to access history instrument records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_PUBLISHED_DATA_GROUPS` | string | Yes | Roles in this list will be able to access history published data records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_POLICIES_GROUPS` | string | Yes | Roles in this list will be able to access history policies data records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_DATABLOCK_GROUPS` | string | Yes | Roles in this list will be able to access history datablock records | \[role1\]\[,role2\]\[,roleN\]... | -|`HISTORY_ACCESS_ATTACHMENT_GROUPS` | string | Yes | Roles in this list will be able to access history attachment records | \[role1\]\[,role2\]\[,roleN\]... | -|`MASK_PERSONAL_INFO` | string | Yes | When enabled all emails and orcid from HTTP responses are masked. Values "yes" or "no". | "no" | + +| Environment Variable | Type | Optional | Description | Default Value | +| -------------------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | +| `ADMIN_GROUPS` | string | Yes | Comma-separated list of admin groups with admin permission assigned to the listed users. Example: "admin, ingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `CREATE_DATASET_GROUPS` | string | Yes | Comma-separated list of create dataset groups. Users belong to the listed groups can create dataset with/without PID. Example: "group1, group2". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `CREATE_DATASET_WITH_PID_GROUPS` | string | Yes | Comma-separated list of create dataset with pid groups. Users belong to the listed groups can create dataset with PID. Example: "group1, group2". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `CREATE_DATASET_PRIVILEGED_GROUPS` | string | Yes | | | +| `CREATE_JOB_PRIVILEGED_GROUPS` | string | Yes | | | +| `UPDATE_JOB_PRIVILEGED_GROUP` | string | Yes | | | +| `DELETE_GROUPS` | string | Yes | Comma-separated list of delete groups. Users belong to the listed groups can delete any dataset, origDatablocks, datablocks, etc. For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `DATASET_CREATION_VALIDATION_ENABLED` | boolean | | Flag to enable/disable dataset validation to validate if requested new dataset is valid with given regular expression. Preconfigure **DATASET_CREATION_VALIDATION_REGEX** variable is required. | false | +| `DATASET_CREATION_VALIDATION_REGEX` | string | | Regular expression validation for new dataset request. | "" | +| `POLICY_GROUPS` | string | Yes | Comma-separated list of policy groups with permission to create any policy. Example: "policyadmin". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `PROPOSAL_GROUPS` | string | Yes | Comma-separated list of proposal groups with permission to create any proposals. Example: "proposaladmin, proposalingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `SAMPLE_GROUPS` | string | Yes | Comma-separated list of sample groups with permission to create any samples. Example: "sampleadmin, sampleingestor". For more details check: [Scicat Documentation](https://scicatproject.github.io/documentation/Development/v4.x/backend/authorization.html) | | +| `SAMPLE_PRIVILEGED_GROUPS` | string | Yes | | "sampleingestor" | +| `ACCESS_GROUPS_GRAPHQL_ENABLED` | string | Yes | Flag to enable/disable the GraphQL service to get access groups. Requires configuration of `ACCESS_GROUP_SERVICE_TOKEN`, `ACCESS_GROUP_SERVICE_API_URL`, and `ACCESS_GROUP_SERVICE_HANDLER`. | true | +| `ACCESS_GROUPS_SERVICE_TOKEN` | string | Yes | Authentication token used if access groups are obtained from a third-party service. Not used by the vanilla installation, but only if the instance is customized to use an external service to provide user groups, like the ESS example. | | +| `ACCESS_GROUP_SERVICE_API_URL` | string | Yes | URL of the service providing the users' access groups. Not used by the vanilla installation, but only if the instance is customized to use an external service to provide user groups, like the ESS example. | | +| `ACCESS_GROUP_SERVICE_HANDLER` | string | Yes | Configuration property that points to the source of a module. This module provides a specific responseProcessor function for handling GraphQL responses and a query template for making GraphQL requests. | | +| `ACCESS_GROUPS_STATIC_ENABLED` | string | Yes | Flag to enable/disable automatic assignment of predefined access groups to all users. | true | +| `ACCESS_GROUPS_STATIC_VALUES` | string | Yes | Comma-separated list of access groups automatically assigned to all users. Example: "scicat, user". | | +| `ACCESS_GROUPS_OIDCPAYLOAD_ENABLED` | string | Yes | Flag to enable/disable fetching access groups directly from OIDC response. Requires specifying a field via `OIDC_ACCESS_GROUPS_PROPERTY` to extract access groups. | false | +| `ACCESS_GROUPS_LDAPPAYLOAD_ENABLED` | string | Yes | Flag to enable/disable fetching access groups directly from Ldap response. Requires specifying a field via `LDAP_ACCESS_GROUPS_PROPERTY` to extract access groups. | false | +| `DOI_PREFIX` | string | | The facility DOI prefix, with trailing slash. | | +| `DOI_SHORT_SUFFIX` | string | | By default `uuidv4` is used to generate the DOI suffix but if this flag is `true` the shorter version of 10 random characters is used as DOI suffix. | | +| `DOI_USERNAME` | string | | The facility DOI DataCite username. | | +| `DOI_PASSWORD` | string | | The facility DOI DataCite password. | | +| `PUBLISHED_DATA_CONFIG_FILE` | string | | Path to the file containing the metadata and UI schemas for PublishedData. | `"publishedDataConfig.json"` | +| `AJV_CUSTOM_DEFINITIONS_FILE` | string | | The path to a JavaScript module that defines custom AJV keywords or dynamic defaults functions. | | +| `EXPRESS_SESSION_SECRET` | string | No | Secret used to set up express session. Required if using OIDC authentication | | +| `EXPRESS_SESSION_STORE` | string | Yes | Where to store the express session. When "mongo" on mongo else in memory | | +| `HTTP_MAX_REDIRECTS` | number | Yes | Max redirects for HTTP requests. | 5 | +| `HTTP_TIMEOUT` | number | Yes | Timeout for HTTP requests in ms. | 5000 | +| `JWT_SECRET` | string | | The secret for your JWT token, used for authorization. | | +| `JWT_EXPIRES_IN` | number | Yes | How long, in seconds, the JWT token is valid. | 3600 | +| `LDAP_URL` | string | Yes | The URL to your LDAP server. | | +| `LDAP_BIND_DN` | string | Yes | Bind*DN for your LDAP server. | | +| `LDAP_BIND_CREDENTIALS` | string | Yes | Credentials for your LDAP server. | | +| `LDAP_SEARCH_BASE` | string | Yes | Search base for your LDAP server. | | +| `LDAP_SEARCH_FILTER` | string | Yes | Search filter for your LDAP server. | | +| `LDAP_GROUP_SEARCH_BASE` | string | Yes | Search base for the user groups. | | +| `LDAP_GROUP_SEARCH_FILTER` | string | Yes | Search filter for the user groups. | | +| `LDAP_ACCESS_GROUPS_PROPERTY` | string | Yes | Target field to get the access groups value from Ldap response. | | +| `LDAP_USERNAME_ATTR` | string | Yes | Target field to get the username from the Ldap response. Defaults to displayName. | | +| `OIDC_ISSUER` | string | Yes | URL of the OIDC server providing the authentication service. Example: https://identity.esss.dk/realm/ess. | | +| `OIDC_CLIENT_ID` | string | Yes | Identity of the client used to obtain the user token. Example: scicat. | | +| `OIDC_ADDITIONAL_AUTHORIZED_PARTIES` | string | No | Comma-separated list of additional OIDC client IDs allowed to present tokens to this backend. Used for token exchange scenarios where a third-party client obtains a token on behalf of a user. The client ID must appear as the `azp` claim in the token. Example: `additional-client1, additional-client2`. | | +| `OIDC_CLIENT_SECRET` | string | Yes | Secret to provide to the OIDC service to obtain the user token. Example: Aa1JIw3kv3mQlGFWhRrE3gOdkH6xreAwro. | | +| `OIDC_CALLBACK_URL` | string | Yes | SciCat callback URL to redirect to after a successful login. Example: http://myscicat/api/v3/oidc/callback. | | +| `OIDC_SCOPE` | string | Yes | Space-separated list of info returned by the OIDC service. Example: "openid profile email". | | +| `OIDC_SUCCESS_URL` | string | Yes | SciCat Frontend auth-callback URL. Required to pass user credentials to SciCat Frontend after OIDC login. Example: https://myscicatfrontend/auth-callback. Must be `frontend-base-url/auth-callback` or `frontend-base-url/login` for the official SciCat frontend. | | +| `OIDC_RETURN_URL` | string | Yes | The path segment within the SciCat Frontend to redirect to, passed as query param in `OIDC_SUCCESS_URL` and handled by frontend. Example: /datasets. | | +| `OIDC_FRONTEND_CLIENTS` | string | Yes | Comma separated list of additional frontend OIDC clients for this backend. Example: scilog,maxiv. Their success and return URLs can be configured by setting `OIDC*${CLIENT}_SUCCESS_URL` (E.g. `OIDC_SCILOG_SUCCESS_URL`) and `OIDC_${CLIENT}\_RETURN_URL` | | +| `OIDC_ACCESS_GROUPS` | string | Yes | Functionality is still unclear. | | +| `OIDC_ACCESS_GROUPS_PROPERTY` | string | Yes | Target field to get the access groups value from OIDC response. | | +| `OIDC_USERINFO_MAPPING_FIELD_USERNAME` | string | Yes | Comma-separated list of fields from the OIDC response to use as the user's profile username. Example:`OIDC_USERINFO_MAPPING_FIELD_USERNAME="iss, sub"`. | "preferred_username" \|\| "name" | +| `OIDC_USERINFO_MAPPING_FIELD_DISPLAYNAME` | string | Yes | Field from the OIDC response to use as the user's profile display name. Example:`OIDC_USERINFO_MAPPING_FIELD_DISPLAYNAME="preferred_username"`. | "name" | +| `OIDC_USERINFO_MAPPING_FIELD_EMAIL` | string | Yes | Field from the OIDC response to use as the user's profile email. | "email" | +| `OIDC_USERINFO_MAPPING_FIELD_FAMILYNAME` | string | Yes | Field from the OIDC response to use as the user's profile family name. | "family_name" | +| `OIDC_USERINFO_MAPPING_FIELD_ID` | string | Yes | Field from the OIDC response to use as the user's profile ID. | "sub" \|\| "user_id" | +| `OIDC_USERINFO_MAPPING_FIELD_THUMBNAILPHOTO` | string | Yes | Field from the OIDC response to use as the user's profile thumbnail photo. | "thumbnailPhoto" | +| `OIDC_USERINFO_MAPPING_FIELD_PROVIDER` | string | Yes | Field from the OIDC response to use as the user's profile provider. | "iss" | +| `OIDC_USERINFO_MAPPING_FIELD_GROUP` | string | Yes | Field from the OIDC response to use as the user's profile group. | "groups" | +| `OIDC_USERQUERY_OPERATOR` | string | Yes | Specifies the operator ("or" or "and") for UserModel.findOne queries, determining the logic used to match fields like "username" or "email". Example:`UserModel.findOne({$or: {"username":"testUser", "email":"test@test.com"}})`. | "or" | +| `OIDC_USERQUERY_FILTER` | string | Yes | Defines key-value pairs for UserModel.findOne queries, using a "key:value" format. Example:`OIDC_USERQUERY_FILTER="username:sub, email:email"`. | "username:username, email:email" | +| `LOGBOOK_ENABLED` | string | Yes | Flag to enable/disable the Logbook endpoints. Values "yes" or "no". | "no" | +| `LOGBOOK_BASE_URL` | string | Yes | The base URL to the Logbook API. Only required if Logbook is enabled. | | +| `METADATA_KEYS_RETURN_LIMIT` | number | Yes | The return limit for the`/Datasets/metadataKeys`endpoint. | | +| `METADATA_PARENT_INSTANCES_RETURN_LIMIT` | number | Yes | The return limit of Datasets to extract metadata keys from for the`/Datasets/metadataKeys`endpoint. | | +| `MONGODB_URI` | string | | The URI for your MongoDB instance. | | +| `OAI_PROVIDER_ROUTE` | string | Yes | URI to OAI provider, used for the`/publisheddata/:id/resync`endpoint. | | +| `PID_PREFIX` | string | | The facility PID prefix, with trailing slash. | | +| `PUBLIC_URL_PREFIX` | string | | The base URL to the facility Landing Page. | | +| `PORT` | number | Yes | The port on which you want to access the app. | 3000 | +| `RABBITMQ_ENABLED` | string | Yes | Flag to enable/disable RabbitMQ consumer. Values "yes" or "no". | "no" | +| `RABBITMQ_HOSTNAME` | string | Yes | The hostname of the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | +| `RABBITMQ_USERNAME` | string | Yes | The username used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | +| `RABBITMQ_PASSWORD` | string | Yes | The password used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | +| `RABBITMQ_PORT` | number | Yes | The port of the RabbitMQ message broker. Only required if RabbitMQ is enabled. | 5672 | +| `REGISTER_DOI_URI` | string | | URI to the organization that registers the facility's DOIs using the new DataCite API. | "https://api.test.datacite.org/dois" | +| `REGISTER_DOI_URI_V3` | string | | URI to the organization that registers the facility's DOIs using the old DataCite API. | "https://mds.test.datacite.org/doi" | +| `REGISTER_METADATA_URI` | string | | URI to the organization that registers the facility's published data metadata. | "https://mds.test.datacite.org/metadata" | +| `DOI_USERNAME` | string | Yes | DOI Username. | "username" | +| `DOI_PASSWORD` | string | Yes | DOI Password. | "password" | +| `SITE` | string | | The name of your site. | | +| `EMAIL_TYPE` | string | Yes | The type of your email provider. Options are "smtp" or "ms365". | "smtp" | +| `EMAIL_FROM` | string | Yes | Email address that emails should be sent from. | | +| `EMAIL_REPLYTO` | string | Yes | Email 'Reply-To' field. | | +| `SMTP_HOST` | string | Yes | Host of SMTP server. | | +| `SMTP_MESSAGE_FROM` | string | Yes | (Deprecated) Alternate spelling of EMAIL_FROM. | | +| `SMTP_PORT` | number | Yes | Port of SMTP server. | 587 | +| `SMTP_SECURE` | bool | Yes | Use encrypted SMTPS. | "no" | +| `MS365_TENANT_ID` | string | Yes | Tenant ID for sending emails over Microsoft Graph API. | | +| `MS365_CLIENT_ID` | string | Yes | Client ID for sending emails over Microsoft Graph API | | +| `MS365_CLIENT_SECRET` | string | Yes | Client Secret for sending emails over Microsoft Graph API | | +| `POLICY_PUBLICATION_SHIFT` | integer | Yes | Embargo period expressed in years. | 3 years | +| `POLICY_RETENTION_SHIFT` | integer | Yes | Retention period (how long the facility will hold on to data) expressed in years. | -1 (indefinitely) | +| `OPENSEARCH_ENABLED` | string | | Controls whether OpenSearch is enabled on application startup. If not provided or set to `no`, OpenSearch will not be instantiated. | "no" | +| `OPENSEARCH_DEFAULT_INDEX` | string | | Specifies the default index. If not provided, a default index named `dataset` will be created automatically. | "dataset" | +| `OPENSEARCH_HOST` | string | | Host of Opensearch server instance. | | +| `OPENSEARCH_USERNAME` | string | Yes | Username for OpenSearch authentication. Defaults to `admin` in standard deployments but can be configured to use a custom user with appropriate permissions. | "admin" | +| `OPENSEARCH_PASSWORD` | string | | Password used for OpenSearch authentication. Must match `OPENSEARCH_INITIAL_ADMIN_PASSWORD` used when creating the OpenSearch container. | | +| `OPENSEARCH_REFRESH` | string | | Controls index refresh behavior. `wait_for`waits for the next refresh cycle before returning, which is useful for development and testing.`false`skips waiting and is recommended for production. Defaults to false. | false | +| `OPENSEARCH_DATA_SYNC_BATCH_SIZE` | number | | Number of documents fetched from MongoDB per batch during OpenSearch data sync. | 1000 | +| `FRONTEND_CONFIG_FILE` | string | | The file name for frontend configuration, located in the`/src/config`directory by default. | "./src/config/frontend.config.json" | +| `FRONTEND_THEME_FILE` | string | | The file name for frontend theme, located in the`/src/config`directory by default. | "./src/config/frontend.theme.json" | +| `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" | +| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" | +| `DATASET_TYPES_FILE` | string | | | "datasetTypes.json" | +| `SWAGGER_PATH` | string | Yes | swaggerPath is the path where the swagger UI will be available. | "explorer" | +| `MAX_FILE_UPLOAD_SIZE` | string | Yes | Maximum allowed file upload size. | "16mb" | +| `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts, relative to the project root directory | "functionalAccounts.json" | +| `JOB_CONFIGURATION_FILE` | string | Yes | Path of a job configuration file (conventionally`"jobConfig.yaml"`). If unset, jobs are disabled | | +| `JOB_DEFAULT_STATUS_CODE` | string | Yes | Default statusCode for new jobs | "jobSubmitted" | +| `JOB_DEFAULT_STATUS_MESSAGE` | string | Yes | Default statusMessage for new jobs | "Job submitted." | +| `TRACKABLE_STRATEGY` | string | Yes | "document" or "delta". Document strategy (default): Stores full document copies in the history collection for both before and after states. Delta strategy: Only stores the fields that changed, saving database space. | "document" | +| `TRACKABLES` | string | Yes | The TRACKABLES environment variable configures which data models are tracked by the history system. When specified, only models listed in this variable will have their changes recorded in the History collection. | Dataset\[,Proposal\]\[,Sample\]\[,Instrument\]\[,PublishedData\] | +| `HISTORY_ACCESS_DATASET_GROUPS` | string | Yes | Roles in this list will be able to access history dataset records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_PROPOSAL_GROUPS` | string | Yes | Roles in this list will be able to access history proposal records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_SAMPLE_GROUPS` | string | Yes | Roles in this list will be able to access history sample records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_INSTRUMENT_GROUPS` | string | Yes | Roles in this list will be able to access history instrument records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_PUBLISHED_DATA_GROUPS` | string | Yes | Roles in this list will be able to access history published data records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_POLICIES_GROUPS` | string | Yes | Roles in this list will be able to access history policies data records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_DATABLOCK_GROUPS` | string | Yes | Roles in this list will be able to access history datablock records | \[role1\]\[,role2\]\[,roleN\]... | +| `HISTORY_ACCESS_ATTACHMENT_GROUPS` | string | Yes | Roles in this list will be able to access history attachment records | \[role1\]\[,role2\]\[,roleN\]... | +| `MASK_PERSONAL_INFO` | string | Yes | When enabled all emails and orcid from HTTP responses are masked. Values "yes" or "no". | "no" | +| `SSE_TICKET_EXPIRES_IN` | number | Yes | How long, in seconds, the server sent event stream connection is valid. | 3600 | ## Migrating from the old SciCat Backend diff --git a/src/serverSentEvent/sse.controller.ts b/src/serverSentEvent/sse.controller.ts index 400bfbf78..332e4063d 100644 --- a/src/serverSentEvent/sse.controller.ts +++ b/src/serverSentEvent/sse.controller.ts @@ -42,7 +42,7 @@ export class SseController { @ApiOperation({ summary: "Subscribe to server-sent events.", description: - "Opens a text/event-stream connection pushing events for documents the authenticated user is allowed to read. Authenticate with a ticket from POST /events/ticket.", + "Opens a event-stream connection pushing events for documents the authenticated user is allowed to read. Authenticate with a ticket from POST /events/ticket.", }) @ApiQuery({ name: "ticket", From e8b2196a83e450de8a877e849a2460ac15ae48ac Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 13:54:22 +0200 Subject: [PATCH 32/34] update docs --- docs/developer-guide/authorization/sse.md | 54 +++++++ docs/developer-guide/sse-module.md | 165 ++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 docs/developer-guide/authorization/sse.md create mode 100644 docs/developer-guide/sse-module.md diff --git a/docs/developer-guide/authorization/sse.md b/docs/developer-guide/authorization/sse.md new file mode 100644 index 000000000..b7a32dc22 --- /dev/null +++ b/docs/developer-guide/authorization/sse.md @@ -0,0 +1,54 @@ +# SSE Authorization Model + +This document describes the authorization model used for the server-sent event stream and associated endpoints. + +## Actions + +The following actions are defined for server-sent events: + +- `SseRead` +- `AccessAny` + +## Permissions + +Permissions are granted cumulatively to users based on their group association. The following permission levels are granted to users: + +### Unauthenticated + +An unauthenticated user has no access to any server-sent event endpoint. + +### Authenticated + +An authenticated user may request a stream ticket and open an event stream. They receive only those documents they are permitted to see, evaluated per event: documents whose `ownerGroup` is one of their groups, or whose `accessGroups` intersect their groups. + +### ADMIN_GROUPS + +If a user is part of a group listed in configuration as part of `ADMIN_GROUPS`, they receive every document emitted on the stream regardless of its groups, and they may read the list of active connections on the instance. + +## Permission Matrix + +Table of the different permission classes defined in casl. For all special permission groups, the full list includes the relevant permissions passed on from generic authenticated user permissions. + +| Operation | Unauthenticated | Authenticated | `ADMIN_GROUPS` | +| ----------- | --------------- | ------------- | -------------- | +| `SseRead` | - | own | any | +| `AccessAny` | - | - | any | + +Legend: + +- any: unrestricted access +- own: access limited to documents owned by or shared with one of the user's groups + +## Implementation Notes + +The definition is implemented in the casl module under `/src/casl/abilities/sse.ability.ts` and accessible elsewhere via `CaslAbilityFactory.sseAccess`. This one function is used to build one casl ability for endpoint and instance authorization: When a user receives permission for an action under some instance-level condition, they should implicitly pass endpoint authorization. + +Unlike other subjects, the casl ability alone does not express the full model. `SseRead` is granted unconditionally to every authenticated user, so endpoint authorization admits anyone logged in. The document-level restriction shown as `own` above is enforced separately in `SseService.emit`, which tests each outgoing document against each connected user before delivering it. Group membership is evaluated against the user captured when the connection opened, so changes take effect on the next connection rather than immediately. + +`SseClass` is an empty marker class rather than a schema. Server-sent events have no persisted document, so it exists only to give casl a subject to attach grants to. + +## Authentication Notes + +The stream endpoint authenticates differently from the rest of the API. `EventSource` cannot set request headers, so `GET /events/stream` accepts a ticket as a query parameter instead of a bearer token. The ticket is a JWT carrying `purpose: "sse"`, minted by `POST /events/ticket`, which is itself guarded by `SseRead`. Authorization is therefore evaluated twice against the same rules, once when the ticket is issued and again when the stream is opened. + +`JwtStrategy.validate` rejects a ticket presented on any other route, and rejects a normal bearer token on the stream route. The first direction is a security property: a credential in a URL reaches access logs and browser history, so it must be useless elsewhere. The second is a consistency choice that makes the ticket the only way to open a stream. diff --git a/docs/developer-guide/sse-module.md b/docs/developer-guide/sse-module.md new file mode 100644 index 000000000..f4e7d558d --- /dev/null +++ b/docs/developer-guide/sse-module.md @@ -0,0 +1,165 @@ +# Server-Sent Events (SSE) + +The backend can push document changes to connected clients over a +`text/event-stream` connection, so a frontend can react to new datasets without +polling. + +## How it works + +``` +MongoDB change stream + │ + SseListener watches the oplog, maps a raw document to a DTO + │ + SseService holds one RxJS Subject per open connection, + │ filters each event per user + SseController exposes the stream over HTTP + │ + client +``` + +Events originate in MongoDB rather than in application code. That is deliberate: +a change stream sees every write regardless of which code path produced it, so a +dataset created through a background job or a migration is broadcast the same way +as one created through the API. The alternative, emitting from service methods, +would silently miss any write that bypassed them. + +## Requirements + +SSE depends on MongoDB change streams, which only exist on a replica set. On +startup `SseListener` probes the connection and, if it finds a standalone +`mongod`, logs a message and leaves SSE disabled. The rest of the API is +unaffected. + +While disabled, the SSE endpoints return `503 Service Unavailable` rather than +failing silently, so a misconfigured deployment is visible instead of merely +quiet. + +To convert a standalone container to a single-node replica set: + +```yaml +services: + mongodb: + image: mongo:latest + command: ["--replSet", "rs0", "--bind_ip_all"] +``` + +then initiate it once: + +```bash +mongosh --eval 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})' +``` + +Existing data in the volume is preserved. + +## Authentication + +`EventSource`, the browser API for consuming SSE, cannot set request headers, so +a bearer token cannot be sent the usual way. The stream therefore uses a +short-lived ticket passed as a query parameter. + +``` +POST /api/v3/events/ticket Authorization: Bearer + → { "ticket": "" } + +GET /api/v3/events/stream?ticket= +``` + +The ticket is a JWT signed with the same secret as a session token, but carrying +`purpose: "sse"`. `JwtStrategy` enforces that claim in both directions: a ticket +is rejected on any route other than the stream, and the stream rejects anything +that is not a ticket. Scoping matters here because a credential in a URL ends up +in access logs, proxy logs, and browser history, so it must be useless anywhere +else. + +Tickets are single-purpose but not single-use. Within their lifetime the same +ticket can open more than one connection, which is why the default lifetime is +short. + +Because tickets expire quickly, a dropped stream cannot simply be reopened at the +same URL. Clients must mint a new ticket for each reconnect. The browser's +built-in `EventSource` reconnect replays the original URL and will fail, so +clients should call `close()` on error and restart the flow from the ticket +request. + +## Authorization + +Two independent checks apply. + +**At connection time**, CASL decides who may open a stream at all. Any +authenticated user has `sse_read`; members of `ADMIN_GROUPS` additionally get +`access_any`, which gates the connections endpoint. + +**Per event**, `SseService` filters every message against every connected user +before sending it. A user receives a document only if one of these holds: + +- the document's `ownerGroup` is one of the user's groups +- the document's `accessGroups` intersect the user's groups +- the user belongs to an admin group + +Filtering happens on emit rather than on subscribe because a user's visibility is +a property of each document, not of the connection. Two users on the same stream +see different subsets of the same event flow. + +## Endpoints + +| Method | Path | Auth | Description | +| ------ | ---------------------------- | ------------------- | ------------------------------------------ | +| `POST` | `/api/v3/events/ticket` | Bearer token | Mints a short-lived stream ticket. | +| `GET` | `/api/v3/events/stream` | Ticket, query param | Opens the event stream. | +| `GET` | `/api/v3/events/connections` | Bearer token, admin | Reports open connections on this instance. | + +### Event format + +```json +{ + "type": "Dataset.created", + "data": { "pid": "20.500.12269/...", "datasetName": "..." } +} +``` + +`type` is `.`. The payload is the document serialized through its +output DTO, not the raw MongoDB document, so clients see the same shape the REST +API returns. + +## What is watched + +A registry in `sse.listener.ts` maps collections to entities and DTOs. Today it +holds one entry: + +| Collection | Entity | Actions | +| ---------- | --------- | --------- | +| `Dataset` | `Dataset` | `created` | + +Only `insert` operations are mapped. The `SseAction` type reserves `updated` and +`deleted` for later, but no change stream operation maps to them yet, so those +events are never emitted. + +Adding a collection means adding a registry entry with its DTO. The change stream +pipeline is built from the registry keys, so no other change is needed. + +## Configuration + +| Variable | Type | Default | Description | +| ----------------------- | ------ | ------- | --------------------------- | +| `SSE_TICKET_EXPIRES_IN` | number | `60` | Ticket lifetime in seconds. | + +## Limits and operational behaviour + +**Connections per user.** Five per user per instance. A sixth request returns +`403 Forbidden`. The cap is per instance, so behind a load balancer the effective +total is five times the number of replicas. + +**Reconnect on change stream failure.** If the change stream errors, the listener +retries up to five times with exponential backoff starting at two seconds. After +that it stops and logs an error, and SSE stays down until the process restarts. +Worth alerting on, since nothing recovers automatically past that point. + +**Multi-instance deployments.** Connections are held in an in-memory `Map`, so +each instance knows only its own clients. Every instance runs its own change +stream and broadcasts to its own connections, so clients receive events wherever +they land. The consequence is that `/events/connections` reports one instance's +view rather than a cluster-wide total. + +**Shutdown.** On module destroy the change stream is closed and every client +Subject is completed, so connections end cleanly rather than being dropped. From 8f69245b067a1dcd903ee7c9bf966ae9c28ec376 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 16:50:00 +0200 Subject: [PATCH 33/34] update doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0c3e163e..4a171645f 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Valid environment variables for the .env file. See [.env.example](/.env.example) | `HISTORY_ACCESS_DATABLOCK_GROUPS` | string | Yes | Roles in this list will be able to access history datablock records | \[role1\]\[,role2\]\[,roleN\]... | | `HISTORY_ACCESS_ATTACHMENT_GROUPS` | string | Yes | Roles in this list will be able to access history attachment records | \[role1\]\[,role2\]\[,roleN\]... | | `MASK_PERSONAL_INFO` | string | Yes | When enabled all emails and orcid from HTTP responses are masked. Values "yes" or "no". | "no" | -| `SSE_TICKET_EXPIRES_IN` | number | Yes | How long, in seconds, the server sent event stream connection is valid. | 3600 | +| `SSE_TICKET_EXPIRES_IN` | number | Yes | How long, in seconds, the server sent event stream connection is valid. | 60 | ## Migrating from the old SciCat Backend From 22d050de9e9a914a2f7498796920395846e9eaad Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 24 Aug 2026 16:58:29 +0200 Subject: [PATCH 34/34] add excludeExtraneousValues: true for message emit to discard excluded fields --- src/serverSentEvent/sse.listener.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serverSentEvent/sse.listener.ts b/src/serverSentEvent/sse.listener.ts index 8cffe9b2c..eb02675de 100644 --- a/src/serverSentEvent/sse.listener.ts +++ b/src/serverSentEvent/sse.listener.ts @@ -96,7 +96,9 @@ export class SseListener implements OnModuleInit, OnModuleDestroy { this.sseService.emit({ entity: registryEntry.entity, action, - message: plainToInstance(registryEntry.dto, rawDoc), + message: plainToInstance(registryEntry.dto, rawDoc, { + excludeExtraneousValues: true, + }), }); } private async onStreamError(error: unknown): Promise {