diff --git a/.env.example b/.env.example index 37d0e45f..82eb424d 100644 --- a/.env.example +++ b/.env.example @@ -5,13 +5,29 @@ HOST_HTTP_PORT=8080 FIRST_INSTALL=yes # Encyption secrets has to be atleast 32 characters long -ENCRYPTION_SECRET=0123456789012345678901234567890123456789 +ENCRYPTION_SECRET=change_me_to_a_long_random_string_of_at_least_32_chars #------------------------------------- # Climsoft web database credentials DB_NAME=climsoft DB_PASSWORD=my_password +#------------------------------------- +# Climate Products (Apache Superset) settings +# Set SUPERSET_ENABLED=true and start with --profile superset to enable +SUPERSET_ENABLED=false +# Must be a random string of at least 42 characters +SUPERSET_SECRET_KEY=change_me_to_a_long_random_string_of_at_least_42_chars +# Human sysadmin account — used to log into the Superset UI +SUPERSET_ADMIN_USERNAME=superset_admin +SUPERSET_ADMIN_PASSWORD=change_me +# Service account used by the Climsoft API to generate guest tokens (keep separate from the sysadmin account) +SUPERSET_SERVICE_USERNAME=climsoft_service +SUPERSET_SERVICE_PASSWORD=change_me_service +# In development Superset runs directly on port 8088 (no nginx), so set this to empty. +# In test/prod it runs behind nginx at /superset/ — leave unset to use the /superset default. +# WEBSERVER_PREFIX= + #------------------------------------- # Climsoft V4 database credentials V4_SAVE=yes diff --git a/back-end/api/package.json b/back-end/api/package.json index dee4023b..029a76d6 100644 --- a/back-end/api/package.json +++ b/back-end/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "preview-3.0.2", + "version": "preview-3.0.4", "description": "Climsoft API", "author": "Patrick Munyoki = { - [AdapterLanguageEnum.PYTHON]: ['requirements.txt'], - [AdapterLanguageEnum.R]: ['renv.lock', 'DESCRIPTION'], - [AdapterLanguageEnum.JAVASCRIPT]: ['package.json', 'package-lock.json'], - [AdapterLanguageEnum.SQL]: ['extensions.txt'], -}; + /** + * Canonical entry-point filename the runner executes. Users don't + * choose this — it's a language convention so the runner never has + * to guess. Matches the filenames shipped by the starter templates. + */ + entryPoint: string; +} -/** - * Canonical entry-point filename required at the root of an uploaded zip, one - * per language. Users don't choose this — the convention is enforced so the - * runner always knows what to execute without a stored `entryPoint` field. - * - * Matches the filenames shipped by the starter templates. - */ -export const CANONICAL_ENTRY_POINT: Record = { - [AdapterLanguageEnum.PYTHON]: 'main.py', - [AdapterLanguageEnum.R]: 'main.R', - [AdapterLanguageEnum.JAVASCRIPT]: 'index.js', - [AdapterLanguageEnum.SQL]: 'transform.sql', +export const LANGUAGE_CONVENTIONS: Record = { + [AdapterLanguageEnum.PYTHON]: { manifest: 'requirements.txt', entryPoint: 'main.py' }, + [AdapterLanguageEnum.R]: { manifest: 'DESCRIPTION', entryPoint: 'main.R' }, + [AdapterLanguageEnum.JAVASCRIPT]: { manifest: 'package.json', entryPoint: 'index.js' }, + [AdapterLanguageEnum.SQL]: { manifest: 'extensions.txt', entryPoint: 'transform.sql' }, }; diff --git a/back-end/api/src/metadata/adapters/dtos/create-adapter-specification.dto.ts b/back-end/api/src/metadata/adapters/dtos/create-adapter-specification.dto.ts index 68ad22d3..c8fcee85 100644 --- a/back-end/api/src/metadata/adapters/dtos/create-adapter-specification.dto.ts +++ b/back-end/api/src/metadata/adapters/dtos/create-adapter-specification.dto.ts @@ -7,7 +7,7 @@ import { AdapterLanguageEnum } from '../enums/adapter-language.enum'; * returned the `scriptDirName` (UUID) the client sends here. * * The entry point is NOT sent by the client — it is a language-level - * convention (see `CANONICAL_ENTRY_POINT`) that the API enforces at + * convention (see `LANGUAGE_CONVENTIONS`) that the API enforces at * upload-preview time. */ export class CreateAdapterSpecificationDto { diff --git a/back-end/api/src/metadata/adapters/dtos/view-adapter-specification.dto.ts b/back-end/api/src/metadata/adapters/dtos/view-adapter-specification.dto.ts index 5b4af4e5..099bddd3 100644 --- a/back-end/api/src/metadata/adapters/dtos/view-adapter-specification.dto.ts +++ b/back-end/api/src/metadata/adapters/dtos/view-adapter-specification.dto.ts @@ -2,6 +2,7 @@ import { AdapterLanguageEnum } from '../enums/adapter-language.enum'; export class ViewAdapterSpecificationDto { id!: number; + systemKey!: string | null; name!: string; description!: string; language!: AdapterLanguageEnum; diff --git a/back-end/api/src/metadata/adapters/entities/adapter-specification.entity.ts b/back-end/api/src/metadata/adapters/entities/adapter-specification.entity.ts index cc48a6b2..7517da66 100644 --- a/back-end/api/src/metadata/adapters/entities/adapter-specification.entity.ts +++ b/back-end/api/src/metadata/adapters/entities/adapter-specification.entity.ts @@ -2,6 +2,15 @@ import { Check, Column, Entity, Index, PrimaryGeneratedColumn } from "typeorm"; import { AppBaseEntity, BaseLogVo } from "src/shared/entity/app-base-entity"; import { AdapterLanguageEnum } from "../enums/adapter-language.enum"; +export interface AdapterSpecificationLogVo extends BaseLogVo { + name: string; + description: string | null; + language: AdapterLanguageEnum; + scriptDirName: string; + disabled: boolean; + comment: string | null; +} + /** * A user-uploaded script that translates a foreign file format to/from the * canonical format the existing import/export pipelines understand. @@ -35,6 +44,9 @@ export class AdapterSpecificationEntity extends AppBaseEntity { @PrimaryGeneratedColumn({ name: "id", type: "int" }) id!: number; + @Column({ name: "system_key", type: "varchar", unique: true, nullable: true }) + systemKey!: string | null; + @Column({ name: "name", type: "varchar", unique: true }) name!: string; @@ -56,7 +68,7 @@ export class AdapterSpecificationEntity extends AppBaseEntity { @Column({ name: "script_dir_name", type: "varchar", unique: true }) scriptDirName!: string; - @Column({ type: "boolean", default: false }) + @Column({ name: "disabled", type: "boolean", default: false }) @Index() disabled!: boolean; @@ -64,5 +76,5 @@ export class AdapterSpecificationEntity extends AppBaseEntity { comment!: string | null; @Column({ name: "log", type: "jsonb", nullable: true }) - log!: BaseLogVo[] | null; + log!: AdapterSpecificationLogVo[] | null; } diff --git a/back-end/api/src/metadata/adapters/services/adapters.service.ts b/back-end/api/src/metadata/adapters/services/adapters.service.ts index 1dd59fba..a59defca 100644 --- a/back-end/api/src/metadata/adapters/services/adapters.service.ts +++ b/back-end/api/src/metadata/adapters/services/adapters.service.ts @@ -1,6 +1,6 @@ import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleInit } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { IsNull, Repository } from 'typeorm'; import fs from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; @@ -19,7 +19,7 @@ import { AdapterRef, AdapterRunMetadata, AdapterRunnerService, AdapterRunResult import { CacheLoadResult, MetadataCache } from 'src/shared/cache/metadata-cache'; import { AdapterTestRunPreviewDto } from '../dtos/adapter-test-run-preview.dto'; import { FileProcessingErrorType } from 'src/metadata/file-processing-error.model'; -import { CANONICAL_ENTRY_POINT, MANIFEST_FILENAMES } from '../adapter-language-conventions'; +import { LANGUAGE_CONVENTIONS } from '../adapter-language-conventions'; @Injectable() export class AdaptersService implements OnModuleInit { @@ -157,9 +157,9 @@ export class AdaptersService implements OnModuleInit { } /** - * Returns an error message if none of the accepted manifest filenames for - * the language are present at the ROOT of the tree (top-level files only, - * no directory prefix). Returns `undefined` if a valid manifest is present. + * Returns an error message if the language's required manifest file is + * not present at the ROOT of the tree (top-level files only, no + * directory prefix). Returns `undefined` if the manifest is present. * * Root-only enforcement is intentional: runners execute from the extracted * script directory and expect the manifest at that same level. A zip whose @@ -167,12 +167,10 @@ export class AdaptersService implements OnModuleInit { * fail here rather than at runtime. */ private checkManifestAtRoot(fileTree: FileTreeEntry[], language: AdapterLanguageEnum): string | undefined { - const acceptedNames = MANIFEST_FILENAMES[language]; - const found = acceptedNames.some(name => - fileTree.some(e => !e.isDirectory && e.path === name), - ); + const expected = LANGUAGE_CONVENTIONS[language].manifest; + const found = fileTree.some(e => !e.isDirectory && e.path === expected); if (found) return undefined; - return `Missing manifest file for '${language}'. Expected one of: ${acceptedNames.join(', ')} at the root of the archive.`; + return `Missing manifest file '${expected}' at the root of the archive for language '${language}'.`; } /** @@ -182,7 +180,7 @@ export class AdaptersService implements OnModuleInit { * `transform.sql`) enforced here so the runner never has to guess. */ private checkEntryPointAtRoot(fileTree: FileTreeEntry[], language: AdapterLanguageEnum): string | undefined { - const expected = CANONICAL_ENTRY_POINT[language]; + const expected = LANGUAGE_CONVENTIONS[language].entryPoint; const found = fileTree.some(e => !e.isDirectory && e.path === expected); if (found) return undefined; return `Missing entry-point file '${expected}' at the root of the archive for language '${language}'.`; @@ -259,16 +257,19 @@ export class AdaptersService implements OnModuleInit { public async update(id: number, dto: UpdateAdapterSpecificationDto, userId: number): Promise { const entity = await this.findEntity(id); - entity.name = dto.name; - entity.description = dto.description ?? null; - entity.disabled = dto.disabled; - entity.comment = dto.comment ?? null; + if (entity.systemKey !== null) { + entity.disabled = dto.disabled; + } else { + entity.name = dto.name; + entity.description = dto.description ?? null; + entity.disabled = dto.disabled; + entity.comment = dto.comment ?? null; + const scriptDir = this.fileIO.getAdapterScriptDir(dto.scriptDirName); + await this.assertDirExists(scriptDir, `Script directory '${dto.scriptDirName}' not found. Please upload the zip file first.`); + entity.scriptDirName = dto.scriptDirName; + } entity.entryUserId = userId; - const scriptDir = this.fileIO.getAdapterScriptDir(dto.scriptDirName); - await this.assertDirExists(scriptDir, `Script directory '${dto.scriptDirName}' not found. Please upload the zip file first.`); - entity.scriptDirName = dto.scriptDirName; - await this.adapterRepo.save(entity); await this.cache.invalidate(); @@ -429,16 +430,19 @@ export class AdaptersService implements OnModuleInit { public async delete(id: number): Promise { const entity: AdapterSpecificationEntity = await this.findEntity(id); + if (entity.systemKey !== null) { + throw new BadRequestException(`Adapter '${entity.name}' is a system adapter and cannot be deleted`); + } await this.adapterRepo.remove(entity); await this.cache.invalidate(); this.logger.log(`Adapter deleted: #${id}. On-disk script directories are retained.`); } public async deleteAll(): Promise { - const entities: AdapterSpecificationEntity[] = await this.adapterRepo.find(); + const entities: AdapterSpecificationEntity[] = await this.adapterRepo.find({ where: { systemKey: IsNull() } }); await this.adapterRepo.remove(entities); await this.cache.invalidate(); - this.logger.log(`All adapters deleted. On-disk script directories are retained.`); + this.logger.log(`All user-defined adapters deleted. System adapters and on-disk script directories are retained.`); } //-------------------------------------------------------------------- @@ -456,6 +460,7 @@ export class AdaptersService implements OnModuleInit { private toViewDto(entity: AdapterSpecificationEntity): ViewAdapterSpecificationDto { return { id: entity.id, + systemKey: entity.systemKey ?? null, name: entity.name, description: entity.description ?? '', language: entity.language, diff --git a/back-end/api/src/metadata/adapters/templates/javascript/package-lock.json b/back-end/api/src/metadata/adapters/templates/javascript/package-lock.json deleted file mode 100644 index 1f3bab90..00000000 --- a/back-end/api/src/metadata/adapters/templates/javascript/package-lock.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "climsoft-adapter", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "climsoft-adapter", - "version": "1.0.0", - "dependencies": {} - } - } -} diff --git a/back-end/api/src/metadata/export-specifications/dtos/raw-export-parameters.dto.ts b/back-end/api/src/metadata/export-specifications/dtos/raw-export-parameters.dto.ts index 9364f1ad..f6bdc2f6 100644 --- a/back-end/api/src/metadata/export-specifications/dtos/raw-export-parameters.dto.ts +++ b/back-end/api/src/metadata/export-specifications/dtos/raw-export-parameters.dto.ts @@ -1,7 +1,6 @@ import { IsBoolean, IsOptional } from "class-validator"; export class RawExportParametersDto { - // Data @IsOptional() @IsBoolean() convertDatetimeToDisplayTimeZone?: boolean; diff --git a/back-end/api/src/metadata/source-specifications/dtos/import-source-tabular-params.dto.ts b/back-end/api/src/metadata/source-specifications/dtos/import-source-tabular-params.dto.ts index 6deee9ff..b60e6e9b 100644 --- a/back-end/api/src/metadata/source-specifications/dtos/import-source-tabular-params.dto.ts +++ b/back-end/api/src/metadata/source-specifications/dtos/import-source-tabular-params.dto.ts @@ -75,6 +75,31 @@ export class FlagDefinition { flagsToFetch?: FlagToFetch[]; } +/** + * When set on {@link ImportSourceTabularParamsDto.inlineFlagRule}, cells in + * the observation `value` column carry a trailing alphabetic run interpreted + * as a flag abbreviation. Example: `0.5T` splits into value `0.5` and flag `T`. + * + * Applies uniformly whether the `value` column comes from an explicit + * `valueDefinition.valueColumnPosition` or from a wide-pivot UNPIVOT + * (multiple elements, day columns range, or hour columns range). + * + * The split rule itself is fixed — trailing `[A-Za-z]+` is the flag; the + * numeric prefix is the value. Both parts may be empty and are handled by + * the existing missing-value logic. + */ +export class InlineFlagRule { + /** + * Optional source flag string → database flag id mapping. When omitted, + * the extracted flag string is matched case-insensitively against + * `flags.abbreviation` — same semantics as `FlagDefinition.flagsToFetch`. + */ + @IsOptional() + @ValidateNested({ each: true }) + @Type(() => FlagToFetch) + flagsToFetch?: FlagToFetch[]; +} + export class ValueDefinition { /** Value column position. */ @IsInt() @@ -279,8 +304,11 @@ export enum DateTimeFormat { // 12-hour clock with AM/PM (Excel and US-locale form entries) YMD_DASH_HM_AMPM = '%Y-%m-%d %I:%M %p', + YMD_DASH_HMS_AMPM = '%Y-%m-%d %I:%M:%S %p', // 2023-12-13 10:30:00 AM DMY_SLASH_HM_AMPM = '%d/%m/%Y %I:%M %p', + DMY_SLASH_HMS_AMPM = '%d/%m/%Y %I:%M:%S %p', // 13/12/2023 10:30:00 AM MDY_SLASH_HM_AMPM = '%m/%d/%Y %I:%M %p', + MDY_SLASH_HMS_AMPM = '%m/%d/%Y %I:%M:%S %p', // 12/13/2023 10:30:00 AM // Dot-separated (German / Russian / Eastern European locales) DMY_DOT_HMS = '%d.%m.%Y %H:%M:%S', @@ -474,6 +502,16 @@ export class ImportSourceTabularParamsDto { @Type(() => ValueDefinition) valueDefinition?: ValueDefinition; + /** + * Opt-in: cells in the `value` column carry a trailing flag suffix. + * Mutually exclusive with `valueDefinition.flagDefinition` — if both are + * set, the explicit flag column takes precedence and this is ignored. + */ + @IsOptional() + @ValidateNested() + @Type(() => InlineFlagRule) + inlineFlagRule?: InlineFlagRule; + @IsOptional() @ValidateNested() @Type(() => CommentDefinition) diff --git a/back-end/api/src/metadata/source-specifications/entities/source-specification.entity.ts b/back-end/api/src/metadata/source-specifications/entities/source-specification.entity.ts index a404a575..f10f6309 100644 --- a/back-end/api/src/metadata/source-specifications/entities/source-specification.entity.ts +++ b/back-end/api/src/metadata/source-specifications/entities/source-specification.entity.ts @@ -35,10 +35,6 @@ export class SourceSpecificationEntity extends AppBaseEntity { @Column({ name: "parameters", type: "jsonb" }) parameters!: SourceParameters; - @Column({ name: "order_number", type: "int", nullable: true }) - @Index() - orderNumber!: number | null; // TODO. Deprecate this in future. Given that auto imports are scheduled by time. This is not relevant - @Column({ type: "boolean", default: false }) disabled!: boolean; diff --git a/back-end/api/src/migrations/migrations.module.ts b/back-end/api/src/migrations/migrations.module.ts index 93dd3172..e87da965 100644 --- a/back-end/api/src/migrations/migrations.module.ts +++ b/back-end/api/src/migrations/migrations.module.ts @@ -7,6 +7,7 @@ import { SharedModule } from 'src/shared/shared.module'; import { SettingsModule } from 'src/settings/settings.module'; import { MetadataModule } from 'src/metadata/metadata.module'; import { SqlScriptsModule } from 'src/sql-scripts/sql-scripts.module'; +import { ProductsModule } from 'src/products/products.module'; @Module({ imports: [ @@ -18,6 +19,7 @@ import { SqlScriptsModule } from 'src/sql-scripts/sql-scripts.module'; MetadataModule, SettingsModule, SqlScriptsModule, + ProductsModule, ], providers: [MigrationsService], exports: [MigrationsService], diff --git a/back-end/api/src/migrations/migrations.service.ts b/back-end/api/src/migrations/migrations.service.ts index abd77759..dea3e47a 100644 --- a/back-end/api/src/migrations/migrations.service.ts +++ b/back-end/api/src/migrations/migrations.service.ts @@ -20,10 +20,12 @@ import { DataSource } from 'typeorm'; import { SourceSpecificationsService } from 'src/metadata/source-specifications/services/source-specifications.service'; import { SourceTypeEnum } from 'src/metadata/source-specifications/enums/source-type.enum'; import { FormSourceDTO } from 'src/metadata/source-specifications/dtos/form-source.dto'; +import { ProductsService } from 'src/products/services/products.service'; +import { SYSTEM_PRODUCTS } from './system-products-defaults'; @Injectable() export class MigrationsService { - private readonly SUPPORTED_DB_VERSION: string = '0.0.6'; // TODO. Should come from a versioning file. + private readonly SUPPORTED_DB_VERSION: string = '0.0.7'; // TODO. Should come from a versioning file. private readonly logger = new Logger(MigrationsService.name); constructor( @@ -39,6 +41,7 @@ export class MigrationsService { private flagsService: FlagsService, private qcSpecsService: QCSpecificationsService, // TODO. Temporary. After all met services have version preview 2.0.5. Remove this. New installations won't need it private sourcesService: SourceSpecificationsService, + private productsService: ProductsService, ) { } @@ -109,6 +112,13 @@ export class MigrationsService { await this.seedFirstUser(); await this.seedMetadata(); await this.seedGeneralSettings(); + + // TODO. seed system products. This is temporary until we have a proper product management system + //await this.seedSystemProducts(); + + //TODO. seed climsoft system adapters authored by climsoft developers + // await this.seedSystemAdapters(); + } private async seedTriggers() { @@ -265,6 +275,24 @@ export class MigrationsService { } } + private async seedSystemProducts(): Promise { + for (const product of SYSTEM_PRODUCTS) { + await this.productsService.upsertSystemProduct( + product.systemKey, + product.supersetUuid, + product.name, + product.description, + product.category, + 1, + ); + } + this.logger.log('System products seeded'); + } + + private async seedSystemAdapters(): Promise { + // TODO. Seed system adapters authored by climsoft developers. This is temporary until we have a proper adapter management system + } + /** * Migrate FORM source parameters from `elementIds: number[]` to * `elementsMetadata: { elementId, hours }[]`. Each existing element id is diff --git a/back-end/api/src/migrations/system-products-defaults.ts b/back-end/api/src/migrations/system-products-defaults.ts new file mode 100644 index 00000000..9b035355 --- /dev/null +++ b/back-end/api/src/migrations/system-products-defaults.ts @@ -0,0 +1,66 @@ +/** + * System climate products shipped with Climsoft. + * + * Each entry maps to a Superset dashboard ZIP committed in superset/products/. + * The supersetUuid MUST match the UUID baked into the corresponding ZIP. + * When authoring a new dashboard, set its UUID in Superset to the value here + * before exporting it, so the seed and the ZIP stay in sync. + * + * To add a new shipped product: + * 1. Generate a UUID and add an entry below. + * 2. In Superset, edit the dashboard settings and set its UUID to match. + * 3. Export the dashboard as a ZIP to superset/products/.zip. + * 4. Commit both the ZIP and this file. + */ +export interface SystemProductSeed { + systemKey: string; + supersetUuid: string; + name: string; + description: string | null; + category: string | null; +} + +export const SYSTEM_PRODUCTS: SystemProductSeed[] = [ + { + systemKey: 'stations_overview', + supersetUuid: '661fab3c-49e6-4a06-92af-7e5b13159d7c', + name: 'Stations Overview', + description: 'Interactive map and directory of all monitoring stations.', + category: 'Network Management', + }, + { + systemKey: 'data_availability', + supersetUuid: 'a2d068da-ae6e-4987-9af6-c83277900c88', + name: 'Data Availability', + description: 'Monthly completeness and QC status by station and element.', + category: 'Data Quality', + }, + { + systemKey: 'observations_explorer', + supersetUuid: 'bcbab172-fd08-4920-a2d7-49ebf02c4697', + name: 'Observations Explorer', + description: 'Browse and filter raw observation records across all stations.', + category: 'Data Exploration', + }, + { + systemKey: 'daily_climate', + supersetUuid: '3739e396-cff4-4524-8cdb-13ce138669ef', + name: 'Daily Climate Summary', + description: 'Daily aggregates (mean, max, min, total) per station and element.', + category: 'Data Exploration', + }, + { + systemKey: 'monthly_climate', + supersetUuid: '79deede1-da9a-473f-a5fb-baf2d83eb101', + name: 'Monthly Climate Summary', + description: 'Monthly aggregates and QC statistics per station and element.', + category: 'Data Exploration', + }, + { + systemKey: 'climate_extremes', + supersetUuid: 'de8f35d9-6a42-438e-9c41-e68aba418efd', + name: 'Climate Extremes', + description: 'Annual maximum, minimum, mean and percentiles per station and element.', + category: 'Climate Analysis', + }, +]; diff --git a/back-end/api/src/observation/controllers/import-preview.controller.ts b/back-end/api/src/observation/controllers/import-preview.controller.ts index c960d6ef..119f6462 100644 --- a/back-end/api/src/observation/controllers/import-preview.controller.ts +++ b/back-end/api/src/observation/controllers/import-preview.controller.ts @@ -19,9 +19,9 @@ export class ImportPreviewController { public async upload( @UploadedFile(new ParseFilePipe({ validators: [ - // 1GB to accomodate preview of large files. Note, should always be same us that used in `observationsController` for upload endpoint to ensure smooth preview of files uploaded for import. + // 5GB to accomodate preview of large files. Note, should always be same us that used in `observationsController` for upload endpoint to ensure smooth preview of files uploaded for import. // In future, this should come from environment. - new MaxFileSizeValidator({ maxSize: 1024 * 1024 * 1024 }), + new MaxFileSizeValidator({ maxSize: (1024 * 1024 * 1024) * 5 }), new FileTypeValidator({ fileType: /(text\/csv|text\/plain|application\/octet-stream)/, fallbackToMimetype: true }), ] })) file: Express.Multer.File, @@ -87,7 +87,7 @@ export class ImportPreviewController { @Param('sourceId', ParseIntPipe) sourceId: number, @UploadedFile(new ParseFilePipe({ validators: [ - new MaxFileSizeValidator({ maxSize: 1024 * 1024 * 1024 }), + new MaxFileSizeValidator({ maxSize: (1024 * 1024 * 1024) * 5 }), // 5GB to accomodate preview of large files. Note, should always be same us that used in `ImportPreviewController` for upload endpoint to ensure smooth preview of files uploaded for import. new FileTypeValidator({ fileType: /(text\/csv|text\/plain|application\/octet-stream)/, fallbackToMimetype: true }), ] })) file: Express.Multer.File, diff --git a/back-end/api/src/observation/controllers/observations.controller.ts b/back-end/api/src/observation/controllers/observations.controller.ts index d8341880..c5fbb496 100644 --- a/back-end/api/src/observation/controllers/observations.controller.ts +++ b/back-end/api/src/observation/controllers/observations.controller.ts @@ -148,8 +148,8 @@ export class ObservationsController { @Param('sourceid', AuthorisedImportsPipe) sourceId: number, @UploadedFile(new ParseFilePipe({ validators: [ - // 1GB to accomodate preview of large files. Note, should always be same us that used in `ImportPreviewController` for upload endpoint to ensure smooth preview of files uploaded for import. - new MaxFileSizeValidator({ maxSize: 1024 * 1024 * 1024 }), + // 5GB to accomodate preview of large files. Note, should always be same us that used in `ImportPreviewController` for upload endpoint to ensure smooth preview of files uploaded for import. + new MaxFileSizeValidator({ maxSize: (1024 * 1024 * 1024) * 5 }), new FileTypeValidator({ fileType: /(text\/csv|text\/plain|application\/octet-stream)/, fallbackToMimetype: true }), ] }) diff --git a/back-end/api/src/observation/entities/observation.entity.ts b/back-end/api/src/observation/entities/observation.entity.ts index b076f174..634b7523 100644 --- a/back-end/api/src/observation/entities/observation.entity.ts +++ b/back-end/api/src/observation/entities/observation.entity.ts @@ -20,7 +20,7 @@ export enum FlagEnum { // TODO. Investigate if a constraints check for level to always not be negative is necessary @Entity("observations") -@Check("CHK_observations_both_value_and_flag_not_null", `"value" IS NOT NULL OR "flag_id" IS NOT NULL`) +//@Check("CHK_observations_both_value_and_flag_id_not_null", `"value" IS NOT NULL OR "flag_id" IS NOT NULL`) // TODO. Temporarily commented out because it prevents inserting a record with both value and flag_id as null. This is needed for the case where a user wants to delete an observation by setting both value and flag_id to null. disable until all countries migrate to using the new flag_id column and the old flag column is deprecated. After that, this check can be re-enabled to prevent inserting a record with both value and flag_id as null. @Check("CHK_observations_no_future_dates", `"date_time" < NOW()`) @Check("CHK_observations_interval_greater_than_zero", `"interval" > 0`) export class ObservationEntity extends AppBaseEntity { diff --git a/back-end/api/src/observation/services/climsoft-web-to-v4-sync.service.ts b/back-end/api/src/observation/services/climsoft-web-to-v4-sync.service.ts index 9bac4186..8287d57c 100644 --- a/back-end/api/src/observation/services/climsoft-web-to-v4-sync.service.ts +++ b/back-end/api/src/observation/services/climsoft-web-to-v4-sync.service.ts @@ -169,7 +169,10 @@ export class ClimsoftWebToV4SyncService { capturedBy = VALUES(capturedBy) `; - const values: (string | number | null | undefined)[][] = []; + // Track each row's source entity alongside its bind values so that + // if the batch fails we can pinpoint the exact row(s) that caused + // it via a per-row diagnostic pass. + const rows: { entity: ObservationEntity; values: (string | number | null | undefined)[] }[] = []; for (const entity of entities) { if (!this.climsoftV4WebSetupService.v4Stations.has(entity.stationId)) { @@ -189,32 +192,35 @@ export class ClimsoftWebToV4SyncService { const v4ValueMap = this.getV4ValueMapping(v4Element, entity); - values.push([ - entity.stationId, - entity.elementId, - v4ValueMap.v4DBDatetime, - v4ValueMap.v4Level, - v4ValueMap.v4Value, - v4ValueMap.v4Flag, - v4ValueMap.v4DBPeriod, - - // V4 qcStatus 1 means data was quality controlled - 1, - - // Web database qc log is not supported by v4 qcTypeLog - null, - - // V4 acquisitionType 7 means data came from climsoft web - 7, - - // Technically, sourceName will never be null - // But put null here to make sure the userEmail goes to the correct column - // This will be mapped to dataForm - source.name, - - // V4 capturedBy supports upto 30 characters only - user.email.substring(0, 30), - ]); + rows.push({ + entity, + values: [ + entity.stationId, + entity.elementId, + v4ValueMap.v4DBDatetime, + v4ValueMap.v4Level, + v4ValueMap.v4Value, + v4ValueMap.v4Flag, + v4ValueMap.v4DBPeriod, + + // V4 qcStatus 1 means data was quality controlled + 1, + + // Web database qc log is not supported by v4 qcTypeLog + null, + + // V4 acquisitionType 7 means data came from climsoft web + 7, + + // Technically, sourceName will never be null + // But put null here to make sure the userEmail goes to the correct column + // This will be mapped to dataForm + source.name, + + // V4 capturedBy supports upto 30 characters only + user.email.substring(0, 30), + ], + }); } @@ -222,16 +228,29 @@ export class ClimsoftWebToV4SyncService { return false; } - // Execute the batch upsert - const results: mariadb.UpsertResult[] = await connection.batch(upsertStatement, values); - const totalAffectedRows = results.reduce((sum, result) => sum + result.affectedRows, 0); - this.logger.log(`V4 affected rows: ${totalAffectedRows}`); - - // As of 03/02/2025, when an existing row is updated MariaDB counts this as a row affected twice - // Once for detecting the conflict (i.e., attempting to insert) - // Once for performing the update - // So more affected rows should return true as well. - return totalAffectedRows >= entities.length; + try { + // Execute the batch upsert + const results: mariadb.UpsertResult[] = await connection.batch(upsertStatement, rows.map(r => r.values)); + const totalAffectedRows = results.reduce((sum, result) => sum + result.affectedRows, 0); + this.logger.log(`V4 affected rows: ${totalAffectedRows}`); + + // As of 03/02/2025, when an existing row is updated MariaDB counts this as a row affected twice + // Once for detecting the conflict (i.e., attempting to insert) + // Once for performing the update + // So more affected rows should return true as well. + return totalAffectedRows >= entities.length; + } catch (batchErr) { + // The batch failed atomically and MariaDB does not tell us + // which row triggered the error. Re-run each row individually + // inside a transaction that is always rolled back, so we can + // log the offending row(s) without side-effects. The sync + // loop will retry the whole backlog on the next tick. + const batchMsg = batchErr instanceof Error ? batchErr.message : String(batchErr); + this.logger.error(`Batch upsert to v4 failed: ${batchMsg}`); + this.logger.error(`Running per-row diagnosis to identify offending row(s). No rows will be committed by this pass.`); + await this.diagnoseFailingV4Rows(connection, upsertStatement, rows); + return false; + } } catch (err) { console.error('Error saving observations to v4 initial table:', err); return false; @@ -240,6 +259,57 @@ export class ClimsoftWebToV4SyncService { } } + /** + * Diagnostic-only per-row execution used after a batch upsert fails. + * Runs each row inside a transaction that is always rolled back at the + * end, so successful rows are not committed here — the next sync tick + * will re-attempt the whole backlog once the offending row(s) are fixed. + * Logs a full identifier tuple + the exact v4 bind values + the MariaDB + * error for each failing row. + */ + private async diagnoseFailingV4Rows( + connection: mariadb.Connection, + statement: string, + rows: { entity: ObservationEntity; values: (string | number | null | undefined)[] }[], + ): Promise { + try { + await connection.beginTransaction(); + } catch (beginErr) { + this.logger.error(`Could not begin diagnostic transaction: ${beginErr instanceof Error ? beginErr.message : String(beginErr)}. Skipping per-row diagnosis.`); + return; + } + + let failedCount = 0; + try { + for (const { entity, values } of rows) { + try { + await connection.query(statement, values); + } catch (rowErr) { + failedCount++; + const rowMsg = rowErr instanceof Error ? rowErr.message : String(rowErr); + this.logger.error( + `V4 sync failing row: ` + + `stationId=${entity.stationId}, elementId=${entity.elementId}, ` + + `level=${entity.level}, datetime=${entity.datetime.toISOString()}, ` + + `interval=${entity.interval}, sourceId=${entity.sourceId}, ` + + `v5 value=${entity.value}. ` + + `V4 bind values: ${JSON.stringify(values)}. ` + + `MariaDB error: ${rowMsg}`, + ); + } + } + } finally { + // Always rollback — this pass is diagnostic only. + try { + await connection.rollback(); + } catch (rollbackErr) { + this.logger.warn(`Failed to rollback diagnostic transaction: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`); + } + } + + this.logger.error(`Per-row diagnosis complete. ${failedCount} of ${rows.length} row(s) failed.`); + } + private getV4ValueMapping(v4Element: V4ElementModel, entity: ObservationEntity): { v4Level: string, v4DBPeriod: number | null, v4Value: number | null, v4Flag: string | null, v4DBDatetime: string } { // V4 database model expects empty for null values let period: number | null = null; diff --git a/back-end/api/src/observation/services/import-preview.service.ts b/back-end/api/src/observation/services/import-preview.service.ts index 65a24ee4..13cbe277 100644 --- a/back-end/api/src/observation/services/import-preview.service.ts +++ b/back-end/api/src/observation/services/import-preview.service.ts @@ -213,7 +213,7 @@ export class ImportPreviewService implements OnModuleDestroy { const importFilePathName = path.posix.join(workingDir, session.workingFileName); const tableName: string = getTableNameFromUUID(crypto.randomUUID()); - await DuckDBUtils.createTableFromFile(this.fileIOService.duckDbConn, importFilePathName, tableName, false, session.rowsToSkip, 0, session.delimiter); + await DuckDBUtils.createTableFromFile(this.fileIOService.duckDbConn, importFilePathName, tableName, false, session.rowsToSkip, this.MAX_PREVIEW_ROWS, session.delimiter); const previewData: PreviewTableData = { columns: await DuckDBUtils.getColumnNames(this.fileIOService.duckDbConn, tableName), @@ -241,7 +241,7 @@ export class ImportPreviewService implements OnModuleDestroy { const importFilePathName = path.posix.join(workingDir, session.workingFileName); const tableName: string = getTableNameFromUUID(crypto.randomUUID()); - await DuckDBUtils.createTableFromFile(this.fileIOService.duckDbConn, importFilePathName, tableName, false, session.rowsToSkip, 0, session.delimiter); + await DuckDBUtils.createTableFromFile(this.fileIOService.duckDbConn, importFilePathName, tableName, false, session.rowsToSkip, this.MAX_PREVIEW_ROWS, session.delimiter); const elements: CreateViewElementDto[] = this.elementsService.find(); const flags: ViewFlagDto[] = this.flagsService.find(); diff --git a/back-end/api/src/observation/services/observations-export.service.ts b/back-end/api/src/observation/services/observations-export.service.ts index 3dc4b200..733d418e 100644 --- a/back-end/api/src/observation/services/observations-export.service.ts +++ b/back-end/api/src/observation/services/observations-export.service.ts @@ -382,7 +382,14 @@ export class ObservationsExportService { columnSelections.push('st.elevation AS station_elevation'); } + //------------------------------------------- + // Elements columnSelections.push('ob.element_id AS element_id'); + + if (exportParams.includeElementAbbreviation) { + columnSelections.push('el.abbreviation AS element_abbreviation'); + } + if (exportParams.includeElementName) { columnSelections.push('el.name AS element_name'); } @@ -390,6 +397,7 @@ export class ObservationsExportService { if (exportParams.includeElementUnits) { columnSelections.push('el.units AS element_units'); } + //------------------------------------------- if (exportParams.includeSourceName) { columnSelections.push('so.name AS source_name'); @@ -412,7 +420,8 @@ export class ObservationsExportService { columnSelections.push(`EXTRACT(MONTH FROM (ob.date_time + INTERVAL '${displayUtcOffset} hours')) AS month`); columnSelections.push(`EXTRACT(DAY FROM (ob.date_time + INTERVAL '${displayUtcOffset} hours')) AS day`); columnSelections.push(`EXTRACT(HOUR FROM (ob.date_time + INTERVAL '${displayUtcOffset} hours')) AS hour`); - columnSelections.push(`TO_CHAR((date_time)::time, 'MI:SS') AS mins_secs`); + columnSelections.push(`EXTRACT(MINUTE FROM (ob.date_time + INTERVAL '${displayUtcOffset} hours')) AS minute`); + columnSelections.push(`TRUNC(EXTRACT(SECOND FROM (ob.date_time + INTERVAL '${displayUtcOffset} hours'))) AS second`); } else { columnSelections.push(`(ob.date_time + INTERVAL '${displayUtcOffset} hours')::timestamp AS date_time`); } @@ -422,7 +431,8 @@ export class ObservationsExportService { columnSelections.push('EXTRACT(MONTH FROM ob.date_time ) AS month'); columnSelections.push('EXTRACT(DAY FROM ob.date_time) AS day'); columnSelections.push('EXTRACT(HOUR FROM ob.date_time) AS hour'); - columnSelections.push(`TO_CHAR((date_time)::time, 'MI:SS') AS mins_secs`); + columnSelections.push('EXTRACT(MINUTE FROM ob.date_time) AS minute'); + columnSelections.push('TRUNC(EXTRACT(SECOND FROM ob.date_time)) AS second'); } else { columnSelections.push('ob.date_time::timestamp AS date_time'); } diff --git a/back-end/api/src/observation/services/tabular-import-transformer.ts b/back-end/api/src/observation/services/tabular-import-transformer.ts index fe0e4ca8..fe9b65b5 100644 --- a/back-end/api/src/observation/services/tabular-import-transformer.ts +++ b/back-end/api/src/observation/services/tabular-import-transformer.ts @@ -1,4 +1,4 @@ -import { ImportSourceTabularParamsDto, DateTimeDefinition, DatePart, DayColumns, TimePart, ValueDefinition, FlagDefinition } from 'src/metadata/source-specifications/dtos/import-source-tabular-params.dto'; +import { ImportSourceTabularParamsDto, DateTimeDefinition, DatePart, DayColumns, TimePart, ValueDefinition, FlagDefinition, FlagToFetch } from 'src/metadata/source-specifications/dtos/import-source-tabular-params.dto'; import { ViewFlagDto } from 'src/metadata/flags/dtos/view-flag.dto'; import { ImportSourceDto } from 'src/metadata/source-specifications/dtos/import-source.dto'; import { DuckDBUtils } from 'src/shared/utils/duckdb.utils'; @@ -135,7 +135,6 @@ export class TabularImportTransformer { return sql; } - private static buildAlterStationColumnSQL(source: ImportSourceTabularParamsDto, tableName: string, stationId: string | null): string[] { const sql: string[] = []; if (source.stationDefinition) { @@ -355,7 +354,7 @@ export class TabularImportTransformer { // Map back to hour-of-day (00..23). Hours start at 0, unlike days at 1, // so the offset is `- firstColumnPosition` not `- firstColumnPosition + 1`. sql.push(`ALTER TABLE ${tableName} ADD COLUMN time_col VARCHAR`); - sql.push(`UPDATE ${tableName} SET time_col = lpad(substr(hour_col, 7)::INTEGER - ${firstColumnPosition}, 2, '0') || ':00:00'`); + sql.push(`UPDATE ${tableName} SET time_col = lpad((substr(hour_col, 7)::INTEGER - ${firstColumnPosition})::VARCHAR, 2, '0') || ':00:00'`); return '%H:%M:%S'; } throw new Error('Time part must define defaultHour, singleColumn, hourAndMinuteColumns, or hourColumnsRange'); @@ -364,6 +363,12 @@ export class TabularImportTransformer { private static buildAlterValueColumnSQL(sourceDef: ViewSourceSpecificationModel, importDef: ImportSourceDto, tabularDef: ImportSourceTabularParamsDto, tableName: string, flags: ViewFlagDto[]): string[] { const sql: string[] = []; + // Whether the flag column has already been set up (renamed from the + // flag-column path, or created + populated by the inline-flag split). + // False means we still need to add a NULL-default flag column so the + // rest of the pipeline sees a uniform `flag_id VARCHAR` column. + let flagColumnConfigured = false; + if (tabularDef.valueDefinition !== undefined) { const valueDefinition: ValueDefinition = tabularDef.valueDefinition; //-------------------------- @@ -372,40 +377,50 @@ export class TabularImportTransformer { //-------------------------- //-------------------------- - // Flag column + // Flag column — explicit column wins over inline-flag rule if both are set. if (valueDefinition.flagDefinition !== undefined) { const flagDefinition: FlagDefinition = valueDefinition.flagDefinition; sql.push(`ALTER TABLE ${tableName} RENAME column${flagDefinition.flagColumnPosition} TO ${this.FLAG_PROPERTY_NAME}`); - - if (flagDefinition.flagsToFetch) { - // flagsToFetch databaseId is already a flag table id (integer), use directly - sql.push(...DuckDBUtils.getDeleteAndUpdateSQL(tableName, this.FLAG_PROPERTY_NAME, flagDefinition.flagsToFetch, false)); - } else { - // No explicit mapping — map string abbreviations to integer IDs using a CASE statement - const caseParts = flags.map(f => `WHEN UPPER(${this.FLAG_PROPERTY_NAME}) = '${f.abbreviation.toUpperCase()}' THEN ${f.id}`); - if (caseParts.length > 0) { - sql.push(`UPDATE ${tableName} SET ${this.FLAG_PROPERTY_NAME} = CASE ${caseParts.join(' ')} ELSE NULL END WHERE ${this.FLAG_PROPERTY_NAME} IS NOT NULL`); - } - } - - } else { - sql.push(`ALTER TABLE ${tableName} ADD COLUMN ${this.FLAG_PROPERTY_NAME} INTEGER DEFAULT NULL`); + sql.push(...this.buildFlagIdMappingSQL(tableName, flagDefinition.flagsToFetch, flags)); + flagColumnConfigured = true; } //-------------------------- + } - } else { - // Just add the flag column because the value column should have been added when stacking elements of date columns - sql.push(`ALTER TABLE ${tableName} ADD COLUMN ${this.FLAG_PROPERTY_NAME} INTEGER DEFAULT NULL`); + // Inline-flag split applies whenever the value column exists AND no + // explicit flag column was configured. Works uniformly for both + // long-format sources (valueDefinition set) and wide-pivot sources + // (value column produced by an UNPIVOT in an earlier step). + if (!flagColumnConfigured && tabularDef.inlineFlagRule !== undefined) { + sql.push(...this.buildInlineFlagSplitSQL(tableName)); + sql.push(...this.buildFlagIdMappingSQL(tableName, tabularDef.inlineFlagRule.flagsToFetch, flags)); + flagColumnConfigured = true; + } + + if (!flagColumnConfigured) { + // No flag information at all. Add a NULL-default VARCHAR column + // so downstream conditions like `flag_id = ''` bind cleanly for + // every path. The final TYPE INTEGER cast at the bottom of this + // method converts VARCHAR NULL → INTEGER NULL without complaint. + sql.push(`ALTER TABLE ${tableName} ADD COLUMN ${this.FLAG_PROPERTY_NAME} VARCHAR DEFAULT NULL`); } // Get all missing value indicators in quoted format const missingValueIndicators: string[] = importDef.sourceMissingValueIndicators.split(',').map(f => `'${f}'`).filter(f => f); - let missingValueCondition: string = `${this.VALUE_PROPERTY_NAME} IS NULL`; + let valueMissingCondition: string = `${this.VALUE_PROPERTY_NAME} IS NULL`; if (missingValueIndicators.length > 0) { - missingValueCondition = `${missingValueCondition} OR ${this.VALUE_PROPERTY_NAME} IN (${missingValueIndicators.join(',')})`; + valueMissingCondition = `${valueMissingCondition} OR ${this.VALUE_PROPERTY_NAME} IN (${missingValueIndicators.join(',')})`; } + // A row is treated as missing only when BOTH the value is empty AND + // there is no meaningful flag. This preserves trace-only cells such + // as ('', 'T'), where the flag alone carries the observation. + // When no flag column is configured, `flag_id` is a NULL default for + // every row, so this narrows to just the value-missing check. + const flagAbsentCondition: string = `${this.FLAG_PROPERTY_NAME} IS NULL OR ${this.FLAG_PROPERTY_NAME} = ''`; + const missingValueCondition: string = `(${valueMissingCondition}) AND (${flagAbsentCondition})`; + if (sourceDef.allowMissingValue) { // Set missing flag if missing are allowed to be imported. const missingFlag = flags.find(f => f.name.toLowerCase() === 'missing'); @@ -430,6 +445,56 @@ export class TabularImportTransformer { return sql; } + /** + * Maps the `flag_id` VARCHAR column from source strings to database flag + * ids. Callers must have already ensured `flag_id` exists on the table + * (either via a column rename or via {@link buildInlineFlagSplitSQL}). + * + * Two modes, same as {@link FlagDefinition.flagsToFetch}: + * - Explicit mapping: rows whose source string isn't in the list are + * deleted; the rest are updated to the mapped integer id (encoded in + * the VARCHAR column; final TYPE INTEGER cast happens later). + * - Fallback: match source string case-insensitively against + * `flags.abbreviation`; unmatched strings become NULL. + */ + private static buildFlagIdMappingSQL( + tableName: string, + flagsToFetch: FlagToFetch[] | undefined, + allFlags: ViewFlagDto[], + ): string[] { + if (flagsToFetch) { + return DuckDBUtils.getDeleteAndUpdateSQL(tableName, this.FLAG_PROPERTY_NAME, flagsToFetch, false); + } + const caseParts = allFlags.map(f => + `WHEN UPPER(${this.FLAG_PROPERTY_NAME}) = '${f.abbreviation.toUpperCase()}' THEN ${f.id}` + ); + if (caseParts.length === 0) return []; + return [ + `UPDATE ${tableName} SET ${this.FLAG_PROPERTY_NAME} = CASE ${caseParts.join(' ')} ELSE NULL END WHERE ${this.FLAG_PROPERTY_NAME} IS NOT NULL`, + ]; + } + + /** + * Splits an inline value+flag `value` column into separate `value` and + * `flag_id` VARCHAR columns using the trailing-alphabetic-run convention. + * '0.5T' → value='0.5', flag_id='T' + * '-1.2' → value='-1.2', flag_id=NULL + * 'T' → value=NULL, flag_id='T' (missing-value logic decides fate) + * '' → value=NULL, flag_id=NULL + * + * NULLIF normalises "empty match" and "empty leftover" to NULL so the + * downstream missing-value check and flag-mapping logic behave uniformly + * regardless of which path produced the columns. + */ + private static buildInlineFlagSplitSQL(tableName: string): string[] { + return [ + `ALTER TABLE ${tableName} ADD COLUMN ${this.FLAG_PROPERTY_NAME} VARCHAR`, + `UPDATE ${tableName} SET ` + + `${this.FLAG_PROPERTY_NAME} = NULLIF(regexp_extract(${this.VALUE_PROPERTY_NAME}, '[A-Za-z]+$'), ''), ` + + `${this.VALUE_PROPERTY_NAME} = NULLIF(regexp_replace(${this.VALUE_PROPERTY_NAME}, '[A-Za-z]+$', ''), '')`, + ]; + } + private static buildYearMonthDaySQL(tableName: string, yearColPos: number, monthColPos: number, dayColumns: DayColumns): string[] { const sql: string[] = []; sql.push(`ALTER TABLE ${tableName} RENAME COLUMN column${yearColPos} TO year_col`); @@ -449,7 +514,7 @@ export class TabularImportTransformer { // Nulls are excluded because they represent non-existent days (e.g. Feb 31st). sql.push(`CREATE OR REPLACE TABLE ${tableName} AS SELECT * FROM ${tableName} UNPIVOT (${this.VALUE_PROPERTY_NAME} FOR day_col IN (${dayColumnNames.join(', ')}))`); // Extract the numeric day part from the column name (e.g. 'column5' -> 5) and zero-pad it. - sql.push(`UPDATE ${tableName} SET day_col = lpad(substr(day_col, 7)::INTEGER - ${firstColumnPosition} + 1, 2, '0')`); + sql.push(`UPDATE ${tableName} SET day_col = lpad((substr(day_col, 7)::INTEGER - ${firstColumnPosition} + 1)::VARCHAR, 2, '0')`); } else { throw new Error('Day columns must define either singleColumn or columnsRange'); } diff --git a/back-end/api/src/observation/services/wis2box-export.service.ts b/back-end/api/src/observation/services/wis2box-export.service.ts index 7380578b..4b8274a6 100644 --- a/back-end/api/src/observation/services/wis2box-export.service.ts +++ b/back-end/api/src/observation/services/wis2box-export.service.ts @@ -9,17 +9,50 @@ export class Wis2BoxExportService implements OnModuleInit { private readonly logger = new Logger(Wis2BoxExportService.name); /** - * Per-element value transforms applied inside pivot expressions. Wrapped in - * ROUND to suppress float-arithmetic noise — e.g. `273.15` is not exactly - * representable in float64, so `value + 273.15` would otherwise produce - * trailing-digit noise. Shared between SYNOP and DAYCLI generators. + * Per-element value transforms applied inside pivot expressions. + * + * Most transforms are conditional on the source element's units (carried + * through the intermediate CSV as `element_units`, one value per input + * row). The conversion only fires when the units string matches an + * accepted spelling of the expected source unit; otherwise the raw + * value passes through unchanged. This guards against double-conversion + * when an admin has already stored values in the WIS2BOX target unit. + * + * `minsToHours` is the exception — it's used only for `sunshine_total_24hr` + * where the WIS2BOX spec commits to minute-storage on the 1hr side, so + * the conversion is unconditional and matches that same assumption. + * + * Conversion branches are wrapped in ROUND to suppress float-arithmetic noise. + * The ELSE branch passes the raw value without rounding. + * + * Shared between SYNOP and DAYCLI generators. */ - private readonly hPaToPa = (v: string) => `ROUND(${v} * 100, 1)`; // Pa to 0.1 precision - private readonly celciusToK = (v: string) => `ROUND(${v} + 273.15, 2)`; // K to 0.01 precision - private readonly knotsToMs = (v: string) => `ROUND(${v} * 0.51444, 2)`; // ms to 0.01 precision - private readonly feetToMeters = (v: string) => `ROUND(${v} * 0.3048, 2)`; // meters to 0.01 precision - private readonly oktasToPerc = (v: string) => `ROUND( (${v} * 100) / 8 , 0)`; // % with no d.p. Important for 9 oktas to be 113% - private readonly minsToHours = (v: string) => `ROUND(${v} / 60, 2)`; // ms to 0.01 precision + private readonly hPaToPa = (v: string) => + this.conditionalConvert(v, ['hpa', 'hectopascal', 'hectopascals', 'mb', 'mbar', 'millibar', 'millibars'], `ROUND(${v} * 100, 1)`); + private readonly celciusToK = (v: string) => + this.conditionalConvert(v, ['c', '°c', 'celsius', 'deg c', 'degc', 'degrees c', 'degrees celsius'], `ROUND(${v} + 273.15, 2)`); + private readonly knotsToMs = (v: string) => + this.conditionalConvert(v, ['kt', 'kts', 'kn', 'knot', 'knots'], `ROUND(${v} * 0.51444, 2)`); + private readonly feetToMeters = (v: string) => + this.conditionalConvert(v, ['ft', 'feet', 'foot'], `ROUND(${v} * 0.3048, 2)`); + // 9 oktas -> 113% is intentional: WMO okta code 9 = "sky obscured". + private readonly oktasToPerc = (v: string) => + this.conditionalConvert(v, ['okta', 'oktas'], `ROUND((${v} * 100) / 8, 0)`); + private readonly minsToHours = (v: string) => `ROUND(${v} / 60, 2)`; // hours to 0.01 precision + /** + * Builds a `CASE WHEN LOWER(TRIM(element_units)) IN (…) THEN ELSE END` + * expression. Called by five of the six unit-transform arrow fields above + * (all except `minsToHours`, which is unconditional). + * NULL/empty element_units matches nothing and falls through to the raw branch. + */ + private conditionalConvert(v: string, acceptedUnits: string[], convertSql: string): string { + // Wrap each accepted spelling as a SQL string literal, doubling any + // embedded single quote (SQL's standard escape) so a spelling like + // "deg's" would splice in safely. None of the current arrays contain + // apostrophes; this is belt-and-braces against future edits. + const list = acceptedUnits.map(u => `'${u.replace(/'/g, "''")}'`).join(', '); + return `CASE WHEN LOWER(TRIM(COALESCE(element_units, ''))) IN (${list}) THEN ${convertSql} ELSE ${v} END`; + } constructor( private fileIOService: FileIOService, @@ -56,10 +89,11 @@ export class Wis2BoxExportService implements OnModuleInit { * Multi-stage SQL: * 1. CTE `pivoted` — group input rows by (station, observation moment) * and pivot each user-mapped element into its own column. - * Per-element transforms applied here: + * Per-element transforms applied here are conditional on the source + * element's units (see `conditionalConvert` above). Common cases: * - Pressure columns: hPa -> Pa (* 100), rounded to 1 decimal * - Temperature columns: C -> K (+ 273.15), rounded to 2 decimals - * Rounding suppresses float-arithmetic noise. + * Values already stored in the WIS2BOX target unit pass through raw. * 2. CTE `station_meta` — collapses pivoted to one row per station and * computes the time bounds for each station's data. * 3. CTE `hourly_grid` — generates a complete hourly timeline per @@ -76,7 +110,9 @@ export class Wis2BoxExportService implements OnModuleInit { * pressure_tendency_fm12() macro (registered in onModuleInit) * - sunshine_total_24hr, total_precipitation_{3,6,12,24}_hour, * solar_radiation24_*: strict SUM over rolling row windows - * (NULL when fewer than N values are present in the window) + * (NULL when fewer than N values are present in the window). + * sunshine_total_24hr additionally divides by 60 because the + * WIS2BOX spec keeps 1hr in minutes but expects hours for 24hr. * * Several columns are emitted as hardcoded constants or NULL (per TODOs * in WIS2BOX_ELEMENTS_BY_REPORT_TYPE[SYNOP]) until station instrument @@ -310,7 +346,8 @@ export class Wis2BoxExportService implements OnModuleInit { `past_weather1`, `past_weather2`, `sunshine_total_1hr`, - // sunshine_total_24hr — strict 24-row trailing sum (NULL if any hour missing). Convert minutes to hours. + // sunshine_total_24hr — strict 24-row trailing sum (NULL if any hour missing). + // Convert minutes to hours. sunshine_total_1hr is assumed to be in minutes. `${this.minsToHours(strictWindowSum('sunshine_total_1hr', 24, 'w_24'))} AS sunshine_total_24hr`, // ── Precipitation ───────────────────────────────────────────── // TODO: rain_sensor_height — to come from station instrument metadata. @@ -488,7 +525,7 @@ export class Wis2BoxExportService implements OnModuleInit { ) TO '${outputFilePathName}' WITH (HEADER, DELIMITER ','); `; - this.logger.debug(`Executing SYNOP WIS2BOX CSV generation SQL`); + this.logger.log(`Executing SYNOP WIS2BOX CSV generation SQL`); await this.fileIOService.duckDbConn.run(sql); @@ -504,9 +541,10 @@ export class Wis2BoxExportService implements OnModuleInit { * 1. CTE `pivoted` — group input rows by (station, day) and, for each * prefix-grouped measurement, emit the 6 spec columns * (`_day_offset`, `_hour`, `_minute`, `_second`, value, `_flag`). - * Per-element transforms applied here: + * Per-element transforms applied here are conditional on the source + * element's units (see `conditionalConvert` above). Common cases: * - Temperature columns: C -> K (+ 273.15), rounded to 2 decimals - * Rounding suppresses float-arithmetic noise. + * Values already stored in the WIS2BOX target unit pass through raw. * 2. Outer SELECT — emits the WIS2BOX spec column order: parsed WIS/WMO * identifiers, location, hardcoded placeholders (siting class, * averaging method, thermometer height) and bare references to the diff --git a/back-end/api/src/products/controllers/products.controller.ts b/back-end/api/src/products/controllers/products.controller.ts new file mode 100644 index 00000000..fc414400 --- /dev/null +++ b/back-end/api/src/products/controllers/products.controller.ts @@ -0,0 +1,59 @@ +import { Body, Controller, Delete, Get, Param, ParseIntPipe, Patch, Post, Req } from "@nestjs/common"; +import { Admin } from "src/user/decorators/admin.decorator"; +import { AuthUtil } from "src/user/services/auth.util"; +import { CreateUpdateProductDto } from "../dtos/create-update-product.dto"; +import { ProductsService } from "../services/products.service"; +import { Request } from 'express'; + +@Controller("products") +export class ProductsController { + + constructor(private readonly productsService: ProductsService) { } + + @Get() + findForUser() { + return this.productsService.findForUser(); + } + + @Admin() + @Get("all") + findAll() { + return this.productsService.findAll(); + } + + @Get(":id") + findOne(@Param("id", ParseIntPipe) id: number) { + return this.productsService.findOneForUser(id); + } + + @Get(":id/guest-token") + getGuestToken( + @Req() request: Request, + @Param("id", ParseIntPipe) id: number + ) { + return this.productsService.getGuestToken(id, AuthUtil.getLoggedInUser(request)); + } + + @Admin() + @Post() + create(@Req() request: Request, @Body() dto: CreateUpdateProductDto) { + return this.productsService.add(dto, AuthUtil.getLoggedInUserId(request)); + } + + @Admin() + @Patch(":id") + update( + @Req() request: Request, + @Param("id", ParseIntPipe) id: number, + @Body() dto: CreateUpdateProductDto + ) { + return this.productsService.update(id, dto, AuthUtil.getLoggedInUserId(request)); + } + + @Admin() + @Delete(":id") + delete(@Param("id", ParseIntPipe) id: number) { + return this.productsService.delete(id); + } + +} diff --git a/back-end/api/src/products/dtos/create-update-product.dto.ts b/back-end/api/src/products/dtos/create-update-product.dto.ts new file mode 100644 index 00000000..7bb45929 --- /dev/null +++ b/back-end/api/src/products/dtos/create-update-product.dto.ts @@ -0,0 +1,24 @@ +import { IsBoolean, IsNotEmpty, IsOptional, IsString } from "class-validator"; + +export class CreateUpdateProductDto { + + @IsString() + @IsNotEmpty() + supersetUuid!: string; + + @IsString() + @IsNotEmpty() + name!: string; + + @IsOptional() + @IsString() + description!: string | null; + + @IsOptional() + @IsString() + category!: string | null; + + @IsBoolean() + disabled!: boolean; + +} diff --git a/back-end/api/src/products/dtos/view-product.dto.ts b/back-end/api/src/products/dtos/view-product.dto.ts new file mode 100644 index 00000000..c6330d27 --- /dev/null +++ b/back-end/api/src/products/dtos/view-product.dto.ts @@ -0,0 +1,6 @@ +import { CreateUpdateProductDto } from "./create-update-product.dto"; + +export class ViewProductDto extends CreateUpdateProductDto { + id!: number; + systemKey!: string | null; +} diff --git a/back-end/api/src/products/entities/climate-product.entity.ts b/back-end/api/src/products/entities/climate-product.entity.ts new file mode 100644 index 00000000..b0ea53ba --- /dev/null +++ b/back-end/api/src/products/entities/climate-product.entity.ts @@ -0,0 +1,41 @@ +import { AppBaseEntity, BaseLogVo } from "src/shared/entity/app-base-entity"; +import { Check, Column, Entity, PrimaryGeneratedColumn } from "typeorm"; + +@Entity("climate_products") +@Check("CHK_climate_products_name_not_empty", `"name" <> ''`) +@Check("CHK_climate_products_uuid_not_empty", `"superset_uuid" <> ''`) +export class ClimateProductEntity extends AppBaseEntity { + + @PrimaryGeneratedColumn({ name: "id", type: "int" }) + id!: number; + + @Column({ name: "system_key", type: "varchar", unique: true, nullable: true }) + systemKey!: string | null; + + @Column({ name: "superset_uuid", type: "varchar" }) + supersetUuid!: string; + + @Column({ name: "name", type: "varchar" }) + name!: string; + + @Column({ name: "description", type: "varchar", nullable: true }) + description!: string | null; + + @Column({ name: "category", type: "varchar", nullable: true }) + category!: string | null; + + @Column({ name: "disabled", type: "boolean", default: false }) + disabled!: boolean; + + @Column({ name: "log", type: "jsonb", nullable: true }) + log!: ClimateProductLogVo[] | null; + +} + +export interface ClimateProductLogVo extends BaseLogVo { + supersetUuid: string; + name: string; + description: string | null; + category: string | null; + disabled: boolean; +} diff --git a/back-end/api/src/products/products.module.ts b/back-end/api/src/products/products.module.ts new file mode 100644 index 00000000..cc8ab5a4 --- /dev/null +++ b/back-end/api/src/products/products.module.ts @@ -0,0 +1,18 @@ +import { Module } from "@nestjs/common"; +import { TypeOrmModule } from "@nestjs/typeorm"; +import { UserModule } from "src/user/user.module"; +import { ProductsController } from "./controllers/products.controller"; +import { ClimateProductEntity } from "./entities/climate-product.entity"; +import { ProductsService } from "./services/products.service"; +import { SupersetService } from "./services/superset.service"; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ClimateProductEntity]), + UserModule, + ], + controllers: [ProductsController], + providers: [ProductsService, SupersetService], + exports: [ProductsService], +}) +export class ProductsModule { } diff --git a/back-end/api/src/products/services/products.service.ts b/back-end/api/src/products/services/products.service.ts new file mode 100644 index 00000000..e53620a7 --- /dev/null +++ b/back-end/api/src/products/services/products.service.ts @@ -0,0 +1,146 @@ +import { Injectable, NotFoundException } from "@nestjs/common"; +import { InjectRepository } from "@nestjs/typeorm"; +import { LoggedInUserDto } from "src/user/dtos/logged-in-user.dto"; +import { Repository } from "typeorm"; +import { CreateUpdateProductDto } from "../dtos/create-update-product.dto"; +import { ViewProductDto } from "../dtos/view-product.dto"; +import { ClimateProductEntity } from "../entities/climate-product.entity"; +import { SupersetService } from "./superset.service"; + +@Injectable() +export class ProductsService { + + constructor( + @InjectRepository(ClimateProductEntity) + private readonly productsRepo: Repository, + private readonly supersetService: SupersetService, + ) { } + + async findForUser(): Promise { + const entities = await this.productsRepo.find({ + where: { disabled: false }, + order: { category: "ASC", name: "ASC" }, + }); + return entities.map(e => this.toViewDto(e)); + } + + async findAll(): Promise { + const entities = await this.productsRepo.find({ order: { category: "ASC", name: "ASC" } }); + return entities.map(e => this.toViewDto(e)); + } + + async findOne(id: number): Promise { + const entity = await this.productsRepo.findOneBy({ id }); + if (!entity) throw new NotFoundException(`Climate Product #${id} not found`); + return this.toViewDto(entity); + } + + async add(dto: CreateUpdateProductDto, userId: number): Promise { + const entity = this.productsRepo.create({ + ...dto, + systemKey: null, + entryUserId: userId, + }); + const saved = await this.productsRepo.save(entity); + return this.toViewDto(saved); + } + + async update(id: number, dto: CreateUpdateProductDto, userId: number): Promise { + const entity = await this.productsRepo.findOneBy({ id }); + if (!entity) throw new NotFoundException(`Climate Product #${id} not found`); + + if (entity.systemKey !== null) { + // Shipped products: only allow toggling disabled state + entity.disabled = dto.disabled; + } else { + Object.assign(entity, dto); + } + entity.entryUserId = userId; + + const saved = await this.productsRepo.save(entity); + return this.toViewDto(saved); + } + + async upsertSystemProduct( + systemKey: string, + supersetUuid: string, + name: string, + description: string | null, + category: string | null, + userId: number, + ): Promise { + const existing = await this.productsRepo.findOneBy({ systemKey }); + if (existing) { + existing.supersetUuid = supersetUuid; + existing.name = name; + existing.description = description; + existing.category = category; + existing.entryUserId = userId; + await this.productsRepo.save(existing); + } else { + const entity = this.productsRepo.create({ + systemKey, + supersetUuid, + name, + description, + category, + disabled: false, + entryUserId: userId, + }); + await this.productsRepo.save(entity); + } + } + + async delete(id: number): Promise { + const entity = await this.productsRepo.findOneBy({ id }); + if (!entity) throw new NotFoundException(`Climate Product #${id} not found`); + await this.productsRepo.remove(entity); + return id; + } + + async findOneForUser(id: number): Promise { + const entity = await this.productsRepo.findOneBy({ id, disabled: false }); + if (!entity) throw new NotFoundException(`Climate Product #${id} not found`); + return this.toViewDto(entity); + } + + async getGuestToken(productId: number, user: LoggedInUserDto): Promise<{ token: string; supersetUuid: string }> { + const entity = await this.productsRepo.findOneBy({ id: productId, disabled: false }); + if (!entity) throw new NotFoundException(`Climate Product #${productId} not found`); + + const rls = this.buildRls(user); + const token = await this.supersetService.generateGuestToken(entity.supersetUuid, rls); + return { token, supersetUuid: entity.supersetUuid }; + } + + private buildRls(user: LoggedInUserDto): { clause: string }[] { + if (user.isSystemAdmin) return []; + + const permissions = user.permissions; + if (!permissions) return [{ clause: "1=0" }]; + + const stationIds = new Set([ + ...(permissions.entryPermissions?.stationIds ?? []), + ...(permissions.qcPermissions?.stationIds ?? []), + ...(permissions.ingestionMonitoringPermissions?.stationIds ?? []), + ]); + + if (stationIds.size === 0) return [{ clause: "1=0" }]; + + const list = [...stationIds].map(id => `'${id.replace(/'/g, "''")}'`).join(","); + return [{ clause: `station_id IN (${list})` }]; + } + + private toViewDto(entity: ClimateProductEntity): ViewProductDto { + return { + id: entity.id, + systemKey: entity.systemKey, + supersetUuid: entity.supersetUuid, + name: entity.name, + description: entity.description, + category: entity.category, + disabled: entity.disabled, + }; + } + +} diff --git a/back-end/api/src/products/services/superset.service.ts b/back-end/api/src/products/services/superset.service.ts new file mode 100644 index 00000000..ab45f2d7 --- /dev/null +++ b/back-end/api/src/products/services/superset.service.ts @@ -0,0 +1,100 @@ +import { Injectable, Logger, ServiceUnavailableException } from "@nestjs/common"; +import axios, { AxiosInstance } from "axios"; +import { AppConfig } from "src/app.config"; + +interface GuestTokenRls { + clause: string; +} + +@Injectable() +export class SupersetService { + private readonly logger = new Logger(SupersetService.name); + + private readonly client: AxiosInstance; + private accessToken: string | null = null; + private tokenExpiresAt: number = 0; + + constructor() { + const { host, port } = AppConfig.superset; + this.client = axios.create({ + baseURL: `http://${host}:${port}`, + timeout: 10_000, + }); + } + + async generateGuestToken(dashboardUuid: string, rls: GuestTokenRls[]): Promise { + if (!AppConfig.superset.enabled) { + throw new ServiceUnavailableException("Climate Products (Superset) is not enabled on this server."); + } + + const accessToken = await this.getAccessToken(); + const { csrfToken, sessionCookie } = await this.fetchCsrfToken(accessToken); + + try { + const response = await this.client.post( + "/api/v1/security/guest_token/", + { + user: { username: "climsoft_guest", first_name: "Climsoft", last_name: "User" }, + resources: [{ type: "dashboard", id: dashboardUuid }], + rls, + }, + { + headers: { + Authorization: `Bearer ${accessToken}`, + "X-CSRFToken": csrfToken, + Referer: `http://${AppConfig.superset.host}:${AppConfig.superset.port}`, + ...(sessionCookie ? { Cookie: sessionCookie } : {}), + }, + } + ); + this.logger.log(`Successfully generated Superset guest token for dashboard ${dashboardUuid}`); + return response.data.token; + } catch (error: any) { + const detail = error?.response?.data ?? error?.message; + this.logger.error("Failed to generate Superset guest token. Status:", error?.response?.status, "Body:", JSON.stringify(detail)); + throw new ServiceUnavailableException("Could not generate access token for Climate Product."); + } + } + + private async getAccessToken(): Promise { + if (this.accessToken && Date.now() < this.tokenExpiresAt) { + return this.accessToken; + } + + const { serviceUsername, servicePassword } = AppConfig.superset; + + try { + const response = await this.client.post("/api/v1/security/login", { + username: serviceUsername, + password: servicePassword, + provider: "db", + refresh: true, + }); + + this.accessToken = response.data.access_token; + // Superset access tokens expire in 1 hour — refresh 5 minutes early + this.tokenExpiresAt = Date.now() + (55 * 60 * 1000); + return this.accessToken!; + } catch (error: any) { + const detail = error?.response?.data ?? error?.message; + this.logger.error("Failed to authenticate with Superset. Status:", error?.response?.status, "Body:", JSON.stringify(detail)); + throw new ServiceUnavailableException("Could not connect to Climate Products service."); + } + } + + private async fetchCsrfToken(accessToken: string): Promise<{ csrfToken: string; sessionCookie: string | null }> { + try { + const response = await this.client.get("/api/v1/security/csrf_token/", { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + const cookies: string[] = response.headers["set-cookie"] ?? []; + const sessionEntry = cookies.find(c => c.startsWith("session=")); + const sessionCookie = sessionEntry ? sessionEntry.split(";")[0] : null; + return { csrfToken: response.data.result, sessionCookie }; + } catch (error: any) { + const detail = error?.response?.data ?? error?.message; + this.logger.error("Failed to fetch CSRF token from Superset. Status:", error?.response?.status, "Body:", JSON.stringify(detail)); + throw new ServiceUnavailableException("Could not connect to Climate Products service."); + } + } +} diff --git a/back-end/api/src/queue/services/cleanup-scheduler.service.ts b/back-end/api/src/queue/services/cleanup-scheduler.service.ts index 7b232f13..9132c35c 100644 --- a/back-end/api/src/queue/services/cleanup-scheduler.service.ts +++ b/back-end/api/src/queue/services/cleanup-scheduler.service.ts @@ -213,6 +213,7 @@ export class CleanupSchedulerService implements OnApplicationBootstrap { this.logger.warn(`Could not delete operation dir ${dir.name}: ${error instanceof Error ? error.message : String(error)}`); } } + this.logger.log(`Operation cleanup completed. Deleted ${deletedCount} unreferenced operation directory(ies)`); } catch (error) { this.logger.error(`Error reading operations directory ${operationsDir}: ${error instanceof Error ? error.message : String(error)}`); } @@ -249,6 +250,7 @@ export class CleanupSchedulerService implements OnApplicationBootstrap { this.logger.warn(`Could not delete sample file ${entry.name}: ${error instanceof Error ? error.message : String(error)}`); } } + this.logger.log(`Sample file cleanup completed. Deleted ${deletedCount} unreferenced sample file(s)`); } catch (error) { this.logger.error(`Error reading samples directory ${samplesDir}: ${error instanceof Error ? error.message : String(error)}`); } @@ -287,6 +289,7 @@ export class CleanupSchedulerService implements OnApplicationBootstrap { this.logger.warn(`Could not delete adapter dir ${dir.name}: ${error instanceof Error ? error.message : String(error)}`); } } + this.logger.log(`Adapter script cleanup completed. Deleted ${deletedCount} unreferenced adapter script directory(ies)`); } catch (error) { this.logger.error(`Error reading adapter scripts dir ${scriptsDir}: ${error instanceof Error ? error.message : String(error)}`); } diff --git a/back-end/api/src/queue/services/connector-import-processor.service.ts b/back-end/api/src/queue/services/connector-import-processor.service.ts index 43c484e8..c3790e9a 100644 --- a/back-end/api/src/queue/services/connector-import-processor.service.ts +++ b/back-end/api/src/queue/services/connector-import-processor.service.ts @@ -38,7 +38,7 @@ export class ConnectorImportProcessorService { try { const payload = job.payload as ConnectorJobPayloadDto; const connector: ViewConnectorSpecificationModel = this.connectorService.find(payload.connectorId, false); - + this.logger.log(`Processing import job: ${job.id} for connector: ${connector.name}. Specs to be processed: ${connector.parameters.specifications.length}`); await this.processImportSpecifications(connector, job.entryUserId); this.logger.log(`Finished processing import job: ${job.id} for connector: ${connector.name}`); @@ -139,7 +139,7 @@ export class ConnectorImportProcessorService { } this.logger.log(`Completed processing and importing file form connector ${connector.name}. Time taken: ${new Date().getTime() - startTime} milliseconds`); - // Step 4. Save the new the connector log + // Step 3. Save the new the connector log newConnectorLog.executionEndDatetime = new Date(); await this.connectorExecutionLogService.create(newConnectorLog); } @@ -314,18 +314,22 @@ export class ConnectorImportProcessorService { ): Promise { for (const spec of connectorParams.specifications) { - // Step 3: Find matching files by converting the user's glob pattern to a regex. - // First, escape all regex-special characters (e.g. "." becomes "\." so it matches a literal dot, not "any character"). - // Then replace the glob wildcard "*" with ".*" (which means "any sequence of characters" in regex). - // Finally, anchor with "^" and "$" so the pattern matches the full file name (e.g. "*.csv" won't match "data.csv.bak"). - // Examples: - // "*.csv" → "^.*\.csv$" → matches "data.csv", "report.csv" - // "data_*.txt" → "^data_.*\.txt$" → matches "data_01.txt", "data_abc.txt" - // "report.csv" → "^report\.csv$" → matches only "report.csv" (not "reportXcsv") - const regexPattern: string = '^' + spec.filePattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'; - const matchingFiles: FileMetadataVo[] = remoteFiles.filter(file => - path.basename(file.fileName).match(new RegExp(regexPattern)) - ); + // Step 3: Find files matching the spec's pattern. + // + // Pattern grammar (Unix-style, only the last segment may glob): + // "*.csv" root-level files matching the glob + // "data.csv" a specific root-level file + // "stationA/" every file directly inside stationA/ (trailing slash = "all files") + // "stationA/*.csv" CSV files directly inside stationA/ + // "folder1/folder2/" every file directly inside folder1/folder2/ + // + // The directory portion is literal — no wildcards there. Matches + // are exact-parent only (not recursive subtree). Directory + // patterns require the connector's `recursive` flag to be on; + // otherwise the flat listing has no path-carrying entries and + // the pattern silently matches nothing (surfaced by the warning + // below). + const matchingFiles: FileMetadataVo[] = remoteFiles.filter(file => this.matchesFilePattern(spec.filePattern, file)); if (matchingFiles.length === 0) { this.logger.warn(`No files found matching pattern ${spec.filePattern} for connector ${connector.name}`); @@ -375,6 +379,43 @@ export class ConnectorImportProcessorService { } } + /** + * Returns true when `file`'s parent directory equals the pattern's + * directory portion AND its basename matches the pattern's filename glob. + * Both checks use posix path semantics regardless of host OS. + * @param pattern + * @param file + * @returns + */ + private matchesFilePattern(pattern: string, file: FileMetadataVo): boolean { + const { dir, glob } = this.splitFilePattern(pattern); + if (path.posix.dirname(file.fileName) !== dir) return false; + // Escape regex-special characters in the glob, then translate the + // glob wildcard "*" to ".*". Anchor with "^"/"$" so partial + // matches don't slip through (e.g. "*.csv" won't match "data.csv.bak"). + const regex = new RegExp('^' + glob.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'); + return regex.test(path.posix.basename(file.fileName)); + } + + /** + * Splits a connector `filePattern` into its literal directory portion + * and its filename glob. See the comment block above the caller for + * the pattern grammar and examples. + */ + private splitFilePattern(pattern: string): { dir: string; glob: string } { + // Trailing "/" means "all files directly inside this directory". + if (pattern.endsWith('/')) { + const dir = pattern.slice(0, -1); + return { dir: dir === '' ? '.' : dir, glob: '*' }; + } + const lastSlash = pattern.lastIndexOf('/'); + if (lastSlash < 0) { + // No directory portion — pattern is a filename glob at the connector root. + return { dir: '.', glob: pattern }; + } + return { dir: pattern.slice(0, lastSlash), glob: pattern.slice(lastSlash + 1) }; + } + /** * Check if a file has changed since the last download * Returns true if the file should be downloaded diff --git a/back-end/api/src/sql-scripts/default-triggers/default-entry-date-time.sql b/back-end/api/src/sql-scripts/default-triggers/default-entry-date-time.sql index 167b0921..34182d80 100644 --- a/back-end/api/src/sql-scripts/default-triggers/default-entry-date-time.sql +++ b/back-end/api/src/sql-scripts/default-triggers/default-entry-date-time.sql @@ -24,6 +24,7 @@ DECLARE 'export_specifications', 'connector_specifications', 'adapter_specifications', + 'climate_products', 'job_queues', 'connector_execution_log', 'station_forms', diff --git a/back-end/api/src/sql-scripts/logging-triggers/adapter-specification-log.sql b/back-end/api/src/sql-scripts/logging-triggers/adapter-specification-log.sql index 37c4aff9..0065a37d 100644 --- a/back-end/api/src/sql-scripts/logging-triggers/adapter-specification-log.sql +++ b/back-end/api/src/sql-scripts/logging-triggers/adapter-specification-log.sql @@ -4,6 +4,7 @@ BEGIN IF ( NEW.name IS DISTINCT FROM OLD.name OR NEW.description IS DISTINCT FROM OLD.description OR + NEW.language IS DISTINCT FROM OLD.language OR NEW.script_dir_name IS DISTINCT FROM OLD.script_dir_name OR NEW.disabled IS DISTINCT FROM OLD.disabled OR NEW.comment IS DISTINCT FROM OLD.comment @@ -11,6 +12,7 @@ BEGIN NEW.log := COALESCE(OLD.log, '[]'::JSONB) || jsonb_build_object( 'name', OLD.name, 'description', OLD.description, + 'language', OLD.language, 'script_dir_name', OLD.script_dir_name, 'disabled', OLD.disabled, 'comment', OLD.comment, diff --git a/back-end/api/src/sql-scripts/logging-triggers/climate-product-log.sql b/back-end/api/src/sql-scripts/logging-triggers/climate-product-log.sql new file mode 100644 index 00000000..ceb09ee9 --- /dev/null +++ b/back-end/api/src/sql-scripts/logging-triggers/climate-product-log.sql @@ -0,0 +1,29 @@ +CREATE OR REPLACE FUNCTION func_update_climate_products_log() +RETURNS TRIGGER AS $$ +BEGIN + IF ( + NEW.superset_uuid IS DISTINCT FROM OLD.superset_uuid OR + NEW.name IS DISTINCT FROM OLD.name OR + NEW.description IS DISTINCT FROM OLD.description OR + NEW.category IS DISTINCT FROM OLD.category OR + NEW.disabled IS DISTINCT FROM OLD.disabled + ) THEN + NEW.log := COALESCE(OLD.log, '[]'::JSONB) || jsonb_build_object( + 'superset_uuid', OLD.superset_uuid, + 'name', OLD.name, + 'description', OLD.description, + 'category', OLD.category, + 'disabled', OLD.disabled, + 'entryUserId', OLD.entry_user_id, + 'entryDateTime', OLD.entry_date_time + ); + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER trg_update_climate_products_log +BEFORE UPDATE ON climate_products +FOR EACH row +EXECUTE FUNCTION func_update_climate_products_log(); diff --git a/back-end/runners/duckdb/server.js b/back-end/runners/duckdb/server.js index 6d80a422..3a76d422 100644 --- a/back-end/runners/duckdb/server.js +++ b/back-end/runners/duckdb/server.js @@ -58,6 +58,9 @@ app.post('/run', async (req, res) => { try { console.log('Received run request with body:', req.body); const body = req.body; + if (!body) { + return res.json(errorSummary('RUNTIME_ERROR', 'Request body must be JSON')); + } const required = ['scriptDirName', 'operationId', 'inputRelPath', 'outputRelPath', 'timeoutSeconds']; const missing = required.filter(k => !(k in body)); if (missing.length > 0) { @@ -103,6 +106,7 @@ app.post('/run', async (req, res) => { const start = Date.now(); let conn; + let timedOut = false; try { const instance = await DuckDBInstance.create(':memory:'); conn = await instance.connect(); @@ -120,14 +124,31 @@ app.post('/run', async (req, res) => { await conn.run(`SET VARIABLE climsoft_metadata = '${metadataFile.replace(/'/g, "''")}';`); await conn.run(`SET VARIABLE climsoft_warnings = '${warningsFile.replace(/'/g, "''")}';`); - // Step 3: TODO. per-statement timeout so a runaway query doesn't hang the runner. - // TODO. DuckDB does not have a native SQL configuration option (like a SET statement_timeout) to limit total query execution time so find a way of enforcing a timeout. - // The node-api has a `statement_timeout` option on the connection, but it is not exposed via SQL. The following line is commented out because it does not work: - //await conn.run(`SET statement_timeout = '${timeoutSeconds}s';`); + // Step 3: enforce a per-run wall-clock timeout. DuckDB has no SQL-level + // `statement_timeout`, so we drive it from JavaScript: a setTimeout fires + // `conn.interrupt()` which signals the running query to cancel at its + // next safe point. The cancellation surfaces as a thrown error that the + // outer catch below translates into a TIMEOUT response via the + // `timedOut` flag. Only the user SQL is bounded here — extension + // install/load runs on its own budget (large first-time downloads must + // not race this wall clock). + const timeoutHandle = setTimeout(() => { + timedOut = true; + console.warn(`Query exceeded timeout of ${timeoutSeconds}s — interrupting.`); + try { + conn.interrupt(); + } catch (interruptErr) { + console.error('conn.interrupt() failed:', interruptErr); + } + }, timeoutSeconds * 1000); // Step 4: execute the user's SQL. - console.log('Executing user SQL:\n', userSql); - await conn.run(userSql); + try { + console.log('Executing user SQL:\n', userSql); + await conn.run(userSql); + } finally { + clearTimeout(timeoutHandle); + } conn.closeSync(); } catch (err) { @@ -140,8 +161,7 @@ app.post('/run', async (req, res) => { fs.writeFileSync(stdoutFile, ''); fs.writeFileSync(stderrFile, stderr); - const isTimeout = /timeout/i.test(stderr) || /interrupt/i.test(stderr); - if (isTimeout) { + if (timedOut) { return res.json({ status: 'timeout', durationMs, diff --git a/back-end/runners/javascript/server.js b/back-end/runners/javascript/server.js index 70dad45c..8541a60f 100644 --- a/back-end/runners/javascript/server.js +++ b/back-end/runners/javascript/server.js @@ -31,6 +31,9 @@ app.get('/health', (_req, res) => { app.post('/run', async (req, res) => { try { const body = req.body; + if (!body) { + return res.json(errorSummary('RUNTIME_ERROR', 'Request body must be JSON')); + } const required = ['scriptDirName', 'operationId', 'inputRelPath', 'outputRelPath', 'timeoutSeconds']; const missing = required.filter(k => !(k in body)); if (missing.length > 0) { @@ -59,32 +62,41 @@ app.post('/run', async (req, res) => { return res.json(errorSummary('RUNTIME_ERROR', `Entry point not found: ${entryPath}`)); } - // Step 1: install dependencies if .installed doesn't exist + // Step 1: install dependencies if .installed doesn't exist. + // Uses `npm install` for symmetry with the other runners' authoring + // model (user declares top-level deps, installer resolves transitives). + // package-lock.json is optional: if present, it's copied alongside so + // power-users get deterministic installs; otherwise `npm install` + // generates its own lockfile in .installed/ during the run. if (!fs.existsSync(envDir)) { const pkgJson = path.join(scriptDir, 'package.json'); const lockFile = path.join(scriptDir, 'package-lock.json'); - if (!fs.existsSync(pkgJson) || !fs.existsSync(lockFile)) { + if (!fs.existsSync(pkgJson)) { fs.mkdirSync(envDir, { recursive: true }); - fs.writeFileSync(installLogFile, 'No package.json/package-lock.json found. Skipping install.\n'); + fs.writeFileSync(installLogFile, 'No package.json found. Skipping install.\n'); } else { // Copy package files to envDir so node_modules lands there fs.mkdirSync(envDir, { recursive: true }); fs.copyFileSync(pkgJson, path.join(envDir, 'package.json')); - fs.copyFileSync(lockFile, path.join(envDir, 'package-lock.json')); + if (fs.existsSync(lockFile)) { + fs.copyFileSync(lockFile, path.join(envDir, 'package-lock.json')); + } const installResult = await runProcess( - 'npm', ['ci', '--no-audit', '--no-fund'], + 'npm', ['install', '--no-audit', '--no-fund'], { cwd: envDir, timeout: Math.max(timeoutSeconds, 600) * 1000 }, installLogFile, ); if (installResult.exitCode !== 0) { + const errorMessage = installResult.timedOut + ? `npm install exceeded the install timeout; see install.log` + : `npm install exited with code ${installResult.exitCode}; see install.log`; return res.json({ status: 'failure', durationMs: 0, exitCode: installResult.exitCode, - errorType: 'INSTALL_FAILED', - errorMessage: `npm ci exited with code ${installResult.exitCode}; see install.log`, + errorMessage, }); } } diff --git a/back-end/runners/r/server.R b/back-end/runners/r/server.R index af6f07a2..9299a111 100644 --- a/back-end/runners/r/server.R +++ b/back-end/runners/r/server.R @@ -62,6 +62,13 @@ run_handler <- function(req) { # Step 1: set up the per-script renv library if it doesn't exist yet if (!dir.exists(env_dir)) { + # Cap network operations. renv has no wall-clock timeout of its own, so + # we set base R's internet timeout (honored by download.file and friends + # under the hood) to bound individual package downloads. Same "at least + # 600s, or the request timeout if larger" convention as the Python and + # JavaScript runners' install budgets. + old_timeout <- getOption("timeout") + options(timeout = max(timeout_secs, 600)) result <- tryCatch({ dir.create(env_dir, recursive = TRUE, showWarnings = FALSE) lockfile <- file.path(script_dir, "renv.lock") @@ -84,6 +91,7 @@ run_handler <- function(req) { writeLines(paste("Install error:", conditionMessage(e)), install_log) list(ok = FALSE, msg = conditionMessage(e)) }) + options(timeout = old_timeout) if (!result$ok) { return(list( diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 1bd75553..2c5ce807 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,7 +1,19 @@ +x-superset-build: &superset-build + build: + context: ./superset + dockerfile: Dockerfile + profiles: ["superset"] + +x-superset-env: &superset-env + SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-superset_dev_secret_change_in_prod} + SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:${DB_PASSWORD:-my_password}@climsoft_db:5432/superset" + REDIS_URL: "redis://climsoft_superset_redis:6379/0" + services: - climsoft_dev_db: + climsoft_db: image: postgis/postgis:17-3.5 + container_name: climsoft-dev-db environment: POSTGRES_DB: climsoft POSTGRES_USER: postgres @@ -11,12 +23,17 @@ services: volumes: - climsoft_dev_data:/var/lib/postgresql/data - ./back-end/api/temp/operations:/var/lib/postgresql/operations + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 5s + timeout: 3s + retries: 10 climsoft_python_runner: build: context: ./back-end/runners/python dockerfile: Dockerfile - container_name: climsoft-python-runner + container_name: climsoft-dev-python-runner profiles: ["adapters-python"] ports: - "5101:5101" @@ -28,7 +45,7 @@ services: build: context: ./back-end/runners/r dockerfile: Dockerfile - container_name: climsoft-r-runner + container_name: climsoft-dev-r-runner profiles: ["adapters-r"] ports: - "5102:5102" @@ -40,7 +57,7 @@ services: build: context: ./back-end/runners/javascript dockerfile: Dockerfile - container_name: climsoft-javascript-runner + container_name: climsoft-dev-javascript-runner profiles: ["adapters-javascript"] ports: - "5103:5103" @@ -60,5 +77,81 @@ services: - ./back-end/api/temp/adapters:/app/adapters - ./back-end/api/temp/operations:/app/operations + climsoft_superset_redis: + image: redis:7 + container_name: climsoft-dev-superset-redis + profiles: ["superset"] + ports: + - "6379:6379" + + climsoft_superset_init: + <<: *superset-build + container_name: climsoft-dev-superset-init + command: ["/app/init.sh"] + environment: + <<: *superset-env + ADMIN_USERNAME: ${SUPERSET_ADMIN_USERNAME:-admin} + ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:-admin} + SERVICE_USERNAME: ${SUPERSET_SERVICE_USERNAME:-climsoft_service} + SERVICE_PASSWORD: ${SUPERSET_SERVICE_PASSWORD:-climsoft_service} + DB_HOST: climsoft_db + DB_NAME: climsoft + DB_PASSWORD: ${DB_PASSWORD:-my_password} + depends_on: + climsoft_db: + condition: service_healthy + climsoft_superset_redis: + condition: service_started + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + - ./superset/init.sh:/app/init.sh + - ./superset/ensure_superset_db.py:/app/ensure_superset_db.py + - ./superset/init_datasets.py:/app/init_datasets.py + - ./superset/datasets:/app/datasets + - ./superset/products:/app/products + + climsoft_superset: + <<: *superset-build + container_name: climsoft-dev-superset + command: ["/app/docker/entrypoints/run-server.sh"] + environment: + <<: *superset-env + SERVER_WORKER_AMOUNT: 2 + SERVER_WORKER_CLASS: gevent + SERVER_THREADS_AMOUNT: 100 + GUNICORN_TIMEOUT: 120 + ports: + - "8088:8088" + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + + climsoft_superset_worker: + <<: *superset-build + container_name: climsoft-dev-superset-worker + command: ["/app/worker.sh"] + environment: + <<: *superset-env + CELERY_CONCURRENCY: 2 + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + + climsoft_superset_beat: + <<: *superset-build + container_name: climsoft-dev-superset-beat + command: ["/app/beat.sh"] + environment: + <<: *superset-env + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + volumes: - climsoft_dev_data: \ No newline at end of file + climsoft_dev_data: diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 6b4f395e..3f3e4e44 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,7 +1,21 @@ +x-superset-image: &superset-image + image: climsoftdevelopers/climsoft-superset:preview-3.0.3 + profiles: ["superset"] + +x-superset-env: &superset-env + SUPERSET_SECRET_KEY: "${SUPERSET_SECRET_KEY}" + SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:${DB_PASSWORD}@climsoft_db:5432/superset" + REDIS_URL: "redis://climsoft_superset_redis:6379/0" + +x-climsoft-db-env: &climsoft-db-env + DB_HOST: climsoft_db + DB_NAME: "${DB_NAME}" + DB_PASSWORD: "${DB_PASSWORD}" + services: climsoft_nginx_proxy: - image: climsoftdevelopers/climsoft-nginx-proxy:preview-3.0.2 + image: climsoftdevelopers/climsoft-nginx-proxy:preview-3.0.3 container_name: climsoft-nginx-proxy restart: always ports: # Expose to external network @@ -13,13 +27,13 @@ services: - climsoft_network climsoft_pwa: - image: climsoftdevelopers/climsoft-pwa:preview-3.0.2 + image: climsoftdevelopers/climsoft-pwa:preview-3.0.3 container_name: climsoft-pwa restart: always environment: HOST_IP_ADDRESS: "${HOST_IP_ADDRESS}" # used in config.json. HOST_HTTP_PORT: "${HOST_HTTP_PORT}" # used in config.json. - expose: + expose: - 80 # Expose to internal network only. depends_on: - climsoft_api @@ -27,15 +41,13 @@ services: - climsoft_network climsoft_api: - image: climsoftdevelopers/climsoft-api:preview-3.0.2 + image: climsoftdevelopers/climsoft-api:preview-3.0.3 container_name: climsoft-api restart: always environment: - DB_HOST: climsoft_db + <<: *climsoft-db-env DB_PORT: 5432 - DB_NAME: "${DB_NAME}" DB_USERNAME: postgres - DB_PASSWORD: "${DB_PASSWORD}" ENCRYPTION_SECRET: "${ENCRYPTION_SECRET}" FIRST_INSTALL: "${FIRST_INSTALL}" V4_SAVE: "${V4_SAVE}" @@ -58,6 +70,11 @@ services: DUCKDB_RUNNER_ENABLED: "${DUCKDB_RUNNER_ENABLED:-false}" DUCKDB_RUNNER_HOST: climsoft_duckdb_runner DUCKDB_RUNNER_PORT: 5104 + SUPERSET_ENABLED: "${SUPERSET_ENABLED:-false}" + SUPERSET_HOST: climsoft_superset + SUPERSET_PORT: 8088 + SUPERSET_SERVICE_USERNAME: "${SUPERSET_SERVICE_USERNAME}" + SUPERSET_SERVICE_PASSWORD: "${SUPERSET_SERVICE_PASSWORD}" expose: - 3000 depends_on: @@ -70,7 +87,7 @@ services: - climsoft_network climsoft_python_runner: - image: climsoftdevelopers/climsoft-python-runner:preview-3.0.2 + image: climsoftdevelopers/climsoft-python-runner:preview-3.0.3 container_name: climsoft-python-runner restart: always profiles: ["adapters-python"] @@ -83,7 +100,7 @@ services: - climsoft_network climsoft_r_runner: - image: climsoftdevelopers/climsoft-r-runner:preview-3.0.2 + image: climsoftdevelopers/climsoft-r-runner:preview-3.0.3 container_name: climsoft-r-runner restart: always profiles: ["adapters-r"] @@ -96,7 +113,7 @@ services: - climsoft_network climsoft_javascript_runner: - image: climsoftdevelopers/climsoft-javascript-runner:preview-3.0.2 + image: climsoftdevelopers/climsoft-javascript-runner:preview-3.0.3 container_name: climsoft-javascript-runner restart: always profiles: ["adapters-javascript"] @@ -109,7 +126,7 @@ services: - climsoft_network climsoft_duckdb_runner: - image: climsoftdevelopers/climsoft-duckdb-runner:preview-3.0.2 + image: climsoftdevelopers/climsoft-duckdb-runner:preview-3.0.3 container_name: climsoft-duckdb-runner restart: always profiles: ["adapters-duckdb"] @@ -121,16 +138,91 @@ services: networks: - climsoft_network + climsoft_superset_redis: + image: redis:7-alpine + container_name: climsoft-superset-redis + restart: always + profiles: ["superset"] + expose: + - 6379 + networks: + - climsoft_network + + climsoft_superset_init: + <<: *superset-image + container_name: climsoft-superset-init + command: ["/app/init.sh"] + environment: + <<: [*superset-env, *climsoft-db-env] + ADMIN_USERNAME: "${SUPERSET_ADMIN_USERNAME}" + ADMIN_PASSWORD: "${SUPERSET_ADMIN_PASSWORD}" + SERVICE_USERNAME: "${SUPERSET_SERVICE_USERNAME}" + SERVICE_PASSWORD: "${SUPERSET_SERVICE_PASSWORD}" + depends_on: + climsoft_db: + condition: service_healthy + climsoft_superset_redis: + condition: service_started + networks: + - climsoft_network + + climsoft_superset: + <<: *superset-image + container_name: climsoft-superset + restart: always + command: ["/app/docker/entrypoints/run-server.sh"] + environment: + <<: *superset-env + SERVER_WORKER_AMOUNT: 10 + SERVER_WORKER_CLASS: gevent + SERVER_THREADS_AMOUNT: 1000 + GUNICORN_TIMEOUT: 120 + expose: + - 8088 + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - climsoft_superset_home:/app/superset_home + networks: + - climsoft_network + + climsoft_superset_worker: + <<: *superset-image + container_name: climsoft-superset-worker + restart: always + command: ["/app/worker.sh"] + environment: + <<: *superset-env + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + networks: + - climsoft_network + + climsoft_superset_beat: + <<: *superset-image + container_name: climsoft-superset-beat + restart: always + command: ["/app/beat.sh"] + environment: + <<: *superset-env + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + networks: + - climsoft_network + climsoft_db: image: postgis/postgis:17-3.5 container_name: climsoft-db restart: always environment: - POSTGRES_DB: "${DB_NAME}" # Default database postgres will create, required when setting up postgres the first time. - POSTGRES_USER: postgres # Uses the default postgres username for simplicity, required when setting up postgres the first time. - POSTGRES_PASSWORD: "${DB_PASSWORD}" # Super user password, required when setting up postgres the first time. + POSTGRES_DB: "${DB_NAME}" + POSTGRES_USER: postgres + POSTGRES_PASSWORD: "${DB_PASSWORD}" ports: - - "5432:5432" # Expose to both external and internal network. + - "5432:5432" volumes: - climsoft_database:/var/lib/postgresql/data - climsoft_operations:/var/lib/postgresql/operations @@ -146,6 +238,7 @@ volumes: climsoft_database: climsoft_operations: climsoft_adapters: + climsoft_superset_home: networks: - climsoft_network: \ No newline at end of file + climsoft_network: diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml index 9b4e6ade..ef126605 100644 --- a/docker-compose.test.yaml +++ b/docker-compose.test.yaml @@ -1,10 +1,26 @@ +x-superset-build: &superset-build + build: + context: ./superset + dockerfile: Dockerfile + profiles: ["superset"] + +x-superset-env: &superset-env + SUPERSET_SECRET_KEY: "${SUPERSET_SECRET_KEY}" + SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:${DB_PASSWORD}@climsoft_db:5432/superset" + REDIS_URL: "redis://climsoft_superset_redis:6379/0" + +x-climsoft-db-env: &climsoft-db-env + DB_HOST: climsoft_db + DB_NAME: "${DB_NAME}" + DB_PASSWORD: "${DB_PASSWORD}" + services: climsoft_nginx_proxy: build: context: ./ dockerfile: Dockerfile - container_name: climsoft-nginx-proxy + container_name: climsoft-test-nginx-proxy ports: # Expose to external network - "${HOST_HTTP_PORT}:80" #- "${HOST_HTTPS_PORT}:443" # Use this if HTTPS is required @@ -20,7 +36,7 @@ services: build: context: ./front-end/pwa dockerfile: Dockerfile - container_name: climsoft-pwa + container_name: climsoft-test-pwa environment: HOST_IP_ADDRESS: "${HOST_IP_ADDRESS}" # used in config.json. HOST_HTTP_PORT: "${HOST_HTTP_PORT}" # used in config.json. @@ -35,13 +51,11 @@ services: build: context: ./back-end/api dockerfile: Dockerfile - container_name: climsoft-api + container_name: climsoft-test-api environment: - DB_HOST: climsoft_db + <<: *climsoft-db-env DB_PORT: 5432 - DB_NAME: "${DB_NAME}" DB_USERNAME: postgres - DB_PASSWORD: "${DB_PASSWORD}" ENCRYPTION_SECRET: "${ENCRYPTION_SECRET}" FIRST_INSTALL: "${FIRST_INSTALL}" V4_SAVE: "${V4_SAVE}" @@ -64,6 +78,11 @@ services: DUCKDB_RUNNER_ENABLED: "true" DUCKDB_RUNNER_HOST: climsoft_duckdb_runner DUCKDB_RUNNER_PORT: 5104 + SUPERSET_ENABLED: "true" + SUPERSET_HOST: climsoft_superset + SUPERSET_PORT: 8088 + SUPERSET_SERVICE_USERNAME: "${SUPERSET_SERVICE_USERNAME}" + SUPERSET_SERVICE_PASSWORD: "${SUPERSET_SERVICE_PASSWORD}" expose: - 3000 depends_on: @@ -79,7 +98,7 @@ services: build: context: ./back-end/runners/python dockerfile: Dockerfile - container_name: climsoft-python-runner + container_name: climsoft-test-python-runner profiles: ["adapters-python"] expose: - 5101 @@ -93,7 +112,7 @@ services: build: context: ./back-end/runners/r dockerfile: Dockerfile - container_name: climsoft-r-runner + container_name: climsoft-test-r-runner profiles: ["adapters-r"] expose: - 5102 @@ -107,7 +126,7 @@ services: build: context: ./back-end/runners/javascript dockerfile: Dockerfile - container_name: climsoft-javascript-runner + container_name: climsoft-test-javascript-runner profiles: ["adapters-javascript"] expose: - 5103 @@ -121,7 +140,7 @@ services: build: context: ./back-end/runners/duckdb dockerfile: Dockerfile - container_name: climsoft-duckdb-runner + container_name: climsoft-test-duckdb-runner profiles: ["adapters-duckdb"] expose: - 5104 @@ -131,9 +150,93 @@ services: networks: - climsoft_network + climsoft_superset_redis: + image: redis:7-alpine + container_name: climsoft-test-superset-redis + profiles: ["superset"] + expose: + - 6379 + networks: + - climsoft_network + + climsoft_superset_init: + <<: *superset-build + container_name: climsoft-test-superset-init + command: ["/app/init.sh"] + environment: + <<: [*superset-env, *climsoft-db-env] + ADMIN_USERNAME: "${SUPERSET_ADMIN_USERNAME}" + ADMIN_PASSWORD: "${SUPERSET_ADMIN_PASSWORD}" + SERVICE_USERNAME: "${SUPERSET_SERVICE_USERNAME}" + SERVICE_PASSWORD: "${SUPERSET_SERVICE_PASSWORD}" + depends_on: + climsoft_db: + condition: service_healthy + climsoft_superset_redis: + condition: service_started + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + - ./superset/init.sh:/app/init.sh + - ./superset/ensure_superset_db.py:/app/ensure_superset_db.py + - ./superset/init_datasets.py:/app/init_datasets.py + - ./superset/datasets:/app/datasets + - ./superset/products:/app/products + networks: + - climsoft_network + + climsoft_superset: + <<: *superset-build + container_name: climsoft-test-superset + command: ["/app/docker/entrypoints/run-server.sh"] + environment: + <<: *superset-env + SERVER_WORKER_AMOUNT: 2 + SERVER_WORKER_CLASS: gevent + SERVER_THREADS_AMOUNT: 100 + GUNICORN_TIMEOUT: 120 + expose: + - 8088 + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + - climsoft_test_superset:/app/superset_home + networks: + - climsoft_network + + climsoft_superset_worker: + <<: *superset-build + container_name: climsoft-test-superset-worker + command: ["/app/worker.sh"] + environment: + <<: *superset-env + CELERY_CONCURRENCY: 2 + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + networks: + - climsoft_network + + climsoft_superset_beat: + <<: *superset-build + container_name: climsoft-test-superset-beat + command: ["/app/beat.sh"] + environment: + <<: *superset-env + depends_on: + climsoft_superset_init: + condition: service_completed_successfully + volumes: + - ./superset/superset_config.py:/app/pythonpath/superset_config.py + networks: + - climsoft_network + climsoft_db: image: postgis/postgis:17-3.5 - container_name: climsoft-db + container_name: climsoft-test-db environment: POSTGRES_DB: "${DB_NAME}" # Default database postgres will create, required when setting up postgres the first time. POSTGRES_USER: postgres # Uses the default postgres username for simplicity, required when setting up postgres the first time. @@ -155,6 +258,7 @@ volumes: climsoft_test_database: climsoft_test_operations: climsoft_test_adapters: + climsoft_test_superset: networks: climsoft_network: \ No newline at end of file diff --git a/front-end/pwa/package-lock.json b/front-end/pwa/package-lock.json index c413902d..b501e62e 100644 --- a/front-end/pwa/package-lock.json +++ b/front-end/pwa/package-lock.json @@ -1,12 +1,13 @@ { "name": "climsoftweb", - "version": "preview-2.0.5", + "version": "preview-3.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "climsoftweb", - "version": "preview-2.0.5", + "version": "preview-3.0.4", + "license": "GPL-3.0-only", "dependencies": { "@angular/animations": "^16.0.0", "@angular/cdk": "^16.0.1", @@ -18,6 +19,7 @@ "@angular/platform-browser-dynamic": "^16.0.0", "@angular/router": "^16.0.0", "@angular/service-worker": "^16.0.0", + "@superset-ui/embedded-sdk": "^0.4.0", "@turf/turf": "^7.1.0", "@types/leaflet": "^1.9.6", "bootstrap": "^5.3.0", @@ -3779,6 +3781,22 @@ "dev": true, "license": "MIT" }, + "node_modules/@superset-ui/embedded-sdk": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@superset-ui/embedded-sdk/-/embedded-sdk-0.4.0.tgz", + "integrity": "sha512-k/PnzvxO0xfeaAO6DRWsC7nG9oH3ooyBQNTRi/EXrK4DIH4ufx6Wy3AD3qDkCjGRw3KMdavCdwAiu2u4TVQ7oA==", + "license": "Apache-2.0", + "dependencies": { + "@superset-ui/switchboard": "^0.20.3", + "jwt-decode": "^4.0.0" + } + }, + "node_modules/@superset-ui/switchboard": { + "version": "0.20.3", + "resolved": "https://registry.npmjs.org/@superset-ui/switchboard/-/switchboard-0.20.3.tgz", + "integrity": "sha512-qEMXFwdRLfXug4gXXdBEGpFtBWZoxdZkCJLBVxj1IR8cQvSqjkWAQOzSSYYdcIeREWqi8iP+iK6apNV1ZQCKcA==", + "license": "Apache-2.0" + }, "node_modules/@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -10696,6 +10714,15 @@ "node": ">= 12" } }, + "node_modules/jwt-decode": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz", + "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/karma": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/karma/-/karma-6.4.4.tgz", diff --git a/front-end/pwa/package.json b/front-end/pwa/package.json index 5923218d..3ca02404 100644 --- a/front-end/pwa/package.json +++ b/front-end/pwa/package.json @@ -1,6 +1,6 @@ { "name": "climsoftweb", - "version": "preview-3.0.2", + "version": "preview-3.0.4", "description": "Climsoft PWA", "author": "Patrick Munyoki
- Climsoft Web (preview-3.0.1) is a free and open-source modern climate and hydrology data management platform. + Climsoft Web (preview-3.0.3) is a free and open-source modern climate and hydrology data management platform.
\ No newline at end of file diff --git a/front-end/pwa/src/app/core/dashboard/dashboard.component.ts b/front-end/pwa/src/app/core/dashboard/dashboard.component.ts index 6d8e1078..c83b2472 100644 --- a/front-end/pwa/src/app/core/dashboard/dashboard.component.ts +++ b/front-end/pwa/src/app/core/dashboard/dashboard.component.ts @@ -50,7 +50,7 @@ export class DashboardComponent implements OnDestroy { // { icon: 'bi-check-circle-fill text-success', message: 'All systems are operational' }, { icon: 'bi-exclamation-circle', message: '0 observations failed QC checks today' }, // { icon: 'bi-exclamation-triangle-fill text-danger', message: 'Station KE005 has not reported in 3 hours' }, - { icon: 'bi-wrench-adjustable-circle ', message: 'A new Climsoft Web update (preview-3.0.2-beta) is available' }, + { icon: 'bi-wrench-adjustable-circle ', message: 'A new Climsoft Web update (preview-3.0.4-beta) is available' }, // { icon: 'bi-cloud-arrow-down-fill text-primary', message: 'Last data import: 2025-05-22 18:15 UTC' } ]; diff --git a/front-end/pwa/src/app/core/home/home.component.ts b/front-end/pwa/src/app/core/home/home.component.ts index b6aded39..70ee1f16 100644 --- a/front-end/pwa/src/app/core/home/home.component.ts +++ b/front-end/pwa/src/app/core/home/home.component.ts @@ -228,6 +228,7 @@ export class HomeComponent implements OnInit, OnDestroy { this.featuresNavItems.push(metadataMenuItems); //------------------------------------------- + } } diff --git a/front-end/pwa/src/app/core/home/menu-items.ts b/front-end/pwa/src/app/core/home/menu-items.ts index b4862f5e..7dca3b57 100644 --- a/front-end/pwa/src/app/core/home/menu-items.ts +++ b/front-end/pwa/src/app/core/home/menu-items.ts @@ -13,7 +13,7 @@ export interface MenuItem { export enum MainMenuNameEnum { DASHBOARD = 'Dashboard', DATA_INGESTION = 'Data Ingestion', - DATA_MONITORING = 'Data Monitoring', + DATA_MONITORING = 'Monitoring & Products', QUALITY_CONTROL = 'Quality Control', DATA_EXTRACTION = 'Data Extraction', METADATA = 'Metadata', @@ -30,6 +30,7 @@ export enum SubMenuNameEnum { DATA_FLOW = 'Data Flow', DATA_AVAILABILTY = 'Data Availabilty', DATA_EXPLORER = 'Data Explorer', + PRODUCT_LIST = 'Product List', QC_ASSESSMENT = 'QC Assessment', @@ -46,6 +47,9 @@ export enum SubMenuNameEnum { EXPORT_SPECIFICATIONS = 'Export Specifications', CONNECTOR_SPECIFICATIONS = 'Connector Specifications', ADAPTERS_SPECIFICATIONS = 'Adapter Specifications', + CLIMATE_PRODUCTS_ADMIN = 'Climate Products', + + USER_GROUPS = 'User Groups', USERS = 'Users', @@ -111,6 +115,10 @@ export class MenuItemsUtil { name: SubMenuNameEnum.DATA_FLOW, url: '/data-flow', }, + { + name: SubMenuNameEnum.PRODUCT_LIST, + url: '/product-list', + }, ] } } @@ -141,10 +149,6 @@ export class MenuItemsUtil { name: SubMenuNameEnum.MANUAL_EXPORT, url: '/manual-export-selection', }, - // { - // name: SubMenuNameEnum.SCHEDULED_EXPORT, - // url: '/auto-export-selection', - // }, ] }; } @@ -200,6 +204,10 @@ export class MenuItemsUtil { name: SubMenuNameEnum.ADAPTERS_SPECIFICATIONS, url: '/view-adapters', }, + { + name: SubMenuNameEnum.CLIMATE_PRODUCTS_ADMIN, + url: '/view-climate-products', + }, ] } } diff --git a/front-end/pwa/src/app/data-ingestion/value-flag-input/value-flag-input.component.ts b/front-end/pwa/src/app/data-ingestion/value-flag-input/value-flag-input.component.ts index e25d7ef6..9e2c4aff 100644 --- a/front-end/pwa/src/app/data-ingestion/value-flag-input/value-flag-input.component.ts +++ b/front-end/pwa/src/app/data-ingestion/value-flag-input/value-flag-input.component.ts @@ -363,8 +363,9 @@ export class ValueFlagInputComponent implements OnChanges { if (`${this.valueFlagInput}-${this.comment}` === this.originalValues) { this.observationEntry.change = 'no_change'; } else if (this.observationEntry.observation.value === null && this.observationEntry.observation.flagId === null) { - this.validationErrorMessage = 'Value and flag cannot be both empty. To clear the field, clear the comment' - this.observationEntry.change = 'invalid_change'; + // Back end does not allow both value and flag to be null. So if both are null then it is considered as no change + // TODO. In future, the user should be given a warning that the deletion changes will not be saved. For now, just consider it as no change. + this.observationEntry.change = 'no_change'; } else { this.observationEntry.change = 'valid_change'; } @@ -389,7 +390,7 @@ export class ValueFlagInputComponent implements OnChanges { // Step 2. // Check if it's a pure integer (no decimals). - if (/^\d+$/.test(input)) { + if (/^[+-]?\d+$/.test(input)) { response.flag = null; response.value = parseInt(input, 10); return response; @@ -398,7 +399,7 @@ export class ValueFlagInputComponent implements OnChanges { // Step 3. // Check if it starts with digits followed by alphanumeric/special chars. // Values should strictly follow the number first then character format. - const mixed = input.match(/^(\d+)([^0-9].*)$/); + const mixed = input.match(/^([+-]?\d+)([^0-9].*)$/); if (mixed) { const flagFound = this.cachedMetadataService.getFlagByAbbreviationOrName(mixed[2]); if (flagFound) { diff --git a/front-end/pwa/src/app/data-monitoring/data-monitoring-routing.module.ts b/front-end/pwa/src/app/data-monitoring/data-monitoring-routing.module.ts index ce4e9246..91289fc1 100644 --- a/front-end/pwa/src/app/data-monitoring/data-monitoring-routing.module.ts +++ b/front-end/pwa/src/app/data-monitoring/data-monitoring-routing.module.ts @@ -4,6 +4,8 @@ import { DataFlowComponent } from './data-flow/data-flow.component'; import { DataExplorerComponent } from './data-explorer/data-explorer.component'; import { stationStatusComponent } from './station-status/stations-status.component'; import { DataAvailabilityComponent } from './data-availability/data-availability.component'; +import { ProductListComponent } from './products/product-list/product-list.component'; +import { ProductViewerComponent } from './products/product-viewer/product-viewer.component'; const routes: Routes = [ { @@ -30,6 +32,14 @@ const routes: Routes = [ path: 'data-explorer', component: DataExplorerComponent }, + { + path: 'product-list', + component: ProductListComponent + }, + { + path: 'product-viewer/:id', + component: ProductViewerComponent + }, ] } diff --git a/front-end/pwa/src/app/data-monitoring/data-monitoring.module.ts b/front-end/pwa/src/app/data-monitoring/data-monitoring.module.ts index 8465448f..9b54475f 100644 --- a/front-end/pwa/src/app/data-monitoring/data-monitoring.module.ts +++ b/front-end/pwa/src/app/data-monitoring/data-monitoring.module.ts @@ -16,6 +16,8 @@ import { DataAvailabilitySummaryComponent } from './data-availability/data-avail import { DataAvailabilityFilterSelectionGeneralComponent } from './data-availability/data-availability-filter-selection-general/data-availability-filter-selection-general.component'; import { DataAvailabilityHeatmapComponent } from './data-availability/data-availability-summary/data-availability-heatmap/data-availability-heatmap.component'; import { DataAvailabilityDetailsDialogComponent } from './data-availability/data-availability-summary/data-availability-details-dialog/data-availability-details-dialog.component'; +import { ProductViewerComponent } from './products/product-viewer/product-viewer.component'; +import { ProductListComponent } from './products/product-list/product-list.component'; @NgModule({ declarations: [ @@ -31,6 +33,8 @@ import { DataAvailabilityDetailsDialogComponent } from './data-availability/data DataAvailabilityHeatmapComponent, DataAvailabilityDetailsDialogComponent, DataExplorerComponent, + ProductListComponent, + ProductViewerComponent, ], imports: [ DataMonitoringRoutingModule, diff --git a/front-end/pwa/src/app/data-monitoring/products/models/view-product.model.ts b/front-end/pwa/src/app/data-monitoring/products/models/view-product.model.ts new file mode 100644 index 00000000..ca6ff42b --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/models/view-product.model.ts @@ -0,0 +1,22 @@ +export interface ViewProductModel { + id: number; + systemKey: string | null; + supersetUuid: string; + name: string; + description: string | null; + category: string | null; + disabled: boolean; +} + +export interface ProductGuestTokenModel { + token: string; + supersetUuid: string; +} + +export interface CreateUpdateProductModel { + supersetUuid: string; + name: string; + description: string | null; + category: string | null; + disabled: boolean; +} diff --git a/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.html b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.html new file mode 100644 index 00000000..27ff6a2a --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.html @@ -0,0 +1,48 @@ +
+
+
+ [{{ products.length | number:'1.0-0' }}] +
+
+ +
+ +
+ +
+
{{ errorMessage }}
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
NameCategoryDescription
{{ product.name }}{{ product.category || '—' }}{{ product.description || '—' }}
+ No climate products are available. +
+
+
+
diff --git a/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.scss b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.scss new file mode 100644 index 00000000..5f47a40e --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.scss @@ -0,0 +1,9 @@ +.product-card { + cursor: pointer; + transition: box-shadow 0.2s, transform 0.1s; + + &:hover { + box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15) !important; + transform: translateY(-2px); + } +} diff --git a/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.ts b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.ts new file mode 100644 index 00000000..a2b1a762 --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-list/product-list.component.ts @@ -0,0 +1,48 @@ +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { Router } from '@angular/router'; +import { Subject } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; +import { ViewProductModel } from '../models/view-product.model'; +import { ProductsService } from '../services/products.service'; + +@Component({ + selector: 'app-product-list', + templateUrl: './product-list.component.html', + styleUrls: ['./product-list.component.scss'], +}) +export class ProductListComponent implements OnInit, OnDestroy { + protected products: ViewProductModel[] = []; + protected loading = true; + protected errorMessage: string | null = null; + + private readonly destroy$ = new Subject(); + + constructor( + private readonly productsService: ProductsService, + private readonly router: Router, + ) { } + + ngOnInit(): void { + this.productsService.findAll() + .pipe(takeUntil(this.destroy$)) + .subscribe({ + next: products => { + this.products = products; + this.loading = false; + }, + error: () => { + this.errorMessage = 'Failed to load climate products.'; + this.loading = false; + }, + }); + } + + protected openProduct(product: ViewProductModel): void { + this.router.navigate(['/data-monitoring/product-viewer', product.id]); + } + + ngOnDestroy(): void { + this.destroy$.next(); + this.destroy$.complete(); + } +} diff --git a/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.html b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.html new file mode 100644 index 00000000..b7271769 --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.html @@ -0,0 +1,19 @@ +
+ +
+ + {{ product.name }} + — {{ product.description }} +
+ +
+ +
+ +
{{ errorMessage }}
+ +
+ +
diff --git a/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.scss b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.scss new file mode 100644 index 00000000..0bef1db4 --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.scss @@ -0,0 +1,12 @@ +.product-viewer-wrapper { + height: calc(100vh - 56px); // subtract nav bar height +} + +.superset-container { + // The SDK injects an iframe into this element + iframe { + width: 100%; + height: 100%; + border: none; + } +} diff --git a/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.ts b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.ts new file mode 100644 index 00000000..12084206 --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/product-viewer/product-viewer.component.ts @@ -0,0 +1,100 @@ +import { Component, ElementRef, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; +import { embedDashboard } from '@superset-ui/embedded-sdk'; +import { firstValueFrom, Subject } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; +import { AppConfigService } from 'src/app/app-config.service'; +import { ViewProductModel } from '../models/view-product.model'; +import { ProductsService } from '../services/products.service'; + +@Component({ + selector: 'app-product-viewer', + templateUrl: './product-viewer.component.html', + styleUrls: ['./product-viewer.component.scss'], +}) +export class ProductViewerComponent implements OnInit, OnDestroy { + @ViewChild('dashboardContainer', { static: true }) container!: ElementRef; + + protected product: ViewProductModel | null = null; + protected loading = true; + protected errorMessage: string | null = null; + + private readonly destroy$ = new Subject(); + private productId!: number; + + constructor( + private readonly route: ActivatedRoute, + private readonly productsService: ProductsService, + private readonly appConfigService: AppConfigService, + ) { } + + ngOnInit(): void { + this.route.params.pipe(takeUntil(this.destroy$)).subscribe(params => { + this.productId = +params['id']; + this.initEmbed(); + }); + } + + private initEmbed(): void { + this.loading = true; + this.errorMessage = null; + + // Fetch initial token and dashboard UUID in one call, then start embedding + this.productsService.getGuestToken(this.productId) + .pipe(takeUntil(this.destroy$)) + .subscribe({ + next: ({ token: initialToken, supersetUuid }) => { + this.productsService.findOne(this.productId) + .pipe(takeUntil(this.destroy$)) + .subscribe({ + next: product => { + this.product = product; + this.embed(supersetUuid, initialToken); + }, + error: () => this.showError(), + }); + }, + error: () => this.showError(), + }); + } + + private embed(supersetUuid: string, initialToken: string): void { + let firstToken = true; + + embedDashboard({ + id: supersetUuid, + supersetDomain: this.appConfigService.supersetBaseUrl, + mountPoint: this.container.nativeElement, + fetchGuestToken: () => { + if (firstToken) { + firstToken = false; + return Promise.resolve(initialToken); + } + return firstValueFrom(this.productsService.getGuestToken(this.productId)) + .then(r => r!.token); + }, + dashboardUiConfig: { + hideTitle: true, + hideChartControls: false, + filters: { visible: true, expanded: false }, + }, + }).then(() => { + this.loading = false; + }).catch(() => { + this.showError(); + }); + } + + private showError(): void { + this.errorMessage = 'Unable to load this climate product. Please try again.'; + this.loading = false; + } + + ngOnDestroy(): void { + this.destroy$.next(); + this.destroy$.complete(); + if (this.container?.nativeElement) { + this.container.nativeElement.innerHTML = ''; + } + } +} diff --git a/front-end/pwa/src/app/data-monitoring/products/services/products.service.ts b/front-end/pwa/src/app/data-monitoring/products/services/products.service.ts new file mode 100644 index 00000000..e2363deb --- /dev/null +++ b/front-end/pwa/src/app/data-monitoring/products/services/products.service.ts @@ -0,0 +1,45 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { AppConfigService } from 'src/app/app-config.service'; +import { CreateUpdateProductModel, ProductGuestTokenModel, ViewProductModel } from '../models/view-product.model'; + +@Injectable({ providedIn: 'root' }) +export class ProductsService { + private readonly endPointUrl: string; + + constructor( + private readonly appConfigService: AppConfigService, + private readonly http: HttpClient, + ) { + this.endPointUrl = `${this.appConfigService.apiBaseUrl}/products`; + } + + public findAll(): Observable { + return this.http.get(this.endPointUrl); + } + + public findAllAdmin(): Observable { + return this.http.get(`${this.endPointUrl}/all`); + } + + public findOne(id: number): Observable { + return this.http.get(`${this.endPointUrl}/${id}`); + } + + public getGuestToken(id: number): Observable { + return this.http.get(`${this.endPointUrl}/${id}/guest-token`); + } + + public create(dto: CreateUpdateProductModel): Observable { + return this.http.post(this.endPointUrl, dto); + } + + public update(id: number, dto: CreateUpdateProductModel): Observable { + return this.http.patch(`${this.endPointUrl}/${id}`, dto); + } + + public deleteProduct(id: number): Observable { + return this.http.delete(`${this.endPointUrl}/${id}`); + } +} diff --git a/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.html b/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.html index c54c3016..38836831 100644 --- a/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.html +++ b/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.html @@ -2,77 +2,108 @@
-
- -
+ + +
+ Name + {{ adapter.name }} +
+
+
+ Description + {{ adapter.description || '—' }} +
+
+
+ Language + {{ languageLabel }} +
+
+
+ Comment + {{ adapter.comment || '—' }} +
+
+
+ +
+
+
+ Shipped adapter — only visibility settings can be changed. +
+ -
+ + +
+ +
-
- -
+
-
+
+ +
-
- -
-
- Language cannot be changed after creation. -
-
+
+ +
+ +
-
-
- - -
- -
-
- - Upload a zip containing the adapter source and a manifest file matching the selected - language. - - - Optional. Upload to replace the current version. - -
-
- +
+ + +
+ +
+
+ + Upload a zip containing the adapter source and a manifest file matching the selected + language. + + + Optional. Upload to replace the current version. + +
+
+ +
-
-
+
-
- -
+
+ +
+
- +
diff --git a/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.ts b/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.ts index c8d85ae5..5c4c2a0e 100644 --- a/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.ts +++ b/front-end/pwa/src/app/metadata/adapters/adapter-detail-dialog/adapter-detail-dialog.component.ts @@ -6,9 +6,9 @@ import { AdaptersService } from '../services/adapters.service'; import { ViewAdapterSpecificationModel } from '../models/view-adapter-specification.model'; import { CreateAdapterSpecificationModel } from '../models/create-adapter-specification.model'; import { UpdateAdapterSpecificationModel } from '../models/update-adapter-specification.model'; -import { AdapterLanguageEnum } from '../models/adapter-language.enum'; +import { ADAPTER_LANGUAGE_LABELS, AdapterLanguageEnum } from '../models/adapter-language.enum'; import { AdapterUploadPreviewResponseModel, FileTreeEntry } from '../models/adapter-upload-preview-response.model'; -import { CANONICAL_ENTRY_POINT, MANIFEST_FILENAMES } from '../models/adapter-language-conventions'; +import { LANGUAGE_CONVENTIONS } from '../models/adapter-language-conventions'; @Component({ selector: 'app-adapter-detail-dialog', @@ -65,6 +65,7 @@ export class AdapterDetailDialogComponent implements OnDestroy { this.title = 'New Adapter'; this.adapter = { id: 0, + systemKey: null, name: '', description: '', language: AdapterLanguageEnum.SQL, @@ -106,6 +107,14 @@ export class AdapterDetailDialogComponent implements OnDestroy { }); } + protected get isSystemAdapter(): boolean { + return !!this.adapter?.systemKey; + } + + protected get languageLabel(): string { + return ADAPTER_LANGUAGE_LABELS[this.adapter?.language] ?? this.adapter?.language ?? ''; + } + protected onLanguageSelected(language: AdapterLanguageEnum | null): void { if (!language || this.adapter.id > 0) return; // Language is immutable for existing adapters this.adapter.language = language; @@ -155,18 +164,19 @@ export class AdapterDetailDialogComponent implements OnDestroy { } protected fileIsEntryPoint(file: FileTreeEntry): boolean { - return !file.isDirectory && file.path === CANONICAL_ENTRY_POINT[this.adapter.language]; + return !file.isDirectory && file.path === LANGUAGE_CONVENTIONS[this.adapter.language].entryPoint; } protected fileIsManifest(file: FileTreeEntry): boolean { if (file.isDirectory) return false; - return MANIFEST_FILENAMES[this.adapter.language].includes(file.path); + return file.path === LANGUAGE_CONVENTIONS[this.adapter.language].manifest; } /** * Whether the save button should be enabled. */ protected get canSave(): boolean { + if (this.isSystemAdapter) return true; if (!this.adapter.name || !this.adapter.language || !this.adapter.scriptDirName || !this.adapterPreviewResponse) return false; if (this.adapterPreviewResponse.manifestError || this.adapterPreviewResponse.entryPointError) return false; return true; @@ -199,7 +209,7 @@ export class AdapterDetailDialogComponent implements OnDestroy { name: this.adapter.name, description: this.adapter.description, language: this.adapter.language, - scriptDirName: this.adapter.scriptDirName!, + scriptDirName: this.adapter.scriptDirName, disabled: this.adapter.disabled, comment: this.adapter.comment || null, }; diff --git a/front-end/pwa/src/app/metadata/adapters/models/adapter-language-conventions.ts b/front-end/pwa/src/app/metadata/adapters/models/adapter-language-conventions.ts index d2824f3e..fcc2adb3 100644 --- a/front-end/pwa/src/app/metadata/adapters/models/adapter-language-conventions.ts +++ b/front-end/pwa/src/app/metadata/adapters/models/adapter-language-conventions.ts @@ -1,28 +1,21 @@ import { AdapterLanguageEnum } from './adapter-language.enum'; /** - * Canonical entry-point filename per language. Mirrors the backend constant - * of the same name — the backend enforces this at upload-preview time; the - * front-end reads it to show the user which file the runner will execute. + * Per-language conventions for uploaded adapter zips. Mirrors the backend + * constant of the same name — the backend enforces these at upload-preview + * time; the front-end reads them to label files in the file-tree preview + * and to display the canonical entry point in the adapter dialog. */ -export const CANONICAL_ENTRY_POINT: Record = { - [AdapterLanguageEnum.PYTHON]: 'main.py', - [AdapterLanguageEnum.R]: 'main.R', - [AdapterLanguageEnum.JAVASCRIPT]: 'index.js', - [AdapterLanguageEnum.SQL]: 'transform.sql', -}; +export interface AdapterLanguageConvention { + /** Required declaration file at the root of the zip. */ + manifest: string; + /** Canonical entry-point filename the runner executes. */ + entryPoint: string; +} -/** - * Accepted manifest filenames per language. Mirrors the backend constant. - * Used by the file tree preview to label which file in the extracted zip is - * the dependency manifest for the selected language. - * - * Multiple entries mean any one of them counts (e.g. R accepts either - * `renv.lock` or `DESCRIPTION`). - */ -export const MANIFEST_FILENAMES: Record = { - [AdapterLanguageEnum.PYTHON]: ['requirements.txt'], - [AdapterLanguageEnum.R]: ['renv.lock', 'DESCRIPTION'], - [AdapterLanguageEnum.JAVASCRIPT]: ['package.json', 'package-lock.json'], - [AdapterLanguageEnum.SQL]: ['extensions.txt'], +export const LANGUAGE_CONVENTIONS: Record = { + [AdapterLanguageEnum.PYTHON]: { manifest: 'requirements.txt', entryPoint: 'main.py' }, + [AdapterLanguageEnum.R]: { manifest: 'DESCRIPTION', entryPoint: 'main.R' }, + [AdapterLanguageEnum.JAVASCRIPT]: { manifest: 'package.json', entryPoint: 'index.js' }, + [AdapterLanguageEnum.SQL]: { manifest: 'extensions.txt', entryPoint: 'transform.sql' }, }; diff --git a/front-end/pwa/src/app/metadata/adapters/models/view-adapter-specification.model.ts b/front-end/pwa/src/app/metadata/adapters/models/view-adapter-specification.model.ts index 3f9ff4cb..147cb2bb 100644 --- a/front-end/pwa/src/app/metadata/adapters/models/view-adapter-specification.model.ts +++ b/front-end/pwa/src/app/metadata/adapters/models/view-adapter-specification.model.ts @@ -2,6 +2,7 @@ import { AdapterLanguageEnum } from './adapter-language.enum'; export interface ViewAdapterSpecificationModel { id: number; + systemKey: string | null; name: string; description: string; language: AdapterLanguageEnum; diff --git a/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.html b/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.html index ddf29beb..81f03612 100644 --- a/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.html +++ b/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.html @@ -19,18 +19,20 @@ - - - + + + + - + + @@ -46,11 +48,15 @@ 'bg-primary': adapter.language === 'sql', 'bg-success': adapter.language === 'python', 'bg-info': adapter.language === 'r', - 'bg-secondary': adapter.language === 'javascript' + 'bg-secondary': adapter.language === 'javascript' }"> {{ adapter.languageLabel }} + diff --git a/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.ts b/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.ts index ec101990..9476cb58 100644 --- a/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.ts +++ b/front-end/pwa/src/app/metadata/adapters/view-adapters/view-adapters.component.ts @@ -91,6 +91,10 @@ export class ViewAdaptersComponent { protected onDeleteClick(adapter: View, event: Event): void { event.stopPropagation(); + if (adapter.systemKey !== null) { + this.pagesDataService.showToast({ title: 'Adapter Specification', message: 'System adapters cannot be deleted', type: ToastEventTypeEnum.ERROR }); + return; + } this.selectedAdapter = adapter; this.dlgDeleteConfirm.openDialog(); } @@ -120,8 +124,8 @@ export class ViewAdaptersComponent { protected onToggleDisabledConfirm(): void { if (!this.selectedAdapter) return; const newDisabledState = !this.selectedAdapter.disabled; - // Destructure to exclude 'id' and 'languageLabel' since API does not expect them - const { id, languageLabel, ...updateDto } = this.selectedAdapter; + // Destructure to exclude 'id', 'languageLabel', and 'systemKey' since API does not expect them + const { id, languageLabel, systemKey, ...updateDto } = this.selectedAdapter; this.adaptersService.update(id, { ...updateDto, disabled: newDisabledState }).pipe( take(1) ).subscribe({ diff --git a/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.html b/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.html new file mode 100644 index 00000000..462dd1b1 --- /dev/null +++ b/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.html @@ -0,0 +1,71 @@ + + +
+ + + +
+ Name + {{ product.name }} +
+
+
+ Category + {{ product.category || '—' }} +
+
+
+ Description + {{ product.description || '—' }} +
+
+
+ Superset UUID + {{ product.supersetUuid }} +
+
+
+ Shipped product — only visibility settings can be changed. +
+ + + + +
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+ + +
+ +
+ +
+ + + +

+ Are you sure you want to delete {{ product.name }}? +

+ + diff --git a/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.scss b/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.ts b/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.ts new file mode 100644 index 00000000..e5ae9d9c --- /dev/null +++ b/front-end/pwa/src/app/metadata/climate-products/product-input-dialog/product-input-dialog.component.ts @@ -0,0 +1,111 @@ +import { Component, EventEmitter, Output, ViewChild } from '@angular/core'; +import { take } from 'rxjs'; +import { PagesDataService, ToastEventTypeEnum } from 'src/app/core/services/pages-data.service'; +import { ConfirmationDialogComponent } from 'src/app/shared/controls/confirmation-dialog/confirmation-dialog.component'; +import { CreateUpdateProductModel, ViewProductModel } from 'src/app/data-monitoring/products/models/view-product.model'; +import { ProductsService } from 'src/app/data-monitoring/products/services/products.service'; + +@Component({ + selector: 'app-product-input-dialog', + templateUrl: './product-input-dialog.component.html', + styleUrls: ['./product-input-dialog.component.scss'] +}) +export class ProductInputDialogComponent { + @ViewChild('dlgDeleteConfirm') dlgDeleteConfirm!: ConfirmationDialogComponent; + @Output() public ok = new EventEmitter(); + + protected open!: boolean; + protected title: 'Edit Product' | 'New Product' = 'New Product'; + protected product!: ViewProductModel; + + constructor( + private productsService: ProductsService, + private pagesDataService: PagesDataService, + ) { } + + public openDialog(productId?: number): void { + if (productId) { + this.title = 'Edit Product'; + this.productsService.findOne(productId).pipe(take(1)).subscribe(res => { + this.product = res; + this.open = true; + }); + } else { + this.title = 'New Product'; + this.product = { + id: 0, + systemKey: null, + supersetUuid: '', + name: '', + description: null, + category: null, + disabled: false, + }; + this.open = true; + } + } + + protected onOkClick(): void { + if (!this.product.supersetUuid) { + this.pagesDataService.showToast({ title: 'Climate Product', message: 'Superset UUID required', type: ToastEventTypeEnum.ERROR }); + return; + } + if (!this.product.name) { + this.pagesDataService.showToast({ title: 'Climate Product', message: 'Name required', type: ToastEventTypeEnum.ERROR }); + return; + } + + const dto: CreateUpdateProductModel = { + supersetUuid: this.product.supersetUuid, + name: this.product.name, + description: this.product.description || null, + category: this.product.category || null, + disabled: this.product.disabled, + }; + + if (this.title === 'New Product') { + this.productsService.create(dto).pipe(take(1)).subscribe({ + next: (data) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: `${data.name} created`, type: ToastEventTypeEnum.SUCCESS }); + this.ok.emit(); + this.open = false; + }, + error: (err) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: err.error?.message || 'Failed to save', type: ToastEventTypeEnum.ERROR }); + } + }); + } else { + this.productsService.update(this.product.id, dto).pipe(take(1)).subscribe({ + next: (data) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: `${data.name} updated`, type: ToastEventTypeEnum.SUCCESS }); + this.ok.emit(); + this.open = false; + }, + error: (err) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: err.error?.message || 'Failed to save', type: ToastEventTypeEnum.ERROR }); + } + }); + } + } + + protected onDelete(): void { + this.dlgDeleteConfirm.openDialog(); + } + + protected onDeleteConfirm(): void { + this.productsService.deleteProduct(this.product.id).pipe(take(1)).subscribe({ + next: () => { + this.pagesDataService.showToast({ title: 'Climate Product', message: 'Product deleted', type: ToastEventTypeEnum.SUCCESS }); + this.open = false; + this.ok.emit(); + }, + error: (err) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: err.error?.message || 'Failed to delete', type: ToastEventTypeEnum.ERROR }); + } + }); + } + + protected onCancelClick(): void { + this.open = false; + } +} diff --git a/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.html b/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.html new file mode 100644 index 00000000..604ea6bc --- /dev/null +++ b/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.html @@ -0,0 +1,96 @@ +
+
+
+ + +
+ +
+
+
+ +
+
+
Name LanguageType Description Actions + Shipped + Custom + - + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameCategoryDescriptionDisabledTypeActions
{{ product.name }}{{ product.category || '—' }}{{ product.description || '—' }}{{ product.disabled ? 'Yes' : 'No' }} + Shipped + Custom + +
+ + + + + +
+
+ No climate products yet. Click "Add" to register one. +
+
+
+ + + + +

Are you sure you want to delete {{ selectedProduct?.name || '' }}?

+ + + +
diff --git a/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.scss b/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.ts b/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.ts new file mode 100644 index 00000000..1a0cb814 --- /dev/null +++ b/front-end/pwa/src/app/metadata/climate-products/view-climate-products/view-climate-products.component.ts @@ -0,0 +1,132 @@ +import { Component, ViewChild } from '@angular/core'; +import { take } from 'rxjs'; +import { PagesDataService, ToastEventTypeEnum } from 'src/app/core/services/pages-data.service'; +import { PagingParameters } from 'src/app/shared/controls/page-input/paging-parameters'; +import { DeleteConfirmationDialogComponent } from 'src/app/shared/controls/delete-confirmation-dialog/delete-confirmation-dialog.component'; +import { ToggleDisabledConfirmationDialogComponent } from 'src/app/shared/controls/toggle-disabled-confirmation-dialog/toggle-disabled-confirmation-dialog.component'; +import { AppConfigService } from 'src/app/app-config.service'; +import { ViewProductModel } from 'src/app/data-monitoring/products/models/view-product.model'; +import { ProductsService } from 'src/app/data-monitoring/products/services/products.service'; +import { ProductInputDialogComponent } from '../product-input-dialog/product-input-dialog.component'; + +@Component({ + selector: 'app-view-climate-products', + templateUrl: './view-climate-products.component.html', + styleUrls: ['./view-climate-products.component.scss'] +}) +export class ViewClimateProductsComponent { + @ViewChild('dlgProductInput') dlgProductInput!: ProductInputDialogComponent; + @ViewChild('dlgDeleteConfirm') dlgDeleteConfirm!: DeleteConfirmationDialogComponent; + @ViewChild('dlgToggleDisabled') dlgToggleDisabled!: ToggleDisabledConfirmationDialogComponent; + + protected products: ViewProductModel[] = []; + protected selectedProduct: ViewProductModel | null = null; + protected pageInputDefinition: PagingParameters = new PagingParameters(); + protected sortColumn: string = ''; + protected sortDirection: 'asc' | 'desc' = 'asc'; + + constructor( + private pagesDataService: PagesDataService, + private productsService: ProductsService, + private appConfigService: AppConfigService, + ) { + this.pagesDataService.setPageHeader('Climate Products'); + this.loadProducts(); + } + + protected loadProducts(): void { + this.productsService.findAllAdmin().pipe(take(1)).subscribe(products => { + this.products = products; + this.applySort(); + this.updatePaging(); + }); + } + + protected get pageItems(): ViewProductModel[] { + const start = (this.pageInputDefinition.page - 1) * this.pageInputDefinition.pageSize; + return this.products.slice(start, start + this.pageInputDefinition.pageSize); + } + + protected onSort(column: string): void { + if (this.sortColumn === column) { + this.sortDirection = this.sortDirection === 'asc' ? 'desc' : 'asc'; + } else { + this.sortColumn = column; + this.sortDirection = 'asc'; + } + this.applySort(); + this.pageInputDefinition.onFirst(); + } + + protected onNewProductClick(): void { + this.dlgProductInput.openDialog(); + } + + protected onEditProductClick(product: ViewProductModel): void { + this.dlgProductInput.openDialog(product.id); + } + + protected onProductSaved(): void { + this.loadProducts(); + } + + protected onOpenSuperset(): void { + window.open(this.appConfigService.supersetBaseUrl, '_blank'); + } + + protected onToggleDisabledClick(product: ViewProductModel, event: Event): void { + event.stopPropagation(); + this.selectedProduct = product; + this.dlgToggleDisabled.showDialog(); + } + + protected onToggleDisabledConfirm(): void { + if (!this.selectedProduct) return; + const { id, systemKey, ...updateDto } = this.selectedProduct; + this.productsService.update(id, { ...updateDto, disabled: !updateDto.disabled }).pipe(take(1)).subscribe({ + next: () => { + const action = !updateDto.disabled ? 'disabled' : 'enabled'; + this.pagesDataService.showToast({ title: 'Climate Product', message: `Product ${action}`, type: ToastEventTypeEnum.SUCCESS }); + this.selectedProduct = null; + this.loadProducts(); + }, + error: (err) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: err.error?.message || 'Something went wrong', type: ToastEventTypeEnum.ERROR }); + } + }); + } + + protected onDeleteClick(product: ViewProductModel, event: Event): void { + event.stopPropagation(); + this.selectedProduct = product; + this.dlgDeleteConfirm.openDialog(); + } + + protected onDeleteConfirm(): void { + if (!this.selectedProduct) return; + this.productsService.deleteProduct(this.selectedProduct.id).pipe(take(1)).subscribe({ + next: () => { + this.pagesDataService.showToast({ title: 'Climate Product', message: 'Product deleted', type: ToastEventTypeEnum.SUCCESS }); + this.selectedProduct = null; + this.loadProducts(); + }, + error: (err) => { + this.pagesDataService.showToast({ title: 'Climate Product', message: err.error?.message || 'Failed to delete', type: ToastEventTypeEnum.ERROR }); + } + }); + } + + private applySort(): void { + if (!this.sortColumn) return; + const dir = this.sortDirection === 'asc' ? 1 : -1; + this.products.sort((a, b) => + String((a as any)[this.sortColumn] ?? '').localeCompare(String((b as any)[this.sortColumn] ?? '')) * dir + ); + } + + private updatePaging(): void { + this.pageInputDefinition = new PagingParameters(); + this.pageInputDefinition.setPageSize(365); + this.pageInputDefinition.setTotalRowCount(this.products.length); + } +} diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.html b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.html index 64c11b30..f2c62a65 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.html +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.html @@ -67,7 +67,7 @@
Server Parameters
-
diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.ts b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.ts index 131bca9f..67139260 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.ts +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/connector-specification-input-dialog.component.ts @@ -14,6 +14,7 @@ import { ImportFileServerParametersModel, ObservationWindowDateFieldEnum, } from '../models/create-connector-specification.model'; +import { FileServerParametersInputComponent } from './file-server-params/file-server-parameters-input.component'; @Component({ selector: 'app-connector-specification-input-dialog', @@ -21,7 +22,8 @@ import { styleUrls: ['./connector-specification-input-dialog.component.scss'] }) export class ConnectorSpecificationInputDialogComponent { - @ViewChild('dlgDeleteConfirm') dlgDeleteConfirm!: DeleteConfirmationDialogComponent; + @ViewChild('dlgDeleteConfirm') private dlgDeleteConfirm!: DeleteConfirmationDialogComponent; + @ViewChild('fileServerParams ') private fileServerParams?: FileServerParametersInputComponent; @Output() public ok = new EventEmitter(); @@ -78,6 +80,15 @@ export class ConnectorSpecificationInputDialogComponent { return; } + // Late-binding validation from the parameters sub-tree — currently + // covers the "directory-shaped filePattern without recursive" case + // in the import file-server params. + const paramsError = this.fileServerParams?.validate() ?? null; + if (paramsError) { + this.showValidationError(paramsError); + return; + } + if (this.parametersErrorMessage) { this.showValidationError(this.parametersErrorMessage); return; diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.html b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.html index 15c90e98..371b9757 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.html +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.html @@ -39,7 +39,7 @@
- +
diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.ts b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.ts index 65e60d40..375cf036 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.ts +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/file-server-parameters-input.component.ts @@ -1,6 +1,7 @@ -import { Component, EventEmitter, Input, OnChanges, Output, SimpleChanges } from '@angular/core'; +import { Component, EventEmitter, Input, OnChanges, Output, SimpleChanges, ViewChild } from '@angular/core'; import { ConnectorTypeEnum, ExportFileServerParametersModel, FileServerParametersModel, FileServerProtocolEnum, ImportFileServerParametersModel } from '../../models/create-connector-specification.model'; import { ViewConnectorSpecificationModel } from '../../models/view-connector-specification.model'; +import { ImportFileServerParamsComponent } from './import-file-server-params/import-file-server-params.component'; /** * Component for managing FTP/SFTP/FTPS connector parameters input. @@ -27,6 +28,9 @@ export class FileServerParametersInputComponent implements OnChanges { @Output() public validationError = new EventEmitter(); + @ViewChild('importFileServerParams') + private importParams?: ImportFileServerParamsComponent; + protected ConnectorTypeEnum = ConnectorTypeEnum; protected newPassword: string = ''; protected confirmPassword: string = ''; @@ -76,6 +80,15 @@ export class FileServerParametersInputComponent implements OnChanges { this.validationError.emit(this.passwordErrormessage); } + /** + * Called by the parent dialog at submit time. Delegates to the import + * file-pattern check on the child component. Returns null when there is + * nothing to validate (export connector, or import with no issues). + */ + public validate(): string | null { + return this.importParams?.validate() ?? null; + } + protected onFileProtocolSelection(protocol: FileServerProtocolEnum): void { this.connector.parameters.protocol = protocol; diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.html b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.html index 24e5a664..a2c7771c 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.html +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.html @@ -2,6 +2,9 @@
+ + Turn on when Climsoft is required to also process files in subdirectories. +
@@ -23,6 +26,12 @@
+ + File Source Pattern examples: *.csv (root files), + stationA/ (all files in directory), + stationA/*.csv (glob inside directory). + Subdirectory patterns require Recursive to be on. + diff --git a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.ts b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.ts index 6a6b100b..bb3abb05 100644 --- a/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.ts +++ b/front-end/pwa/src/app/metadata/connector-specifications/connector-specification-input-dialog/file-server-params/import-file-server-params/import-file-server-params.component.ts @@ -41,4 +41,22 @@ export class ImportFileServerParamsComponent { protected onSpecificationIdChange(index: number, specificationId: number | null): void { this.importFileServerParameters.specifications[index].specificationId = specificationId ?? 0; } + + /** + * Called by the parent dialog at submit time. Returns an error message + * if any spec's `filePattern` targets a subdirectory (contains "/") + * while `recursive` is off — that combination silently matches nothing + * at runtime, so we catch it here at authoring time. + */ + public validate(): string | null { + const specs = this.importFileServerParameters?.specifications; + if (!specs || specs.length === 0) return null; + if (this.importFileServerParameters.recursive) return null; + + const offending = specs.find(s => s.filePattern && s.filePattern.includes('/')); + if (offending) { + return `Pattern "${offending.filePattern}" targets a subdirectory but "Recursive" is off. Enable Recursive or drop the "/" from the pattern.`; + } + return null; + } } diff --git a/front-end/pwa/src/app/metadata/metadata-routing.module.ts b/front-end/pwa/src/app/metadata/metadata-routing.module.ts index a2fb0ecd..a6bd1910 100644 --- a/front-end/pwa/src/app/metadata/metadata-routing.module.ts +++ b/front-end/pwa/src/app/metadata/metadata-routing.module.ts @@ -11,6 +11,7 @@ import { ViewNetworkAffiliationsComponent } from './network-affiliations/view-ne import { ViewQCSpecificationsComponent } from './qc-tests/view-qc-specifications/view-qc-specifications.component'; import { ViewFlagsComponent } from './flags/view-flags/view-flags.component'; import { ViewAdaptersComponent } from './adapters/view-adapters/view-adapters.component'; +import { ViewClimateProductsComponent } from './climate-products/view-climate-products/view-climate-products.component'; const routes: Routes = [ { path: '', @@ -67,6 +68,10 @@ const routes: Routes = [ path: 'view-adapters', component: ViewAdaptersComponent }, + { + path: 'view-climate-products', + component: ViewClimateProductsComponent + }, ] } diff --git a/front-end/pwa/src/app/metadata/metadata.module.ts b/front-end/pwa/src/app/metadata/metadata.module.ts index 7611c8ed..feecb1a2 100644 --- a/front-end/pwa/src/app/metadata/metadata.module.ts +++ b/front-end/pwa/src/app/metadata/metadata.module.ts @@ -103,6 +103,8 @@ import { AdapterDetailDialogComponent } from './adapters/adapter-detail-dialog/a import { AdapterTestRunPaneComponent } from './adapters/adapter-detail-dialog/adapter-test-run-pane/adapter-test-run-pane.component'; import { AdapterLanguageSelectorSingleComponent } from './adapters/adapter-detail-dialog/adapter-language-selector-single/adapter-language-selector-single.component'; import { AdapterSpecificationSelectorSingleComponent } from './adapters/adapter-specification-selector-single/adapter-specification-selector-single.component'; +import { ViewClimateProductsComponent } from './climate-products/view-climate-products/view-climate-products.component'; +import { ProductInputDialogComponent } from './climate-products/product-input-dialog/product-input-dialog.component'; @NgModule({ declarations: [ ViewNetworkAffiliationsComponent, @@ -235,6 +237,9 @@ import { AdapterSpecificationSelectorSingleComponent } from './adapters/adapter- AdapterTestRunPaneComponent, AdapterSpecificationSelectorSingleComponent, + ViewClimateProductsComponent, + ProductInputDialogComponent, + ], imports: [ MetadataRoutingModule, diff --git a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.html b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.html index 00f2c246..415cb735 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.html +++ b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.html @@ -1,22 +1,56 @@
- -
- - -
- - -
- +
Flags:
+ + + +
+ + +
+ + +
+ + +
+ +
+ + +
+

+ Cells like 0.5T are split into value 0.5 and flag T + (trailing letters become the flag). Values without a trailing letter carry no flag. +

+ +
+ + +
+ +
+
+
+
diff --git a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.ts b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.ts index b484f488..5d4d93e8 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.ts +++ b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-flag-detail/import-source-flag-detail.component.ts @@ -1,32 +1,119 @@ -import { Component, EventEmitter, Input, Output } from '@angular/core'; -import { FlagDefinition } from 'src/app/metadata/source-specifications/models/import-source-tabular-params.model'; +import { Component, EventEmitter, Input, OnChanges, Output } from '@angular/core'; +import { FlagDefinition, InlineFlagRule } from 'src/app/metadata/source-specifications/models/import-source-tabular-params.model'; import { CachedMetadataService } from 'src/app/metadata/metadata-updates/cached-metadata.service'; import { IdMapping } from '../../id-mapping-table/id-mapping-table.component'; -// Inclusion-control rule: this step uses a checkbox because the "off" state means -// literally nothing (no flag column). Steps whose "off" state still carries -// sub-config (default value or default id) use a radio instead. +/** + * Three-way flag configuration for a tabular import spec. + * + * - `NONE` — the file has no flag information. + * - `SEPARATE` — a dedicated flag column, position + optional source→db mapping. hidden when the parent is in a wide-pivot mode (no explicit value column exists to attach a separate flag column to) + * - `INLINE` — cells in the value column carry a trailing letter as the flag + * (e.g. `0.5T` → value 0.5, flag T). Works for both column-based + * value sources and wide-pivot sources. + */ +export enum FlagMode { + NONE = 'NONE', + SEPARATE = 'SEPARATE', + INLINE = 'INLINE', +} + @Component({ selector: 'app-import-source-flag-detail', templateUrl: './import-source-flag-detail.component.html', styleUrls: ['./import-source-flag-detail.component.scss'] }) -export class ImportSourceFlagDetailComponent { +export class ImportSourceFlagDetailComponent implements OnChanges { @Input() public flagDefinition: FlagDefinition | undefined; @Output() public flagDefinitionChange = new EventEmitter(); - constructor(private cachedMetadataService: CachedMetadataService) { } + @Input() public inlineFlagRule: InlineFlagRule | undefined; + @Output() public inlineFlagRuleChange = new EventEmitter(); + + /** True unless the parent has committed to a wide-pivot value source. */ + @Input() public canUseSeparateColumn: boolean = true; + + /** + * Local, authoritative mode for rendering. Seeded from the @Input values in + * ngOnChanges so it stays in sync when the parent updates the model, but + * updated eagerly inside `onModeSelection` so the *ngIf blocks flip on the + * same tick as the click — no dependency on the emit → parent → change- + * detection round trip. + */ + protected mode: FlagMode = FlagMode.NONE; + + /** Exposed to the template so it can reference enum members. */ + protected readonly FlagMode = FlagMode; + + protected modeButtons: { label: FlagMode; checked: boolean }[] = []; + + constructor(private cachedMetadataService: CachedMetadataService) { + this.rebuildModeButtons(); + } + + ngOnChanges(): void { + // Reseed local mode from the (possibly-updated) inputs, unless we already + // have a matching mode set from a click. This handles two cases: + // 1. Dialog opened with an existing spec — inputs arrive after construction. + // 2. Parent clears state (e.g. wizard restart) — we mirror the reset. + const derived: FlagMode = this.deriveModeFromInputs(); + if (derived !== this.mode) { + this.mode = derived; + } + this.rebuildModeButtons(); + } protected get defaultFlagId(): number { const missingFlag = this.cachedMetadataService.getMissingFlag(); return missingFlag ? missingFlag.id : 1; } - protected onIncludesFlag(include: boolean): void { - this.flagDefinition = include ? { flagColumnPosition: 0, flagsToFetch: undefined } : undefined; + protected onModeSelection(mode: FlagMode): void { + this.mode = mode; + + // Reset both slots; only the chosen one gets populated. The parent's + // change handlers keep the model in sync via the emits below. + this.flagDefinition = undefined; + this.inlineFlagRule = undefined; + + if (mode === FlagMode.SEPARATE) { + this.flagDefinition = { flagColumnPosition: 0, flagsToFetch: undefined }; + } else if (mode === FlagMode.INLINE) { + this.inlineFlagRule = { flagsToFetch: undefined }; + } + + this.rebuildModeButtons(); this.flagDefinitionChange.emit(this.flagDefinition); + this.inlineFlagRuleChange.emit(this.inlineFlagRule); } + protected readonly modeLabel = (mode: FlagMode): string => { + switch (mode) { + case FlagMode.NONE: return 'No Flag'; + case FlagMode.SEPARATE: return 'Separate Column'; + case FlagMode.INLINE: return 'Inline With Value (e.g. 0.5T)'; + } + }; + + private deriveModeFromInputs(): FlagMode { + if (this.flagDefinition) return FlagMode.SEPARATE; + if (this.inlineFlagRule) return FlagMode.INLINE; + return FlagMode.NONE; + } + + private rebuildModeButtons(): void { + const buttons: { label: FlagMode; checked: boolean }[] = [ + { label: FlagMode.NONE, checked: this.mode === FlagMode.NONE }, + ]; + if (this.canUseSeparateColumn) { + buttons.push({ label: FlagMode.SEPARATE, checked: this.mode === FlagMode.SEPARATE }); + } + buttons.push({ label: FlagMode.INLINE, checked: this.mode === FlagMode.INLINE }); + this.modeButtons = buttons; + } + + // ─── Separate-column mode ────────────────────────────────────────────── + protected onFetchFlagsChange(fetch: boolean): void { if (!this.flagDefinition) return; this.flagDefinition.flagsToFetch = fetch ? [] : undefined; @@ -35,11 +122,27 @@ export class ImportSourceFlagDetailComponent { protected onMappingsChange(mappings: IdMapping[]): void { if (!this.flagDefinition) return; - // Flag database IDs are numbers. this.flagDefinition.flagsToFetch = mappings.map(m => ({ sourceId: m.sourceId, databaseId: Number(m.databaseId), })); this.flagDefinitionChange.emit(this.flagDefinition); } + + // ─── Inline-flag mode ────────────────────────────────────────────────── + + protected onInlineFetchFlagsChange(fetch: boolean): void { + if (!this.inlineFlagRule) return; + this.inlineFlagRule.flagsToFetch = fetch ? [] : undefined; + this.inlineFlagRuleChange.emit(this.inlineFlagRule); + } + + protected onInlineMappingsChange(mappings: IdMapping[]): void { + if (!this.inlineFlagRule) return; + this.inlineFlagRule.flagsToFetch = mappings.map(m => ({ + sourceId: m.sourceId, + databaseId: Number(m.databaseId), + })); + this.inlineFlagRuleChange.emit(this.inlineFlagRule); + } } diff --git a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.html b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.html index e9b1e192..a64fc1c4 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.html +++ b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.html @@ -136,6 +136,15 @@
+
+ +
+
diff --git a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.ts b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.ts index 04c4f265..0290ed53 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.ts +++ b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-input-dialog.component.ts @@ -1,5 +1,5 @@ import { Component, EventEmitter, OnDestroy, Output, ViewChild } from '@angular/core'; -import { DateTimeFormat, ImportSourceTabularParamsModel } from '../models/import-source-tabular-params.model'; +import { DateTimeFormat, FlagDefinition, ImportSourceTabularParamsModel, InlineFlagRule } from '../models/import-source-tabular-params.model'; import { PagesDataService, ToastEventTypeEnum } from 'src/app/core/services/pages-data.service'; import { SourceTypeEnum } from 'src/app/metadata/source-specifications/models/source-type.enum'; import { Observable, switchMap, take } from 'rxjs'; @@ -12,6 +12,7 @@ import { RawPreviewResponse, TransformedPreviewResponse } from '../models/import import { StringUtils } from 'src/app/shared/utils/string.utils'; import { ViewAdapterSpecificationModel } from 'src/app/metadata/adapters/models/view-adapter-specification.model'; import { ConfirmationDialogComponent } from 'src/app/shared/controls/confirmation-dialog/confirmation-dialog.component'; +import { AdaptersService } from '../../adapters/services/adapters.service'; type WizardStep = 'upload' | 'station' | 'element' | 'level' | 'datetime' | 'interval' | 'value' | 'review'; @@ -59,6 +60,7 @@ export class ImportSourceInputDialogComponent implements OnDestroy { private pagesDataService: PagesDataService, private sourcesCacheService: SourcesCacheService, private importPreviewService: ImportPreviewService, + private adaptersService: AdaptersService, ) { // Reset all state this.resetSamplePreview(); @@ -77,10 +79,16 @@ export class ImportSourceInputDialogComponent implements OnDestroy { if (source) { this.title = 'Edit Import Specification'; this.viewSource = structuredClone(source); - console.log('Loaded source specification for editing:', this.viewSource.parameters); this.initPreviewFromSavedFile(); // Mark all steps as visited for existing specifications this.wizardSteps.forEach(s => this.visitedSteps.add(s)); + + if (this.viewSource.adapterId) { + this.adaptersService.findOne(this.viewSource.adapterId).pipe( + take(1) + ).subscribe(res => { this.selectedAdapter = res; }); + } + } else { this.title = 'New Import Specification'; @@ -439,6 +447,22 @@ export class ImportSourceInputDialogComponent implements OnDestroy { } } + /** + * Handles flag-mode transitions from the flag-detail component. The + * `flagDefinition` slot lives nested inside `valueDefinition`, so it can + * only be set when we're not in a wide-pivot mode (the flag-detail + * component already hides 'Separate Column' in that case). + */ + protected onFlagDefinitionChange(flagDef: FlagDefinition | undefined): void { + if (this.tabularImportParams.valueDefinition) { + this.tabularImportParams.valueDefinition.flagDefinition = flagDef; + } + } + + protected onInlineFlagRuleChange(rule: InlineFlagRule | undefined): void { + this.tabularImportParams.inlineFlagRule = rule; + } + protected get showNavigation(): boolean { return !!this.rawPreviewResponse.sessionId || (this.viewSource?.id > 0); } @@ -474,7 +498,7 @@ export class ImportSourceInputDialogComponent implements OnDestroy { sampleFileOperationId: this.rawPreviewResponse.sessionId, utcOffset: this.viewSource.utcOffset, parameters: this.viewSource.parameters, - adapterId: this.selectedAdapter?.id || null, + adapterId: this.viewSource.adapterId || null, disabled: this.viewSource.disabled, comment: this.viewSource.comment || null, }; diff --git a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-value-detail/import-source-value-detail.component.html b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-value-detail/import-source-value-detail.component.html index e4d010e2..64860e64 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-value-detail/import-source-value-detail.component.html +++ b/front-end/pwa/src/app/metadata/source-specifications/import-source-input-dialog/import-source-value-detail/import-source-value-detail.component.html @@ -2,9 +2,4 @@
- -
- -
- -
\ No newline at end of file +
diff --git a/front-end/pwa/src/app/metadata/source-specifications/models/import-source-tabular-params.model.ts b/front-end/pwa/src/app/metadata/source-specifications/models/import-source-tabular-params.model.ts index 60112b08..dc2f1ce6 100644 --- a/front-end/pwa/src/app/metadata/source-specifications/models/import-source-tabular-params.model.ts +++ b/front-end/pwa/src/app/metadata/source-specifications/models/import-source-tabular-params.model.ts @@ -3,6 +3,22 @@ export interface FlagDefinition { flagsToFetch?: { sourceId: string, databaseId: number }[]; } +/** + * When set on {@link ImportSourceTabularParamsModel.inlineFlagRule}, cells in + * the observation `value` column carry a trailing alphabetic run interpreted + * as a flag abbreviation. Example: `0.5T` splits into value `0.5` and flag `T`. + * + * Applies uniformly whether the `value` column comes from an explicit + * `valueDefinition.valueColumnPosition` or from a wide-pivot UNPIVOT. + * Mutually exclusive with `valueDefinition.flagDefinition`. + */ +export interface InlineFlagRule { + /** Optional source flag string → database flag id mapping. When omitted, + * the extracted flag string is matched case-insensitively against + * `flags.abbreviation`. */ + flagsToFetch?: { sourceId: string, databaseId: number }[]; +} + export interface ValueDefinition { /** Value column position. */ @@ -179,8 +195,11 @@ export enum DateTimeFormat { // 12-hour clock with AM/PM (Excel and US-locale form entries) YMD_DASH_HM_AMPM = '%Y-%m-%d %I:%M %p', + YMD_DASH_HMS_AMPM = '%Y-%m-%d %I:%M:%S %p', // 2023-12-13 10:30:00 AM DMY_SLASH_HM_AMPM = '%d/%m/%Y %I:%M %p', + DMY_SLASH_HMS_AMPM = '%d/%m/%Y %I:%M:%S %p', // 13/12/2023 10:30:00 AM MDY_SLASH_HM_AMPM = '%m/%d/%Y %I:%M %p', + MDY_SLASH_HMS_AMPM = '%m/%d/%Y %I:%M:%S %p', // 12/13/2023 10:30:00 AM // Dot-separated (German / Russian / Eastern European locales) DMY_DOT_HMS = '%d.%m.%Y %H:%M:%S', @@ -349,6 +368,12 @@ export interface ImportSourceTabularParamsModel { valueDefinition?: ValueDefinition; + /** + * Opt-in: cells in the `value` column carry a trailing flag suffix. + * Mutually exclusive with `valueDefinition.flagDefinition`. + */ + inlineFlagRule?: InlineFlagRule; + commentDefinition?: CommentDefinition; /** diff --git a/front-end/pwa/src/app/shared/controls/datetime-format-selectors/datetime-format-selector-single/datetime-format-selector-single.component.ts b/front-end/pwa/src/app/shared/controls/datetime-format-selectors/datetime-format-selector-single/datetime-format-selector-single.component.ts index e49b7ccd..827d83a2 100644 --- a/front-end/pwa/src/app/shared/controls/datetime-format-selectors/datetime-format-selector-single/datetime-format-selector-single.component.ts +++ b/front-end/pwa/src/app/shared/controls/datetime-format-selectors/datetime-format-selector-single/datetime-format-selector-single.component.ts @@ -54,9 +54,12 @@ export class DatetimeFormatSelectorSingleComponent implements OnChanges { [DateTimeFormat.MDY_SLASH_HM]: '01/15/2024 14:30 (%m/%d/%Y %H:%M)', // 12-hour AM/PM - [DateTimeFormat.YMD_DASH_HM_AMPM]: '2024-01-15 02:30 PM (%Y-%m-%d %I:%M %p)', - [DateTimeFormat.DMY_SLASH_HM_AMPM]: '15/01/2024 02:30 PM (%d/%m/%Y %I:%M %p)', - [DateTimeFormat.MDY_SLASH_HM_AMPM]: '01/15/2024 02:30 PM (%m/%d/%Y %I:%M %p)', + [DateTimeFormat.YMD_DASH_HM_AMPM]: '2024-01-15 02:30 AM/PM (%Y-%m-%d %I:%M %p)', + [DateTimeFormat.YMD_DASH_HMS_AMPM]: '2024-01-15 02:30:00 AM/PM (%Y-%m-%d %I:%M:%S %p)', + [DateTimeFormat.DMY_SLASH_HM_AMPM]: '15/01/2024 02:30 AM/PM (%d/%m/%Y %I:%M %p)', + [DateTimeFormat.DMY_SLASH_HMS_AMPM]: '15/01/2024 02:30:00 AM/PM (%d/%m/%Y %I:%M:%S %p)', + [DateTimeFormat.MDY_SLASH_HM_AMPM]: '01/15/2024 02:30 AM/PM (%m/%d/%Y %I:%M %p)', + [DateTimeFormat.MDY_SLASH_HMS_AMPM]: '01/15/2024 02:30:00 AM/PM (%m/%d/%Y %I:%M:%S %p)', // Dot-separated (European) [DateTimeFormat.DMY_DOT_HMS]: '15.01.2024 14:30:00 (%d.%m.%Y %H:%M:%S)', diff --git a/front-end/pwa/src/app/shared/controls/datetime-format-selectors/time-format-selector-single/time-format-selector-single.component.ts b/front-end/pwa/src/app/shared/controls/datetime-format-selectors/time-format-selector-single/time-format-selector-single.component.ts index c2bd5545..e6507153 100644 --- a/front-end/pwa/src/app/shared/controls/datetime-format-selectors/time-format-selector-single/time-format-selector-single.component.ts +++ b/front-end/pwa/src/app/shared/controls/datetime-format-selectors/time-format-selector-single/time-format-selector-single.component.ts @@ -28,14 +28,14 @@ export class TimeFormatSelectorSingleComponent implements OnChanges { } private readonly displayLabels: Record = { - [TimeFormat.HMS]: '14:30:00 (%H:%M:%S)', - [TimeFormat.HM]: '14:30 (%H:%M)', + [TimeFormat.HMS]: '09:30:00 (%H:%M:%S)', + [TimeFormat.HM]: '09:30 (%H:%M)', [TimeFormat.HM_UNPADDED]: '9:30 (%-H:%M, no padding)', - [TimeFormat.H]: '14 (%H, zero-padded hour)', + [TimeFormat.H]: '09 (%H, zero-padded hour)', [TimeFormat.H_UNPADDED]: '9 (%-H, unpadded hour)', - [TimeFormat.HMS_FRAC]: '14:30:00.123456 (%H:%M:%S.%f)', - [TimeFormat.HMS_COMPACT]: '143000 (%H%M%S)', - [TimeFormat.HM_COMPACT]: '1430 (%H%M)', + [TimeFormat.HMS_FRAC]: '09:30:00.123456 (%H:%M:%S.%f)', + [TimeFormat.HMS_COMPACT]: '093000 (%H%M%S)', + [TimeFormat.HM_COMPACT]: '0930 (%H%M)', [TimeFormat.HMS_AMPM]: '02:30:00 PM (%I:%M:%S %p)', [TimeFormat.HM_AMPM]: '02:30 PM (%I:%M %p)', }; diff --git a/front-end/pwa/src/app/shared/controls/interval-selector/Intervals.util.ts b/front-end/pwa/src/app/shared/controls/interval-selector/Intervals.util.ts index 2dec3df9..7a71fa7d 100644 --- a/front-end/pwa/src/app/shared/controls/interval-selector/Intervals.util.ts +++ b/front-end/pwa/src/app/shared/controls/interval-selector/Intervals.util.ts @@ -3,7 +3,15 @@ export class IntervalsUtil { // these should be changed to enums, the the api will translate the enums to their correct minute interval. // When querying the database, the API should be able to know that months vary from 28 to 31 days and yearly vary from 365 to 366 public static possibleIntervals: Interval[] = [ + { id: 1, name: "1 min" }, + { id: 2, name: "2 mins" }, + { id: 3, name: "3 mins" }, + { id: 4, name: "4 mins" }, { id: 5, name: "5 mins" }, + { id: 6, name: "6 mins" }, + { id: 7, name: "7 mins" }, + { id: 8, name: "8 mins" }, + { id: 9, name: "9 mins" }, { id: 10, name: "10 mins" }, { id: 15, name: "15 mins" }, { id: 30, name: "30 mins" }, @@ -15,8 +23,8 @@ export class IntervalsUtil { { id: 2880, name: "2 Days" }, { id: 1080, name: "Weekly" }, { id: 14400, name: "Dekadal" }, - { id: 44640, name: "Monthly" }, // TODO. Abandon use of minutes at fron end level - { id: 527040, name: "Yearly" }, // TODO. Abandon use of minutes at fron end level + { id: 44640, name: "Monthly" }, // TODO. Factor leap years and non-leap years. + { id: 527040, name: "Yearly" }, // TODO. Factor leap years and non-leap years. ]; public static findInterval(minutes: number): Interval | undefined { diff --git a/nginx.conf b/nginx.conf index 65ec233a..37bf1ffd 100644 --- a/nginx.conf +++ b/nginx.conf @@ -31,6 +31,24 @@ http { error_page 404 = @angular_fallback; } + # Proxy requests to Apache Superset (Climate Products — direct access and embedded iframe) + # Only active when the superset profile is started. Returns 502 when Superset is not running. + location /superset/ { + proxy_pass http://climsoft_superset:8088/superset/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support for Superset live updates + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_connect_timeout 30s; + proxy_read_timeout 300s; + } + # Proxy API Requests to NestJS location /api/ { proxy_pass http://climsoft_api:3000/; diff --git a/superset/Dockerfile b/superset/Dockerfile new file mode 100644 index 00000000..f8cf2990 --- /dev/null +++ b/superset/Dockerfile @@ -0,0 +1,32 @@ +FROM apache/superset:6.1.0 + +USER root + +ENV PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright-browsers + +RUN . /app/.venv/bin/activate && \ + uv pip install \ + psycopg2-binary \ + gevent \ + Authlib \ + openpyxl \ + Pillow \ + playwright \ + && playwright install-deps \ + && PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright-browsers playwright install chromium + +# Bake in config and init scripts so the image works without a repo clone. +# In dev, volume mounts in docker-compose shadow these files for live editing. +COPY --chown=superset:superset superset_config.py /app/pythonpath/superset_config.py +COPY --chown=superset:superset init.sh /app/init.sh +COPY --chown=superset:superset worker.sh /app/worker.sh +COPY --chown=superset:superset beat.sh /app/beat.sh +RUN chmod +x /app/init.sh /app/worker.sh /app/beat.sh +COPY --chown=superset:superset ensure_superset_db.py /app/ensure_superset_db.py +COPY --chown=superset:superset init_datasets.py /app/init_datasets.py +COPY --chown=superset:superset datasets/ /app/datasets/ +COPY --chown=superset:superset climsoft_logo.png /app/superset/static/assets/images/climsoft_logo.png +# TODO (Phase 6): Replace with COPY --chown=superset:superset products/ /app/products/ once shipped dashboard ZIPs are committed +RUN mkdir -p /app/products && chown superset:superset /app/products + +USER superset diff --git a/superset/beat.sh b/superset/beat.sh new file mode 100644 index 00000000..0e1e45ef --- /dev/null +++ b/superset/beat.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +exec celery --app=superset.tasks.celery_app:app beat \ + --pidfile /tmp/celerybeat.pid \ + --schedule /tmp/celerybeat-schedule diff --git a/superset/climsoft_logo.png b/superset/climsoft_logo.png new file mode 100644 index 00000000..330cc3d0 Binary files /dev/null and b/superset/climsoft_logo.png differ diff --git a/superset/datasets/climate_extremes.sql b/superset/datasets/climate_extremes.sql new file mode 100644 index 00000000..bc659a9f --- /dev/null +++ b/superset/datasets/climate_extremes.sql @@ -0,0 +1,39 @@ +SELECT + o.station_id AS "Station ID", + s.name AS "Station Name", + o.element_id AS "Element ID", + e.name AS "Element", + e.abbreviation AS "Element Abbreviation", + e.units AS "Units", + EXTRACT(YEAR FROM o.date_time)::int AS "Year", + MAX( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Annual Maximum", + MIN( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Annual Minimum", + AVG( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Annual Mean", + PERCENTILE_CONT(0.10) WITHIN GROUP (ORDER BY + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "10th Percentile", + PERCENTILE_CONT(0.90) WITHIN GROUP (ORDER BY + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "90th Percentile", + COUNT(o.value) AS "Observation Count" +FROM observations o +JOIN stations s ON s.id = o.station_id +JOIN elements e ON e.id = o.element_id +WHERE o.deleted = FALSE + AND o.value IS NOT NULL + AND o.qc_status != 'failed' +GROUP BY + o.station_id, s.name, + o.element_id, e.name, e.abbreviation, e.units, + EXTRACT(YEAR FROM o.date_time) diff --git a/superset/datasets/data_availability.sql b/superset/datasets/data_availability.sql new file mode 100644 index 00000000..0bd9c5ec --- /dev/null +++ b/superset/datasets/data_availability.sql @@ -0,0 +1,30 @@ +SELECT + o.station_id AS "Station ID", + s.name AS "Station Name", + o.element_id AS "Element ID", + e.name AS "Element", + e.abbreviation AS "Element Abbreviation", + o.interval AS "Interval (min)", + DATE_TRUNC('month', o.date_time) AS "Month", + EXTRACT(YEAR FROM o.date_time)::int AS "Year", + EXTRACT(MONTH FROM o.date_time)::int AS "Month Number", + COUNT(*) AS "Total Slots", + COUNT(o.value) AS "Values Present", + COUNT(*) - COUNT(o.value) AS "Missing Values", + ROUND( + 100.0 * COUNT(o.value) / NULLIF(COUNT(*), 0), 1 + ) AS "Completeness (%)", + COUNT(*) FILTER (WHERE o.qc_status = 'passed') AS "QC Passed", + COUNT(*) FILTER (WHERE o.qc_status = 'failed') AS "QC Failed", + COUNT(*) FILTER (WHERE o.qc_status = 'none') AS "Not QC Checked" +FROM observations o +JOIN stations s ON s.id = o.station_id +JOIN elements e ON e.id = o.element_id +WHERE o.deleted = FALSE +GROUP BY + o.station_id, s.name, + o.element_id, e.name, e.abbreviation, + o.interval, + DATE_TRUNC('month', o.date_time), + EXTRACT(YEAR FROM o.date_time), + EXTRACT(MONTH FROM o.date_time) diff --git a/superset/datasets/observations_daily.sql b/superset/datasets/observations_daily.sql new file mode 100644 index 00000000..41315f6d --- /dev/null +++ b/superset/datasets/observations_daily.sql @@ -0,0 +1,35 @@ +SELECT + o.station_id AS "Station ID", + s.name AS "Station Name", + o.element_id AS "Element ID", + e.name AS "Element", + e.abbreviation AS "Element Abbreviation", + e.units AS "Units", + DATE_TRUNC('day', o.date_time) AS "Date", + AVG( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Daily Mean", + MAX( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Daily Maximum", + MIN( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Daily Minimum", + SUM( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Daily Total", + COUNT(o.value) AS "Observation Count", + COUNT(*) FILTER (WHERE o.qc_status = 'failed') AS "QC Failed Count" +FROM observations o +JOIN stations s ON s.id = o.station_id +JOIN elements e ON e.id = o.element_id +WHERE o.deleted = FALSE + AND o.value IS NOT NULL +GROUP BY + o.station_id, s.name, + o.element_id, e.name, e.abbreviation, e.units, + DATE_TRUNC('day', o.date_time) diff --git a/superset/datasets/observations_monthly.sql b/superset/datasets/observations_monthly.sql new file mode 100644 index 00000000..0f3f4733 --- /dev/null +++ b/superset/datasets/observations_monthly.sql @@ -0,0 +1,39 @@ +SELECT + o.station_id AS "Station ID", + s.name AS "Station Name", + o.element_id AS "Element ID", + e.name AS "Element", + e.abbreviation AS "Element Abbreviation", + e.units AS "Units", + DATE_TRUNC('month', o.date_time) AS "Month", + EXTRACT(YEAR FROM o.date_time)::int AS "Year", + EXTRACT(MONTH FROM o.date_time)::int AS "Month Number", + AVG( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Monthly Mean", + MAX( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Monthly Maximum", + MIN( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Monthly Minimum", + SUM( + CASE WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision ELSE o.value END + ) AS "Monthly Total", + COUNT(o.value) AS "Observation Count", + COUNT(*) FILTER (WHERE o.qc_status = 'failed') AS "QC Failed Count" +FROM observations o +JOIN stations s ON s.id = o.station_id +JOIN elements e ON e.id = o.element_id +WHERE o.deleted = FALSE + AND o.value IS NOT NULL +GROUP BY + o.station_id, s.name, + o.element_id, e.name, e.abbreviation, e.units, + DATE_TRUNC('month', o.date_time), + EXTRACT(YEAR FROM o.date_time), + EXTRACT(MONTH FROM o.date_time) diff --git a/superset/datasets/observations_raw.sql b/superset/datasets/observations_raw.sql new file mode 100644 index 00000000..b6bebd00 --- /dev/null +++ b/superset/datasets/observations_raw.sql @@ -0,0 +1,26 @@ +SELECT + o.station_id AS "Station ID", + s.name AS "Station Name", + o.element_id AS "Element ID", + e.name AS "Element", + e.abbreviation AS "Element Abbreviation", + e.units AS "Units", + o.date_time AS "Date Time", + o.interval AS "Interval (min)", + o.level AS "Level", + o.value AS "Raw Value", + CASE + WHEN e.entry_scale_factor IS NOT NULL AND e.entry_scale_factor != 0 + THEN o.value / e.entry_scale_factor::double precision + ELSE o.value + END AS "Value", + o.qc_status AS "QC Status", + f.name AS "QC Flag", + src.name AS "Source", + o.comment AS "Comment" +FROM observations o +JOIN stations s ON s.id = o.station_id +JOIN elements e ON e.id = o.element_id +LEFT JOIN flags f ON f.id = o.flag_id +LEFT JOIN source_templates src ON src.id = o.source_id +WHERE o.deleted = FALSE diff --git a/superset/datasets/station_metadata.sql b/superset/datasets/station_metadata.sql new file mode 100644 index 00000000..c89e0454 --- /dev/null +++ b/superset/datasets/station_metadata.sql @@ -0,0 +1,23 @@ +SELECT + s.id AS "Station ID", + s.name AS "Station Name", + s.description AS "Description", + s.wmo_id AS "WMO ID", + s.wigos_id AS "WIGOS ID", + s.icao_id AS "ICAO ID", + s.status AS "Status", + s.observation_processing_method AS "Processing Method", + ST_Y(s.location::geometry) AS "Latitude", + ST_X(s.location::geometry) AS "Longitude", + s.elevation AS "Elevation (m)", + s.date_established AS "Date Established", + s.date_closed AS "Date Closed", + env.name AS "Observation Environment", + foc.name AS "Observation Focus", + owner_org.name AS "Owner Organisation", + op_org.name AS "Operator Organisation" +FROM stations s +LEFT JOIN station_observation_environments env ON env.id = s.observation_environment_id +LEFT JOIN station_observation_focuses foc ON foc.id = s.observation_focus_id +LEFT JOIN organisations owner_org ON owner_org.id = s.owner_id +LEFT JOIN organisations op_org ON op_org.id = s.operator_id diff --git a/superset/ensure_superset_db.py b/superset/ensure_superset_db.py new file mode 100644 index 00000000..ed701e47 --- /dev/null +++ b/superset/ensure_superset_db.py @@ -0,0 +1,30 @@ +""" +Creates the 'superset' database in PostgreSQL if it does not already exist. +Run before `superset db upgrade` so Alembic has a database to connect to. +""" +import os +import sys +import psycopg2 +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + +try: + conn = psycopg2.connect( + host=os.environ.get("DB_HOST", "climsoft_db"), + port=5432, + user="postgres", + password=os.environ.get("DB_PASSWORD", ""), + database="postgres", + ) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + cur = conn.cursor() + cur.execute("SELECT 1 FROM pg_database WHERE datname = 'superset'") + if not cur.fetchone(): + cur.execute("CREATE DATABASE superset") + print("[Superset] Created superset database.") + else: + print("[Superset] Superset database already exists.") + cur.close() + conn.close() +except Exception as e: + print(f"[Superset] Failed to ensure database: {e}", file=sys.stderr) + sys.exit(1) diff --git a/superset/init.sh b/superset/init.sh new file mode 100644 index 00000000..3260155f --- /dev/null +++ b/superset/init.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +echo "[Superset] Ensuring superset database exists..." +python3 /app/ensure_superset_db.py + +echo "[Superset] Running database migrations..." +superset db upgrade + +echo "[Superset] Creating admin user..." +superset fab create-admin \ + --username "${ADMIN_USERNAME:-admin}" \ + --firstname "Admin" \ + --lastname "Admin" \ + --email "admin@climsoft.org" \ + --password "${ADMIN_PASSWORD:-admin}" 2>/dev/null || echo "[Superset] Admin user already exists." + +echo "[Superset] Creating API service account..." +superset fab create-admin \ + --username "${SERVICE_USERNAME:-climsoft_service}" \ + --firstname "Climsoft" \ + --lastname "Service" \ + --email "service@climsoft.org" \ + --password "${SERVICE_PASSWORD:-climsoft_service}" 2>/dev/null || echo "[Superset] Service account already exists." + +echo "[Superset] Initialising roles and permissions..." +superset init + +echo "[Superset] Registering Climsoft database connection and virtual datasets..." +python3 /app/init_datasets.py + +if compgen -G "/app/products/*.zip" > /dev/null 2>&1; then + echo "[Superset] Importing climate products..." + for zip in /app/products/*.zip; do + superset import-dashboards -p "$zip" --overwrite + echo "[Superset] Imported: $(basename "$zip")" + done +else + echo "[Superset] No climate products to import." +fi + +echo "[Superset] Initialisation complete." diff --git a/superset/init_datasets.py b/superset/init_datasets.py new file mode 100644 index 00000000..8d476e0e --- /dev/null +++ b/superset/init_datasets.py @@ -0,0 +1,79 @@ +""" +Registers virtual datasets from superset/datasets/*.sql into Superset. +Run via the entrypoint before starting gunicorn (uses Superset's app context directly). +""" +import os +import sys + +DATASETS_DIR = "/app/datasets" +DB_HOST = os.environ.get("DB_HOST", "climsoft_db") +DB_NAME = os.environ.get("DB_NAME", "climsoft") +DB_PASSWORD = os.environ.get("DB_PASSWORD", "") +CLIMSOFT_DB_URI = f"postgresql://postgres:{DB_PASSWORD}@{DB_HOST}:5432/{DB_NAME}" + + +def main(): + sql_files = sorted( + f for f in os.listdir(DATASETS_DIR) if f.endswith(".sql") + ) if os.path.isdir(DATASETS_DIR) else [] + + if not sql_files: + print("[Superset] No SQL dataset files found, skipping.") + return + + from superset import create_app + from superset.extensions import db + + app = create_app() + with app.app_context(): + from superset.models.core import Database + from superset.connectors.sqla.models import SqlaTable + + # Ensure the Climsoft database connection exists in Superset + climsoft_db = db.session.query(Database).filter_by( + database_name="Climsoft" + ).first() + + if not climsoft_db: + climsoft_db = Database( + database_name="Climsoft", + sqlalchemy_uri=CLIMSOFT_DB_URI, + expose_in_sqllab=True, + allow_run_async=True, + allow_dml=False, + ) + db.session.add(climsoft_db) + db.session.flush() + print(f"[Superset] Registered Climsoft database connection.") + else: + print("[Superset] Climsoft database connection already registered.") + + for filename in sql_files: + dataset_name = filename.replace(".sql", "") + with open(os.path.join(DATASETS_DIR, filename)) as f: + sql = f.read().strip() + + existing = db.session.query(SqlaTable).filter_by( + table_name=dataset_name, + database_id=climsoft_db.id, + ).first() + + if existing: + existing.sql = sql + print(f"[Superset] Updated dataset: {dataset_name}") + else: + table = SqlaTable( + table_name=dataset_name, + sql=sql, + database_id=climsoft_db.id, + schema="public", + is_managed_externally=False, + ) + db.session.add(table) + print(f"[Superset] Created dataset: {dataset_name}") + + db.session.commit() + + +if __name__ == "__main__": + main() diff --git a/superset/superset_config.py b/superset/superset_config.py new file mode 100644 index 00000000..04b2eabe --- /dev/null +++ b/superset/superset_config.py @@ -0,0 +1,108 @@ +import os +from urllib.parse import urlparse + +SECRET_KEY = os.environ.get("SUPERSET_SECRET_KEY", "change_me") + +APP_NAME = "Climsoft Web" +FAVICONS = [{"href": "/static/assets/images/climsoft_logo.png"}] + +# APP_ICON is deprecated in Superset 5+. Use THEME_DEFAULT token instead. +# Set both light and dark themes so the logo persists regardless of user mode. +_BRAND_LOGO = { + "brandLogoUrl": "/static/assets/images/climsoft_logo.png", + "brandLogoAlt": "Climsoft Web", + "brandLogoHref": "/", + "brandLogoHeight": "40px", +} +THEME_DEFAULT = {"token": _BRAND_LOGO} +THEME_DARK = {"token": _BRAND_LOGO} + +SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI") +if not SQLALCHEMY_DATABASE_URI: + raise RuntimeError("SQLALCHEMY_DATABASE_URI environment variable is required") + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + +_redis = urlparse(REDIS_URL) + +CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_URL": REDIS_URL, +} + +DATA_CACHE_CONFIG = { + **CACHE_CONFIG, + "CACHE_KEY_PREFIX": "superset_data_", +} + +FEATURE_FLAGS = { + "EMBEDDED_SUPERSET": True, +} + +# Grant the guest/public role the same dataset-viewing permissions as Gamma. +# Required for embedded dashboards — guests are assigned the Public role and +# need at least Gamma-level permissions to load dashboards and query datasets. +PUBLIC_ROLE_LIKE = "Gamma" + +# Sub-path prefix when Superset is served at /superset/ behind Nginx. +# Must match the Nginx location block prefix. +WEBSERVER_PREFIX = os.environ.get("WEBSERVER_PREFIX", "/superset") + +# Required when running behind a reverse proxy so Superset trusts +# X-Forwarded-For / X-Forwarded-Proto headers from Nginx. +ENABLE_PROXY_FIX = True +PROXY_FIX_CONFIG = { + "x_for": 1, + "x_proto": 1, + "x_host": 1, + "x_prefix": 1, +} + +# Allow Climsoft Angular to embed dashboards via iframe +ENABLE_CORS = True +CORS_OPTIONS = { + "supports_credentials": True, + "allow_headers": ["*"], + "resources": ["*"], + "origins": ["*"], # Restrict to Climsoft's domain in production +} + +# Disable Talisman so Nginx controls security headers (including frame-ancestors) +TALISMAN_ENABLED = False +HTTP_HEADERS = {} + +# Cookie settings for cross-origin iframe embedding. +# In production with HTTPS set SESSION_COOKIE_SECURE = True and +# SESSION_COOKIE_SAMESITE = "None". +SESSION_COOKIE_SAMESITE = os.environ.get("SESSION_COOKIE_SAMESITE", "Lax") +SESSION_COOKIE_SECURE = os.environ.get("SESSION_COOKIE_SECURE", "false").lower() == "true" +SESSION_COOKIE_HTTPONLY = True + +# Guest token settings (used for embedded mode) +GUEST_TOKEN_JWT_SECRET = SECRET_KEY +GUEST_TOKEN_JWT_ALGO = "HS256" +GUEST_TOKEN_JWT_EXP_SECONDS = 300 # 5 minutes — Angular must refresh before expiry + +# Celery configuration for async queries, alerts, reports, and thumbnails +class CeleryConfig: + broker_url = REDIS_URL + imports = ("superset.sql_lab", "superset.tasks.scheduler") + result_backend = REDIS_URL + worker_prefetch_multiplier = 10 + task_acks_late = True + task_annotations = { + "sql_lab.get_sql_results": {"rate_limit": "100/s"}, + } + +CELERY_CONFIG = CeleryConfig + +# Store async SQL Lab query results in Redis +from cachelib.redis import RedisCache +RESULTS_BACKEND = RedisCache( + host=_redis.hostname, + port=_redis.port or 6379, + db=int(_redis.path.lstrip("/") or 0), + key_prefix="superset_results", +) diff --git a/superset/worker.sh b/superset/worker.sh new file mode 100644 index 00000000..0dce79f5 --- /dev/null +++ b/superset/worker.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +exec celery --app=superset.tasks.celery_app:app worker \ + --pool=gevent \ + --concurrency="${CELERY_CONCURRENCY:-4}" \ + --max-tasks-per-child=128 \ + -Ofair