Improves Data Ingestion and adds beta implementation of Climsoft Web products

.env.example

@@ -5,13 +5,29 @@ HOST_HTTP_PORT=8080
 FIRST_INSTALL=yes
 
 # Encyption secrets has to be atleast 32 characters long
-ENCRYPTION_SECRET=0123456789012345678901234567890123456789
+ENCRYPTION_SECRET=change_me_to_a_long_random_string_of_at_least_32_chars
 
 #-------------------------------------
 # Climsoft web database credentials
 DB_NAME=climsoft
 DB_PASSWORD=my_password
 
+#-------------------------------------
+# Climate Products (Apache Superset) settings
+# Set SUPERSET_ENABLED=true and start with --profile superset to enable
+SUPERSET_ENABLED=false
+# Must be a random string of at least 42 characters
+SUPERSET_SECRET_KEY=change_me_to_a_long_random_string_of_at_least_42_chars
+# Human sysadmin account — used to log into the Superset UI
+SUPERSET_ADMIN_USERNAME=superset_admin
+SUPERSET_ADMIN_PASSWORD=change_me
+# Service account used by the Climsoft API to generate guest tokens (keep separate from the sysadmin account)
+SUPERSET_SERVICE_USERNAME=climsoft_service
+SUPERSET_SERVICE_PASSWORD=change_me_service
+# In development Superset runs directly on port 8088 (no nginx), so set this to empty.
+# In test/prod it runs behind nginx at /superset/ — leave unset to use the /superset default.
+# WEBSERVER_PREFIX=
+
 #-------------------------------------
 # Climsoft V4 database credentials
 V4_SAVE=yes

back-end/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "preview-3.0.2",
+  "version": "preview-3.0.4",
   "description": "Climsoft API",
   "author": "Patrick Munyoki <patrickmunyoki3@gmail.com",
   "contributors": [

back-end/api/src/app.config.ts

@@ -62,6 +62,14 @@ export class AppConfig {
   // It should be atleast 32 chracters long
   public static readonly encryptionSecret: string = AppConfig.devMode ? '0123456789012345678901234567890123456789' : (process.env.ENCRYPTION_SECRET ? process.env.ENCRYPTION_SECRET : '');
 
+  public static readonly superset = {
+    enabled: AppConfig.devMode ? true : process.env.SUPERSET_ENABLED === 'true',
+    host: AppConfig.devMode ? 'localhost' : (process.env.SUPERSET_HOST ?? 'climsoft_superset'),
+    port: process.env.SUPERSET_PORT ? +process.env.SUPERSET_PORT : 8088,
+    serviceUsername: process.env.SUPERSET_SERVICE_USERNAME ?? 'climsoft_service',
+    servicePassword: process.env.SUPERSET_SERVICE_PASSWORD ?? 'climsoft_service',
+  };
+
   public static readonly v4DbCredentials = {
     v4Save: AppConfig.devMode ? true : (process.env.V4_SAVE ? (process.env.V4_SAVE === 'yes') : false),
     v4Import: AppConfig.devMode ? true : (process.env.V4_IMPORT ? (process.env.V4_IMPORT === 'yes') : false),

back-end/api/src/app.module.ts

@@ -8,6 +8,7 @@ import { UserModule } from './user/user.module';
 import { SettingsModule } from './settings/settings.module';
 import { MigrationsModule } from './migrations/migrations.module';
 import { QueueModule } from './queue/queue.module';
+import { ProductsModule } from './products/products.module';
 import { AppConfig } from './app.config';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { ScheduleModule } from '@nestjs/schedule';
@@ -22,6 +23,7 @@ import { ScheduleModule } from '@nestjs/schedule';
     SettingsModule,
     MigrationsModule,
     QueueModule,
+    ProductsModule,
     TypeOrmModule.forRoot({
       type: "postgres",
       host: AppConfig.dbCredentials.host,

back-end/api/src/metadata/adapters/adapter-language-conventions.ts

@@ -1,40 +1,39 @@
 import { AdapterLanguageEnum } from './enums/adapter-language.enum';
 
 /**
- * Language conventions for uploaded adapter zips. Every zip must place these
- * files at its ROOT (top-level, no directory prefix) — the runner reads the
- * entry point from the extracted script directory and the manifest is
- * detected by the extraction preview before save.
+ * Per-language conventions for uploaded adapter zips. Every zip must
+ * place both files at its ROOT (top-level, no directory prefix) — the
+ * runner reads the entry point from the extracted script directory and
+ * the manifest is detected by the extraction preview before save.
  *
  * Kept in a shared file (not the service or the runner) because both
- * `AdaptersService` (metadata) and `AdapterRunnerService` (shared) need to
- * agree on the mapping.
+ * `AdaptersService` (metadata) and `AdapterRunnerService` (shared) need
+ * to agree on the mapping.
  */
+export interface AdapterLanguageConvention {
+    /**
+     * Required declaration file at the root of the zip. Analogous across
+     * languages: Python `requirements.txt`, R `DESCRIPTION`, JavaScript
+     * `package.json`, SQL `extensions.txt`. Users declare top-level
+     * dependencies; the runner resolves transitives at install time.
+     *
+     * Optional lockfiles (renv.lock, package-lock.json) may be shipped
+     * alongside for deterministic installs but are not required — the
+     * runner honors them when present.
+     */
+    manifest: string;
 
-/**
- * Manifest filenames accepted at the root of an uploaded zip, one entry per
- * language. Multiple values mean any one of them satisfies the check
- * (e.g. R adapters accept either `renv.lock` or `DESCRIPTION`).
- *
- * The API only checks existence; runners parse dependencies at first-run time.
- */
-export const MANIFEST_FILENAMES: Record<AdapterLanguageEnum, string[]> = {
-    [AdapterLanguageEnum.PYTHON]: ['requirements.txt'],
-    [AdapterLanguageEnum.R]: ['renv.lock', 'DESCRIPTION'],
-    [AdapterLanguageEnum.JAVASCRIPT]: ['package.json', 'package-lock.json'],
-    [AdapterLanguageEnum.SQL]: ['extensions.txt'],
-};
+    /**
+     * Canonical entry-point filename the runner executes. Users don't
+     * choose this — it's a language convention so the runner never has
+     * to guess. Matches the filenames shipped by the starter templates.
+     */
+    entryPoint: string;
+}
 
-/**
- * Canonical entry-point filename required at the root of an uploaded zip, one
- * per language. Users don't choose this — the convention is enforced so the
- * runner always knows what to execute without a stored `entryPoint` field.
- *
- * Matches the filenames shipped by the starter templates.
- */
-export const CANONICAL_ENTRY_POINT: Record<AdapterLanguageEnum, string> = {
-    [AdapterLanguageEnum.PYTHON]: 'main.py',
-    [AdapterLanguageEnum.R]: 'main.R',
-    [AdapterLanguageEnum.JAVASCRIPT]: 'index.js',
-    [AdapterLanguageEnum.SQL]: 'transform.sql',
+export const LANGUAGE_CONVENTIONS: Record<AdapterLanguageEnum, AdapterLanguageConvention> = {
+    [AdapterLanguageEnum.PYTHON]:     { manifest: 'requirements.txt', entryPoint: 'main.py' },
+    [AdapterLanguageEnum.R]:          { manifest: 'DESCRIPTION',      entryPoint: 'main.R' },
+    [AdapterLanguageEnum.JAVASCRIPT]: { manifest: 'package.json',     entryPoint: 'index.js' },
+    [AdapterLanguageEnum.SQL]:        { manifest: 'extensions.txt',   entryPoint: 'transform.sql' },
 };

back-end/api/src/metadata/adapters/dtos/create-adapter-specification.dto.ts

@@ -7,7 +7,7 @@ import { AdapterLanguageEnum } from '../enums/adapter-language.enum';
  * returned the `scriptDirName` (UUID) the client sends here.
  *
  * The entry point is NOT sent by the client — it is a language-level
- * convention (see `CANONICAL_ENTRY_POINT`) that the API enforces at
+ * convention (see `LANGUAGE_CONVENTIONS`) that the API enforces at
  * upload-preview time.
  */
 export class CreateAdapterSpecificationDto {

back-end/api/src/metadata/adapters/dtos/view-adapter-specification.dto.ts

@@ -2,6 +2,7 @@ import { AdapterLanguageEnum } from '../enums/adapter-language.enum';
 
 export class ViewAdapterSpecificationDto {
     id!: number;
+    systemKey!: string | null;
     name!: string;
     description!: string;
     language!: AdapterLanguageEnum;

back-end/api/src/metadata/adapters/entities/adapter-specification.entity.ts

@@ -2,6 +2,15 @@ import { Check, Column, Entity, Index, PrimaryGeneratedColumn } from "typeorm";
 import { AppBaseEntity, BaseLogVo } from "src/shared/entity/app-base-entity";
 import { AdapterLanguageEnum } from "../enums/adapter-language.enum";
 
+export interface AdapterSpecificationLogVo extends BaseLogVo {
+    name: string;
+    description: string | null;
+    language: AdapterLanguageEnum;
+    scriptDirName: string;
+    disabled: boolean;
+    comment: string | null;
+}
+
 /**
  * A user-uploaded script that translates a foreign file format to/from the
  * canonical format the existing import/export pipelines understand.
@@ -35,6 +44,9 @@ export class AdapterSpecificationEntity extends AppBaseEntity {
     @PrimaryGeneratedColumn({ name: "id", type: "int" })
     id!: number;
 
+    @Column({ name: "system_key", type: "varchar", unique: true, nullable: true })
+    systemKey!: string | null;
+
     @Column({ name: "name", type: "varchar", unique: true })
     name!: string;
 
@@ -56,13 +68,13 @@ export class AdapterSpecificationEntity extends AppBaseEntity {
     @Column({ name: "script_dir_name", type: "varchar", unique: true })
     scriptDirName!: string;
 
-    @Column({ type: "boolean", default: false })
+    @Column({ name: "disabled", type: "boolean", default: false })
     @Index()
     disabled!: boolean;
 
     @Column({ name: "comment", type: "varchar", nullable: true })
     comment!: string | null;
 
     @Column({ name: "log", type: "jsonb", nullable: true })
-    log!: BaseLogVo[] | null;
+    log!: AdapterSpecificationLogVo[] | null;
 }

back-end/api/src/metadata/adapters/services/adapters.service.ts

@@ -1,6 +1,6 @@
 import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleInit } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import { Repository } from 'typeorm';
+import { IsNull, Repository } from 'typeorm';
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
@@ -19,7 +19,7 @@ import { AdapterRef, AdapterRunMetadata, AdapterRunnerService, AdapterRunResult
 import { CacheLoadResult, MetadataCache } from 'src/shared/cache/metadata-cache';
 import { AdapterTestRunPreviewDto } from '../dtos/adapter-test-run-preview.dto';
 import { FileProcessingErrorType } from 'src/metadata/file-processing-error.model';
-import { CANONICAL_ENTRY_POINT, MANIFEST_FILENAMES } from '../adapter-language-conventions';
+import { LANGUAGE_CONVENTIONS } from '../adapter-language-conventions';
 
 @Injectable()
 export class AdaptersService implements OnModuleInit {
@@ -157,22 +157,20 @@ export class AdaptersService implements OnModuleInit {
     }
 
     /**
-     * Returns an error message if none of the accepted manifest filenames for
-     * the language are present at the ROOT of the tree (top-level files only,
-     * no directory prefix). Returns `undefined` if a valid manifest is present.
+     * Returns an error message if the language's required manifest file is
+     * not present at the ROOT of the tree (top-level files only, no
+     * directory prefix). Returns `undefined` if the manifest is present.
      *
      * Root-only enforcement is intentional: runners execute from the extracted
      * script directory and expect the manifest at that same level. A zip whose
      * files are nested inside a wrapper folder (a common user mistake) must
      * fail here rather than at runtime.
      */
     private checkManifestAtRoot(fileTree: FileTreeEntry[], language: AdapterLanguageEnum): string | undefined {
-        const acceptedNames = MANIFEST_FILENAMES[language];
-        const found = acceptedNames.some(name =>
-            fileTree.some(e => !e.isDirectory && e.path === name),
-        );
+        const expected = LANGUAGE_CONVENTIONS[language].manifest;
+        const found = fileTree.some(e => !e.isDirectory && e.path === expected);
         if (found) return undefined;
-        return `Missing manifest file for '${language}'. Expected one of: ${acceptedNames.join(', ')} at the root of the archive.`;
+        return `Missing manifest file '${expected}' at the root of the archive for language '${language}'.`;
     }
 
     /**
@@ -182,7 +180,7 @@ export class AdaptersService implements OnModuleInit {
      * `transform.sql`) enforced here so the runner never has to guess.
      */
     private checkEntryPointAtRoot(fileTree: FileTreeEntry[], language: AdapterLanguageEnum): string | undefined {
-        const expected = CANONICAL_ENTRY_POINT[language];
+        const expected = LANGUAGE_CONVENTIONS[language].entryPoint;
         const found = fileTree.some(e => !e.isDirectory && e.path === expected);
         if (found) return undefined;
         return `Missing entry-point file '${expected}' at the root of the archive for language '${language}'.`;
@@ -259,16 +257,19 @@ export class AdaptersService implements OnModuleInit {
     public async update(id: number, dto: UpdateAdapterSpecificationDto, userId: number): Promise<ViewAdapterSpecificationDto> {
         const entity = await this.findEntity(id);
 
-        entity.name = dto.name;
-        entity.description = dto.description ?? null;
-        entity.disabled = dto.disabled;
-        entity.comment = dto.comment ?? null;
+        if (entity.systemKey !== null) {
+            entity.disabled = dto.disabled;
+        } else {
+            entity.name = dto.name;
+            entity.description = dto.description ?? null;
+            entity.disabled = dto.disabled;
+            entity.comment = dto.comment ?? null;
+            const scriptDir = this.fileIO.getAdapterScriptDir(dto.scriptDirName);
+            await this.assertDirExists(scriptDir, `Script directory '${dto.scriptDirName}' not found. Please upload the zip file first.`);
+            entity.scriptDirName = dto.scriptDirName;
+        }
         entity.entryUserId = userId;
 
-        const scriptDir = this.fileIO.getAdapterScriptDir(dto.scriptDirName);
-        await this.assertDirExists(scriptDir, `Script directory '${dto.scriptDirName}' not found. Please upload the zip file first.`);
-        entity.scriptDirName = dto.scriptDirName;
-
         await this.adapterRepo.save(entity);
         await this.cache.invalidate();
 
@@ -429,16 +430,19 @@ export class AdaptersService implements OnModuleInit {
 
     public async delete(id: number): Promise<void> {
         const entity: AdapterSpecificationEntity = await this.findEntity(id);
+        if (entity.systemKey !== null) {
+            throw new BadRequestException(`Adapter '${entity.name}' is a system adapter and cannot be deleted`);
+        }
         await this.adapterRepo.remove(entity);
         await this.cache.invalidate();
         this.logger.log(`Adapter deleted: #${id}. On-disk script directories are retained.`);
     }
 
     public async deleteAll(): Promise<void> {
-        const entities: AdapterSpecificationEntity[] = await this.adapterRepo.find();
+        const entities: AdapterSpecificationEntity[] = await this.adapterRepo.find({ where: { systemKey: IsNull() } });
         await this.adapterRepo.remove(entities);
         await this.cache.invalidate();
-        this.logger.log(`All adapters deleted. On-disk script directories are retained.`);
+        this.logger.log(`All user-defined adapters deleted. System adapters and on-disk script directories are retained.`);
     }
 
     //--------------------------------------------------------------------
@@ -456,6 +460,7 @@ export class AdaptersService implements OnModuleInit {
     private toViewDto(entity: AdapterSpecificationEntity): ViewAdapterSpecificationDto {
         return {
             id: entity.id,
+            systemKey: entity.systemKey ?? null,
             name: entity.name,
             description: entity.description ?? '',
             language: entity.language,

back-end/api/src/metadata/export-specifications/dtos/raw-export-parameters.dto.ts

@@ -1,7 +1,6 @@
 import { IsBoolean, IsOptional } from "class-validator";
 
 export class RawExportParametersDto {
-  // Data
   @IsOptional()
   @IsBoolean()
   convertDatetimeToDisplayTimeZone?: boolean;

back-end/api/src/metadata/source-specifications/dtos/import-source-tabular-params.dto.ts

@@ -75,6 +75,31 @@ export class FlagDefinition {
     flagsToFetch?: FlagToFetch[];
 }
 
+/**
+ * When set on {@link ImportSourceTabularParamsDto.inlineFlagRule}, cells in
+ * the observation `value` column carry a trailing alphabetic run interpreted
+ * as a flag abbreviation. Example: `0.5T` splits into value `0.5` and flag `T`.
+ *
+ * Applies uniformly whether the `value` column comes from an explicit
+ * `valueDefinition.valueColumnPosition` or from a wide-pivot UNPIVOT
+ * (multiple elements, day columns range, or hour columns range).
+ *
+ * The split rule itself is fixed — trailing `[A-Za-z]+` is the flag; the
+ * numeric prefix is the value. Both parts may be empty and are handled by
+ * the existing missing-value logic.
+ */
+export class InlineFlagRule {
+    /**
+     * Optional source flag string → database flag id mapping. When omitted,
+     * the extracted flag string is matched case-insensitively against
+     * `flags.abbreviation` — same semantics as `FlagDefinition.flagsToFetch`.
+     */
+    @IsOptional()
+    @ValidateNested({ each: true })
+    @Type(() => FlagToFetch)
+    flagsToFetch?: FlagToFetch[];
+}
+
 export class ValueDefinition {
     /** Value column position. */
     @IsInt()
@@ -279,8 +304,11 @@ export enum DateTimeFormat {
 
     // 12-hour clock with AM/PM (Excel and US-locale form entries)
     YMD_DASH_HM_AMPM = '%Y-%m-%d %I:%M %p',
+    YMD_DASH_HMS_AMPM = '%Y-%m-%d %I:%M:%S %p',    // 2023-12-13 10:30:00 AM
     DMY_SLASH_HM_AMPM = '%d/%m/%Y %I:%M %p',
+    DMY_SLASH_HMS_AMPM = '%d/%m/%Y %I:%M:%S %p',   // 13/12/2023 10:30:00 AM
     MDY_SLASH_HM_AMPM = '%m/%d/%Y %I:%M %p',
+    MDY_SLASH_HMS_AMPM = '%m/%d/%Y %I:%M:%S %p',   // 12/13/2023 10:30:00 AM
 
     // Dot-separated (German / Russian / Eastern European locales)
     DMY_DOT_HMS = '%d.%m.%Y %H:%M:%S',
@@ -474,6 +502,16 @@ export class ImportSourceTabularParamsDto {
     @Type(() => ValueDefinition)
     valueDefinition?: ValueDefinition;
 
+    /**
+     * Opt-in: cells in the `value` column carry a trailing flag suffix.
+     * Mutually exclusive with `valueDefinition.flagDefinition` — if both are
+     * set, the explicit flag column takes precedence and this is ignored.
+     */
+    @IsOptional()
+    @ValidateNested()
+    @Type(() => InlineFlagRule)
+    inlineFlagRule?: InlineFlagRule;
+
     @IsOptional()
     @ValidateNested()
     @Type(() => CommentDefinition)