diff --git a/charts/streaming-hub/Chart.lock b/charts/streaming-hub/Chart.lock new file mode 100644 index 000000000..fcf430ae5 --- /dev/null +++ b/charts/streaming-hub/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: lerian-common-helm + repository: file://../lerian-common + version: 1.4.0 +digest: sha256:55cbfcfef14768a47be9e794cb86f07cecdd320cb5ce40b363527d7c465f97e6 +generated: "2026-08-04T16:17:01.689166-03:00" diff --git a/charts/streaming-hub/Chart.yaml b/charts/streaming-hub/Chart.yaml index fed353409..0d9ce5938 100644 --- a/charts/streaming-hub/Chart.yaml +++ b/charts/streaming-hub/Chart.yaml @@ -29,8 +29,13 @@ keywords: - lerian icon: https://avatars.githubusercontent.com/u/148895005?s=200&v=4 -# NO dependencies block by deliberate design (deviation from reporter/midaz): -# OTEL is env-wired via the OTEL_EXPORTER_OTLP_ENDPOINT downward-API override, -# not bundled as an otel-collector-lerian subchart. Declaring it would force an -# OCI pull on every `helm lint` / `helm template`. Kafka/Redpanda and Postgres -# are shared external infra, never subcharts here. +# lerian-common is the ONLY dependency: a local library chart (no OCI pull, no +# runtime objects) providing the cfgValue/globalValue config helpers the +# ConfigMap is productized onto. Deliberately NO otel-collector / Kafka / Postgres +# subcharts: OTEL is env-wired via the OTEL_EXPORTER_OTLP_ENDPOINT downward-API +# override, and Kafka/Redpanda + Postgres are shared external infra — declaring +# any of them would force an OCI pull on every `helm lint` / `helm template`. +dependencies: + - name: lerian-common-helm + version: "1.4.0" + repository: "file://../lerian-common" diff --git a/charts/streaming-hub/README.params.md b/charts/streaming-hub/README.params.md new file mode 100644 index 000000000..9c6e793b2 --- /dev/null +++ b/charts/streaming-hub/README.params.md @@ -0,0 +1,92 @@ +# Parameters + +## Parameters + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `nameOverride` | string | `""` | Override the chart name component of resource names. | +| `fullnameOverride` | string | `""` | Override the fully-qualified release name (wins verbatim). | +| `namespaceOverride` | string | `""` | Override the namespace (defaults to .Release.Namespace). | +| `global.externalPostgresDefinitions` | string | `{}` | Bootstrap Job for an external/shared PostgreSQL: creates the hub's ONE database + role and grants privileges. The hub owns a SINGLE database with a tenant_id column (NOT per-tenant DB). Default OFF — dev-st may instead point STREAMING_HUB_POSTGRES_DSN at a pre-provisioned managed host. | +| `global.externalPostgresDefinitions.enabled` | bool | `false` | Enable or disable the PostgreSQL bootstrap Job. | +| `global.externalPostgresDefinitions.database` | string | `"streaming-hub"` | Name of the database the Job creates (must match the DSN dbname). | +| `global.externalPostgresDefinitions.role` | string | `"streaming-hub"` | Name of the login role the Job creates (must match the DSN user). | +| `global.externalPostgresDefinitions.connection` | string | `{}` | PostgreSQL connection settings for the bootstrap Job. | +| `global.externalPostgresDefinitions.connection.host` | string | `"streaming-hub-postgresql"` | PostgreSQL host. | +| `global.externalPostgresDefinitions.connection.port` | string | `"5432"` | PostgreSQL port. | +| `global.externalPostgresDefinitions.postgresAdminLogin` | string | `{}` | Admin credentials used by the Job to create the DB/role. | +| `global.externalPostgresDefinitions.postgresAdminLogin.useExistingSecret.name` | string | `""` | Existing secret with DB_USER_ADMIN and DB_ADMIN_PASSWORD keys. | +| `global.externalPostgresDefinitions.postgresAdminLogin.username` | string | `"postgres"` | Admin username (ignored if useExistingSecret.name is set). | +| `global.externalPostgresDefinitions.postgresAdminLogin.password` | string | `""` | Admin password (ignored if useExistingSecret.name is set). | +| `global.externalPostgresDefinitions.hubCredentials` | string | `{}` | Credentials for the hub role created by the Job. | +| `global.externalPostgresDefinitions.hubCredentials.useExistingSecret.name` | string | `""` | Existing secret with DB_PASSWORD_HUB key. | +| `global.externalPostgresDefinitions.hubCredentials.password` | string | `""` | Password for the hub role (ignored if useExistingSecret.name is set). | +| `global.auth` | object | `{}` | Env-wide inbound auth (lib-auth / plugin-auth), consumed by lerian-common.globalValue. Declare once at the umbrella level; a component streamingHub.common.configmap.PLUGIN_AUTH_* still overrides per-service. Precedence: common.configmap. > global.auth. > chart default. Leave empty ({}) to keep the chart defaults. | +| `streamingHub.mode` | enum: all|split | `all` | Deployment topology switch. One of: all | split. all (default) -> ONE Deployment with STREAMING_HUB_ROLE=all (ingest + delivery co-resident; the dev-st target). Byte-equivalent to the historical single binary. split -> TWO Deployments: ingest (role=ingest) and delivery (role=delivery), each scaled independently. !!! NEVER run both an `all` Deployment AND ingest/delivery against the same Kafka cluster: they join ONE consumer group and DOUBLE-CONSUME every event. The mode switch enforces either/or — do not work around it. !!! The values.schema.json constrains this to the enum ["all","split"]. | +| `streamingHub.image.repository` | string | `ghcr.io/lerianstudio/streaming-hub` | Container image repository. | +| `streamingHub.image.pullPolicy` | string | `IfNotPresent` | Image pull policy. | +| `streamingHub.image.tag` | string | `""` | Image tag. Empty falls back to Chart.appVersion via the defaultTag helper. | +| `streamingHub.imagePullSecrets` | string | `{}` | Secrets for pulling the image from a private registry. | +| `streamingHub.revisionHistoryLimit` | int | `10` | Number of old ReplicaSets to retain for rollback. | +| `streamingHub.annotations` | object | `{}` | Annotations applied to every Deployment resource. | +| `streamingHub.podAnnotations` | object | `{}` | Annotations applied to every pod. | +| `streamingHub.deploymentStrategy` | string | `{}` | Deployment update strategy (shared by all roles). | +| `streamingHub.podSecurityContext` | object | `{}` | Pod-level security context. Empty by default (the hub needs no fsGroup). | +| `streamingHub.securityContext` | string | `{}` | Container-level security context (distroless:nonroot, uid/gid 65532). | +| `streamingHub.securityContext.runAsGroup` | int | `65532` | Group ID for the process inside the container. | +| `streamingHub.securityContext.runAsUser` | int | `65532` | User ID for the process inside the container. | +| `streamingHub.securityContext.runAsNonRoot` | bool | `true` | Never run as root. | +| `streamingHub.securityContext.readOnlyRootFilesystem` | bool | `true` | Read-only root filesystem (the image carries no writable state). | +| `streamingHub.service.type` | string | `ClusterIP` | Service type. MUST be ClusterIP (Lerian convention; Ingress fronts external). | +| `streamingHub.service.port` | int | `8080` | Control-plane HTTP port (the hub listens on :8080; see Dockerfile EXPOSE). | +| `streamingHub.service.annotations` | object | `{}` | Annotations for every Service. | +| `streamingHub.ingress.enabled` | bool | `false` | Enable or disable the control-plane Ingress (opt-in per env in gitops). | +| `streamingHub.ingress.className` | string | `"nginx"` | Ingress class name. | +| `streamingHub.ingress.annotations` | object | `{}` | Additional ingress annotations. | +| `streamingHub.ingress.hosts` | list | `[]` | Hosts (default empty; the control API is served on every role). | +| `streamingHub.ingress.tls` | list | `[]` | TLS configuration. | +| `streamingHub.serviceAccount.create` | bool | `true` | Whether a ServiceAccount is created. | +| `streamingHub.serviceAccount.annotations` | object | `{}` | Annotations for the ServiceAccount (e.g. AWS IRSA role-arn). | +| `streamingHub.serviceAccount.name` | string | `""` | ServiceAccount name. Empty defaults to the chart fullname. | +| `streamingHub.serviceAccount.automountServiceAccountToken` | bool | `false` | Mount the SA API token into pods. Default false — the hub makes no in-cluster Kubernetes API calls. (IRSA's projected token is injected by the EKS webhook independently of this, so it stays functional.) | +| `streamingHub.terminationGracePeriodSeconds` | int | `80` | Graceful-shutdown window. Defaults to the hub's derived SIGTERM drain ceiling (80s) at STREAMING_HUB_SHUTDOWN_TIMEOUT=30s + STREAMING_HUB_PRE_STOP_DRAIN_TIMEOUT=5s: 30s HTTP + 5s consumer-commit + 30s dispatcher + 10s slack = 75s, + 5s pre-stop = 80s (see .env.reference). If you tune those knobs up, recompute and keep this AT OR ABOVE the new ceiling so the orchestrator never SIGKILLs a still-draining replica. NO preStop hook is used — the hub self-drains on SIGTERM (PID 1 receives it directly; exec-form ENTRYPOINT). | +| `streamingHub.livenessProbe` | string | `{}` | Liveness probe tuning (GET /healthz on the http port; stays 200 during drain). | +| `streamingHub.readinessProbe` | string | `{}` | Readiness probe tuning (GET /readyz; flips NotReady first on SIGTERM). | +| `streamingHub.nodeSelector` | object | `{}` | Shared default scheduling (per-role blocks may override). | +| `streamingHub.extraEnvVars` | list | `[]` | Extra non-sensitive env vars injected on every Deployment (list of {name,value}). | +| `streamingHub.telemetry.enabled` | bool | `false` | Inject the per-pod OTLP endpoint override (HOST_IP downward API). | +| `streamingHub.common` | string | `{}` | Native per-key escape hatch (highest precedence). Any UPPER_SNAKE app env var can be pinned here verbatim, overriding the grouped field + default. | +| `streamingHub.extraEnvVars` | object | `{}` | Unmodeled extra env vars appended verbatim to the ConfigMap. | +| `streamingHub.app` | object | `{}` | Application / lifecycle (STREAMING_HUB_ENV|LOG_LEVEL|HEALTH_WINDOW| SWAGGER_ENABLED|SHUTDOWN_TIMEOUT|PRE_STOP_DRAIN_TIMEOUT). | +| `streamingHub.server` | object | `{}` | HTTP server / metrics (STREAMING_HUB_HTTP_LISTEN_ADDR|METRICS_ENABLED). | +| `streamingHub.kafka` | object | `{}` | Kafka / Redpanda (brokers|scramMechanism|scramUsername|tlsEnabled|caCert). The SCRAM password is a Secret (see secrets below). | +| `streamingHub.kek` | object | `{}` | Crypto / KEK config (source|ref). The KEK material itself is a Secret. | +| `streamingHub.dispatch` | object | `{}` | Dispatch / poison worker pool (workers|claimBatch|idleIntervalMs|poisonThreshold). | +| `streamingHub.pull` | object | `{}` | Pull rate limit (rate|burst). | +| `streamingHub.dlq` | object | `{}` | DLQ visibility (group|retention|pruneInterval). | +| `streamingHub.manifest` | object | `{}` | Event manifest sources (sources|refreshInterval). | +| `streamingHub.reconciler` | object | `{}` | Reconciler / topic-drift (enabled|interval). | +| `streamingHub.partition` | object | `{}` | Partition lifecycle cron (cronInterval|futureBufferWeeks|retentionEnabled|retentionHorizon). | +| `streamingHub.idempotency` | object | `{}` | Idempotency (ttl|reapInterval). | +| `streamingHub.autodisable` | object | `{}` | Auto-disable subscriber circuit (enabled|failureWindow|failureSpread). | +| `streamingHub.multiTenant` | object | `{}` | Tenancy (F4). libEnabled=MULTI_TENANT_ENABLED (lib-commons request tenancy); the rest is the hub SaaS tenant-manager roster (STREAMING_HUB_MULTI_TENANT_*/ TENANT_MANAGER_*/TENANT_ID/ENVIRONMENT_NAME). Default = BYOC single-tenant. | +| `streamingHub.security` | object | `{}` | Security posture (allowInsecureKafka|allowInsecureDbTls|allowPrivateSinks). All fail-closed; MUST stay false in staging/production. | +| `streamingHub.aws` | object | `{}` | AWS SaaS setup metadata (hubPrincipalArn|setupTemplateUrl). | +| `streamingHub.observability` | object | `{}` | Observability (OTEL_*: libraryName|serviceName|deploymentEnvironment| exporterOtlpEndpoint|insecureExporter). OTEL_EXPORTER_OTLP_ENDPOINT is overridden per-pod when telemetry.enabled=true (downward API). | +| `streamingHub.migrations.enabled` | bool | `false` | Enable or disable the migrations Job. Default false — opt-in per env (consistent with the chart's other optional features). | +| `streamingHub.migrations.useExistingSecret` | bool | `false` | Optional. When true, the Job reads STREAMING_HUB_POSTGRES_DSN from a pre-existing Secret (existingSecretName) instead of the chart-managed migration-secret hook. Independent of streamingHub.useExistingSecret; set this to point migrations at the app's existing (e.g. Vault) Secret. | +| `streamingHub.migrations.existingSecretName` | string | `""` | Name of the pre-existing Secret holding STREAMING_HUB_POSTGRES_DSN (only used when migrations.useExistingSecret=true). | +| `streamingHub.migrations.image.repository` | string | `ghcr.io/lerianstudio/streaming-hub-migrations` | Migrations image (FROM migrate/migrate + COPY migrations/ /migrations/). | +| `streamingHub.migrations.image.tag` | string | `""` | Tag for the migrations image. Empty falls back to the app image tag (streamingHub.image.tag, or the chart appVersion). An explicit tag or digest overrides this. | +| `streamingHub.migrations.image.digest` | string | `""` | Pin by digest (sha256:...) instead of tag. Wins over tag when set. | +| `streamingHub.migrations.image.pullPolicy` | string | `IfNotPresent` | Image pull policy. | +| `streamingHub.migrations.backoffLimit` | int | `3` | Maximum retries before the Job is considered failed. | +| `streamingHub.migrations.activeDeadlineSeconds` | int | `600` | Hard wall-clock cap on the Job (seconds). | +| `streamingHub.migrations.ttlSecondsAfterFinished` | int | `600` | TTL after which a finished Job is garbage-collected (seconds). | +| `streamingHub.migrations.annotations` | object | `{}` | Extra annotations on the Job (merged after the hook annotations). | +| `streamingHub.migrations.podAnnotations` | object | `{}` | Extra annotations on the migration pod. | +| `streamingHub.migrations.resources` | string | `{}` | Resource requests/limits for the migration container. | +| `streamingHub.all.autoscaling.enabled` | bool | `false` | HPA off by default; replicaCount governs. maxReplicas × poolMaxOpenConns must respect the connection-budget invariant above. | +| `streamingHub.ingest.autoscaling.maxReplicas` | int | `4` | maxReplicas × 8 (poolMaxOpenConns) must fit the connection budget. | +| `streamingHub.delivery.autoscaling.maxReplicas` | int | `4` | maxReplicas × 16 (poolMaxOpenConns) must fit the connection budget. | + diff --git a/charts/streaming-hub/templates/configmap.yaml b/charts/streaming-hub/templates/configmap.yaml index 20b9c86ed..8866eba78 100644 --- a/charts/streaming-hub/templates/configmap.yaml +++ b/charts/streaming-hub/templates/configmap.yaml @@ -5,11 +5,33 @@ roles, so it is NOT duplicated per role (DRY). The role-specific vars they are injected as explicit per-Deployment env in templates//deployment.yaml (see templates/_deployment.tpl), which wins over this envFrom layer. -Ranged over .Values.streamingHub.common.configmap so operators can add/override -keys without editing this template. Every key in that map MUST exist verbatim in -streaming-hub/.env.reference (the env-parity cross-check enforces this); OTEL_* -keys are the documented exception (lib-observability vars, not LoadConfig vars). +Productized onto lerian-common (cfgValue precedence): every key resolves + common.configmap. (escape hatch) > streamingHub.. > chart default. +Unset groups fall through to the chart default. Grouped params give a typed, +schema-validated API; common.configmap stays as the native per-key escape hatch and +streamingHub.extraEnvVars carries unmodeled keys. Every emitted key exists verbatim +in streaming-hub/.env.reference (the env-parity cross-check) EXCEPT the OTEL +identity keys OTEL_RESOURCE_* and OTEL_LIBRARY_NAME (lib-observability passthrough) +and ENVIRONMENT_NAME (lib-commons SaaS var). PLUGIN_AUTH_* resolve via global.auth +so the env-wide auth endpoint is shared across services. */}} +{{- $cm := .Values.streamingHub.common.configmap | default dict }} +{{- $app := .Values.streamingHub.app | default dict }} +{{- $srv := .Values.streamingHub.server | default dict }} +{{- $kafka := .Values.streamingHub.kafka | default dict }} +{{- $kek := .Values.streamingHub.kek | default dict }} +{{- $disp := .Values.streamingHub.dispatch | default dict }} +{{- $pull := .Values.streamingHub.pull | default dict }} +{{- $dlq := .Values.streamingHub.dlq | default dict }} +{{- $mani := .Values.streamingHub.manifest | default dict }} +{{- $rec := .Values.streamingHub.reconciler | default dict }} +{{- $part := .Values.streamingHub.partition | default dict }} +{{- $idem := .Values.streamingHub.idempotency | default dict }} +{{- $auto := .Values.streamingHub.autodisable | default dict }} +{{- $ten := .Values.streamingHub.multiTenant | default dict }} +{{- $sec := .Values.streamingHub.security | default dict }} +{{- $aws := .Values.streamingHub.aws | default dict }} +{{- $obs := .Values.streamingHub.observability | default dict }} apiVersion: v1 kind: ConfigMap metadata: @@ -18,6 +40,155 @@ metadata: labels: {{- include "streaming-hub.labels" (dict "context" . "component" "all") | nindent 4 }} data: - {{- range $key, $value := .Values.streamingHub.common.configmap }} + # ===================================================================== + # APPLICATION / LIFECYCLE + # ===================================================================== + STREAMING_HUB_ENV: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_ENV" "params" $app "field" "env" "default" "production") | quote }} + STREAMING_HUB_LOG_LEVEL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_LOG_LEVEL" "params" $app "field" "logLevel" "default" "info") | quote }} + STREAMING_HUB_HEALTH_WINDOW: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_HEALTH_WINDOW" "params" $app "field" "healthWindow" "default" "720h") | quote }} + STREAMING_HUB_SWAGGER_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_SWAGGER_ENABLED" "params" $app "field" "swaggerEnabled" "default" "false") | quote }} + STREAMING_HUB_SHUTDOWN_TIMEOUT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_SHUTDOWN_TIMEOUT" "params" $app "field" "shutdownTimeout" "default" "30s") | quote }} + STREAMING_HUB_PRE_STOP_DRAIN_TIMEOUT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PRE_STOP_DRAIN_TIMEOUT" "params" $app "field" "preStopDrainTimeout" "default" "5s") | quote }} + + # ===================================================================== + # HTTP SERVER / METRICS + # ===================================================================== + STREAMING_HUB_HTTP_LISTEN_ADDR: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_HTTP_LISTEN_ADDR" "params" $srv "field" "httpListenAddr" "default" ":8080") | quote }} + STREAMING_HUB_METRICS_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_METRICS_ENABLED" "params" $srv "field" "metricsEnabled" "default" "true") | quote }} + + # ===================================================================== + # KAFKA / REDPANDA (external). SCRAM password lives in the Secret. + # ===================================================================== + STREAMING_HUB_KAFKA_BROKERS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KAFKA_BROKERS" "params" $kafka "field" "brokers" "default" "localhost:19092") | quote }} + STREAMING_HUB_KAFKA_SCRAM_MECHANISM: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KAFKA_SCRAM_MECHANISM" "params" $kafka "field" "scramMechanism" "default" "") | quote }} + STREAMING_HUB_KAFKA_SCRAM_USERNAME: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KAFKA_SCRAM_USERNAME" "params" $kafka "field" "scramUsername" "default" "") | quote }} + STREAMING_HUB_KAFKA_TLS_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KAFKA_TLS_ENABLED" "params" $kafka "field" "tlsEnabled" "default" "false") | quote }} + STREAMING_HUB_KAFKA_CA_CERT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KAFKA_CA_CERT" "params" $kafka "field" "caCert" "default" "") | quote }} + + # ===================================================================== + # CRYPTO / KEK (the material itself is a Secret; source/ref are config) + # ===================================================================== + STREAMING_HUB_KEK_SOURCE: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KEK_SOURCE" "params" $kek "field" "source" "default" "env") | quote }} + STREAMING_HUB_KEK_REF: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_KEK_REF" "params" $kek "field" "ref" "default" "") | quote }} + + # ===================================================================== + # DISPATCH / POISON (delivery worker pool) + # ===================================================================== + STREAMING_HUB_DISPATCH_WORKERS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DISPATCH_WORKERS" "params" $disp "field" "workers" "default" "8") | quote }} + STREAMING_HUB_DISPATCH_CLAIM_BATCH: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DISPATCH_CLAIM_BATCH" "params" $disp "field" "claimBatch" "default" "16") | quote }} + STREAMING_HUB_DISPATCH_IDLE_INTERVAL_MS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DISPATCH_IDLE_INTERVAL_MS" "params" $disp "field" "idleIntervalMs" "default" "250") | quote }} + STREAMING_HUB_POISON_THRESHOLD: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_POISON_THRESHOLD" "params" $disp "field" "poisonThreshold" "default" "5") | quote }} + + # ===================================================================== + # PULL RATE LIMIT (source consumption) + # ===================================================================== + STREAMING_HUB_PULL_RATE: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PULL_RATE" "params" $pull "field" "rate" "default" "100") | quote }} + STREAMING_HUB_PULL_BURST: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PULL_BURST" "params" $pull "field" "burst" "default" "200") | quote }} + + # ===================================================================== + # DLQ VISIBILITY + # ===================================================================== + STREAMING_HUB_DLQ_GROUP: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DLQ_GROUP" "params" $dlq "field" "group" "default" "streaming-hub-dlq.local") | quote }} + STREAMING_HUB_DLQ_RETENTION: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DLQ_RETENTION" "params" $dlq "field" "retention" "default" "168h") | quote }} + STREAMING_HUB_DLQ_PRUNE_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_DLQ_PRUNE_INTERVAL" "params" $dlq "field" "pruneInterval" "default" "1h") | quote }} + + # ===================================================================== + # MANIFEST (event manifest sources) + # ===================================================================== + STREAMING_HUB_MANIFEST_SOURCES: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MANIFEST_SOURCES" "params" $mani "field" "sources" "default" "") | quote }} + STREAMING_HUB_MANIFEST_REFRESH_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MANIFEST_REFRESH_INTERVAL" "params" $mani "field" "refreshInterval" "default" "300") | quote }} + + # ===================================================================== + # RECONCILER (topic-drift detection; dark-ship, defaults on) + # ===================================================================== + STREAMING_HUB_RECONCILER_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_RECONCILER_ENABLED" "params" $rec "field" "enabled" "default" "true") | quote }} + STREAMING_HUB_RECONCILER_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_RECONCILER_INTERVAL" "params" $rec "field" "interval" "default" "300") | quote }} + + # ===================================================================== + # PARTITION LIFECYCLE CRON + # ===================================================================== + STREAMING_HUB_PARTITION_CRON_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PARTITION_CRON_INTERVAL" "params" $part "field" "cronInterval" "default" "6h") | quote }} + STREAMING_HUB_PARTITION_FUTURE_BUFFER_WEEKS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PARTITION_FUTURE_BUFFER_WEEKS" "params" $part "field" "futureBufferWeeks" "default" "4") | quote }} + STREAMING_HUB_PARTITION_RETENTION_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PARTITION_RETENTION_ENABLED" "params" $part "field" "retentionEnabled" "default" "false") | quote }} + STREAMING_HUB_PARTITION_RETENTION_HORIZON: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_PARTITION_RETENTION_HORIZON" "params" $part "field" "retentionHorizon" "default" "0") | quote }} + + # ===================================================================== + # IDEMPOTENCY + # ===================================================================== + STREAMING_HUB_IDEMPOTENCY_TTL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_IDEMPOTENCY_TTL" "params" $idem "field" "ttl" "default" "24h") | quote }} + STREAMING_HUB_IDEMPOTENCY_REAP_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_IDEMPOTENCY_REAP_INTERVAL" "params" $idem "field" "reapInterval" "default" "5m") | quote }} + + # ===================================================================== + # AUTO-DISABLE (subscriber failure circuit) + # ===================================================================== + STREAMING_HUB_AUTODISABLE_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_AUTODISABLE_ENABLED" "params" $auto "field" "enabled" "default" "true") | quote }} + STREAMING_HUB_AUTODISABLE_FAILURE_WINDOW: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_AUTODISABLE_FAILURE_WINDOW" "params" $auto "field" "failureWindow" "default" "120h") | quote }} + STREAMING_HUB_AUTODISABLE_FAILURE_SPREAD: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_AUTODISABLE_FAILURE_SPREAD" "params" $auto "field" "failureSpread" "default" "12h") | quote }} + + # ===================================================================== + # TENANCY (F4). Bare MULTI_TENANT_ENABLED is the lib-commons request-tenancy + # toggle; the STREAMING_HUB_MULTI_TENANT_* block is the hub's SaaS tenant-manager + # roster (a distinct concern). Default = BYOC single-tenant. A non-default + # TENANT_ID quarantines all other ce-tenantid events — confirm the producer. + # ===================================================================== + MULTI_TENANT_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "MULTI_TENANT_ENABLED" "params" $ten "field" "libEnabled" "default" "false") | quote }} + STREAMING_HUB_TENANT_ID: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_TENANT_ID" "params" $ten "field" "tenantId" "default" "default") | quote }} + STREAMING_HUB_MULTI_TENANT_ENABLED: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_ENABLED" "params" $ten "field" "enabled" "default" "false") | quote }} + STREAMING_HUB_MULTI_TENANT_AWS_REGION: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_AWS_REGION" "params" $ten "field" "awsRegion" "default" "") | quote }} + # Source SELECTOR for tenant secrets (env|secretsmanager reference), NOT secret + # material — the `| default ""` marks it as intentional config for the standard's + # secret-in-configmap gate. The actual secrets live in the app Secret. + STREAMING_HUB_MULTI_TENANT_SECRETS_ENV: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_SECRETS_ENV" "params" $ten "field" "secretsEnv" "default" "") | default "" | quote }} + STREAMING_HUB_TENANT_MANAGER_URL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_TENANT_MANAGER_URL" "params" $ten "field" "tenantManagerUrl" "default" "") | quote }} + STREAMING_HUB_MULTI_TENANT_ALLOW_INSECURE_TM: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_ALLOW_INSECURE_TM" "params" $ten "field" "allowInsecureTm" "default" "false") | quote }} + STREAMING_HUB_MULTI_TENANT_TM_STALENESS_THRESHOLD: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_TM_STALENESS_THRESHOLD" "params" $ten "field" "tmStalenessThreshold" "default" "12h") | quote }} + STREAMING_HUB_MULTI_TENANT_LISTENER_STALENESS_THRESHOLD: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_LISTENER_STALENESS_THRESHOLD" "params" $ten "field" "listenerStalenessThreshold" "default" "12h") | quote }} + STREAMING_HUB_MULTI_TENANT_RECONCILE_INTERVAL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_RECONCILE_INTERVAL" "params" $ten "field" "reconcileInterval" "default" "6h") | quote }} + STREAMING_HUB_MULTI_TENANT_CACHE_TTL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_CACHE_TTL" "params" $ten "field" "cacheTtl" "default" "12h") | quote }} + STREAMING_HUB_TENANT_MANAGER_CB_THRESHOLD: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_TENANT_MANAGER_CB_THRESHOLD" "params" $ten "field" "cbThreshold" "default" "0") | quote }} + STREAMING_HUB_TENANT_MANAGER_CB_TIMEOUT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_TENANT_MANAGER_CB_TIMEOUT" "params" $ten "field" "cbTimeout" "default" "30s") | quote }} + STREAMING_HUB_MULTI_TENANT_REDIS_HOST: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_REDIS_HOST" "params" $ten "field" "redisHost" "default" "") | quote }} + STREAMING_HUB_MULTI_TENANT_REDIS_PORT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_REDIS_PORT" "params" $ten "field" "redisPort" "default" "") | quote }} + STREAMING_HUB_MULTI_TENANT_REDIS_TLS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_REDIS_TLS" "params" $ten "field" "redisTls" "default" "false") | quote }} + STREAMING_HUB_MULTI_TENANT_REDIS_CA_CERT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_MULTI_TENANT_REDIS_CA_CERT" "params" $ten "field" "redisCaCert" "default" "") | quote }} + ENVIRONMENT_NAME: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "ENVIRONMENT_NAME" "params" $ten "field" "environmentName" "default" "") | quote }} + + # ===================================================================== + # SECURITY POSTURE (fail-closed insecure opt-ins; MUST stay false in prod) + # ===================================================================== + STREAMING_HUB_ALLOW_INSECURE_KAFKA: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_ALLOW_INSECURE_KAFKA" "params" $sec "field" "allowInsecureKafka" "default" "false") | quote }} + STREAMING_HUB_ALLOW_INSECURE_DB_TLS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_ALLOW_INSECURE_DB_TLS" "params" $sec "field" "allowInsecureDbTls" "default" "false") | quote }} + STREAMING_HUB_ALLOW_PRIVATE_SINKS: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_ALLOW_PRIVATE_SINKS" "params" $sec "field" "allowPrivateSinks" "default" "false") | quote }} + + # ===================================================================== + # AWS (SaaS setup metadata) + # ===================================================================== + STREAMING_HUB_AWS_HUB_PRINCIPAL_ARN: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_AWS_HUB_PRINCIPAL_ARN" "params" $aws "field" "hubPrincipalArn" "default" "") | quote }} + STREAMING_HUB_AWS_SETUP_TEMPLATE_URL: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "STREAMING_HUB_AWS_SETUP_TEMPLATE_URL" "params" $aws "field" "setupTemplateUrl" "default" "") | quote }} + + # ===================================================================== + # AUTHENTICATION (lib-auth / plugin-auth JWT). ENABLED + ADDRESS resolve via + # global.auth (globalValue: configmap. > global.auth. > default) so + # the env-wide auth endpoint is shared across services. + # ===================================================================== + PLUGIN_AUTH_ENABLED: {{ include "lerian-common.globalValue" (dict "context" $ "configmap" $cm "block" "auth" "field" "enabled" "nativeKey" "PLUGIN_AUTH_ENABLED" "default" "true") | quote }} + PLUGIN_AUTH_ADDRESS: {{ include "lerian-common.globalValue" (dict "context" $ "configmap" $cm "block" "auth" "field" "address" "nativeKey" "PLUGIN_AUTH_ADDRESS" "default" "http://plugin-access-manager-auth:4000") | quote }} + + # ===================================================================== + # OBSERVABILITY (OTEL_*). OTEL_EXPORTER_OTLP_ENDPOINT is overridden per-pod with + # $(HOST_IP):4317 via the downward API when streamingHub.telemetry.enabled=true + # (a CHART-level toggle — not an app env var; see templates/_deployment.tpl). + # OTEL_LIBRARY_NAME / OTEL_RESOURCE_* are lib-observability identity passthrough. + # ===================================================================== + OTEL_LIBRARY_NAME: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "OTEL_LIBRARY_NAME" "params" $obs "field" "libraryName" "default" "github.com/LerianStudio/streaming-hub") | quote }} + OTEL_RESOURCE_SERVICE_NAME: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "OTEL_RESOURCE_SERVICE_NAME" "params" $obs "field" "serviceName" "default" "streaming-hub") | quote }} + OTEL_RESOURCE_DEPLOYMENT_ENVIRONMENT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "OTEL_RESOURCE_DEPLOYMENT_ENVIRONMENT" "params" $obs "field" "deploymentEnvironment" "default" "production") | quote }} + OTEL_EXPORTER_OTLP_ENDPOINT: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "OTEL_EXPORTER_OTLP_ENDPOINT" "params" $obs "field" "exporterOtlpEndpoint" "default" "") | quote }} + OTEL_INSECURE_EXPORTER: {{ include "lerian-common.cfgValue" (dict "configmap" $cm "nativeKey" "OTEL_INSECURE_EXPORTER" "params" $obs "field" "insecureExporter" "default" "false") | quote }} + + # ===================================================================== + # EXTRA OVERRIDES (escape hatch for optional knobs not modeled above) + # ===================================================================== + {{- range $key, $value := .Values.streamingHub.extraEnvVars }} {{ $key }}: {{ $value | quote }} {{- end }} diff --git a/charts/streaming-hub/values.schema.json b/charts/streaming-hub/values.schema.json index 7d0c17a0e..0e2c7e236 100644 --- a/charts/streaming-hub/values.schema.json +++ b/charts/streaming-hub/values.schema.json @@ -1,16 +1,22 @@ { "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Values schema (generated by productize-chart-env/gen-schema.py)", "type": "object", - "additionalProperties": true, "properties": { "nameOverride": { - "type": "string" + "type": "string", + "default": "", + "description": "Override the chart name component of resource names." }, "fullnameOverride": { - "type": "string" + "type": "string", + "default": "", + "description": "Override the fully-qualified release name (wins verbatim)." }, "namespaceOverride": { - "type": "string" + "type": "string", + "default": "", + "description": "Override the namespace (defaults to .Release.Namespace)." }, "global": { "type": "object", @@ -18,26 +24,1077 @@ }, "streamingHub": { "type": "object", - "additionalProperties": true, "properties": { "mode": { "type": "string", - "enum": ["all", "split"], - "description": "Topology switch. 'all' = one Deployment (role=all); 'split' = ingest + delivery. NEVER run both against one Kafka cluster (double-consume)." + "default": "all", + "description": "Deployment topology switch. One of: all | split.", + "enum": [ + "all", + "split" + ] + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "default": "ghcr.io/lerianstudio/streaming-hub", + "description": "Container image repository." + }, + "pullPolicy": { + "type": "string", + "default": "IfNotPresent", + "description": "Image pull policy." + }, + "tag": { + "type": "string", + "default": "", + "description": "Image tag. Empty falls back to Chart.appVersion via the defaultTag helper." + } + }, + "additionalProperties": false + }, + "imagePullSecrets": { + "type": "array", + "description": "Secrets for pulling the image from a private registry." + }, + "revisionHistoryLimit": { + "type": "integer", + "default": 10, + "description": "Number of old ReplicaSets to retain for rollback." + }, + "annotations": { + "type": "object", + "additionalProperties": true, + "description": "Annotations applied to every Deployment resource." + }, + "podAnnotations": { + "type": "object", + "additionalProperties": true, + "description": "Annotations applied to every pod." + }, + "deploymentStrategy": { + "type": "object", + "description": "Deployment update strategy (shared by all roles).", + "properties": { + "type": { + "type": "string", + "default": "RollingUpdate" + }, + "rollingUpdate": { + "type": "object", + "properties": { + "maxSurge": { + "type": "integer", + "default": 1 + }, + "maxUnavailable": { + "type": "integer", + "default": 0 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "podSecurityContext": { + "type": "object", + "additionalProperties": true, + "description": "Pod-level security context. Empty by default (the hub needs no fsGroup)." + }, + "securityContext": { + "type": "object", + "description": "Container-level security context (distroless:nonroot, uid/gid 65532).", + "properties": { + "runAsGroup": { + "type": "integer", + "default": 65532, + "description": "Group ID for the process inside the container." + }, + "runAsUser": { + "type": "integer", + "default": 65532, + "description": "User ID for the process inside the container." + }, + "runAsNonRoot": { + "type": "boolean", + "default": true, + "description": "Never run as root." + }, + "capabilities": { + "type": "object", + "properties": { + "drop": { + "type": "array" + } + }, + "additionalProperties": false + }, + "readOnlyRootFilesystem": { + "type": "boolean", + "default": true, + "description": "Read-only root filesystem (the image carries no writable state)." + }, + "allowPrivilegeEscalation": { + "type": "boolean", + "default": false + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "RuntimeDefault" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "service": { + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "ClusterIP", + "description": "Service type. MUST be ClusterIP (Lerian convention; Ingress fronts external)." + }, + "port": { + "type": "integer", + "default": 8080, + "description": "Control-plane HTTP port (the hub listens on :8080; see Dockerfile EXPOSE)." + }, + "annotations": { + "type": "object", + "additionalProperties": true, + "description": "Annotations for every Service." + } + }, + "additionalProperties": false + }, + "ingress": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false, + "description": "Enable or disable the control-plane Ingress (opt-in per env in gitops)." + }, + "className": { + "type": "string", + "default": "nginx", + "description": "Ingress class name." + }, + "annotations": { + "type": "object", + "additionalProperties": true, + "description": "Additional ingress annotations." + }, + "hosts": { + "type": "array", + "description": "Hosts (default empty; the control API is served on every role)." + }, + "tls": { + "type": "array", + "description": "TLS configuration." + } + }, + "additionalProperties": false + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean", + "default": true, + "description": "Whether a ServiceAccount is created." + }, + "annotations": { + "type": "object", + "additionalProperties": true, + "description": "Annotations for the ServiceAccount (e.g. AWS IRSA role-arn)." + }, + "name": { + "type": "string", + "default": "", + "description": "ServiceAccount name. Empty defaults to the chart fullname." + }, + "automountServiceAccountToken": { + "type": "boolean", + "default": false, + "description": "Mount the SA API token into pods. Default false — the hub makes no" + } + }, + "additionalProperties": false + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "default": 80, + "description": "Graceful-shutdown window. Defaults to the hub's derived SIGTERM drain" + }, + "livenessProbe": { + "type": "object", + "description": "Liveness probe tuning (GET /healthz on the http port; stays 200 during drain).", + "properties": { + "initialDelaySeconds": { + "type": "integer", + "default": 15 + }, + "periodSeconds": { + "type": "integer", + "default": 20 + }, + "timeoutSeconds": { + "type": "integer", + "default": 5 + }, + "successThreshold": { + "type": "integer", + "default": 1 + }, + "failureThreshold": { + "type": "integer", + "default": 3 + } + }, + "additionalProperties": false + }, + "readinessProbe": { + "type": "object", + "description": "Readiness probe tuning (GET /readyz; flips NotReady first on SIGTERM).", + "properties": { + "initialDelaySeconds": { + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "type": "integer", + "default": 5 + }, + "successThreshold": { + "type": "integer", + "default": 1 + }, + "failureThreshold": { + "type": "integer", + "default": 3 + } + }, + "additionalProperties": false + }, + "nodeSelector": { + "type": "object", + "additionalProperties": true, + "description": "Shared default scheduling (per-role blocks may override)." + }, + "tolerations": { + "type": "object", + "additionalProperties": true + }, + "affinity": { + "type": "object", + "additionalProperties": true + }, + "extraEnvVars": { + "type": "object", + "additionalProperties": true, + "description": "Unmodeled extra env vars appended verbatim to the ConfigMap." + }, + "telemetry": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false, + "description": "Inject the per-pod OTLP endpoint override (HOST_IP downward API)." + } + }, + "additionalProperties": false + }, + "useExistingSecret": { + "type": "boolean", + "default": false + }, + "existingSecretName": { + "type": "string", + "default": "" }, "common": { "type": "object", "additionalProperties": true, + "description": "Native per-key escape hatch (highest precedence). Any UPPER_SNAKE app env" + }, + "app": { + "type": "object", + "properties": { + "env": { + "default": "production" + }, + "healthWindow": { + "default": "720h" + }, + "logLevel": { + "default": "info" + }, + "preStopDrainTimeout": { + "default": "5s" + }, + "shutdownTimeout": { + "default": "30s" + }, + "swaggerEnabled": { + "default": "false" + } + }, + "additionalProperties": false, + "description": "Application / lifecycle (STREAMING_HUB_ENV|LOG_LEVEL|HEALTH_WINDOW|" + }, + "server": { + "type": "object", "properties": { - "configmap": { - "type": "object" + "httpListenAddr": { + "default": ":8080" + }, + "metricsEnabled": { + "default": "true" } - } + }, + "additionalProperties": false, + "description": "HTTP server / metrics (STREAMING_HUB_HTTP_LISTEN_ADDR|METRICS_ENABLED)." + }, + "kafka": { + "type": "object", + "properties": { + "brokers": { + "default": "localhost:19092" + }, + "caCert": { + "default": "" + }, + "scramMechanism": { + "default": "" + }, + "scramUsername": { + "default": "" + }, + "tlsEnabled": { + "default": "false" + } + }, + "additionalProperties": false, + "description": "Kafka / Redpanda (brokers|scramMechanism|scramUsername|tlsEnabled|caCert)." + }, + "kek": { + "type": "object", + "properties": { + "ref": { + "default": "" + }, + "source": { + "default": "env" + } + }, + "additionalProperties": false, + "description": "Crypto / KEK config (source|ref). The KEK material itself is a Secret." + }, + "dispatch": { + "type": "object", + "properties": { + "claimBatch": { + "default": "16" + }, + "idleIntervalMs": { + "default": "250" + }, + "poisonThreshold": { + "default": "5" + }, + "workers": { + "default": "8" + } + }, + "additionalProperties": false, + "description": "Dispatch / poison worker pool (workers|claimBatch|idleIntervalMs|poisonThreshold)." + }, + "pull": { + "type": "object", + "properties": { + "burst": { + "default": "200" + }, + "rate": { + "default": "100" + } + }, + "additionalProperties": false, + "description": "Pull rate limit (rate|burst)." + }, + "dlq": { + "type": "object", + "properties": { + "group": { + "default": "streaming-hub-dlq.local" + }, + "pruneInterval": { + "default": "1h" + }, + "retention": { + "default": "168h" + } + }, + "additionalProperties": false, + "description": "DLQ visibility (group|retention|pruneInterval)." + }, + "manifest": { + "type": "object", + "properties": { + "refreshInterval": { + "default": "300" + }, + "sources": { + "default": "" + } + }, + "additionalProperties": false, + "description": "Event manifest sources (sources|refreshInterval)." + }, + "reconciler": { + "type": "object", + "properties": { + "enabled": { + "default": "true" + }, + "interval": { + "default": "300" + } + }, + "additionalProperties": false, + "description": "Reconciler / topic-drift (enabled|interval)." + }, + "partition": { + "type": "object", + "properties": { + "cronInterval": { + "default": "6h" + }, + "futureBufferWeeks": { + "default": "4" + }, + "retentionEnabled": { + "default": "false" + }, + "retentionHorizon": { + "default": "0" + } + }, + "additionalProperties": false, + "description": "Partition lifecycle cron (cronInterval|futureBufferWeeks|retentionEnabled|retentionHorizon)." + }, + "idempotency": { + "type": "object", + "properties": { + "reapInterval": { + "default": "5m" + }, + "ttl": { + "default": "24h" + } + }, + "additionalProperties": false, + "description": "Idempotency (ttl|reapInterval)." + }, + "autodisable": { + "type": "object", + "properties": { + "enabled": { + "default": "true" + }, + "failureSpread": { + "default": "12h" + }, + "failureWindow": { + "default": "120h" + } + }, + "additionalProperties": false, + "description": "Auto-disable subscriber circuit (enabled|failureWindow|failureSpread)." + }, + "multiTenant": { + "type": "object", + "properties": { + "allowInsecureTm": { + "default": "false" + }, + "awsRegion": { + "default": "" + }, + "cacheTtl": { + "default": "12h" + }, + "cbThreshold": { + "default": "0" + }, + "cbTimeout": { + "default": "30s" + }, + "enabled": { + "default": "false" + }, + "environmentName": { + "default": "" + }, + "libEnabled": { + "default": "false" + }, + "listenerStalenessThreshold": { + "default": "12h" + }, + "reconcileInterval": { + "default": "6h" + }, + "redisCaCert": { + "default": "" + }, + "redisHost": { + "default": "" + }, + "redisPort": { + "default": "" + }, + "redisTls": { + "default": "false" + }, + "secretsEnv": { + "default": "" + }, + "tenantId": { + "default": "default" + }, + "tenantManagerUrl": { + "default": "" + }, + "tmStalenessThreshold": { + "default": "12h" + } + }, + "additionalProperties": false, + "description": "Tenancy (F4). libEnabled=MULTI_TENANT_ENABLED (lib-commons request tenancy);" + }, + "security": { + "type": "object", + "properties": { + "allowInsecureDbTls": { + "default": "false" + }, + "allowInsecureKafka": { + "default": "false" + }, + "allowPrivateSinks": { + "default": "false" + } + }, + "additionalProperties": false, + "description": "Security posture (allowInsecureKafka|allowInsecureDbTls|allowPrivateSinks)." + }, + "aws": { + "type": "object", + "properties": { + "hubPrincipalArn": { + "default": "" + }, + "setupTemplateUrl": { + "default": "" + } + }, + "additionalProperties": false, + "description": "AWS SaaS setup metadata (hubPrincipalArn|setupTemplateUrl)." + }, + "observability": { + "type": "object", + "properties": { + "deploymentEnvironment": { + "default": "production" + }, + "exporterOtlpEndpoint": { + "default": "" + }, + "insecureExporter": { + "default": "false" + }, + "libraryName": { + "default": "github.com/LerianStudio/streaming-hub" + }, + "serviceName": { + "default": "streaming-hub" + } + }, + "additionalProperties": false, + "description": "Observability (OTEL_*: libraryName|serviceName|deploymentEnvironment|" }, "secrets": { - "type": "object" + "type": "object", + "additionalProperties": true + }, + "migrations": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false, + "description": "Enable or disable the migrations Job. Default false — opt-in per env" + }, + "useExistingSecret": { + "type": "boolean", + "default": false, + "description": "Optional. When true, the Job reads STREAMING_HUB_POSTGRES_DSN from a" + }, + "existingSecretName": { + "type": "string", + "default": "", + "description": "Name of the pre-existing Secret holding STREAMING_HUB_POSTGRES_DSN" + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "default": "ghcr.io/lerianstudio/streaming-hub-migrations", + "description": "Migrations image (FROM migrate/migrate + COPY migrations/ /migrations/)." + }, + "tag": { + "type": "string", + "default": "", + "description": "Tag for the migrations image. Empty falls back to the app image tag" + }, + "digest": { + "type": "string", + "default": "", + "description": "Pin by digest (sha256:...) instead of tag. Wins over tag when set." + }, + "pullPolicy": { + "type": "string", + "default": "IfNotPresent", + "description": "Image pull policy." + } + }, + "additionalProperties": false + }, + "backoffLimit": { + "type": "integer", + "default": 3, + "description": "Maximum retries before the Job is considered failed." + }, + "activeDeadlineSeconds": { + "type": "integer", + "default": 600, + "description": "Hard wall-clock cap on the Job (seconds)." + }, + "ttlSecondsAfterFinished": { + "type": "integer", + "default": 600, + "description": "TTL after which a finished Job is garbage-collected (seconds)." + }, + "annotations": { + "type": "object", + "additionalProperties": true, + "description": "Extra annotations on the Job (merged after the hook annotations)." + }, + "podAnnotations": { + "type": "object", + "additionalProperties": true, + "description": "Extra annotations on the migration pod." + }, + "resources": { + "type": "object", + "description": "Resource requests/limits for the migration container.", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "250m" + }, + "memory": { + "type": "string", + "default": "256Mi" + } + }, + "additionalProperties": false + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "50m" + }, + "memory": { + "type": "string", + "default": "64Mi" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "all": { + "type": "object", + "properties": { + "replicaCount": { + "type": "integer", + "default": 1 + }, + "poolMaxOpenConns": { + "type": "integer", + "default": 25 + }, + "poolMaxIdleConns": { + "type": "integer", + "default": 12 + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "500m" + }, + "memory": { + "type": "string", + "default": "512Mi" + } + }, + "additionalProperties": false + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "100m" + }, + "memory": { + "type": "string", + "default": "128Mi" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "autoscaling": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false, + "description": "HPA off by default; replicaCount governs. maxReplicas × poolMaxOpenConns" + }, + "minReplicas": { + "type": "integer", + "default": 1 + }, + "maxReplicas": { + "type": "integer", + "default": 3 + }, + "targetCPUUtilizationPercentage": { + "type": "integer", + "default": 80 + }, + "targetMemoryUtilizationPercentage": { + "type": "integer", + "default": 80 + } + }, + "additionalProperties": false + }, + "pdb": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "minAvailable": { + "type": "integer", + "default": 1 + }, + "annotations": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "nodeSelector": { + "type": "object", + "additionalProperties": true + }, + "tolerations": { + "type": "object", + "additionalProperties": true + }, + "affinity": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "ingest": { + "type": "object", + "properties": { + "replicaCount": { + "type": "integer", + "default": 1 + }, + "poolMaxOpenConns": { + "type": "integer", + "default": 8 + }, + "poolMaxIdleConns": { + "type": "integer", + "default": 4 + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "500m" + }, + "memory": { + "type": "string", + "default": "512Mi" + } + }, + "additionalProperties": false + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "100m" + }, + "memory": { + "type": "string", + "default": "128Mi" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "autoscaling": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "minReplicas": { + "type": "integer", + "default": 1 + }, + "maxReplicas": { + "type": "integer", + "default": 4, + "description": "maxReplicas × 8 (poolMaxOpenConns) must fit the connection budget." + }, + "targetCPUUtilizationPercentage": { + "type": "integer", + "default": 80 + }, + "targetMemoryUtilizationPercentage": { + "type": "integer", + "default": 80 + } + }, + "additionalProperties": false + }, + "pdb": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "minAvailable": { + "type": "integer", + "default": 1 + }, + "annotations": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "nodeSelector": { + "type": "object", + "additionalProperties": true + }, + "tolerations": { + "type": "object", + "additionalProperties": true + }, + "affinity": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "delivery": { + "type": "object", + "properties": { + "replicaCount": { + "type": "integer", + "default": 1 + }, + "poolMaxOpenConns": { + "type": "integer", + "default": 16 + }, + "poolMaxIdleConns": { + "type": "integer", + "default": 10 + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "500m" + }, + "memory": { + "type": "string", + "default": "512Mi" + } + }, + "additionalProperties": false + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "100m" + }, + "memory": { + "type": "string", + "default": "128Mi" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "autoscaling": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "minReplicas": { + "type": "integer", + "default": 1 + }, + "maxReplicas": { + "type": "integer", + "default": 4, + "description": "maxReplicas × 16 (poolMaxOpenConns) must fit the connection budget." + }, + "targetCPUUtilizationPercentage": { + "type": "integer", + "default": 80 + }, + "targetMemoryUtilizationPercentage": { + "type": "integer", + "default": 80 + } + }, + "additionalProperties": false + }, + "pdb": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "minAvailable": { + "type": "integer", + "default": 1 + }, + "annotations": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "nodeSelector": { + "type": "object", + "additionalProperties": true + }, + "tolerations": { + "type": "object", + "additionalProperties": true + }, + "affinity": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "streaming-hub": { + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "default": "1.0.1" + } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": true } diff --git a/charts/streaming-hub/values.yaml b/charts/streaming-hub/values.yaml index 6921ebd0f..ccada005a 100644 --- a/charts/streaming-hub/values.yaml +++ b/charts/streaming-hub/values.yaml @@ -49,11 +49,19 @@ global: name: "" # -- Password for the hub role (ignored if useExistingSecret.name is set). password: "" + # -- Env-wide inbound auth (lib-auth / plugin-auth), consumed by + # lerian-common.globalValue. Declare once at the umbrella level; a component + # streamingHub.common.configmap.PLUGIN_AUTH_* still overrides per-service. + # Precedence: common.configmap. > global.auth. > chart default. + # Leave empty ({}) to keep the chart defaults. + auth: {} + # enabled: true # PLUGIN_AUTH_ENABLED + # address: "http://plugin-access-manager-auth:4000" # PLUGIN_AUTH_ADDRESS # ============================================================================= # streamingHub — the single root key for everything the hub owns. # ============================================================================= streamingHub: - # -- Deployment topology switch. One of: all | split. + # -- (enum: all|split) Deployment topology switch. One of: all | split. # all (default) -> ONE Deployment with STREAMING_HUB_ROLE=all (ingest + # delivery co-resident; the dev-st target). Byte-equivalent to the # historical single binary. @@ -190,78 +198,64 @@ streamingHub: useExistingSecret: false existingSecretName: "" # =========================================================================== - # common.configmap — role-INVARIANT, NON-SENSITIVE env (one shared ConfigMap). - # Every key MUST exist verbatim in streaming-hub/.env.reference, EXCEPT the - # OTEL_* keys (lib-observability vars, not read by LoadConfig). + # Configuration API (productized onto lerian-common). + # + # Every non-sensitive key is emitted by templates/configmap.yaml via + # lerian-common.cfgValue with precedence: + # common.configmap. (escape hatch) > streamingHub.. > chart default. + # Chart defaults live in the TEMPLATE, so the grouped blocks below ship EMPTY + # ({}) — set a field only to override. common.configmap stays as the per-key + # native escape hatch; streamingHub.extraEnvVars carries unmodeled keys. # # DELIBERATELY ABSENT (injected per-Deployment, see the all/ingest/delivery # blocks below): STREAMING_HUB_ROLE, STREAMING_HUB_POSTGRES_MAX_OPEN_CONNS, - # STREAMING_HUB_POSTGRES_MAX_IDLE_CONNS. + # STREAMING_HUB_POSTGRES_MAX_IDLE_CONNS, STREAMING_HUB_POSTGRES_CONN_MAX_LIFETIME. # =========================================================================== + # -- Native per-key escape hatch (highest precedence). Any UPPER_SNAKE app env + # var can be pinned here verbatim, overriding the grouped field + default. common: - configmap: - # --- core --- - # Deployment environment: local | staging | production. - STREAMING_HUB_ENV: "production" - STREAMING_HUB_LOG_LEVEL: "info" - STREAMING_HUB_HTTP_LISTEN_ADDR: ":8080" - STREAMING_HUB_METRICS_ENABLED: "true" - # --- kafka / redpanda (external) --- - STREAMING_HUB_KAFKA_BROKERS: "localhost:19092" - STREAMING_HUB_KAFKA_SCRAM_MECHANISM: "" - # Plaintext Kafka opt-in. MUST stay "false" for staging/production - # (rejected at boot when STREAMING_HUB_ENV=production). - STREAMING_HUB_ALLOW_INSECURE_KAFKA: "false" - # --- crypto / KEK (the material itself is a Secret) --- - # env | secretsmanager. KEK material arrives via the var named by KEK_REF. - STREAMING_HUB_KEK_SOURCE: "env" - # --- auth (lib-auth / plugin-auth JWT; PLUGIN_AUTH_ prefix, NOT STREAMING_HUB_) --- - PLUGIN_AUTH_ENABLED: "true" - PLUGIN_AUTH_ADDRESS: "http://plugin-access-manager-auth:4000" - # --- tenancy (F4) --- - # default = BYOC single-tenant. A non-default value quarantines all - # other ce-tenantid events (zero-delivery risk) — confirm the producer. - STREAMING_HUB_TENANT_ID: "default" - # SaaS roster off by default (BYOC). When "true" the SaaS-only knobs below - # become required-when-SaaS (and several SECRET counterparts in secrets:). - STREAMING_HUB_MULTI_TENANT_ENABLED: "false" - STREAMING_HUB_TENANT_MANAGER_URL: "" - STREAMING_HUB_MULTI_TENANT_ALLOW_INSECURE_TM: "false" - STREAMING_HUB_MULTI_TENANT_TM_STALENESS_THRESHOLD: "12h" - STREAMING_HUB_MULTI_TENANT_LISTENER_STALENESS_THRESHOLD: "12h" - STREAMING_HUB_MULTI_TENANT_RECONCILE_INTERVAL: "6h" - STREAMING_HUB_MULTI_TENANT_CACHE_TTL: "12h" - STREAMING_HUB_TENANT_MANAGER_CB_THRESHOLD: "0" - STREAMING_HUB_TENANT_MANAGER_CB_TIMEOUT: "30s" - STREAMING_HUB_MULTI_TENANT_REDIS_HOST: "" - STREAMING_HUB_MULTI_TENANT_REDIS_PORT: "" - STREAMING_HUB_MULTI_TENANT_REDIS_TLS: "false" - STREAMING_HUB_MULTI_TENANT_REDIS_CA_CERT: "" - # lib-commons env name (NOT STREAMING_HUB_-prefixed). Required-in-SaaS: - # staging | production. Inert under BYOC. Leave empty unless SaaS. - ENVIRONMENT_NAME: "" - # --- reconciler (topic-drift detection; dark-ship, defaults on) --- - STREAMING_HUB_RECONCILER_ENABLED: "true" - STREAMING_HUB_RECONCILER_INTERVAL: "300" - # --- partition lifecycle cron --- - STREAMING_HUB_PARTITION_CRON_INTERVAL: "6h" - STREAMING_HUB_PARTITION_FUTURE_BUFFER_WEEKS: "4" - STREAMING_HUB_PARTITION_RETENTION_ENABLED: "false" - STREAMING_HUB_PARTITION_RETENTION_HORIZON: "0" - # --- DLQ visibility --- - STREAMING_HUB_DLQ_RETENTION: "168h" - STREAMING_HUB_DLQ_PRUNE_INTERVAL: "1h" - # --- lifecycle / drain (kept coherent with terminationGracePeriodSeconds) --- - STREAMING_HUB_SHUTDOWN_TIMEOUT: "30s" - STREAMING_HUB_PRE_STOP_DRAIN_TIMEOUT: "5s" - # --- observability (OTEL_* are lib-observability vars, NOT in .env.reference) --- - # OTEL_EXPORTER_OTLP_ENDPOINT is overridden per-pod with $(HOST_IP):4317 via - # the downward API when streamingHub.telemetry.enabled=true (a CHART-level - # toggle, see above — not an app env var; see templates/_deployment.tpl). - OTEL_LIBRARY_NAME: "github.com/LerianStudio/streaming-hub" - OTEL_RESOURCE_SERVICE_NAME: "streaming-hub" - OTEL_RESOURCE_DEPLOYMENT_ENVIRONMENT: "production" - OTEL_EXPORTER_OTLP_ENDPOINT: "" + configmap: {} + # -- Unmodeled extra env vars appended verbatim to the ConfigMap. + extraEnvVars: {} + # -- Application / lifecycle (STREAMING_HUB_ENV|LOG_LEVEL|HEALTH_WINDOW| + # SWAGGER_ENABLED|SHUTDOWN_TIMEOUT|PRE_STOP_DRAIN_TIMEOUT). + app: {} + # -- HTTP server / metrics (STREAMING_HUB_HTTP_LISTEN_ADDR|METRICS_ENABLED). + server: {} + # -- Kafka / Redpanda (brokers|scramMechanism|scramUsername|tlsEnabled|caCert). + # The SCRAM password is a Secret (see secrets below). + kafka: {} + # -- Crypto / KEK config (source|ref). The KEK material itself is a Secret. + kek: {} + # -- Dispatch / poison worker pool (workers|claimBatch|idleIntervalMs|poisonThreshold). + dispatch: {} + # -- Pull rate limit (rate|burst). + pull: {} + # -- DLQ visibility (group|retention|pruneInterval). + dlq: {} + # -- Event manifest sources (sources|refreshInterval). + manifest: {} + # -- Reconciler / topic-drift (enabled|interval). + reconciler: {} + # -- Partition lifecycle cron (cronInterval|futureBufferWeeks|retentionEnabled|retentionHorizon). + partition: {} + # -- Idempotency (ttl|reapInterval). + idempotency: {} + # -- Auto-disable subscriber circuit (enabled|failureWindow|failureSpread). + autodisable: {} + # -- Tenancy (F4). libEnabled=MULTI_TENANT_ENABLED (lib-commons request tenancy); + # the rest is the hub SaaS tenant-manager roster (STREAMING_HUB_MULTI_TENANT_*/ + # TENANT_MANAGER_*/TENANT_ID/ENVIRONMENT_NAME). Default = BYOC single-tenant. + multiTenant: {} + # -- Security posture (allowInsecureKafka|allowInsecureDbTls|allowPrivateSinks). + # All fail-closed; MUST stay false in staging/production. + security: {} + # -- AWS SaaS setup metadata (hubPrincipalArn|setupTemplateUrl). + aws: {} + # -- Observability (OTEL_*: libraryName|serviceName|deploymentEnvironment| + # exporterOtlpEndpoint|insecureExporter). OTEL_EXPORTER_OTLP_ENDPOINT is + # overridden per-pod when telemetry.enabled=true (downward API). + observability: {} # =========================================================================== # secrets — role-INVARIANT, SENSITIVE env (one shared Secret). All default "". # Empty values are SKIPPED at render so unset SaaS/dev keys never ship blank. @@ -271,12 +265,11 @@ streamingHub: secrets: # Single hub-owned Postgres DSN (password embedded -> whole DSN is secret). STREAMING_HUB_POSTGRES_DSN: "" - # SASL/SCRAM credentials (required when KAFKA_SCRAM_MECHANISM is set). - STREAMING_HUB_KAFKA_SCRAM_USERNAME: "" + # SASL/SCRAM password (required when KAFKA_SCRAM_MECHANISM is set). The + # username + mechanism are non-sensitive config (streamingHub.kafka.*). STREAMING_HUB_KAFKA_SCRAM_PASSWORD: "" - # KEK reference: the NAME of the env var holding the KEK material. - STREAMING_HUB_KEK_REF: "" # DEV-ONLY KEK material holder (local BYOC). Leave empty in real deploys. + # (KEK_SOURCE/KEK_REF are non-sensitive config — streamingHub.kek.*.) STREAMING_HUB_DEV_KEK: "" # SaaS tenant-manager X-API-Key (required-when-SaaS). STREAMING_HUB_TENANT_MANAGER_SERVICE_API_KEY: ""