Replace DuckLake with Hoglake in Trino perf tests

.github/workflows/scenario-dev.yml

@@ -77,7 +77,7 @@ jobs:
       # Frozen perf uses a namespace-local Trino cell. The dedicated role is
       # consumed only when E2E_SUITE selects that isolated deployment.
       TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }}
-      TRINO_IMAGE: ghcr.io/posthog/trino:59184e0c58c2fdbde031f3a08787c926dc4d7f69@sha256:a7072f7c86fcbb815c2502bb455b27bc4679d201ad6ce81725378a7c5ce84325
+      TRINO_IMAGE: ghcr.io/posthog/trino:a2943f5ec37f1d5a9ab90b9bec56695a00de4584@sha256:7a57712498446bd97393cadece90ce0bc297f6feab8e0098510668fe2667ac39
       E2E_SUITE: ${{ (matrix.scenario == 'posthog_frozen_perf' || matrix.scenario == 'posthog_frozen_perf_trino_cached') && 'trino' || 'neutral' }}
       PR_NUMBER: ${{ github.run_id }}${{ strategy.job-index }}
       NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ strategy.job-index }}
@@ -128,8 +128,8 @@ jobs:
       - name: Update kubeconfig
         run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT"
 
-      - name: Load Athena perf configuration
-        if: env.SCENARIO_NAME == 'posthog_frozen_perf'
+      - name: Load frozen perf identity and Athena configuration
+        if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_trino_cached'
         run: bash scripts/scenario_athena_config.sh >> "$GITHUB_ENV"
 
       - name: Deploy isolated Duckgres stack

README.md

@@ -65,6 +65,11 @@ A PostgreSQL wire protocol compatible server backed by DuckDB. Connect with any
 
 ## Trino API identity
 
+`DUCKGRES_TRINO_HOGLAKE_URI` defaults to empty (DuckLake catalog provisioning).
+The frozen perf deployments set it automatically to their namespace-local Hoglake
+service, replacing the Trino backend in the existing cached and uncached scenarios.
+See the [scenario runbook](docs/runbooks/scenario-runner.md).
+
 The existing Trino deployment appears as `legacy` in the Trino console API.
 This name does not change its stored org assignments or catalog-store key.
 `DUCKGRES_TRINO_CELL_ID` remains the ownership setting, with the existing default

controlplane/provisioner/trino_provisioner.go

@@ -327,6 +327,10 @@ type TrinoProvisionerOpts struct {
 	// FilesystemCacheEnabled enables the node-local filesystem cache for new
 	// catalogs. Defaults to false. Trino nodes must configure a cache manager.
 	FilesystemCacheEnabled bool
+
+	// HoglakeURI selects Hoglake catalogs using the pod's S3 credentials.
+	// Empty retains the default DuckLake catalog configuration.
+	HoglakeURI string
 }
 
 // TrinoBootstrapSentinelStore is the narrow configstore surface the
@@ -414,6 +418,7 @@ type TrinoProvisioner struct {
 	awsRegion               string
 	s3MaxConnections        int
 	filesystemCacheEnabled  bool
+	hoglakeURI              string
 
 	// adminPasswordHash is cached on each Reconcile from the
 	// trino-auth K8s Secret and prepended to password.db on projection.
@@ -536,6 +541,7 @@ func NewTrinoProvisioner(opts TrinoProvisionerOpts) (*TrinoProvisioner, error) {
 		awsRegion:               opts.AWSRegion,
 		s3MaxConnections:        maxConns,
 		filesystemCacheEnabled:  opts.FilesystemCacheEnabled,
+		hoglakeURI:              opts.HoglakeURI,
 	}, nil
 }
 
@@ -1725,6 +1731,15 @@ func (p *TrinoProvisioner) buildCatalogProperties(orgID string, w *configstore.M
 	if region == "" {
 		region = p.awsRegion
 	}
+	if p.hoglakeURI != "" {
+		return map[string]string{
+			"connector.name":    "hoglake",
+			"hoglake.uri":       p.hoglakeURI,
+			"hoglake.catalog":   orgID,
+			"fs.cache.enabled":  strconv.FormatBool(p.filesystemCacheEnabled),
+			"hoglake.s3.region": region,
+		}
+	}
 	return map[string]string{
 		"connector.name":                    "ducklake",
 		"ducklake.metadata.connection-url":  ducklakeMetadataJDBCURL(d),

controlplane/provisioner/trino_provisioner_test.go

@@ -1897,9 +1897,34 @@ func TestCatalogFilesystemCacheSetting(t *testing.T) {
 				t.Fatal(err)
 			}
 			props := p.buildCatalogProperties("42", readyWarehouse("42"), readyDuckling("42"))
+			if props["connector.name"] != "ducklake" || props["ducklake.metadata.connection-url"] == "" || props["hoglake.uri"] != "" {
+				t.Fatalf("default catalog must remain DuckLake: %v", props)
+			}
 			if got, want := props["fs.cache.enabled"], strconv.FormatBool(enabled); got != want {
 				t.Fatalf("fs.cache.enabled = %q, want %q", got, want)
 			}
 		})
 	}
 }
+
+func TestTrinoHoglakeCatalogProperties(t *testing.T) {
+	for _, enabled := range []bool{false, true} {
+		t.Run(strconv.FormatBool(enabled), func(t *testing.T) {
+			opts := baseTestOpts()
+			opts.HoglakeURI = "http://hoglake:8080"
+			opts.FilesystemCacheEnabled = enabled
+			p, err := NewTrinoProvisioner(opts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			d := readyDuckling("42")
+			want := map[string]string{
+				"connector.name": "hoglake", "hoglake.uri": opts.HoglakeURI, "hoglake.catalog": "42",
+				"fs.cache.enabled": strconv.FormatBool(enabled), "hoglake.s3.region": d.DataStore.S3Region,
+			}
+			if got := p.buildCatalogProperties("42", readyWarehouse("42"), d); !reflect.DeepEqual(got, want) {
+				t.Fatalf("catalog properties = %v, want %v", got, want)
+			}
+		})
+	}
+}

controlplane/trino_inputs.go

@@ -5,6 +5,7 @@ package controlplane
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -109,6 +110,9 @@ const (
 	// envTrinoFilesystemCacheEnabled enables caching for newly created catalogs.
 	// Empty defaults to false; Trino nodes also need a configured cache manager.
 	envTrinoFilesystemCacheEnabled = "DUCKGRES_TRINO_FILESYSTEM_CACHE_ENABLED"
+
+	// envTrinoHoglakeURI selects Hoglake catalogs when set. Empty preserves DuckLake.
+	envTrinoHoglakeURI = "DUCKGRES_TRINO_HOGLAKE_URI"
 )
 
 // trinoProvisionerEnabled recognizes legacy or registry configuration.
@@ -273,6 +277,11 @@ func buildTrinoCellWiring(store trinoWiringStore, kc kubernetes.Interface, duckl
 		return nil, err
 	}
 
+	hoglakeURI, err := trinoHoglakeURI()
+	if err != nil {
+		return nil, err
+	}
+
 	if ducklings == nil {
 		// Without it every catalog sits pending forever waiting on a
 		// duckling status that nothing resolves — a silent, permanent
@@ -316,6 +325,7 @@ func buildTrinoCellWiring(store trinoWiringStore, kc kubernetes.Interface, duckl
 		AWSRegion:               strings.TrimSpace(os.Getenv(envTrinoAWSRegion)),
 		S3MaxConnections:        envInt(envTrinoS3MaxConnections),
 		FilesystemCacheEnabled:  filesystemCacheEnabled,
+		HoglakeURI:              hoglakeURI,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("construct Trino provisioner: %w", err)
@@ -413,3 +423,15 @@ func trinoFilesystemCacheEnabled() (bool, error) {
 	}
 	return enabled, nil
 }
+
+func trinoHoglakeURI() (string, error) {
+	value := strings.TrimSpace(os.Getenv(envTrinoHoglakeURI))
+	if value == "" {
+		return "", nil
+	}
+	parsed, err := url.Parse(value)
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" {
+		return "", fmt.Errorf("%s must be an HTTP(S) URL without credentials, query, or fragment", envTrinoHoglakeURI)
+	}
+	return value, nil
+}

controlplane/trino_inputs_test.go

@@ -42,3 +42,25 @@ func TestBuildTrinoWiringRejectsInvalidFilesystemCacheSetting(t *testing.T) {
 		t.Fatalf("expected invalid cache setting error before constructing dependencies, got %v", err)
 	}
 }
+
+func TestTrinoHoglakeURI(t *testing.T) {
+	for _, tc := range []struct {
+		value   string
+		invalid bool
+	}{
+		{"", false}, {"http://hoglake:8080", false}, {" https://example.com/api ", false},
+		{"hoglake:8080", true}, {"ftp://example.com", true}, {"http:///missing", true},
+		{"https://user:password@example.com", true}, {"https://example.com?token=x", true}, {"https://example.com#fragment", true},
+	} {
+		t.Run(tc.value, func(t *testing.T) {
+			t.Setenv(envTrinoHoglakeURI, tc.value)
+			got, err := trinoHoglakeURI()
+			if (err != nil) != tc.invalid {
+				t.Fatalf("URI %q: error = %v, want invalid %v", tc.value, err, tc.invalid)
+			}
+			if !tc.invalid && got != strings.TrimSpace(tc.value) {
+				t.Fatalf("URI = %q", got)
+			}
+		})
+	}
+}

docs/runbooks/scenario-runner.md

@@ -6,15 +6,19 @@ The scenario runner executes end-to-end managed-warehouse flows against a config
 
 The default full workload uses one `full-suite.yaml` scenario: it provisions a fresh dev warehouse, creates read-only views over frozen persons/events parquet supplied by `DUCKGRES_SCENARIO_FROZEN_S3_URI`, runs metadata exploration, perf queries, and dbt models, then deprovisions. `fast-suite.yaml` follows the same flow without dbt. The standalone provisioning, frozen metadata, perf, and dbt scenarios remain available for focused debugging.
 
-The targeted frozen-perf scenario additionally creates production-shaped DuckLake
-tables, `posthog.events` and `posthog.persons`, from those raw views. It uses the
-PostHog backfill schema pinned at `056583335dc739b9e025efede811c9b4f5e153f5`,
-rewritten inserts, and `year(timestamp), month(timestamp), day(timestamp)` /
-`year(_timestamp), month(_timestamp)` partitioning. The raw views are deliberately
-retained as the later paired-query performance-control target.
-
-`project_id` is the one mapping exception: it is derived from `team_id`, exactly
-as the pinned production exporter does, so no `project_id` fixture column is needed.
+The existing frozen-perf scenarios compare Duckgres raw views and DuckLake tables,
+Trino Hoglake tables, and Athena external tables over the same immutable S3 files.
+`posthog_frozen_perf` runs `trino` with `fs.cache.enabled=false`;
+`posthog_frozen_perf_trino_cached` runs `trino_cached` with it set to `true`.
+The pinned Hoglake connector currently ignores that flag; enabling actual cache
+support is deferred. Both modes run the full seven-query corpus, with one warmup
+and four measured iterations.
+
+The scenario deployment creates a namespace-local Hoglake server and metadata
+PostgreSQL database. Setup registers the frozen `events` and `persons` Parquet
+files in the org's Hoglake catalog without copying or rewriting S3 data. Namespace
+teardown removes the server and metadata database. Duckgres continues to use
+DuckLake for its PGWire measurements.
 
 ## Required Environment
 
@@ -61,7 +65,7 @@ export DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI="s3://<results-bucket>/<prefix>/"
 ```
 
 The full and fast suites exercise PGWire only. The targeted frozen perf
-scenario compares PGWire, Trino, and on-demand Athena. It records per-query
+scenario compares PGWire on DuckLake, Trino on Hoglake, and on-demand Athena. It records per-query
 success and failure rows in `query_results.csv` and Athena service details in
 `query_service_metrics.csv`.
 Measured query errors fail the perf DAG step after its artifacts are written;
@@ -72,6 +76,14 @@ catalog's targets. Optional `with.worker_cpu` and `with.worker_memory` values
 are sent as PGWire startup options. Both default to empty, which leaves worker
 selection to the server; set both for resource-controlled comparisons.
 
+For the two frozen-perf scenarios, `tests/mw-dev/run.sh` automatically supplies
+`DUCKGRES_TRINO_HOGLAKE_URI` to the control plane and
+`DUCKGRES_SCENARIO_HOGLAKE_URI` to the runner. No catalog configuration is supplied
+by the caller. Outside these deployments, the control-plane URI defaults to empty
+and catalog provisioning continues to use DuckLake. `HOGLAKE_IMAGE` can override
+the pinned server image. The scenario image includes Python, boto3, and pyarrow
+for reading Parquet footers from S3 during setup.
+
 Do not commit concrete dev endpoints, secrets, org IDs, or private bucket names.
 
 ## Run
@@ -114,9 +126,9 @@ just scenario-frozen-perf
 
 This runs, in order: raw-view setup, source-column preflight, explicit PostHog
 table DDL, registration of the frozen Parquet files in DuckLake, then partition
-and file-metadata validation. Registration reads Parquet footers but does not
+and file-metadata validation, followed by Hoglake registration for Trino. Registration reads Parquet footers but does not
 rewrite the fixture rows, so the raw-view and DuckLake-table queries use the
-same frozen S3 objects. Validation checks the declared schema and partition
+same frozen S3 objects as Hoglake. Validation checks the declared schema and partition
 metadata plus exact source/registered file-list equality. Neither `fast-suite`
 nor `full-suite` enables these tables yet.
 

tests/mw-dev/manifests.hoglake.tmpl.yaml

@@ -0,0 +1,108 @@
+# Frozen-perf metadata only. Parquet stays in the shared, read-only S3 fixture.
+# Both deployments and the catalog PVC are removed with the scenario namespace.
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: duckgres-hoglake-postgres
+  namespace: ${NAMESPACE}
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: duckgres-hoglake-postgres
+  namespace: ${NAMESPACE}
+spec:
+  replicas: 1
+  selector:
+    matchLabels: { app: duckgres-hoglake-postgres }
+  template:
+    metadata:
+      labels: { app: duckgres-hoglake-postgres }
+      annotations:
+        karpenter.sh/do-not-disrupt: "true"
+    spec:
+      containers:
+        - name: postgres
+          image: public.ecr.aws/docker/library/postgres:16-alpine
+          env:
+            - { name: POSTGRES_USER, value: hoglake }
+            - name: POSTGRES_PASSWORD
+              valueFrom: { secretKeyRef: { name: duckgres-config-store-credentials, key: password } }
+            - { name: POSTGRES_DB, value: hoglake }
+            - { name: PGDATA, value: /var/lib/postgresql/data/pgdata }
+          ports: [{ containerPort: 5432 }]
+          resources:
+            requests: { cpu: 250m, memory: 256Mi }
+            limits: { memory: 1Gi }
+          volumeMounts: [{ name: data, mountPath: /var/lib/postgresql/data }]
+          readinessProbe:
+            exec: { command: ["pg_isready", "-U", "hoglake"] }
+            periodSeconds: 3
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: duckgres-hoglake-postgres
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: duckgres-hoglake-postgres
+  namespace: ${NAMESPACE}
+spec:
+  selector: { app: duckgres-hoglake-postgres }
+  ports: [{ port: 5432, targetPort: 5432 }]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: duckgres-hoglake
+  namespace: ${NAMESPACE}
+spec:
+  # run.sh admits the server only after its Pod Identity association exists.
+  replicas: 0
+  selector:
+    matchLabels: { app: duckgres-hoglake }
+  template:
+    metadata:
+      labels: { app: duckgres-hoglake }
+      annotations:
+        karpenter.sh/do-not-disrupt: "true"
+    spec:
+      serviceAccountName: trino
+      containers:
+        - name: hoglake
+          image: ${HOGLAKE_IMAGE}
+          env:
+            - { name: HOGLAKE_JDBC_URL, value: "jdbc:postgresql://duckgres-hoglake-postgres.${NAMESPACE}.svc:5432/hoglake" }
+            - { name: HOGLAKE_DB_USER, value: hoglake }
+            - name: HOGLAKE_DB_PASSWORD
+              valueFrom: { secretKeyRef: { name: duckgres-config-store-credentials, key: password } }
+            - { name: HOGLAKE_S3_REGION, value: "${AWS_REGION}" }
+            - { name: HOGLAKE_S3_PATH_STYLE, value: "false" }
+            - { name: HOGLAKE_COMPACTION_INTERVAL_MS, value: "0" }
+          ports: [{ containerPort: 8080 }]
+          resources:
+            requests: { cpu: 500m, memory: 1Gi }
+            limits: { memory: 2Gi }
+          startupProbe:
+            httpGet: { path: /healthz, port: 8080 }
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: { path: /healthz, port: 8080 }
+            periodSeconds: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: duckgres-hoglake
+  namespace: ${NAMESPACE}
+spec:
+  selector: { app: duckgres-hoglake }
+  ports: [{ port: 8080, targetPort: 8080 }]