diff --git a/.github/workflows/scenario-dev.yml b/.github/workflows/scenario-dev.yml index 8c0c99b4..c64e5058 100644 --- a/.github/workflows/scenario-dev.yml +++ b/.github/workflows/scenario-dev.yml @@ -77,7 +77,7 @@ jobs: # Frozen perf uses a namespace-local Trino cell. The dedicated role is # consumed only when E2E_SUITE selects that isolated deployment. TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }} - TRINO_IMAGE: ghcr.io/posthog/trino:59184e0c58c2fdbde031f3a08787c926dc4d7f69@sha256:a7072f7c86fcbb815c2502bb455b27bc4679d201ad6ce81725378a7c5ce84325 + TRINO_IMAGE: ghcr.io/posthog/trino:a2943f5ec37f1d5a9ab90b9bec56695a00de4584@sha256:7a57712498446bd97393cadece90ce0bc297f6feab8e0098510668fe2667ac39 E2E_SUITE: ${{ (matrix.scenario == 'posthog_frozen_perf' || matrix.scenario == 'posthog_frozen_perf_trino_cached') && 'trino' || 'neutral' }} PR_NUMBER: ${{ github.run_id }}${{ strategy.job-index }} NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ strategy.job-index }} @@ -128,8 +128,8 @@ jobs: - name: Update kubeconfig run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT" - - name: Load Athena perf configuration - if: env.SCENARIO_NAME == 'posthog_frozen_perf' + - name: Load frozen perf identity and Athena configuration + if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_trino_cached' run: bash scripts/scenario_athena_config.sh >> "$GITHUB_ENV" - name: Deploy isolated Duckgres stack diff --git a/README.md b/README.md index a5c3d5dd..cb4936b4 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,11 @@ A PostgreSQL wire protocol compatible server backed by DuckDB. Connect with any ## Trino API identity +`DUCKGRES_TRINO_HOGLAKE_URI` defaults to empty (DuckLake catalog provisioning). +The frozen perf deployments set it automatically to their namespace-local Hoglake +service, replacing the Trino backend in the existing cached and uncached scenarios. +See the [scenario runbook](docs/runbooks/scenario-runner.md). + The existing Trino deployment appears as `legacy` in the Trino console API. This name does not change its stored org assignments or catalog-store key. `DUCKGRES_TRINO_CELL_ID` remains the ownership setting, with the existing default diff --git a/controlplane/provisioner/trino_provisioner.go b/controlplane/provisioner/trino_provisioner.go index fe768f52..4b114b1c 100644 --- a/controlplane/provisioner/trino_provisioner.go +++ b/controlplane/provisioner/trino_provisioner.go @@ -327,6 +327,10 @@ type TrinoProvisionerOpts struct { // FilesystemCacheEnabled enables the node-local filesystem cache for new // catalogs. Defaults to false. Trino nodes must configure a cache manager. FilesystemCacheEnabled bool + + // HoglakeURI selects Hoglake catalogs using the pod's S3 credentials. + // Empty retains the default DuckLake catalog configuration. + HoglakeURI string } // TrinoBootstrapSentinelStore is the narrow configstore surface the @@ -414,6 +418,7 @@ type TrinoProvisioner struct { awsRegion string s3MaxConnections int filesystemCacheEnabled bool + hoglakeURI string // adminPasswordHash is cached on each Reconcile from the // trino-auth K8s Secret and prepended to password.db on projection. @@ -536,6 +541,7 @@ func NewTrinoProvisioner(opts TrinoProvisionerOpts) (*TrinoProvisioner, error) { awsRegion: opts.AWSRegion, s3MaxConnections: maxConns, filesystemCacheEnabled: opts.FilesystemCacheEnabled, + hoglakeURI: opts.HoglakeURI, }, nil } @@ -1725,6 +1731,15 @@ func (p *TrinoProvisioner) buildCatalogProperties(orgID string, w *configstore.M if region == "" { region = p.awsRegion } + if p.hoglakeURI != "" { + return map[string]string{ + "connector.name": "hoglake", + "hoglake.uri": p.hoglakeURI, + "hoglake.catalog": orgID, + "fs.cache.enabled": strconv.FormatBool(p.filesystemCacheEnabled), + "hoglake.s3.region": region, + } + } return map[string]string{ "connector.name": "ducklake", "ducklake.metadata.connection-url": ducklakeMetadataJDBCURL(d), diff --git a/controlplane/provisioner/trino_provisioner_test.go b/controlplane/provisioner/trino_provisioner_test.go index 4895e308..9456c7f1 100644 --- a/controlplane/provisioner/trino_provisioner_test.go +++ b/controlplane/provisioner/trino_provisioner_test.go @@ -1897,9 +1897,34 @@ func TestCatalogFilesystemCacheSetting(t *testing.T) { t.Fatal(err) } props := p.buildCatalogProperties("42", readyWarehouse("42"), readyDuckling("42")) + if props["connector.name"] != "ducklake" || props["ducklake.metadata.connection-url"] == "" || props["hoglake.uri"] != "" { + t.Fatalf("default catalog must remain DuckLake: %v", props) + } if got, want := props["fs.cache.enabled"], strconv.FormatBool(enabled); got != want { t.Fatalf("fs.cache.enabled = %q, want %q", got, want) } }) } } + +func TestTrinoHoglakeCatalogProperties(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(strconv.FormatBool(enabled), func(t *testing.T) { + opts := baseTestOpts() + opts.HoglakeURI = "http://hoglake:8080" + opts.FilesystemCacheEnabled = enabled + p, err := NewTrinoProvisioner(opts) + if err != nil { + t.Fatal(err) + } + d := readyDuckling("42") + want := map[string]string{ + "connector.name": "hoglake", "hoglake.uri": opts.HoglakeURI, "hoglake.catalog": "42", + "fs.cache.enabled": strconv.FormatBool(enabled), "hoglake.s3.region": d.DataStore.S3Region, + } + if got := p.buildCatalogProperties("42", readyWarehouse("42"), d); !reflect.DeepEqual(got, want) { + t.Fatalf("catalog properties = %v, want %v", got, want) + } + }) + } +} diff --git a/controlplane/trino_inputs.go b/controlplane/trino_inputs.go index 72b2fd0e..72940fc3 100644 --- a/controlplane/trino_inputs.go +++ b/controlplane/trino_inputs.go @@ -5,6 +5,7 @@ package controlplane import ( "context" "fmt" + "net/url" "os" "strconv" "strings" @@ -109,6 +110,9 @@ const ( // envTrinoFilesystemCacheEnabled enables caching for newly created catalogs. // Empty defaults to false; Trino nodes also need a configured cache manager. envTrinoFilesystemCacheEnabled = "DUCKGRES_TRINO_FILESYSTEM_CACHE_ENABLED" + + // envTrinoHoglakeURI selects Hoglake catalogs when set. Empty preserves DuckLake. + envTrinoHoglakeURI = "DUCKGRES_TRINO_HOGLAKE_URI" ) // trinoProvisionerEnabled recognizes legacy or registry configuration. @@ -273,6 +277,11 @@ func buildTrinoCellWiring(store trinoWiringStore, kc kubernetes.Interface, duckl return nil, err } + hoglakeURI, err := trinoHoglakeURI() + if err != nil { + return nil, err + } + if ducklings == nil { // Without it every catalog sits pending forever waiting on a // duckling status that nothing resolves — a silent, permanent @@ -316,6 +325,7 @@ func buildTrinoCellWiring(store trinoWiringStore, kc kubernetes.Interface, duckl AWSRegion: strings.TrimSpace(os.Getenv(envTrinoAWSRegion)), S3MaxConnections: envInt(envTrinoS3MaxConnections), FilesystemCacheEnabled: filesystemCacheEnabled, + HoglakeURI: hoglakeURI, }) if err != nil { return nil, fmt.Errorf("construct Trino provisioner: %w", err) @@ -413,3 +423,15 @@ func trinoFilesystemCacheEnabled() (bool, error) { } return enabled, nil } + +func trinoHoglakeURI() (string, error) { + value := strings.TrimSpace(os.Getenv(envTrinoHoglakeURI)) + if value == "" { + return "", nil + } + parsed, err := url.Parse(value) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return "", fmt.Errorf("%s must be an HTTP(S) URL without credentials, query, or fragment", envTrinoHoglakeURI) + } + return value, nil +} diff --git a/controlplane/trino_inputs_test.go b/controlplane/trino_inputs_test.go index 0912d588..89b15fc2 100644 --- a/controlplane/trino_inputs_test.go +++ b/controlplane/trino_inputs_test.go @@ -42,3 +42,25 @@ func TestBuildTrinoWiringRejectsInvalidFilesystemCacheSetting(t *testing.T) { t.Fatalf("expected invalid cache setting error before constructing dependencies, got %v", err) } } + +func TestTrinoHoglakeURI(t *testing.T) { + for _, tc := range []struct { + value string + invalid bool + }{ + {"", false}, {"http://hoglake:8080", false}, {" https://example.com/api ", false}, + {"hoglake:8080", true}, {"ftp://example.com", true}, {"http:///missing", true}, + {"https://user:password@example.com", true}, {"https://example.com?token=x", true}, {"https://example.com#fragment", true}, + } { + t.Run(tc.value, func(t *testing.T) { + t.Setenv(envTrinoHoglakeURI, tc.value) + got, err := trinoHoglakeURI() + if (err != nil) != tc.invalid { + t.Fatalf("URI %q: error = %v, want invalid %v", tc.value, err, tc.invalid) + } + if !tc.invalid && got != strings.TrimSpace(tc.value) { + t.Fatalf("URI = %q", got) + } + }) + } +} diff --git a/docs/runbooks/scenario-runner.md b/docs/runbooks/scenario-runner.md index a8480789..f0d30da6 100644 --- a/docs/runbooks/scenario-runner.md +++ b/docs/runbooks/scenario-runner.md @@ -6,15 +6,19 @@ The scenario runner executes end-to-end managed-warehouse flows against a config The default full workload uses one `full-suite.yaml` scenario: it provisions a fresh dev warehouse, creates read-only views over frozen persons/events parquet supplied by `DUCKGRES_SCENARIO_FROZEN_S3_URI`, runs metadata exploration, perf queries, and dbt models, then deprovisions. `fast-suite.yaml` follows the same flow without dbt. The standalone provisioning, frozen metadata, perf, and dbt scenarios remain available for focused debugging. -The targeted frozen-perf scenario additionally creates production-shaped DuckLake -tables, `posthog.events` and `posthog.persons`, from those raw views. It uses the -PostHog backfill schema pinned at `056583335dc739b9e025efede811c9b4f5e153f5`, -rewritten inserts, and `year(timestamp), month(timestamp), day(timestamp)` / -`year(_timestamp), month(_timestamp)` partitioning. The raw views are deliberately -retained as the later paired-query performance-control target. - -`project_id` is the one mapping exception: it is derived from `team_id`, exactly -as the pinned production exporter does, so no `project_id` fixture column is needed. +The existing frozen-perf scenarios compare Duckgres raw views and DuckLake tables, +Trino Hoglake tables, and Athena external tables over the same immutable S3 files. +`posthog_frozen_perf` runs `trino` with `fs.cache.enabled=false`; +`posthog_frozen_perf_trino_cached` runs `trino_cached` with it set to `true`. +The pinned Hoglake connector currently ignores that flag; enabling actual cache +support is deferred. Both modes run the full seven-query corpus, with one warmup +and four measured iterations. + +The scenario deployment creates a namespace-local Hoglake server and metadata +PostgreSQL database. Setup registers the frozen `events` and `persons` Parquet +files in the org's Hoglake catalog without copying or rewriting S3 data. Namespace +teardown removes the server and metadata database. Duckgres continues to use +DuckLake for its PGWire measurements. ## Required Environment @@ -61,7 +65,7 @@ export DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI="s3:////" ``` The full and fast suites exercise PGWire only. The targeted frozen perf -scenario compares PGWire, Trino, and on-demand Athena. It records per-query +scenario compares PGWire on DuckLake, Trino on Hoglake, and on-demand Athena. It records per-query success and failure rows in `query_results.csv` and Athena service details in `query_service_metrics.csv`. Measured query errors fail the perf DAG step after its artifacts are written; @@ -72,6 +76,14 @@ catalog's targets. Optional `with.worker_cpu` and `with.worker_memory` values are sent as PGWire startup options. Both default to empty, which leaves worker selection to the server; set both for resource-controlled comparisons. +For the two frozen-perf scenarios, `tests/mw-dev/run.sh` automatically supplies +`DUCKGRES_TRINO_HOGLAKE_URI` to the control plane and +`DUCKGRES_SCENARIO_HOGLAKE_URI` to the runner. No catalog configuration is supplied +by the caller. Outside these deployments, the control-plane URI defaults to empty +and catalog provisioning continues to use DuckLake. `HOGLAKE_IMAGE` can override +the pinned server image. The scenario image includes Python, boto3, and pyarrow +for reading Parquet footers from S3 during setup. + Do not commit concrete dev endpoints, secrets, org IDs, or private bucket names. ## Run @@ -114,9 +126,9 @@ just scenario-frozen-perf This runs, in order: raw-view setup, source-column preflight, explicit PostHog table DDL, registration of the frozen Parquet files in DuckLake, then partition -and file-metadata validation. Registration reads Parquet footers but does not +and file-metadata validation, followed by Hoglake registration for Trino. Registration reads Parquet footers but does not rewrite the fixture rows, so the raw-view and DuckLake-table queries use the -same frozen S3 objects. Validation checks the declared schema and partition +same frozen S3 objects as Hoglake. Validation checks the declared schema and partition metadata plus exact source/registered file-list equality. Neither `fast-suite` nor `full-suite` enables these tables yet. diff --git a/tests/mw-dev/manifests.hoglake.tmpl.yaml b/tests/mw-dev/manifests.hoglake.tmpl.yaml new file mode 100644 index 00000000..fb453e45 --- /dev/null +++ b/tests/mw-dev/manifests.hoglake.tmpl.yaml @@ -0,0 +1,108 @@ +# Frozen-perf metadata only. Parquet stays in the shared, read-only S3 fixture. +# Both deployments and the catalog PVC are removed with the scenario namespace. +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: duckgres-hoglake-postgres + namespace: ${NAMESPACE} +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: duckgres-hoglake-postgres + namespace: ${NAMESPACE} +spec: + replicas: 1 + selector: + matchLabels: { app: duckgres-hoglake-postgres } + template: + metadata: + labels: { app: duckgres-hoglake-postgres } + annotations: + karpenter.sh/do-not-disrupt: "true" + spec: + containers: + - name: postgres + image: public.ecr.aws/docker/library/postgres:16-alpine + env: + - { name: POSTGRES_USER, value: hoglake } + - name: POSTGRES_PASSWORD + valueFrom: { secretKeyRef: { name: duckgres-config-store-credentials, key: password } } + - { name: POSTGRES_DB, value: hoglake } + - { name: PGDATA, value: /var/lib/postgresql/data/pgdata } + ports: [{ containerPort: 5432 }] + resources: + requests: { cpu: 250m, memory: 256Mi } + limits: { memory: 1Gi } + volumeMounts: [{ name: data, mountPath: /var/lib/postgresql/data }] + readinessProbe: + exec: { command: ["pg_isready", "-U", "hoglake"] } + periodSeconds: 3 + volumes: + - name: data + persistentVolumeClaim: + claimName: duckgres-hoglake-postgres +--- +apiVersion: v1 +kind: Service +metadata: + name: duckgres-hoglake-postgres + namespace: ${NAMESPACE} +spec: + selector: { app: duckgres-hoglake-postgres } + ports: [{ port: 5432, targetPort: 5432 }] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: duckgres-hoglake + namespace: ${NAMESPACE} +spec: + # run.sh admits the server only after its Pod Identity association exists. + replicas: 0 + selector: + matchLabels: { app: duckgres-hoglake } + template: + metadata: + labels: { app: duckgres-hoglake } + annotations: + karpenter.sh/do-not-disrupt: "true" + spec: + serviceAccountName: trino + containers: + - name: hoglake + image: ${HOGLAKE_IMAGE} + env: + - { name: HOGLAKE_JDBC_URL, value: "jdbc:postgresql://duckgres-hoglake-postgres.${NAMESPACE}.svc:5432/hoglake" } + - { name: HOGLAKE_DB_USER, value: hoglake } + - name: HOGLAKE_DB_PASSWORD + valueFrom: { secretKeyRef: { name: duckgres-config-store-credentials, key: password } } + - { name: HOGLAKE_S3_REGION, value: "${AWS_REGION}" } + - { name: HOGLAKE_S3_PATH_STYLE, value: "false" } + - { name: HOGLAKE_COMPACTION_INTERVAL_MS, value: "0" } + ports: [{ containerPort: 8080 }] + resources: + requests: { cpu: 500m, memory: 1Gi } + limits: { memory: 2Gi } + startupProbe: + httpGet: { path: /healthz, port: 8080 } + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: { path: /healthz, port: 8080 } + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: duckgres-hoglake + namespace: ${NAMESPACE} +spec: + selector: { app: duckgres-hoglake } + ports: [{ port: 8080, targetPort: 8080 }] diff --git a/tests/mw-dev/run.sh b/tests/mw-dev/run.sh index ed46e5ef..e7aba3a9 100755 --- a/tests/mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -35,8 +35,14 @@ case "$E2E_SUITE" in neutral|duckdb|trino|reshard) ;; *) echo "E2E_SUITE must be neutral, duckdb, trino, or reshard (got $E2E_SUITE)" >&2; exit 2 ;; esac -TRINO_IMAGE="${TRINO_IMAGE:-ghcr.io/posthog/trino:b239980432446a9893a811282217039bab24f1c4@sha256:4e459a87deb4f567858c6d537e143ef4e9411c17325269231a5a2074e0c135d8}" +# Frozen perf requires the Hoglake connector; other E2E lanes retain their pin. +if [ "$SCENARIO_NAME" = posthog_frozen_perf ] || [ "$SCENARIO_NAME" = posthog_frozen_perf_trino_cached ]; then + TRINO_IMAGE="${TRINO_IMAGE:-ghcr.io/posthog/trino:a2943f5ec37f1d5a9ab90b9bec56695a00de4584@sha256:7a57712498446bd97393cadece90ce0bc297f6feab8e0098510668fe2667ac39}" +else + TRINO_IMAGE="${TRINO_IMAGE:-ghcr.io/posthog/trino:b239980432446a9893a811282217039bab24f1c4@sha256:4e459a87deb4f567858c6d537e143ef4e9411c17325269231a5a2074e0c135d8}" +fi TRINO_TLS_PASSWORD="${TRINO_TLS_PASSWORD:-duckgres-e2e-keystore}" +HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:f10c34f9c779e2794fca662d5302f97dc26e48a6b2a601ae344cad945e70483c}" # Derive cache mode from the scenario so reported protocol and catalog agree. TRINO_FILESYSTEM_CACHE_ENABLED=false if [ "$SCENARIO_NAME" = "posthog_frozen_perf_trino_cached" ]; then @@ -85,6 +91,14 @@ require_pr_identity() { TRINO_CELL_NS="duckgres-ci-pr-0${PR_NUMBER}" } +frozen_perf_scenario() { + [ "$SCENARIO_NAME" = posthog_frozen_perf ] || [ "$SCENARIO_NAME" = posthog_frozen_perf_trino_cached ] +} + +hoglake_perf_enabled() { + [ "$E2E_SUITE" = trino ] && frozen_perf_scenario +} + trino_multicell_enabled() { [ "$E2E_SUITE" = trino ] && [ "$SCENARIO_NAME" = full-suite ] } @@ -174,6 +188,10 @@ render() { NAMESPACE="$NS" PR_NUMBER="$PR_NUMBER" \ envsubst '$NAMESPACE $PR_NUMBER $TRINO_IMAGE $TRINO_TLS_PASSWORD $TRINO_CA_CERT_B64 $TRINO_SERVER_P12_B64 $CONFIG_STORE_PASSWORD' \ < "$HERE/manifests.trino.tmpl.yaml" + if hoglake_perf_enabled; then + NAMESPACE="$NS" HOGLAKE_IMAGE="$HOGLAKE_IMAGE" AWS_REGION="$AWS_REGION" \ + envsubst '$NAMESPACE $HOGLAKE_IMAGE $AWS_REGION' < "$HERE/manifests.hoglake.tmpl.yaml" + fi if trino_multicell_enabled; then render_trino_multicell; fi fi } @@ -248,7 +266,7 @@ ensure_trino_pod_identity() { } ensure_scenario_pod_identity() { - : "${SCENARIO_POD_IDENTITY_ROLE:?SCENARIO_POD_IDENTITY_ROLE is required for the Athena perf scenario}" + : "${SCENARIO_POD_IDENTITY_ROLE:?SCENARIO_POD_IDENTITY_ROLE is required for frozen perf scenarios}" create_pod_identity "$SCENARIO_SA_NAME" "$SCENARIO_POD_IDENTITY_ROLE" # Pod Identity is injected only at pod admission. Let the association reach # the node agent before test-scenario creates the runner Job. @@ -420,7 +438,7 @@ cmd_deploy() { ensure_pod_identity restart_cp_with_identity - if [ "$SCENARIO_NAME" = "posthog_frozen_perf" ]; then + if frozen_perf_scenario; then ensure_scenario_pod_identity fi @@ -428,6 +446,16 @@ cmd_deploy() { # Associate before admitting Trino pods: the Pod Identity agent injects # credentials only at admission and never retrofits an existing pod. ensure_trino_pod_identity + if hoglake_perf_enabled; then + "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-hoglake-postgres --timeout=120s + # Allow the association to reach the node agent before pod admission. + sleep 15 + "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-hoglake \ + --type=merge -p '{"spec":{"replicas":1}}' + "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-hoglake --timeout=300s + "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \ + "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"controlplane\",\"env\":[{\"name\":\"DUCKGRES_TRINO_HOGLAKE_URI\",\"value\":\"http://duckgres-hoglake.$NS.svc:8080\"}]}]}}}}" + fi if trino_multicell_enabled; then NS="$TRINO_CELL_NS" ensure_trino_pod_identity "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \ @@ -619,6 +647,9 @@ run_scenario() { internal_secret="$(cat "$internal_secret_file")" job="$(scenario_job_name "$scenario_name")" + local hoglake_uri="" + if hoglake_perf_enabled; then hoglake_uri="http://duckgres-hoglake.$NS.svc:8080"; fi + delete_scenario_job "$job" cat < 2**63 - 1 + ): + raise ValueError( + f"uint64 column {field.name} cannot be proven safe as signed long from every row-group footer; lossless rewrite required" + ) + files.append( + dict(obj, rows=metadata.num_rows, footer_size=metadata.serialized_size) + ) + expected_ids = {name: i + 1 for i, name in enumerate(columns)} + if any(expected_ids[name] != field_id for name, field_id in field_ids): + raise ValueError( + "Parquet field IDs differ from the new Hoglake schema; lossless rewrite required" + ) + return list(columns.values()), files + + +def run(store, api, source, catalog): + source_bucket, source_prefix = location(source) + if not re.fullmatch(r"[a-z][a-z0-9_-]{0,62}", catalog): + raise ValueError("invalid catalog identifier") + objects = sorted(store.objects(source), key=lambda obj: obj["key"]) + plans = {} + for table in ("events", "persons"): + prefix = source_prefix + table + "/" + table_objects = [ + obj + for obj in objects + if obj["key"].startswith(prefix) + and obj["key"].endswith(".parquet") + and "/" not in obj["key"][len(prefix) :] + ] + if not table_objects: + raise ValueError(f"no {table}/*.parquet files in source") + plans[table] = inspect_table( + table_objects, lambda obj: store.footer(source_bucket, obj) + ) + # Validate all files before registering the catalog. + root = "/v1/catalogs/" + catalog + api.post( + "/v1/catalogs", {"name": catalog, "data_path": source.rstrip("/") + "/"} + ) + api.post(root + "/namespaces", {"name": "posthog"}) + registrations = [] + for table, (columns, files) in plans.items(): + info = api.post( + root + "/namespaces/posthog/tables", {"name": table, "columns": columns} + ) + if [(c["name"], c["field_id"]) for c in info["columns"]] != [ + (c["name"], i + 1) for i, c in enumerate(columns) + ]: + raise ValueError( + "server assigned unexpected field IDs; no source files registered" + ) + registered = [] + for obj in files: + registered.append( + { + "path": f"s3://{source_bucket}/{obj['key']}", + "record_count": obj["rows"], + "file_size_bytes": obj["size"], + "footer_size": obj["footer_size"], + } + ) + registrations.append( + { + "namespace": "posthog", + "table": table, + "expected_table_uuid": info["table_uuid"], + "files": registered, + } + ) + # Both relations become visible in one snapshot; no writes during benchmarks. + result = api.post( + root + "/commit", + { + "appends": registrations, + "author": "perf-fixture", + "message": "Register immutable benchmark fixtures", + }, + ) + return result + + +class S3Store: + def __init__(self, client): + self.client = client + + def objects(self, uri): + bucket, prefix = location(uri) + return [ + {"key": obj["Key"], "size": obj["Size"], "etag": obj["ETag"]} + for page in self.client.get_paginator("list_objects_v2").paginate( + Bucket=bucket, Prefix=prefix + ) + for obj in page.get("Contents", []) + ] + + def footer(self, bucket, obj): + def read(start, end): + response = self.client.get_object( + Bucket=bucket, + Key=obj["key"], + Range=f"bytes={start}-{end}", + IfMatch=obj["etag"], + ) + with response["Body"] as body: + return body.read() + + size = obj["size"] + if size < 12: + raise ValueError("invalid Parquet file size") + trailer = read(size - 8, size - 1) + if len(trailer) != 8 or trailer[4:] != b"PAR1": + raise ValueError("invalid or encrypted Parquet footer") + length = struct.unpack(" size - 12: + raise ValueError("invalid Parquet footer length") + raw = read(size - 8 - length, size - 9) + if len(raw) != length: + raise ValueError("truncated Parquet footer") + return pq.read_metadata(io.BytesIO(b"PAR1" + raw + trailer)) + +class RestAPI: + def __init__(self, uri): + parsed = urlparse(uri) + if ( + parsed.scheme not in ("https", "http") + or not parsed.hostname + or parsed.query + or parsed.fragment + or parsed.username + ): + raise ValueError("invalid Hoglake API URI") + self.uri = uri.rstrip("/") + + def post(self, path, body): + headers = {"Content-Type": "application/json"} + request = Request( + self.uri + path, + data=json.dumps(body).encode(), + headers=headers, + method="POST", + ) + with urlopen(request, timeout=120) as response: + return json.load(response) + + +def main(): + import boto3 + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", required=True) + parser.add_argument("--catalog", required=True) + parser.add_argument("--uri", required=True) + args = parser.parse_args() + result = run(S3Store(boto3.client("s3")), RestAPI(args.uri), args.source, args.catalog) + print(f"Registered frozen fixtures at snapshot {result['snapshot_id']}") + + +if __name__ == "__main__": + main() diff --git a/tests/mw-dev/scenario/perf/steps.go b/tests/mw-dev/scenario/perf/steps.go index 0d46c8c9..72756e7d 100644 --- a/tests/mw-dev/scenario/perf/steps.go +++ b/tests/mw-dev/scenario/perf/steps.go @@ -135,6 +135,9 @@ func (s *State) Result(stepID string) (StepResult, bool) { } func (e *Executor) ExecuteStep(ctx context.Context, step core.Step) error { + if step.Type == StepTypeSetupHoglake { + return e.setupHoglake(ctx, step) + } if step.Type != StepTypePerfQueries { return classified(ErrorClassUnsupportedStep, fmt.Errorf("unsupported perf step type %q", step.Type)) } diff --git a/tests/mw-dev/scenario/runner_test.go b/tests/mw-dev/scenario/runner_test.go index 216c627f..b0681d28 100644 --- a/tests/mw-dev/scenario/runner_test.go +++ b/tests/mw-dev/scenario/runner_test.go @@ -124,6 +124,7 @@ func TestScenarioRunner(t *testing.T) { func TestProvisionSmokeScenarioUsesIsolatedStackWarehouseIdentityAndSupportedSteps(t *testing.T) { const scenarioOrgID = "ci-pr-123-cnpg" + t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/") t.Setenv("DUCKGRES_SCENARIO_ORG_ID", scenarioOrgID) scenario, err := core.LoadScenario(filepath.Join("scenarios", "provision_smoke.yaml")) @@ -163,7 +164,9 @@ func TestProvisionSmokeScenarioUsesIsolatedStackWarehouseIdentityAndSupportedSte } func TestFrozenSuccessScenariosUseIsolatedStackWarehouseIdentity(t *testing.T) { + t.Setenv("DUCKGRES_SCENARIO_HOGLAKE_URI", "http://hoglake:8080") const scenarioOrgID = "ci-pr-123-cnpg" + t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/") t.Setenv("DUCKGRES_SCENARIO_TRINO_CA_CERT", "/tmp/test-trino-ca.crt") setAthenaPerfEnv(t) t.Setenv("DUCKGRES_K8S_WORKER_CPU_REQUEST", "3") @@ -451,6 +454,7 @@ func TestLoadScenarioForRunResolvesScenarioRelativeFiles(t *testing.T) { } func TestFrozenPerfScenarioUsesSupportedStepsAndRelativeCatalog(t *testing.T) { + t.Setenv("DUCKGRES_SCENARIO_HOGLAKE_URI", "http://hoglake:8080") t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/") t.Setenv("DUCKGRES_SCENARIO_ORG_ID", "ci-pr-123-cnpg") t.Setenv("DUCKGRES_SCENARIO_TRINO_CA_CERT", "/tmp/test-trino-ca.crt") @@ -513,6 +517,7 @@ func TestFrozenPerfScenarioUsesSupportedStepsAndRelativeCatalog(t *testing.T) { } func TestFrozenPerfScenarioBuildsAndValidatesPostHogTablesBeforePerf(t *testing.T) { + t.Setenv("DUCKGRES_SCENARIO_HOGLAKE_URI", "http://hoglake:8080") t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/") t.Setenv("DUCKGRES_SCENARIO_ORG_ID", "ci-pr-123-cnpg") t.Setenv("DUCKGRES_SCENARIO_TRINO_CA_CERT", "/tmp/test-trino-ca.crt") @@ -566,8 +571,17 @@ func TestFrozenPerfScenarioBuildsAndValidatesPostHogTablesBeforePerf(t *testing. if enabled, _ := trinoRequest["enabled"].(bool); !enabled { t.Fatalf("Trino provision request = %#v, want enabled", provisionRequest["trino"]) } - if got := steps["perf_queries"].DependsOn; len(got) != 2 || got[0] != "validate_posthog_tables" || got[1] != "wait_trino_ready" { - t.Fatalf("perf dependencies = %#v, want [validate_posthog_tables wait_trino_ready]", got) + hoglake := steps["setup_hoglake"] + if hoglake.Type != scenarioperf.StepTypeSetupHoglake || len(hoglake.DependsOn) != 1 || hoglake.DependsOn[0] != "validate_posthog_tables" { + t.Fatalf("Hoglake setup must follow fixture validation: %#v", hoglake) + } + if file, _ := hoglake.With["file"].(string); file == "" { + t.Fatal("missing Hoglake setup script") + } else if _, err := os.Stat(file); err != nil { + t.Fatal(err) + } + if got := steps["perf_queries"].DependsOn; len(got) != 2 || got[0] != "setup_hoglake" || got[1] != "wait_trino_ready" { + t.Fatalf("perf dependencies = %#v, want [setup_hoglake wait_trino_ready]", got) } } @@ -914,7 +928,7 @@ func (e dispatchExecutor) ExecuteStep(ctx context.Context, step core.Step) error return e.provision.ExecuteStep(ctx, step) case scenariosql.StepTypeSQL, scenariosql.StepTypeSQLCatalog: return e.sql.ExecuteStep(ctx, step) - case scenarioperf.StepTypePerfQueries: + case scenarioperf.StepTypePerfQueries, scenarioperf.StepTypeSetupHoglake: return e.perf.ExecuteStep(ctx, step) case scenariodbt.StepTypeDBTRun: return e.dbt.ExecuteStep(ctx, step) @@ -936,7 +950,7 @@ func dispatchSupports(stepType string) bool { return true case scenariosql.StepTypeSQL, scenariosql.StepTypeSQLCatalog: return true - case scenarioperf.StepTypePerfQueries: + case scenarioperf.StepTypePerfQueries, scenarioperf.StepTypeSetupHoglake: return true case scenariodbt.StepTypeDBTRun: return true diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml index 6a93be21..22b9fcca 100644 --- a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml @@ -2,6 +2,7 @@ name: posthog-frozen-perf run_id_prefix: scenario-frozen-perf required_env: - DUCKGRES_SCENARIO_ORG_ID + - DUCKGRES_SCENARIO_HOGLAKE_URI - DUCKGRES_SCENARIO_FROZEN_S3_URI - DUCKGRES_SCENARIO_TRINO_CA_CERT - DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_DSN @@ -74,9 +75,18 @@ steps: max_attempts: 3 retry_interval: 10s + - id: setup_hoglake + type: setup_hoglake + depends_on: [validate_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI} + source: ${env:DUCKGRES_SCENARIO_FROZEN_S3_URI} + file: ../perf/setup_hoglake.py + - id: perf_queries type: perf_queries - depends_on: [validate_posthog_tables, wait_trino_ready] + depends_on: [setup_hoglake, wait_trino_ready] with: org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} catalog: ducklake diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_trino_cached.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_trino_cached.yaml index ab852d9f..8e230367 100644 --- a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_trino_cached.yaml +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_trino_cached.yaml @@ -2,6 +2,7 @@ name: posthog-frozen-perf-trino-cached run_id_prefix: scenario-frozen-perf-trino-cached required_env: - DUCKGRES_SCENARIO_ORG_ID + - DUCKGRES_SCENARIO_HOGLAKE_URI - DUCKGRES_SCENARIO_FROZEN_S3_URI - DUCKGRES_SCENARIO_TRINO_CA_CERT - DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_DSN @@ -70,9 +71,18 @@ steps: max_attempts: 3 retry_interval: 10s + - id: setup_hoglake + type: setup_hoglake + depends_on: [validate_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI} + source: ${env:DUCKGRES_SCENARIO_FROZEN_S3_URI} + file: ../perf/setup_hoglake.py + - id: perf_queries type: perf_queries - depends_on: [validate_posthog_tables, wait_trino_ready] + depends_on: [setup_hoglake, wait_trino_ready] with: org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} catalog: ducklake diff --git a/tests/mw-dev/scenario/script_test.go b/tests/mw-dev/scenario/script_test.go index 523426c0..c1dc5faf 100644 --- a/tests/mw-dev/scenario/script_test.go +++ b/tests/mw-dev/scenario/script_test.go @@ -101,8 +101,8 @@ func TestDevScenarioWorkflowUsesUnifiedMwDevHarness(t *testing.T) { "EKS_CLUSTER_NAME: posthog-mw-dev", "CP_POD_IDENTITY_ROLE: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/duckgres-control-plane-dev", "TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }}", - "- name: Load Athena perf configuration", - "if: env.SCENARIO_NAME == 'posthog_frozen_perf'", + "- name: Load frozen perf identity and Athena configuration", + "if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_trino_cached'", "bash scripts/scenario_athena_config.sh >> \"$GITHUB_ENV\"", "TRINO_IMAGE: ghcr.io/posthog/trino:", "E2E_SUITE: ${{ (matrix.scenario == 'posthog_frozen_perf' || matrix.scenario == 'posthog_frozen_perf_trino_cached') && 'trino' || 'neutral' }}", @@ -173,10 +173,10 @@ func TestDevScenarioWorkflowUsesUnifiedMwDevHarness(t *testing.T) { } authIndex := strings.Index(workflow, "- name: Configure AWS credentials (OIDC)") - configIndex := strings.Index(workflow, "- name: Load Athena perf configuration") + configIndex := strings.Index(workflow, "- name: Load frozen perf identity and Athena configuration") deployIndex := strings.Index(workflow, "- name: Deploy isolated Duckgres stack") if authIndex < 0 || configIndex < authIndex || deployIndex < configIndex { - t.Fatal("Athena config must load after AWS authentication and before deploying the stack") + t.Fatal("Frozen perf configuration must load after AWS authentication and before deploying the stack") } teardownIndex := strings.Index(workflow, "- name: Teardown") diff --git a/tests/perf/core/catalog.go b/tests/perf/core/catalog.go index 4c6ef1b5..d5ca2903 100644 --- a/tests/perf/core/catalog.go +++ b/tests/perf/core/catalog.go @@ -141,6 +141,12 @@ func validateRelationVariants(targets []Protocol, variants map[StorageTarget]map return nil } requiredTargets := []StorageTarget{StorageTargetRawView, StorageTargetDuckLakeTable} + for _, target := range targets { + if target == ProtocolTrino || target == ProtocolTrinoCached { + requiredTargets = append(requiredTargets, StorageTargetHoglakeTable) + break + } + } for _, target := range targets { if target == ProtocolAthena { requiredTargets = append(requiredTargets, StorageTargetAthenaExternal) @@ -182,6 +188,9 @@ func expandPairedQuery(def pairedQueryDefinition, variants map[StorageTarget]map } targets := []StorageTarget{StorageTargetRawView, StorageTargetDuckLakeTable} + if _, ok := variants[StorageTargetHoglakeTable]; ok { + targets = append(targets, StorageTargetHoglakeTable) + } if _, ok := variants[StorageTargetAthenaExternal]; ok { targets = append(targets, StorageTargetAthenaExternal) } @@ -204,12 +213,10 @@ func expandPairedQuery(def pairedQueryDefinition, variants map[StorageTarget]map StorageTarget: target, }) } - renderedSQL := make(map[string]struct{}, len(queries)) - for _, query := range queries { - if _, exists := renderedSQL[query.PGWireSQL]; exists { - return nil, fmt.Errorf("paired query %s relation bindings must differ between storage targets", def.QueryIDBase) - } - renderedSQL[query.PGWireSQL] = struct{}{} + // Identical relation names are valid across engines, but the raw-view and + // DuckLake variants run on the same PGWire target and must differ. + if queries[0].PGWireSQL == queries[1].PGWireSQL { + return nil, fmt.Errorf("paired query %s relation bindings must differ between storage targets", def.QueryIDBase) } return queries, nil } diff --git a/tests/perf/core/catalog_test.go b/tests/perf/core/catalog_test.go index 8cb3316d..6fe69347 100644 --- a/tests/perf/core/catalog_test.go +++ b/tests/perf/core/catalog_test.go @@ -51,24 +51,31 @@ func TestCheckedInPostHogCatalogPublishesCompleteStablePairs(t *testing.T) { want := []string{ "q_events_total_balanced_v4__raw_view", "q_events_total_balanced_v4__ducklake_table", + "q_events_total_balanced_v4__hoglake_table", "q_events_total_balanced_v4__athena_external", "q_events_count_one_day_balanced_v4__raw_view", "q_events_count_one_day_balanced_v4__ducklake_table", + "q_events_count_one_day_balanced_v4__hoglake_table", "q_events_count_one_day_balanced_v4__athena_external", "q_events_by_name_march_2026_balanced_v4__raw_view", "q_events_by_name_march_2026_balanced_v4__ducklake_table", + "q_events_by_name_march_2026_balanced_v4__hoglake_table", "q_events_by_name_march_2026_balanced_v4__athena_external", "q_events_distinct_persons_balanced_v4__raw_view", "q_events_distinct_persons_balanced_v4__ducklake_table", + "q_events_distinct_persons_balanced_v4__hoglake_table", "q_events_distinct_persons_balanced_v4__athena_external", "q_persons_total_balanced_v4__raw_view", "q_persons_total_balanced_v4__ducklake_table", + "q_persons_total_balanced_v4__hoglake_table", "q_persons_total_balanced_v4__athena_external", "q_persons_daily_april_2026_balanced_v4__raw_view", "q_persons_daily_april_2026_balanced_v4__ducklake_table", + "q_persons_daily_april_2026_balanced_v4__hoglake_table", "q_persons_daily_april_2026_balanced_v4__athena_external", "q_events_daily_march_2026_balanced_v4__raw_view", "q_events_daily_march_2026_balanced_v4__ducklake_table", + "q_events_daily_march_2026_balanced_v4__hoglake_table", "q_events_daily_march_2026_balanced_v4__athena_external", } if got := queryIDs(catalog); !reflect.DeepEqual(got, want) { @@ -86,16 +93,20 @@ func TestCheckedInPostHogCatalogPublishesCompleteStablePairs(t *testing.T) { } } for index, query := range catalog.Queries { - wantTarget := []StorageTarget{StorageTargetRawView, StorageTargetDuckLakeTable, StorageTargetAthenaExternal}[index%3] + wantTarget := []StorageTarget{StorageTargetRawView, StorageTargetDuckLakeTable, StorageTargetHoglakeTable, StorageTargetAthenaExternal}[index%4] if query.StorageTarget != wantTarget { t.Fatalf("query %s storage target = %q, want %q", query.QueryID, query.StorageTarget, wantTarget) } - if index%3 != 2 { + if index%4 != 3 { continue } - rawQuery := catalog.Queries[index-2] - duckLakeQuery := catalog.Queries[index-1] + rawQuery := catalog.Queries[index-3] + duckLakeQuery := catalog.Queries[index-2] + hoglakeQuery := catalog.Queries[index-1] + if hoglakeQuery.PGWireSQL != duckLakeQuery.PGWireSQL || hoglakeQuery.IntentID != duckLakeQuery.IntentID { + t.Fatalf("Hoglake query %s must use the shared SQL and intent", hoglakeQuery.QueryID) + } if query.IntentID != rawQuery.IntentID || duckLakeQuery.IntentID != rawQuery.IntentID { t.Fatalf("query variants have mismatched intents: %q/%q/%q", rawQuery.IntentID, duckLakeQuery.IntentID, query.IntentID) } @@ -131,10 +142,10 @@ paired_queries: if err != nil { t.Fatalf("ParseCatalog returned error: %v", err) } - if got, want := queryIDs(catalog), []string{"q_events__raw_view", "q_events__ducklake_table", "q_events__athena_external"}; !reflect.DeepEqual(got, want) { + if got, want := queryIDs(catalog), []string{"q_events__raw_view", "q_events__ducklake_table", "q_events__hoglake_table", "q_events__athena_external"}; !reflect.DeepEqual(got, want) { t.Fatalf("unexpected generated query order: got %v want %v", got, want) } - athenaQuery := catalog.Queries[2] + athenaQuery := catalog.Queries[3] if got, want := athenaQuery.StorageTarget, StorageTargetAthenaExternal; got != want { t.Fatalf("Athena query target: got %q want %q", got, want) } @@ -478,6 +489,9 @@ relation_variants: ducklake_table: events: posthog.events persons: posthog.persons + hoglake_table: + events: posthog.events + persons: posthog.persons athena_external: events: events persons: persons diff --git a/tests/perf/core/runner.go b/tests/perf/core/runner.go index f93c8679..713e13a5 100644 --- a/tests/perf/core/runner.go +++ b/tests/perf/core/runner.go @@ -161,7 +161,7 @@ func (r *QueryRunner) executeIteration(ctx context.Context, protocol Protocol, m // Each physical relation family is routed only to protocols which expose it. // PGWire measures both the raw Parquet view and production-shaped DuckLake -// table, Trino measures the shared DuckLake table, and Athena measures its +// table, Trino measures the Hoglake table, and Athena measures its // Glue external table over the same immutable Parquet files. func querySupportsProtocol(query Query, protocol Protocol) bool { switch query.StorageTarget { @@ -170,7 +170,9 @@ func querySupportsProtocol(query Query, protocol Protocol) bool { case StorageTargetRawView: return protocol == ProtocolPGWire || protocol == ProtocolPGWireUncached || protocol == ProtocolPGWireCached case StorageTargetDuckLakeTable: - return protocol == ProtocolPGWire || protocol == ProtocolPGWireUncached || protocol == ProtocolPGWireCached || protocol == ProtocolTrino || protocol == ProtocolTrinoCached + return protocol == ProtocolPGWire || protocol == ProtocolPGWireUncached || protocol == ProtocolPGWireCached + case StorageTargetHoglakeTable: + return protocol == ProtocolTrino || protocol == ProtocolTrinoCached case StorageTargetAthenaExternal: return protocol == ProtocolAthena default: diff --git a/tests/perf/core/runner_test.go b/tests/perf/core/runner_test.go index 2f56a720..3b8d56f9 100644 --- a/tests/perf/core/runner_test.go +++ b/tests/perf/core/runner_test.go @@ -181,6 +181,7 @@ func TestRunnerBalancesPairedQueryOrderAcrossMeasuredIterations(t *testing.T) { PGWireSQL: "SELECT COUNT(*) FROM posthog.events", StorageTarget: StorageTargetDuckLakeTable, }, + {QueryID: "q_events__hoglake_table", IntentID: "intent_events", PGWireSQL: "SELECT COUNT(*) FROM posthog.events", StorageTarget: StorageTargetHoglakeTable}, }, }, Drivers: map[Protocol]ProtocolDriver{ @@ -211,7 +212,7 @@ func TestRunnerBalancesPairedQueryOrderAcrossMeasuredIterations(t *testing.T) { } } -func TestRunnerKeepsRawViewsOnPGWireAndRunsDuckLakeTablesOnEveryTarget(t *testing.T) { +func TestRunnerKeepsDuckLakeOnPGWireAndRunsHoglakeOnTrino(t *testing.T) { const trino Protocol = "trino" pg := &testDriver{protocol: ProtocolPGWire} trinoDriver := &testDriver{protocol: trino} @@ -234,6 +235,12 @@ func TestRunnerKeepsRawViewsOnPGWireAndRunsDuckLakeTablesOnEveryTarget(t *testin PGWireSQL: "SELECT COUNT(*) FROM posthog.events", StorageTarget: StorageTargetDuckLakeTable, }, + { + QueryID: "q_events__hoglake_table", + IntentID: "intent_events", + PGWireSQL: "SELECT COUNT(*) FROM posthog.events", + StorageTarget: StorageTargetHoglakeTable, + }, }, }, Drivers: map[Protocol]ProtocolDriver{ @@ -251,7 +258,7 @@ func TestRunnerKeepsRawViewsOnPGWireAndRunsDuckLakeTablesOnEveryTarget(t *testin if got, want := pg.queryIDs, []string{"q_events__raw_view", "q_events__ducklake_table"}; !reflect.DeepEqual(got, want) { t.Fatalf("pgwire query IDs: got %v want %v", got, want) } - if got, want := trinoDriver.queryIDs, []string{"q_events__ducklake_table"}; !reflect.DeepEqual(got, want) { + if got, want := trinoDriver.queryIDs, []string{"q_events__hoglake_table"}; !reflect.DeepEqual(got, want) { t.Fatalf("trino query IDs: got %v want %v", got, want) } if summary.TotalQueries != 3 { @@ -276,6 +283,7 @@ func TestRunnerRoutesEachStorageVariantOnlyToItsComparableProtocol(t *testing.T) Queries: []Query{ {QueryID: "q__raw_view", IntentID: "intent", StorageTarget: StorageTargetRawView}, {QueryID: "q__ducklake_table", IntentID: "intent", StorageTarget: StorageTargetDuckLakeTable}, + {QueryID: "q__hoglake_table", IntentID: "intent", StorageTarget: StorageTargetHoglakeTable}, {QueryID: "q__athena_external", IntentID: "intent", StorageTarget: StorageTargetAthenaExternal}, }, }, @@ -296,13 +304,13 @@ func TestRunnerRoutesEachStorageVariantOnlyToItsComparableProtocol(t *testing.T) if got, want := pg.queryIDs, []string{"q__raw_view", "q__ducklake_table"}; !reflect.DeepEqual(got, want) { t.Fatalf("PGWire query IDs: got %v want %v", got, want) } - if got, want := trinoDriver.queryIDs, []string{"q__ducklake_table"}; !reflect.DeepEqual(got, want) { + if got, want := trinoDriver.queryIDs, []string{"q__hoglake_table"}; !reflect.DeepEqual(got, want) { t.Fatalf("Trino query IDs: got %v want %v", got, want) } if got, want := athenaDriver.queryIDs, []string{"q__athena_external"}; !reflect.DeepEqual(got, want) { t.Fatalf("Athena query IDs: got %v want %v", got, want) } - if got, want := cachedDriver.queryIDs, []string{"q__ducklake_table"}; !reflect.DeepEqual(got, want) { + if got, want := cachedDriver.queryIDs, []string{"q__hoglake_table"}; !reflect.DeepEqual(got, want) { t.Fatalf("Cached Trino query IDs: got %v want %v", got, want) } if summary.TotalQueries != 5 { @@ -323,6 +331,7 @@ func TestRunnerRunsUncachedAndCachedPGWireAsDistinctComparableResults(t *testing Queries: []Query{ {QueryID: "q__raw_view", IntentID: "intent", StorageTarget: StorageTargetRawView}, {QueryID: "q__ducklake_table", IntentID: "intent", StorageTarget: StorageTargetDuckLakeTable}, + {QueryID: "q__hoglake_table", IntentID: "intent", StorageTarget: StorageTargetHoglakeTable}, {QueryID: "q__athena_external", IntentID: "intent", StorageTarget: StorageTargetAthenaExternal}, }, }, diff --git a/tests/perf/core/types.go b/tests/perf/core/types.go index 8b52f3af..a6157421 100644 --- a/tests/perf/core/types.go +++ b/tests/perf/core/types.go @@ -21,6 +21,7 @@ type StorageTarget string const ( StorageTargetRawView StorageTarget = "raw_view" StorageTargetDuckLakeTable StorageTarget = "ducklake_table" + StorageTargetHoglakeTable StorageTarget = "hoglake_table" StorageTargetAthenaExternal StorageTarget = "athena_external" ) diff --git a/tests/perf/queries/ducklake_posthog_tables.yaml b/tests/perf/queries/ducklake_posthog_tables.yaml index 5eba2974..79dd17a7 100644 --- a/tests/perf/queries/ducklake_posthog_tables.yaml +++ b/tests/perf/queries/ducklake_posthog_tables.yaml @@ -1,5 +1,5 @@ name: posthog-frozen-ducklake-golden-v4 -description: Identical query shapes with protocol-isolated, balanced execution order over frozen raw Parquet views and production-shaped DuckLake tables. +description: Identical query shapes with protocol-isolated, balanced execution order over frozen raw Parquet views and DuckLake tables on PGWire, Hoglake tables on Trino, and Athena external tables. seed: 42 dataset_scale: 1 targets: @@ -18,6 +18,9 @@ relation_variants: ducklake_table: events: posthog.events persons: posthog.persons + hoglake_table: + events: posthog.events + persons: posthog.persons athena_external: events: events persons: persons