This repository was archived by the owner on Oct 8, 2026. It is now read-only.
Repository navigation
scenario-dev #201
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scenario-dev | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| scenario: | |
| description: Scenario YAML name under tests/mw-dev/scenario/scenarios, without .yaml | |
| required: true | |
| default: full-suite | |
| type: string | |
| coverage_target: | |
| description: Extended target; all runs each configuration in a separate job (manual only) | |
| required: false | |
| default: all | |
| type: choice | |
| options: [all, pgwire_uncached, pgwire_cached, trino, trino_cached, athena] | |
| profile_distinct: | |
| description: Profile exact distinct on isolated Trino workers | |
| type: boolean | |
| default: false | |
| distinct_variant: | |
| description: Exact distinct diagnostic treatment | |
| type: choice | |
| options: [baseline, memory64, dictionary] | |
| default: baseline | |
| profile_ordered_funnel: | |
| description: Capture ordered funnel diagnostics after timings (extended scenario only) | |
| required: false | |
| default: false | |
| type: boolean | |
| experiment_ordered_funnel: | |
| description: Compare single-start rewrite with original (requires profiling; Trino only) | |
| required: false | |
| default: false | |
| type: boolean | |
| profile_recipient: | |
| description: age public recipient for encrypted diagnostic artifacts | |
| required: false | |
| default: "" | |
| type: string | |
| properties_s3_uri: | |
| description: Properties Parquet S3 prefix override; blank uses the existing Athena table location | |
| required: false | |
| default: "" | |
| type: string | |
| duckgres_image: | |
| description: Existing Duckgres image to deploy instead of building this SHA | |
| required: false | |
| default: "" | |
| type: string | |
| trino_image: | |
| description: Trino image for posthog_frozen_perf; blank uses the newest PostHog/trino master build | |
| required: false | |
| default: "" | |
| type: string | |
| schedule: | |
| - cron: "17 8 * * *" | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: | |
| group: scenario-dev-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| plan: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| matrix: ${{ steps.targets.outputs.matrix }} | |
| trino_image: ${{ steps.trino.outputs.image }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Select scenario targets | |
| id: targets | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| python3 - <<'PYTHON' | |
| import json, os, re | |
| with open(os.environ["GITHUB_EVENT_PATH"]) as event_file: | |
| inputs = json.load(event_file).get("inputs") or {} | |
| scenario = "posthog_frozen_perf" if os.environ["EVENT_NAME"] == "schedule" else inputs.get("scenario", "") | |
| targets = ["pgwire_uncached", "pgwire_cached", "trino", "trino_cached", "athena"] | |
| profile = str(inputs.get("profile_ordered_funnel", "false")).lower() == "true" | |
| experiment = str(inputs.get("experiment_ordered_funnel", "false")).lower() == "true" | |
| if experiment and not profile: | |
| raise SystemExit("Funnel experiment requires profiling") | |
| distinct = str(inputs.get("profile_distinct", "false")).lower() == "true" | |
| if distinct and (profile or experiment): | |
| raise SystemExit("Distinct profiling cannot combine with funnel profiling") | |
| if distinct: | |
| if inputs.get("distinct_variant", "baseline") not in ("baseline", "memory64", "dictionary"): | |
| raise SystemExit("Unsupported distinct diagnostic variant") | |
| targets = ["trino", "trino_cached"] | |
| if profile or distinct: | |
| if scenario != "posthog_frozen_perf_extended": | |
| raise SystemExit("Profiling requires posthog_frozen_perf_extended") | |
| if not re.fullmatch(r"age1[a-z0-9]+", inputs.get("profile_recipient", "")): | |
| raise SystemExit("Profiling requires an age public recipient") | |
| if "athena" in targets: targets.remove("athena") | |
| if experiment: | |
| targets = ["trino", "trino_cached"] | |
| if scenario == "posthog_frozen_perf_extended": | |
| selected = inputs.get("coverage_target") or "all" | |
| if selected not in ["all", *targets]: | |
| raise SystemExit("Unsupported extended coverage target") | |
| jobs = [{"target": target, "id_suffix": str(i + 1), "artifact_suffix": "-" + target} | |
| for i, target in enumerate(targets) if selected in ("all", target)] | |
| else: | |
| jobs = [{"target": "pgwire_uncached", "id_suffix": "", "artifact_suffix": ""}] | |
| needs_trino = scenario == "posthog_frozen_perf" or ( | |
| scenario == "posthog_frozen_perf_extended" and any(j["target"].startswith("trino") for j in jobs)) | |
| if needs_trino and inputs.get("duckgres_image"): | |
| raise SystemExit("Trino comparisons require the control-plane image built from this revision; omit duckgres_image") | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as output: | |
| print("matrix=" + json.dumps({"include": jobs}, separators=(",", ":")), file=output) | |
| print("needs_trino=" + str(needs_trino).lower(), file=output) | |
| PYTHON | |
| - name: Resolve shared Trino image | |
| id: trino | |
| if: steps.targets.outputs.needs_trino == 'true' | |
| env: | |
| TRINO_IMAGE_OVERRIDE: ${{ inputs.trino_image }} | |
| run: | | |
| set -euo pipefail | |
| image="${TRINO_IMAGE_OVERRIDE:-$(scripts/resolve_trino_master_image.sh)}" | |
| echo "image=$image" >> "$GITHUB_OUTPUT" | |
| echo "Trino image: \`$image\`" >> "$GITHUB_STEP_SUMMARY" | |
| scenario-runner-image: | |
| uses: ./.github/workflows/_image-build.yml | |
| with: | |
| dockerfile: tests/mw-dev/scenario/Dockerfile | |
| image-name: duckgres-prs | |
| tag: scenario-runner-${{ github.run_id }}-${{ github.run_attempt }}-arm64 | |
| platform: linux/arm64 | |
| cache-scope: scenario-runner-arm64 | |
| secrets: | |
| ecr-role: ${{ vars.AWS_ECR_PRS_PUBLISH_IAM_ROLE }} | |
| duckgres-image: | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.duckgres_image == '' }} | |
| uses: ./.github/workflows/_image-build.yml | |
| with: | |
| dockerfile: Dockerfile | |
| image-name: duckgres-prs | |
| tag: scenario-duckgres-${{ github.run_id }}-${{ github.run_attempt }}-arm64 | |
| platform: linux/arm64 | |
| cache-scope: scenario-duckgres-arm64 | |
| build-args: | | |
| DUCKDB_EXTENSION_VERSION=1.5.5 | |
| HTTPFS_EXTENSION_TAG=v1.5.5-cred-refresh-write-retry | |
| DUCKLAKE_EXTENSION_TAG=v1.0-posthog.7 | |
| DUCKDB_EXTENSION_REPOSITORY=https://extensions.duckdb.org | |
| POSTGRES_SCANNER_TAG=v1.5.5-a3516c0 | |
| secrets: | |
| ecr-role: ${{ vars.AWS_ECR_PRS_PUBLISH_IAM_ROLE }} | |
| scenario: | |
| name: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }}${{ matrix.artifact_suffix }} | |
| needs: [plan, scenario-runner-image, duckgres-image] | |
| if: ${{ always() && needs.plan.result == 'success' && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 270 | |
| strategy: | |
| fail-fast: false | |
| # Sequential targets avoid benchmark resource contention. Each gets a fresh timeout. | |
| max-parallel: 1 | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| env: | |
| SCENARIO_NAME: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }} | |
| SCENARIO_RUNNER_IMAGE: ${{ needs.scenario-runner-image.outputs.image }} | |
| WORKER_IMAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.duckgres_image) || needs.duckgres-image.outputs.image }} | |
| CONTROLPLANE_IMAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.duckgres_image) || needs.duckgres-image.outputs.image }} | |
| KUBE_CONTEXT: posthog-mw-dev | |
| CLUSTER_NAME: posthog-mw-dev | |
| EKS_CLUSTER_NAME: posthog-mw-dev | |
| AWS_REGION: us-east-1 | |
| CP_POD_IDENTITY_ROLE: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/duckgres-control-plane-dev | |
| # Frozen perf uses a namespace-local Trino cell. The dedicated role is | |
| # consumed only when E2E_SUITE selects that isolated deployment. | |
| TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }} | |
| E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || (inputs.scenario == 'posthog_frozen_perf_extended' && (matrix.target == 'trino' || matrix.target == 'trino_cached'))) && 'trino' || 'neutral' }} | |
| DUCKGRES_SCENARIO_COVERAGE_TARGET: ${{ matrix.target }} | |
| DUCKGRES_SCENARIO_PROFILE_DISTINCT: ${{ inputs.profile_distinct || false }} | |
| DUCKGRES_SCENARIO_DISTINCT_PARTIAL_MEMORY: ${{ inputs.profile_distinct && inputs.distinct_variant == 'memory64' && '64MB' || '16MB' }} | |
| DUCKGRES_SCENARIO_DISTINCT_DICTIONARY: ${{ inputs.profile_distinct && inputs.distinct_variant == 'dictionary' || false }} | |
| DUCKGRES_SCENARIO_PROFILE_ORDERED_FUNNEL: ${{ inputs.profile_ordered_funnel || false }} | |
| DUCKGRES_SCENARIO_EXPERIMENT_ORDERED_FUNNEL: ${{ inputs.experiment_ordered_funnel || false }} | |
| DUCKGRES_SCENARIO_PROFILE_RECIPIENT: ${{ (inputs.profile_ordered_funnel || inputs.profile_distinct) && inputs.profile_recipient || '' }} | |
| PR_NUMBER: ${{ github.run_id }}${{ matrix.id_suffix }} | |
| NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.id_suffix }} | |
| TRINO_IMAGE: ${{ needs.plan.outputs.trino_image }} | |
| DUCKGRES_SCENARIO_MAX_RUNTIME: 4h | |
| DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: 4h15m | |
| # Match the benchmark Trino fleet: three 7-CPU / 28Gi workers. | |
| # Non-benchmark scenarios retain their existing small resource budget. | |
| DUCKGRES_K8S_WORKER_CPU_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '3' }} | |
| DUCKGRES_K8S_WORKER_MEMORY_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '12Gi' }} | |
| # The perf driver passes explicit sizing GUCs: keep their admission caps aligned. | |
| DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '8' }} | |
| DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '16Gi' }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Scenario dev target selected | |
| run: echo "Deploying an isolated Duckgres stack for $SCENARIO_NAME." | |
| - name: Set up Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Test scenario workflow scripts | |
| run: go test -count=1 ./tests/mw-dev/scenario ./tests/mw-dev ./tests/perf/publishercli | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/github-duckgres-e2e | |
| aws-region: us-east-1 | |
| # Covers this job's 270-minute timeout so long dbt runs can still | |
| # collect artifacts and tear down the isolated namespace. | |
| role-duration-seconds: 16200 | |
| - name: Discover managed Hoglake configuration | |
| if: env.E2E_SUITE == 'trino' | |
| run: bash tests/mw-dev/discover-hoglake.sh | |
| - name: Connect to Tailscale | |
| uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 | |
| with: | |
| oauth-client-id: ${{ vars.TS_WIF_CLIENT_ID_MW_DEV }} | |
| audience: ${{ vars.TS_WIF_AUDIENCE_MW_DEV }} | |
| tags: tag:github-runner | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v5.1.0 | |
| - name: Update kubeconfig | |
| run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT" | |
| - name: Load frozen perf identity and Athena configuration | |
| if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended_uncached' | |
| run: bash scripts/scenario_athena_config.sh >> "$GITHUB_ENV" | |
| - name: Load properties fixture override | |
| if: env.SCENARIO_NAME == 'posthog_frozen_perf' | |
| run: | | |
| source=$(jq -r '.inputs.properties_s3_uri // ""' "$GITHUB_EVENT_PATH") | |
| if [ -n "$source" ]; then | |
| echo "::add-mask::$source" | |
| if [[ "$source" =~ ^s3://[^[:space:]]+$ ]]; then | |
| echo "DUCKGRES_SCENARIO_PROPERTIES_S3_URI=$source" >> "$GITHUB_ENV" | |
| else | |
| echo "::warning::Invalid properties prefix; the properties comparison will fail after original benchmarks complete." | |
| echo "DUCKGRES_SCENARIO_PROPERTIES_S3_URI=invalid" >> "$GITHUB_ENV" | |
| fi | |
| fi | |
| - name: Deploy isolated Duckgres stack | |
| run: tests/mw-dev/run.sh deploy | |
| - name: Run selected scenario | |
| run: tests/mw-dev/run.sh test-scenario | |
| - name: Publish scenario summary | |
| if: ${{ always() && !inputs.profile_distinct }} | |
| run: | | |
| set -euo pipefail | |
| summary_file="$(find artifacts/scenario-dev -type f -name scenario_summary.md -print -quit 2>/dev/null || true)" | |
| if [ -n "$summary_file" ]; then | |
| cat "$summary_file" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| { | |
| echo "## Scenario result" | |
| echo | |
| echo "Scenario summary unavailable. Inspect the job log and any uploaded partial artifact." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Collect diagnostics | |
| if: ${{ failure() && !inputs.profile_distinct }} | |
| run: tests/mw-dev/run.sh diagnostics | |
| - name: Teardown | |
| if: always() | |
| run: tests/mw-dev/run.sh teardown | |
| - name: Upload scenario artifacts | |
| if: ${{ always() && !inputs.profile_distinct }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: scenario-dev-${{ env.SCENARIO_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}${{ matrix.artifact_suffix }} | |
| path: artifacts/scenario-dev/ | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| - name: Upload encrypted distinct profiles | |
| if: ${{ always() && inputs.profile_distinct }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: distinct-profile-${{ github.run_id }}-${{ github.run_attempt }}${{ matrix.artifact_suffix }} | |
| path: artifacts/scenario-dev/**/profile-*.json.age | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| - name: Publish scenario perf results | |
| if: ${{ always() && github.ref == 'refs/heads/main' && !inputs.profile_distinct }} | |
| timeout-minutes: 10 | |
| env: | |
| MW_DEV_SCENARIO_PERF_SECRET_ID: ${{ vars.MW_DEV_SCENARIO_PERF_SECRET_ID }} | |
| run: | | |
| set -euo pipefail | |
| mapfile -d '' perf_summaries < <( | |
| find artifacts/scenario-dev -type f \( -path '*/perf/summary.json' -o -path '*/perf-properties/summary.json' -o -path '*/perf-coverage/summary.json' \) -print0 2>/dev/null | sort -z | |
| ) | |
| if [ "${#perf_summaries[@]}" -eq 0 ]; then | |
| echo "No scenario perf artifact was produced; skipping historical publish." | |
| exit 0 | |
| fi | |
| : "${MW_DEV_SCENARIO_PERF_SECRET_ID:?MW_DEV_SCENARIO_PERF_SECRET_ID repository variable is required}" | |
| publish_failed=0 | |
| for perf_summary in "${perf_summaries[@]}"; do | |
| perf_dir="$(dirname "$perf_summary")" | |
| echo "Publishing scenario perf artifacts from $perf_dir" | |
| if ! aws secretsmanager get-secret-value \ | |
| --secret-id "$MW_DEV_SCENARIO_PERF_SECRET_ID" \ | |
| --query SecretString \ | |
| --output text \ | |
| | go run ./cmd/duckgres-perf-publisher \ | |
| --run-dir "$perf_dir" \ | |
| --connection-secret-stdin \ | |
| --publish-timeout 2m \ | |
| --schema duckgres_scenario_perf \ | |
| --bootstrap-schema=true; then | |
| echo "::error::Failed to publish scenario perf artifacts from $perf_dir" | |
| publish_failed=1 | |
| fi | |
| done | |
| exit "$publish_failed" |