From a7c847db9ba89bf63c07cab63af67628a4570785 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 29 Aug 2026 22:47:41 +0100 Subject: [PATCH 1/4] chore(signals): sync prod snapshots to run the inbox ranking training job locally Claude-Session: https://claude.ai/code/session_01DciwXDou2MVrmrGupjjJCh --- products/signals/dags/inbox_ranking/README.md | 33 ++++++++++++ .../inbox_ranking/bin/sync_snapshots_local.sh | 53 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100755 products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md index a52fcbc4623a..0a1e912424b3 100644 --- a/products/signals/dags/inbox_ranking/README.md +++ b/products/signals/dags/inbox_ranking/README.md @@ -76,6 +76,39 @@ s3://// - **Candidate**: per-head XGBoost with fixed params, holdout = the last `holdout_days` of reports (cut by report, never by row), AUC + a label-permutation null. A head is _readable_ when it has enough holdout positives and clears its null by 0.05. The shipped booster (`.ubj`) is refit on everything; the train-only fit is kept as `.holdout.ubj` so a later candidate can grade this model on its own holdout. - **Champion**: `promotion.decide_promotion` — promote when the candidate has a readable head, is within 0.02 AUC of the champion on every head the champion could read, and the champion is at least `INBOX_RANKING_PROMOTION_MIN_DAYS` old. The champion's AUCs come from its `.holdout.ubj` scored on the candidate's holdout (`paired_champion_aucs`), so both models are compared on one set of reports; a champion without that file falls back to its stored AUC. The pointer is rewritten only when `INBOX_RANKING_AUTO_PROMOTE` is on; otherwise the decision is logged and surfaced as asset metadata, so the daily candidate series is monitoring while the first shadow read runs on a frozen champion. To promote by hand, copy a candidate's `metadata.json` to `champion.json` with a `promoted_at`. +### Running the training job locally + +The training job is S3-only, so it can run on a laptop against copies of the prod snapshots. The dataset job cannot: it needs the dogfood project's ClickHouse and cross-region Postgres. + +1. Sync the two prefixes the job reads into the local object-storage bucket (needs an SSO session with the `secrets-editor` role on `prod-us-secrets`, granted through Access Elevator): + + ```bash + aws sso login --profile prod-us-secrets + products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh + ``` + + This copies `inbox_report_state/v1/dt=*` and `inbox_report_labels/v1/dt=*` to `~/.cache/posthog/inbox_ranking/` and from there into `s3://posthog/inbox_ranking/` on SeaweedFS (`localhost:19000`). It prints the partition days present in both tables; pick one of those as the partition to run. Re-runs only move new days. + +2. Leave `INBOX_RANKING_DATASET_S3_BUCKET` unset: `common.s3_client()` then talks to SeaweedFS and `dataset_bucket()` is `posthog`. + +3. With the dev stack up (`bin/start` runs `dagster dev` on http://localhost:3030 with the signals location loaded), materialize `inbox_ranking_training_job` for that partition from the UI, or from the CLI: + + ```bash + dagster job launch -w .dagster_home/workspace.yaml --location posthog.dags.locations.signals \ + -j inbox_ranking_training_job --tags '{"dagster/partition": "2026-08-25"}' + ``` + + The schedule is stopped outside prod US, so nothing runs unasked. If the run sits in `QUEUED`, check the daemon log for `Maximum is 10, won't launch more`: runs from a killed `dagster dev` stay `STARTED` forever and count against the local queue. Terminate them from the Runs page (force termination). + +4. Read the result from `s3://posthog/inbox_ranking/inbox_ranking_models/v1/dt=/metadata.json` (per-head AUCs, readability) and the examples parquet next to it: + + ```bash + AWS_ACCESS_KEY_ID=object_storage_root_user AWS_SECRET_ACCESS_KEY=object_storage_root_password \ + aws --endpoint-url http://localhost:19000 s3 cp s3://posthog/inbox_ranking/inbox_ranking_models/v1/dt=2026-08-25/metadata.json - + ``` + +Nothing here touches the prod bucket: the reader credential is read-only and the dag writes only to the local bucket. + ### Configuration | Setting | Default | Meaning | diff --git a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh new file mode 100755 index 000000000000..c6574940eb58 --- /dev/null +++ b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copy the prod inbox_report_state and inbox_report_labels partitions into local SeaweedFS so +# inbox_ranking_training_job can run from the local Dagster UI against real snapshots. +# +# products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh +# +# Two hops: prod S3 -> a disk cache -> the local object-storage bucket. The disk copy is what +# makes a re-sync cheap (aws s3 sync only moves new partitions) and doubles as input for +# notebooks. Only the dt= partitions the training job reads are copied; latest/ is skipped. +# S3 object metadata (snapshot-date, row-count) does not survive the download; the training +# job never reads it, so that is fine here. +# +# The prod read needs an active SSO session on the secrets profile: +# aws sso login --profile prod-us-secrets +set -euo pipefail + +SECRETS_PROFILE="${SECRETS_PROFILE:-prod-us-secrets}" +PROD_BUCKET="${INBOX_RANKING_PROD_BUCKET:-posthog-inbox-ranking-dataset-prod-us}" +PREFIX="${INBOX_RANKING_DATASET_S3_PREFIX:-inbox_ranking}" +LOCAL_ENDPOINT="${INBOX_RANKING_LOCAL_ENDPOINT:-http://localhost:19000}" +LOCAL_BUCKET="${OBJECT_STORAGE_BUCKET:-posthog}" +CACHE_DIR="${INBOX_RANKING_SYNC_DIR:-$HOME/.cache/posthog/inbox_ranking}" +TABLES=(inbox_report_state inbox_report_labels) + +if ! aws --profile "$SECRETS_PROFILE" sts get-caller-identity >/dev/null 2>&1; then + echo "No active SSO session. Run: aws sso login --profile $SECRETS_PROFILE" >&2 + exit 1 +fi + +secret_json=$(aws --profile "$SECRETS_PROFILE" secretsmanager get-secret-value \ + --secret-id posthog-inbox-ranking-dataset-reader \ + --query SecretString --output text) +reader_key=$(printf '%s' "$secret_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_key_id"])') +reader_secret=$(printf '%s' "$secret_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["secret_access_key"])') + +echo "==> prod -> $CACHE_DIR" +for table in "${TABLES[@]}"; do + AWS_ACCESS_KEY_ID="$reader_key" AWS_SECRET_ACCESS_KEY="$reader_secret" AWS_SESSION_TOKEN= \ + aws s3 sync "s3://$PROD_BUCKET/$PREFIX/$table/v1/" "$CACHE_DIR/$table/v1/" --region us-east-1 \ + --exclude "*" --include "dt=*/part-00000.parquet" +done + +echo "==> $CACHE_DIR -> $LOCAL_ENDPOINT/$LOCAL_BUCKET/$PREFIX" +for table in "${TABLES[@]}"; do + AWS_ACCESS_KEY_ID=object_storage_root_user AWS_SECRET_ACCESS_KEY=object_storage_root_password AWS_SESSION_TOKEN= \ + aws s3 sync "$CACHE_DIR/$table/v1/" "s3://$LOCAL_BUCKET/$PREFIX/$table/v1/" \ + --endpoint-url "$LOCAL_ENDPOINT" --region us-east-1 +done + +echo "==> partitions present in both tables locally:" +comm -12 \ + <(ls "$CACHE_DIR/${TABLES[0]}/v1" | sort) \ + <(ls "$CACHE_DIR/${TABLES[1]}/v1" | sort) From cf00435c8bd9d80e0b7995c56a6d8bbd2aaff837 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 29 Aug 2026 22:57:34 +0100 Subject: [PATCH 2/4] chore(signals): take the reader secret id from the environment Claude-Session: https://claude.ai/code/session_01DciwXDou2MVrmrGupjjJCh --- products/signals/dags/inbox_ranking/README.md | 4 ++-- .../dags/inbox_ranking/bin/sync_snapshots_local.sh | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md index 0a1e912424b3..608d52c65cab 100644 --- a/products/signals/dags/inbox_ranking/README.md +++ b/products/signals/dags/inbox_ranking/README.md @@ -80,11 +80,11 @@ s3://// The training job is S3-only, so it can run on a laptop against copies of the prod snapshots. The dataset job cannot: it needs the dogfood project's ClickHouse and cross-region Postgres. -1. Sync the two prefixes the job reads into the local object-storage bucket (needs an SSO session with the `secrets-editor` role on `prod-us-secrets`, granted through Access Elevator): +1. Sync the two prefixes the job reads into the local object-storage bucket. This needs an SSO session with the `secrets-editor` role on `prod-us-secrets`, and the Secrets Manager id of the dataset reader credential (provisioned with the bucket; ask the owning team): ```bash aws sso login --profile prod-us-secrets - products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh + INBOX_RANKING_READER_SECRET_ID= products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh ``` This copies `inbox_report_state/v1/dt=*` and `inbox_report_labels/v1/dt=*` to `~/.cache/posthog/inbox_ranking/` and from there into `s3://posthog/inbox_ranking/` on SeaweedFS (`localhost:19000`). It prints the partition days present in both tables; pick one of those as the partition to run. Re-runs only move new days. diff --git a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh index c6574940eb58..bfa649bef5aa 100755 --- a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh +++ b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh @@ -10,8 +10,10 @@ # S3 object metadata (snapshot-date, row-count) does not survive the download; the training # job never reads it, so that is fine here. # -# The prod read needs an active SSO session on the secrets profile: +# The prod read needs an active SSO session on the secrets profile and the id of the +# Secrets Manager secret that holds the reader credential (see the dag README): # aws sso login --profile prod-us-secrets +# INBOX_RANKING_READER_SECRET_ID=... products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh set -euo pipefail SECRETS_PROFILE="${SECRETS_PROFILE:-prod-us-secrets}" @@ -22,13 +24,18 @@ LOCAL_BUCKET="${OBJECT_STORAGE_BUCKET:-posthog}" CACHE_DIR="${INBOX_RANKING_SYNC_DIR:-$HOME/.cache/posthog/inbox_ranking}" TABLES=(inbox_report_state inbox_report_labels) +if [ -z "${INBOX_RANKING_READER_SECRET_ID:-}" ]; then + echo "Set INBOX_RANKING_READER_SECRET_ID to the Secrets Manager id of the dataset reader credential." >&2 + exit 1 +fi + if ! aws --profile "$SECRETS_PROFILE" sts get-caller-identity >/dev/null 2>&1; then echo "No active SSO session. Run: aws sso login --profile $SECRETS_PROFILE" >&2 exit 1 fi secret_json=$(aws --profile "$SECRETS_PROFILE" secretsmanager get-secret-value \ - --secret-id posthog-inbox-ranking-dataset-reader \ + --secret-id "$INBOX_RANKING_READER_SECRET_ID" \ --query SecretString --output text) reader_key=$(printf '%s' "$secret_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_key_id"])') reader_secret=$(printf '%s' "$secret_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["secret_access_key"])') From 2d9d5465f97512c3415fa9d49ec1255be5886de6 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 29 Aug 2026 23:10:44 +0100 Subject: [PATCH 3/4] chore(signals): wait for local object storage, honor its credentials, and mirror deletions in the snapshot sync Claude-Session: https://claude.ai/code/session_01DciwXDou2MVrmrGupjjJCh --- products/signals/dags/inbox_ranking/README.md | 8 +++--- .../inbox_ranking/bin/sync_snapshots_local.sh | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md index 608d52c65cab..e5b18e9c7e28 100644 --- a/products/signals/dags/inbox_ranking/README.md +++ b/products/signals/dags/inbox_ranking/README.md @@ -80,7 +80,7 @@ s3://// The training job is S3-only, so it can run on a laptop against copies of the prod snapshots. The dataset job cannot: it needs the dogfood project's ClickHouse and cross-region Postgres. -1. Sync the two prefixes the job reads into the local object-storage bucket. This needs an SSO session with the `secrets-editor` role on `prod-us-secrets`, and the Secrets Manager id of the dataset reader credential (provisioned with the bucket; ask the owning team): +1. With the dev stack up (`bin/start`; the script waits for object storage to accept requests), sync the two prefixes the job reads into the local object-storage bucket. This needs an SSO session with the `secrets-editor` role on `prod-us-secrets`, and the Secrets Manager id of the dataset reader credential (provisioned with the bucket; ask the owning team): ```bash aws sso login --profile prod-us-secrets @@ -89,9 +89,9 @@ The training job is S3-only, so it can run on a laptop against copies of the pro This copies `inbox_report_state/v1/dt=*` and `inbox_report_labels/v1/dt=*` to `~/.cache/posthog/inbox_ranking/` and from there into `s3://posthog/inbox_ranking/` on SeaweedFS (`localhost:19000`). It prints the partition days present in both tables; pick one of those as the partition to run. Re-runs only move new days. -2. Leave `INBOX_RANKING_DATASET_S3_BUCKET` unset: `common.s3_client()` then talks to SeaweedFS and `dataset_bucket()` is `posthog`. +2. Make sure `INBOX_RANKING_DATASET_S3_BUCKET` is unset in the shell that runs Dagster (`env | grep INBOX_RANKING`): `common.s3_client()` then talks to SeaweedFS and `dataset_bucket()` is `posthog`. If it is set, the dag uses your ambient AWS credentials against that bucket, and the read-only credential the sync used protects nothing. -3. With the dev stack up (`bin/start` runs `dagster dev` on http://localhost:3030 with the signals location loaded), materialize `inbox_ranking_training_job` for that partition from the UI, or from the CLI: +3. `bin/start` runs `dagster dev` on http://localhost:3030 with the signals location loaded; materialize `inbox_ranking_training_job` for that partition from the UI, or from the CLI: ```bash dagster job launch -w .dagster_home/workspace.yaml --location posthog.dags.locations.signals \ @@ -100,7 +100,7 @@ The training job is S3-only, so it can run on a laptop against copies of the pro The schedule is stopped outside prod US, so nothing runs unasked. If the run sits in `QUEUED`, check the daemon log for `Maximum is 10, won't launch more`: runs from a killed `dagster dev` stay `STARTED` forever and count against the local queue. Terminate them from the Runs page (force termination). -4. Read the result from `s3://posthog/inbox_ranking/inbox_ranking_models/v1/dt=/metadata.json` (per-head AUCs, readability) and the examples parquet next to it: +4. Read the result from `s3://posthog/inbox_ranking/inbox_ranking_models/v1/dt=/metadata.json` (per-head AUCs, readability); the examples are at `s3://posthog/inbox_ranking/inbox_ranking_training_examples/v1/dt=/part-00000.parquet`: ```bash AWS_ACCESS_KEY_ID=object_storage_root_user AWS_SECRET_ACCESS_KEY=object_storage_root_password \ diff --git a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh index bfa649bef5aa..a035a93f50a2 100755 --- a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh +++ b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh @@ -7,6 +7,8 @@ # Two hops: prod S3 -> a disk cache -> the local object-storage bucket. The disk copy is what # makes a re-sync cheap (aws s3 sync only moves new partitions) and doubles as input for # notebooks. Only the dt= partitions the training job reads are copied; latest/ is skipped. +# Both hops delete what the source no longer has, so a partition scrubbed from prod (see the +# README's retention section) leaves the cache and the local bucket on the next sync. # S3 object metadata (snapshot-date, row-count) does not survive the download; the training # job never reads it, so that is fine here. # @@ -21,6 +23,8 @@ PROD_BUCKET="${INBOX_RANKING_PROD_BUCKET:-posthog-inbox-ranking-dataset-prod-us} PREFIX="${INBOX_RANKING_DATASET_S3_PREFIX:-inbox_ranking}" LOCAL_ENDPOINT="${INBOX_RANKING_LOCAL_ENDPOINT:-http://localhost:19000}" LOCAL_BUCKET="${OBJECT_STORAGE_BUCKET:-posthog}" +LOCAL_KEY="${OBJECT_STORAGE_ACCESS_KEY_ID:-object_storage_root_user}" +LOCAL_SECRET="${OBJECT_STORAGE_SECRET_ACCESS_KEY:-object_storage_root_password}" CACHE_DIR="${INBOX_RANKING_SYNC_DIR:-$HOME/.cache/posthog/inbox_ranking}" TABLES=(inbox_report_state inbox_report_labels) @@ -34,6 +38,21 @@ if ! aws --profile "$SECRETS_PROFILE" sts get-caller-identity >/dev/null 2>&1; t exit 1 fi +# Checked before the download so a stack that is not up fails in seconds, not after the pull. +# The objectstorage container registers its credentials in a bootstrap loop after it starts, so +# a fresh stack rejects signed requests for a short while; retry rather than fail on the first. +for attempt in 1 2 3 4 5 6; do + if AWS_ACCESS_KEY_ID="$LOCAL_KEY" AWS_SECRET_ACCESS_KEY="$LOCAL_SECRET" AWS_SESSION_TOKEN= \ + aws --endpoint-url "$LOCAL_ENDPOINT" --region us-east-1 s3 ls "s3://$LOCAL_BUCKET/" >/dev/null 2>&1; then + break + fi + if [ "$attempt" -eq 6 ]; then + echo "Local object storage at $LOCAL_ENDPOINT is not accepting requests for bucket $LOCAL_BUCKET. Is the dev stack up (bin/start)?" >&2 + exit 1 + fi + sleep 5 +done + secret_json=$(aws --profile "$SECRETS_PROFILE" secretsmanager get-secret-value \ --secret-id "$INBOX_RANKING_READER_SECRET_ID" \ --query SecretString --output text) @@ -44,14 +63,14 @@ echo "==> prod -> $CACHE_DIR" for table in "${TABLES[@]}"; do AWS_ACCESS_KEY_ID="$reader_key" AWS_SECRET_ACCESS_KEY="$reader_secret" AWS_SESSION_TOKEN= \ aws s3 sync "s3://$PROD_BUCKET/$PREFIX/$table/v1/" "$CACHE_DIR/$table/v1/" --region us-east-1 \ - --exclude "*" --include "dt=*/part-00000.parquet" + --delete --exclude "*" --include "dt=*/part-00000.parquet" done echo "==> $CACHE_DIR -> $LOCAL_ENDPOINT/$LOCAL_BUCKET/$PREFIX" for table in "${TABLES[@]}"; do - AWS_ACCESS_KEY_ID=object_storage_root_user AWS_SECRET_ACCESS_KEY=object_storage_root_password AWS_SESSION_TOKEN= \ + AWS_ACCESS_KEY_ID="$LOCAL_KEY" AWS_SECRET_ACCESS_KEY="$LOCAL_SECRET" AWS_SESSION_TOKEN= \ aws s3 sync "$CACHE_DIR/$table/v1/" "s3://$LOCAL_BUCKET/$PREFIX/$table/v1/" \ - --endpoint-url "$LOCAL_ENDPOINT" --region us-east-1 + --endpoint-url "$LOCAL_ENDPOINT" --region us-east-1 --delete done echo "==> partitions present in both tables locally:" From ee66b05defcfeb078aefe0152978fc971508bf35 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Mon, 31 Aug 2026 12:30:07 +0100 Subject: [PATCH 4/4] chore(signals): harden the local snapshot sync script and its docs - prefer OBJECT_STORAGE_ENDPOINT over the localhost default for the local hop - turn xtrace off before the reader credential enters the shell - list partitions from parquet files, not directories a scrub leaves empty - README: check .env.local for INBOX_RANKING_DATASET_S3_BUCKET, pick the newest common partition, note the dev bucket is unauthenticated Claude-Session: https://claude.ai/code/session_01Y9VyXUkutac4vDhgEsDjDT --- products/signals/dags/inbox_ranking/README.md | 6 ++++-- .../inbox_ranking/bin/sync_snapshots_local.sh | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md index e5b18e9c7e28..707e8d7b85cf 100644 --- a/products/signals/dags/inbox_ranking/README.md +++ b/products/signals/dags/inbox_ranking/README.md @@ -87,9 +87,11 @@ The training job is S3-only, so it can run on a laptop against copies of the pro INBOX_RANKING_READER_SECRET_ID= products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh ``` - This copies `inbox_report_state/v1/dt=*` and `inbox_report_labels/v1/dt=*` to `~/.cache/posthog/inbox_ranking/` and from there into `s3://posthog/inbox_ranking/` on SeaweedFS (`localhost:19000`). It prints the partition days present in both tables; pick one of those as the partition to run. Re-runs only move new days. + This copies `inbox_report_state/v1/dt=*` and `inbox_report_labels/v1/dt=*` to `~/.cache/posthog/inbox_ranking/` and from there into `s3://posthog/inbox_ranking/` on SeaweedFS (`localhost:19000`). It prints the partition days present in both tables; pick the **newest** of those as the partition to run. The examples asset reads the `INBOX_RANKING_TRAINING_LOOKBACK_DAYS` snapshots behind the partition it runs for, so an early day has little history behind it and trains on almost nothing. Re-runs only move new days. -2. Make sure `INBOX_RANKING_DATASET_S3_BUCKET` is unset in the shell that runs Dagster (`env | grep INBOX_RANKING`): `common.s3_client()` then talks to SeaweedFS and `dataset_bucket()` is `posthog`. If it is set, the dag uses your ambient AWS credentials against that bucket, and the read-only credential the sync used protects nothing. + Note that the dev stack's object storage accepts unsigned requests and publishes port 19000 on every host interface, so anything synced here (and the disk cache) is readable by whoever can reach your machine. The snapshots are internal dogfood telemetry, not customer data, but run the sync on a network you trust. + +2. Make sure `INBOX_RANKING_DATASET_S3_BUCKET` is unset for the Dagster process: `common.s3_client()` then talks to SeaweedFS and `dataset_bucket()` is `posthog`. Checking your shell (`env | grep INBOX_RANKING`) is not enough, because `bin/start` sources `.env.local` into the processes it starts; check that file too. If the variable reaches Dagster, the dag uses your ambient AWS credentials against that bucket, and the read-only credential the sync used protects nothing. 3. `bin/start` runs `dagster dev` on http://localhost:3030 with the signals location loaded; materialize `inbox_ranking_training_job` for that partition from the UI, or from the CLI: diff --git a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh index a035a93f50a2..2e8a01fac342 100755 --- a/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh +++ b/products/signals/dags/inbox_ranking/bin/sync_snapshots_local.sh @@ -21,7 +21,7 @@ set -euo pipefail SECRETS_PROFILE="${SECRETS_PROFILE:-prod-us-secrets}" PROD_BUCKET="${INBOX_RANKING_PROD_BUCKET:-posthog-inbox-ranking-dataset-prod-us}" PREFIX="${INBOX_RANKING_DATASET_S3_PREFIX:-inbox_ranking}" -LOCAL_ENDPOINT="${INBOX_RANKING_LOCAL_ENDPOINT:-http://localhost:19000}" +LOCAL_ENDPOINT="${INBOX_RANKING_LOCAL_ENDPOINT:-${OBJECT_STORAGE_ENDPOINT:-http://localhost:19000}}" LOCAL_BUCKET="${OBJECT_STORAGE_BUCKET:-posthog}" LOCAL_KEY="${OBJECT_STORAGE_ACCESS_KEY_ID:-object_storage_root_user}" LOCAL_SECRET="${OBJECT_STORAGE_SECRET_ACCESS_KEY:-object_storage_root_password}" @@ -53,6 +53,9 @@ for attempt in 1 2 3 4 5 6; do sleep 5 done +# An inherited xtrace (bash -x) would print the SecretString and every later expansion of the +# reader keys; force it off before the credential enters the shell. +{ set +x; } 2>/dev/null secret_json=$(aws --profile "$SECRETS_PROFILE" secretsmanager get-secret-value \ --secret-id "$INBOX_RANKING_READER_SECRET_ID" \ --query SecretString --output text) @@ -74,6 +77,13 @@ for table in "${TABLES[@]}"; do done echo "==> partitions present in both tables locally:" -comm -12 \ - <(ls "$CACHE_DIR/${TABLES[0]}/v1" | sort) \ - <(ls "$CACHE_DIR/${TABLES[1]}/v1" | sort) +# aws s3 sync --delete removes the parquet of a scrubbed partition but leaves its empty dt= +# directory, so enumerate the files, not the directories. +list_partitions() { + local f + for f in "$CACHE_DIR/$1/v1"/dt=*/part-00000.parquet; do + [ -e "$f" ] || continue + basename "$(dirname "$f")" + done | sort +} +comm -12 <(list_partitions "${TABLES[0]}") <(list_partitions "${TABLES[1]}")