fix(session): raise session-init timeout to 60s and drop the duplicate catalog probe (#1192)

Two fixes for client-facing session-creation failures on tenants with large
DuckLake catalogs, found while root-causing a mw-prod-us incident on portola.

1. DefaultSessionInitTimeout 10s -> 60s.

Session init runs its probes with DuckLake as the session default, so each
statement starts a DuckLake transaction. When the catalog's schema version
moved since the DuckDB instance last loaded it, that transaction pays a full
catalog reload first. Measured on mw-prod-us (~5,900 live tables, ~1.0M live
columns) the reload takes 12-22s, so the 10s budget expired mid-reload. The
failure is fatal and client-facing: the worker returns "detect ducklake
attachment: INTERRUPT Error", flight_handler maps it to ResourceExhausted, and
the CP refuses the connection with 58000 without retrying this error class. It
hit ~3% of worker spawns for that tenant.

60s rather than 30s because one reload is not the worst case: these tenants
churn DDL continuously, so a second schema-version bump inside one init window
is plausible. Nothing upstream clips it - the CP derives the init context from
the cancel-only connection context, the worker queue timeout is minutes, and
workerSpawnActivateTimeout governs the spawn/activate phase.

2. Stop probing for the DuckLake catalog twice per session create.

Every session-init caller ran HasAttachedCatalog and then called
InitSessionDatabaseMetadata*, which ran it again internally - two
duckdb_databases() queries per session create, each starting a DuckLake
transaction on the path above. InitSessionDatabaseMetadataWithAttached takes
the caller's answer; the probe-or-reuse branch lives in an unexported helper so
the probe keeps its original position in the statement order.

Tests: sessionmeta gains coverage asserting zero probe queries and the exact
deferred statements (USE ducklake / SET search_path) for attached and
unattached, plus the probing entry point still issuing exactly one probe. The
e2e-skip rationale is recorded in harness.sh above logical_catalog_alias, which
already drives this path. Docs and help text follow the 60s default.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: fuziontech

README.md

@@ -357,7 +357,7 @@ Create a `duckgres.yaml` file (see `duckgres.example.yaml` for a complete exampl
 host: "0.0.0.0"
 port: 5432
 data_dir: "./data"
-session_init_timeout: "10s"
+session_init_timeout: "60s"
 admission_reclaimer_max_reservations: 4096
 
 tls:
@@ -424,7 +424,7 @@ Run with config file:
 | `DUCKGRES_PROCESS_RETIRE_ON_SESSION_END` | Retire a process worker immediately after its last session ends instead of keeping it warm for reuse | `false` |
 | `DUCKGRES_IDLE_TIMEOUT` | Connection idle timeout (e.g., `30m`, `1h`, `-1` to disable) | `24h` standalone, `5m` control-plane |
 | `DUCKGRES_CLIENT_IDLE_TIMEOUT_MAX` | Maximum client-requested `duckgres.idle_timeout`; unset disables client overrides | disabled |
-| `DUCKGRES_SESSION_INIT_TIMEOUT` | Session startup metadata initialization and catalog probe timeout | `10s` |
+| `DUCKGRES_SESSION_INIT_TIMEOUT` | Session startup metadata initialization and catalog probe timeout | `60s` |
 | `DUCKGRES_WORKER_QUEUE_TIMEOUT` | Max time to wait for worker acquisition and per-org/per-user vCPU resource admission; the managed K8s queue TTL uses this value | `60s` |
 | `DUCKGRES_ADMISSION_RECLAIMER_MAX_RESERVATIONS` | Max queued/live admission identities whose cleanup ownership one control plane may retain; new admissions are rejected before enqueue when full | `4096` |
 | `DUCKGRES_HANDOVER_DRAIN_TIMEOUT` | Max time to drain planned shutdowns and upgrades before forcing exit | `24h` in process mode, `15m` in remote K8s mode |

controlplane/control.go

@@ -2070,7 +2070,9 @@ func (cp *ControlPlane) initSessionMetadata(
 		// the connection selected one — so current_database() and the
 		// information_schema views agree with the name the client connected
 		// with. Everything that executes below still uses effectiveCatalog.
-		if err := sessionmeta.InitSessionDatabaseMetadataWithAccess(initCtx, exec, res.visibleCatalog, metadataAccess); err != nil {
+		// duckLakeAttached comes from the probe above; passing it keeps
+		// sessionmeta from repeating that `duckdb_databases()` query.
+		if err := sessionmeta.InitSessionDatabaseMetadataWithAttached(initCtx, exec, res.visibleCatalog, metadataAccess, duckLakeAttached); err != nil {
 			initContextErr := initCtx.Err()
 			initCancel()
 			outcome, reason := controlPlaneSessionStartOperationResult(

duckdbservice/service.go

@@ -1845,7 +1845,9 @@ func initAttachedDuckLakeSessionMetadata(conn *sql.Conn, db *sql.DB, cfg server.
 	if !duckLakeAttached {
 		return nil
 	}
-	if err := sessionmeta.InitSessionDatabaseMetadata(initCtx, executor, "ducklake"); err != nil {
+	// Reuse the probe above instead of letting sessionmeta run a second
+	// `duckdb_databases()` query against the DuckLake catalog.
+	if err := sessionmeta.InitSessionDatabaseMetadataWithAttached(initCtx, executor, "ducklake", nil, duckLakeAttached); err != nil {
 		return fmt.Errorf("initialize ducklake session metadata: %w", err)
 	}
 	return nil

k8s/README.md

@@ -63,7 +63,7 @@ Key flags for Kubernetes multitenant mode:
 |------|---------|-------------|
 | `--worker-backend remote` | - | Use K8s remote workers in config-store-backed multitenant mode |
 | `--config-store` | `DUCKGRES_CONFIG_STORE` | PostgreSQL config-store connection string required for remote mode |
-| `--session-init-timeout` | `DUCKGRES_SESSION_INIT_TIMEOUT` | Session startup metadata initialization and catalog probe timeout (`10s` default) |
+| `--session-init-timeout` | `DUCKGRES_SESSION_INIT_TIMEOUT` | Session startup metadata initialization and catalog probe timeout (`60s` default) |
 | `--handover-drain-timeout` | `DUCKGRES_HANDOVER_DRAIN_TIMEOUT` | Max time to drain planned shutdowns/upgrades before forced exit (`15m` default in remote mode) |
 | `--sni-routing-mode` | `DUCKGRES_SNI_ROUTING_MODE` | Managed-hostname routing: `off`, `passthrough`, or `enforce`. Postgres uses requested dbname first; managed SNI must resolve to the same org, and SNI supplies the database only when dbname is empty |
 | `--managed-hostname-suffixes` | `DUCKGRES_MANAGED_HOSTNAME_SUFFIXES` | Comma-separated managed hostname suffixes such as `.dw.test.local` |

main.go

@@ -129,7 +129,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_PROCESS_ISOLATION  Enable process isolation (1 or true)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_IDLE_TIMEOUT       Connection idle timeout (e.g., 30m, 1h, -1 to disable)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_CLIENT_IDLE_TIMEOUT_MAX  Maximum client-requested idle timeout; unset disables client overrides\n")
-		fmt.Fprintf(os.Stderr, "  DUCKGRES_SESSION_INIT_TIMEOUT  Session startup metadata/probe timeout (default: 10s)\n")
+		fmt.Fprintf(os.Stderr, "  DUCKGRES_SESSION_INIT_TIMEOUT  Session startup metadata/probe timeout (default: 60s)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_MEMORY_LIMIT       DuckDB memory_limit per session (e.g., 4GB)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_THREADS            DuckDB threads per session\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_MEMORY_BUDGET      Total memory for all DuckDB sessions (e.g., 24GB)\n")
@@ -149,7 +149,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_DUCKDB_TOKEN       DuckDB service bearer token (duckdb-service mode)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_DUCKDB_MAX_SESSIONS  DuckDB service max sessions (duckdb-service mode)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_CONFIG_STORE       PostgreSQL connection string for config store (multi-tenant)\n")
-		fmt.Fprintf(os.Stderr, "  DUCKGRES_CONFIG_POLL_INTERVAL  Config store poll interval (default: 30s)\n")
+		fmt.Fprintf(os.Stderr, "  DUCKGRES_CONFIG_POLL_INTERVAL  Config store poll interval (default: 60s)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_INTERNAL_SECRET    Shared secret for API authentication\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_INTERNAL_SECRET_FALLBACKS  Comma-separated previous internal secrets still accepted during rotation\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_AWS_REGION         AWS region for STS client\n")

main_test.go

@@ -594,11 +594,11 @@ func TestResolveEffectiveConfigInvalidWorkerEnvVars(t *testing.T) {
 
 func TestResolveEffectiveConfigSessionInitTimeout(t *testing.T) {
 	resolved := configresolve.ResolveEffective(nil, configresolve.CLIInputs{}, nil, nil)
-	if resolved.SessionInitTimeout != 10*time.Second {
-		t.Fatalf("expected default session init timeout 10s, got %s", resolved.SessionInitTimeout)
+	if resolved.SessionInitTimeout != 60*time.Second {
+		t.Fatalf("expected default session init timeout 60s, got %s", resolved.SessionInitTimeout)
 	}
-	if resolved.Server.SessionInitTimeout != 10*time.Second {
-		t.Fatalf("expected server default session init timeout 10s, got %s", resolved.Server.SessionInitTimeout)
+	if resolved.Server.SessionInitTimeout != 60*time.Second {
+		t.Fatalf("expected server default session init timeout 60s, got %s", resolved.Server.SessionInitTimeout)
 	}
 }
 

server/conn.go

@@ -1052,7 +1052,9 @@ func (c *clientConn) serve() error {
 		if duckLakeAttached {
 			catalog = physicalDuckLakeCatalog
 		}
-		if err := sessionmeta.InitSessionDatabaseMetadata(initCtx, c.executor, catalog); err != nil {
+		// duckLakeAttached comes from the probe above; passing it keeps
+		// sessionmeta from repeating that `duckdb_databases()` query.
+		if err := sessionmeta.InitSessionDatabaseMetadataWithAttached(initCtx, c.executor, catalog, nil, duckLakeAttached); err != nil {
 			initCancel()
 			c.sendError("FATAL", "XX000", fmt.Sprintf("failed to initialize session database metadata: %v", err))
 			return err

server/server.go

@@ -43,7 +43,33 @@ type DuckLakeConfig = ducklake.Config
 const DefaultDuckLakeSpecVersion = ducklake.DefaultSpecVersion
 
 // DefaultSessionInitTimeout bounds startup metadata initialization and catalog probes.
-const DefaultSessionInitTimeout = 10 * time.Second
+//
+// 10s → 60s (2026-09-16): session init runs its probes with the DuckLake
+// catalog as the session default, so each statement starts a DuckLake
+// transaction. When the catalog's schema version changed since the DuckDB
+// instance last loaded it, that transaction pays a full catalog reload before
+// the probe returns. On a large tenant catalog (measured on mw-prod-us: ~5,900
+// live tables, ~1.0M live columns) the reload takes 12-22s, so a 10s budget
+// expired mid-reload. The failure is client-facing and fatal: the worker
+// returns "detect ducklake attachment: INTERRUPT Error", flight_handler maps
+// it to ResourceExhausted, and the control plane refuses the connection with
+// 58000 rather than retrying. It hit ~3% of worker spawns for that tenant.
+//
+// 60s rather than 30s because one reload is not the worst case: the tenants
+// that hit this churn DDL (sqlmesh creates and drops tables continuously), so
+// a second schema-version bump inside a single init window is plausible, and
+// 30s covers one 12-22s reload but not two.
+//
+// This bounds a pathological metadata store, not the steady state: a warm
+// instance completes session init in milliseconds, so raising the ceiling
+// costs nothing on a healthy path. The only price is a slower time-to-FATAL
+// for the one client already waiting on that worker. Override per deployment
+// with DUCKGRES_SESSION_INIT_TIMEOUT / --session-init-timeout.
+//
+// Nothing upstream clips this: the control plane derives the init context from
+// the cancel-only connection context, the worker queue timeout is minutes, and
+// workerSpawnActivateTimeout governs the spawn/activate phase instead.
+const DefaultSessionInitTimeout = 60 * time.Second
 
 // Re-exports of the migration / backup / delta-path entry points so callers
 // that referenced them under the server package continue to compile after

server/sessionmeta/sessionmeta.go

@@ -37,8 +37,31 @@ func InitSessionDatabaseMetadata(ctx context.Context, executor sqlcore.QueryExec
 }
 
 // InitSessionDatabaseMetadataWithAccess installs metadata views filtered to
-// the relations the current principal can query.
+// the relations the current principal can query. It probes for the DuckLake
+// catalog itself; callers that already hold that answer should pass it to
+// InitSessionDatabaseMetadataWithAttached instead.
 func InitSessionDatabaseMetadataWithAccess(ctx context.Context, executor sqlcore.QueryExecutor, catalog string, access *MetadataAccessPolicy) error {
+	return initSessionDatabaseMetadata(ctx, executor, catalog, access, nil)
+}
+
+// InitSessionDatabaseMetadataWithAttached is InitSessionDatabaseMetadataWithAccess
+// with the DuckLake attachment state supplied by the caller.
+//
+// Every session-init caller already runs HasAttachedCatalog before this
+// function, which then ran it a second time — two `duckdb_databases()` queries
+// per session create. With DuckLake as the session default each of those
+// starts a DuckLake transaction, and a transaction on a catalog whose schema
+// version moved pays a full reload. Passing the known value drops the second
+// probe.
+func InitSessionDatabaseMetadataWithAttached(ctx context.Context, executor sqlcore.QueryExecutor, catalog string, access *MetadataAccessPolicy, duckLakeAttached bool) error {
+	return initSessionDatabaseMetadata(ctx, executor, catalog, access, &duckLakeAttached)
+}
+
+// initSessionDatabaseMetadata does the work for both exported entry points. A
+// nil `attached` probes; a non-nil one reuses the caller's answer. The pointer
+// stays unexported so no caller has to reason about a nil-able bool, and the
+// probe keeps its original position in the statement order.
+func initSessionDatabaseMetadata(ctx context.Context, executor sqlcore.QueryExecutor, catalog string, access *MetadataAccessPolicy, attached *bool) error {
 	if executor == nil {
 		return fmt.Errorf("session executor is required")
 	}
@@ -55,9 +78,15 @@ func InitSessionDatabaseMetadataWithAccess(ctx context.Context, executor sqlcore
 		return fmt.Errorf("create current_database() macro: %w", err)
 	}
 
-	duckLakeAttached, err := HasAttachedCatalog(ctx, executor, "ducklake")
-	if err != nil {
-		return fmt.Errorf("detect ducklake attachment: %w", err)
+	var duckLakeAttached bool
+	if attached != nil {
+		duckLakeAttached = *attached
+	} else {
+		probed, err := HasAttachedCatalog(ctx, executor, "ducklake")
+		if err != nil {
+			return fmt.Errorf("detect ducklake attachment: %w", err)
+		}
+		duckLakeAttached = probed
 	}
 
 	if _, err := executor.ExecContext(ctx, "USE memory"); err != nil {

server/sessionmeta/sessionmeta_test.go

@@ -12,10 +12,12 @@ import (
 type countingExecutor struct {
 	execCalls   int
 	execQueries []string
+	queryCalls  int
 	queryRows   sqlcore.RowSet
 }
 
 func (e *countingExecutor) QueryContext(_ context.Context, _ string, _ ...any) (sqlcore.RowSet, error) {
+	e.queryCalls++
 	if e.queryRows == nil {
 		return nil, errors.New("no query rows configured")
 	}
@@ -122,3 +124,82 @@ func TestBuildSessionMetadataSQLContainsAllExpectedStatements(t *testing.T) {
 		t.Errorf("buildSessionMetadataSQL did not interpolate database literal")
 	}
 }
+
+// TestInitSessionDatabaseMetadataWithAttachedSkipsProbe pins the duplicate-probe
+// fix. Every session-init caller runs HasAttachedCatalog before calling in here,
+// and this function used to run it a second time — two `duckdb_databases()`
+// queries per session create. With DuckLake as the session default each one
+// starts a DuckLake transaction, and a transaction on a catalog whose schema
+// version moved pays a full catalog reload. The executor below has no queryRows
+// configured, so any probe would both bump queryCalls and fail the call.
+func TestInitSessionDatabaseMetadataWithAttachedSkipsProbe(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		attached bool
+		// The deferred restore runs only when DuckLake is attached. Assert on
+		// the statements themselves: a bare count would still pass if the
+		// restore ran against the wrong catalog.
+		wantRestore bool
+	}{
+		{name: "attached", attached: true, wantRestore: true},
+		{name: "not attached", attached: false, wantRestore: false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			exec := &countingExecutor{}
+
+			if err := InitSessionDatabaseMetadataWithAttached(
+				context.Background(), exec, "analytics", nil, tc.attached,
+			); err != nil {
+				t.Fatalf("InitSessionDatabaseMetadataWithAttached: %v", err)
+			}
+
+			if exec.queryCalls != 0 {
+				t.Errorf("issued %d probe queries, want 0 — the caller's answer must be reused", exec.queryCalls)
+			}
+
+			joined := strings.Join(exec.execQueries, "\n---\n")
+			gotRestore := false
+			gotSearchPath := false
+			for _, q := range exec.execQueries {
+				if strings.EqualFold(strings.TrimSpace(q), "USE ducklake") {
+					gotRestore = true
+				}
+				if strings.HasPrefix(strings.TrimSpace(q), "SET search_path") {
+					gotSearchPath = true
+				}
+			}
+			if gotRestore != tc.wantRestore {
+				t.Errorf("`USE ducklake` issued = %v, want %v.\nQueries:\n%s", gotRestore, tc.wantRestore, joined)
+			}
+			if gotSearchPath != tc.wantRestore {
+				t.Errorf("`SET search_path` issued = %v, want %v.\nQueries:\n%s", gotSearchPath, tc.wantRestore, joined)
+			}
+			// The session must always leave the memory catalog and install the
+			// metadata surfaces, attached or not.
+			if !strings.Contains(joined, "USE memory") {
+				t.Errorf("did not switch to the memory catalog.\nQueries:\n%s", joined)
+			}
+			if !strings.Contains(joined, "current_database()") {
+				t.Errorf("did not create the current_database() macro.\nQueries:\n%s", joined)
+			}
+		})
+	}
+}
+
+// TestInitSessionDatabaseMetadataProbesWhenAttachmentUnknown keeps the nil case
+// probing, so the exported wrappers and any caller that does not already hold
+// the answer behave exactly as before.
+func TestInitSessionDatabaseMetadataProbesWhenAttachmentUnknown(t *testing.T) {
+	exec := &countingExecutor{queryRows: &singleIntRow{v: 1}}
+
+	// The probing entry point must still issue exactly one probe.
+	if err := InitSessionDatabaseMetadataWithAccess(
+		context.Background(), exec, "analytics", nil,
+	); err != nil {
+		t.Fatalf("InitSessionDatabaseMetadataWithAccess: %v", err)
+	}
+
+	if exec.queryCalls != 1 {
+		t.Errorf("issued %d probe queries, want 1", exec.queryCalls)
+	}
+}