fix: log multi-statement batches and add data_inlining_row_limit config (#367)

Fix a logging gap: executeSingleStatement (used for multi-statement
batches like sqlmesh's "BEGIN; ALTER TABLE; INSERT; COMMIT") had zero
logQuery calls. All statements in batches are now logged with
protocol="simple-batch", making sqlmesh operations visible in
system.query_log.

Also adds a configurable data_inlining_row_limit option for DuckLake,
settable via YAML config or DUCKGRES_DUCKLAKE_DATA_INLINING_ROW_LIMIT
env var. Set to 0 to disable data inlining (new default in DuckLake 0.4).
Negative values are rejected with a warning.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: fuziontech

config_resolution.go

@@ -271,6 +271,13 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 				warn("Invalid ducklake.checkpoint_interval duration: " + err.Error())
 			}
 		}
+		if fileCfg.DuckLake.DataInliningRowLimit != nil {
+			if *fileCfg.DuckLake.DataInliningRowLimit < 0 {
+				warn("ducklake.data_inlining_row_limit must be >= 0")
+			} else {
+				cfg.DuckLake.DataInliningRowLimit = fileCfg.DuckLake.DataInliningRowLimit
+			}
+		}
 
 		cfg.ProcessIsolation = fileCfg.ProcessIsolation
 		if fileCfg.IdleTimeout != "" {
@@ -503,6 +510,17 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 			warn("Invalid DUCKGRES_DUCKLAKE_CHECKPOINT_INTERVAL duration: " + err.Error())
 		}
 	}
+	if v := getenv("DUCKGRES_DUCKLAKE_DATA_INLINING_ROW_LIMIT"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			if n < 0 {
+				warn("DUCKGRES_DUCKLAKE_DATA_INLINING_ROW_LIMIT must be >= 0")
+			} else {
+				cfg.DuckLake.DataInliningRowLimit = &n
+			}
+		} else {
+			warn("Invalid DUCKGRES_DUCKLAKE_DATA_INLINING_ROW_LIMIT: " + err.Error())
+		}
+	}
 	if v := getenv("DUCKGRES_PROCESS_ISOLATION"); v != "" {
 		if b, err := strconv.ParseBool(v); err == nil {
 			cfg.ProcessIsolation = b

main.go

@@ -125,6 +125,10 @@ type DuckLakeFileConfig struct {
 
 	// Checkpoint interval for DuckLake maintenance (e.g., "24h", "6h")
 	CheckpointInterval string `yaml:"checkpoint_interval"`
+
+	// DataInliningRowLimit controls max rows inlined in metadata per insert.
+	// Set to 0 to disable inlining. Omit to use DuckLake default (10).
+	DataInliningRowLimit *int `yaml:"data_inlining_row_limit"`
 }
 
 // loadConfigFile loads configuration from a YAML file

server/conn.go

@@ -1252,6 +1252,8 @@ func (c *clientConn) handleMultiStatementQuery(tree *pg_query.ParseResult) error
 // client (so the caller can stop processing a batch), or (false, err) for
 // fatal connection errors.
 func (c *clientConn) executeSingleStatement(query string) (errSent bool, fatalErr error) {
+	start := time.Now()
+
 	// Check for cursor operations before transpilation
 	tree, parseErr := pg_query.Parse(query)
 	if parseErr == nil && len(tree.Stmts) == 1 {
@@ -1466,20 +1468,29 @@ func (c *clientConn) executeSingleStatement(query string) (errSent bool, fatalEr
 				}
 			}
 			if err != nil {
+				errCode := "42000"
+				errMsg := err.Error()
 				if isQueryCancelled(err) {
-					c.sendError("ERROR", "57014", "canceling statement due to user request")
+					errCode = "57014"
+					errMsg = "canceling statement due to user request"
 				} else {
 					slog.Error("Query execution failed.", "user", c.username, "query", executedQuery, "error", err)
-					c.sendError("ERROR", "42000", err.Error())
 				}
+				c.sendError("ERROR", errCode, errMsg)
 				c.setTxError()
+				c.logQuery(start, query, executedQuery, cmdType, 0, 0, errCode, errMsg, "simple-batch")
 				return true, nil
 			}
 		}
 
+		var writtenRows int64
+		if execResult != nil {
+			writtenRows, _ = execResult.RowsAffected()
+		}
 		c.updateTxStatus(cmdType)
 		tag := c.buildCommandTag(cmdType, execResult)
 		_ = writeCommandComplete(c.writer, tag)
+		c.logQuery(start, query, executedQuery, cmdType, 0, writtenRows, "", "", "simple-batch")
 		return false, nil
 	}
 
@@ -1489,13 +1500,17 @@ func (c *clientConn) executeSingleStatement(query string) (errSent bool, fatalEr
 
 	rows, err := c.executor.QueryContext(ctx, executedQuery)
 	if err != nil {
+		errCode := "42000"
+		errMsg := err.Error()
 		if isQueryCancelled(err) {
-			c.sendError("ERROR", "57014", "canceling statement due to user request")
+			errCode = "57014"
+			errMsg = "canceling statement due to user request"
 		} else {
 			slog.Error("Query execution failed.", "user", c.username, "query", executedQuery, "error", err)
-			c.sendError("ERROR", "42000", err.Error())
 		}
+		c.sendError("ERROR", errCode, errMsg)
 		c.setTxError()
+		c.logQuery(start, query, executedQuery, cmdType, 0, 0, errCode, errMsg, "simple-batch")
 		return true, nil
 	}
 	defer func() { _ = rows.Close() }()
@@ -1546,6 +1561,7 @@ func (c *clientConn) executeSingleStatement(query string) (errSent bool, fatalEr
 	c.updateTxStatus(cmdType)
 	tag := buildCommandTagFromRowCount(cmdType, int64(rowCount))
 	_ = writeCommandComplete(c.writer, tag)
+	c.logQuery(start, query, executedQuery, cmdType, int64(rowCount), 0, "", "", "simple-batch")
 	return false, nil
 }
 

server/ducklake_migration.go

@@ -531,6 +531,9 @@ func buildDuckLakeAttachStmt(dlCfg DuckLakeConfig, migrate bool) string {
 	if migrate {
 		options = append(options, "AUTOMATIC_MIGRATION TRUE")
 	}
+	if dlCfg.DataInliningRowLimit != nil {
+		options = append(options, fmt.Sprintf("DATA_INLINING_ROW_LIMIT %d", *dlCfg.DataInliningRowLimit))
+	}
 
 	if len(options) > 0 {
 		return fmt.Sprintf("ATTACH 'ducklake:%s' AS ducklake (%s)",

server/ducklake_migration_test.go

@@ -79,6 +79,33 @@ func TestBuildDuckLakeAttachStmt(t *testing.T) {
 			migrate: false,
 			want:    "ATTACH 'ducklake:postgres:host=localhost password=it''s_secret' AS ducklake",
 		},
+		{
+			name: "with data inlining disabled",
+			dlCfg: DuckLakeConfig{
+				MetadataStore:        "postgres:host=localhost dbname=dl",
+				DataInliningRowLimit: intPtr(0),
+			},
+			migrate: false,
+			want:    "ATTACH 'ducklake:postgres:host=localhost dbname=dl' AS ducklake (DATA_INLINING_ROW_LIMIT 0)",
+		},
+		{
+			name: "with data inlining custom limit",
+			dlCfg: DuckLakeConfig{
+				MetadataStore:        "postgres:host=localhost dbname=dl",
+				ObjectStore:          "s3://bucket/path",
+				DataInliningRowLimit: intPtr(100),
+			},
+			migrate: false,
+			want:    "ATTACH 'ducklake:postgres:host=localhost dbname=dl' AS ducklake (DATA_PATH 's3://bucket/path', DATA_INLINING_ROW_LIMIT 100)",
+		},
+		{
+			name: "nil data inlining uses default",
+			dlCfg: DuckLakeConfig{
+				MetadataStore: "postgres:host=localhost dbname=dl",
+			},
+			migrate: false,
+			want:    "ATTACH 'ducklake:postgres:host=localhost dbname=dl' AS ducklake",
+		},
 	}
 
 	for _, tt := range tests {
@@ -91,6 +118,8 @@ func TestBuildDuckLakeAttachStmt(t *testing.T) {
 	}
 }
 
+func intPtr(n int) *int { return &n }
+
 func TestFormatSQLValue(t *testing.T) {
 	tests := []struct {
 		name string

server/server.go

@@ -240,6 +240,11 @@ type DuckLakeConfig struct {
 	// Set to 0 to disable. Default: 24h.
 	CheckpointInterval time.Duration
 
+	// DataInliningRowLimit controls the maximum number of rows to inline
+	// in DuckLake metadata instead of writing to Parquet files.
+	// Set to pointer-to-0 to disable inlining. nil uses DuckLake default (10).
+	DataInliningRowLimit *int
+
 	// Migrate is set by the control plane after running the migration check.
 	// When true, AttachDuckLake uses AUTOMATIC_MIGRATION TRUE without
 	// re-running the version check. This avoids redundant backups and