diff --git a/sei-db/seiwal/metrics.go b/sei-db/seiwal/metrics.go new file mode 100644 index 0000000000..b845fb599d --- /dev/null +++ b/sei-db/seiwal/metrics.go @@ -0,0 +1,95 @@ +package seiwal + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + commonmetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" +) + +// The name of the OpenTelemetry meter for WAL metrics. +const walMeterName = "seidb_seiwal" + +// Instruments are shared process-wide (created once); individual WAL instances are distinguished by the +// "wal" attribute attached at each recording (see walNameAttr), mirroring LittDB's per-table labeling. This +// keeps metrics from multiple instances in one process from clobbering each other. +var ( + walMeter = otel.Meter(walMeterName) + + // The number of records appended to the WAL. + walRecordsWritten = must(walMeter.Int64Counter( + "seiwal_records_written", + metric.WithDescription("Number of records appended to the WAL"), + metric.WithUnit("{count}"), + )) + + // The number of record bytes appended to the WAL (including framing). + walBytesWritten = must(walMeter.Int64Counter( + "seiwal_bytes_written", + metric.WithDescription("Number of bytes written to the WAL"), + metric.WithUnit("By"), + )) + + // The number of WAL files sealed (rotated) after reaching the target size. + walFilesSealed = must(walMeter.Int64Counter( + "seiwal_files_sealed", + metric.WithDescription("Number of WAL files sealed on rotation"), + metric.WithUnit("{count}"), + )) + + // The number of sealed WAL files deleted by pruning. + walFilesPruned = must(walMeter.Int64Counter( + "seiwal_files_pruned", + metric.WithDescription("Number of WAL files removed by pruning"), + metric.WithUnit("{count}"), + )) + + // The time spent serializing a payload in the generic serializing WAL. + walSerializeDuration = must(walMeter.Float64Histogram( + "seiwal_serialize_duration_seconds", + metric.WithDescription("Time spent serializing a payload in the generic WAL"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(commonmetrics.LatencyBuckets...), + )) + + // The number of payload bytes produced by serialization in the generic serializing WAL. + walSerializedBytes = must(walMeter.Int64Counter( + "seiwal_serialized_bytes", + metric.WithDescription("Number of payload bytes produced by serialization in the generic WAL"), + metric.WithUnit("By"), + )) + + // The number of serialization failures in the generic serializing WAL. + walSerializeErrors = must(walMeter.Int64Counter( + "seiwal_serialize_errors", + metric.WithDescription("Number of serialization failures in the generic WAL"), + metric.WithUnit("{count}"), + )) + + // The buffered depth of a WAL's internal channel, sampled periodically. + walQueueDepth = must(walMeter.Int64Gauge( + "seiwal_queue_depth", + metric.WithDescription("Buffered depth of a WAL internal channel, sampled periodically"), + metric.WithUnit("{count}"), + )) +) + +// walNameAttr returns the measurement option that tags an observation with a WAL instance's name, so metrics +// from distinct instances in the same process remain distinguishable. +func walNameAttr(name string) metric.MeasurementOption { + return metric.WithAttributeSet(attribute.NewSet(attribute.String("wal", name))) +} + +// queueDepthAttrs tags a queue-depth observation with the WAL instance name and which internal channel +// ("writer" or "serializer") is being measured. +func queueDepthAttrs(name string, queue string) metric.MeasurementOption { + return metric.WithAttributeSet(attribute.NewSet(attribute.String("wal", name), attribute.String("queue", queue))) +} + +func must[V any](v V, err error) V { + if err != nil { + panic(err) + } + return v +} diff --git a/sei-db/seiwal/seiwal.go b/sei-db/seiwal/seiwal.go new file mode 100644 index 0000000000..b76a1e15f0 --- /dev/null +++ b/sei-db/seiwal/seiwal.go @@ -0,0 +1,92 @@ +package seiwal + +// WAL is a generic, index-keyed, append-only write-ahead log over payloads of type T. +// +// Each record is tagged with a caller-provided monotonic index. The index is what makes garbage +// collection ("drop everything below N"), iteration ("start at N"), and rollback ("drop everything +// above N") expressible without the WAL ever interpreting a payload. +// +// A WAL instance is not safe for concurrent use: its methods must not be called from multiple +// goroutines simultaneously. Callers that share a WAL across goroutines must serialize access +// themselves. +// +// Slices are not copied at the call boundary. Any slice passed into a WAL method — the payload and every +// slice reachable through it — must not be modified after the call: the WAL may retain it and read it +// asynchronously, so mutating it races the WAL and can corrupt what is persisted. Likewise every slice +// returned from a WAL or its iterator is owned by the WAL and must be treated as read-only. Callers that +// need to mutate such data must copy it first. +type WAL[T any] interface { + + // Append a record with the given index and payload. + // + // The index must be strictly greater than the index of the most recently appended record (indices + // need not be contiguous, but they must strictly increase). + // + // This method only schedules the append; it does not block until the record is durable. Durability is + // achieved by a subsequent Flush. + // + // data, and every slice reachable through it, must not be modified after this call: the payload may be + // retained and serialized asynchronously, so mutating it races the WAL and can corrupt what is + // persisted. Callers that need to reuse or mutate the buffer must copy it first. + Append(index uint64, data T) error + + // Flush blocks until all previously scheduled appends are durable. + Flush() error + + // Bounds reports the range of record indices currently stored in the WAL. + Bounds() ( + // If true, there is at least one record in the WAL and first/last are valid. If false, the WAL is + // empty and first/last are undefined. + ok bool, + // The lowest stored record index, inclusive. Only valid if ok is true. + first uint64, + // The highest stored record index, inclusive. Only valid if ok is true. + last uint64, + // Any error encountered while retrieving the range. + err error, + ) + + // Prune removes all records with an index less than lowestIndexToKeep. + // + // This method merely schedules the prune; it does not block until the prune is complete. Pruning is + // async and lazy, and implementations are free to delay it arbitrarily long. Pruning removes whole + // sealed files only, so records may survive above the requested threshold until their containing file + // is fully below it. + Prune(lowestIndexToKeep uint64) error + + // Iterator returns an iterator over the WAL starting at the given index. + // + // The iterator reads a consistent, point-in-time snapshot of the WAL taken at some instant between the + // start and the return of this call. Records appended before that instant are included; records + // appended after it are not. For records appended concurrently with this call, whether they are + // included is unspecified. + Iterator(startIndex uint64) (Iterator[T], error) + + // Close flushes pending appends, seals the current file, and releases resources. + Close() error +} + +// Iterator iterates over the records of a WAL in ascending index order. +// +// An Iterator is single-consumer and not safe for concurrent use: all of its methods, including Close, must +// be called from a single goroutine (or with external serialization). In particular, Close must not be +// called concurrently with Next from another goroutine. +// +// Every payload returned by an Iterator — and every slice reachable through it — is owned by the WAL and +// must be treated as read-only. Callers that need to retain or mutate the data must copy it first. +type Iterator[T any] interface { + // Next advances the iterator to the next record. It returns false when iteration is complete (no more + // records), and returns an error if advancing failed. After Next returns (false, nil), iteration is + // complete; after it returns an error, the iterator must not be used further (other than Close). + Next() (bool, error) + + // Entry returns the index and payload of the record at the iterator's current position. It is only + // valid to call Entry after Next has returned (true, nil). + // + // The returned payload must be treated as read-only and must not be modified. Callers that need to + // retain or mutate the data must copy it first. + Entry() (index uint64, data T) + + // Close releases the resources held by the iterator. + Close() error +} diff --git a/sei-db/seiwal/seiwal_config.go b/sei-db/seiwal/seiwal_config.go new file mode 100644 index 0000000000..3487522e0d --- /dev/null +++ b/sei-db/seiwal/seiwal_config.go @@ -0,0 +1,81 @@ +package seiwal + +import ( + "fmt" + "regexp" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/common/unit" +) + +// The permitted shape of a WAL instance name: it becomes a metric attribute value, so it is restricted to +// characters safe for label values. +var nameRegex = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + +// Config configures a WAL. +type Config struct { + // The directory where the WAL writes its files. + Path string + + // A short identifier for this WAL instance, used to distinguish its metrics from those of other + // instances in the same process. Required; must match [a-zA-Z0-9_-]+. + Name string + + // The size of the channel used to send framed records and control messages to the writer goroutine. + WriteBufferSize uint + + // The depth of the serialization request queue. Used only by the generic serializing WAL + // (NewGenericWAL); the byte-oriented engine ignores it. + SerializerBufferSize uint + + // The size a WAL file may reach before it is sealed and a fresh one is opened. Rotation happens after a + // record is appended, so a file may exceed this by the size of a single record — and because a record + // is written atomically to a single file, a record larger than this threshold produces a file that + // overshoots it by that record's size. Must be greater than 0. + TargetFileSize uint + + // When true, Flush calls fsync on the underlying file so that flushed data survives a power loss, not + // merely a process crash. When false, Flush only flushes the in-process buffer to the OS. + FsyncOnFlush bool + + // The number of records an iterator's reader thread may prefetch ahead of the consumer. A larger value + // keeps the reader busy while the consumer processes records, which matters for startup replay speed. + // Must be greater than 0. + IteratorPrefetchSize uint + + // The interval at which the WAL samples the buffered depth of its internal channel into the + // seiwal_queue_depth gauge. Zero or negative disables sampling. + MetricsSampleInterval time.Duration +} + +// DefaultConfig returns a default WAL configuration for the WAL at path, identified by name. +func DefaultConfig(path string, name string) *Config { + return &Config{ + Path: path, + Name: name, + WriteBufferSize: 16, + SerializerBufferSize: 16, + TargetFileSize: 64 * unit.MB, + FsyncOnFlush: true, + IteratorPrefetchSize: 32, + MetricsSampleInterval: 15 * time.Second, + } +} + +// Validate the configuration, returning nil if valid, or an error describing the problem if invalid. +func (c *Config) Validate() error { + if c.Path == "" { + return fmt.Errorf("path is required") + } + if !nameRegex.MatchString(c.Name) { + return fmt.Errorf("name %q is required and must match %s", c.Name, nameRegex.String()) + } + if c.TargetFileSize == 0 { + // A zero target would seal and rotate a fresh file after every single record. + return fmt.Errorf("target file size must be greater than 0") + } + if c.IteratorPrefetchSize == 0 { + return fmt.Errorf("iterator prefetch size must be greater than 0") + } + return nil +} diff --git a/sei-db/seiwal/seiwal_config_test.go b/sei-db/seiwal/seiwal_config_test.go new file mode 100644 index 0000000000..281f5ffbac --- /dev/null +++ b/sei-db/seiwal/seiwal_config_test.go @@ -0,0 +1,40 @@ +package seiwal + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigValidate(t *testing.T) { + t.Run("default config is valid", func(t *testing.T) { + require.NoError(t, DefaultConfig("/tmp/wal", "test").Validate()) + }) + + t.Run("empty path is rejected", func(t *testing.T) { + cfg := DefaultConfig("", "test") + require.Error(t, cfg.Validate()) + }) + + t.Run("empty name is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "") + require.Error(t, cfg.Validate()) + }) + + t.Run("malformed name is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "bad name!") + require.Error(t, cfg.Validate()) + }) + + t.Run("zero target file size is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "test") + cfg.TargetFileSize = 0 + require.Error(t, cfg.Validate()) + }) + + t.Run("zero iterator prefetch size is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "test") + cfg.IteratorPrefetchSize = 0 + require.Error(t, cfg.Validate()) + }) +} diff --git a/sei-db/seiwal/seiwal_file.go b/sei-db/seiwal/seiwal_file.go new file mode 100644 index 0000000000..99a8582da6 --- /dev/null +++ b/sei-db/seiwal/seiwal_file.go @@ -0,0 +1,559 @@ +package seiwal + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" +) + +// WAL files use the following naming schema: +// +// For mutable files: {file sequence}.wal.u +// For sealed files: {file sequence}-{first index}-{last index}.wal +// +// The file sequence is a unique, monotonically increasing counter identifying the file; the first and last +// index are the record indices the sealed file spans. The on-disk serialization version is recorded in each +// file's header rather than its name. + +const ( + // The file extension for unsealed (mutable) WAL files. + walUnsealedExtension = ".wal.u" + // The file extension for sealed (immutable) WAL files. + walSealedExtension = ".wal" + // The serialization version written into each file's header. Bumped if the on-disk format changes. + walFormatVersion = byte(1) +) + +// The magic prefix written at the start of every WAL file, followed by a single format-version byte. +var walFileMagic = []byte("SEIWL") + +// The length of a WAL file header: magic prefix plus the format-version byte. +const walHeaderSize = 6 // len("SEIWL") + 1 + +var ( + unsealedFileRegex = regexp.MustCompile(`^(\d+)\.wal\.u$`) + sealedFileRegex = regexp.MustCompile(`^(\d+)-(\d+)-(\d+)\.wal$`) +) + +// The result of parsing a WAL file name. +type parsedFileName struct { + fileSeq uint64 + firstIndex uint64 + lastIndex uint64 + sealed bool +} + +// Parse a WAL file name into its components. Returns false if the name is not a WAL file name. +func parseFileName(fileName string) (parsedFileName, bool) { + if m := sealedFileRegex.FindStringSubmatch(fileName); m != nil { + seq, err1 := strconv.ParseUint(m[1], 10, 64) + first, err2 := strconv.ParseUint(m[2], 10, 64) + last, err3 := strconv.ParseUint(m[3], 10, 64) + if err1 != nil || err2 != nil || err3 != nil { + return parsedFileName{}, false + } + return parsedFileName{fileSeq: seq, firstIndex: first, lastIndex: last, sealed: true}, true + } + if m := unsealedFileRegex.FindStringSubmatch(fileName); m != nil { + seq, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + return parsedFileName{}, false + } + return parsedFileName{fileSeq: seq, sealed: false}, true + } + return parsedFileName{}, false +} + +// Build the name of an unsealed (mutable) WAL file. +func unsealedFileName(fileSeq uint64) string { + return fmt.Sprintf("%d%s", fileSeq, walUnsealedExtension) +} + +// Build the name of a sealed WAL file. +func sealedFileName(fileSeq uint64, firstIndex uint64, lastIndex uint64) string { + return fmt.Sprintf("%d-%d-%d%s", fileSeq, firstIndex, lastIndex, walSealedExtension) +} + +// A single WAL file on disk, either the current mutable file being appended to or a sealed file. +type walFile struct { + // The directory this file lives in. + directory string + + // The open file handle and buffered writer. Only set for the mutable file being written this session. + file *os.File + writer *bufio.Writer + + // The unique, monotonically increasing sequence number of this file. + fileSeq uint64 + + // If true, this file is sealed and rejects writes. + sealed bool + + // The first and last record indices that appear in this file, valid only when hasRecords is true. + firstIndex uint64 + lastIndex uint64 + hasRecords bool + + // The number of bytes written to this file so far, including the header. + size uint64 +} + +// Create a new mutable WAL file on disk, writing its header, ready to accept records. +func newWalFile(directory string, fileSeq uint64) (*walFile, error) { + path := filepath.Join(directory, unsealedFileName(fileSeq)) + file, err := os.Create(path) //nolint:gosec // path derived from a validated directory + if err != nil { + return nil, fmt.Errorf("failed to create WAL file %s: %w", path, err) + } + + // Persist the new directory entry so a later fsync of the file's contents (via flush) cannot be undone by a + // power loss that drops the unsynced create. Without this, flushed data could be lost if the file is never + // sealed (a seal fsyncs the directory via the atomic rename) before a crash. + if err := util.SyncParentPath(path); err != nil { + _ = file.Close() + return nil, fmt.Errorf("failed to fsync WAL directory after creating %s: %w", path, err) + } + + writer := bufio.NewWriter(file) + header := append(append([]byte(nil), walFileMagic...), walFormatVersion) + if _, err := writer.Write(header); err != nil { + _ = file.Close() + return nil, fmt.Errorf("failed to write WAL header to %s: %w", path, err) + } + + return &walFile{ + directory: directory, + file: file, + writer: writer, + fileSeq: fileSeq, + size: walHeaderSize, + }, nil +} + +// frameRecord wraps a payload in its on-disk framing: +// [uvarint index][uvarint payload length][payload][uint32 CRC32(index+length+payload)]. +// The checksum covers the index and length prefixes as well as the payload, so a torn or corrupt index is +// detected on recovery exactly as a torn payload is. +func frameRecord(index uint64, payload []byte) []byte { + var idxBuf [binary.MaxVarintLen64]byte + idxN := binary.PutUvarint(idxBuf[:], index) + var lenBuf [binary.MaxVarintLen64]byte + lenN := binary.PutUvarint(lenBuf[:], uint64(len(payload))) + + record := make([]byte, 0, idxN+lenN+len(payload)+4) + record = append(record, idxBuf[:idxN]...) + record = append(record, lenBuf[:lenN]...) + record = append(record, payload...) + var crcBuf [4]byte + binary.BigEndian.PutUint32(crcBuf[:], crc32.ChecksumIEEE(record)) + record = append(record, crcBuf[:]...) + return record +} + +// Append a pre-framed record (see frameRecord) with the given index to this file. +func (f *walFile) writeRecord(record []byte, index uint64) error { + if f.sealed { + return fmt.Errorf("cannot write to a sealed WAL file") + } + if _, err := f.writer.Write(record); err != nil { + return fmt.Errorf("failed to write WAL record: %w", err) + } + f.size += uint64(len(record)) + + if !f.hasRecords { + f.firstIndex = index + f.hasRecords = true + } + f.lastIndex = index + return nil +} + +// close releases the file handle without sealing. Used on the fatal-error path, where the file is left +// unsealed for recoverOrphans to seal (truncating any torn tail) on the next open. Idempotent. +func (f *walFile) close() error { + if f.file == nil { + return nil + } + err := f.file.Close() + f.file = nil + return err +} + +// Flush buffered data to the OS. When fsync is true, also fsync the file so the data survives power loss. +func (f *walFile) flush(fsync bool) error { + if f.writer != nil { + if err := f.writer.Flush(); err != nil { + return fmt.Errorf("failed to flush WAL file: %w", err) + } + } + if fsync && f.file != nil { + if err := f.file.Sync(); err != nil { + return fmt.Errorf("failed to fsync WAL file: %w", err) + } + } + return nil +} + +// Seal this file: flush it, then atomically rename it to its sealed name. Every record written in-process is +// whole (records are framed and written atomically), so there is nothing to truncate. A file with no records +// (including one that never received a record) is removed rather than sealed. Idempotent. Returns the sealed +// file name, or "" if the file was removed. +func (f *walFile) seal() (string, error) { + if f.sealed { + return "", nil + } + if err := f.flush(true); err != nil { + return "", fmt.Errorf("failed to flush before sealing: %w", err) + } + + unsealedPath := filepath.Join(f.directory, unsealedFileName(f.fileSeq)) + if !f.hasRecords { + if f.file != nil { + if err := f.file.Close(); err != nil { + return "", fmt.Errorf("failed to close WAL file: %w", err) + } + } + if err := removeAndSyncDir(f.directory, unsealedFileName(f.fileSeq)); err != nil { + return "", fmt.Errorf("failed to remove empty WAL file: %w", err) + } + f.sealed = true + return "", nil + } + + if f.file != nil { + if err := f.file.Close(); err != nil { + return "", fmt.Errorf("failed to close WAL file: %w", err) + } + } + + sealedName := sealedFileName(f.fileSeq, f.firstIndex, f.lastIndex) + sealedPath := filepath.Join(f.directory, sealedName) + if err := util.AtomicRename(unsealedPath, sealedPath, true); err != nil { + return "", fmt.Errorf("failed to seal WAL file: %w", err) + } + f.sealed = true + return sealedName, nil +} + +// A single record read from a WAL file: its index, its payload (a sub-slice of the file's bytes), and the +// byte offset just past its framing. +type walRecord struct { + index uint64 + payload []byte + end int64 +} + +// The result of reading a WAL file from disk. +type walFileContents struct { + // The parsed file name components. + parsed parsedFileName + + // The intact records read from the file, in order. Excludes any torn trailing record. + records []walRecord + + // The first and last record indices across the intact records, valid only when hasRecords is true. + firstIndex uint64 + lastIndex uint64 + hasRecords bool + + // The byte offset just past the last intact record. Data beyond this offset is a torn trailing record and + // is discarded on recovery. + validEnd int64 +} + +// Read a WAL file from disk, tolerating a torn trailing record (a crash mid-write can leave a final record +// whose index prefix, length prefix, payload, or checksum is incomplete). Any bytes past the last intact +// record are discarded; the last intact record's boundary is reported so callers can truncate the torn tail. +func readWalFile(path string) (*walFileContents, error) { + name := filepath.Base(path) + parsed, ok := parseFileName(name) + if !ok { + return nil, fmt.Errorf("not a WAL file: %s", name) + } + + data, err := os.ReadFile(path) //nolint:gosec // caller-supplied path + if err != nil { + return nil, fmt.Errorf("failed to read WAL file %s: %w", path, err) + } + + return parseWalFileData(data, parsed, path) +} + +// readWalFileFromHandle reads and parses a WAL file from an already-open handle, then closes the handle. +func readWalFileFromHandle(file *os.File, parsed parsedFileName) (*walFileContents, error) { + defer func() { _ = file.Close() }() + data, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read WAL file %s: %w", file.Name(), err) + } + return parseWalFileData(data, parsed, file.Name()) +} + +// parseWalFileData parses the raw bytes of a WAL file (already read into memory) into its intact records, +// tolerating a torn trailing record. name is used only for error messages. It is shared by readWalFile (which +// reads by path) and the iterator (which reads through a file handle opened on the writer goroutine). +func parseWalFileData(data []byte, parsed parsedFileName, name string) (*walFileContents, error) { + contents := &walFileContents{parsed: parsed} + + if len(data) < walHeaderSize { + // A file too short to even contain a header carries no committed data. + return contents, nil + } + if !bytes.Equal(data[:len(walFileMagic)], walFileMagic) { + return nil, fmt.Errorf("WAL file %s has an invalid magic prefix", name) + } + if version := data[len(walFileMagic)]; version != walFormatVersion { + return nil, fmt.Errorf("WAL file %s has unsupported format version %d", name, version) + } + contents.validEnd = walHeaderSize + + // A torn trailing record is expected only in the mutable file after a crash, where discarding it is correct. + // A sealed file is durable, fsync'd, and truncated to a record boundary on seal, so any framing or checksum + // failure in it is corruption (e.g. bit-rot) that must be surfaced rather than silently discarded — otherwise + // records past the fault vanish while the file's name keeps promising them. torn() turns each tolerated break + // into a hard error for a sealed file. + torn := func(offset int, reason string) error { + if !parsed.sealed { + return nil + } + return fmt.Errorf("sealed WAL file %s is corrupt: %s at offset %d", name, reason, offset) + } + + offset := walHeaderSize + for offset < len(data) { + index, idxN := binary.Uvarint(data[offset:]) + if idxN <= 0 { + if err := torn(offset, "torn record index prefix"); err != nil { + return nil, err + } + break // torn or incomplete index prefix + } + lenStart := offset + idxN + length, lenN := binary.Uvarint(data[lenStart:]) + if lenN <= 0 { + if err := torn(offset, "torn record length prefix"); err != nil { + return nil, err + } + break // torn or incomplete length prefix + } + payloadStart := lenStart + lenN + remaining := uint64(len(data) - payloadStart) //nolint:gosec // payloadStart <= len(data), so non-negative + if remaining < 4 || length > remaining-4 { + if err := torn(offset, "truncated record payload or checksum"); err != nil { + return nil, err + } + break // torn record: payload or checksum truncated (4 trailing bytes are the CRC32) + } + payloadLen := int(length) //nolint:gosec // bounded above by remaining-4, which is <= len(data) + payloadEnd := payloadStart + payloadLen + recordEnd := payloadEnd + 4 + gotCRC := binary.BigEndian.Uint32(data[payloadEnd:recordEnd]) + if gotCRC != crc32.ChecksumIEEE(data[offset:payloadEnd]) { + if err := torn(offset, "record checksum mismatch"); err != nil { + return nil, err + } + break // torn or corrupt record + } + + contents.records = append(contents.records, + walRecord{index: index, payload: data[payloadStart:payloadEnd], end: int64(recordEnd)}) + if !contents.hasRecords { + contents.firstIndex = index + contents.hasRecords = true + } + contents.lastIndex = index + contents.validEnd = int64(recordEnd) + + offset = recordEnd + } + + return contents, nil +} + +// verifySealedContents confirms that a sealed file's intact content exactly covers the [first, last] index +// range promised by its name. A sealed file is durable and complete, so a shortfall means interior corruption +// (e.g. a record truncated to a clean boundary) that leaves parseWalFileData reading fewer records than the +// name promises, while Bounds/GetStoredRange keep reporting the full range. fileSeq is used only for error +// messages. +func verifySealedContents(contents *walFileContents, fileSeq uint64, first uint64, last uint64) error { + if !contents.hasRecords { + return fmt.Errorf( + "WAL file (sequence %d) is corrupt: name promises indices [%d, %d] but no intact records were read", + fileSeq, first, last) + } + if contents.firstIndex != first || contents.lastIndex != last { + return fmt.Errorf( + "WAL file (sequence %d) is corrupt: name promises indices [%d, %d] but content holds [%d, %d]", + fileSeq, first, last, contents.firstIndex, contents.lastIndex) + } + return nil +} + +// truncateAndSync truncates the file at path to size and fsyncs it, so the shorter length is durable on its +// own — before any subsequent rename. Without the fsync, a crash could persist a rename while losing the +// truncation, leaving a file whose name promises more records than its content actually holds. +func truncateAndSync(path string, size int64) error { + file, err := os.OpenFile(path, os.O_RDWR, 0) //nolint:gosec // caller-supplied path + if err != nil { + return fmt.Errorf("failed to open %s for truncation: %w", path, err) + } + if err := file.Truncate(size); err != nil { + _ = file.Close() + return fmt.Errorf("failed to truncate %s: %w", path, err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("failed to fsync %s after truncation: %w", path, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("failed to close %s after truncation: %w", path, err) + } + return nil +} + +// removeAndSyncDir removes the named file and fsyncs its parent directory, so the removal is durable before the +// caller proceeds. Callers rely on this to keep the sealed-file sequence gap-free across a crash. +func removeAndSyncDir(directory string, name string) error { + path := filepath.Join(directory, name) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove WAL file %s: %w", path, err) + } + if err := util.SyncParentPath(path); err != nil { + return fmt.Errorf("failed to fsync directory after removing %s: %w", path, err) + } + return nil +} + +// Seal an orphaned mutable file discovered on disk at startup (left behind by a crash before it could be +// sealed). Any torn trailing record is truncated away first, so the sealed file ends cleanly on a record +// boundary. A file left with no records is removed. +func sealOrphanFile(directory string, name string) error { + path := filepath.Join(directory, name) + contents, err := readWalFile(path) + if err != nil { + return fmt.Errorf("failed to read orphaned WAL file %s: %w", path, err) + } + + if !contents.hasRecords { + return removeAndSyncDir(directory, name) + } + + if err := truncateAndSync(path, contents.validEnd); err != nil { + return fmt.Errorf("failed to truncate orphaned WAL file %s: %w", path, err) + } + sealedPath := filepath.Join(directory, + sealedFileName(contents.parsed.fileSeq, contents.firstIndex, contents.lastIndex)) + if err := util.AtomicRename(path, sealedPath, true); err != nil { + return fmt.Errorf("failed to seal orphaned WAL file %s: %w", path, err) + } + return nil +} + +// rollbackStraddlingFile handles the single sealed file that spans rollbackThrough: it drops every record +// beyond the rollback point and reduces the file's range accordingly. The name of a sealed file encodes its +// index range, so the reduced content and the reduced name must become durable together — a crash must never +// leave a file whose name promises more records than its content holds (the iterator bounds sealed reads by +// content and would otherwise under-yield, while Bounds trusts the name and would over-report). Because the +// reduced range means a different file name, this cannot be a single in-place rename: the kept prefix is +// written to a fresh correctly-named file via AtomicWrite (durable on its own), and only then is the old, +// larger-named file removed. A crash after the write but before the removal leaves both files under the same +// file sequence; recovery's reconcileRollbackRemnants resolves that deterministically. Files entirely beyond +// the rollback point are removed by the caller; this handles only the boundary file. +func rollbackStraddlingFile(directory string, name string, rollbackThrough uint64) error { + path := filepath.Join(directory, name) + parsed, ok := parseFileName(name) + if !ok { + return fmt.Errorf("not a WAL file: %s", name) + } + data, err := os.ReadFile(path) //nolint:gosec // path derived from a scanned WAL directory entry + if err != nil { + return fmt.Errorf("failed to read WAL file %s during rollback: %w", path, err) + } + contents, err := parseWalFileData(data, parsed, path) + if err != nil { + return fmt.Errorf("failed to parse WAL file %s during rollback: %w", path, err) + } + + truncateTo := int64(walHeaderSize) + var lastKept uint64 + kept := false + for _, record := range contents.records { + if record.index > rollbackThrough { + break + } + truncateTo = record.end + lastKept = record.index + kept = true + } + + if !kept { + // The content holds no record at or below the rollback point after all; drop the whole file. + return removeAndSyncDir(directory, name) + } + if lastKept == contents.lastIndex { + return nil // nothing beyond the rollback point; leave the file untouched + } + + // Write the kept prefix to a fresh file under its reduced name, made durable before the old file is removed. + newName := sealedFileName(parsed.fileSeq, contents.firstIndex, lastKept) + newPath := filepath.Join(directory, newName) + if err := util.AtomicWrite(newPath, data[:truncateTo], true); err != nil { + return fmt.Errorf("failed to write rolled-back WAL file %s: %w", newPath, err) + } + if err := removeAndSyncDir(directory, name); err != nil { + return fmt.Errorf("failed to remove old WAL file %s during rollback: %w", path, err) + } + return nil +} + +// reconcileRollbackRemnants resolves the one crash window left by rollbackStraddlingFile: a crash after the +// reduced file was written but before the old, larger-named file was removed leaves two sealed files sharing a +// file sequence. This can happen only from an interrupted rollback swap (healthy operation never assigns a +// sequence to two files), so the reduced file (the one with the smaller last index) is the intended survivor; +// the larger one is removed. Both files are internally name/content consistent, so the choice is made from +// names alone without reading contents. A no-op in the common case where every sealed sequence is unique. +func reconcileRollbackRemnants(directory string) error { + entries, err := os.ReadDir(directory) + if err != nil { + return fmt.Errorf("failed to read WAL directory %s: %w", directory, err) + } + + // The name kept for each sealed sequence so far. A duplicate sequence means an interrupted rollback swap. + kept := make(map[uint64]parsedFileName) + for _, entry := range entries { + if entry.IsDir() { + continue + } + parsed, ok := parseFileName(entry.Name()) + if !ok || !parsed.sealed { + continue + } + prev, seen := kept[parsed.fileSeq] + if !seen { + kept[parsed.fileSeq] = parsed + continue + } + // Two sealed files share this sequence. Keep the one with the smaller last index (the rolled-back + // result) and remove the other. A sealed name is a deterministic function of its parsed fields, so + // each file's name is recoverable without tracking the raw directory entry. + keep, drop := parsed, prev + if prev.lastIndex <= parsed.lastIndex { + keep, drop = prev, parsed + } + dropName := sealedFileName(drop.fileSeq, drop.firstIndex, drop.lastIndex) + if err := removeAndSyncDir(directory, dropName); err != nil { + return fmt.Errorf("failed to remove rollback remnant %s: %w", dropName, err) + } + kept[parsed.fileSeq] = keep + } + return nil +} diff --git a/sei-db/seiwal/seiwal_file_test.go b/sei-db/seiwal/seiwal_file_test.go new file mode 100644 index 0000000000..c7a515f641 --- /dev/null +++ b/sei-db/seiwal/seiwal_file_test.go @@ -0,0 +1,153 @@ +package seiwal + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFileNaming(t *testing.T) { + require.Equal(t, "3.wal.u", unsealedFileName(3)) + require.Equal(t, "3-10-20.wal", sealedFileName(3, 10, 20)) + + parsed, ok := parseFileName("3.wal.u") + require.True(t, ok) + require.Equal(t, parsedFileName{fileSeq: 3, sealed: false}, parsed) + + parsed, ok = parseFileName("3-10-20.wal") + require.True(t, ok) + require.Equal(t, parsedFileName{fileSeq: 3, firstIndex: 10, lastIndex: 20, sealed: true}, parsed) + + _, ok = parseFileName("not-a-wal-file.txt") + require.False(t, ok) +} + +// writeMutableFile creates a mutable file at sequence 0, applies fn to it, then flushes and closes the +// underlying handle without sealing, leaving an unsealed file on disk. It returns the file path. +func writeMutableFile(t *testing.T, dir string, fn func(f *walFile)) string { + t.Helper() + f, err := newWalFile(dir, 0) + require.NoError(t, err) + fn(f) + require.NoError(t, f.flush(true)) + require.NoError(t, f.file.Close()) + return filepath.Join(dir, unsealedFileName(0)) +} + +// writeRecordTo frames and appends a record for the given index to f. +func writeRecordTo(t *testing.T, f *walFile, index uint64, payload []byte) { + t.Helper() + require.NoError(t, f.writeRecord(frameRecord(index, payload), index)) +} + +func TestReadWalFileCleanTail(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + writeRecordTo(t, f, 2, []byte("two")) + }) + + contents, err := readWalFile(path) + require.NoError(t, err) + require.True(t, contents.hasRecords) + require.Equal(t, uint64(1), contents.firstIndex) + require.Equal(t, uint64(2), contents.lastIndex) + require.Len(t, contents.records, 2) +} + +func TestReadWalFileTornTrailingRecord(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + writeRecordTo(t, f, 2, []byte("two")) + // A third record whose framing is truncated mid-payload, as a torn write would leave. + frame := frameRecord(3, []byte("three")) + require.NoError(t, f.flush(false)) + _, err := f.writer.Write(frame[:len(frame)-3]) + require.NoError(t, err) + }) + + contents, err := readWalFile(path) + require.NoError(t, err) + // The torn record 3 is dropped; the last intact record is 2. + require.True(t, contents.hasRecords) + require.Equal(t, uint64(2), contents.lastIndex) + require.Len(t, contents.records, 2) +} + +func TestReadWalFilePartialLengthPrefix(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + }) + + // Append a lone 0x80 byte: an incomplete uvarint prefix, as a torn write would leave. + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + require.NoError(t, err) + _, err = f.Write([]byte{0x80}) + require.NoError(t, err) + require.NoError(t, f.Close()) + + contents, err := readWalFile(path) + require.NoError(t, err) + require.True(t, contents.hasRecords) + require.Equal(t, uint64(1), contents.lastIndex) + require.Len(t, contents.records, 1) +} + +func TestReadWalFileMidRecordTruncation(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + writeRecordTo(t, f, 2, []byte("two")) + }) + + info, err := os.Stat(path) + require.NoError(t, err) + // Lop a few bytes off the end, tearing record 2. + require.NoError(t, os.Truncate(path, info.Size()-3)) + + contents, err := readWalFile(path) + require.NoError(t, err) + require.True(t, contents.hasRecords) + require.Equal(t, uint64(1), contents.lastIndex) + require.Len(t, contents.records, 1) +} + +func TestReadWalFileChecksumMismatch(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + writeRecordTo(t, f, 2, []byte("two")) + }) + + // Flip the final byte (part of record 2's CRC), so that record fails its checksum. + data, err := os.ReadFile(path) + require.NoError(t, err) + data[len(data)-1] ^= 0xFF + require.NoError(t, os.WriteFile(path, data, 0o600)) + + contents, err := readWalFile(path) + require.NoError(t, err) + // Record 1 survives; the corrupt record 2 is dropped. + require.True(t, contents.hasRecords) + require.Equal(t, uint64(1), contents.lastIndex) + require.Len(t, contents.records, 1) +} + +func TestReadWalFileBadMagic(t *testing.T) { + dir := t.TempDir() + path := writeMutableFile(t, dir, func(f *walFile) { + writeRecordTo(t, f, 1, []byte("one")) + }) + + data, err := os.ReadFile(path) + require.NoError(t, err) + data[0] ^= 0xFF + require.NoError(t, os.WriteFile(path, data, 0o600)) + + _, err = readWalFile(path) + require.Error(t, err) +} diff --git a/sei-db/seiwal/seiwal_impl.go b/sei-db/seiwal/seiwal_impl.go new file mode 100644 index 0000000000..a3b0c0f221 --- /dev/null +++ b/sei-db/seiwal/seiwal_impl.go @@ -0,0 +1,780 @@ +package seiwal + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/metric" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" + "github.com/sei-protocol/seilog" +) + +var _ WAL[[]byte] = (*walImpl)(nil) + +var logger = seilog.NewLogger("db", "seiwal") + +// dataToBeWritten carries a framed record from a caller to the writer to be appended. +type dataToBeWritten struct { + record []byte + index uint64 +} + +// flushRequest asks the writer to flush (and optionally fsync) the mutable file, signaling done when durable. +type flushRequest struct { + done chan error +} + +// rangeQuery asks the writer to report the stored index range. +type rangeQuery struct { + reply chan storedRange +} + +// pruneRequest asks the writer to drop whole sealed files below `through`. +type pruneRequest struct { + through uint64 +} + +// closeRequest asks the writer to seal the mutable file and shut down, signaling done when sealed. +type closeRequest struct { + done chan error +} + +// unpinRequest releases a read lease previously registered when an iterator was created. +type unpinRequest struct { + index uint64 +} + +// iteratorStartRequest asks the writer to construct an iterator. The writer flushes the mutable file (so the +// iterator observes all prior appends), snapshots the current set of files, registers the read lease, and +// builds the iterator, all on its own goroutine so construction is serialized with rotation/seal/prune. +type iteratorStartRequest struct { + startIndex uint64 + reply chan iteratorStartResponse +} + +// The iterator (or an error) produced by the writer in response to an iteratorStartRequest. +type iteratorStartResponse struct { + iterator *walIterator + err error +} + +// The index range reported by Bounds. +type storedRange struct { + ok bool + first uint64 + last uint64 +} + +// Bookkeeping for a sealed WAL file, owned by the writer goroutine. +type sealedFileInfo struct { + fileSeq uint64 + name string + firstIndex uint64 + lastIndex uint64 +} + +// A generic write-ahead log implementation. +type walImpl struct { + // The configuration this WAL was opened with. Read-only after construction. + config *Config + + // The measurement option tagging this instance's metrics with its name. Read-only after construction. + metricAttrs metric.MeasurementOption + + // Callers funnel framed records and control messages through writerChan as a single ordered stream to + // the writer goroutine. + writerChan chan any + + // The hard-stop context the writer watches. Cancelled by fail() on a fatal error and by Close() once + // everything has drained. + ctx context.Context + // Cancels ctx, tearing down the writer goroutine. + cancel context.CancelFunc + + // A child of ctx that the writerChan producers watch, cancelled once the writer stops reading so an + // in-flight or future push aborts rather than deadlocking. + senderCtx context.Context + // Cancels senderCtx. + senderCancel context.CancelFunc + + // Tracks the writer and queue-depth sampler goroutines so Close() can wait for them to exit. + wg sync.WaitGroup + + // Closed by Close() to stop the queue-depth sampler goroutine. + samplerStop chan struct{} + + // Guarantees the Close() shutdown sequence runs at most once. + closeOnce sync.Once + + // Set by Close() so subsequent scheduling calls fail fast. + closed atomic.Bool + + // The first unrecoverable background-goroutine error, surfaced to the caller by Close(). + asyncErr atomic.Pointer[error] + + // Guards the caller-side append-ordering state below, which is read/written synchronously in Append + // (not on the writer goroutine). This gate is a best-effort, time-of-call convenience for the + // (contractually single-threaded) caller; it is not the authoritative ordering guard, since concurrent + // callers could still reorder records past it. The writer goroutine holds the authoritative check. + appendMu sync.Mutex + // The index of the most recently appended record. + lastAppendIndex uint64 + // Whether any record has been appended (this session or recovered from disk). + hasAppended bool + + // The following fields are owned exclusively by the writer goroutine. + + // The index of the most recently written record. A writer-owned backstop that rejects out-of-order + // records that slip past the caller-side gate (e.g. under concurrent misuse), turning silent + // corruption into a fatal error. + lastWrittenIndex uint64 + + // Whether any record has been written (this session or recovered from disk). + hasWritten bool + + // The current mutable file accepting records. + mutableFile *walFile + + // The sequence number to assign the next mutable file. + nextFileSeq uint64 + + // Sealed files in ascending index order. Rotation appends to the back; pruning removes from the front. + sealedFiles *util.RandomAccessDeque[*sealedFileInfo] + + // Read leases held by live iterators: record index -> reference count. Pruning will not delete a file + // whose index range contains a leased index. Mutated only by the writer goroutine. + indexRefs map[uint64]int +} + +// NewWAL opens (or creates) a byte-oriented WAL in the configured directory, recovering any files left +// behind by a previous session. Operates on []byte payloads. +func NewWAL(config *Config) (WAL[[]byte], error) { + return newWAL(config, nil) +} + +// NewWALWithRollback opens a byte-oriented WAL and deletes all records with an index greater than +// rollbackIndex before returning, so the WAL contains no record with an index greater than rollbackIndex. +func NewWALWithRollback(config *Config, rollbackIndex uint64) (WAL[[]byte], error) { + return newWAL(config, &rollbackIndex) +} + +func newWAL(config *Config, rollbackThrough *uint64) (WAL[[]byte], error) { + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid WAL config: %w", err) + } + if err := util.EnsureDirectoryExists(config.Path, true); err != nil { + return nil, fmt.Errorf("failed to ensure WAL directory %s: %w", config.Path, err) + } + + // Clean up remnants of a rollback swap interrupted by a crash before scanning (see rollbackStraddlingFile): + // a leftover swap file from an unfinished AtomicWrite, or two sealed files sharing a sequence because the old + // file was not yet removed. This leaves a set where every sealed sequence is unique and name matches content. + if err := util.DeleteOrphanedSwapFiles(config.Path); err != nil { + return nil, fmt.Errorf("failed to delete orphaned swap files: %w", err) + } + if err := reconcileRollbackRemnants(config.Path); err != nil { + return nil, fmt.Errorf("failed to reconcile rollback remnants: %w", err) + } + if err := recoverOrphans(config.Path); err != nil { + return nil, fmt.Errorf("failed to recover orphaned WAL files: %w", err) + } + if rollbackThrough != nil { + if err := rollbackDirectory(config.Path, *rollbackThrough); err != nil { + return nil, fmt.Errorf("failed to roll back WAL beyond index %d: %w", *rollbackThrough, err) + } + } + + sealedFiles, nextFileSeq, err := scanSealedFiles(config.Path) + if err != nil { + return nil, fmt.Errorf("failed to scan sealed WAL files: %w", err) + } + if err := validateSealedFiles(config.Path, sealedFiles); err != nil { + return nil, fmt.Errorf("corrupt sealed WAL file: %w", err) + } + + mutable, err := newWalFile(config.Path, nextFileSeq) + if err != nil { + return nil, fmt.Errorf("failed to open mutable WAL file: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + senderCtx, senderCancel := context.WithCancel(ctx) + + w := &walImpl{ + config: config, + metricAttrs: walNameAttr(config.Name), + writerChan: make(chan any, config.WriteBufferSize), + ctx: ctx, + cancel: cancel, + senderCtx: senderCtx, + senderCancel: senderCancel, + mutableFile: mutable, + nextFileSeq: nextFileSeq + 1, + sealedFiles: sealedFiles, + indexRefs: make(map[uint64]int), + samplerStop: make(chan struct{}), + } + // Recover the append-ordering position from the highest index already on disk. + if r := w.bounds(); r.ok { + w.lastAppendIndex = r.last + w.hasAppended = true + w.lastWrittenIndex = r.last + w.hasWritten = true + } + + w.wg.Add(1) + go w.writerLoop() + + if config.MetricsSampleInterval > 0 { + w.wg.Add(1) + go w.sampleQueueDepth(config.MetricsSampleInterval) + } + + return w, nil +} + +// sampleQueueDepth periodically records the writer channel's buffered depth until Close stops it (samplerStop) +// or a fatal shutdown cancels ctx. +func (w *walImpl) sampleQueueDepth(interval time.Duration) { + defer w.wg.Done() + attrs := queueDepthAttrs(w.config.Name, "writer") + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-w.ctx.Done(): + return + case <-w.samplerStop: + return + case <-ticker.C: + walQueueDepth.Record(w.ctx, int64(len(w.writerChan)), attrs) + } + } +} + +// Append frames a record and schedules it for the writer, after enforcing that indices strictly increase. +func (w *walImpl) Append(index uint64, data []byte) error { + if w.closed.Load() { + return fmt.Errorf("WAL is closed") + } + // Fail fast on a bricked WAL. Without this, a fire-and-forget append can win the sendToWriter select + // against the already-cancelled senderCtx and enqueue onto a dead writer's buffer, silently dropping the + // record and returning nil. asyncErr is recorded before the cancel, so any caller that has observed the + // shutdown also observes the error here. + if err := w.asyncError(); err != nil { + return fmt.Errorf("WAL failed: %w", err) + } + + w.appendMu.Lock() + if w.hasAppended && index <= w.lastAppendIndex { + last := w.lastAppendIndex + w.appendMu.Unlock() + return fmt.Errorf("append rejected: index %d is not greater than last appended index %d", index, last) + } + w.lastAppendIndex = index + w.hasAppended = true + w.appendMu.Unlock() + + record := frameRecord(index, data) + if err := w.sendToWriter(dataToBeWritten{record: record, index: index}); err != nil { + return fmt.Errorf("failed to schedule append for index %d: %w", index, err) + } + return nil +} + +// Flush blocks until all previously scheduled appends are durable. +func (w *walImpl) Flush() error { + done := make(chan error, 1) + if err := w.sendToWriter(flushRequest{done: done}); err != nil { + return fmt.Errorf("failed to schedule flush: %w", err) + } + select { + case err := <-done: + return err // already wrapped by the writer, or nil on success + case <-w.ctx.Done(): + if err := w.asyncError(); err != nil { + return fmt.Errorf("flush aborted: %w", err) + } + return fmt.Errorf("flush aborted: %w", w.ctx.Err()) + } +} + +// Bounds reports the range of record indices stored in the WAL. +func (w *walImpl) Bounds() (bool, uint64, uint64, error) { + reply := make(chan storedRange, 1) + if err := w.sendToWriter(rangeQuery{reply: reply}); err != nil { + return false, 0, 0, fmt.Errorf("failed to schedule bounds query: %w", err) + } + select { + case r := <-reply: + return r.ok, r.first, r.last, nil + case <-w.ctx.Done(): + if err := w.asyncError(); err != nil { + return false, 0, 0, fmt.Errorf("bounds query aborted: %w", err) + } + return false, 0, 0, fmt.Errorf("bounds query aborted: %w", w.ctx.Err()) + } +} + +// Prune schedules removal of whole sealed files below lowestIndexToKeep. It does not block on completion. +func (w *walImpl) Prune(lowestIndexToKeep uint64) error { + if err := w.sendToWriter(pruneRequest{through: lowestIndexToKeep}); err != nil { + return fmt.Errorf("failed to schedule prune below index %d: %w", lowestIndexToKeep, err) + } + return nil +} + +// Iterator returns an iterator over the WAL starting at startIndex. Construction runs on the writer goroutine +// (see iteratorStartRequest): the writer flushes so all previously scheduled appends are visible, registers a +// read lease so pruning cannot delete files out from under the iterator, and builds the iterator. The lease is +// released by the iterator's Close. +func (w *walImpl) Iterator(startIndex uint64) (Iterator[[]byte], error) { + reply := make(chan iteratorStartResponse, 1) + if err := w.sendToWriter(iteratorStartRequest{startIndex: startIndex, reply: reply}); err != nil { + return nil, fmt.Errorf("failed to schedule iterator creation: %w", err) + } + select { + case resp := <-reply: + if resp.err != nil { + return nil, fmt.Errorf("failed to create iterator: %w", resp.err) + } + return resp.iterator, nil + case <-w.ctx.Done(): + if err := w.asyncError(); err != nil { + return nil, fmt.Errorf("iterator creation aborted: %w", err) + } + return nil, fmt.Errorf("iterator creation aborted: %w", w.ctx.Err()) + } +} + +// unpinIndex releases a read lease. Best-effort: if the WAL is already shutting down the lease is moot. +func (w *walImpl) unpinIndex(index uint64) { + _ = w.sendToWriter(unpinRequest{index: index}) +} + +// Close flushes pending appends, seals the mutable file, and releases resources. +func (w *walImpl) Close() error { + var closeErr error + w.closeOnce.Do(func() { + w.closed.Store(true) + close(w.samplerStop) // stop the queue-depth sampler before waiting for goroutines + done := make(chan error, 1) + if err := w.sendToWriter(closeRequest{done: done}); err == nil { + select { + case closeErr = <-done: + case <-w.ctx.Done(): + } + } + w.wg.Wait() + w.cancel() + }) + if err := w.asyncError(); err != nil { + return fmt.Errorf("WAL closed with error: %w", err) + } + return closeErr // already wrapped by the writer, or nil on a clean seal +} + +// sendToWriter enqueues a message onto the writer's input channel, aborting if the WAL is shutting down or has +// failed. +func (w *walImpl) sendToWriter(msg any) error { + select { + case w.writerChan <- msg: + return nil + case <-w.senderCtx.Done(): + if err := w.asyncError(); err != nil { + return fmt.Errorf("WAL failed: %w", err) + } + return fmt.Errorf("WAL is closed") + } +} + +// writerLoop consumes messages, appending records to the mutable file and handling control messages. It owns +// all file bookkeeping and runs on its own goroutine until close or a fatal error. +func (w *walImpl) writerLoop() { + defer w.wg.Done() + for { + var msg any + select { + case <-w.ctx.Done(): + return + case msg = <-w.writerChan: + } + + switch m := msg.(type) { + case dataToBeWritten: + if err := w.appendRecord(m); err != nil { + w.fail(err) + return + } + case flushRequest: + err := w.mutableFile.flush(w.config.FsyncOnFlush) + m.done <- err + if err != nil { + w.fail(err) + return + } + case rangeQuery: + m.reply <- w.bounds() + case pruneRequest: + if err := w.pruneSealedFiles(m.through); err != nil { + w.fail(err) + return + } + case iteratorStartRequest: + resp := w.startIterator(m.startIndex) + m.reply <- resp + if resp.err != nil { + w.fail(resp.err) + return + } + case unpinRequest: + w.releaseIndex(m.index) + case closeRequest: + _, err := w.mutableFile.seal() + m.done <- err + // FIFO guarantees every prior append has been processed. Forbid further pushes so any + // racing/future schedule aborts instead of deadlocking against the now-exiting writer. + w.senderCancel() + return + } + } +} + +// appendRecord appends a record to the mutable file, updates bookkeeping, and rotates once the file exceeds +// the target size. Every record is complete, so any record is a valid rotation boundary. +func (w *walImpl) appendRecord(m dataToBeWritten) error { + // Authoritative ordering check: the caller-side gate in Append can be bypassed by concurrent callers + // (the WAL is documented single-threaded), so re-assert strict increase here on the one writer + // goroutine to reject a reordered record rather than write a file with inverted index bounds. + if w.hasWritten && m.index <= w.lastWrittenIndex { + return fmt.Errorf("append out of order: index %d is not greater than last written index %d", + m.index, w.lastWrittenIndex) + } + if err := w.mutableFile.writeRecord(m.record, m.index); err != nil { + return fmt.Errorf("failed to append record for index %d: %w", m.index, err) + } + w.lastWrittenIndex = m.index + w.hasWritten = true + walBytesWritten.Add(w.ctx, int64(len(m.record)), w.metricAttrs) + walRecordsWritten.Add(w.ctx, 1, w.metricAttrs) + + if w.mutableFile.size >= uint64(w.config.TargetFileSize) { + if err := w.rotate(); err != nil { + return fmt.Errorf("failed to rotate after index %d: %w", m.index, err) + } + } + return nil +} + +// rotate seals the current mutable file, records its bookkeeping, and opens a fresh mutable file. It is only +// called when the mutable file holds at least one record (immediately after an append, or from sealForIterator +// when it has records), so the seal always produces a sealed file rather than removing an empty one. +func (w *walImpl) rotate() error { + fileSeq := w.mutableFile.fileSeq + first := w.mutableFile.firstIndex + last := w.mutableFile.lastIndex + sealedName, err := w.mutableFile.seal() + if err != nil { + return fmt.Errorf("failed to seal WAL file during rotation: %w", err) + } + w.sealedFiles.PushBack(&sealedFileInfo{fileSeq: fileSeq, name: sealedName, firstIndex: first, lastIndex: last}) + walFilesSealed.Add(w.ctx, 1, w.metricAttrs) + + mutable, err := newWalFile(w.config.Path, w.nextFileSeq) + if err != nil { + return fmt.Errorf("failed to open new mutable WAL file during rotation: %w", err) + } + w.mutableFile = mutable + w.nextFileSeq++ + return nil +} + +// pruneSealedFiles deletes sealed files whose highest index is below pruneThrough. Files are removed +// oldest-first (from the front of the deque) with a directory fsync after each removal, so a crash mid-prune +// leaves a contiguous suffix of files rather than a gap in the sequence. The mutable file is never pruned. +// +// A live iterator holds a read lease at some index R and may still read every record from R onward, so no file +// whose range reaches R or higher may be removed. A file [first, last] is needed iff it overlaps [R, ∞), i.e. +// iff last >= R. Comparing the lowest live reservation against each file's last index (rather than testing +// whether the reservation falls inside a file's range) protects exactly the files an iterator can still open — +// even when the reservation lands in a gap between files or strictly inside a file's range. Because +// reservations never fall below the lowest stored index (see pinLowestReadableIndex), a file left below the +// lowest reservation is one the iterator has already moved past and can safely be dropped. +// +// Iteration stops at the first retained file: index ranges grow toward the back, so once a file is kept (by +// pruneThrough or by the lowest reservation) every later file is kept too. +func (w *walImpl) pruneSealedFiles(pruneThrough uint64) error { + // Reservations are mutated only on this (the writer) goroutine, so the lowest reservation is stable for the + // duration of this prune and can be computed once. + reservation, hasReservation := w.lowestReservation() + for { + front, ok := w.sealedFiles.TryPeekFront() + if !ok || front.lastIndex >= pruneThrough { + break + } + if hasReservation && front.lastIndex >= reservation { + break // a live iterator may still read this file (or a later one); keep it and everything after + } + path := filepath.Join(w.config.Path, front.name) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to prune WAL file %s: %w", path, err) + } + if err := util.SyncParentPath(path); err != nil { + return fmt.Errorf("failed to fsync directory after pruning %s: %w", path, err) + } + w.sealedFiles.PopFront() + walFilesPruned.Add(w.ctx, 1, w.metricAttrs) + } + return nil +} + +// startIterator builds an iterator on the writer goroutine. It first seals the mutable file (see +// sealForIterator) so every record written so far lives in an immutable sealed file, then snapshots the sealed +// files in ascending index order, registers the read lease, and constructs the iterator (which launches its +// reader goroutine). Running here serializes construction with rotation, seal, and prune, so the snapshot is a +// consistent point-in-time view: every file the iterator reads is sealed and immutable, opened lazily by name +// and protected from pruning by the lease, so its contents cannot change underneath the reader. +func (w *walImpl) startIterator(startIndex uint64) iteratorStartResponse { + if err := w.sealForIterator(); err != nil { + return iteratorStartResponse{err: fmt.Errorf("failed to seal mutable file before creating iterator: %w", err)} + } + + files := make([]iteratorFile, 0, w.sealedFiles.Size()) + for _, info := range w.sealedFiles.Iterator() { + files = append(files, iteratorFile{ + fileSeq: info.fileSeq, + name: info.name, + firstIndex: info.firstIndex, + lastIndex: info.lastIndex, + }) + } + + pinned := w.pinLowestReadableIndex(startIndex) + it := newWalIterator(w, startIndex, pinned, files, w.config.IteratorPrefetchSize) + return iteratorStartResponse{iterator: it} +} + +// sealForIterator seals the mutable file so a newly-created iterator sees a snapshot that cannot change +// underneath it: after this call every record lives in an immutable sealed file. It is a no-op when the +// mutable file holds no records — the iterator reads only sealed files, so an empty mutable file is simply +// left in place. +func (w *walImpl) sealForIterator() error { + if !w.mutableFile.hasRecords { + return nil + } + if err := w.rotate(); err != nil { + return fmt.Errorf("failed to seal mutable file: %w", err) + } + return nil +} + +// pinLowestReadableIndex records a read lease and returns the pinned index. An iterator reads records at or +// above startIndex but never below the oldest record actually stored, so the lease is clamped up to that: a +// stale low start must not pin files that no longer exist (or wedge pruning forever). Clamping to the oldest +// stored index also establishes the invariant pruneSealedFiles relies on: a reservation never falls below the +// lowest stored index, so a file entirely below the lowest reservation is one every iterator has moved past. +func (w *walImpl) pinLowestReadableIndex(startIndex uint64) uint64 { + pinned := startIndex + if r := w.bounds(); r.ok && r.first > pinned { + pinned = r.first + } + w.indexRefs[pinned]++ + return pinned +} + +// releaseIndex drops one reference to a leased index, forgetting it once the count reaches zero. +func (w *walImpl) releaseIndex(index uint64) { + if w.indexRefs[index] <= 1 { + delete(w.indexRefs, index) + return + } + w.indexRefs[index]-- +} + +// lowestReservation returns the smallest index currently leased by a live iterator, and ok=false when no lease +// is held. A lease at index R means some iterator may still read records at or above R, so every sealed file +// whose range reaches R or higher must be retained by pruning. +func (w *walImpl) lowestReservation() (uint64, bool) { + var lowest uint64 + found := false + for index := range w.indexRefs { + if !found || index < lowest { + lowest = index + found = true + } + } + return lowest, found +} + +// bounds reports the range of record indices across all files. Owned by the writer goroutine. +func (w *walImpl) bounds() storedRange { + var r storedRange + + // The highest index is in the mutable file if it has records, otherwise in the newest sealed file. + if w.mutableFile.hasRecords { + r = storedRange{ok: true, last: w.mutableFile.lastIndex} + } else if back, ok := w.sealedFiles.TryPeekBack(); ok { + r = storedRange{ok: true, last: back.lastIndex} + } else { + return storedRange{} // nothing stored yet + } + + // The lowest index is in the oldest sealed file if any, otherwise in the mutable file. + if front, ok := w.sealedFiles.TryPeekFront(); ok { + r.first = front.firstIndex + } else { + r.first = w.mutableFile.firstIndex + } + return r +} + +// fail records the first fatal background error and triggers shutdown of the pipeline. +func (w *walImpl) fail(err error) { + w.asyncErr.CompareAndSwap(nil, &err) + w.cancel() + if cerr := w.mutableFile.close(); cerr != nil { + logger.Error("failed to close mutable WAL file after fatal error", "err", cerr) + } + logger.Error("WAL encountered a fatal error", "err", err) +} + +// asyncError returns the first fatal background error, or nil if none occurred. +func (w *walImpl) asyncError() error { + if p := w.asyncErr.Load(); p != nil { + return *p + } + return nil +} + +// recoverOrphans seals any unsealed WAL files left behind by a crash. +func recoverOrphans(directory string) error { + entries, err := os.ReadDir(directory) + if err != nil { + return fmt.Errorf("failed to read WAL directory %s: %w", directory, err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + parsed, ok := parseFileName(entry.Name()) + if !ok || parsed.sealed { + continue + } + if err := sealOrphanFile(directory, entry.Name()); err != nil { + return fmt.Errorf("failed to seal orphan %s: %w", entry.Name(), err) + } + } + return nil +} + +// rollbackDirectory drops all records beyond rollbackThrough from the sealed files. Assumes orphans are already +// sealed. Files are processed highest-sequence-first: the files entirely beyond the rollback point (a suffix of +// the sequence) are removed one at a time, each removal made durable before the next, and finally the single +// file straddling the rollback point is truncated. This ordering guarantees that a crash mid-rollback always +// leaves a contiguous prefix of files — never a gap that scanSealedFiles would reject — mirroring the +// contiguous-suffix guarantee that pruning provides from the other end. +func rollbackDirectory(directory string, rollbackThrough uint64) error { + entries, err := os.ReadDir(directory) + if err != nil { + return fmt.Errorf("failed to read WAL directory %s: %w", directory, err) + } + + sealed := make([]parsedFileName, 0, len(entries)) + names := make(map[uint64]string, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + parsed, ok := parseFileName(entry.Name()) + if !ok || !parsed.sealed { + continue + } + sealed = append(sealed, parsed) + names[parsed.fileSeq] = entry.Name() + } + sort.Slice(sealed, func(i int, j int) bool { return sealed[i].fileSeq > sealed[j].fileSeq }) + + for _, parsed := range sealed { + if parsed.lastIndex <= rollbackThrough { + // This file lies entirely at or below the rollback point; so does every lower-sequence file. Done. + break + } + if parsed.firstIndex > rollbackThrough { + // Entirely beyond the rollback point: remove the whole file, durably, before the next-lower one. + if err := removeAndSyncDir(directory, names[parsed.fileSeq]); err != nil { + return fmt.Errorf("failed to roll back %s: %w", names[parsed.fileSeq], err) + } + continue + } + // Straddles the rollback point: truncate away the records beyond it. This is the last file to process. + if err := rollbackStraddlingFile(directory, names[parsed.fileSeq], rollbackThrough); err != nil { + return fmt.Errorf("failed to roll back %s: %w", names[parsed.fileSeq], err) + } + } + return nil +} + +// scanSealedFiles loads the sealed files in a directory into an ascending-order deque and returns the sequence +// to assign the next mutable file (one past the highest sealed sequence, or 0 when there are none). File +// sequences must be contiguous: a gap means a sealed file went missing, which is unrecoverable corruption, so +// this fails with an informative error rather than silently leaving a hole in the index sequence. +func scanSealedFiles(directory string) (*util.RandomAccessDeque[*sealedFileInfo], uint64, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, 0, fmt.Errorf("failed to read WAL directory %s: %w", directory, err) + } + + parsed := make([]parsedFileName, 0, len(entries)) + names := make(map[uint64]string, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + p, ok := parseFileName(entry.Name()) + if !ok || !p.sealed { + continue + } + parsed = append(parsed, p) + names[p.fileSeq] = entry.Name() + } + sort.Slice(parsed, func(i int, j int) bool { return parsed[i].fileSeq < parsed[j].fileSeq }) + + sealedFiles := util.NewRandomAccessDeque[*sealedFileInfo](uint64(len(parsed))) + var nextFileSeq uint64 + for i, p := range parsed { + if i > 0 && p.fileSeq != parsed[i-1].fileSeq+1 { + return nil, 0, fmt.Errorf( + "WAL is corrupt: sealed file sequences are not contiguous (gap between %d and %d)", + parsed[i-1].fileSeq, p.fileSeq) + } + sealedFiles.PushBack(&sealedFileInfo{ + fileSeq: p.fileSeq, name: names[p.fileSeq], firstIndex: p.firstIndex, lastIndex: p.lastIndex}) + nextFileSeq = p.fileSeq + 1 + } + return sealedFiles, nextFileSeq, nil +} + +// validateSealedFiles reads and checksums every sealed file, confirming each file's content exactly covers the +// [first, last] index range its name promises. This surfaces corruption (bit-rot, truncation) at open — where +// it demands human intervention — rather than lazily at iteration time, by which point Bounds/GetStoredRange +// would already be reporting a range that iteration cannot deliver. Cost is O(total sealed bytes): every sealed +// file is read and CRC-verified on open. +func validateSealedFiles(directory string, sealedFiles *util.RandomAccessDeque[*sealedFileInfo]) error { + for _, info := range sealedFiles.Iterator() { + contents, err := readWalFile(filepath.Join(directory, info.name)) + if err != nil { + return fmt.Errorf("failed to read sealed WAL file %s: %w", info.name, err) + } + if err := verifySealedContents(contents, info.fileSeq, info.firstIndex, info.lastIndex); err != nil { + return err + } + } + return nil +} diff --git a/sei-db/seiwal/seiwal_impl_test.go b/sei-db/seiwal/seiwal_impl_test.go new file mode 100644 index 0000000000..4e438b724f --- /dev/null +++ b/sei-db/seiwal/seiwal_impl_test.go @@ -0,0 +1,883 @@ +package seiwal + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestQueueDepthSamplerRunsAndStops exercises the queue-depth sampler goroutine on a tiny interval: it must +// sample the writer channel concurrently with appends (validated by the race detector) and shut down cleanly +// on Close. +func TestQueueDepthSamplerRunsAndStops(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.MetricsSampleInterval = time.Millisecond + w := openWAL(t, cfg) + for index := uint64(1); index <= 300; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) +} + +func testConfig(dir string) *Config { + return DefaultConfig(dir, "test") +} + +func openWAL(t *testing.T, cfg *Config) WAL[[]byte] { + t.Helper() + w, err := NewWAL(cfg) + require.NoError(t, err) + return w +} + +// recordPayload returns a deterministic payload for a record index. +func recordPayload(index uint64) []byte { + return []byte(fmt.Sprintf("payload-%d", index)) +} + +// appendRecord appends a record with recordPayload(index) at the given index. +func appendRecord(t *testing.T, w WAL[[]byte], index uint64) { + t.Helper() + require.NoError(t, w.Append(index, recordPayload(index))) +} + +// collectIndices iterates from start and returns the index of each record, verifying that indices are +// strictly increasing and never below start. +func collectIndices(t *testing.T, w WAL[[]byte], start uint64) []uint64 { + t.Helper() + it, err := w.Iterator(start) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + var indices []uint64 + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + index, _ := it.Entry() + require.GreaterOrEqual(t, index, start) + if len(indices) > 0 { + require.Greater(t, index, indices[len(indices)-1]) + } + indices = append(indices, index) + } + return indices +} + +func countSealedFiles(t *testing.T, dir string) int { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + count := 0 + for _, entry := range entries { + if parsed, ok := parseFileName(entry.Name()); ok && parsed.sealed { + count++ + } + } + return count +} + +// sealedFileNames returns the names of all sealed WAL files in dir, sorted for stable assertions. +func sealedFileNames(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if parsed, ok := parseFileName(entry.Name()); ok && parsed.sealed { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + return names +} + +func TestAppendFlushReopenBounds(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(5), last) + require.NoError(t, w.Close()) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + ok, first, last, err = w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(5), last) + + require.Equal(t, []uint64{1, 2, 3, 4, 5}, collectIndices(t, w2, 1)) +} + +func TestAppendOrdering(t *testing.T) { + t.Run("index must strictly increase", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Append(5, recordPayload(5))) + require.Error(t, w.Append(4, recordPayload(4))) + require.Error(t, w.Append(5, recordPayload(5))) + }) + + t.Run("non-contiguous indices are allowed", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Append(1, recordPayload(1))) + require.NoError(t, w.Append(3, recordPayload(3))) + require.NoError(t, w.Append(100, recordPayload(100))) + require.NoError(t, w.Flush()) + require.Equal(t, []uint64{1, 3, 100}, collectIndices(t, w, 0)) + }) + + t.Run("empty payload is allowed", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Append(1, nil)) + require.NoError(t, w.Append(2, []byte{})) + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + index, data := it.Entry() + require.Equal(t, uint64(1), index) + require.Empty(t, data) + }) +} + +// TestWriterRejectsOutOfOrderRecord exercises the writer-goroutine backstop directly. The caller-side gate +// in Append can be bypassed by concurrent misuse (the reordering race is non-deterministic), so appendRecord +// re-asserts strict index increase itself. Driving appendRecord on a standalone walImpl (no running writer +// loop) verifies that a non-increasing index is rejected rather than written with inverted bounds. +func TestWriterRejectsOutOfOrderRecord(t *testing.T) { + dir := t.TempDir() + mf, err := newWalFile(dir, 0) + require.NoError(t, err) + w := &walImpl{ + config: testConfig(dir), + metricAttrs: walNameAttr("test"), + ctx: context.Background(), + mutableFile: mf, + } + defer func() { _, _ = w.mutableFile.seal() }() + + write := func(index uint64) error { + return w.appendRecord(dataToBeWritten{record: frameRecord(index, recordPayload(index)), index: index}) + } + + require.NoError(t, write(5)) + require.Error(t, write(4)) // lower than last written + require.Error(t, write(5)) // equal to last written + require.NoError(t, write(6)) + require.Equal(t, uint64(6), w.lastWrittenIndex) +} + +// TestFailReleasesMutableFile verifies that a fatal error releases the mutable file's handle (rather than +// leaking the fd until process exit) and that the release is idempotent. +func TestFailReleasesMutableFile(t *testing.T) { + dir := t.TempDir() + mf, err := newWalFile(dir, 0) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + w := &walImpl{ + config: testConfig(dir), + metricAttrs: walNameAttr("test"), + ctx: ctx, + cancel: cancel, + mutableFile: mf, + } + require.NoError(t, w.appendRecord(dataToBeWritten{record: frameRecord(1, recordPayload(1)), index: 1})) + + w.fail(fmt.Errorf("boom")) + + require.Nil(t, w.mutableFile.file) // fd released + require.Error(t, w.asyncError()) // failure recorded + require.NoError(t, w.mutableFile.close()) // idempotent +} + +// TestFlushIOFailureBricksWAL verifies that an IO error during Flush is fatal: the failure is surfaced to the +// flushing caller, the WAL then refuses all further work, and Close reports the original error — matching how +// every other writer IO error is handled, so a broken durability guarantee is never silently tolerated. +func TestFlushIOFailureBricksWAL(t *testing.T) { + dir := t.TempDir() + w := openWAL(t, testConfig(dir)) + + impl, ok := w.(*walImpl) + require.True(t, ok) + + // Force the next flush to fail by closing the mutable file's descriptor out from under the writer. The + // writer is idle (blocked awaiting a message) and never reassigns the handle, and appending only buffers + // bytes, so this affects nothing until the flush attempts to write/fsync the closed descriptor. + require.NoError(t, impl.mutableFile.file.Close()) + + require.NoError(t, w.Append(1, recordPayload(1))) + require.Error(t, w.Flush(), "flush must surface the IO failure") + + // Bricking cancels the context; wait for it so the "refuses further work" assertions are deterministic + // (Flush may return the moment the error is sent, a hair before fail() finishes tearing down). + select { + case <-impl.ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("WAL did not brick after flush failure") + } + + require.Error(t, w.Append(2, recordPayload(2)), "appends must fail on a bricked WAL") + require.Error(t, w.Flush(), "flush must fail on a bricked WAL") + _, _, _, err := w.Bounds() + require.Error(t, err, "bounds must fail on a bricked WAL") + + require.Error(t, w.Close(), "Close must surface the fatal flush error") + require.Error(t, impl.asyncError()) +} + +// TestIteratorRotateFailureBricksWAL verifies that when the rotation performed during iterator creation fails +// at opening the fresh mutable file (after the current file was already sealed), the WAL bricks itself rather +// than limping on with an inconsistent mutable file and later staging a phantom sealed entry. +func TestIteratorRotateFailureBricksWAL(t *testing.T) { + dir := t.TempDir() + w := openWAL(t, testConfig(dir)) + + impl, ok := w.(*walImpl) + require.True(t, ok) + + require.NoError(t, w.Append(1, recordPayload(1))) + require.NoError(t, w.Flush()) + + // Make the rotation's newWalFile step fail while its seal step still succeeds: occupy the exact path the + // next mutable file wants (fileSeq 1 -> "1.wal.u") with a directory, so os.Create there fails with EISDIR. + // The seal renames the current file to "0-1-1.wal", unaffected by this blocker. + blocker := filepath.Join(dir, unsealedFileName(1)) + require.NoError(t, os.Mkdir(blocker, 0o755)) + + _, err := w.Iterator(1) + require.Error(t, err, "iterator creation must surface the rotation failure") + + select { + case <-impl.ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("WAL did not brick after rotation failure during iterator creation") + } + + require.Error(t, w.Append(2, recordPayload(2)), "appends must fail on a bricked WAL") + require.Error(t, w.Close(), "Close must surface the fatal error") + require.Error(t, impl.asyncError()) +} + +func TestOrphanFileRecovery(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + // Fabricate an orphaned unsealed file: records 1 and 2 intact, a torn record 3, left unsealed as if the + // process crashed before it could seal. + f, err := newWalFile(dir, 0) + require.NoError(t, err) + writeRecordTo(t, f, 1, recordPayload(1)) + writeRecordTo(t, f, 2, recordPayload(2)) + frame := frameRecord(3, recordPayload(3)) + require.NoError(t, f.flush(false)) + _, err = f.writer.Write(frame[:len(frame)-3]) // torn record 3 + require.NoError(t, err) + require.NoError(t, f.flush(true)) + require.NoError(t, f.file.Close()) + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(2), last) + require.Equal(t, []uint64{1, 2}, collectIndices(t, w, 1)) +} + +func TestRotationProducesContiguousSealedFiles(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // rotate after every record + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(6), last) + require.Equal(t, []uint64{1, 2, 3, 4, 5, 6}, collectIndices(t, w, 1)) + require.NoError(t, w.Close()) + + // Every record should have produced its own sealed file with a clean [k,k] range. + var sealed []parsedFileName + entries, err := os.ReadDir(dir) + require.NoError(t, err) + for _, entry := range entries { + if parsed, okName := parseFileName(entry.Name()); okName && parsed.sealed { + sealed = append(sealed, parsed) + require.Equal(t, parsed.firstIndex, parsed.lastIndex) + } + } + require.Len(t, sealed, 6) +} + +func TestRecordNeverSplitAcrossFiles(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 128 // tiny, so a single record dwarfs the rotation threshold + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + + // Two records, each far larger than TargetFileSize. + big1 := make([]byte, 4096) + big2 := make([]byte, 4096) + for i := range big1 { + big1[i] = byte(i) + big2[i] = byte(i + 1) + } + require.NoError(t, w.Append(1, big1)) + require.NoError(t, w.Append(2, big2)) + require.NoError(t, w.Flush()) + + // Each oversized record rotated into its own file, intact — never split across files. + require.Equal(t, 2, countSealedFiles(t, dir)) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + index, data := it.Entry() + require.Equal(t, uint64(1), index) + require.Equal(t, big1, data) + + ok, err = it.Next() + require.NoError(t, err) + require.True(t, ok) + index, data = it.Entry() + require.Equal(t, uint64(2), index) + require.Equal(t, big2, data) + + ok, err = it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +func TestPruneDropsWholeFiles(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per file, so pruning can drop whole files + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 10; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + require.NoError(t, w.Prune(5)) + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first) + require.Equal(t, uint64(10), last) + require.Equal(t, []uint64{5, 6, 7, 8, 9, 10}, collectIndices(t, w, 0)) +} + +func TestPrunePastAllRecordsEmptiesRange(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per file so every record sits in a prunable sealed file + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + require.NoError(t, w.Prune(100)) + + ok, _, _, err := w.Bounds() + require.NoError(t, err) + require.False(t, ok) +} + +func TestActiveIteratorBlocksPruningOfNeededFiles(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per sealed file, so pruning works file-by-file + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 10; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + // Hold an iterator anchored at index 1 (the oldest). Its read lease must keep index 1's file alive. + it, err := w.Iterator(1) + require.NoError(t, err) + + require.NoError(t, w.Prune(5)) + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first, "index 1 must survive pruning while a live iterator pins it") + require.Equal(t, uint64(10), last) + + // The iterator still sees the full, intact sequence. + require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, collectIndices(t, w, 1)) + + // Releasing the lease lets the same prune make progress. + require.NoError(t, it.Close()) + require.NoError(t, w.Prune(5)) + ok, first, _, err = w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first) +} + +func TestIteratorAnchoredAboveKeepPointDoesNotBlockPruning(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 10; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + // An iterator anchored at index 8 does not need records below 5, so pruning to 5 proceeds. + it, err := w.Iterator(8) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + require.NoError(t, w.Prune(5)) + ok, first, _, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first) +} + +// TestIteratorInGapBlocksPruningAcrossGap covers the index gap case: indices may jump, so an iterator's read +// lease can land in a gap between stored files. Pruning must still protect every file the iterator will read +// (those reaching the lease index or higher), even though no file's range contains the lease index itself. +// The directory is inspected directly rather than relying on iterator output, since the reader goroutine may +// have buffered the files into memory before an unsafe delete. +func TestIteratorInGapBlocksPruningAcrossGap(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per sealed file + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + // Indices 1,2,3 then a legal jump to 10,11,12. The lease index 5 falls in the gap (3, 10). + for _, index := range []uint64{1, 2, 3, 10, 11, 12} { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(5) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + // Prune(12) would remove every file with last index < 12, but the live lease at 5 must keep the files for + // indices 10 and 11 (both >= 5). Only the files entirely below the lease (indices 1,2,3) may be dropped. + require.NoError(t, w.Prune(12)) + _, _, _, err = w.Bounds() // synchronous round-trip forces the async prune to complete + require.NoError(t, err) + + names := sealedFileNames(t, dir) + require.Contains(t, names, sealedFileName(3, 10, 10), "file for index 10 must survive while iterator(5) is live") + require.Contains(t, names, sealedFileName(4, 11, 11), "file for index 11 must survive while iterator(5) is live") + require.NotContains(t, names, sealedFileName(0, 1, 1), "file for index 1 is below the lease and should be pruned") + + require.Equal(t, []uint64{10, 11, 12}, collectIndices(t, w, 5)) +} + +// TestIteratorLeaseInsideFileRangeBlocksPruning checks the boundary where the lease index sits within the kept +// window: an iterator anchored at 5 must keep indices 5..10 even as pruning is asked to drop through a higher +// point, because those files reach the lease index or higher. +func TestIteratorLeaseInsideFileRangeBlocksPruning(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per sealed file + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 10; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(5) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + require.NoError(t, w.Prune(8)) + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first, "lease at 5 must keep records from 5 onward") + require.Equal(t, uint64(10), last) + require.Equal(t, []uint64{5, 6, 7, 8, 9, 10}, collectIndices(t, w, 5)) +} + +func TestScanRejectsGapInSealedFiles(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per sealed file + + w := openWAL(t, cfg) + for index := uint64(1); index <= 4; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + // Delete a middle sealed file to punch a gap in the sequence, simulating corruption. + var sealed []parsedFileName + entries, err := os.ReadDir(dir) + require.NoError(t, err) + for _, entry := range entries { + if p, ok := parseFileName(entry.Name()); ok && p.sealed { + sealed = append(sealed, p) + } + } + require.GreaterOrEqual(t, len(sealed), 3) + sort.Slice(sealed, func(i int, j int) bool { return sealed[i].fileSeq < sealed[j].fileSeq }) + victim := sealed[len(sealed)/2] + require.NoError(t, os.Remove(filepath.Join(dir, sealedFileName(victim.fileSeq, victim.firstIndex, victim.lastIndex)))) + + _, err = NewWAL(cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "not contiguous") +} + +// TestNewWALRejectsMidStreamCorruptSealedFile verifies that a checksum mismatch in a non-final record of a +// sealed file is surfaced as a hard error at open. Corrupted durable data demands human intervention, so the +// WAL must refuse to open rather than silently serving a view truncated at the corruption point. +func TestNewWALRejectsMidStreamCorruptSealedFile(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) // seals records 1..5 into a single file + + names := sealedFileNames(t, dir) + require.Len(t, names, 1) + path := filepath.Join(dir, names[0]) + data, err := os.ReadFile(path) + require.NoError(t, err) + // Flip a byte in the first record's payload so the fault is mid-stream, not a torn trailing record. The + // first record's payload begins just past the header and its two single-byte uvarint prefixes (index 1, + // length 9), so walHeaderSize+2 lands inside the payload. + data[walHeaderSize+2] ^= 0xFF + require.NoError(t, os.WriteFile(path, data, 0o600)) + + _, err = NewWAL(cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "corrupt") +} + +// TestNewWALRejectsTruncatedSealedFile verifies that a sealed file truncated at a clean record boundary — all +// remaining records checksum correctly, but the content stops short of the last index its name promises — is +// rejected at open. This is the case parse-strictness alone cannot catch (no torn record remains); the +// content/name range check must. +func TestNewWALRejectsTruncatedSealedFile(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) // seals records 1..5 into a single file named 0-1-5 + + names := sealedFileNames(t, dir) + require.Len(t, names, 1) + path := filepath.Join(dir, names[0]) + + // Truncate at the boundary just past the 4th record, leaving indices 1..4 intact while the name still + // promises [1, 5]. + contents, err := readWalFile(path) + require.NoError(t, err) + require.Len(t, contents.records, 5) + require.NoError(t, os.Truncate(path, contents.records[3].end)) + + _, err = NewWAL(cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "corrupt") +} + +// TestNewWALRejectsSealedFileBadMagic verifies that a sealed file with a clobbered header (invalid magic +// prefix) is rejected at open rather than treated as empty. +func TestNewWALRejectsSealedFileBadMagic(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for index := uint64(1); index <= 3; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + names := sealedFileNames(t, dir) + require.Len(t, names, 1) + path := filepath.Join(dir, names[0]) + data, err := os.ReadFile(path) + require.NoError(t, err) + data[0] ^= 0xFF // clobber the magic prefix + require.NoError(t, os.WriteFile(path, data, 0o600)) + + _, err = NewWAL(cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "corrupt") +} + +func TestBoundsEmpty(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + ok, _, _, err := w.Bounds() + require.NoError(t, err) + require.False(t, ok) +} + +func TestRollbackConstructor(t *testing.T) { + t.Run("drops whole files beyond the rollback point", func(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one record per file + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + w2, err := NewWALWithRollback(cfg, 3) + require.NoError(t, err) + defer func() { require.NoError(t, w2.Close()) }() + + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []uint64{1, 2, 3}, collectIndices(t, w2, 1)) + }) + + t.Run("truncates within a file at the rollback point", func(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) // large target: all records land in one file + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + w2, err := NewWALWithRollback(cfg, 3) + require.NoError(t, err) + defer func() { require.NoError(t, w2.Close()) }() + + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []uint64{1, 2, 3}, collectIndices(t, w2, 1)) + + // Appending continues cleanly after the rollback point. + appendRecord(t, w2, 4) + require.NoError(t, w2.Flush()) + _, _, last, err = w2.Bounds() + require.NoError(t, err) + require.Equal(t, uint64(4), last) + }) + + // After a rollback, a subsequent *normal* open (not another rollback) must observe exactly the rolled-back + // range. This is the path that would expose a name/content mismatch left by a non-crash-safe rollback: + // Bounds is name-derived while iteration is content-bound, so the two agree only if the truncation and + // rename were applied consistently. Exercises both rollback shapes: whole-file removal and in-file + // truncation of the straddling file. + t.Run("rolled-back state is consistent under a normal reopen", func(t *testing.T) { + for _, tc := range []struct { + name string + targetSize uint + }{ + {"whole-file removal", 1}, // one record per file: rollback removes whole trailing files + {"in-file truncation", 64 * 1024 * 1024}, // all records in one file: rollback truncates it in place + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = tc.targetSize + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + w2, err := NewWALWithRollback(cfg, 3) + require.NoError(t, err) + require.NoError(t, w2.Close()) + + // Reopen normally; the rollback must have durably and consistently reduced the range to [1,3]. + w3 := openWAL(t, cfg) + defer func() { require.NoError(t, w3.Close()) }() + + ok, first, last, err := w3.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []uint64{1, 2, 3}, collectIndices(t, w3, 1)) + }) + } + }) +} + +// recordPrefixBytes reads the sealed file at path and returns the raw bytes of the prefix ending just past the +// record for lastKeep — i.e. the exact content rollbackStraddlingFile's AtomicWrite would install for a +// rollback to lastKeep. It is the test's stand-in for "the truncated copy the rollback would produce". +func recordPrefixBytes(t *testing.T, path string, lastKeep uint64) []byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + contents, err := readWalFile(path) + require.NoError(t, err) + var truncateTo int64 + found := false + for _, r := range contents.records { + if r.index == lastKeep { + truncateTo = r.end + found = true + break + } + } + require.True(t, found, "index %d has no record boundary in %s", lastKeep, path) + return data[:truncateTo] +} + +// TestRollbackCrashAfterSwapReconciledOnReopen simulates a crash in rollbackStraddlingFile after the reduced +// file was durably written (AtomicWrite) but before the old, larger-named file was removed. That leaves two +// sealed files sharing a sequence. A subsequent open must reconcile them — keeping the reduced file — so the +// name-derived Bounds and the content-derived iterator agree on the rolled-back range. +func TestRollbackCrashAfterSwapReconciledOnReopen(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) // large target: all six records land in one file, sequence 0 + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + oldName := sealedFileName(0, 1, 6) + require.Equal(t, []string{oldName}, sealedFileNames(t, dir)) + + // Reproduce the crash state: the reduced file [1,3] exists next to the untouched original [1,6]. + reducedName := sealedFileName(0, 1, 3) + prefix := recordPrefixBytes(t, filepath.Join(dir, oldName), 3) + require.NoError(t, os.WriteFile(filepath.Join(dir, reducedName), prefix, 0o600)) + require.Equal(t, []string{reducedName, oldName}, sealedFileNames(t, dir)) + + // A plain reopen must reconcile the duplicate sequence down to the rolled-back file. + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + require.Equal(t, []string{reducedName}, sealedFileNames(t, dir)) + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []uint64{1, 2, 3}, collectIndices(t, w2, 1)) +} + +// TestRollbackCrashDuringSwapWindowRecovers simulates a crash mid-rollback in the earlier window: the +// AtomicWrite's swap file was created but not yet renamed into place, so only a leftover ".swap" exists beside +// the still-intact original. A reopen must drop the swap and leave the original range intact (the rollback +// simply did not take effect), and a subsequent rollback must then complete cleanly and durably. +func TestRollbackCrashDuringSwapWindowRecovers(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) // large target: all six records in one file, sequence 0 + + w := openWAL(t, cfg) + for index := uint64(1); index <= 6; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) + + oldName := sealedFileName(0, 1, 6) + + // Reproduce the crash state: an unfinished AtomicWrite left a swap file for the reduced name, alongside + // the untouched original. util.AtomicWrite names its temp ".swap". + prefix := recordPrefixBytes(t, filepath.Join(dir, oldName), 3) + swapName := sealedFileName(0, 1, 3) + ".swap" + require.NoError(t, os.WriteFile(filepath.Join(dir, swapName), prefix, 0o600)) + + // A plain reopen drops the swap; the original range survives (rollback did not take effect). + w2 := openWAL(t, cfg) + require.Equal(t, []string{oldName}, sealedFileNames(t, dir)) + _, err := os.Stat(filepath.Join(dir, swapName)) + require.True(t, os.IsNotExist(err), "leftover swap file should have been removed") + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(6), last) + require.NoError(t, w2.Close()) + + // The subsequent rollback completes cleanly, and a normal reopen sees the consistent rolled-back range. + w3, err := NewWALWithRollback(cfg, 3) + require.NoError(t, err) + require.NoError(t, w3.Close()) + + w4 := openWAL(t, cfg) + defer func() { require.NoError(t, w4.Close()) }() + require.Equal(t, []string{sealedFileName(0, 1, 3)}, sealedFileNames(t, dir)) + ok, first, last, err = w4.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []uint64{1, 2, 3}, collectIndices(t, w4, 1)) +} diff --git a/sei-db/seiwal/seiwal_iterator.go b/sei-db/seiwal/seiwal_iterator.go new file mode 100644 index 0000000000..e83006b830 --- /dev/null +++ b/sei-db/seiwal/seiwal_iterator.go @@ -0,0 +1,269 @@ +package seiwal + +import ( + "fmt" + "os" + "path/filepath" + "sync" +) + +var _ Iterator[[]byte] = (*walIterator)(nil) + +// A record produced by the reader goroutine, or a terminal error. +type iteratorResult struct { + index uint64 + payload []byte + err error +} + +// iteratorFile is one entry in an iterator's file snapshot, captured on the writer goroutine when the iterator +// is created (see startIterator). Every snapshot file is sealed and immutable: it carries its immutable name +// and is opened lazily by the reader, held against pruning by the iterator's read lease. +type iteratorFile struct { + fileSeq uint64 + name string + firstIndex uint64 + lastIndex uint64 +} + +// walIterator iterates the WAL a record at a time, in ascending index order. A dedicated reader goroutine +// reads WAL files from disk and pushes each record onto a buffered channel; Next simply dequeues. Decoupling +// disk reads from the consumer keeps the reader busy while the consumer works, which matters for startup +// replay speed. The reader loads one file at a time, so its memory use is bounded by a single WAL file plus +// the prefetch buffer. +// +// The set of files to read is snapshotted once at creation (files), so the reader walks it in O(n) without +// re-scanning the directory. Every snapshot file is sealed and immutable (the mutable file is sealed at +// creation, see startIterator), so its contents cannot change under the reader. A read lease (pinnedIndex) +// holds the files the reader needs against concurrent pruning; Close releases it. +type walIterator struct { + // The WAL this iterator reads from. + wal *walImpl + + // The lowest index the consumer asked for; records below it are skipped. + start uint64 + + // The index pinned as this iterator's read lease, released on Close. + pinnedIndex uint64 + + // Records produced by the reader goroutine. Closed by the reader on clean EOF. + results chan iteratorResult + + // Closed by Close to tell the reader goroutine to stop early. + stop chan struct{} + + // Closed by the reader goroutine when it exits, so Close can wait for it. + readerExited chan struct{} + + // Ensures the shutdown sequence runs at most once. + closeOnce sync.Once + + // The index and payload returned by Entry, set by the most recent successful Next. Owned by the single + // consumer goroutine (see the Iterator concurrency contract); never touched by Close or the reader. + resultIndex uint64 + resultPayload []byte + + // Set once iteration is complete. Owned by the single consumer goroutine (see the Iterator concurrency + // contract): Next and Close both set it, which is why they must not run concurrently. + done bool + + // The following fields are owned exclusively by the reader goroutine. + + // The point-in-time snapshot of files to read, in ascending index order. Set once at construction. + files []iteratorFile + // The position into files of the next file to load. + filePos int + // The records loaded from the current file, filtered to indices at or beyond start. + buffer []walRecord + // The position within buffer; -1 before the first record is read. + pos int +} + +// newWalIterator creates an iterator over wal starting at startIndex and launches its reader goroutine. +// pinnedIndex is the read lease registered on the iterator's behalf, released by Close. files is the snapshot +// of files to read (captured on the writer goroutine). prefetch is the number of records the reader may buffer +// ahead of the consumer. +func newWalIterator( + wal *walImpl, + startIndex uint64, + pinnedIndex uint64, + files []iteratorFile, + prefetch uint, +) *walIterator { + it := &walIterator{ + wal: wal, + start: startIndex, + pinnedIndex: pinnedIndex, + results: make(chan iteratorResult, prefetch), + stop: make(chan struct{}), + readerExited: make(chan struct{}), + files: files, + pos: -1, + } + go it.read() + return it +} + +func (it *walIterator) Next() (bool, error) { + if it.done { + return false, nil + } + // Drain an already-available result first (non-blocking), so a clean EOF — the reader closing results + // with records still buffered — is never lost to a concurrent WAL shutdown in the select below. + select { + case result, ok := <-it.results: + return it.deliver(result, ok) + default: + } + // Otherwise wait for the next result, but don't block forever if the WAL is torn down. The reader + // goroutine watches the same context (see send) and stops producing when it fires, so the results + // channel would never advance again; surface the shutdown as an error rather than hang. + select { + case result, ok := <-it.results: + return it.deliver(result, ok) + case <-it.wal.ctx.Done(): + it.done = true + return false, fmt.Errorf("WAL shut down during iteration: %w", it.wal.ctx.Err()) + } +} + +// deliver turns a dequeued reader result (or a closed-channel signal) into a Next return value. +func (it *walIterator) deliver(result iteratorResult, ok bool) (bool, error) { + if !ok { + it.done = true + return false, nil + } + if result.err != nil { + it.done = true + return false, result.err + } + it.resultIndex = result.index + it.resultPayload = result.payload + return true, nil +} + +func (it *walIterator) Entry() (uint64, []byte) { + return it.resultIndex, it.resultPayload +} + +func (it *walIterator) Close() error { + it.closeOnce.Do(func() { + close(it.stop) // tell the reader to stop if it is mid-read + <-it.readerExited // wait for it to actually exit before releasing resources + it.wal.unpinIndex(it.pinnedIndex) + }) + it.done = true + return nil +} + +// read is the reader goroutine: it produces records onto the results channel until the WAL is exhausted (then +// closes the channel), a read fails (then sends the error), Close signals a stop, or the WAL context is +// cancelled (see send). It never blocks indefinitely, so it cannot outlive the WAL as a zombie. +func (it *walIterator) read() { + defer close(it.readerExited) + for { + record, ok, err := it.nextRecord() + if err != nil { + it.send(iteratorResult{err: err}) + return + } + if !ok { + close(it.results) + return + } + if !it.send(iteratorResult{index: record.index, payload: record.payload}) { + return // Close signalled a stop + } + } +} + +// send pushes a result onto the channel, returning false if Close signalled a stop or the WAL was torn down +// first. Watching the WAL context here is what keeps the reader from becoming a zombie: if an iterator is +// orphaned (Iterator aborted via ctx.Done before the caller ever received it, so Close is never called) the +// prefetch buffer eventually fills and this send would otherwise block forever with no one to close stop. +func (it *walIterator) send(result iteratorResult) bool { + select { + case it.results <- result: + return true + case <-it.stop: + return false + case <-it.wal.ctx.Done(): + return false + } +} + +// nextRecord returns the next record in ascending order, advancing across files as needed. It returns +// ok=false once no further records remain. +func (it *walIterator) nextRecord() (walRecord, bool, error) { + for { + it.pos++ + if it.pos < len(it.buffer) { + return it.buffer[it.pos], true, nil + } + loaded, err := it.loadNextFile() + if err != nil { + return walRecord{}, false, err + } + if !loaded { + return walRecord{}, false, nil + } + it.pos = -1 + } +} + +// loadNextFile walks the file snapshot from filePos, loading the next file's records (filtered to indices at +// or beyond start) into buffer and advancing filePos. It returns false when the snapshot is exhausted. Sealed +// files entirely below start are skipped without being opened; a file that yields no matching records leaves +// buffer empty (still reported as loaded). +func (it *walIterator) loadNextFile() (bool, error) { + for { + if it.filePos >= len(it.files) { + return false, nil + } + f := &it.files[it.filePos] + it.filePos++ + it.buffer = nil + + if f.lastIndex < it.start { + continue // entirely below the start index; skip without opening + } + + handle, err := it.openFile(f) + if err != nil { + return false, err + } + + parsed := parsedFileName{fileSeq: f.fileSeq, firstIndex: f.firstIndex, lastIndex: f.lastIndex, sealed: true} + contents, err := readWalFileFromHandle(handle, parsed) + if err != nil { + return false, fmt.Errorf("failed to read WAL file (sequence %d) during iteration: %w", f.fileSeq, err) + } + + // A sealed file is durable and complete, so its content must span the [first, last] range its name + // promises. Fail loudly on any shortfall (interior corruption) instead of silently under-yielding while + // Bounds/GetStoredRange keep reporting the full range. This mirrors the check run eagerly at open by + // validateSealedFiles. + if err := verifySealedContents(contents, f.fileSeq, f.firstIndex, f.lastIndex); err != nil { + return false, err + } + + for _, record := range contents.records { + if record.index < it.start { + continue + } + it.buffer = append(it.buffer, record) + } + return true, nil + } +} + +// openFile opens a snapshot file by its immutable sealed name. The read lease keeps the file alive against +// pruning, so the open cannot miss it. readWalFileFromHandle closes the returned handle after reading. +func (it *walIterator) openFile(f *iteratorFile) (*os.File, error) { + path := filepath.Join(it.wal.config.Path, f.name) + handle, err := os.Open(path) //nolint:gosec // path derived from the writer's file snapshot + if err != nil { + return nil, fmt.Errorf("failed to open WAL file %s during iteration: %w", f.name, err) + } + return handle, nil +} diff --git a/sei-db/seiwal/seiwal_iterator_test.go b/sei-db/seiwal/seiwal_iterator_test.go new file mode 100644 index 0000000000..05afd4af5b --- /dev/null +++ b/sei-db/seiwal/seiwal_iterator_test.go @@ -0,0 +1,312 @@ +package seiwal + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestIteratorRejectsCorruptSealedFile verifies that interior corruption appearing in a sealed file after the +// WAL is already open (e.g. bit-rot on disk after a clean startup validation) is surfaced as an error rather +// than silently truncating iteration short of the file's name-promised last index. The corruption is +// introduced after open so it slips past validateSealedFiles and must be caught by the iterator's re-read. +func TestIteratorRejectsCorruptSealedFile(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Close()) // seals records 1..5 into a single file + + names := sealedFileNames(t, dir) + require.Len(t, names, 1) + path := filepath.Join(dir, names[0]) + + w2 := openWAL(t, cfg) // healthy at open; validation passes + defer func() { require.NoError(t, w2.Close()) }() + + // Corrupt the last record's CRC only now, after open, so the parser stops short of index 5 on the + // iterator's re-read of the file from disk. + data, err := os.ReadFile(path) + require.NoError(t, err) + data[len(data)-1] ^= 0xFF + require.NoError(t, os.WriteFile(path, data, 0o600)) + + it, err := w2.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + var iterErr error + for { + ok, err := it.Next() + if err != nil { + iterErr = err + break + } + if !ok { + break + } + } + require.Error(t, iterErr) + require.Contains(t, iterErr.Error(), "corrupt") +} + +func TestIteratorEmpty(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + it, err := w.Iterator(0) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +func TestIteratorFromMiddle(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + require.Equal(t, []uint64{3, 4, 5}, collectIndices(t, w, 3)) +} + +func TestIteratorAcrossFiles(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 // one record per file + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + require.Equal(t, []uint64{2, 3, 4, 5}, collectIndices(t, w, 2)) +} + +func TestIteratorWithTinyPrefetchBuffer(t *testing.T) { + // A prefetch buffer smaller than the number of records exercises reader backpressure: the reader must + // block on a full channel and resume as the consumer drains, without deadlocking or dropping records. + cfg := testConfig(t.TempDir()) + cfg.IteratorPrefetchSize = 1 + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 20; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + collectIndices(t, w, 1)) +} + +func TestIteratorCloseBeforeDrainDoesNotLeak(t *testing.T) { + // Closing an iterator before consuming it must unblock and shut down the reader goroutine cleanly. + cfg := testConfig(t.TempDir()) + cfg.IteratorPrefetchSize = 1 + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for index := uint64(1); index <= 20; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + // Consume just one record, then close while the reader is still mid-stream (blocked on the full buffer). + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, it.Close()) + require.NoError(t, it.Close()) // idempotent +} + +func TestIteratorReaderExitsWhenWALTornDownWhileOrphaned(t *testing.T) { + // Regression: an iterator whose reader fills the prefetch buffer and is never consumed or Closed — as + // happens when Iterator() is aborted via ctx.Done() and the constructed iterator is returned to no one — + // must not leave its reader goroutine blocked on send() forever. Watching the WAL context lets the reader + // exit when the WAL is torn down, so it cannot become a zombie. + cfg := testConfig(t.TempDir()) + cfg.IteratorPrefetchSize = 1 + w := openWAL(t, cfg) + for index := uint64(1); index <= 20; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + iter := it.(*walIterator) + + // Do not consume the iterator: the reader fills the prefetch buffer (size 1) and blocks on send. Tear + // down the WAL context out from under it, as fail() or Close() would. + w.(*walImpl).cancel() + + select { + case <-iter.readerExited: + case <-time.After(5 * time.Second): + t.Fatal("reader goroutine did not exit after the WAL context was cancelled") + } + + // A consumer that races the teardown drains whatever the reader had already buffered, then observes an + // error rather than a clean EOF: a truncated iteration must never masquerade as fully consumed. + var termErr error + for i := 0; i < 25; i++ { + ok, err := it.Next() + if err != nil { + termErr = err + break + } + if !ok { + break + } + } + require.Error(t, termErr, "truncated iteration must surface an error, not a clean EOF") +} + +func TestIteratorYieldsRecordContents(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + require.NoError(t, w.Append(1, []byte("hello world"))) + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + index, data := it.Entry() + require.Equal(t, uint64(1), index) + require.Equal(t, []byte("hello world"), data) + + ok, err = it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +// TestConcurrentIterationDuringRotation hammers the writer with rotate-on-every-record churn while several +// iterators read concurrently. Each iterator seals the mutable file and snapshots its file set at creation on +// the writer goroutine, so every file it reads is sealed and immutable and can never be renamed or rewritten +// out from under an in-flight read; every iteration must be error-free and gap-free. +func TestConcurrentIterationDuringRotation(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 // rotate (rename) after every record, maximizing the seal/rename churn + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + + const totalRecords = 300 + const readers = 4 + const iterationsPerReader = 40 + + var wg sync.WaitGroup + + writeErr := make(chan error, 1) + wg.Add(1) + go func() { + defer wg.Done() + for index := uint64(1); index <= totalRecords; index++ { + if err := w.Append(index, recordPayload(index)); err != nil { + writeErr <- err + return + } + } + writeErr <- nil + }() + + readerErr := make(chan error, readers) + for r := 0; r < readers; r++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterationsPerReader; i++ { + if err := drainContiguousFrom(w, 1); err != nil { + readerErr <- err + return + } + } + readerErr <- nil + }() + } + + wg.Wait() + require.NoError(t, <-writeErr) + for r := 0; r < readers; r++ { + require.NoError(t, <-readerErr) + } +} + +// drainContiguousFrom fully consumes an iterator anchored at start, verifying the yielded indices form a +// gap-free, strictly-increasing run beginning at start (an empty run is allowed: the writer may not have +// produced start yet). Returns the first error encountered. +func drainContiguousFrom(w WAL[[]byte], start uint64) error { + it, err := w.Iterator(start) + if err != nil { + return fmt.Errorf("create iterator: %w", err) + } + prev := start - 1 + for { + ok, err := it.Next() + if err != nil { + _ = it.Close() + return fmt.Errorf("next: %w", err) + } + if !ok { + break + } + index, _ := it.Entry() + if index != prev+1 { + _ = it.Close() + return fmt.Errorf("non-contiguous iteration: got index %d after %d (start %d)", index, prev, start) + } + prev = index + } + return it.Close() +} + +// TestIteratorDoesNotSeePostConstructionRecords pins down the snapshot contract: an iterator yields only +// records that existed when it was created, never records appended afterward. Because Iterator() seals the +// mutable file at creation, a record appended after lands in a fresh file outside the snapshot and must not +// appear. +func TestIteratorDoesNotSeePostConstructionRecords(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) // large target: records begin in one mutable file + defer func() { require.NoError(t, w.Close()) }() + + for index := uint64(1); index <= 3; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + // Appended after the iterator exists, before draining: must not be observed. + require.NoError(t, w.Append(4, recordPayload(4))) + require.NoError(t, w.Flush()) + + var got []uint64 + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + index, _ := it.Entry() + got = append(got, index) + } + require.Equal(t, []uint64{1, 2, 3}, got, "post-construction record 4 must not be iterated") +} diff --git a/sei-db/seiwal/seiwal_serializing.go b/sei-db/seiwal/seiwal_serializing.go new file mode 100644 index 0000000000..80a6470987 --- /dev/null +++ b/sei-db/seiwal/seiwal_serializing.go @@ -0,0 +1,420 @@ +package seiwal + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/metric" +) + +var _ WAL[[]byte] = (*serializingWAL[[]byte])(nil) + +// serAppend carries a framed-payload producer to the serializer goroutine. The closure captures the typed +// item so this message type stays non-generic — T never enters the channel's dynamic type, which keeps the +// serializer loop's type switch free of type parameters. +type serAppend struct { + index uint64 + serialize func() ([]byte, error) +} + +// serFlush asks the serializer goroutine to flush the inner WAL, signaling done when durable. +type serFlush struct { + done chan error +} + +// serBounds asks the serializer goroutine to report the inner WAL's stored index range. +type serBounds struct { + reply chan serBoundsResult +} + +// The index range (and any error) reported by the inner WAL's Bounds. +type serBoundsResult struct { + ok bool + first uint64 + last uint64 + err error +} + +// serPrune asks the serializer goroutine to prune the inner WAL below `through`. +type serPrune struct { + through uint64 +} + +// serIterator asks the serializer goroutine to create an inner iterator, ordered after every prior append. +type serIterator struct { + startIndex uint64 + reply chan serIteratorResult +} + +// The inner iterator (or an error) produced in response to a serIterator request. +type serIteratorResult struct { + it Iterator[[]byte] + err error +} + +// serClose asks the serializer goroutine to close the inner WAL and shut down, signaling done when closed. +type serClose struct { + done chan error +} + +// serializingWAL is a WAL[T] that serializes each payload to []byte on a background goroutine. +type serializingWAL[T any] struct { + // The inner byte-oriented WAL that framed records are delegated to. + inner WAL[[]byte] + + // Serializes a payload to bytes; runs on the serializer goroutine. + serialize func(T) ([]byte, error) + // Deserializes stored bytes back to a payload; runs inline in the iterator. + deserialize func([]byte) (T, error) + + // The measurement option tagging this instance's metrics with its name. Read-only after construction. + metricAttrs metric.MeasurementOption + + // Caller entry points funnel through serializerChan as a single ordered stream to the serializer. + serializerChan chan any + + // The hard-stop context the serializer watches. Cancelled by fail() on a fatal error and by Close() once + // everything has drained. + ctx context.Context + // Cancels ctx, tearing down the serializer goroutine. + cancel context.CancelFunc + + // A child of ctx that the serializerChan producers watch, cancelled once the serializer stops reading so + // an in-flight or future push aborts rather than deadlocking. + senderCtx context.Context + // Cancels senderCtx. + senderCancel context.CancelFunc + + // Tracks the serializer and queue-depth sampler goroutines so Close() can wait for them to exit. + wg sync.WaitGroup + + // Closed by Close() to stop the queue-depth sampler goroutine. + samplerStop chan struct{} + + // Guarantees the Close() shutdown sequence runs at most once. + closeOnce sync.Once + + // Set by Close() so subsequent scheduling calls fail fast. + closed atomic.Bool + + // The first unrecoverable background-goroutine error, surfaced to the caller by Close(). + asyncErr atomic.Pointer[error] +} + +// NewGenericWAL opens a WAL over payloads of type T that does serialization on a background goroutine. +func NewGenericWAL[T any]( + config *Config, + serialize func(T) ([]byte, error), + deserialize func([]byte) (T, error), +) (WAL[T], error) { + inner, err := NewWAL(config) + if err != nil { + return nil, fmt.Errorf("failed to open inner WAL: %w", err) + } + return newSerializingWAL(config, inner, serialize, deserialize), nil +} + +// NewGenericWALWithRollback is like NewGenericWAL but first rolls the inner WAL back so it contains no record +// with an index greater than rollbackIndex. +func NewGenericWALWithRollback[T any]( + config *Config, + rollbackIndex uint64, + serialize func(T) ([]byte, error), + deserialize func([]byte) (T, error), +) (WAL[T], error) { + inner, err := NewWALWithRollback(config, rollbackIndex) + if err != nil { + return nil, fmt.Errorf("failed to open inner WAL: %w", err) + } + return newSerializingWAL(config, inner, serialize, deserialize), nil +} + +func newSerializingWAL[T any]( + config *Config, + inner WAL[[]byte], + serialize func(T) ([]byte, error), + deserialize func([]byte) (T, error), +) *serializingWAL[T] { + ctx, cancel := context.WithCancel(context.Background()) + senderCtx, senderCancel := context.WithCancel(ctx) + + s := &serializingWAL[T]{ + inner: inner, + serialize: serialize, + deserialize: deserialize, + metricAttrs: walNameAttr(config.Name), + serializerChan: make(chan any, config.SerializerBufferSize), + ctx: ctx, + cancel: cancel, + senderCtx: senderCtx, + senderCancel: senderCancel, + samplerStop: make(chan struct{}), + } + + s.wg.Add(1) + go s.serializerLoop() + + if config.MetricsSampleInterval > 0 { + s.wg.Add(1) + go s.sampleQueueDepth(config.Name, config.MetricsSampleInterval) + } + + return s +} + +// sampleQueueDepth periodically records the serializer channel's buffered depth until Close stops it +// (samplerStop) or a fatal shutdown cancels ctx. +func (s *serializingWAL[T]) sampleQueueDepth(name string, interval time.Duration) { + defer s.wg.Done() + attrs := queueDepthAttrs(name, "serializer") + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-s.samplerStop: + return + case <-ticker.C: + walQueueDepth.Record(s.ctx, int64(len(s.serializerChan)), attrs) + } + } +} + +// Append schedules a payload to be serialized and appended at the given index. +func (s *serializingWAL[T]) Append(index uint64, data T) error { + if s.closed.Load() { + return fmt.Errorf("WAL is closed") + } + // Fail fast on a bricked WAL, so a fire-and-forget append cannot win the submit select against the + // cancelled senderCtx and be silently dropped onto a dead serializer (see walImpl.Append for detail). + if err := s.asyncError(); err != nil { + return fmt.Errorf("WAL failed: %w", err) + } + req := serAppend{ + index: index, + serialize: func() ([]byte, error) { return s.serialize(data) }, + } + if err := s.submit(req); err != nil { + return fmt.Errorf("failed to schedule append for index %d: %w", index, err) + } + return nil +} + +// Flush blocks until all previously scheduled appends are durable. +func (s *serializingWAL[T]) Flush() error { + done := make(chan error, 1) + if err := s.submit(serFlush{done: done}); err != nil { + return fmt.Errorf("failed to schedule flush: %w", err) + } + select { + case err := <-done: + return err // already wrapped by the inner WAL, or nil on success + case <-s.ctx.Done(): + if err := s.asyncError(); err != nil { + return fmt.Errorf("flush aborted: %w", err) + } + return fmt.Errorf("flush aborted: %w", s.ctx.Err()) + } +} + +// Bounds reports the range of record indices stored in the WAL. +func (s *serializingWAL[T]) Bounds() (bool, uint64, uint64, error) { + reply := make(chan serBoundsResult, 1) + if err := s.submit(serBounds{reply: reply}); err != nil { + return false, 0, 0, fmt.Errorf("failed to schedule bounds query: %w", err) + } + select { + case r := <-reply: + if r.err != nil { + return false, 0, 0, fmt.Errorf("bounds query failed: %w", r.err) + } + return r.ok, r.first, r.last, nil + case <-s.ctx.Done(): + if err := s.asyncError(); err != nil { + return false, 0, 0, fmt.Errorf("bounds query aborted: %w", err) + } + return false, 0, 0, fmt.Errorf("bounds query aborted: %w", s.ctx.Err()) + } +} + +// Prune schedules removal of whole inner files below lowestIndexToKeep. It does not block on completion. +func (s *serializingWAL[T]) Prune(lowestIndexToKeep uint64) error { + if err := s.submit(serPrune{through: lowestIndexToKeep}); err != nil { + return fmt.Errorf("failed to schedule prune below index %d: %w", lowestIndexToKeep, err) + } + return nil +} + +// Iterator returns an iterator over the WAL starting at startIndex. Construction is ordered on the serializer +// goroutine after every prior append, so the iterator observes all previously scheduled appends. +func (s *serializingWAL[T]) Iterator(startIndex uint64) (Iterator[T], error) { + reply := make(chan serIteratorResult, 1) + if err := s.submit(serIterator{startIndex: startIndex, reply: reply}); err != nil { + return nil, fmt.Errorf("failed to schedule iterator creation: %w", err) + } + select { + case r := <-reply: + if r.err != nil { + return nil, fmt.Errorf("failed to create iterator: %w", r.err) + } + return &serializingIterator[T]{inner: r.it, deserialize: s.deserialize}, nil + case <-s.ctx.Done(): + if err := s.asyncError(); err != nil { + return nil, fmt.Errorf("iterator creation aborted: %w", err) + } + return nil, fmt.Errorf("iterator creation aborted: %w", s.ctx.Err()) + } +} + +// Close flushes pending appends, closes the inner WAL, and releases resources. +func (s *serializingWAL[T]) Close() error { + var closeErr error + s.closeOnce.Do(func() { + s.closed.Store(true) + close(s.samplerStop) // stop the queue-depth sampler before waiting for goroutines + done := make(chan error, 1) + if err := s.submit(serClose{done: done}); err == nil { + select { + case closeErr = <-done: + case <-s.ctx.Done(): + } + } + s.wg.Wait() + s.cancel() + }) + if err := s.asyncError(); err != nil { + return fmt.Errorf("WAL closed with error: %w", err) + } + return closeErr // already wrapped by the inner WAL, or nil on a clean close +} + +// submit enqueues a message onto the serializer's input channel, aborting if the WAL is shutting down or has +// failed. +func (s *serializingWAL[T]) submit(msg any) error { + select { + case s.serializerChan <- msg: + return nil + case <-s.senderCtx.Done(): + if err := s.asyncError(); err != nil { + return fmt.Errorf("WAL failed: %w", err) + } + return fmt.Errorf("WAL is closed") + } +} + +// serializerLoop serializes each append's payload and delegates it to the inner WAL, handling control +// messages (flush, bounds, prune, iterator, close) in FIFO order relative to appends so they observe a +// consistent view. Runs on its own goroutine until close or a fatal error. +func (s *serializingWAL[T]) serializerLoop() { + defer s.wg.Done() + for { + var msg any + select { + case <-s.ctx.Done(): + return + case msg = <-s.serializerChan: + } + + switch m := msg.(type) { + case serAppend: + start := time.Now() + data, err := m.serialize() + if err != nil { + walSerializeErrors.Add(s.ctx, 1, s.metricAttrs) + s.fail(fmt.Errorf("failed to serialize record for index %d: %w", m.index, err)) + return + } + walSerializeDuration.Record(s.ctx, time.Since(start).Seconds(), s.metricAttrs) + walSerializedBytes.Add(s.ctx, int64(len(data)), s.metricAttrs) + if err := s.inner.Append(m.index, data); err != nil { + s.fail(fmt.Errorf("failed to append record for index %d: %w", m.index, err)) + return + } + case serFlush: + err := s.inner.Flush() + m.done <- err + if err != nil { + s.fail(fmt.Errorf("failed to flush: %w", err)) + return + } + case serBounds: + ok, first, last, err := s.inner.Bounds() + m.reply <- serBoundsResult{ok: ok, first: first, last: last, err: err} + case serPrune: + if err := s.inner.Prune(m.through); err != nil { + s.fail(fmt.Errorf("failed to prune below index %d: %w", m.through, err)) + return + } + case serIterator: + it, err := s.inner.Iterator(m.startIndex) + m.reply <- serIteratorResult{it: it, err: err} + case serClose: + m.done <- s.inner.Close() + // FIFO guarantees every prior append has been delegated. Forbid further pushes so any + // racing/future schedule aborts instead of deadlocking against the now-exiting serializer. + s.senderCancel() + return + } + } +} + +// fail records the first fatal background error and triggers shutdown of the pipeline. +func (s *serializingWAL[T]) fail(err error) { + s.asyncErr.CompareAndSwap(nil, &err) + s.cancel() + if cerr := s.inner.Close(); cerr != nil { + logger.Error("failed to close inner WAL after fatal error", "err", cerr) + } + logger.Error("serializing WAL encountered a fatal error", "err", err) +} + +// asyncError returns the first fatal background error, or nil if none occurred. +func (s *serializingWAL[T]) asyncError() error { + if p := s.asyncErr.Load(); p != nil { + return *p + } + return nil +} + +var _ Iterator[[]byte] = (*serializingIterator[[]byte])(nil) + +// serializingIterator adapts an inner byte iterator to a typed iterator by running deserialize inline in Next. +// Like the inner iterator, it is single-consumer and not safe for concurrent use (see the Iterator +// concurrency contract). +type serializingIterator[T any] struct { + inner Iterator[[]byte] + deserialize func([]byte) (T, error) + index uint64 + entry T +} + +func (it *serializingIterator[T]) Next() (bool, error) { + ok, err := it.inner.Next() + if err != nil || !ok { + var zero T + it.entry = zero + return false, err + } + index, data := it.inner.Entry() + value, err := it.deserialize(data) + if err != nil { + var zero T + it.entry = zero + return false, fmt.Errorf("failed to deserialize record at index %d: %w", index, err) + } + it.index = index + it.entry = value + return true, nil +} + +func (it *serializingIterator[T]) Entry() (uint64, T) { + return it.index, it.entry +} + +func (it *serializingIterator[T]) Close() error { + return it.inner.Close() +} diff --git a/sei-db/seiwal/seiwal_serializing_test.go b/sei-db/seiwal/seiwal_serializing_test.go new file mode 100644 index 0000000000..be8900d225 --- /dev/null +++ b/sei-db/seiwal/seiwal_serializing_test.go @@ -0,0 +1,258 @@ +package seiwal + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestGenericWALQueueDepthSampler exercises the queue-depth samplers (both the serializing layer's and the +// inner byte engine's) on a tiny interval, validating concurrent sampling under the race detector and a clean +// shutdown on Close. +func TestGenericWALQueueDepthSampler(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.MetricsSampleInterval = time.Millisecond + w := openStringWAL(t, cfg) + for i := uint64(1); i <= 300; i++ { + require.NoError(t, w.Append(i, fmt.Sprintf("v%d", i))) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) +} + +func stringSerialize(s string) ([]byte, error) { return []byte(s), nil } +func stringDeserialize(b []byte) (string, error) { return string(b), nil } + +func openStringWAL(t *testing.T, cfg *Config) WAL[string] { + t.Helper() + w, err := NewGenericWAL[string](cfg, stringSerialize, stringDeserialize) + require.NoError(t, err) + return w +} + +type indexedString struct { + index uint64 + value string +} + +func collectStrings(t *testing.T, w WAL[string], start uint64) []indexedString { + t.Helper() + it, err := w.Iterator(start) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + var out []indexedString + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + index, value := it.Entry() + out = append(out, indexedString{index: index, value: value}) + } + return out +} + +// TestSerializeFailureClosesInnerWAL verifies that a serialize error tears the inner byte WAL down instead of +// orphaning its writer goroutine and mutable-file handle. The inner WAL is healthy (only serialization failed), +// so fail() closes it gracefully — observable as the inner mutable file being sealed. +func TestSerializeFailureClosesInnerWAL(t *testing.T) { + cfg := testConfig(t.TempDir()) + boom := errors.New("serialize boom") + serialize := func(s string) ([]byte, error) { return nil, boom } + w, err := NewGenericWAL[string](cfg, serialize, stringDeserialize) + require.NoError(t, err) + + require.NoError(t, w.Append(1, "one")) // scheduling succeeds; serialize fails on the serializer goroutine + + // Close drains the serializer goroutine, which by now has run fail() -> inner.Close(). + err = w.Close() + require.Error(t, err) + require.ErrorIs(t, err, boom) + + sw := w.(*serializingWAL[string]) + inner := sw.inner.(*walImpl) + require.True(t, inner.mutableFile.sealed) // inner cleanly closed by fail(), not orphaned +} + +// TestGenericWALFlushIOFailureBricksWAL verifies that an inner flush IO failure bricks the serializing WAL too: +// the inner byte engine tears itself down, and the serializing layer mirrors that rather than delegating +// subsequent appends to a dead inner WAL. +func TestGenericWALFlushIOFailureBricksWAL(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openStringWAL(t, cfg) + + ser, ok := w.(*serializingWAL[string]) + require.True(t, ok) + inner, ok := ser.inner.(*walImpl) + require.True(t, ok) + + // Close the inner mutable file's descriptor so the flush the inner engine performs fails. + require.NoError(t, inner.mutableFile.file.Close()) + + require.NoError(t, w.Append(1, "one")) + require.Error(t, w.Flush(), "flush must surface the inner IO failure") + + // Bricking cancels the serializing layer's context; wait for it so the assertions below are deterministic. + select { + case <-ser.ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("serializing WAL did not brick after flush failure") + } + + require.Error(t, w.Append(2, "two"), "appends must fail on a bricked WAL") + require.Error(t, w.Flush(), "flush must fail on a bricked WAL") + require.Error(t, w.Close(), "Close must surface the fatal flush error") +} + +func TestGenericWALRoundTrip(t *testing.T) { + w := openStringWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + require.NoError(t, w.Append(1, "one")) + require.NoError(t, w.Append(2, "two")) + require.NoError(t, w.Append(5, "five")) // non-contiguous index is allowed + require.NoError(t, w.Flush()) + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(5), last) + + require.Equal(t, []indexedString{{1, "one"}, {2, "two"}, {5, "five"}}, collectStrings(t, w, 0)) + require.Equal(t, []indexedString{{2, "two"}, {5, "five"}}, collectStrings(t, w, 2)) +} + +func TestGenericWALReopen(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openStringWAL(t, cfg) + for i := uint64(1); i <= 3; i++ { + require.NoError(t, w.Append(i, fmt.Sprintf("v%d", i))) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + w2 := openStringWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []indexedString{{1, "v1"}, {2, "v2"}, {3, "v3"}}, collectStrings(t, w2, 0)) +} + +func TestGenericWALPrune(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 // one record per file so pruning drops whole files + + w := openStringWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for i := uint64(1); i <= 10; i++ { + require.NoError(t, w.Append(i, fmt.Sprintf("v%d", i))) + } + require.NoError(t, w.Flush()) + + require.NoError(t, w.Prune(5)) + + ok, first, last, err := w.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first) + require.Equal(t, uint64(10), last) +} + +func TestGenericWALRollback(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 + + w := openStringWAL(t, cfg) + for i := uint64(1); i <= 6; i++ { + require.NoError(t, w.Append(i, fmt.Sprintf("v%d", i))) + } + require.NoError(t, w.Close()) + + w2, err := NewGenericWALWithRollback[string](cfg, 3, stringSerialize, stringDeserialize) + require.NoError(t, err) + defer func() { require.NoError(t, w2.Close()) }() + + ok, first, last, err := w2.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(3), last) + require.Equal(t, []indexedString{{1, "v1"}, {2, "v2"}, {3, "v3"}}, collectStrings(t, w2, 0)) +} + +func TestGenericWALSerializeErrorSurfaces(t *testing.T) { + serErr := errors.New("serialize boom") + serialize := func(s string) ([]byte, error) { + if s == "bad" { + return nil, serErr + } + return []byte(s), nil + } + w, err := NewGenericWAL[string](testConfig(t.TempDir()), serialize, stringDeserialize) + require.NoError(t, err) + + require.NoError(t, w.Append(1, "good")) + require.NoError(t, w.Append(2, "bad")) // async; the serialize failure tears the pipeline down + + // The fatal serialization error surfaces on the next synchronous operation. + require.Error(t, w.Flush()) + require.Error(t, w.Close()) +} + +func TestGenericWALDeserializeErrorSurfaces(t *testing.T) { + deErr := errors.New("deserialize boom") + deserialize := func(b []byte) (string, error) { + if string(b) == "poison" { + return "", deErr + } + return string(b), nil + } + w, err := NewGenericWAL[string](testConfig(t.TempDir()), stringSerialize, deserialize) + require.NoError(t, err) + defer func() { require.NoError(t, w.Close()) }() + + require.NoError(t, w.Append(1, "ok")) + require.NoError(t, w.Append(2, "poison")) + require.NoError(t, w.Flush()) + + it, err := w.Iterator(0) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + index, value := it.Entry() + require.Equal(t, uint64(1), index) + require.Equal(t, "ok", value) + + // The poison record fails to deserialize; the error surfaces from Next, not a clean EOF. + ok, err = it.Next() + require.Error(t, err) + require.False(t, ok) +} + +func TestGenericWALAppendOrdering(t *testing.T) { + w := openStringWAL(t, testConfig(t.TempDir())) + + require.NoError(t, w.Append(5, "five")) + require.NoError(t, w.Flush()) + // The inner byte engine enforces strictly-increasing indices; a stale index tears the pipeline down. + require.NoError(t, w.Append(4, "four")) // async, will fail on the goroutine + require.Error(t, w.Flush()) + // Close surfaces the same fatal error rather than succeeding. + require.Error(t, w.Close()) +} diff --git a/sei-db/state_db/statewal/state_wal.go b/sei-db/state_db/statewal/state_wal.go new file mode 100644 index 0000000000..35e87dc7b3 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal.go @@ -0,0 +1,86 @@ +package statewal + +import ( + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/seiwal" +) + +// A WAL for state. +// +// A StateWAL is not safe for concurrent use. Callers must serialize their calls to a single instance; +// in particular Write and SignalEndOfBlock share write-ordering state that is not internally locked. +// +// Slices are not copied at the call boundary. Changesets passed to Write — and every byte slice reachable +// through them — must not be modified after the call: the WAL retains them and serializes them +// asynchronously, so mutating them races the WAL and can corrupt what is persisted. Likewise the changesets +// returned by the iterator are owned by the WAL and must be treated as read-only. Callers that need to +// mutate such data must copy it first. +type StateWAL interface { + + // Write a set of changes to the WAL. + // + // This method only schedules the write, it does not block until the write is complete. + // + // cs, and every byte slice reachable through it (changeset keys and values), must not be modified after + // this call. Callers that need to modify those buffers must copy them first. + // + // The StateWAL rejects writes for blocks if provided out of order. To avoid errors, observe + // the following rules: + // + // - The block numbers passed to Write() may never decrease. + // - If data has been written for block N, you cannot write data for block N+1 until you have called + // SignalEndOfBlock(). + Write( + // The block number associated with the changeset. + blockNumber uint64, + // The changeset to write. + cs []*proto.NamedChangeSet, + ) error + + // Signal that there will be no more writes for the current block number. Attempting to write additional + // changes for the same block number after calling this method may result in an error. + // + // Similar to Write(), this method is asynchronous. Calling this method does not, by itself, make data immediately + // crash durable. + SignalEndOfBlock() error + + // Flush the WAL to disk. All data previously passed to Write() before this call will be crash durable + // after this call returns. + Flush() error + + // Get the range of block numbers stored in the WAL. + GetStoredRange() ( + // If true, there is data in the WAL. If false, the WAL is empty and startBlockNumber and + // endBlockNumber are undefined. + ok bool, + // The lowest block number stored in the WAL, inclusive. Only valid if ok is true. + startBlockNumber uint64, + // The highest block number stored in the WAL, inclusive. Only valid if ok is true. + endBlockNumber uint64, + // Any error encountered while retrieving the range. + err error, + ) + + // Prune the WAL, removing all entries with block numbers less than lowestBlockNumberToKeep. + // + // This method merely schedules the prune operation, it does not block until the prune is complete. Pruning + // is async and lazy, and implementations are free to delay pruning arbitrarily long. If crashed or closed + // before the prune is complete, the WAL may not attempt to prune again on the next open unless Prune() is + // called again or for a higher block number. + Prune(lowestBlockNumberToKeep uint64) error + + // Create an iterator over the WAL, starting at the given block number. + // + // The iterator reads a consistent, point-in-time snapshot of the WAL taken at some instant between the + // start and the return of this call. Data written before that instant is included; data written after it + // is not. For data written concurrently with this call, whether it is included is unspecified. + // + // The iterator yields one entry per block in ascending block order. Its Entry() returns (blockNumber, + // changesets), where changesets are all the changes written for that block (across one or more Write + // calls) combined in write order. Blocks that were never ended with SignalEndOfBlock are not yielded. + // The returned changesets, and every byte slice reachable through them, must be treated as read-only. + Iterator(startingBlockNumber uint64) (seiwal.Iterator[[]*proto.NamedChangeSet], error) + + // Close the WAL, flushing any pending writes and releasing resources. + Close() error +} diff --git a/sei-db/state_db/statewal/state_wal_config.go b/sei-db/state_db/statewal/state_wal_config.go new file mode 100644 index 0000000000..7b5bfa1700 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_config.go @@ -0,0 +1,76 @@ +package statewal + +import ( + "time" + + "github.com/sei-protocol/sei-chain/sei-db/seiwal" +) + +// Configuration for a state WAL. +type Config struct { + // The directory where the WAL writes its files. + Path string + + // A short identifier for this WAL instance, used to distinguish its metrics from those of other + // instances in the same process. Required; must match [a-zA-Z0-9_-]+. + Name string + + // The size of the channel used to send work from the caller to the serialization goroutine. + RequestBufferSize uint + + // The size of the channel used to send framed records from the underlying WAL's serialization to its + // writer goroutine. + WriteBufferSize uint + + // The size a WAL file may reach before it is sealed and a fresh one is opened. Because each block is + // written as a single record, a file may exceed this by the size of one block's serialized changesets. + // Must be greater than 0. + TargetFileSize uint + + // When true, Flush calls fsync on the underlying file so that flushed data survives a power loss, not + // merely a process crash. When false, Flush only flushes the in-process buffer to the OS. + FsyncOnFlush bool + + // The number of blocks an iterator's reader thread may prefetch ahead of the consumer. A larger value + // keeps the reader busy while the consumer processes blocks, which matters for startup replay speed. + // Must be greater than 0. + IteratorPrefetchSize uint + + // The interval at which the underlying WAL samples the buffered depth of its internal channels into the + // seiwal_queue_depth gauge. Zero or negative disables sampling. + MetricsSampleInterval time.Duration +} + +// Constructor for a default state WAL configuration for the WAL at path, identified by name. +func DefaultConfig(path string, name string) *Config { + s := seiwal.DefaultConfig(path, name) + return &Config{ + Path: path, + Name: name, + RequestBufferSize: 16, + WriteBufferSize: s.WriteBufferSize, + TargetFileSize: s.TargetFileSize, + FsyncOnFlush: s.FsyncOnFlush, + IteratorPrefetchSize: s.IteratorPrefetchSize, + MetricsSampleInterval: s.MetricsSampleInterval, + } +} + +// Validate the configuration, returning nil if valid, or an error describing the problem if invalid. +func (c *Config) Validate() error { + return c.toSeiwalConfig().Validate() +} + +// toSeiwalConfig maps this configuration onto the underlying generic WAL's configuration. +func (c *Config) toSeiwalConfig() *seiwal.Config { + return &seiwal.Config{ + Path: c.Path, + Name: c.Name, + WriteBufferSize: c.WriteBufferSize, + SerializerBufferSize: c.RequestBufferSize, + TargetFileSize: c.TargetFileSize, + FsyncOnFlush: c.FsyncOnFlush, + IteratorPrefetchSize: c.IteratorPrefetchSize, + MetricsSampleInterval: c.MetricsSampleInterval, + } +} diff --git a/sei-db/state_db/statewal/state_wal_config_test.go b/sei-db/state_db/statewal/state_wal_config_test.go new file mode 100644 index 0000000000..4d28024e7b --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_config_test.go @@ -0,0 +1,40 @@ +package statewal + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigValidate(t *testing.T) { + t.Run("default config is valid", func(t *testing.T) { + require.NoError(t, DefaultConfig("/tmp/wal", "test").Validate()) + }) + + t.Run("empty path is rejected", func(t *testing.T) { + cfg := DefaultConfig("", "test") + require.Error(t, cfg.Validate()) + }) + + t.Run("empty name is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "") + require.Error(t, cfg.Validate()) + }) + + t.Run("malformed name is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "bad name!") + require.Error(t, cfg.Validate()) + }) + + t.Run("zero target file size is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "test") + cfg.TargetFileSize = 0 + require.Error(t, cfg.Validate()) + }) + + t.Run("zero iterator prefetch size is rejected", func(t *testing.T) { + cfg := DefaultConfig("/tmp/wal", "test") + cfg.IteratorPrefetchSize = 0 + require.Error(t, cfg.Validate()) + }) +} diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go new file mode 100644 index 0000000000..5ddcb42c91 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -0,0 +1,163 @@ +package statewal + +import ( + "fmt" + "sync/atomic" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/seiwal" +) + +var _ StateWAL = (*stateWALImpl)(nil) + +// A WAL for storing state changesets by block number. +// +// Not safe for concurrent use; see the StateWAL interface doc. +type stateWALImpl struct { + // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. + wal seiwal.WAL[[]*proto.NamedChangeSet] + + // Set by Close() so subsequent Write/SignalEndOfBlock calls fail fast. + closed atomic.Bool + + // The write-ordering contract state and the accumulation buffer below are mutated by Write and + // SignalEndOfBlock, which callers must not invoke concurrently. + + // The block number of the most recent Write or SignalEndOfBlock. + currentBlock uint64 + // Whether currentBlock has been finalized by SignalEndOfBlock. + currentBlockEnded bool + // Whether any block has been observed (this session or recovered from disk). + hasCurrentBlock bool + // The changesets accumulated for the current block across its Write calls, appended as one record at + // end-of-block. Ownership is handed to the WAL at end-of-block and a fresh buffer starts for the next + // block, so the serialization goroutine never races the wrapper over the backing array. + buf []*proto.NamedChangeSet +} + +// New opens (or creates) a state WAL in the configured directory, recovering any files left behind by a +// previous session. +func New(config *Config) (StateWAL, error) { + wal, err := seiwal.NewGenericWAL[[]*proto.NamedChangeSet]( + config.toSeiwalConfig(), serializeChangesets, deserializeChangesets) + if err != nil { + return nil, fmt.Errorf("failed to open state WAL: %w", err) + } + return newStateWAL(wal) +} + +// NewWithRollback opens a state WAL and deletes all data for blocks beyond rollbackBlockNumber before +// returning, so the WAL contains no block greater than rollbackBlockNumber. +func NewWithRollback(config *Config, rollbackBlockNumber uint64) (StateWAL, error) { + wal, err := seiwal.NewGenericWALWithRollback[[]*proto.NamedChangeSet]( + config.toSeiwalConfig(), rollbackBlockNumber, serializeChangesets, deserializeChangesets) + if err != nil { + return nil, fmt.Errorf("failed to open state WAL: %w", err) + } + return newStateWAL(wal) +} + +func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { + w := &stateWALImpl{wal: wal} + + // Recover the write-ordering position from the highest block already on disk. + ok, _, last, err := wal.Bounds() + if err != nil { + _ = wal.Close() + return nil, fmt.Errorf("failed to read WAL bounds: %w", err) + } + if ok { + w.currentBlock = last + w.currentBlockEnded = true + w.hasCurrentBlock = true + } + return w, nil +} + +// Write accumulates a set of changes for the given block number in memory. +func (w *stateWALImpl) Write(blockNumber uint64, cs []*proto.NamedChangeSet) error { + if w.closed.Load() { + return fmt.Errorf("state WAL is closed") + } + if err := w.enforceWriteOrdering(blockNumber); err != nil { + return fmt.Errorf("write rejected: %w", err) + } + w.buf = append(w.buf, cs...) + return nil +} + +// SignalEndOfBlock appends the current block's accumulated changesets to the WAL as a single record. +func (w *stateWALImpl) SignalEndOfBlock() error { + if w.closed.Load() { + return fmt.Errorf("state WAL is closed") + } + + if !w.hasCurrentBlock || w.currentBlockEnded { + return fmt.Errorf("no block in progress to end") + } + blockNumber := w.currentBlock + w.currentBlockEnded = true + changeset := w.buf + w.buf = nil // hand ownership to the WAL; the next block starts a fresh buffer + + if err := w.wal.Append(blockNumber, changeset); err != nil { + return fmt.Errorf("failed to append block %d: %w", blockNumber, err) + } + return nil +} + +// enforceWriteOrdering rejects a Write that violates the block-ordering rules (no decreasing block numbers; no +// advancing to a new block before the current one is ended) and records the new position when it is allowed. +func (w *stateWALImpl) enforceWriteOrdering(blockNumber uint64) error { + if !w.hasCurrentBlock { + w.currentBlock = blockNumber + w.currentBlockEnded = false + w.hasCurrentBlock = true + return nil + } + if blockNumber < w.currentBlock { + return fmt.Errorf("block number %d is less than the current block number %d", blockNumber, w.currentBlock) + } + if blockNumber == w.currentBlock { + if w.currentBlockEnded { + return fmt.Errorf("block number %d has already ended; cannot write more changes to it", blockNumber) + } + return nil + } + // blockNumber > currentBlock + if !w.currentBlockEnded { + return fmt.Errorf( + "cannot write block %d before calling SignalEndOfBlock for block %d", blockNumber, w.currentBlock) + } + w.currentBlock = blockNumber + w.currentBlockEnded = false + return nil +} + +// Flush blocks until all previously scheduled writes are durable. +func (w *stateWALImpl) Flush() error { + return w.wal.Flush() +} + +// GetStoredRange reports the range of complete blocks stored in the WAL. +func (w *stateWALImpl) GetStoredRange() (bool, uint64, uint64, error) { + return w.wal.Bounds() +} + +// Prune schedules removal of whole underlying files below lowestBlockNumberToKeep. It does not block on +// completion. +func (w *stateWALImpl) Prune(lowestBlockNumberToKeep uint64) error { + return w.wal.Prune(lowestBlockNumberToKeep) +} + +// Iterator returns an iterator over the WAL starting at startingBlockNumber. It yields (blockNumber, +// changesets) directly from the underlying generic WAL. +func (w *stateWALImpl) Iterator(startingBlockNumber uint64) (seiwal.Iterator[[]*proto.NamedChangeSet], error) { + return w.wal.Iterator(startingBlockNumber) +} + +// Close flushes pending writes, closes the underlying WAL, and releases resources. +func (w *stateWALImpl) Close() error { + w.closed.Store(true) + return w.wal.Close() +} diff --git a/sei-db/state_db/statewal/state_wal_impl_test.go b/sei-db/state_db/statewal/state_wal_impl_test.go new file mode 100644 index 0000000000..6a20f6da38 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_impl_test.go @@ -0,0 +1,250 @@ +package statewal + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func testConfig(dir string) *Config { + return DefaultConfig(dir, "test") +} + +func openWAL(t *testing.T, cfg *Config) StateWAL { + t.Helper() + w, err := New(cfg) + require.NoError(t, err) + return w +} + +// writeBlock writes a single changeset for the block and signals end of block. +func writeBlock(t *testing.T, w StateWAL, block uint64) { + t.Helper() + cs := []*proto.NamedChangeSet{makeChangeSet("evm", []byte{byte(block)}, []byte{byte(block)})} + require.NoError(t, w.Write(block, cs)) + require.NoError(t, w.SignalEndOfBlock()) +} + +// collectBlocks iterates from start and returns the block number of each entry, verifying that entries are +// strictly increasing and never below start. +func collectBlocks(t *testing.T, w StateWAL, start uint64) []uint64 { + t.Helper() + it, err := w.Iterator(start) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + var blocks []uint64 + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + blockNumber, _ := it.Entry() + require.GreaterOrEqual(t, blockNumber, start) + if len(blocks) > 0 { + require.Greater(t, blockNumber, blocks[len(blocks)-1]) + } + blocks = append(blocks, blockNumber) + } + return blocks +} + +func TestWriteFlushReopenGetRange(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + ok, start, end, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(5), end) + require.NoError(t, w.Close()) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + ok, start, end, err = w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(5), end) + + require.Equal(t, []uint64{1, 2, 3, 4, 5}, collectBlocks(t, w2, 1)) +} + +func TestContractViolations(t *testing.T) { + t.Run("block numbers may not decrease", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + writeBlock(t, w, 5) + require.Error(t, w.Write(4, nil)) + }) + + t.Run("cannot advance block without ending the previous one", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Write(1, nil)) + require.Error(t, w.Write(2, nil)) + }) + + t.Run("cannot write to an ended block", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Write(1, nil)) + require.NoError(t, w.SignalEndOfBlock()) + require.Error(t, w.Write(1, nil)) + }) + + t.Run("end of block with no block in progress is an error", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.Error(t, w.SignalEndOfBlock()) + }) + + t.Run("multiple writes to the same block are allowed before end of block", func(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + require.NoError(t, w.Write(1, []*proto.NamedChangeSet{makeChangeSet("a", []byte("k1"), []byte("v1"))})) + require.NoError(t, w.Write(1, []*proto.NamedChangeSet{makeChangeSet("b", []byte("k2"), []byte("v2"))})) + require.NoError(t, w.SignalEndOfBlock()) + }) +} + +func TestIncompleteBlockDiscardedOnReopen(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + + w := openWAL(t, cfg) + for block := uint64(1); block <= 3; block++ { + writeBlock(t, w, block) + } + // Block 4 is written but never ended (a crash mid-block): it was never appended as a record. + require.NoError(t, w.Write(4, []*proto.NamedChangeSet{makeChangeSet("evm", []byte{0x04}, []byte{0x04})})) + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + ok, start, end, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(3), end) + require.Equal(t, []uint64{1, 2, 3}, collectBlocks(t, w2, 1)) + + // Block 4 may now be re-executed cleanly. + writeBlock(t, w2, 4) + require.NoError(t, w2.Flush()) + ok, _, end, err = w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(4), end) +} + +func TestGetStoredRangeEmpty(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + ok, _, _, err := w.GetStoredRange() + require.NoError(t, err) + require.False(t, ok) +} + +func TestEmptyChangesetBlockIsStored(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + // A block with an empty changeset that is properly ended is a real, stored block. + require.NoError(t, w.Write(1, []*proto.NamedChangeSet{})) + require.NoError(t, w.SignalEndOfBlock()) + require.NoError(t, w.Flush()) + + ok, start, end, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(1), end) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + ok, err = it.Next() + require.NoError(t, err) + require.True(t, ok) + blockNumber, changeset := it.Entry() + require.Equal(t, uint64(1), blockNumber) + require.Empty(t, changeset) +} + +func TestPruneDropsOldBlocks(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = 1 // one block per file, so pruning can drop whole files + + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, w.Prune(5)) + + ok, start, end, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), start) + require.Equal(t, uint64(10), end) + require.Equal(t, []uint64{5, 6, 7, 8, 9, 10}, collectBlocks(t, w, 0)) +} + +// TestRollbackConstructor is a wrapper-level smoke test that NewWithRollback drops blocks beyond the rollback +// point end to end (the crash-safety details are exercised in the seiwal package). +func TestRollbackConstructor(t *testing.T) { + for _, tc := range []struct { + name string + targetSize uint + }{ + {"whole-file removal", 1}, // one block per file: rollback removes whole trailing files + {"in-file truncation", 64 * 1024 * 1024}, // all blocks in one file: rollback truncates it in place + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + cfg := testConfig(dir) + cfg.TargetFileSize = tc.targetSize + + w := openWAL(t, cfg) + for block := uint64(1); block <= 6; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Close()) + + w2, err := NewWithRollback(cfg, 3) + require.NoError(t, err) + defer func() { require.NoError(t, w2.Close()) }() + + ok, start, end, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(3), end) + require.Equal(t, []uint64{1, 2, 3}, collectBlocks(t, w2, 1)) + + // Writing continues cleanly after the rollback point. + writeBlock(t, w2, 4) + require.NoError(t, w2.Flush()) + _, _, end, err = w2.GetStoredRange() + require.NoError(t, err) + require.Equal(t, uint64(4), end) + }) + } +} diff --git a/sei-db/state_db/statewal/state_wal_iterator_test.go b/sei-db/state_db/statewal/state_wal_iterator_test.go new file mode 100644 index 0000000000..47bd423d1f --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_iterator_test.go @@ -0,0 +1,140 @@ +package statewal + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func TestIteratorEmpty(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + it, err := w.Iterator(0) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +func TestIteratorFromMiddle(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.Equal(t, []uint64{3, 4, 5}, collectBlocks(t, w, 3)) +} + +func TestIteratorYieldsChangesetContents(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + cs := []*proto.NamedChangeSet{makeChangeSet("evm", []byte("key"), []byte("value"))} + require.NoError(t, w.Write(1, cs)) + require.NoError(t, w.SignalEndOfBlock()) + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + blockNumber, changeset := it.Entry() + require.Equal(t, uint64(1), blockNumber) + require.Len(t, changeset, 1) + require.Equal(t, "evm", changeset[0].Name) + require.Equal(t, []byte("key"), changeset[0].Changeset.Pairs[0].Key) + require.Equal(t, []byte("value"), changeset[0].Changeset.Pairs[0].Value) + + ok, err = it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +// TestIteratorCombinesMultipleWritesInOrder verifies that all changesets written for one block across several +// Write calls appear, in write order, in that block's single entry. +func TestIteratorCombinesMultipleWritesInOrder(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + require.NoError(t, w.Write(1, []*proto.NamedChangeSet{makeChangeSet("a", []byte("k1"), []byte("v1"))})) + require.NoError(t, w.Write(1, []*proto.NamedChangeSet{ + makeChangeSet("b", []byte("k2"), []byte("v2")), + makeChangeSet("c", []byte("k3"), []byte("v3")), + })) + require.NoError(t, w.SignalEndOfBlock()) + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + ok, err := it.Next() + require.NoError(t, err) + require.True(t, ok) + + blockNumber, changeset := it.Entry() + require.Equal(t, uint64(1), blockNumber) + // Three changesets total (1 from the first Write, 2 from the second), in write order. + require.Len(t, changeset, 3) + require.Equal(t, "a", changeset[0].Name) + require.Equal(t, "b", changeset[1].Name) + require.Equal(t, "c", changeset[2].Name) + + ok, err = it.Next() + require.NoError(t, err) + require.False(t, ok) +} + +func TestIteratorStopsBeforeIncompleteBlock(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + for block := uint64(1); block <= 3; block++ { + writeBlock(t, w, block) + } + // Block 4 written but not ended: it was never appended, so it must not be yielded. + require.NoError(t, w.Write(4, []*proto.NamedChangeSet{makeChangeSet("evm", []byte{4}, []byte{4})})) + require.NoError(t, w.Flush()) + + require.Equal(t, []uint64{1, 2, 3}, collectBlocks(t, w, 1)) +} + +// TestIteratorDoesNotSeePostConstructionBlocks confirms the snapshot contract at the wrapper level: an +// iterator yields only blocks that were complete when it was created. +func TestIteratorDoesNotSeePostConstructionBlocks(t *testing.T) { + w := openWAL(t, testConfig(t.TempDir())) + defer func() { require.NoError(t, w.Close()) }() + + for block := uint64(1); block <= 3; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + it, err := w.Iterator(1) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + // Written after the iterator exists, before draining: must not be observed. + writeBlock(t, w, 4) + require.NoError(t, w.Flush()) + + var got []uint64 + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + blockNumber, _ := it.Entry() + got = append(got, blockNumber) + } + require.Equal(t, []uint64{1, 2, 3}, got, "post-construction block 4 must not be iterated") +} diff --git a/sei-db/state_db/statewal/state_wal_serialization.go b/sei-db/state_db/statewal/state_wal_serialization.go new file mode 100644 index 0000000000..6fd667ed8d --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_serialization.go @@ -0,0 +1,77 @@ +package statewal + +import ( + "encoding/binary" + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// The version byte prefixed to every serialized changeset payload. Bumped if the changeset encoding changes, +// so deserialization can detect and reject an unknown format rather than misparsing it. +const changesetFormatVersion = byte(1) + +// appendChangeset appends the framing [uvarint marshaled length][marshaled NamedChangeSet] for ncs to buf and +// returns the extended buffer. It frames a single changeset; serializeChangesets calls it once per changeset +// to build a block's WAL record payload. +func appendChangeset(buf []byte, ncs *proto.NamedChangeSet) ([]byte, error) { + if ncs == nil { + return nil, fmt.Errorf("changeset is nil") + } + marshaled, err := ncs.Marshal() + if err != nil { + return nil, fmt.Errorf("failed to marshal changeset: %w", err) + } + var scratch [binary.MaxVarintLen64]byte + n := binary.PutUvarint(scratch[:], uint64(len(marshaled))) + buf = append(buf, scratch[:n]...) + buf = append(buf, marshaled...) + return buf, nil +} + +// serializeChangesets encodes a changeset list as a version byte followed by the concatenation +// [version]([uvarint length][marshaled])* — the payload of a single block's WAL record. The block number is +// not encoded: it is the WAL record's index. +func serializeChangesets(cs []*proto.NamedChangeSet) ([]byte, error) { + buf := []byte{changesetFormatVersion} + var err error + for _, ncs := range cs { + buf, err = appendChangeset(buf, ncs) + if err != nil { + return nil, err + } + } + return buf, nil +} + +// deserializeChangesets decodes the payload produced by serializeChangesets, after checking its leading +// version byte. Because the enclosing WAL record is length-delimited and CRC-verified by the underlying WAL, +// any truncation encountered here indicates corruption and is reported as an error rather than tolerated. +func deserializeChangesets(data []byte) ([]*proto.NamedChangeSet, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty changeset payload: missing version byte") + } + if version := data[0]; version != changesetFormatVersion { + return nil, fmt.Errorf("unsupported changeset format version %d", version) + } + + var result []*proto.NamedChangeSet + rest := data[1:] + for len(rest) > 0 { + length, n := binary.Uvarint(rest) + if n <= 0 { + return nil, fmt.Errorf("corrupt changeset length prefix") + } + rest = rest[n:] + if uint64(len(rest)) < length { + return nil, fmt.Errorf("changeset payload truncated: need %d bytes, have %d", length, len(rest)) + } + ncs := &proto.NamedChangeSet{} + if err := ncs.Unmarshal(rest[:length]); err != nil { + return nil, fmt.Errorf("failed to unmarshal changeset: %w", err) + } + rest = rest[length:] + result = append(result, ncs) + } + return result, nil +} diff --git a/sei-db/state_db/statewal/state_wal_serialization_test.go b/sei-db/state_db/statewal/state_wal_serialization_test.go new file mode 100644 index 0000000000..30c3c995df --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_serialization_test.go @@ -0,0 +1,79 @@ +package statewal + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func makeChangeSet(name string, key []byte, value []byte) *proto.NamedChangeSet { + return &proto.NamedChangeSet{ + Name: name, + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: key, Value: value}}, + }, + } +} + +func TestChangesetsRoundTrip(t *testing.T) { + t.Run("multiple named change sets", func(t *testing.T) { + cs := []*proto.NamedChangeSet{ + makeChangeSet("bank", []byte("a"), []byte("1")), + makeChangeSet("evm", []byte("b"), []byte("2")), + } + + data, err := serializeChangesets(cs) + require.NoError(t, err) + require.Equal(t, changesetFormatVersion, data[0], "payload must begin with the format version") + + got, err := deserializeChangesets(data) + require.NoError(t, err) + require.Equal(t, cs, got) + }) + + t.Run("empty changeset list", func(t *testing.T) { + data, err := serializeChangesets([]*proto.NamedChangeSet{}) + require.NoError(t, err) + require.Equal(t, []byte{changesetFormatVersion}, data) // just the version byte + + got, err := deserializeChangesets(data) + require.NoError(t, err) + require.Empty(t, got) + }) +} + +func TestDeserializeUnknownVersion(t *testing.T) { + // A payload whose leading version byte is not recognized must be rejected before any decoding. + _, err := deserializeChangesets([]byte{changesetFormatVersion + 1}) + require.Error(t, err) + + // An empty payload is missing the version byte entirely. + _, err = deserializeChangesets(nil) + require.Error(t, err) +} + +func TestDeserializeChangesetsTruncated(t *testing.T) { + cs := []*proto.NamedChangeSet{ + makeChangeSet("bank", []byte("hello"), []byte("world")), + } + data, err := serializeChangesets(cs) + require.NoError(t, err) + + // Every strict prefix that reaches past the version byte is truncated. Because the enclosing record is + // length-delimited by the underlying WAL, a truncated payload here is corruption and must surface an + // error, never a silent partial decode. (Length 1 is the bare version byte, a valid empty payload.) + for length := 2; length < len(data); length++ { + _, err := deserializeChangesets(data[:length]) + require.Error(t, err) + } +} + +func TestDeserializeCorruptChangeset(t *testing.T) { + // A length prefix pointing at bytes that are not a valid NamedChangeSet protobuf must surface an error. + // Layout: [version][len=2][0x08 0xFF] where 0x08 is a varint field tag (field 1, wire type 0) followed by + // a truncated varint, which the protobuf decoder rejects. + payload := []byte{changesetFormatVersion, 0x02, 0x08, 0xFF} + _, err := deserializeChangesets(payload) + require.Error(t, err) +}