From cc9c8a2e93a211c0dc56a3000376790fdb61a9dd Mon Sep 17 00:00:00 2001 From: Daniel Liu <139250065@qq.com> Date: Fri, 7 Aug 2026 18:46:32 +0800 Subject: [PATCH] fix(consensus/XDPoS,core,eth): repair missing V2 gap snapshots at startup A node can lose its persisted V2 gap snapshot when the process exits between writeHeadBlock and UpdateM1 in writeBlockWithState: the head markers are already on disk, the snapshot is not, and nothing recreates it. Every consumer of that snapshot then fails for a whole epoch. A node that keeps up with the chain head still imports blocks, because the fetcher path verifies headers with fullVerify disabled and trusts header.Validators, but it can no longer mine or vote. A node that falls behind takes the downloader path, where fullVerify is enabled, and stalls on the epoch switch block while reporting it as a bad block. Repair the hole once at startup, before the chain is used: for each gap block whose snapshot is missing, rebuild it from the state committed at that block and store it. Only the last two gap blocks at or below the head are considered. A gap block G is among them exactly while the head is in [G, G+2*Epoch), and G's snapshot is consulted exactly while the head is in [G+Gap, G+Gap+Epoch), so every hole that can still affect the running chain is covered. Checking a single gap block is not enough: a node killed at G restarts with the head on G itself, while a node that stalled on the following epoch switch restarts several hundred blocks later, and no single derivation yields both. The derivation is shared with core.BlockChain.UpdateM1 and the downloader through the unstable xdc_sort ordering, which must not drift, since a different equal-stake order yields a different masternode set. It also refuses to produce an empty snapshot, which would load back fine and permanently mask the missing masternode list. The rebuild is skipped when a snapshot is already stored, so a decode or I/O error never overwrites a masternode set persisted through a reorg, and at or below the V2 switch block, where the snapshot still comes from Initial(). It needs the gap block's state root to still be readable, which the targeted case implies: a node only keeps that root when it was committed, and this trie database never prunes what it has written. Otherwise loadLastState finds no head state, repair() rewinds to an ancestor that has one, and re-importing the gap block runs UpdateM1 and writes the snapshot again. Roots that are gone for good, such as after offline pruning or below a fast sync pivot, cannot be recovered, so every failure is only logged and startup continues. --- consensus/XDPoS/engines/engine_v2/engine.go | 9 +- consensus/XDPoS/engines/engine_v2/snapshot.go | 111 +++++++ .../XDPoS/engines/engine_v2/snapshot_test.go | 306 ++++++++++++++++++ core/blockchain_reader.go | 3 + core/rawdb/accessors_xdc.go | 5 + 5 files changed, 433 insertions(+), 1 deletion(-) diff --git a/consensus/XDPoS/engines/engine_v2/engine.go b/consensus/XDPoS/engines/engine_v2/engine.go index c64f493a801d..f9833ecc1cfe 100644 --- a/consensus/XDPoS/engines/engine_v2/engine.go +++ b/consensus/XDPoS/engines/engine_v2/engine.go @@ -198,7 +198,14 @@ func (x *XDPoS_v2) Initial(chain consensus.ChainReader, header *types.Header) er x.lock.Lock() defer x.lock.Unlock() - return x.initial(chain, header) + if err := x.initial(chain, header); err != nil { + return err + } + // Startup-only repair, skipped for chain readers that cannot open state. + if gapChain, ok := chain.(GapStateReader); ok { + x.RepairGapSnapshots(gapChain) + } + return nil } func (x *XDPoS_v2) initial(chain consensus.ChainReader, header *types.Header) error { diff --git a/consensus/XDPoS/engines/engine_v2/snapshot.go b/consensus/XDPoS/engines/engine_v2/snapshot.go index 9e27523d5da4..203875db6f8b 100644 --- a/consensus/XDPoS/engines/engine_v2/snapshot.go +++ b/consensus/XDPoS/engines/engine_v2/snapshot.go @@ -2,11 +2,15 @@ package engine_v2 import ( "encoding/json" + "errors" "fmt" "github.com/XinFinOrg/XDPoSChain/common" + xdc_sort "github.com/XinFinOrg/XDPoSChain/common/sort" "github.com/XinFinOrg/XDPoSChain/consensus" + "github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/utils" "github.com/XinFinOrg/XDPoSChain/core/rawdb" + "github.com/XinFinOrg/XDPoSChain/core/state" "github.com/XinFinOrg/XDPoSChain/ethdb" "github.com/XinFinOrg/XDPoSChain/log" ) @@ -111,3 +115,110 @@ func (x *XDPoS_v2) getSnapshot(chain consensus.ChainReader, number uint64, isGap x.snapshots.Add(snap.Hash, snap) return snap, nil } + +// GapStateReader is a chain reader that can also open historical state, which +// the startup repair needs to rebuild a snapshot from its gap block. +type GapStateReader interface { + consensus.ChainReader + StateAt(root common.Hash) (*state.StateDB, error) +} + +var errNoCandidates = errors.New("no masternode candidates in state") + +// BuildSnapshotFromState derives a gap block snapshot from the state committed +// at that block. The ordering must stay identical to core.BlockChain.UpdateM1 +// and Downloader.generateSnapshot: a different equal-stake order yields a +// different masternode set. +func BuildSnapshotFromState(statedb *state.StateDB, number uint64, hash common.Hash) (*SnapshotV2, error) { + var ms []utils.Masternode + for _, candidate := range statedb.GetCandidates() { + if candidate.IsZero() { + continue + } + ms = append(ms, utils.Masternode{Address: candidate, Stake: statedb.GetCandidateCap(candidate)}) + } + if len(ms) == 0 { + // An empty snapshot loads back fine and would permanently mask the hole. + return nil, errNoCandidates + } + xdc_sort.Slice(ms, func(i, j int) bool { + return ms[i].Stake.Cmp(ms[j].Stake) >= 0 + }) + + candidates := make([]common.Address, len(ms)) + for i, m := range ms { + candidates[i] = m.Address + } + return NewSnapshot(number, hash, candidates), nil +} + +// repairGapCandidates returns the gap block numbers at or below head whose +// snapshot can still matter to the running chain. getSnapshot maps head to the +// gap block between Gap and Gap+Epoch blocks back, which is always one of these. +func (x *XDPoS_v2) repairGapCandidates(head uint64) []uint64 { + epoch, gap := x.config.Epoch, x.config.Gap + if epoch == 0 || gap >= epoch { + return nil + } + offset := epoch - gap + if head < offset { + return nil + } + latest := head - (head-offset)%epoch + if latest < epoch { + return []uint64{latest} + } + return []uint64{latest - epoch, latest} +} + +// RepairGapSnapshots restores gap block snapshots missing from the database, +// which happens when the process exits between writeHeadBlock and UpdateM1. +// Meant to run once at startup. Failures are only logged: a node that is still +// syncing legitimately has no state to rebuild from. +func (x *XDPoS_v2) RepairGapSnapshots(chain GapStateReader) { + head := chain.CurrentHeader() + if head == nil { + return + } + for _, gapNum := range x.repairGapCandidates(head.Number.Uint64()) { + // The snapshot at V2 SwitchBlock-Gap is owned by initial(), and gap + // blocks below the switch belong to the v1 engine. + if gapNum <= x.config.V2.SwitchBlock.Uint64() { + continue + } + gapHeader := chain.GetHeaderByNumber(gapNum) + if gapHeader == nil { + // gapNum is at or below the current head, so the canonical header must exist. + log.Warn("[RepairGapSnapshots] missing canonical gap header", "number", gapNum, "head", head.Number) + continue + } + gapHash := gapHeader.Hash() + // Only a genuinely absent key is repaired. If we cannot reliably determine + // whether a snapshot is present (e.g. I/O error), skip repair to avoid + // overwriting a snapshot that may have been persisted through a reorg. + has, err := rawdb.HasXdposV2Snapshot(x.db, gapHash) + if err != nil { + log.Debug("[RepairGapSnapshots] cannot probe stored snapshot", "number", gapNum, "hash", gapHash, "err", err) + continue + } + if has { + continue + } + statedb, err := chain.StateAt(gapHeader.Root) + if err != nil { + log.Warn("[RepairGapSnapshots] gap block state unavailable", "number", gapNum, "hash", gapHash, "root", gapHeader.Root, "err", err) + continue + } + snap, err := BuildSnapshotFromState(statedb, gapNum, gapHash) + if err != nil { + log.Error("[RepairGapSnapshots] cannot derive snapshot", "number", gapNum, "hash", gapHash, "err", err) + continue + } + if err := StoreSnapshot(snap, x.db); err != nil { + log.Error("[RepairGapSnapshots] cannot store snapshot", "number", gapNum, "hash", gapHash, "err", err) + continue + } + x.snapshots.Add(snap.Hash, snap) + log.Warn("Repaired missing V2 gap snapshot", "number", gapNum, "hash", gapHash, "candidates", len(snap.NextEpochCandidates)) + } +} diff --git a/consensus/XDPoS/engines/engine_v2/snapshot_test.go b/consensus/XDPoS/engines/engine_v2/snapshot_test.go index 66289eb9f660..87086b4c02bd 100644 --- a/consensus/XDPoS/engines/engine_v2/snapshot_test.go +++ b/consensus/XDPoS/engines/engine_v2/snapshot_test.go @@ -1,12 +1,20 @@ package engine_v2 import ( + "errors" "fmt" + "math/big" "testing" "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/common/lru" + "github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/utils" "github.com/XinFinOrg/XDPoSChain/core/rawdb" + "github.com/XinFinOrg/XDPoSChain/core/state" + "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/ethdb" "github.com/XinFinOrg/XDPoSChain/ethdb/leveldb" + "github.com/XinFinOrg/XDPoSChain/params" ) func TestGetMasterNodes(t *testing.T) { @@ -40,3 +48,301 @@ func TestStoreLoadSnapshot(t *testing.T) { t.Error("load snapshot failed", err) } } + +const ( + testRepairEpoch = uint64(900) + testRepairGap = uint64(450) + testRepairSwitchBlock = uint64(450) +) + +// newCandidateState returns a state holding the masternode voting contract +// storage that BuildSnapshotFromState reads, with candidates in the given order. +func newCandidateState(t *testing.T, candidates []common.Address, caps []*big.Int) *state.StateDB { + t.Helper() + statedb, err := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase())) + if err != nil { + t.Fatalf("create state: %v", err) + } + slotHash := common.BigToHash(new(big.Int).SetUint64(8)) // slotValidatorMapping["candidates"] + statedb.SetState(common.MasternodeVotingSMCBinary, slotHash, common.BigToHash(new(big.Int).SetUint64(uint64(len(candidates))))) + for i, candidate := range candidates { + statedb.SetState(common.MasternodeVotingSMCBinary, state.GetLocDynamicArrAtElement(slotHash, uint64(i), 1), candidate.Hash()) + if candidate.IsZero() { + continue + } + locCap := new(big.Int).Add(state.GetLocMappingAtKey(candidate.Hash(), 1), big.NewInt(1)) + statedb.SetState(common.MasternodeVotingSMCBinary, common.BigToHash(locCap), common.BigToHash(caps[i])) + } + return statedb +} + +// repairTestChain is a minimal GapStateReader over a handful of headers. +type repairTestChain struct { + headers map[uint64]*types.Header + head *types.Header + stateErr error + stateCalls int + statedb *state.StateDB +} + +func (c *repairTestChain) addHeader(header *types.Header) { + if c.headers == nil { + c.headers = make(map[uint64]*types.Header) + } + c.headers[header.Number.Uint64()] = header + if c.head == nil || header.Number.Uint64() > c.head.Number.Uint64() { + c.head = header + } +} + +func (c *repairTestChain) Config() *params.ChainConfig { return nil } +func (c *repairTestChain) CurrentHeader() *types.Header { return c.head } +func (c *repairTestChain) GetHeaderByNumber(number uint64) *types.Header { + return c.headers[number] +} +func (c *repairTestChain) GetHeader(common.Hash, uint64) *types.Header { return nil } +func (c *repairTestChain) GetHeaderByHash(common.Hash) *types.Header { return nil } +func (c *repairTestChain) GetBlock(common.Hash, uint64) *types.Block { return nil } + +func (c *repairTestChain) StateAt(common.Hash) (*state.StateDB, error) { + c.stateCalls++ + if c.stateErr != nil { + return nil, c.stateErr + } + return c.statedb, nil +} + +func newRepairEngine(db ethdb.Database) *XDPoS_v2 { + return &XDPoS_v2{ + config: ¶ms.XDPoSConfig{ + Epoch: testRepairEpoch, + Gap: testRepairGap, + V2: ¶ms.V2{ + SwitchBlock: new(big.Int).SetUint64(testRepairSwitchBlock), + }, + }, + db: db, + snapshots: lru.NewCache[common.Hash, *SnapshotV2](utils.InMemorySnapshots), + } +} + +// newRepairFixture wires an engine and a chain whose head is at headNumber, with +// gap block headers present for every gap number at or below it. +func newRepairFixture(t *testing.T, headNumber uint64, candidates []common.Address, caps []*big.Int) (*XDPoS_v2, *repairTestChain, ethdb.Database) { + t.Helper() + db := rawdb.NewMemoryDatabase() + x := newRepairEngine(db) + chain := &repairTestChain{statedb: newCandidateState(t, candidates, caps)} + for num := testRepairEpoch - testRepairGap; num <= headNumber; num += testRepairEpoch { + chain.addHeader(&types.Header{Number: new(big.Int).SetUint64(num), Extra: []byte{byte(num), byte(num >> 8)}}) + } + chain.addHeader(&types.Header{Number: new(big.Int).SetUint64(headNumber), Extra: []byte("head")}) + return x, chain, db +} + +func gapHash(t *testing.T, chain *repairTestChain, number uint64) common.Hash { + t.Helper() + header := chain.GetHeaderByNumber(number) + if header == nil { + t.Fatalf("no header at %d", number) + } + return header.Hash() +} + +func assertSnapshotStored(t *testing.T, db ethdb.Database, hash common.Hash, want []common.Address) { + t.Helper() + snap, err := loadSnapshot(db, hash) + if err != nil { + t.Fatalf("load snapshot: %v", err) + } + if len(snap.NextEpochCandidates) != len(want) { + t.Fatalf("candidates = %v, want %v", snap.NextEpochCandidates, want) + } + for i, addr := range want { + if snap.NextEpochCandidates[i] != addr { + t.Fatalf("candidates = %v, want %v", snap.NextEpochCandidates, want) + } + } +} + +func TestBuildSnapshotFromStateSortsByStakeDescending(t *testing.T) { + low, mid, high := common.Address{0x1}, common.Address{0x2}, common.Address{0x3} + statedb := newCandidateState(t, + []common.Address{low, mid, high}, + []*big.Int{big.NewInt(10), big.NewInt(20), big.NewInt(30)}, + ) + + snap, err := BuildSnapshotFromState(statedb, 1350, common.Hash{0xaa}) + if err != nil { + t.Fatalf("build snapshot: %v", err) + } + want := []common.Address{high, mid, low} + for i, addr := range want { + if snap.NextEpochCandidates[i] != addr { + t.Fatalf("candidates = %v, want %v", snap.NextEpochCandidates, want) + } + } + if snap.Number != 1350 || snap.Hash != (common.Hash{0xaa}) { + t.Fatalf("snapshot = (%d, %s), want (1350, 0xaa..)", snap.Number, snap.Hash.Hex()) + } +} + +// Equal stakes must keep the exact order xdc_sort produces, otherwise nodes +// derive different masternode sets from the same state. +func TestBuildSnapshotFromStateEqualStakeOrder(t *testing.T) { + a, b, c := common.Address{0x1}, common.Address{0x2}, common.Address{0x3} + statedb := newCandidateState(t, + []common.Address{a, b, c}, + []*big.Int{big.NewInt(10), big.NewInt(10), big.NewInt(10)}, + ) + + snap, err := BuildSnapshotFromState(statedb, 1350, common.Hash{0xaa}) + if err != nil { + t.Fatalf("build snapshot: %v", err) + } + want := []common.Address{c, b, a} + for i, addr := range want { + if snap.NextEpochCandidates[i] != addr { + t.Fatalf("candidates = %v, want %v", snap.NextEpochCandidates, want) + } + } +} + +func TestBuildSnapshotFromStateSkipsZeroCandidates(t *testing.T) { + real := common.Address{0x1} + statedb := newCandidateState(t, + []common.Address{{}, real}, + []*big.Int{big.NewInt(0), big.NewInt(10)}, + ) + + snap, err := BuildSnapshotFromState(statedb, 1350, common.Hash{0xaa}) + if err != nil { + t.Fatalf("build snapshot: %v", err) + } + if len(snap.NextEpochCandidates) != 1 || snap.NextEpochCandidates[0] != real { + t.Fatalf("candidates = %v, want [%s]", snap.NextEpochCandidates, real.Hex()) + } +} + +func TestBuildSnapshotFromStateRejectsEmptyCandidates(t *testing.T) { + statedb := newCandidateState(t, nil, nil) + + if _, err := BuildSnapshotFromState(statedb, 1350, common.Hash{0xaa}); !errors.Is(err, errNoCandidates) { + t.Fatalf("err = %v, want errNoCandidates", err) + } +} + +func TestRepairGapCandidates(t *testing.T) { + x := newRepairEngine(rawdb.NewMemoryDatabase()) + // Gap block G=1350 backs heads in [1800, 2700); it must stay a candidate + // across that whole span plus the margin on either side. + tests := []struct { + head uint64 + want []uint64 + }{ + {head: 0, want: nil}, + {head: 449, want: nil}, + {head: 450, want: []uint64{450}}, + {head: 1350, want: []uint64{450, 1350}}, + {head: 1799, want: []uint64{450, 1350}}, + {head: 1800, want: []uint64{450, 1350}}, + {head: 2250, want: []uint64{1350, 2250}}, + {head: 2699, want: []uint64{1350, 2250}}, + {head: 3150, want: []uint64{2250, 3150}}, + } + for _, tt := range tests { + got := x.repairGapCandidates(tt.head) + if len(got) != len(tt.want) { + t.Fatalf("head %d: got %v, want %v", tt.head, got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("head %d: got %v, want %v", tt.head, got, tt.want) + } + } + } +} + +func TestRepairGapSnapshotsHeadIsGapBlock(t *testing.T) { + candidate := common.Address{0x1} + x, chain, db := newRepairFixture(t, 1350, []common.Address{candidate}, []*big.Int{big.NewInt(10)}) + + x.RepairGapSnapshots(chain) + + assertSnapshotStored(t, db, gapHash(t, chain, 1350), []common.Address{candidate}) +} + +// A node that kept importing past the gap block and only restarted later must +// still have its missing snapshot repaired. +func TestRepairGapSnapshotsHeadPastGapBlock(t *testing.T) { + candidate := common.Address{0x1} + x, chain, db := newRepairFixture(t, 1799, []common.Address{candidate}, []*big.Int{big.NewInt(10)}) + + x.RepairGapSnapshots(chain) + + assertSnapshotStored(t, db, gapHash(t, chain, 1350), []common.Address{candidate}) +} + +func TestRepairGapSnapshotsChecksBothCandidates(t *testing.T) { + candidate := common.Address{0x1} + x, chain, db := newRepairFixture(t, 2400, []common.Address{candidate}, []*big.Int{big.NewInt(10)}) + + x.RepairGapSnapshots(chain) + + for _, gapNum := range []uint64{1350, 2250} { + assertSnapshotStored(t, db, gapHash(t, chain, gapNum), []common.Address{candidate}) + } +} + +// A stored snapshot may come from the reorg path and disagree with the state +// derived one, so it must never be overwritten. +func TestRepairGapSnapshotsKeepsStoredSnapshot(t *testing.T) { + stored := common.Address{0x9} + x, chain, db := newRepairFixture(t, 1350, []common.Address{{0x1}}, []*big.Int{big.NewInt(10)}) + if err := StoreSnapshot(NewSnapshot(1350, gapHash(t, chain, 1350), []common.Address{stored}), db); err != nil { + t.Fatalf("store snapshot: %v", err) + } + + x.RepairGapSnapshots(chain) + + assertSnapshotStored(t, db, gapHash(t, chain, 1350), []common.Address{stored}) + if chain.stateCalls != 0 { + t.Fatalf("stateCalls = %d, want 0", chain.stateCalls) + } +} + +func TestRepairGapSnapshotsSkipsMissingState(t *testing.T) { + x, chain, db := newRepairFixture(t, 1350, []common.Address{{0x1}}, []*big.Int{big.NewInt(10)}) + chain.stateErr = errors.New("missing trie node") + + x.RepairGapSnapshots(chain) + + if _, err := loadSnapshot(db, gapHash(t, chain, 1350)); err == nil { + t.Fatal("snapshot was stored despite unavailable state") + } +} + +func TestRepairGapSnapshotsSkipsSwitchBlockGap(t *testing.T) { + x, chain, db := newRepairFixture(t, 450, []common.Address{{0x1}}, []*big.Int{big.NewInt(10)}) + + x.RepairGapSnapshots(chain) + + if _, err := loadSnapshot(db, gapHash(t, chain, 450)); err == nil { + t.Fatal("snapshot at V2 switch block gap should be left to initial()") + } + if chain.stateCalls != 0 { + t.Fatalf("stateCalls = %d, want 0", chain.stateCalls) + } +} + +func TestRepairGapSnapshotsBeforeFirstGap(t *testing.T) { + x := newRepairEngine(rawdb.NewMemoryDatabase()) + chain := &repairTestChain{} + chain.addHeader(&types.Header{Number: big.NewInt(100)}) + + x.RepairGapSnapshots(chain) + + if chain.stateCalls != 0 { + t.Fatalf("stateCalls = %d, want 0", chain.stateCalls) + } +} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 2484bda6905e..789aea15c579 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -21,6 +21,7 @@ import ( "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/consensus" + "github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/engines/engine_v2" "github.com/XinFinOrg/XDPoSChain/core/rawdb" "github.com/XinFinOrg/XDPoSChain/core/state" "github.com/XinFinOrg/XDPoSChain/core/types" @@ -296,6 +297,8 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { return statedb, nil } +var _ engine_v2.GapStateReader = (*BlockChain)(nil) + // Config retrieves the chain's fork configuration. func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig } diff --git a/core/rawdb/accessors_xdc.go b/core/rawdb/accessors_xdc.go index cc628439b375..b623deaf04f6 100644 --- a/core/rawdb/accessors_xdc.go +++ b/core/rawdb/accessors_xdc.go @@ -50,6 +50,11 @@ func ReadXdposV2Snapshot(db ethdb.KeyValueReader, hash common.Hash) ([]byte, err return data, nil } +// HasXdposV2Snapshot reports whether a snapshot is stored for the given hash. +func HasXdposV2Snapshot(db ethdb.KeyValueReader, hash common.Hash) (bool, error) { + return db.Has(xdposV2Key(hash)) +} + // WriteXdposV2Snapshot writes the SnapshotV2 into the database. func WriteXdposV2Snapshot(db ethdb.KeyValueWriter, hash common.Hash, blob []byte) error { if err := db.Put(xdposV2Key(hash), blob); err != nil {