fix(core): store gap snapshot before writeHeadBlock to close crash window

UpdateMasternodes (called via UpdateM1) was invoked after writeHeadBlock,
leaving a window where the node head pointed to a gap block but no
corresponding snapshot existed in the database. If the process was killed
or restarted during the lengthy per-candidate EVM calls in UpdateM1, the
snapshot would never be written. On the next boot the node would load the
gap block as its head, then fail with:

  Cannot find snapshot from last gap block err="leveldb: not found"

on every block in the following epoch, silently dropping out of consensus.

Fix: add updateM1ForBlock(block, statedb) which reads candidates and stakes
directly from the committed state trie (same as downloader.generateSnapshot)
without depending on bc.CurrentBlock() or bc.CurrentHeader(). Both the
canonical-chain path (writeBlockWithState) and the reorg path (reorg) now
call updateM1ForBlock with bc.StateAt(block.Root()) before writeHeadBlock,
so the snapshot is durable before the head markers are persisted.

The original UpdateM1 is retained unchanged for external callers and tests.

Author: gzliudan

consensus/tests/engine_v1_tests/block_signer_test.go

@@ -201,6 +201,119 @@ func TestCallUpdateM1WithSmartContractTranscation(t *testing.T) {
 	}
 }
 
+// Regression: for canonical gap blocks, the block promoted to head must always
+// have a readable snapshot.
+func TestCanonicalGapBlockHeadSnapshotConsistency(t *testing.T) {
+	blockchain, backend, parentBlock, signer, signFn := PrepareXDCTestBlockChain(t, GAP-1, params.TestXDPoSMockChainConfig)
+	tx, err := voteTX(37117, 0, acc1Addr.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	header := &types.Header{
+		Root:       common.HexToHash("46234e9cd7e85a267f7f0435b15256a794a2f6d65cc98cdbd21dcd10a01d9772"),
+		Number:     big.NewInt(int64(GAP)),
+		ParentHash: parentBlock.Hash(),
+		Coinbase:   common.HexToAddress("0xaaa0000000000000000000000000000000000450"),
+	}
+	gapBlock, err := createBlockFromHeader(blockchain, header, []*types.Transaction{tx}, signer, signFn, blockchain.Config())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = blockchain.InsertBlock(gapBlock)
+	assert.Nil(t, err)
+
+	assert.Equal(t, gapBlock.Hash(), blockchain.CurrentBlock().Hash())
+	headSigners, err := GetSnapshotSigner(blockchain, blockchain.CurrentBlock())
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, common.MaxMasternodes, len(headSigners))
+	if headSigners[acc1Addr.Hex()] != true {
+		debugMessage(backend, headSigners, t)
+		t.Fatalf("head snapshot should include account 1 after canonical gap block")
+	}
+}
+
+// Regression: after reorg to a chain that includes a gap block, the new head
+// and its predecessor gap block must both have the reorged snapshot.
+func TestReorgGapBlockHeadSnapshotConsistency(t *testing.T) {
+	blockchain, backend, parentBlock, signer, signFn := PrepareXDCTestBlockChain(t, GAP-1, params.TestXDPoSMockChainConfig)
+
+	// Build canonical block 450 A (vote acc1).
+	txA, err := voteTX(37117, 0, acc1Addr.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	headerA := &types.Header{
+		Root:       common.HexToHash("46234e9cd7e85a267f7f0435b15256a794a2f6d65cc98cdbd21dcd10a01d9772"),
+		Number:     big.NewInt(int64(GAP)),
+		ParentHash: parentBlock.Hash(),
+		Coinbase:   common.HexToAddress("0xaaa0000000000000000000000000000000000450"),
+	}
+	block450A, err := createBlockFromHeader(blockchain, headerA, []*types.Transaction{txA}, signer, signFn, blockchain.Config())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = blockchain.InsertBlock(block450A)
+	assert.Nil(t, err)
+	assert.Equal(t, block450A.Hash(), blockchain.CurrentBlock().Hash())
+
+	// Build forked block 450 B (vote acc2), should stay side chain for now.
+	txB, err := voteTX(37117, 0, acc2Addr.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	headerB := &types.Header{
+		Root:       common.HexToHash("068dfa09d7b4093441c0cc4d9807a71bc586f6101c072d939b214c21cd136eb3"),
+		Number:     big.NewInt(int64(GAP)),
+		ParentHash: parentBlock.Hash(),
+		Coinbase:   common.HexToAddress("0xbbb0000000000000000000000000000000000450"),
+	}
+	block450B, err := createBlockFromHeader(blockchain, headerB, []*types.Transaction{txB}, signer, signFn, blockchain.Config())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = blockchain.InsertBlock(block450B)
+	assert.Nil(t, err)
+	assert.Equal(t, block450A.Hash(), blockchain.CurrentBlock().Hash())
+
+	// Extend fork chain with 451 B to trigger reorg.
+	header451B := &types.Header{
+		Root:       common.HexToHash("068dfa09d7b4093441c0cc4d9807a71bc586f6101c072d939b214c21cd136eb3"),
+		Number:     big.NewInt(int64(GAP + 1)),
+		ParentHash: block450B.Hash(),
+		Coinbase:   common.HexToAddress("0xbbb0000000000000000000000000000000000451"),
+	}
+	block451B, err := createBlockFromHeader(blockchain, header451B, nil, signer, signFn, blockchain.Config())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = blockchain.InsertBlock(block451B)
+	assert.Nil(t, err)
+	assert.Equal(t, block451B.Hash(), blockchain.CurrentBlock().Hash())
+
+	gapSigners, err := GetSnapshotSigner(blockchain, block450B.Header())
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, common.MaxMasternodes, len(gapSigners))
+	if gapSigners[acc2Addr.Hex()] != true {
+		debugMessage(backend, gapSigners, t)
+		t.Fatalf("reorged gap snapshot should include account 2")
+	}
+
+	headSigners, err := GetSnapshotSigner(blockchain, blockchain.CurrentBlock())
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, common.MaxMasternodes, len(headSigners))
+	if headSigners[acc2Addr.Hex()] != true {
+		debugMessage(backend, headSigners, t)
+		t.Fatalf("reorged head snapshot should include account 2")
+	}
+}
+
 // Should call updateM1 and update snapshot when a forked block(at gap block number) is inserted back into main chain (Edge case)
 func TestCallUpdateM1WhenForkedBlockBackToMainChain(t *testing.T) {
 	blockchain, backend, currentBlock, signer, signFn := PrepareXDCTestBlockChain(t, GAP-1, params.TestXDPoSMockChainConfig)

core/blockchain.go

@@ -1684,14 +1684,23 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 
 	// Set new head.
 	if status == CanonStatTy {
-		// WriteBlock has already been called, no need to write again
-		bc.writeHeadBlock(block, false)
-		// prepare set of masternodes for the next epoch
+		// For gap blocks, store the masternode snapshot before persisting head markers.
+		// This eliminates the crash window between writeHeadBlock and UpdateM1 that
+		// would leave the head pointing to a gap block with no snapshot in the database.
+		// We open a fresh StateDB from the committed root (equivalent to what UpdateM1
+		// does via bc.State() after writeHeadBlock) so we do not depend on the
+		// post-Commit statedb object, whose internal caches may have been cleared.
 		if bc.chainConfig.XDPoS != nil && ((block.NumberU64() % bc.chainConfig.XDPoS.Epoch) == (bc.chainConfig.XDPoS.Epoch - bc.chainConfig.XDPoS.Gap)) {
-			if err := bc.UpdateM1(); err != nil {
-				log.Crit("Fail to update masternodes during writeBlockWithState", "number", block.Number, "hash", block.Hash().Hex(), "err", err)
+			gapState, stateErr := bc.StateAt(block.Root())
+			if stateErr != nil {
+				log.Warn("Fail to open gap block state during writeBlockWithState, fallback to contract reads", "number", block.NumberU64(), "hash", block.Hash().Hex(), "err", stateErr)
+			}
+			if err := bc.updateM1ForBlock(block, gapState); err != nil {
+				return NonStatTy, fmt.Errorf("failed to update masternodes during writeBlockWithState at block %d (%s): %w", block.NumberU64(), block.Hash().Hex(), err)
 			}
 		}
+		// WriteBlock has already been called, no need to write again
+		bc.writeHeadBlock(block, false)
 	}
 	// save cache BlockSigners
 	if bc.chainConfig.XDPoS != nil && bc.chainConfig.IsTIPSigning(block.Number()) {
@@ -2589,14 +2598,18 @@ func (bc *BlockChain) reorg(oldHead, newHead *types.Header) error {
 			bc.logsFeed.Send(rebirthLogs)
 			rebirthLogs = nil
 		}
-		// Update the head block
-		bc.writeHeadBlock(block, true)
-		// prepare set of masternodes for the next epoch
+		// For gap blocks, store the masternode snapshot before persisting head markers.
 		if bc.chainConfig.XDPoS != nil && ((block.NumberU64() % bc.chainConfig.XDPoS.Epoch) == (bc.chainConfig.XDPoS.Epoch - bc.chainConfig.XDPoS.Gap)) {
-			if err := bc.UpdateM1(); err != nil {
-				log.Crit("Fail to update masternodes during reorg", "number", block.Number, "hash", block.Hash().Hex(), "err", err)
+			gapState, stateErr := bc.StateAt(block.Root())
+			if stateErr != nil {
+				log.Warn("Fail to open gap block state during reorg, fallback to contract reads", "number", block.NumberU64(), "hash", block.Hash().Hex(), "err", stateErr)
+			}
+			if err := bc.updateM1ForBlock(block, gapState); err != nil {
+				return fmt.Errorf("failed to update masternodes during reorg at block %d (%s): %w", block.NumberU64(), block.Hash().Hex(), err)
 			}
 		}
+		// Update the head block
+		bc.writeHeadBlock(block, true)
 	}
 	if len(rebirthLogs) > 0 {
 		bc.logsFeed.Send(rebirthLogs)
@@ -2758,6 +2771,73 @@ func (bc *BlockChain) GetClient() (bind.ContractBackend, error) {
 	return bc.Client, nil
 }
 
+// updateM1ForBlock computes the masternode candidate set for a gap block using the
+// provided committed StateDB, then asks the consensus engine to update/store the
+// corresponding snapshot. Unlike UpdateM1, this method reads candidates and stakes
+// directly from the state trie and does not depend on bc.CurrentBlock() or
+// bc.CurrentHeader(), so it can safely be called before writeHeadBlock.
+func (bc *BlockChain) updateM1ForBlock(block *types.Block, statedb *state.StateDB) error {
+	engine, ok := bc.Engine().(*XDPoS.XDPoS)
+	if bc.Config().XDPoS == nil || !ok {
+		return ErrNotXDPoS
+	}
+	log.Info("It's time to update new set of masternodes for the next epoch...", "number", block.Number(), "hash", block.Hash().Hex())
+
+	var (
+		ms         []utils.Masternode
+		candidates []common.Address
+	)
+	if statedb != nil {
+		candidates = statedb.GetCandidates()
+		for _, candidate := range candidates {
+			v := statedb.GetCandidateCap(candidate)
+			if !candidate.IsZero() {
+				ms = append(ms, utils.Masternode{Address: candidate, Stake: v})
+			}
+		}
+	} else {
+		client, err := bc.GetClient()
+		if err != nil {
+			return fmt.Errorf("failed to get client for fallback candidate query: %w", err)
+		}
+		validator, err := contractValidator.NewXDCValidator(common.MasternodeVotingSMCBinary, client)
+		if err != nil {
+			return fmt.Errorf("failed to create validator contract for fallback candidate query: %w", err)
+		}
+		opts := &bind.CallOpts{BlockNumber: block.Number()}
+		candidates, err = validator.GetCandidates(opts)
+		if err != nil {
+			return fmt.Errorf("failed to get fallback candidate list: %w", err)
+		}
+		for _, candidate := range candidates {
+			v, err := validator.GetCandidateCap(opts, candidate)
+			if err != nil {
+				return fmt.Errorf("failed to get fallback candidate cap for %s: %w", candidate.Hex(), err)
+			}
+			if !candidate.IsZero() {
+				ms = append(ms, utils.Masternode{Address: candidate, Stake: v})
+			}
+		}
+	}
+	if len(ms) == 0 {
+		log.Error("No masternode found. Stopping node")
+		return errors.New("no masternode found")
+	}
+	xdc_sort.Slice(ms, func(i, j int) bool {
+		return ms[i].Stake.Cmp(ms[j].Stake) >= 0
+	})
+	log.Info("Ordered list of masternode candidates")
+	for _, m := range ms {
+		log.Info("", "address", m.Address, "stake", m.Stake)
+	}
+	log.Info("Updating new set of masternodes")
+	if err := engine.UpdateMasternodes(bc, block.Header(), ms); err != nil {
+		return err
+	}
+	log.Info("Masternodes are ready for the next epoch")
+	return nil
+}
+
 func (bc *BlockChain) UpdateM1() error {
 	engine, ok := bc.Engine().(*XDPoS.XDPoS)
 	if bc.Config().XDPoS == nil || !ok {