From 105f82749778d2197e487217ad0fd33f5412e377 Mon Sep 17 00:00:00 2001 From: polymorpher Date: Thu, 13 Aug 2026 14:31:34 -0700 Subject: [PATCH 1/9] Add read-only inplace recovery libraries for preflight. Introduce the harmony-recovery inplace stack (rodb, chainread, certverify, statecheck, anchor, report, and fixtures) so a live shard-0 LevelDB can be sampled without writes. Co-authored-by: Cursor --- internal/recovery/inplace/anchor/anchor.go | 143 ++++ .../recovery/inplace/anchor/anchor_test.go | 86 ++ .../inplace/anchor/releasecheck_test.go | 72 ++ .../recovery/inplace/certverify/certverify.go | 106 +++ .../inplace/certverify/certverify_test.go | 197 +++++ internal/recovery/inplace/chainread/checks.go | 390 +++++++++ .../inplace/chainread/checks_unit_test.go | 118 +++ .../inplace/chainread/headsample_test.go | 127 +++ internal/recovery/inplace/chainread/keys.go | 64 ++ .../recovery/inplace/chainread/keys_test.go | 78 ++ internal/recovery/inplace/chainread/reader.go | 223 ++++++ .../recovery/inplace/chainread/readers.go | 115 +++ internal/recovery/inplace/fixture/fixture.go | 755 ++++++++++++++++++ .../recovery/inplace/fixture/fixture_test.go | 75 ++ internal/recovery/inplace/fixture/gen/main.go | 72 ++ internal/recovery/inplace/fixture/mutate.go | 159 ++++ internal/recovery/inplace/report/report.go | 265 ++++++ .../recovery/inplace/report/report_test.go | 92 +++ internal/recovery/inplace/rodb/adapter.go | 240 ++++++ internal/recovery/inplace/rodb/errors.go | 117 +++ internal/recovery/inplace/rodb/layout.go | 68 ++ internal/recovery/inplace/rodb/open.go | 83 ++ internal/recovery/inplace/rodb/probe_other.go | 8 + internal/recovery/inplace/rodb/probe_unix.go | 36 + internal/recovery/inplace/rodb/retry.go | 130 +++ internal/recovery/inplace/rodb/rodb_test.go | 658 +++++++++++++++ internal/recovery/inplace/rodb/storage.go | 206 +++++ .../recovery/inplace/statecheck/anomaly.go | 91 +++ .../recovery/inplace/statecheck/digest.go | 117 +++ .../recovery/inplace/statecheck/walker.go | 535 +++++++++++++ .../inplace/statecheck/walker_test.go | 341 ++++++++ 31 files changed, 5767 insertions(+) create mode 100644 internal/recovery/inplace/anchor/anchor.go create mode 100644 internal/recovery/inplace/anchor/anchor_test.go create mode 100644 internal/recovery/inplace/anchor/releasecheck_test.go create mode 100644 internal/recovery/inplace/certverify/certverify.go create mode 100644 internal/recovery/inplace/certverify/certverify_test.go create mode 100644 internal/recovery/inplace/chainread/checks.go create mode 100644 internal/recovery/inplace/chainread/checks_unit_test.go create mode 100644 internal/recovery/inplace/chainread/headsample_test.go create mode 100644 internal/recovery/inplace/chainread/keys.go create mode 100644 internal/recovery/inplace/chainread/keys_test.go create mode 100644 internal/recovery/inplace/chainread/reader.go create mode 100644 internal/recovery/inplace/chainread/readers.go create mode 100644 internal/recovery/inplace/fixture/fixture.go create mode 100644 internal/recovery/inplace/fixture/fixture_test.go create mode 100644 internal/recovery/inplace/fixture/gen/main.go create mode 100644 internal/recovery/inplace/fixture/mutate.go create mode 100644 internal/recovery/inplace/report/report.go create mode 100644 internal/recovery/inplace/report/report_test.go create mode 100644 internal/recovery/inplace/rodb/adapter.go create mode 100644 internal/recovery/inplace/rodb/errors.go create mode 100644 internal/recovery/inplace/rodb/layout.go create mode 100644 internal/recovery/inplace/rodb/open.go create mode 100644 internal/recovery/inplace/rodb/probe_other.go create mode 100644 internal/recovery/inplace/rodb/probe_unix.go create mode 100644 internal/recovery/inplace/rodb/retry.go create mode 100644 internal/recovery/inplace/rodb/rodb_test.go create mode 100644 internal/recovery/inplace/rodb/storage.go create mode 100644 internal/recovery/inplace/statecheck/anomaly.go create mode 100644 internal/recovery/inplace/statecheck/digest.go create mode 100644 internal/recovery/inplace/statecheck/walker.go create mode 100644 internal/recovery/inplace/statecheck/walker_test.go diff --git a/internal/recovery/inplace/anchor/anchor.go b/internal/recovery/inplace/anchor/anchor.go new file mode 100644 index 0000000000..01b18eabfc --- /dev/null +++ b/internal/recovery/inplace/anchor/anchor.go @@ -0,0 +1,143 @@ +// Package anchor holds the compiled-in incident constants for the shard-0 +// in-place recovery preflight and derives the epoch geometry from the +// network schedule at runtime. +// +// The anchor is deliberately minimal: network, shard, target height and the +// operator-provided target block hash. Everything else (state root, epoch, +// ViewID, parent hash) is read from the locally stored target header, which +// must recompute (header.Hash()) to the anchored hash - the externally known +// block hash pins the entire header content, so there is no manifest file +// and nothing for validators to fetch or verify. +package anchor + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/shard" +) + +// Compiled-in incident constants (mainnet shard 0). +const ( + // MainnetTargetHeight is the recovery target block height. + MainnetTargetHeight uint64 = 92730034 + // MainnetTargetHashHex is the operator-provided target block hash + // (authoritative, cross-checked against explorer/API at release time via + // `go test -tags releasecheck ./internal/recovery/inplace/anchor/...`). + MainnetTargetHashHex = "0x30c35d2f2291e4b27debe7862956cf7a0cc7abefc044273d6823567335086d8d" + + // MainnetAbandonedChildHashHex is the memo's abandoned child block + // 92,730,035 hash. Informational only: the head-sample receipt rows + // compare against it; it never gates any check. + MainnetAbandonedChildHashHex = "0x5de06979a333f20afb8b245a8cf44472dc5bfc7383a57ddee48e1809bcee7c5d" +) + +// MainnetTargetHash is the parsed operator-provided target hash. +var MainnetTargetHash = common.HexToHash(MainnetTargetHashHex) + +// Anchor is the resolved verification anchor: compiled constants plus +// schedule-derived epoch geometry. +type Anchor struct { + Network nodeconfig.NetworkType + ShardID uint32 + TargetHeight uint64 + TargetHash common.Hash + + // Derived from the schedule (never compiled in): + Epoch *big.Int // CalcEpochNumber(TargetHeight) + BoundaryHeight uint64 // EpochLastBlock(Epoch-1): carries ss + + ChainConfig *params.ChainConfig + Schedule shardingconfig.Schedule +} + +// Overrides are the hidden, test-only anchor overrides. On mainnet they are +// refused: the compiled constants are authoritative. +type Overrides struct { + TargetHeight uint64 // 0 = unset + TargetHash string // "" = unset +} + +// Resolve builds the anchor for the given network, applying test-only +// overrides on non-mainnet networks. It also installs the process-global +// shard.Schedule required by the committee/quorum code. +func Resolve(network string, shardID uint32, ov Overrides) (*Anchor, error) { + networkType := nodeconfig.NetworkType(network) + schedule, chainConfig, err := scheduleForNetwork(networkType) + if err != nil { + return nil, err + } + // Process-global schedule initialization, as cmd/harmony/main.go does; + // votepower.Compute and committee reads consult shard.Schedule. + shard.Schedule = schedule + + a := &Anchor{ + Network: networkType, + ShardID: shardID, + ChainConfig: chainConfig, + Schedule: schedule, + } + if networkType == nodeconfig.Mainnet { + if ov.TargetHeight != 0 || ov.TargetHash != "" { + return nil, fmt.Errorf("--target-height/--target-hash are test-only overrides and are refused on --network mainnet (compiled constants are authoritative)") + } + if shardID != 0 { + return nil, fmt.Errorf("the compiled mainnet anchor is for shard 0; --shard %d is not supported", shardID) + } + a.TargetHeight = MainnetTargetHeight + a.TargetHash = MainnetTargetHash + } else { + if ov.TargetHeight == 0 || ov.TargetHash == "" { + return nil, fmt.Errorf("--network %s requires the test-only --target-height and --target-hash overrides", network) + } + if len(common.FromHex(ov.TargetHash)) != common.HashLength { + return nil, fmt.Errorf("--target-hash %q is not a 32-byte hex hash", ov.TargetHash) + } + a.TargetHeight = ov.TargetHeight + a.TargetHash = common.HexToHash(ov.TargetHash) + } + + a.Epoch = schedule.CalcEpochNumber(a.TargetHeight) + if a.Epoch.Sign() <= 0 { + return nil, fmt.Errorf("target height %d resolves to epoch %s; the preflight requires a target above epoch 0", a.TargetHeight, a.Epoch) + } + if !chainConfig.IsStaking(a.Epoch) { + return nil, fmt.Errorf("target epoch %s is pre-staking on %s; the certificate check requires a staking-era committee", a.Epoch, network) + } + a.BoundaryHeight = schedule.EpochLastBlock(a.Epoch.Uint64() - 1) + // Cross-check the derived geometry against the schedule. + if a.BoundaryHeight >= a.TargetHeight { + return nil, fmt.Errorf("schedule inconsistency: boundary %d >= target %d", a.BoundaryHeight, a.TargetHeight) + } + if last := schedule.EpochLastBlock(a.Epoch.Uint64()); last < a.TargetHeight { + return nil, fmt.Errorf("schedule inconsistency: target %d beyond epoch %s last block %d", a.TargetHeight, a.Epoch, last) + } + return a, nil +} + +func scheduleForNetwork(nt nodeconfig.NetworkType) (shardingconfig.Schedule, *params.ChainConfig, error) { + switch nt { + case nodeconfig.Mainnet: + return shardingconfig.MainnetSchedule, params.MainnetChainConfig, nil + case nodeconfig.Testnet: + return shardingconfig.TestnetSchedule, params.TestnetChainConfig, nil + case nodeconfig.Localnet: + // The localnet schedule needs its blocks-per-epoch configuration + // installed before use (panics otherwise). 16/16 are the harmony + // config defaults; fixtures are generated with the same values. + shardingconfig.InitLocalnetConfig(16, 16) + return shardingconfig.LocalnetSchedule, params.LocalnetChainConfig, nil + case nodeconfig.Partner: + return shardingconfig.PartnerSchedule, params.PartnerChainConfig, nil + case nodeconfig.Stressnet: + return shardingconfig.StressNetSchedule, params.StressnetChainConfig, nil + case nodeconfig.Pangaea: + return shardingconfig.PangaeaSchedule, params.PangaeaChainConfig, nil + default: + return nil, nil, fmt.Errorf("unsupported --network %q (mainnet, testnet, localnet, partner, stressnet, pangaea)", string(nt)) + } +} diff --git a/internal/recovery/inplace/anchor/anchor_test.go b/internal/recovery/inplace/anchor/anchor_test.go new file mode 100644 index 0000000000..9e66a48588 --- /dev/null +++ b/internal/recovery/inplace/anchor/anchor_test.go @@ -0,0 +1,86 @@ +package anchor + +import ( + "strings" + "testing" + + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" +) + +// TestMainnetScheduleProperties pins the derived epoch geometry the plan +// relies on: CalcEpochNumber(92,730,034) = 3002 and EpochLastBlock(3001) = +// 92,700,671 under the compiled mainnet schedule. +func TestMainnetScheduleProperties(t *testing.T) { + s := shardingconfig.MainnetSchedule + if got := s.CalcEpochNumber(MainnetTargetHeight); got.Uint64() != 3002 { + t.Fatalf("CalcEpochNumber(%d) = %s, want 3002", MainnetTargetHeight, got) + } + if got := s.EpochLastBlock(3001); got != 92700671 { + t.Fatalf("EpochLastBlock(3001) = %d, want 92700671", got) + } + if got := s.EpochLastBlock(3002); got != 92733439 { + t.Fatalf("EpochLastBlock(3002) = %d, want 92733439", got) + } +} + +// TestMainnetResolve checks the compiled constants land in the anchor +// (reviewed literals) and the derived fields match the schedule. +func TestMainnetResolve(t *testing.T) { + a, err := Resolve("mainnet", 0, Overrides{}) + if err != nil { + t.Fatal(err) + } + if a.TargetHeight != 92730034 { + t.Fatalf("target height %d", a.TargetHeight) + } + if a.TargetHash.Hex() != "0x30c35d2f2291e4b27debe7862956cf7a0cc7abefc044273d6823567335086d8d" { + t.Fatalf("target hash %s", a.TargetHash.Hex()) + } + if a.Epoch.Uint64() != 3002 { + t.Fatalf("epoch %s", a.Epoch) + } + if a.BoundaryHeight != 92700671 { + t.Fatalf("boundary %d", a.BoundaryHeight) + } + if !a.ChainConfig.IsStaking(a.Epoch) { + t.Fatal("target epoch must be staking-era") + } +} + +// TestMainnetOverridesRefused: the compiled constants are authoritative on +// mainnet. +func TestMainnetOverridesRefused(t *testing.T) { + if _, err := Resolve("mainnet", 0, Overrides{TargetHeight: 42}); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("height override not refused: %v", err) + } + if _, err := Resolve("mainnet", 0, Overrides{TargetHash: "0xdead"}); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("hash override not refused: %v", err) + } + if _, err := Resolve("mainnet", 1, Overrides{}); err == nil || !strings.Contains(err.Error(), "shard 0") { + t.Fatalf("non-zero shard not refused: %v", err) + } +} + +func TestNonMainnetResolve(t *testing.T) { + hash := "0x30c35d2f2291e4b27debe7862956cf7a0cc7abefc044273d6823567335086d8d" + if _, err := Resolve("localnet", 0, Overrides{}); err == nil { + t.Fatal("localnet without overrides must be refused") + } + if _, err := Resolve("localnet", 0, Overrides{TargetHeight: 44, TargetHash: "0x123"}); err == nil { + t.Fatal("short hash must be refused") + } + a, err := Resolve("localnet", 0, Overrides{TargetHeight: 44, TargetHash: hash}) + if err != nil { + t.Fatal(err) + } + if a.Epoch.Uint64() != 3 || a.BoundaryHeight != 36 { + t.Fatalf("localnet target 44: epoch %s boundary %d, want 3/36", a.Epoch, a.BoundaryHeight) + } + if _, err := Resolve("neptune", 0, Overrides{TargetHeight: 44, TargetHash: hash}); err == nil { + t.Fatal("unknown network must be refused") + } + // Pre-staking target refused (localnet epoch 1 < staking epoch 2). + if _, err := Resolve("localnet", 0, Overrides{TargetHeight: 10, TargetHash: hash}); err == nil || !strings.Contains(err.Error(), "staking") { + t.Fatalf("pre-staking target not refused: %v", err) + } +} diff --git a/internal/recovery/inplace/anchor/releasecheck_test.go b/internal/recovery/inplace/anchor/releasecheck_test.go new file mode 100644 index 0000000000..986fe2554a --- /dev/null +++ b/internal/recovery/inplace/anchor/releasecheck_test.go @@ -0,0 +1,72 @@ +//go:build releasecheck + +package anchor + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" +) + +// TestReleaseCrossCheckTargetHash is the pre-release cross-check build step: +// it fetches the target block from the public explorer/API endpoints and +// asserts the returned hash equals the compiled constant. +// +// It is guarded by the releasecheck build tag because it needs network +// access; the releaser runs it manually: +// +// go test -tags releasecheck -run TestReleaseCrossCheck ./internal/recovery/inplace/anchor/... +// +// Validators never run it; CI never needs network. +func TestReleaseCrossCheckTargetHash(t *testing.T) { + endpoints := []string{ + "https://api.harmony.one", + "https://api.s0.t.hmny.io", + } + reqBody := fmt.Sprintf( + `{"jsonrpc":"2.0","id":1,"method":"hmyv2_getBlockByNumber","params":[%d,{}]}`, + MainnetTargetHeight, + ) + client := &http.Client{Timeout: 30 * time.Second} + var lastErr error + for _, url := range endpoints { + resp, err := client.Post(url, "application/json", bytes.NewBufferString(reqBody)) + if err != nil { + lastErr = err + continue + } + var parsed struct { + Result struct { + Hash string `json:"hash"` + Number uint64 `json:"number"` + } `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + err = json.NewDecoder(resp.Body).Decode(&parsed) + resp.Body.Close() + if err != nil { + lastErr = err + continue + } + if parsed.Error != nil { + lastErr = fmt.Errorf("%s: rpc error: %s", url, parsed.Error.Message) + continue + } + if parsed.Result.Number != MainnetTargetHeight { + t.Fatalf("%s returned block %d, requested %d", url, parsed.Result.Number, MainnetTargetHeight) + } + if !strings.EqualFold(parsed.Result.Hash, MainnetTargetHashHex) { + t.Fatalf("compiled target hash %s does NOT match %s block %d hash %s", + MainnetTargetHashHex, url, MainnetTargetHeight, parsed.Result.Hash) + } + t.Logf("compiled target hash confirmed by %s at height %d", url, MainnetTargetHeight) + return + } + t.Fatalf("no endpoint reachable: %v", lastErr) +} diff --git a/internal/recovery/inplace/certverify/certverify.go b/internal/recovery/inplace/certverify/certverify.go new file mode 100644 index 0000000000..414d5b8a2b --- /dev/null +++ b/internal/recovery/inplace/certverify/certverify.go @@ -0,0 +1,106 @@ +// Package certverify verifies the target block's commit certificate from +// its two possible local sources, using the audited +// chain.NewEngine().VerifyHeaderSignature path against the minimal +// fail-closed ChainReader (BlockChainImpl is never constructed). +package certverify + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/harmony-one/harmony/internal/chain" + "github.com/harmony-one/harmony/internal/recovery/inplace/anchor" + "github.com/harmony-one/harmony/internal/recovery/inplace/chainread" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" +) + +// Result records which certificate sources were present and which satisfied +// the check. +type Result struct { + Sources report.CertificateSources +} + +// Verify runs check 6 (target certificate, two sources): +// +// - source A: the exact raw key "block-sig-"+BE64(target height) +// - source B: the child header's LastCommitSignature+LastCommitBitmap +// +// Every present source is verified cryptographically: stake-weighted quorum +// over the committee decoded from the walk-authenticated ss record, payload +// binding blockNum+blockHash+viewID. Decision: at least one source present +// and verifying passes; two present sources that differ byte-wise FAIL (two +// different valid aggregates for one block must not be papered over); zero +// present sources or any present-but-failing source FAIL. +func Verify(kv ethdb.KeyValueReader, a *anchor.Anchor, out *chainread.Outcome) (*Result, error) { + res := &Result{} + + exact, foundExact, err := chainread.ReadBlockCommitSig(kv, a.TargetHeight) + if err != nil { + return res, err + } + res.Sources.ExactKeyPresent = foundExact + + // A present-but-malformed child header (undecodable, or failing hash + // authentication) is a failing source, not an absent one: every present + // source must verify, even when the exact key alone would pass. + if out.ChildSourceErr != "" { + res.Sources.ChildHeaderPresent = true + return res, report.Failf("certificate", + "child-header source at %d is present but malformed: %s", a.TargetHeight+1, out.ChildSourceErr) + } + + var childPayload []byte + if out.ChildHeader != nil { + sig := out.ChildHeader.LastCommitSignature() + childPayload = append(sig[:], out.ChildHeader.LastCommitBitmap()...) + res.Sources.ChildHeaderPresent = true + } + + if !foundExact && childPayload == nil { + return res, report.Failf("certificate", + "no certificate source present: block-sig-%d missing and no child header at %d", + a.TargetHeight, a.TargetHeight+1) + } + + reader := chainread.NewMinimalChainReader(a.ChainConfig, a.ShardID, out.TargetHeader, a.Epoch, out.ShardState) + engine := chain.NewEngine() + + verify := func(name string, payload []byte) error { + sig, bitmap, err := chain.ParseCommitSigAndBitmap(payload) + if err != nil { + return report.Failf("certificate", "source %s does not parse: %v", name, err) + } + if err := engine.VerifyHeaderSignature(reader, out.TargetHeader, sig, bitmap); err != nil { + return report.Failf("certificate", "source %s fails verification: %v", name, err) + } + return nil + } + + var satisfied []string + if foundExact { + if err := verify("exact-key", exact); err != nil { + return res, err + } + satisfied = append(satisfied, "exact-key") + } + if childPayload != nil { + if err := verify("child-header", childPayload); err != nil { + return res, err + } + satisfied = append(satisfied, "child-header") + } + if foundExact && childPayload != nil && !bytes.Equal(exact, childPayload) { + return res, report.Failf("certificate", + "exact-key and child-header certificates are both valid but differ byte-wise (%d vs %d bytes)", + len(exact), len(childPayload)) + } + if len(satisfied) == 1 { + res.Sources.SatisfiedBy = satisfied[0] + } else { + res.Sources.SatisfiedBy = "exact-key+child-header" + } + // Note for Workstream B (informational): if source A was missing, apply + // time can materialize the exact key from the verified child header. + return res, nil +} diff --git a/internal/recovery/inplace/certverify/certverify_test.go b/internal/recovery/inplace/certverify/certverify_test.go new file mode 100644 index 0000000000..e2be260470 --- /dev/null +++ b/internal/recovery/inplace/certverify/certverify_test.go @@ -0,0 +1,197 @@ +package certverify_test + +import ( + "math/big" + "path/filepath" + "testing" + "time" + + blockfactory "github.com/harmony-one/harmony/block/factory" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/internal/chain" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/recovery/inplace/anchor" + "github.com/harmony-one/harmony/internal/recovery/inplace/certverify" + "github.com/harmony-one/harmony/internal/recovery/inplace/chainread" + "github.com/harmony-one/harmony/internal/recovery/inplace/fixture" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" +) + +func fixtureOutcome(t *testing.T) (*fixture.Manifest, *anchor.Anchor, *rodb.DB, *chainread.Outcome) { + t.Helper() + dir := filepath.Join(t.TempDir(), "harmony_db_0") + m, err := fixture.Build(dir, fixture.VariantBase) + if err != nil { + t.Fatal(err) + } + a, err := anchor.Resolve("localnet", 0, anchor.Overrides{ + TargetHeight: fixture.TargetHeight, + TargetHash: m.TargetHash.Hex(), + }) + if err != nil { + t.Fatal(err) + } + db, err := rodb.Open(dir, rodb.Options{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + out, err := chainread.RunChecks(db.KV(&rodb.Latch{}), a, nil) + if err != nil { + t.Fatalf("chain checks: %v", err) + } + return m, a, db, out +} + +// TestVerifyBothSources: the fixture carries both certificate sources with +// identical bytes; verification must pass via the production engine. +func TestVerifyBothSources(t *testing.T) { + _, a, db, out := fixtureOutcome(t) + res, err := certverify.Verify(db.KV(&rodb.Latch{}), a, out) + if err != nil { + t.Fatalf("verify: %v", err) + } + if res.Sources.SatisfiedBy != "exact-key+child-header" { + t.Fatalf("satisfied by %q", res.Sources.SatisfiedBy) + } +} + +// TestBlockChainImplDifferential: the independent certificate differential. +// A real core.BlockChainImpl is constructed over a scratch COPY of the +// fixture (it may write there - production preflight never constructs one), +// and the engine's verdict through it must agree with the verdict through +// the minimal fail-closed ChainReader, on both the accept and the reject +// side. A shared bug in the minimal reader's committee sourcing would break +// the agreement here. +func TestBlockChainImplDifferential(t *testing.T) { + m, a, _, out := fixtureOutcome(t) + + scratch := filepath.Join(t.TempDir(), "harmony_db_0") + if err := fixture.CopyDB(m.Dir, scratch); err != nil { + t.Fatal(err) + } + db, err := rawdb.NewLevelDBDatabase(scratch, 64, 128, "", false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // BlockChainImpl refuses to construct without a genesis block; the + // recovery fixture deliberately starts at the epoch boundary, so plant + // a minimal genesis in the scratch copy. + factory := blockfactory.NewFactory(params.LocalnetChainConfig) + gh := factory.NewHeader(big.NewInt(0)) + gh.SetNumber(big.NewInt(0)) + gh.SetShardID(0) + if err := rawdb.WriteHeader(db, gh); err != nil { + t.Fatal(err) + } + if err := rawdb.WriteCanonicalHash(db, gh.Hash(), 0); err != nil { + t.Fatal(err) + } + gbody, err := types.NewBodyForMatchingHeader(gh) + if err != nil { + t.Fatal(err) + } + if err := rawdb.WriteBody(db, gh.Hash(), 0, gbody); err != nil { + t.Fatal(err) + } + // Point the head block at the target so CurrentHeader matches what the + // minimal reader serves (the fixture's head is the child, whose header + // carries no state root). + if err := rawdb.WriteHeadBlockHash(db, m.TargetHash); err != nil { + t.Fatal(err) + } + + // The constructor's leader-rotation init walk resolves each header's + // coinbase to a committee BLS key - metadata the minimal recovery + // fixture does not carry, and irrelevant to certificate verification. + // Push the activation epoch out of reach for the scratch chain only; + // every field the signature path consults is untouched. + cfg := *params.LocalnetChainConfig + cfg.LeaderRotationInternalValidatorsEpoch = big.NewInt(1 << 30) + cfg.LeaderRotationExternalValidatorsEpoch = big.NewInt(1 << 30) + + engine := chain.NewEngine() + bc, err := core.NewBlockChain(db, nil, nil, &core.CacheConfig{ + // Disabled skips Stop()'s recent-trie commit sweep, which would + // dereference the zero roots the fixture's non-target headers carry. + Disabled: true, + TrieCleanLimit: 16, + TrieDirtyLimit: 16, + TrieTimeLimit: time.Minute, + TriesInMemory: 128, + SnapshotLimit: 0, + }, &cfg, engine, vm.Config{}) + if err != nil { + t.Fatalf("scratch BlockChainImpl: %v", err) + } + defer bc.Stop() + if got := bc.CurrentHeader().Hash(); got != m.TargetHash { + t.Fatalf("scratch chain current header %s, want target %s", got.Hex(), m.TargetHash.Hex()) + } + + minimal := chainread.NewMinimalChainReader( + params.LocalnetChainConfig, a.ShardID, out.TargetHeader, a.Epoch, out.ShardState) + + // Accept side: the genuine certificate verifies through both readers. + sig, bitmap, err := chain.ParseCommitSigAndBitmap(m.CertPayload) + if err != nil { + t.Fatal(err) + } + if err := engine.VerifyHeaderSignature(bc, out.TargetHeader, sig, bitmap); err != nil { + t.Fatalf("BlockChainImpl path rejects the genuine certificate: %v", err) + } + if err := engine.VerifyHeaderSignature(minimal, out.TargetHeader, sig, bitmap); err != nil { + t.Fatalf("minimal-reader path rejects the genuine certificate: %v", err) + } + + // Reject side: a tampered aggregate fails through both readers. + bad := append([]byte(nil), m.CertPayload...) + bad[10] ^= 0x01 + badSig, badBitmap, err := chain.ParseCommitSigAndBitmap(bad) + if err != nil { + t.Fatal(err) + } + errFull := engine.VerifyHeaderSignature(bc, out.TargetHeader, badSig, badBitmap) + errMin := engine.VerifyHeaderSignature(minimal, out.TargetHeader, badSig, badBitmap) + if errFull == nil || errMin == nil { + t.Fatalf("tampered certificate accepted: full=%v minimal=%v", errFull, errMin) + } +} + +// TestChainReaderMethodPin: the certificate verification path exercises +// exactly {Config, CurrentHeader, ShardID, ReadShardState} on the minimal +// ChainReader. An upstream internal/chain change that starts calling more +// methods fails here (and fails closed at runtime). +func TestChainReaderMethodPin(t *testing.T) { + m, a, _, out := fixtureOutcome(t) + + reader := chainread.NewMinimalChainReader( + params.LocalnetChainConfig, a.ShardID, out.TargetHeader, a.Epoch, out.ShardState) + engine := chain.NewEngine() + sig, bitmap, err := chain.ParseCommitSigAndBitmap(m.CertPayload) + if err != nil { + t.Fatal(err) + } + if err := engine.VerifyHeaderSignature(reader, out.TargetHeader, sig, bitmap); err != nil { + t.Fatalf("verification failed: %v", err) + } + called := reader.CalledMethods() + audited := map[string]bool{ + "Config": true, "CurrentHeader": true, "ShardID": true, "ReadShardState": true, + } + for method := range called { + if !audited[method] { + t.Fatalf("engine exercised unaudited ChainReader method %s (called set %v)", method, called) + } + } + for _, must := range []string{"Config", "CurrentHeader", "ReadShardState"} { + if called[must] == 0 { + t.Fatalf("expected engine to call %s (called set %v)", must, called) + } + } +} diff --git a/internal/recovery/inplace/chainread/checks.go b/internal/recovery/inplace/chainread/checks.go new file mode 100644 index 0000000000..fa0675ec9c --- /dev/null +++ b/internal/recovery/inplace/chainread/checks.go @@ -0,0 +1,390 @@ +package chainread + +import ( + "bytes" + "fmt" + "io" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/recovery/inplace/anchor" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" + "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" +) + +// Outcome carries what the chain checks proved, feeding the certificate +// check and the state walk. +type Outcome struct { + TargetHeader *block.Header + StateRoot common.Hash + ViewID uint64 + BoundaryHeader *block.Header + ShardState *shard.State + + // ChildHeader is the header at target height+1 whose parent is the + // target (certificate source B), nil when absent. + ChildHeader *block.Header + + // ChildSourceErr is set when the canonical slot at target+1 holds a + // present-but-malformed header: undecodable bytes, or content that does + // not recompute to its canonical hash. Certificate verification fails + // closed on it - a present source must verify, and malformed stored + // bytes must not be papered over as absence. + ChildSourceErr string + + // Head sample rows (informational, never gate). + Head report.HeadSample + + // Checks records "ok"/"fail: ..."/"skipped" per check id as the ordered + // checks progress. + Checks map[string]string +} + +// maxUpwardWalk bounds the informational head-to-target walk (the incident +// cites ~21k abandoned blocks above the target; walk a bit further, never +// unbounded). +const maxUpwardWalk = 1 << 18 + +// RunChecks performs the ordered chain checks (target tuple, body, +// downward ancestry, shard state, upward head sample). Certificate +// verification (check 6) lives in certverify and consumes this Outcome. +// The returned error is a *report.Failure for verification FAILs, or a read +// error to be classified by the retry runner. +func RunChecks(kv ethdb.KeyValueReader, a *anchor.Anchor, progress io.Writer) (*Outcome, error) { + out := &Outcome{Checks: report.NewChecks()} + + // mark records a verification FAIL against a check id; read errors are + // NOT verification results and leave the check "skipped" (the stage may + // be retried against a fresh manifest generation). + mark := func(id string, err error) error { + if f, ok := err.(*report.Failure); ok { + out.Checks[id] = "fail: " + f.Reason + } + return err + } + + // Check 1: target tuple. + if err := out.checkTargetHeader(kv, a); err != nil { + return out, mark("target_header", err) + } + out.Checks["target_header"] = "ok" + + // Check 2: body integrity. + if err := out.checkBody(kv, a); err != nil { + return out, mark("body", err) + } + out.Checks["body"] = "ok" + + // Check 3: downward ancestry walk to the epoch boundary. + if err := out.walkAncestryToBoundary(kv, a, progress); err != nil { + return out, mark("ancestry_to_boundary", err) + } + out.Checks["ancestry_to_boundary"] = "ok" + + // Check 4: ss byte-equality against the walk-authenticated + // boundary header. + if err := out.checkShardState(kv, a); err != nil { + return out, mark("shard_state", err) + } + out.Checks["shard_state"] = "ok" + + // Check 5: upward sample - informational, never gates. Read errors on a + // moving head are recorded as strings, not returned (the head is + // expected to move on a live DB), and they must not dirty the shared + // read-error latch either - a latched head error would send an + // otherwise clean run into the retry machinery and exit 3. When the + // reader can scope its latch (the rodb adapter), sample through an + // unlatched view. + informational := kv + if u, ok := kv.(interface{ Unlatched() ethdb.KeyValueReader }); ok { + informational = u.Unlatched() + } + out.sampleHeads(informational, a, progress) + + // Locate the child header for certificate source B. + if err := out.locateChild(kv, a); err != nil { + return out, err + } + return out, nil +} + +func (o *Outcome) checkTargetHeader(kv ethdb.KeyValueReader, a *anchor.Anchor) error { + canonical, found, err := ReadCanonicalHash(kv, a.TargetHeight) + if err != nil { + return err + } + if !found { + return report.Failf("target_header", "no canonical hash at target height %d", a.TargetHeight) + } + if canonical != a.TargetHash { + return report.Failf("target_header", "canonical hash at %d is %s, want anchored %s", a.TargetHeight, canonical.Hex(), a.TargetHash.Hex()) + } + num, found, err := ReadHeaderNumber(kv, a.TargetHash) + if err != nil { + return err + } + if !found { + return report.Failf("target_header", "no reverse number mapping for target hash %s", a.TargetHash.Hex()) + } + if num != a.TargetHeight { + return report.Failf("target_header", "reverse mapping of %s is height %d, want %d", a.TargetHash.Hex(), num, a.TargetHeight) + } + header, found, err := ReadHeader(kv, a.TargetHeight, a.TargetHash) + if err != nil { + if de, ok := err.(*DecodeErr); ok { + return report.Failf("target_header", "%v", de) + } + return err + } + if !found { + return report.Failf("target_header", "target header %d %s not present", a.TargetHeight, a.TargetHash.Hex()) + } + // The recomputed hash pins the entire header content to the anchor. + if got := header.Hash(); got != a.TargetHash { + return report.Failf("target_header", "stored target header recomputes to %s, want anchored %s", got.Hex(), a.TargetHash.Hex()) + } + if header.Number() == nil || header.Number().Uint64() != a.TargetHeight { + return report.Failf("target_header", "target header embeds number %v, want %d", header.Number(), a.TargetHeight) + } + if header.ShardID() != a.ShardID { + return report.Failf("target_header", "target header shard %d, want %d", header.ShardID(), a.ShardID) + } + if header.Epoch() == nil || header.Epoch().Cmp(a.Epoch) != 0 { + return report.Failf("target_header", "target header epoch %v, want %s (schedule cross-check)", header.Epoch(), a.Epoch) + } + o.TargetHeader = header + o.StateRoot = header.Root() + o.ViewID = header.ViewID().Uint64() + return nil +} + +func (o *Outcome) checkBody(kv ethdb.KeyValueReader, a *anchor.Anchor) error { + body, found, err := ReadBody(kv, a.TargetHeight, a.TargetHash) + if err != nil { + if de, ok := err.(*DecodeErr); ok { + return report.Failf("body", "%v", de) + } + return err + } + if !found { + return report.Failf("body", "target block body not present") + } + txs := types.Transactions(body.Transactions()) + stks := staking.StakingTransactions(body.StakingTransactions()) + if got := types.DeriveSha(txs, stks); got != o.TargetHeader.TxHash() { + return report.Failf("body", "transaction root mismatch: derived %s, header %s", got.Hex(), o.TargetHeader.TxHash().Hex()) + } + incoming := types.CXReceiptsProofs(body.IncomingReceipts()) + if got := types.DeriveSha(incoming); got != o.TargetHeader.IncomingReceiptHash() { + return report.Failf("body", "incoming-receipt root mismatch: derived %s, header %s", got.Hex(), o.TargetHeader.IncomingReceiptHash().Hex()) + } + // The v3 header has no UncleHash field, so stored uncles are + // unauthenticated bytes; the check gates on emptiness. + if n := len(body.Uncles()); n != 0 { + return report.Failf("body", "target block body carries %d uncles, want 0 (uncles are unauthenticated)", n) + } + return nil +} + +func (o *Outcome) walkAncestryToBoundary(kv ethdb.KeyValueReader, a *anchor.Anchor, progress io.Writer) error { + steps := a.TargetHeight - a.BoundaryHeight + current := o.TargetHeader + for n := a.TargetHeight; n > a.BoundaryHeight; n-- { + parentHash := current.ParentHash() + parentNum := n - 1 + parent, found, err := ReadHeader(kv, parentNum, parentHash) + if err != nil { + if de, ok := err.(*DecodeErr); ok { + return report.Failf("ancestry_to_boundary", "%v", de) + } + return err + } + if !found { + return report.Failf("ancestry_to_boundary", "broken parent link: header %d %s (parent of %d) not present", parentNum, parentHash.Hex(), n) + } + // Recompute the parent hash: a valid-RLP-wrong-content header under + // the right key must not pass. + if got := parent.Hash(); got != parentHash { + return report.Failf("ancestry_to_boundary", "header stored at %d %s recomputes to %s", parentNum, parentHash.Hex(), got.Hex()) + } + if parent.Number() == nil || parent.Number().Uint64() != parentNum { + return report.Failf("ancestry_to_boundary", "header %s embeds number %v, want %d", parentHash.Hex(), parent.Number(), parentNum) + } + // Canonical-mapping agreement: long-final heights must agree even on + // a live DB. + canonical, found, err := ReadCanonicalHash(kv, parentNum) + if err != nil { + return err + } + if !found { + return report.Failf("ancestry_to_boundary", "no canonical mapping at height %d", parentNum) + } + if canonical != parentHash { + return report.Failf("ancestry_to_boundary", "canonical mapping at %d is %s, ancestry expects %s", parentNum, canonical.Hex(), parentHash.Hex()) + } + current = parent + if progress != nil && (a.TargetHeight-parentNum)%8192 == 0 { + fmt.Fprintf(progress, "ancestry walk: %d/%d parent steps\n", a.TargetHeight-parentNum, steps) + } + } + o.BoundaryHeader = current + if progress != nil { + fmt.Fprintf(progress, "ancestry walk: authenticated boundary header %d %s (%d steps)\n", + a.BoundaryHeight, current.Hash().Hex(), steps) + } + return nil +} + +func (o *Outcome) checkShardState(kv ethdb.KeyValueReader, a *anchor.Anchor) error { + raw, found, err := ReadShardStateBytes(kv, a.Epoch) + if err != nil { + return err + } + if !found { + return report.Failf("shard_state", "ss<%s> record not present", a.Epoch) + } + boundaryBytes := o.BoundaryHeader.ShardState() + if len(boundaryBytes) == 0 { + return report.Failf("shard_state", "boundary header %d carries no shard state", a.BoundaryHeight) + } + if !bytes.Equal(raw, boundaryBytes) { + return report.Failf("shard_state", "ss<%s> record (%d bytes) differs from boundary header %d ShardState (%d bytes)", + a.Epoch, len(raw), a.BoundaryHeight, len(boundaryBytes)) + } + decoded, err := shard.DecodeWrapper(raw) + if err != nil { + return report.Failf("shard_state", "ss<%s> does not decode: %v", a.Epoch, err) + } + if decoded.Epoch == nil || decoded.Epoch.Cmp(a.Epoch) != 0 { + return report.Failf("shard_state", "ss<%s> decodes with epoch %v, want %s", a.Epoch, decoded.Epoch, a.Epoch) + } + committee, err := decoded.FindCommitteeByID(a.ShardID) + if err != nil { + return report.Failf("shard_state", "ss<%s> has no committee for shard %d: %v", a.Epoch, a.ShardID, err) + } + if len(committee.Slots) == 0 { + return report.Failf("shard_state", "ss<%s> committee for shard %d is empty", a.Epoch, a.ShardID) + } + o.ShardState = decoded + return nil +} + +// sampleHeads reads the head pointers and walks head-to-target parent links. +// Informational only: on a live DB heads move and abandoned-branch shapes +// vary - gating here would produce false FAILs, and target-block eligibility +// does not depend on what sits above the target. All errors are recorded as +// strings, never returned. +func (o *Outcome) sampleHeads(kv ethdb.KeyValueReader, a *anchor.Anchor, progress io.Writer) { + headHeader, foundHH, errHH := ReadHeadPointer(kv, HeadHeaderKey) + switch { + case errHH != nil: + o.Head.LastHeader = "read-error: " + errHH.Error() + case !foundHH: + o.Head.LastHeader = "absent" + default: + o.Head.LastHeader = headHeader.Hex() + } + headBlock, foundHB, errHB := ReadHeadPointer(kv, HeadBlockKey) + switch { + case errHB != nil: + o.Head.LastBlock = "read-error: " + errHB.Error() + case !foundHB: + o.Head.LastBlock = "absent" + default: + o.Head.LastBlock = headBlock.Hex() + } + + start := common.Hash{} + if foundHH && errHH == nil { + start = headHeader + } else if foundHB && errHB == nil { + start = headBlock + } + if start == (common.Hash{}) { + o.Head.WalkToTarget = "not-walked: no resolvable head pointer" + return + } + num, found, err := ReadHeaderNumber(kv, start) + if err != nil || !found { + o.Head.WalkToTarget = "not-walked: head has no number mapping" + return + } + if num < a.TargetHeight { + o.Head.WalkToTarget = fmt.Sprintf("head-below-target: head height %d < target %d", num, a.TargetHeight) + return + } + if num-a.TargetHeight > maxUpwardWalk { + o.Head.WalkToTarget = fmt.Sprintf("not-walked: head %d is %d blocks above target (bound %d)", num, num-a.TargetHeight, maxUpwardWalk) + return + } + if progress != nil { + fmt.Fprintf(progress, "head sample: walking %d parent steps from head %d to target height\n", num-a.TargetHeight, num) + } + cursor, cursorNum := start, num + for cursorNum > a.TargetHeight { + h, found, err := ReadHeader(kv, cursorNum, cursor) + if err != nil || !found { + o.Head.WalkToTarget = fmt.Sprintf("walk-broken at height %d (%s)", cursorNum, cursor.Hex()) + return + } + cursor = h.ParentHash() + cursorNum-- + } + if cursor == a.TargetHash { + o.Head.WalkToTarget = "reached-target" + } else { + o.Head.WalkToTarget = fmt.Sprintf("diverged: head ancestry at target height is %s, target is %s", cursor.Hex(), a.TargetHash.Hex()) + } +} + +// locateChild finds the header at target+1 whose parent is the target, for +// certificate source B. Absence is not an error; read errors propagate. +func (o *Outcome) locateChild(kv ethdb.KeyValueReader, a *anchor.Anchor) error { + childNum := a.TargetHeight + 1 + childHash, found, err := ReadCanonicalHash(kv, childNum) + if err != nil { + return err + } + if !found { + o.Head.ChildAtTargetPlus = "absent" + return nil + } + child, found, err := ReadHeader(kv, childNum, childHash) + if err != nil { + if de, ok := err.(*DecodeErr); ok { + o.Head.ChildAtTargetPlus = fmt.Sprintf("undecodable header at %d (%s)", childNum, childHash.Hex()) + o.ChildSourceErr = fmt.Sprintf("header at %d (%s) does not decode: %v", childNum, childHash.Hex(), de) + return nil + } + return err + } + if !found { + // The canonical mapping outliving the header is a transient shape + // during above-target cleanup on a live DB: the source payload is + // genuinely absent, not malformed. + o.Head.ChildAtTargetPlus = fmt.Sprintf("canonical %s at %d without stored header", childHash.Hex(), childNum) + return nil + } + if got := child.Hash(); got != childHash { + o.Head.ChildAtTargetPlus = fmt.Sprintf("stored header at %d recomputes to %s, canonical says %s", childNum, got.Hex(), childHash.Hex()) + o.ChildSourceErr = fmt.Sprintf("header stored at %d recomputes to %s, canonical says %s", childNum, got.Hex(), childHash.Hex()) + return nil + } + if child.ParentHash() != a.TargetHash { + // A well-formed foreign block at target+1: its commit signature + // certifies a different parent, so it is not a certificate source + // for the target (informational only). + o.Head.ChildAtTargetPlus = fmt.Sprintf("present-not-child: %s (parent %s != target)", childHash.Hex(), child.ParentHash().Hex()) + return nil + } + o.Head.ChildAtTargetPlus = childHash.Hex() + if a.Network == "mainnet" && childHash == common.HexToHash(anchor.MainnetAbandonedChildHashHex) { + o.Head.ChildAtTargetPlus += " (matches memo abandoned-child hash)" + } + o.ChildHeader = child + return nil +} diff --git a/internal/recovery/inplace/chainread/checks_unit_test.go b/internal/recovery/inplace/chainread/checks_unit_test.go new file mode 100644 index 0000000000..52e0c9c920 --- /dev/null +++ b/internal/recovery/inplace/chainread/checks_unit_test.go @@ -0,0 +1,118 @@ +package chainread + +import ( + "math/big" + "strings" + "testing" + + "github.com/harmony-one/harmony/block/factory" + "github.com/harmony-one/harmony/core/rawdb" + bls "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/recovery/inplace/anchor" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" +) + +func testCommittee(epoch int64) *shard.State { + stake := numeric.NewDec(1) + return &shard.State{ + Epoch: big.NewInt(epoch), + Shards: []shard.Committee{{ + ShardID: 0, + Slots: shard.SlotList{{ + BLSPublicKey: bls.SerializedPublicKey{0x01}, + EffectiveStake: &stake, + }}, + }}, + } +} + +// TestShardStateWrongEpoch: byte-equality passes (boundary header carries +// the same bytes) but the decoded epoch does not match the anchor - FAIL. +func TestShardStateWrongEpoch(t *testing.T) { + db := rawdb.NewMemoryDatabase() + a := &anchor.Anchor{Epoch: big.NewInt(3), ShardID: 0} + + wrongEpochState := testCommittee(4) // decodes fine, epoch mismatch + raw, err := shard.EncodeWrapper(*wrongEpochState, true) + if err != nil { + t.Fatal(err) + } + if err := rawdb.WriteShardStateBytes(db, big.NewInt(3), raw); err != nil { + t.Fatal(err) + } + boundary := blockfactory.NewFactory(params.LocalnetChainConfig).NewHeader(big.NewInt(2)) + boundary.SetShardState(raw) + + o := &Outcome{BoundaryHeader: boundary, Checks: report.NewChecks()} + err = o.checkShardState(db, a) + f, ok := err.(*report.Failure) + if !ok || !strings.Contains(f.Reason, "epoch") { + t.Fatalf("want epoch-mismatch failure, got %v", err) + } +} + +// TestShardStateUndecodable: present, byte-equal, but not valid RLP. +func TestShardStateUndecodable(t *testing.T) { + db := rawdb.NewMemoryDatabase() + a := &anchor.Anchor{Epoch: big.NewInt(3), ShardID: 0} + raw := []byte{0xde, 0xad} + if err := rawdb.WriteShardStateBytes(db, big.NewInt(3), raw); err != nil { + t.Fatal(err) + } + boundary := blockfactory.NewFactory(params.LocalnetChainConfig).NewHeader(big.NewInt(2)) + boundary.SetShardState(raw) + o := &Outcome{BoundaryHeader: boundary, Checks: report.NewChecks()} + err := o.checkShardState(db, a) + f, ok := err.(*report.Failure) + if !ok || !strings.Contains(f.Reason, "decode") { + t.Fatalf("want decode failure, got %v", err) + } +} + +// TestMinimalChainReaderFailsClosed: every method outside the audited set +// panics with UnexpectedCallError. +func TestMinimalChainReaderFailsClosed(t *testing.T) { + r := NewMinimalChainReader(params.LocalnetChainConfig, 0, nil, big.NewInt(3), testCommittee(3)) + + mustPanic := func(name string, fn func()) { + t.Helper() + defer func() { + rec := recover() + uce, ok := rec.(*UnexpectedCallError) + if !ok { + t.Fatalf("%s: want UnexpectedCallError panic, got %v", name, rec) + } + if uce.Method != name { + t.Fatalf("panic names %s, want %s", uce.Method, name) + } + }() + fn() + } + mustPanic("GetHeaderByNumber", func() { r.GetHeaderByNumber(1) }) + mustPanic("TrieDB", func() { r.TrieDB() }) + mustPanic("CurrentBlock", func() { r.CurrentBlock() }) + mustPanic("ReadCommitSig", func() { _, _ = r.ReadCommitSig(1) }) + mustPanic("Snapshots", func() { r.Snapshots() }) + + // Audited set answers. + if r.Config() != params.LocalnetChainConfig || r.ShardID() != 0 { + t.Fatal("audited methods broken") + } + if _, err := r.ReadShardState(big.NewInt(3)); err != nil { + t.Fatal(err) + } + if _, err := r.ReadShardState(big.NewInt(4)); err == nil { + t.Fatal("foreign epoch must be refused") + } + called := r.CalledMethods() + for m := range called { + switch m { + case "Config", "CurrentHeader", "ShardID", "ReadShardState": + default: + t.Fatalf("unexpected recorded method %s", m) + } + } +} diff --git a/internal/recovery/inplace/chainread/headsample_test.go b/internal/recovery/inplace/chainread/headsample_test.go new file mode 100644 index 0000000000..cecdea6a82 --- /dev/null +++ b/internal/recovery/inplace/chainread/headsample_test.go @@ -0,0 +1,127 @@ +package chainread + +import ( + "bytes" + "errors" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/harmony-one/harmony/internal/recovery/inplace/anchor" + "github.com/harmony-one/harmony/internal/recovery/inplace/fixture" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" +) + +// headFaultCounters tracks which adapter view served the failing head-key +// reads. +type headFaultCounters struct { + latchedHeadReads int + unlatchedHeadReads int +} + +func isHeadKey(key []byte) bool { + return bytes.Equal(key, HeadHeaderKey) || bytes.Equal(key, HeadBlockKey) +} + +// headFaultReader wraps the latched rodb adapter and injects a read error +// on the head-pointer keys; its Unlatched view injects the same error but +// counts separately, so the test can pin which view sampleHeads used. +type headFaultReader struct { + kv *rodb.KV + counters *headFaultCounters +} + +func (r *headFaultReader) Get(key []byte) ([]byte, error) { + if isHeadKey(key) { + r.counters.latchedHeadReads++ + return nil, errors.New("injected head read error") + } + return r.kv.Get(key) +} + +func (r *headFaultReader) Has(key []byte) (bool, error) { + if isHeadKey(key) { + r.counters.latchedHeadReads++ + return false, errors.New("injected head read error") + } + return r.kv.Has(key) +} + +func (r *headFaultReader) Unlatched() ethdb.KeyValueReader { + return &headFaultUnlatched{inner: r.kv.Unlatched(), counters: r.counters} +} + +type headFaultUnlatched struct { + inner ethdb.KeyValueReader + counters *headFaultCounters +} + +func (r *headFaultUnlatched) Get(key []byte) ([]byte, error) { + if isHeadKey(key) { + r.counters.unlatchedHeadReads++ + return nil, errors.New("injected head read error") + } + return r.inner.Get(key) +} + +func (r *headFaultUnlatched) Has(key []byte) (bool, error) { + if isHeadKey(key) { + r.counters.unlatchedHeadReads++ + return false, errors.New("injected head read error") + } + return r.inner.Has(key) +} + +// TestHeadSampleReadErrorsDoNotGate: head-pointer read errors are strictly +// informational - the run completes, nothing reaches the shared read-error +// latch, and the sampling happened through the unlatched view. +func TestHeadSampleReadErrorsDoNotGate(t *testing.T) { + m, err := fixture.Build(filepath.Join(t.TempDir(), "harmony_db_0"), fixture.VariantBase) + if err != nil { + t.Fatalf("build fixture: %v", err) + } + db, err := rodb.Open(m.Dir, rodb.Options{}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + latch := &rodb.Latch{} + kv := db.KV(latch) + + a, err := anchor.Resolve("localnet", 0, anchor.Overrides{ + TargetHeight: fixture.TargetHeight, + TargetHash: m.TargetHash.Hex(), + }) + if err != nil { + t.Fatal(err) + } + + counters := &headFaultCounters{} + out, err := RunChecks(&headFaultReader{kv: kv, counters: counters}, a, nil) + if err != nil { + t.Fatalf("head read errors gated the run: %v", err) + } + if latch.First() != nil { + t.Fatalf("head read errors dirtied the shared latch: %v", latch.First()) + } + if counters.latchedHeadReads != 0 { + t.Fatalf("%d head reads went through the latched view, want 0", counters.latchedHeadReads) + } + if counters.unlatchedHeadReads == 0 { + t.Fatal("head sampling never touched the unlatched view") + } + if !bytes.Contains([]byte(out.Head.LastHeader), []byte("read-error")) || + !bytes.Contains([]byte(out.Head.LastBlock), []byte("read-error")) { + t.Fatalf("head sample did not record the errors: %+v", out.Head) + } + if out.Head.WalkToTarget != "not-walked: no resolvable head pointer" { + t.Fatalf("walk = %q", out.Head.WalkToTarget) + } + // The gating checks all completed against the intact fixture. + for _, id := range []string{"target_header", "body", "ancestry_to_boundary", "shard_state"} { + if out.Checks[id] != "ok" { + t.Fatalf("check %s = %q", id, out.Checks[id]) + } + } +} diff --git a/internal/recovery/inplace/chainread/keys.go b/internal/recovery/inplace/chainread/keys.go new file mode 100644 index 0000000000..c40db59211 --- /dev/null +++ b/internal/recovery/inplace/chainread/keys.go @@ -0,0 +1,64 @@ +// Package chainread provides strict raw readers over the exact rawdb key +// schema plus the minimal fail-closed engine.ChainReader used for +// certificate verification. Stock rawdb readers swallow read errors into +// "absence"; on a live DB transient errors are expected, so every reader +// here propagates errors and distinguishes them from genuine absence. +package chainread + +import ( + "encoding/binary" + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +// Exact raw keys, re-derived from core/rawdb/schema.go (the builders there +// are unexported). TestKeySchemaAgainstRawdb pins byte-equality with what +// the production rawdb writers put on disk. + +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + +// HeaderHashKey is the canonical mapping: "h" + BE64(number) + "n" -> hash. +func HeaderHashKey(number uint64) []byte { + return append(append([]byte("h"), encodeBlockNumber(number)...), 'n') +} + +// HeaderNumberKey is the reverse mapping: "H" + hash -> BE64(number). +func HeaderNumberKey(hash common.Hash) []byte { + return append([]byte("H"), hash.Bytes()...) +} + +// HeaderKey: "h" + BE64(number) + hash -> header RLP. +func HeaderKey(number uint64, hash common.Hash) []byte { + return append(append([]byte("h"), encodeBlockNumber(number)...), hash.Bytes()...) +} + +// BlockBodyKey: "b" + BE64(number) + hash -> body RLP. +func BlockBodyKey(number uint64, hash common.Hash) []byte { + return append(append([]byte("b"), encodeBlockNumber(number)...), hash.Bytes()...) +} + +// BlockCommitSigKey: "block-sig-" + BE64(number) -> 96-byte BLS aggregate +// signature followed by the bitmap. The legacy "LastCommits" fallback inside +// rawdb.ReadBlockCommitSig is deliberately not consulted: the preflight +// reads exact keys only. +func BlockCommitSigKey(number uint64) []byte { + return append([]byte("block-sig-"), encodeBlockNumber(number)...) +} + +// ShardStateKey: "ss" + epoch.Bytes() -> shard state RLP (the boundary +// header's raw ShardState() bytes, stored unmodified by all three +// production write sites). +func ShardStateKey(epoch *big.Int) []byte { + return append([]byte("ss"), epoch.Bytes()...) +} + +// HeadHeaderKey is the "LastHeader" head pointer (informational only). +var HeadHeaderKey = []byte("LastHeader") + +// HeadBlockKey is the "LastBlock" head pointer (informational only). +var HeadBlockKey = []byte("LastBlock") diff --git a/internal/recovery/inplace/chainread/keys_test.go b/internal/recovery/inplace/chainread/keys_test.go new file mode 100644 index 0000000000..6dcd4dc306 --- /dev/null +++ b/internal/recovery/inplace/chainread/keys_test.go @@ -0,0 +1,78 @@ +package chainread + +import ( + "bytes" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/harmony-one/harmony/core/rawdb" +) + +// TestKeySchemaAgainstRawdb pins the re-derived exact keys byte-for-byte +// against what the production rawdb writers put on disk. +func TestKeySchemaAgainstRawdb(t *testing.T) { + db := rawdb.NewMemoryDatabase() + hash := common.HexToHash("0x30c35d2f2291e4b27debe7862956cf7a0cc7abefc044273d6823567335086d8d") + const height = uint64(92730034) + + if err := rawdb.WriteCanonicalHash(db, hash, height); err != nil { + t.Fatal(err) + } + got, err := db.Get(HeaderHashKey(height)) + if err != nil || !bytes.Equal(got, hash.Bytes()) { + t.Fatalf("canonical key mismatch: %x %v", got, err) + } + + if err := rawdb.WriteHeaderNumber(db, hash, height); err != nil { + t.Fatal(err) + } + if _, err := db.Get(HeaderNumberKey(hash)); err != nil { + t.Fatalf("reverse-number key mismatch: %v", err) + } + + sig := []byte("aggregate-signature-and-bitmap") + if err := rawdb.WriteBlockCommitSig(db, height, sig); err != nil { + t.Fatal(err) + } + got, err = db.Get(BlockCommitSigKey(height)) + if err != nil || !bytes.Equal(got, sig) { + t.Fatalf("block-sig key mismatch: %x %v", got, err) + } + + epoch := big.NewInt(3002) + ss := []byte("shard-state-bytes") + if err := rawdb.WriteShardStateBytes(db, epoch, ss); err != nil { + t.Fatal(err) + } + got, err = db.Get(ShardStateKey(epoch)) + if err != nil || !bytes.Equal(got, ss) { + t.Fatalf("ss key mismatch: %x %v", got, err) + } + + if err := rawdb.WriteHeadHeaderHash(db, hash); err != nil { + t.Fatal(err) + } + if _, err := db.Get(HeadHeaderKey); err != nil { + t.Fatalf("LastHeader key mismatch: %v", err) + } + if err := rawdb.WriteHeadBlockHash(db, hash); err != nil { + t.Fatal(err) + } + if _, err := db.Get(HeadBlockKey); err != nil { + t.Fatalf("LastBlock key mismatch: %v", err) + } + + // Literal shapes. + if !bytes.Equal(BlockCommitSigKey(height)[:10], []byte("block-sig-")) { + t.Fatal("block-sig prefix") + } + if !bytes.Equal(ShardStateKey(epoch), append([]byte("ss"), 0x0b, 0xba)) { + t.Fatalf("ss<3002> literal: %x", ShardStateKey(epoch)) + } + wantCanonical := append(append([]byte("h"), 0, 0, 0, 0, 0x05, 0x86, 0xf2, 0xb2), 'n') // 92730034 = 0x0586F2B2 + if !bytes.Equal(HeaderHashKey(height), wantCanonical) { + t.Fatalf("canonical literal: %x want %x", HeaderHashKey(height), wantCanonical) + } +} diff --git a/internal/recovery/inplace/chainread/reader.go b/internal/recovery/inplace/chainread/reader.go new file mode 100644 index 0000000000..e23960286b --- /dev/null +++ b/internal/recovery/inplace/chainread/reader.go @@ -0,0 +1,223 @@ +package chainread + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/trie" + + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/state/snapshot" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" +) + +// UnexpectedCallError is panicked by every MinimalChainReader method outside +// the audited set. The pipeline converts it to exit code 2: an upstream +// internal/chain change that starts calling more ChainReader methods must +// fail closed, never silently return junk. +type UnexpectedCallError struct { + Method string +} + +func (e *UnexpectedCallError) Error() string { + return fmt.Sprintf("minimal ChainReader: unexpected method call %s (fail-closed; the certificate verification path changed upstream)", e.Method) +} + +// MinimalChainReader implements engine.ChainReader with exactly the state +// the certificate check needs: the chain config, the pinned target header +// (as CurrentHeader), the shard ID, and the walk-authenticated shard state +// for the target epoch. BlockChainImpl is never constructed (its open path +// calls loadLastState, which can reset or repair the chain on disk). +type MinimalChainReader struct { + config *params.ChainConfig + shardID uint32 + current *block.Header + epoch *big.Int + shardState *shard.State + + mu sync.Mutex + called map[string]int +} + +// NewMinimalChainReader builds the reader from the outcome of the chain +// checks: header is the hash-verified target header and ss the +// byte-equality-authenticated shard state for the target epoch. +func NewMinimalChainReader(config *params.ChainConfig, shardID uint32, header *block.Header, epoch *big.Int, ss *shard.State) *MinimalChainReader { + return &MinimalChainReader{ + config: config, + shardID: shardID, + current: header, + epoch: epoch, + shardState: ss, + called: make(map[string]int), + } +} + +var _ engine.ChainReader = (*MinimalChainReader)(nil) + +func (r *MinimalChainReader) record(method string) { + r.mu.Lock() + defer r.mu.Unlock() + r.called[method]++ +} + +// CalledMethods returns the set of methods exercised so far (pin test). +func (r *MinimalChainReader) CalledMethods() map[string]int { + r.mu.Lock() + defer r.mu.Unlock() + out := make(map[string]int, len(r.called)) + for k, v := range r.called { + out[k] = v + } + return out +} + +// Config is in the audited set. +func (r *MinimalChainReader) Config() *params.ChainConfig { + r.record("Config") + return r.config +} + +// CurrentHeader is in the audited set (the engine's Number()<=1 guard). +func (r *MinimalChainReader) CurrentHeader() *block.Header { + r.record("CurrentHeader") + return r.current +} + +// ShardID is in the audited set. +func (r *MinimalChainReader) ShardID() uint32 { + r.record("ShardID") + return r.shardID +} + +// ReadShardState is in the audited set; only the target epoch is served. +func (r *MinimalChainReader) ReadShardState(epoch *big.Int) (*shard.State, error) { + r.record("ReadShardState") + if epoch == nil || r.epoch.Cmp(epoch) != 0 { + return nil, fmt.Errorf("minimal ChainReader: shard state requested for epoch %s, only epoch %s is available", epoch, r.epoch) + } + return r.shardState, nil +} + +// --- everything below is outside the audited set and fails closed --- + +func (r *MinimalChainReader) unexpected(method string) { + panic(&UnexpectedCallError{Method: method}) +} + +func (r *MinimalChainReader) TrieDB() *trie.Database { + r.unexpected("TrieDB") + return nil +} + +func (r *MinimalChainReader) TrieNode(hash common.Hash) ([]byte, error) { + r.unexpected("TrieNode") + return nil, nil +} + +func (r *MinimalChainReader) ContractCode(hash common.Hash) ([]byte, error) { + r.unexpected("ContractCode") + return nil, nil +} + +func (r *MinimalChainReader) ValidatorCode(hash common.Hash) ([]byte, error) { + r.unexpected("ValidatorCode") + return nil, nil +} + +func (r *MinimalChainReader) GetReceiptsByHash(hash common.Hash) types.Receipts { + r.unexpected("GetReceiptsByHash") + return nil +} + +func (r *MinimalChainReader) GetHeader(hash common.Hash, number uint64) *block.Header { + r.unexpected("GetHeader") + return nil +} + +func (r *MinimalChainReader) GetHeaderByNumber(number uint64) *block.Header { + r.unexpected("GetHeaderByNumber") + return nil +} + +func (r *MinimalChainReader) GetHeaderByHash(hash common.Hash) *block.Header { + r.unexpected("GetHeaderByHash") + return nil +} + +func (r *MinimalChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { + r.unexpected("GetBlock") + return nil +} + +func (r *MinimalChainReader) Snapshots() *snapshot.Tree { + r.unexpected("Snapshots") + return nil +} + +func (r *MinimalChainReader) ReadValidatorList() ([]common.Address, error) { + r.unexpected("ReadValidatorList") + return nil, nil +} + +func (r *MinimalChainReader) CurrentBlock() *types.Block { + r.unexpected("CurrentBlock") + return nil +} + +func (r *MinimalChainReader) StateAt(root common.Hash) (*state.DB, error) { + r.unexpected("StateAt") + return nil, nil +} + +func (r *MinimalChainReader) ReadValidatorInformation(addr common.Address) (*staking.ValidatorWrapper, error) { + r.unexpected("ReadValidatorInformation") + return nil, nil +} + +func (r *MinimalChainReader) ReadValidatorInformationAtState(addr common.Address, state *state.DB) (*staking.ValidatorWrapper, error) { + r.unexpected("ReadValidatorInformationAtState") + return nil, nil +} + +func (r *MinimalChainReader) ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorSnapshot, error) { + r.unexpected("ReadValidatorSnapshot") + return nil, nil +} + +func (r *MinimalChainReader) ValidatorCandidates() []common.Address { + r.unexpected("ValidatorCandidates") + return nil +} + +func (r *MinimalChainReader) ReadValidatorSnapshotAtEpoch(epoch *big.Int, addr common.Address) (*staking.ValidatorSnapshot, error) { + r.unexpected("ReadValidatorSnapshotAtEpoch") + return nil, nil +} + +func (r *MinimalChainReader) ReadBlockRewardAccumulator(number uint64) (*big.Int, error) { + r.unexpected("ReadBlockRewardAccumulator") + return nil, nil +} + +func (r *MinimalChainReader) ReadValidatorStats(addr common.Address) (*staking.ValidatorStats, error) { + r.unexpected("ReadValidatorStats") + return nil, nil +} + +func (r *MinimalChainReader) SuperCommitteeForNextEpoch(beacon engine.ChainReader, header *block.Header, isVerify bool) (*shard.State, error) { + r.unexpected("SuperCommitteeForNextEpoch") + return nil, nil +} + +func (r *MinimalChainReader) ReadCommitSig(blockNum uint64) ([]byte, error) { + r.unexpected("ReadCommitSig") + return nil, nil +} diff --git a/internal/recovery/inplace/chainread/readers.go b/internal/recovery/inplace/chainread/readers.go new file mode 100644 index 0000000000..9f05680e12 --- /dev/null +++ b/internal/recovery/inplace/chainread/readers.go @@ -0,0 +1,115 @@ +package chainread + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" +) + +// Get is the strict read primitive: (value, found, error). A missing key is +// (nil, false, nil); any other error propagates and is never treated as +// absence. +func Get(db ethdb.KeyValueReader, key []byte) ([]byte, bool, error) { + val, err := db.Get(key) + if err != nil { + if rodb.IsNotFound(err) { + return nil, false, nil + } + return nil, false, err + } + return val, true, nil +} + +// ReadCanonicalHash reads the canonical hash for a height. +func ReadCanonicalHash(db ethdb.KeyValueReader, number uint64) (common.Hash, bool, error) { + raw, found, err := Get(db, HeaderHashKey(number)) + if err != nil || !found { + return common.Hash{}, found, err + } + if len(raw) != common.HashLength { + return common.Hash{}, true, fmt.Errorf("canonical mapping for height %d has %d bytes, want 32", number, len(raw)) + } + return common.BytesToHash(raw), true, nil +} + +// ReadHeaderNumber reads the reverse hash->number mapping. +func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) (uint64, bool, error) { + raw, found, err := Get(db, HeaderNumberKey(hash)) + if err != nil || !found { + return 0, found, err + } + if len(raw) != 8 { + return 0, true, fmt.Errorf("reverse number mapping for %s has %d bytes, want 8", hash.Hex(), len(raw)) + } + return binary.BigEndian.Uint64(raw), true, nil +} + +// DecodeErr marks bytes that were present but failed to decode - a data +// integrity FAIL, not a read error. +type DecodeErr struct { + What string + Err error +} + +func (e *DecodeErr) Error() string { return fmt.Sprintf("%s: undecodable: %v", e.What, e.Err) } + +func (e *DecodeErr) Unwrap() error { return e.Err } + +// ReadHeader reads and decodes the header stored under (number, hash). +// Returns (nil, true, *DecodeErr) when present but undecodable. +func ReadHeader(db ethdb.KeyValueReader, number uint64, hash common.Hash) (*block.Header, bool, error) { + raw, found, err := Get(db, HeaderKey(number, hash)) + if err != nil || !found { + return nil, found, err + } + header := new(block.Header) + if err := rlp.Decode(bytes.NewReader(raw), header); err != nil { + return nil, true, &DecodeErr{What: fmt.Sprintf("header %d %s", number, hash.Hex()), Err: err} + } + return header, true, nil +} + +// ReadBody reads and decodes the block body stored under (number, hash). +func ReadBody(db ethdb.KeyValueReader, number uint64, hash common.Hash) (*types.Body, bool, error) { + raw, found, err := Get(db, BlockBodyKey(number, hash)) + if err != nil || !found { + return nil, found, err + } + body := new(types.Body) + if err := rlp.Decode(bytes.NewReader(raw), body); err != nil { + return nil, true, &DecodeErr{What: fmt.Sprintf("body %d %s", number, hash.Hex()), Err: err} + } + return body, true, nil +} + +// ReadShardStateBytes reads the raw ss record. +func ReadShardStateBytes(db ethdb.KeyValueReader, epoch *big.Int) ([]byte, bool, error) { + return Get(db, ShardStateKey(epoch)) +} + +// ReadBlockCommitSig reads the exact block-sig- record (no legacy +// LastCommits fallback). +func ReadBlockCommitSig(db ethdb.KeyValueReader, number uint64) ([]byte, bool, error) { + return Get(db, BlockCommitSigKey(number)) +} + +// ReadHeadPointer reads a 32-byte head pointer ("LastHeader"/"LastBlock"). +func ReadHeadPointer(db ethdb.KeyValueReader, key []byte) (common.Hash, bool, error) { + raw, found, err := Get(db, key) + if err != nil || !found { + return common.Hash{}, found, err + } + if len(raw) != common.HashLength { + return common.Hash{}, true, fmt.Errorf("head pointer %q has %d bytes, want 32", string(key), len(raw)) + } + return common.BytesToHash(raw), true, nil +} diff --git a/internal/recovery/inplace/fixture/fixture.go b/internal/recovery/inplace/fixture/fixture.go new file mode 100644 index 0000000000..7790f65634 --- /dev/null +++ b/internal/recovery/inplace/fixture/fixture.go @@ -0,0 +1,755 @@ +// Package fixture builds the deterministic preflight test fixtures: a small +// shard-0 LevelDB carrying a BLS-signed header chain (with real, verifiable +// certificates), the epoch-boundary shard state, and a state trie populated +// with EOA / contract / validator / legacy-code / crafted flag-edge +// accounts. +// +// SECURITY NOTE - fixture-only secrets: the committee BLS secret keys are +// fixed small nonzero scalars (secret i = 32-byte little-endian of i+1), +// exploiting that the pinned SecretKey.SetLittleEndian performs no modular +// reduction. They exist so fixtures are byte-reproducible; they must never +// be used outside tests. +package fixture + +import ( + "fmt" + "math/big" + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + bls_core "github.com/harmony-one/bls/ffi/go/bls" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/harmony-one/harmony/block" + blockfactory "github.com/harmony-one/harmony/block/factory" + "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/types" + bls_cosi "github.com/harmony-one/harmony/crypto/bls" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/staking" + staketest "github.com/harmony-one/harmony/staking/types/test" +) + +// Variant selects a build-time state mutation (cases that cannot be created +// by post-hoc key surgery because trie hashes must remain consistent). +type Variant int + +const ( + // VariantBase is the passing fixture. + VariantBase Variant = iota + // VariantBadAccountLeaf plants an undecodable account leaf value. + VariantBadAccountLeaf + // VariantBadStorageLeaf plants a storage leaf whose value is not an RLP + // byte string. + VariantBadStorageLeaf + // VariantFlaggedEmptyCode plants a canonical IsValidator flag on an + // account with the empty code hash (walker must FAIL). + VariantFlaggedEmptyCode + // VariantManyAnomalies plants >20 decoded-zero flag accounts (anomaly + // truncation row: 20 examples, correct total/by_kind/omitted). + VariantManyAnomalies + // VariantWrapperUnbound plants a flagged account whose code is a valid + // wrapper bound to a DIFFERENT address (walker must FAIL on the + // address binding). + VariantWrapperUnbound +) + +// ManyAnomaliesCount is the number of decoded-zero accounts planted by +// VariantManyAnomalies (in addition to the base fixture's one). +const ManyAnomaliesCount = 25 + +// Chain geometry (localnet schedule, 16 blocks/epoch, epoch 3 is staking). +const ( + Epoch = 3 + BoundaryHeight = 36 // EpochLastBlock(2) + TargetHeight = 44 + ChildHeight = 45 + NumEOA = 1000 + shard0Slots = 9 // 6 harmony (nil stake) + 3 external + shard0Harmony = 6 +) + +// Manifest reports what was built, for test assertions and mutations. +type Manifest struct { + Dir string + + StateRoot common.Hash + TargetHash common.Hash + ChildHash common.Hash + Hashes map[uint64]common.Hash // height -> hash, Boundary..Child + + CertPayload []byte // aggregate signature || bitmap for the target block + + // Committee secrets in slot order (fixture-only scalars, shard 0). + Secrets []*bls_core.SecretKey + // PubKeys are the wrappers in committee slot order. + PubKeys []bls_cosi.PublicKeyWrapper + + ContractAddr common.Address // multi-node storage + c-namespace code + Contract2Addr common.Address // second contract (shares nothing) + LegacyCodeAddr common.Address // code stored ONLY at the legacy bare-hash key + ValidatorAddrs []common.Address + FlagZeroAddr common.Address // crafted decoded-zero flag leaf + FlagOddAddr common.Address // crafted non-canonical non-zero flag value + DualClassAddr common.Address // unflagged account sharing the FlagOdd wrapper code hash + FlaggedEmptyAddr common.Address // only in VariantFlaggedEmptyCode + BadLeafAddr common.Address // variant-dependent crafted account + + LegacyCodeHash common.Hash + ContractCodeHash common.Hash + ValidatorCodeHashes []common.Hash // vc-namespace wrapper code hashes (slot order) + OddWrapperCodeHash common.Hash // FlagOddAddr's (and DualClassAddr's) code hash + + committee *shard.State +} + +// addr derives a deterministic address from a label. +func addr(label string) common.Address { + return common.BytesToAddress(crypto.Keccak256([]byte("hmy-preflight-fixture/" + label))[12:]) +} + +// secretScalar builds the fixture-only secret for slot index i (little-endian +// of i+1, or base+i for other shards). +func secretScalar(n uint64) *bls_core.SecretKey { + var buf [32]byte + for i := 0; i < 8; i++ { + buf[i] = byte(n >> (8 * i)) + } + sec := &bls_core.SecretKey{} + if err := sec.SetLittleEndian(buf[:]); err != nil { + panic(fmt.Sprintf("fixture secret scalar: %v", err)) + } + return sec +} + +// Build creates the fixture database in dir (which must not exist or be +// empty) and returns the manifest. Deterministic for a given variant. +func Build(dir string, variant Variant) (*Manifest, error) { + shardingconfig.InitLocalnetConfig(16, 16) + shard.Schedule = shardingconfig.LocalnetSchedule + + db, err := rawdb.NewLevelDBDatabase(dir, 64, 128, "", false) + if err != nil { + return nil, fmt.Errorf("open fixture db: %w", err) + } + + m := &Manifest{Dir: dir, Hashes: make(map[uint64]common.Hash)} + + build := func() error { + if err := m.buildCommittee(); err != nil { + return err + } + if err := m.buildState(db, variant); err != nil { + return err + } + if err := m.buildChain(db); err != nil { + return err + } + // Flush the memtable/journal into table files: the corruption and + // relocation test rows operate on .ldb files, and real validator + // DBs hold their data in tables. + if err := db.Compact(nil, nil); err != nil { + return fmt.Errorf("compact fixture: %w", err) + } + return nil + } + if err := build(); err != nil { + db.Close() + return nil, err + } + if err := db.Close(); err != nil { + return nil, fmt.Errorf("close fixture: %w", err) + } + if err := canonicalize(dir); err != nil { + return nil, fmt.Errorf("canonicalize fixture: %w", err) + } + return m, nil +} + +// canonicalize rewrites the freshly built database into a byte-reproducible +// canonical form. LevelDB embeds per-write sequence numbers in its tables, +// so the physical bytes depend on the write ORDER, which upstream map +// iteration makes nondeterministic even for identical logical content. +// Re-inserting every key-value pair in sorted key order makes the sequence +// numbers (and hence the tables and the manifest) a pure function of the +// content; the timestamped LOG file is dropped. Two generations of the same +// variant are byte-identical afterwards. +func canonicalize(dir string) error { + src, err := leveldb.OpenFile(dir, &opt.Options{ReadOnly: true, ErrorIfMissing: true}) + if err != nil { + return fmt.Errorf("open source: %w", err) + } + tmp := dir + ".canonical" + if err := os.RemoveAll(tmp); err != nil { + src.Close() + return err + } + dst, err := leveldb.OpenFile(tmp, &opt.Options{ErrorIfExist: true}) + if err != nil { + src.Close() + return fmt.Errorf("open canonical: %w", err) + } + + it := src.NewIterator(nil, nil) + batch := new(leveldb.Batch) + n := 0 + for it.Next() { + batch.Put(append([]byte(nil), it.Key()...), append([]byte(nil), it.Value()...)) + n++ + if n%1024 == 0 { + if err := dst.Write(batch, nil); err != nil { + it.Release() + src.Close() + dst.Close() + return err + } + batch.Reset() + } + } + it.Release() + if err := it.Error(); err != nil { + src.Close() + dst.Close() + return fmt.Errorf("iterate source: %w", err) + } + if err := dst.Write(batch, nil); err != nil { + src.Close() + dst.Close() + return err + } + if err := src.Close(); err != nil { + dst.Close() + return err + } + if err := dst.CompactRange(util.Range{}); err != nil { + dst.Close() + return fmt.Errorf("compact canonical: %w", err) + } + if err := dst.Close(); err != nil { + return err + } + if err := os.Remove(filepath.Join(tmp, "LOG")); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.RemoveAll(dir); err != nil { + return err + } + return os.Rename(tmp, dir) +} + +func (m *Manifest) buildCommittee() error { + stake := numeric.NewDec(20000) + var slots0 shard.SlotList + for i := 0; i < shard0Slots; i++ { + sec := secretScalar(uint64(i + 1)) + pub := sec.GetPublicKey() + wrapper := bls_cosi.PublicKeyWrapper{Object: pub} + if err := wrapper.Bytes.FromLibBLSPublicKey(pub); err != nil { + return err + } + m.Secrets = append(m.Secrets, sec) + m.PubKeys = append(m.PubKeys, wrapper) + slot := shard.Slot{ + EcdsaAddress: addr(fmt.Sprintf("slot0-%d", i)), + BLSPublicKey: wrapper.Bytes, + } + if i >= shard0Harmony { + s := stake + slot.EffectiveStake = &s // external, stake-weighted + } + slots0 = append(slots0, slot) + } + var slots1 shard.SlotList + for i := 0; i < 3; i++ { + sec := secretScalar(uint64(101 + i)) + pub := sec.GetPublicKey() + var ser bls_cosi.SerializedPublicKey + if err := ser.FromLibBLSPublicKey(pub); err != nil { + return err + } + slots1 = append(slots1, shard.Slot{ + EcdsaAddress: addr(fmt.Sprintf("slot1-%d", i)), + BLSPublicKey: ser, + }) + } + m.committee = &shard.State{ + Epoch: big.NewInt(Epoch), + Shards: []shard.Committee{ + {ShardID: 0, Slots: slots0}, + {ShardID: 1, Slots: slots1}, + }, + } + return nil +} + +func (m *Manifest) buildState(db ethdb.Database, variant Variant) error { + sdb := state.NewDatabase(db) + st, err := state.New(common.Hash{}, sdb, nil) + if err != nil { + return err + } + + // ~1k EOAs for multi-level tries. + for i := 0; i < NumEOA; i++ { + a := addr(fmt.Sprintf("eoa-%d", i)) + st.SetBalance(a, big.NewInt(int64(i)*1_000_000_007+13)) + st.SetNonce(a, uint64(i%7)) + } + + // Contract with a multi-node storage trie and c-namespace code. + m.ContractAddr = addr("contract-1") + contractCode := deterministicCode("contract-1-code", 2048) + m.ContractCodeHash = crypto.Keccak256Hash(contractCode) + st.SetCode(m.ContractAddr, contractCode, false) + for j := 0; j < 96; j++ { + key := crypto.Keccak256Hash([]byte(fmt.Sprintf("slot-%d", j))) + val := crypto.Keccak256Hash([]byte(fmt.Sprintf("value-%d", j))) + st.SetState(m.ContractAddr, key, val) + } + m.Contract2Addr = addr("contract-2") + st.SetCode(m.Contract2Addr, deterministicCode("contract-2-code", 512), false) + st.SetState(m.Contract2Addr, common.HexToHash("0x01"), common.HexToHash("0x02")) + + // Legacy bare-hash code account: created normally, then relocated to + // the legacy location by key surgery after commit. + m.LegacyCodeAddr = addr("legacy-code") + legacyCode := deterministicCode("legacy-code-bytes", 300) + m.LegacyCodeHash = crypto.Keccak256Hash(legacyCode) + st.SetCode(m.LegacyCodeAddr, legacyCode, false) + + // Validator accounts: canonical flag + vc-namespace wrapper code. + for i := 0; i < 2; i++ { + a := addr(fmt.Sprintf("validator-%d", i)) + m.ValidatorAddrs = append(m.ValidatorAddrs, a) + w := staketest.GetDefaultValidatorWrapperWithAddr(a, []bls_cosi.SerializedPublicKey{m.PubKeys[i].Bytes}) + wBytes, err := rlp.EncodeToBytes(&w) + if err != nil { + return err + } + m.ValidatorCodeHashes = append(m.ValidatorCodeHashes, crypto.Keccak256Hash(wBytes)) + if err := st.UpdateValidatorWrapper(a, &w); err != nil { + return fmt.Errorf("update validator wrapper: %w", err) + } + st.SetValidatorFlag(a) + st.SetBalance(a, big.NewInt(1_000_000)) + } + + root, err := st.Commit(false) + if err != nil { + return err + } + if err := sdb.TrieDB().Commit(root, false); err != nil { + return err + } + + // Post-commit crafted accounts via direct trie manipulation (stock + // SetState cannot write these shapes). + root, err = m.craftAccounts(db, sdb, root, variant) + if err != nil { + return err + } + + // Legacy-code relocation surgery: move the blob from the c-namespace + // key to the bare-hash legacy key. + cKey := append([]byte("c"), m.LegacyCodeHash.Bytes()...) + blob, err := db.Get(cKey) + if err != nil { + return fmt.Errorf("read code for legacy relocation: %w", err) + } + if err := db.Put(m.LegacyCodeHash.Bytes(), blob); err != nil { + return err + } + if err := db.Delete(cKey); err != nil { + return err + } + + m.StateRoot = root + return nil +} + +// craftAccounts writes the flag-edge accounts by direct trie manipulation: +// - FlagZeroAddr: IsValidatorKey leaf whose RLP decodes to zero (0x80) - +// presence-testing would call it flagged; decode-testing keeps it +// unflagged (anomaly, passing) +// - FlagOddAddr: non-canonical non-zero flag value (0x02) - flagged + +// anomaly, wrapper code required and provided +// - variant-specific FAIL shapes +func (m *Manifest) craftAccounts(db ethdb.Database, sdb state.Database, root common.Hash, variant Variant) (common.Hash, error) { + triedb := sdb.TrieDB() + + put := func(root common.Hash, address common.Address, tweak func(st *trie.StateTrie) (common.Hash, []byte, error)) (common.Hash, error) { + addrHash := crypto.Keccak256Hash(address.Bytes()) + storageTrie, err := trie.NewStateTrie(trie.StorageTrieID(root, addrHash, common.Hash{}), triedb) + if err != nil { + return common.Hash{}, err + } + storageRoot, codeHash, err := tweak(storageTrie) + if err != nil { + return common.Hash{}, err + } + accountTrie, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) + if err != nil { + return common.Hash{}, err + } + acct := ðtypes.StateAccount{ + Nonce: 1, + Balance: big.NewInt(42), + Root: storageRoot, + CodeHash: codeHash, + } + if err := accountTrie.TryUpdateAccount(address, acct); err != nil { + return common.Hash{}, err + } + // collectLeaf=false: triedb.Update decodes collected account leaves + // for reference tracking, which the crafted shapes would trip; the + // fixture flushes to disk immediately and needs no references. + newRoot, nodes := accountTrie.Commit(false) + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return common.Hash{}, err + } + } + if err := triedb.Commit(newRoot, false); err != nil { + return common.Hash{}, err + } + return newRoot, nil + } + + commitStorage := func(st *trie.StateTrie) (common.Hash, error) { + newRoot, nodes := st.Commit(false) + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return common.Hash{}, err + } + } + // Flush the storage nodes to disk directly: with collectLeaf=false + // on the account commit there is no leaf-derived reference from the + // account trie to this storage root, so the account-root Commit + // would not reach these nodes. + if err := triedb.Commit(newRoot, false); err != nil { + return common.Hash{}, err + } + return newRoot, nil + } + + emptyCodeHash := crypto.Keccak256(nil) + + // Decoded-zero flag leaf: storage value RLP 0x80 (empty byte string). + m.FlagZeroAddr = addr("flag-decoded-zero") + var err error + root, err = put(root, m.FlagZeroAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + zeroVal, _ := rlp.EncodeToBytes([]byte{}) + if err := st.TryUpdate(staking.IsValidatorKey.Bytes(), zeroVal); err != nil { + return common.Hash{}, nil, err + } + // A second slot so the trie is not single-leaf. + other, _ := rlp.EncodeToBytes([]byte{0x33}) + if err := st.TryUpdate(common.HexToHash("0x07").Bytes(), other); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, emptyCodeHash, err + }) + if err != nil { + return common.Hash{}, err + } + + // Non-canonical non-zero flag value: flagged + anomaly; must carry a + // valid address-bound wrapper (vc namespace). + m.FlagOddAddr = addr("flag-noncanonical") + oddWrapper := staketest.GetDefaultValidatorWrapperWithAddr(m.FlagOddAddr, nil) + oddBytes, err := rlp.EncodeToBytes(&oddWrapper) + if err != nil { + return common.Hash{}, err + } + oddCodeHash := crypto.Keccak256Hash(oddBytes) + if err := db.Put(append([]byte("vc"), oddCodeHash.Bytes()...), oddBytes); err != nil { + return common.Hash{}, err + } + root, err = put(root, m.FlagOddAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + odd, _ := rlp.EncodeToBytes([]byte{0x02}) + if err := st.TryUpdate(staking.IsValidatorKey.Bytes(), odd); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, oddCodeHash.Bytes(), err + }) + if err != nil { + return common.Hash{}, err + } + m.OddWrapperCodeHash = oddCodeHash + + // Dual-class code: an unflagged account referencing the same wrapper + // code hash (contract class + wrapper-shaped anomaly + dual-class + // anomaly; all passing). + m.DualClassAddr = addr("dual-class-contract") + root, err = put(root, m.DualClassAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + benign, _ := rlp.EncodeToBytes([]byte{0x44}) + if err := st.TryUpdate(common.HexToHash("0x21").Bytes(), benign); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, oddCodeHash.Bytes(), err + }) + if err != nil { + return common.Hash{}, err + } + + switch variant { + case VariantBadAccountLeaf: + // Plant a garbage account leaf value directly in the account trie. + m.BadLeafAddr = addr("bad-account-leaf") + accountTrie, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) + if err != nil { + return common.Hash{}, err + } + if err := accountTrie.TryUpdate(m.BadLeafAddr.Bytes(), []byte{0xde, 0xad, 0xbe, 0xef}); err != nil { + return common.Hash{}, err + } + newRoot, nodes := accountTrie.Commit(false) + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return common.Hash{}, err + } + } + if err := triedb.Commit(newRoot, false); err != nil { + return common.Hash{}, err + } + root = newRoot + case VariantBadStorageLeaf: + // Storage leaf value that is not an RLP byte string (an RLP list). + m.BadLeafAddr = addr("bad-storage-leaf") + root, err = put(root, m.BadLeafAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + listVal, _ := rlp.EncodeToBytes([]interface{}{[]byte{0x01}, []byte{0x02}}) + if err := st.TryUpdate(common.HexToHash("0x11").Bytes(), listVal); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, emptyCodeHash, err + }) + if err != nil { + return common.Hash{}, err + } + case VariantFlaggedEmptyCode: + // Canonical flag, empty code hash: the walker must FAIL. + m.FlaggedEmptyAddr = addr("flagged-empty-code") + root, err = put(root, m.FlaggedEmptyAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + canonical, _ := rlp.EncodeToBytes(staking.IsValidator.Bytes()) + if err := st.TryUpdate(staking.IsValidatorKey.Bytes(), canonical); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, emptyCodeHash, err + }) + if err != nil { + return common.Hash{}, err + } + case VariantManyAnomalies: + for i := 0; i < ManyAnomaliesCount; i++ { + a := addr(fmt.Sprintf("many-anomalies-%d", i)) + root, err = put(root, a, func(st *trie.StateTrie) (common.Hash, []byte, error) { + zeroVal, _ := rlp.EncodeToBytes([]byte{}) + if err := st.TryUpdate(staking.IsValidatorKey.Bytes(), zeroVal); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, emptyCodeHash, err + }) + if err != nil { + return common.Hash{}, err + } + } + case VariantWrapperUnbound: + // Flagged account whose (hash-consistent) code is a wrapper bound + // to a different address. + m.BadLeafAddr = addr("wrapper-unbound") + foreignWrapper := staketest.GetDefaultValidatorWrapperWithAddr(addr("some-other-validator"), nil) + foreignBytes, err2 := rlp.EncodeToBytes(&foreignWrapper) + if err2 != nil { + return common.Hash{}, err2 + } + foreignHash := crypto.Keccak256Hash(foreignBytes) + if err2 := db.Put(append([]byte("vc"), foreignHash.Bytes()...), foreignBytes); err2 != nil { + return common.Hash{}, err2 + } + root, err = put(root, m.BadLeafAddr, func(st *trie.StateTrie) (common.Hash, []byte, error) { + canonical, _ := rlp.EncodeToBytes(staking.IsValidator.Bytes()) + if err := st.TryUpdate(staking.IsValidatorKey.Bytes(), canonical); err != nil { + return common.Hash{}, nil, err + } + r, err := commitStorage(st) + return r, foreignHash.Bytes(), err + }) + if err != nil { + return common.Hash{}, err + } + } + return root, nil +} + +func deterministicCode(label string, size int) []byte { + out := make([]byte, 0, size) + seed := crypto.Keccak256([]byte(label)) + for len(out) < size { + out = append(out, seed...) + seed = crypto.Keccak256(seed) + } + return out[:size] +} + +// buildChain writes headers Boundary..Child with real BLS certificates, +// canonical + reverse mappings, empty bodies, the boundary ss record, the +// exact target block-sig record, and the head pointers. +func (m *Manifest) buildChain(db ethdb.Database) error { + config := params.LocalnetChainConfig + factory := blockfactory.NewFactory(config) + + ssBytes, err := shard.EncodeWrapper(*m.committee, true) + if err != nil { + return err + } + + type built struct { + header *block.Header + hash common.Hash + } + var prev *built + for n := uint64(BoundaryHeight); n <= uint64(ChildHeight); n++ { + epoch := shardingconfig.LocalnetSchedule.CalcEpochNumber(n) + h := factory.NewHeader(epoch) + h.SetNumber(new(big.Int).SetUint64(n)) + h.SetShardID(0) + h.SetViewID(new(big.Int).SetUint64(n)) + h.SetTime(new(big.Int).SetUint64(1_700_000_000 + 2*n)) + h.SetTxHash(types.EmptyRootHash) + h.SetReceiptHash(types.EmptyRootHash) + h.SetIncomingReceiptHash(types.EmptyRootHash) + h.SetCoinbase(addr("leader")) + if n == uint64(BoundaryHeight) { + h.SetShardState(ssBytes) + } + if n == uint64(TargetHeight) { + h.SetRoot(m.StateRoot) + } + if prev != nil { + h.SetParentHash(prev.hash) + sigAndBitmap, err := m.SignPayload(prev.header) + if err != nil { + return err + } + var sig [96]byte + copy(sig[:], sigAndBitmap[:96]) + h.SetLastCommitSignature(sig) + h.SetLastCommitBitmap(sigAndBitmap[96:]) + } + hash := h.Hash() + m.Hashes[n] = hash + if err := rawdb.WriteHeader(db, h); err != nil { + return err + } + if err := rawdb.WriteCanonicalHash(db, hash, n); err != nil { + return err + } + body, err := types.NewBodyForMatchingHeader(h) + if err != nil { + return err + } + if err := rawdb.WriteBody(db, hash, n, body); err != nil { + return err + } + prev = &built{header: h, hash: hash} + } + + m.TargetHash = m.Hashes[TargetHeight] + m.ChildHash = m.Hashes[ChildHeight] + + // The boundary ss record: byte-identical to the boundary header's + // ShardState (as all production write sites store it). + if err := rawdb.WriteShardStateBytes(db, big.NewInt(Epoch), ssBytes); err != nil { + return err + } + // Exact block-sig record for the target (certificate source A; the + // child header carries the same bytes as source B). + certPayload, err := m.SignPayloadAt(m.TargetHash, TargetHeight) + if err != nil { + return err + } + m.CertPayload = certPayload + if err := rawdb.WriteBlockCommitSig(db, TargetHeight, certPayload); err != nil { + return err + } + // Head pointers at the child (upward sample lands on the target). + if err := rawdb.WriteHeadHeaderHash(db, m.ChildHash); err != nil { + return err + } + return rawdb.WriteHeadBlockHash(db, m.ChildHash) +} + +// SignPayload builds the aggregate commit signature || full bitmap for the +// given header. +func (m *Manifest) SignPayload(h *block.Header) ([]byte, error) { + return m.SignPayloadAt(h.Hash(), h.Number().Uint64()) +} + +// SignPayloadAt signs the commit payload for (hash, height) with the full +// committee (viewID = height by fixture convention). +func (m *Manifest) SignPayloadAt(hash common.Hash, height uint64) ([]byte, error) { + epoch := shardingconfig.LocalnetSchedule.CalcEpochNumber(height) + payload := signature.ConstructCommitPayload(params.LocalnetChainConfig, epoch, hash, height, height) + return m.SignRaw(payload, nil) +} + +// SignRaw signs an arbitrary payload with the committee; signers selects +// slot indices (nil = all). +func (m *Manifest) SignRaw(payload []byte, signers []int) ([]byte, error) { + if signers == nil { + signers = make([]int, len(m.Secrets)) + for i := range signers { + signers[i] = i + } + } + mask := bls_cosi.NewMask(m.PubKeys) + agg := &bls_core.Sign{} + for _, idx := range signers { + sig := m.Secrets[idx].SignHash(payload) + if sig == nil { + return nil, fmt.Errorf("bls sign failed for slot %d", idx) + } + agg.Add(sig) + if err := mask.SetBit(idx, true); err != nil { + return nil, err + } + } + out := append([]byte(nil), agg.Serialize()...) + if len(out) != 96 { + return nil, fmt.Errorf("aggregate signature is %d bytes, want 96", len(out)) + } + return append(out, mask.Bitmap...), nil +} + +// committee is kept on the manifest for signing helpers. +func (m *Manifest) Committee() *shard.State { return m.committee } + +// ValidatorWrapperBytes returns RLP of a default wrapper bound to addr +// (helper for tests crafting code blobs). +func ValidatorWrapperBytes(a common.Address) ([]byte, common.Hash, error) { + w := staketest.GetDefaultValidatorWrapperWithAddr(a, nil) + b, err := rlp.EncodeToBytes(&w) + if err != nil { + return nil, common.Hash{}, err + } + return b, crypto.Keccak256Hash(b), nil +} diff --git a/internal/recovery/inplace/fixture/fixture_test.go b/internal/recovery/inplace/fixture/fixture_test.go new file mode 100644 index 0000000000..30d37172d6 --- /dev/null +++ b/internal/recovery/inplace/fixture/fixture_test.go @@ -0,0 +1,75 @@ +package fixture_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/harmony-one/harmony/internal/recovery/inplace/fixture" +) + +// TestBuildByteReproducible: two independent generations of the same +// variant must produce byte-identical database trees (the canonical +// rewrite makes LevelDB sequence numbers a pure function of the content +// and drops the timestamped LOG). +func TestBuildByteReproducible(t *testing.T) { + dirA := filepath.Join(t.TempDir(), "harmony_db_0") + dirB := filepath.Join(t.TempDir(), "harmony_db_0") + ma, err := fixture.Build(dirA, fixture.VariantBase) + if err != nil { + t.Fatalf("build A: %v", err) + } + mb, err := fixture.Build(dirB, fixture.VariantBase) + if err != nil { + t.Fatalf("build B: %v", err) + } + if ma.TargetHash != mb.TargetHash || ma.StateRoot != mb.StateRoot { + t.Fatalf("logical content differs: %s/%s vs %s/%s", + ma.TargetHash.Hex(), ma.StateRoot.Hex(), mb.TargetHash.Hex(), mb.StateRoot.Hex()) + } + CompareTrees(t, dirA, dirB) +} + +// CompareTrees fails the test unless the two flat directories hold the same +// file names with byte-identical contents. +func CompareTrees(t *testing.T, a, b string) { + t.Helper() + filesA := readTree(t, a) + filesB := readTree(t, b) + for name := range filesB { + if _, ok := filesA[name]; !ok { + t.Errorf("file %s only in %s", name, b) + } + } + for name, data := range filesA { + got, ok := filesB[name] + if !ok { + t.Errorf("file %s only in %s", name, a) + continue + } + if !bytes.Equal(data, got) { + t.Errorf("file %s differs (%d vs %d bytes)", name, len(data), len(got)) + } + } +} + +func readTree(t *testing.T, dir string) map[string][]byte { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + files := map[string][]byte{} + for _, ent := range entries { + if ent.IsDir() { + t.Fatalf("unexpected subdirectory %s in %s", ent.Name(), dir) + } + data, err := os.ReadFile(filepath.Join(dir, ent.Name())) + if err != nil { + t.Fatal(err) + } + files[ent.Name()] = data + } + return files +} diff --git a/internal/recovery/inplace/fixture/gen/main.go b/internal/recovery/inplace/fixture/gen/main.go new file mode 100644 index 0000000000..11e0f10718 --- /dev/null +++ b/internal/recovery/inplace/fixture/gen/main.go @@ -0,0 +1,72 @@ +// Command gen materializes the deterministic preflight fixtures into +// testdata/recovery/preflight/ (invoked via +// scripts/recovery/gen-preflight-fixtures.sh). Tests build the same +// fixtures hermetically in temp dirs through the fixture package; the +// materialized copies exist for manual inspection and ad-hoc runs of the +// preflight binary against a known-good database. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/harmony-one/harmony/internal/recovery/inplace/fixture" +) + +func main() { + out := "testdata/recovery/preflight" + if len(os.Args) > 1 { + out = os.Args[1] + } + variants := map[string]fixture.Variant{ + "base": fixture.VariantBase, + "bad-account-leaf": fixture.VariantBadAccountLeaf, + "bad-storage-leaf": fixture.VariantBadStorageLeaf, + "flagged-empty-code": fixture.VariantFlaggedEmptyCode, + "many-anomalies": fixture.VariantManyAnomalies, + "wrapper-unbound": fixture.VariantWrapperUnbound, + } + type summary struct { + StateRoot string `json:"state_root"` + TargetHash string `json:"target_hash"` + ChildHash string `json:"child_hash"` + Height uint64 `json:"target_height"` + Network string `json:"network"` + } + all := map[string]summary{} + for name, v := range variants { + dir := filepath.Join(out, name, "harmony_db_0") + if err := os.RemoveAll(dir); err != nil { + fatal(err) + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + fatal(err) + } + m, err := fixture.Build(dir, v) + if err != nil { + fatal(fmt.Errorf("build %s: %w", name, err)) + } + all[name] = summary{ + StateRoot: m.StateRoot.Hex(), + TargetHash: m.TargetHash.Hex(), + ChildHash: m.ChildHash.Hex(), + Height: fixture.TargetHeight, + Network: "localnet", + } + fmt.Printf("%-20s target %s state %s -> %s\n", name, m.TargetHash.Hex(), m.StateRoot.Hex(), dir) + } + data, err := json.MarshalIndent(all, "", " ") + if err != nil { + fatal(err) + } + if err := os.WriteFile(filepath.Join(out, "fixtures.json"), append(data, '\n'), 0o644); err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "gen-preflight-fixtures:", err) + os.Exit(1) +} diff --git a/internal/recovery/inplace/fixture/mutate.go b/internal/recovery/inplace/fixture/mutate.go new file mode 100644 index 0000000000..cc74493eb0 --- /dev/null +++ b/internal/recovery/inplace/fixture/mutate.go @@ -0,0 +1,159 @@ +package fixture + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/syndtr/goleveldb/leveldb" + + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" +) + +// Mutate opens the fixture database read-write (the tool under test is +// never running at this point) and applies fn. Used to derive corruption +// variants from the pristine fixture. +func Mutate(dir string, fn func(db *leveldb.DB) error) error { + db, err := leveldb.OpenFile(dir, nil) + if err != nil { + return fmt.Errorf("open fixture for mutation: %w", err) + } + defer db.Close() + return fn(db) +} + +// DeleteKey removes one exact key. +func DeleteKey(dir string, key []byte) error { + return Mutate(dir, func(db *leveldb.DB) error { return db.Delete(key, nil) }) +} + +// PutKey writes one exact key. +func PutKey(dir string, key, value []byte) error { + return Mutate(dir, func(db *leveldb.DB) error { return db.Put(key, value, nil) }) +} + +// GetKey reads one exact key from the (stopped) fixture. +func GetKey(dir string, key []byte) ([]byte, error) { + var out []byte + err := Mutate(dir, func(db *leveldb.DB) error { + v, err := db.Get(key, nil) + if err != nil { + return err + } + out = append([]byte(nil), v...) + return nil + }) + return out, err +} + +// CopyDB copies a fixture database directory (file-level copy of a stopped +// DB) so tests can derive mutation variants without rebuilding. +func CopyDB(src, dst string) error { + entries, err := os.ReadDir(src) + if err != nil { + return err + } + if err := os.MkdirAll(dst, 0o755); err != nil { + return err + } + for _, ent := range entries { + if ent.IsDir() { + continue + } + data, err := os.ReadFile(filepath.Join(src, ent.Name())) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dst, ent.Name()), data, 0o644); err != nil { + return err + } + } + return nil +} + +// TrieNodes describes the physical trie-node keys of the fixture state, +// classified for targeted deletion/corruption. +type TrieNodes struct { + AccountRoot common.Hash + AccountInternal []common.Hash // non-root standalone nodes that are not leaf-carrying + AccountAll []common.Hash // every standalone account-trie node incl. root + + StorageRoot common.Hash // storage root node of the probe account + StorageInternal []common.Hash // non-root standalone storage nodes + StorageAll []common.Hash +} + +// EnumerateTrieNodes walks the pristine fixture read-only and returns the +// standalone node hashes of the account trie and of probeAddr's storage +// trie, in deterministic iteration order. +func EnumerateTrieNodes(dir string, stateRoot common.Hash, probeAddr common.Address) (*TrieNodes, error) { + db, err := rodb.Open(dir, rodb.Options{}) + if err != nil { + return nil, err + } + defer db.Close() + latch := &rodb.Latch{} + sdb := state.NewDatabase(rawdb.NewDatabase(db.KV(latch))) + + out := &TrieNodes{AccountRoot: stateRoot} + + accountTrie, err := sdb.OpenTrie(stateRoot) + if err != nil { + return nil, err + } + var probeStorageRoot common.Hash + probeKey := crypto.Keccak256(probeAddr.Bytes()) + it := accountTrie.NodeIterator(nil) + for it.Next(true) { + if it.Hash() != (common.Hash{}) { + out.AccountAll = append(out.AccountAll, it.Hash()) + if it.Hash() != stateRoot && !it.Leaf() { + out.AccountInternal = append(out.AccountInternal, it.Hash()) + } + } + if it.Leaf() && string(it.LeafKey()) == string(probeKey) { + var acct state.Account + if err := decodeAccount(it.LeafBlob(), &acct); err != nil { + return nil, err + } + probeStorageRoot = acct.Root + } + } + if err := it.Error(); err != nil { + return nil, err + } + if probeStorageRoot == (common.Hash{}) { + return nil, fmt.Errorf("probe account %s not found or without storage", probeAddr.Hex()) + } + out.StorageRoot = probeStorageRoot + + storageTrie, err := sdb.OpenStorageTrie(stateRoot, common.BytesToHash(probeKey), probeStorageRoot) + if err != nil { + return nil, err + } + sit := storageTrie.NodeIterator(nil) + for sit.Next(true) { + if sit.Hash() != (common.Hash{}) { + out.StorageAll = append(out.StorageAll, sit.Hash()) + if sit.Hash() != probeStorageRoot { + out.StorageInternal = append(out.StorageInternal, sit.Hash()) + } + } + } + if err := sit.Error(); err != nil { + return nil, err + } + if latch.First() != nil { + return nil, latch.First() + } + return out, nil +} + +func decodeAccount(blob []byte, acct *state.Account) error { + return rlp.DecodeBytes(blob, acct) +} diff --git a/internal/recovery/inplace/report/report.go b/internal/recovery/inplace/report/report.go new file mode 100644 index 0000000000..780cfdc249 --- /dev/null +++ b/internal/recovery/inplace/report/report.go @@ -0,0 +1,265 @@ +// Package report holds the validator-facing output contract: the PASS/FAIL +// console line, the small flat JSON receipt, and the exit codes. +package report + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// Exit codes (fixed contract). +const ( + ExitPass = 0 // all checks passed (point-in-time sample if the node was running) + ExitFail = 1 // a verification check failed (one-line reason on the console) + ExitUnusable = 2 // bad flags, missing DB, unsupported layout, not a LevelDB + ExitReadError = 3 // persistent read errors after retries +) + +// RemedyLine is printed to stderr on exit 3. +const RemedyLine = "remedy: re-run; if it keeps failing, stop the node briefly and re-run" + +// SampleNote is the point-in-time live-sample disclaimer embedded in every +// receipt. +const SampleNote = "result reflects the database at scan time; if the node was running, this is a point-in-time sample - authoritative verification happens at apply time on a stopped node" + +// Schema is the receipt schema identifier. +const Schema = "preflight-result-v2" + +// Tool is the receipt tool identifier. +const Tool = "harmony-recovery preflight" + +// Failure is a verification-check FAIL (as opposed to an I/O read error). +// The first failing check wins the console line; the receipt carries the +// rest. +type Failure struct { + Check string // stable check id, e.g. "target_header" + Reason string // one line +} + +// Failf builds a Failure. +func Failf(check, format string, args ...interface{}) *Failure { + return &Failure{Check: check, Reason: fmt.Sprintf(format, args...)} +} + +func (f *Failure) Error() string { return f.Check + ": " + f.Reason } + +// VerificationFailure marks this error as a check FAIL for the retry runner +// (rodb.IsVerificationFailure). +func (f *Failure) VerificationFailure() bool { return true } + +// Build carries informational build-stamp fields (no gating or refusal +// paths). +type Build struct { + GitDescribe string `json:"git_describe,omitempty"` + VCSRevision string `json:"vcs_revision,omitempty"` + VCSModified bool `json:"vcs_modified,omitempty"` + GoVersion string `json:"go_version,omitempty"` +} + +// Target is the verified target tuple. +type Target struct { + Height uint64 `json:"height"` + Hash string `json:"hash"` + StateRoot string `json:"state_root,omitempty"` + Epoch uint64 `json:"epoch,omitempty"` + ViewID uint64 `json:"view_id,omitempty"` +} + +// HeadSample is the informational upward sample (never gates). +type HeadSample struct { + LastHeader string `json:"last_header,omitempty"` + LastBlock string `json:"last_block,omitempty"` + WalkToTarget string `json:"walk_to_target,omitempty"` + ChildAtTargetPlus string `json:"child_at_target_plus_1,omitempty"` +} + +// CertificateSources records which certificate sources were present and +// which satisfied the check. +type CertificateSources struct { + ExactKeyPresent bool `json:"exact_key_present"` + ChildHeaderPresent bool `json:"child_header_present"` + SatisfiedBy string `json:"satisfied_by,omitempty"` +} + +// Anomaly is one bounded example entry. +type Anomaly struct { + Kind string `json:"kind"` + Detail string `json:"detail"` +} + +// Anomalies is the bounded anomaly report: full counters, first-seen +// examples, and the omitted count. Anomalies never gate. +type Anomalies struct { + Total int `json:"total"` + ByKind map[string]int `json:"by_kind,omitempty"` + Example []Anomaly `json:"examples,omitempty"` + Omitted int `json:"omitted"` +} + +// StateCounts are the state-walk counters. +type StateCounts struct { + Accounts uint64 `json:"accounts"` + AccountTrieNodes uint64 `json:"account_trie_nodes"` + StorageTries uint64 `json:"storage_tries"` + StorageTrieNodes uint64 `json:"storage_trie_nodes"` + StorageLeaves uint64 `json:"storage_leaves"` + CodeRefsContract uint64 `json:"code_refs_contract"` + CodeRefsValidator uint64 `json:"code_refs_validator"` + UniqueCodeContract uint64 `json:"unique_code_contract"` + UniqueCodeValidator uint64 `json:"unique_code_validator"` + UniqueCodeBytes uint64 `json:"unique_code_bytes"` +} + +// State is the state-walk section of the receipt. +type State struct { + Digest string `json:"digest,omitempty"` + DigestAlgorithm string `json:"digest_algorithm,omitempty"` + Counts StateCounts `json:"counts"` + Anomalies Anomalies `json:"anomalies"` +} + +// Retries reports live-race reopen activity. +type Retries struct { + ReopenCount int `json:"reopen_count"` +} + +// Receipt is the one small flat JSON file a validator attaches in Telegram. +type Receipt struct { + Tool string `json:"tool"` + Schema string `json:"schema"` + Build Build `json:"build"` + + Name string `json:"name,omitempty"` + Hostname string `json:"hostname,omitempty"` + Network string `json:"network"` + Shard uint32 `json:"shard"` + DBPath string `json:"db_path"` + + NodeProbablyRunning *bool `json:"node_probably_running,omitempty"` + SampleNote string `json:"sample_note"` + + StartedAt string `json:"started_at"` + DurationS float64 `json:"duration_s"` + Retries Retries `json:"retries"` + + Target Target `json:"target"` + + // Checks maps stable check ids to "ok" | "fail: " | "skipped". + Checks map[string]string `json:"checks"` + + HeadSample HeadSample `json:"head_sample"` + CertificateSources CertificateSources `json:"certificate_sources"` + + State State `json:"state"` + + Result string `json:"result"` // "PASS" | "FAIL" (exit_code 3 marks a read-error FAIL) + FailReason string `json:"fail_reason,omitempty"` + ExitCode int `json:"exit_code"` +} + +// CheckIDs is the stable, ordered check id list. +var CheckIDs = []string{ + "target_header", + "body", + "ancestry_to_boundary", + "shard_state", + "certificate", + "state_walk", +} + +// NewChecks returns a check map with every check "skipped". +func NewChecks() map[string]string { + m := make(map[string]string, len(CheckIDs)) + for _, id := range CheckIDs { + m[id] = "skipped" + } + return m +} + +// ValidateReportPath refuses a report path that resolves inside the DB +// directory (writing into a live LevelDB directory could confuse the node). +// Both the raw and the symlink-resolved forms are compared: the report's +// parent may not exist yet, and partially-resolvable paths must not slip +// through (e.g. /var vs /private/var on macOS). +func ValidateReportPath(reportPath, dbPath string) error { + absReport, err := filepath.Abs(reportPath) + if err != nil { + return fmt.Errorf("resolve report path: %w", err) + } + absDB, err := filepath.Abs(dbPath) + if err != nil { + return fmt.Errorf("resolve db path: %w", err) + } + within := func(base, p string) bool { + rel, err := filepath.Rel(base, p) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) + } + resolvedDB := absDB + if r, err := filepath.EvalSymlinks(absDB); err == nil { + resolvedDB = r + } + resolvedReport := absReport + // Resolve the deepest existing ancestor of the report path and rejoin + // the non-existing remainder. + dir, rest := filepath.Dir(absReport), filepath.Base(absReport) + for { + if r, err := filepath.EvalSymlinks(dir); err == nil { + resolvedReport = filepath.Join(r, rest) + break + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + rest = filepath.Join(filepath.Base(dir), rest) + dir = parent + } + if within(absDB, absReport) || within(resolvedDB, resolvedReport) { + return fmt.Errorf("report path %s resolves inside the DB directory %s; choose a path outside the database", reportPath, dbPath) + } + return nil +} + +// Write atomically writes the receipt: temp file + rename in the target +// directory. +func (r *Receipt) Write(path string) error { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".preflight-result-*.tmp") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + os.Remove(tmpName) + return err + } + return nil +} + +// FinalLine prints the exactly-one-line stdout contract for completed +// verification runs: "PASS" or "FAIL: ". +func FinalLine(stdout io.Writer, pass bool, failReason string) { + if pass { + fmt.Fprintln(stdout, "PASS") + return + } + fmt.Fprintf(stdout, "FAIL: %s\n", failReason) +} diff --git a/internal/recovery/inplace/report/report_test.go b/internal/recovery/inplace/report/report_test.go new file mode 100644 index 0000000000..e5f1da907e --- /dev/null +++ b/internal/recovery/inplace/report/report_test.go @@ -0,0 +1,92 @@ +package report + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateReportPath(t *testing.T) { + db := t.TempDir() + if err := ValidateReportPath(filepath.Join(db, "r.json"), db); err == nil { + t.Fatal("path inside the DB directory must be refused") + } + if err := ValidateReportPath(filepath.Join(db, "sub", "r.json"), db); err == nil { + t.Fatal("nested path inside the DB directory must be refused") + } + outside := filepath.Join(t.TempDir(), "r.json") + if err := ValidateReportPath(outside, db); err != nil { + t.Fatalf("outside path refused: %v", err) + } + // A symlinked parent that resolves into the DB directory is refused. + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(db, link); err == nil { + if err := ValidateReportPath(filepath.Join(link, "r.json"), db); err == nil { + t.Fatal("symlinked path into the DB directory must be refused") + } + } +} + +func TestReceiptAtomicWrite(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "receipt.json") + rec := &Receipt{ + Tool: Tool, + Schema: Schema, + Checks: NewChecks(), + Result: "PASS", + } + if err := rec.Write(path); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var back Receipt + if err := json.Unmarshal(data, &back); err != nil { + t.Fatalf("written receipt does not parse: %v", err) + } + if back.Schema != Schema { + t.Fatalf("schema %q", back.Schema) + } + // No temp files left behind. + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("stray files after atomic write: %v", entries) + } + for _, id := range CheckIDs { + if back.Checks[id] != "skipped" { + t.Fatalf("check %s = %q", id, back.Checks[id]) + } + } +} + +func TestFinalLine(t *testing.T) { + var buf bytes.Buffer + FinalLine(&buf, true, "") + if buf.String() != "PASS\n" { + t.Fatalf("pass line %q", buf.String()) + } + buf.Reset() + FinalLine(&buf, false, "target_header: gone") + if buf.String() != "FAIL: target_header: gone\n" { + t.Fatalf("fail line %q", buf.String()) + } + if strings.Count(buf.String(), "\n") != 1 { + t.Fatal("final line must be exactly one line") + } +} + +func TestFailureError(t *testing.T) { + f := Failf("body", "root mismatch %d", 42) + if f.Error() != "body: root mismatch 42" || !f.VerificationFailure() { + t.Fatalf("%v", f) + } +} diff --git a/internal/recovery/inplace/rodb/adapter.go b/internal/recovery/inplace/rodb/adapter.go new file mode 100644 index 0000000000..978960daac --- /dev/null +++ b/internal/recovery/inplace/rodb/adapter.go @@ -0,0 +1,240 @@ +package rodb + +import ( + "sync" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/util" +) + +// Latch records non-not-found read errors seen through the adapter. Stock +// rawdb readers and the trie resolver swallow read errors into "absence"; +// the latch keeps a transient I/O error distinguishable from genuine +// absence, so it surfaces as a read error (exit 3), never as a false FAIL. +type Latch struct { + mu sync.Mutex + first error + count int + writeAttempts int +} + +// Record notes a non-nil, non-not-found error. +func (l *Latch) Record(err error) { + if err == nil || IsNotFound(err) { + return + } + l.mu.Lock() + defer l.mu.Unlock() + if l.first == nil { + l.first = err + } + l.count++ +} + +func (l *Latch) recordWriteAttempt() { + l.mu.Lock() + defer l.mu.Unlock() + l.writeAttempts++ +} + +// First returns the first recorded error (nil if clean). +func (l *Latch) First() error { + l.mu.Lock() + defer l.mu.Unlock() + return l.first +} + +// Count returns the number of recorded errors. +func (l *Latch) Count() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.count +} + +// WriteAttempts returns how many times a write method was invoked (each one +// was refused; a non-zero value indicates a programming error somewhere in +// the read pipeline). +func (l *Latch) WriteAttempts() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.writeAttempts +} + +// Reset clears the latch (used when a stage is retried after reopen). +func (l *Latch) Reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.first = nil + l.count = 0 +} + +// KV is the write-refusing ethdb.KeyValueStore adapter over the read-only +// goleveldb handle (never-write layer 3). +type KV struct { + ldb *leveldb.DB + latch *Latch +} + +var _ ethdb.KeyValueStore = (*KV)(nil) + +// Get passes through with error propagation; a missing key returns +// leveldb.ErrNotFound untouched, any other error is latched. +func (kv *KV) Get(key []byte) ([]byte, error) { + dat, err := kv.ldb.Get(key, nil) + if err != nil { + kv.latch.Record(err) + return nil, err + } + return dat, nil +} + +// Has passes through with error propagation. +func (kv *KV) Has(key []byte) (bool, error) { + ret, err := kv.ldb.Has(key, nil) + if err != nil { + kv.latch.Record(err) + return false, err + } + return ret, nil +} + +// Put is refused. +func (kv *KV) Put(key []byte, value []byte) error { + kv.latch.recordWriteAttempt() + return ErrWriteRefused +} + +// Delete is refused. +func (kv *KV) Delete(key []byte) error { + kv.latch.recordWriteAttempt() + return ErrWriteRefused +} + +// Compact is refused (it is a write path). +func (kv *KV) Compact(start []byte, limit []byte) error { + kv.latch.recordWriteAttempt() + return ErrWriteRefused +} + +// Stat passes through to goleveldb's property reader. +func (kv *KV) Stat(property string) (string, error) { + return kv.ldb.GetProperty(property) +} + +// NewBatch returns a batch whose write methods are refused (the Batcher +// signature admits no error, so refusal happens on the batch's methods). +func (kv *KV) NewBatch() ethdb.Batch { + return &refusingBatch{latch: kv.latch} +} + +// NewBatchWithSize returns a write-refusing batch. +func (kv *KV) NewBatchWithSize(size int) ethdb.Batch { + return &refusingBatch{latch: kv.latch} +} + +// NewIterator wraps goleveldb's iterator over a binary-alphabetical prefix +// range, mirroring geth's leveldb wrapper semantics. +func (kv *KV) NewIterator(prefix []byte, start []byte) ethdb.Iterator { + r := util.BytesPrefix(prefix) + r.Start = append(r.Start, start...) + return &latchingIterator{it: kv.ldb.NewIterator(r, nil), latch: kv.latch} +} + +// NewSnapshot wraps goleveldb's native read-only snapshot. +func (kv *KV) NewSnapshot() (ethdb.Snapshot, error) { + snap, err := kv.ldb.GetSnapshot() + if err != nil { + kv.latch.Record(err) + return nil, err + } + return &roSnapshot{snap: snap, latch: kv.latch}, nil +} + +// Close is a no-op: the rodb.DB owner controls the database lifecycle (the +// adapter is handed to library code that must not close it). +func (kv *KV) Close() error { return nil } + +// Unlatched returns a view over the same database whose read errors are NOT +// recorded in the shared latch (writes are still refused). It backs +// strictly informational reads - the head sample - which must never gate +// the run or engage the retry machinery: a latched error from a moving head +// would send an otherwise clean run into retries and exit 3. +func (kv *KV) Unlatched() ethdb.KeyValueReader { + return &KV{ldb: kv.ldb, latch: &Latch{}} +} + +type latchingIterator struct { + it iterator.Iterator + latch *Latch +} + +func (i *latchingIterator) Next() bool { return i.it.Next() } +func (i *latchingIterator) Key() []byte { return i.it.Key() } +func (i *latchingIterator) Value() []byte { return i.it.Value() } +func (i *latchingIterator) Release() { i.err(); i.it.Release() } + +func (i *latchingIterator) Error() error { return i.err() } + +func (i *latchingIterator) err() error { + err := i.it.Error() + if err != nil { + i.latch.Record(err) + } + return err +} + +type roSnapshot struct { + snap *leveldb.Snapshot + latch *Latch +} + +func (s *roSnapshot) Has(key []byte) (bool, error) { + ok, err := s.snap.Has(key, nil) + if err != nil { + s.latch.Record(err) + } + return ok, err +} + +func (s *roSnapshot) Get(key []byte) ([]byte, error) { + dat, err := s.snap.Get(key, nil) + if err != nil { + s.latch.Record(err) + return nil, err + } + return dat, nil +} + +func (s *roSnapshot) Release() { s.snap.Release() } + +type refusingBatch struct { + latch *Latch +} + +var _ ethdb.Batch = (*refusingBatch)(nil) + +func (b *refusingBatch) Put(key, value []byte) error { + b.latch.recordWriteAttempt() + return ErrWriteRefused +} + +func (b *refusingBatch) Delete(key []byte) error { + b.latch.recordWriteAttempt() + return ErrWriteRefused +} + +func (b *refusingBatch) ValueSize() int { return 0 } + +func (b *refusingBatch) Write() error { + b.latch.recordWriteAttempt() + return ErrWriteRefused +} + +func (b *refusingBatch) Reset() {} + +func (b *refusingBatch) Replay(w ethdb.KeyValueWriter) error { + b.latch.recordWriteAttempt() + return ErrWriteRefused +} diff --git a/internal/recovery/inplace/rodb/errors.go b/internal/recovery/inplace/rodb/errors.go new file mode 100644 index 0000000000..e445b8222f --- /dev/null +++ b/internal/recovery/inplace/rodb/errors.go @@ -0,0 +1,117 @@ +// Package rodb opens a (possibly live) harmony_db_0 LevelDB strictly +// read-only, without taking the OS flock and without ever writing to the +// database directory. +// +// Never-write property, three layers: +// 1. goleveldb opened with opt.Options{ReadOnly: true, ErrorIfMissing: true} +// 2. the custom storage.Storage refuses Create/Remove/Rename/SetMeta and +// never opens, creates or probes the LOCK and LOG files +// 3. the ethdb adapter refuses Put/Delete/Compact and returns +// write-refusing batches +// +// leveldb.RecoverFile is never called (a RecoverFile against a live +// validator DB rewrites the MANIFEST on disk and corrupts the node). +package rodb + +import ( + "errors" + "fmt" + "os" + + lverrors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/storage" +) + +// ErrWriteRefused is returned by every write path of the read-only storage +// and the ethdb adapter. +var ErrWriteRefused = errors.New("rodb: write refused (recovery preflight is strictly read-only)") + +// LayoutError marks an unusable/unsupported database layout (exit code 2). +type LayoutError struct { + Reason string +} + +func (e *LayoutError) Error() string { return "unsupported database layout: " + e.Reason } + +// ReadError marks a persistent read error (exit code 3). Remedy: re-run; if +// it keeps failing, stop the node briefly and re-run. +type ReadError struct { + Err error + Detail string // e.g. the corrupt table file name + Retries int // reopen attempts consumed before giving up +} + +func (e *ReadError) Error() string { + if e.Detail != "" { + return fmt.Sprintf("persistent read error (%s): %v", e.Detail, e.Err) + } + return fmt.Sprintf("persistent read error: %v", e.Err) +} + +func (e *ReadError) Unwrap() error { return e.Err } + +// Class is the retry classification of a read error observed on a live DB. +type Class int + +const ( + // ClassNone: no error. + ClassNone Class = iota + // ClassRetryableRace: exactly the error classes a concurrent live + // writer can cause: (a) ENOENT on a referenced file (compaction deleted + // a table under our pinned manifest), and (b) ErrCorrupted attributed + // to journal or manifest files (torn .log tail during in-memory journal + // recovery, CURRENT/MANIFEST rotation mid-open). + ClassRetryableRace + // ClassCorruptTable: a checksum/content error inside an existing + // immutable SST. No live-writer race explains it; never retried. + ClassCorruptTable + // ClassPersistent: everything else (EACCES, EIO, ...); never retried. + ClassPersistent +) + +// Classify sorts an error into retry classes. The string is a short detail +// (the corrupt table name for ClassCorruptTable). +func Classify(err error) (Class, string) { + if err == nil { + return ClassNone, "" + } + if os.IsNotExist(err) { + return ClassRetryableRace, "" + } + var fd storage.FileDesc + var isCorrupted bool + var lvErr *lverrors.ErrCorrupted + var stErr *storage.ErrCorrupted + if errors.As(err, &lvErr) { + fd, isCorrupted = lvErr.Fd, true + } else if errors.As(err, &stErr) { + fd, isCorrupted = stErr.Fd, true + } + if isCorrupted { + switch fd.Type { + case storage.TypeTable: + return ClassCorruptTable, fd.String() + case storage.TypeJournal, storage.TypeManifest: + return ClassRetryableRace, fd.String() + default: + // Corruption not attributable to a specific live-file class: + // fail closed, do not retry. + return ClassPersistent, "" + } + } + return ClassPersistent, "" +} + +// IsNotFound reports whether err is a key-absence error from the underlying +// key-value store (as opposed to an I/O or corruption error). +func IsNotFound(err error) bool { + if err == nil { + return false + } + if errors.Is(err, lverrors.ErrNotFound) { + return true + } + // geth's memorydb returns a private errors.New("not found"); tolerate it + // so strict readers behave identically under test databases. + return err.Error() == "not found" +} diff --git a/internal/recovery/inplace/rodb/layout.go b/internal/recovery/inplace/rodb/layout.go new file mode 100644 index 0000000000..307da3e2a2 --- /dev/null +++ b/internal/recovery/inplace/rodb/layout.go @@ -0,0 +1,68 @@ +package rodb + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// CheckLayout gates on the simple default layout: a single-directory +// goleveldb database whose basename is exactly harmony_db_. Sharded +// multi-LevelDB (harmony_sharddb_*), pebble and TiKV layouts have no safe +// read-only open path here, and a wrong-shard or renamed directory would +// produce confusing FAILs instead of a clear refusal; all are rejected with +// a *LayoutError (exit code 2). +func CheckLayout(dir string, shardID uint32) error { + st, err := os.Stat(dir) + if err != nil { + return &LayoutError{Reason: fmt.Sprintf("cannot stat --db path: %v", err)} + } + if !st.IsDir() { + return &LayoutError{Reason: "--db path is not a directory"} + } + base := filepath.Base(filepath.Clean(dir)) + if strings.HasPrefix(base, "harmony_sharddb") { + return &LayoutError{Reason: "sharded multi-LevelDB layout (harmony_sharddb*) is not supported; only the default harmony_db_0 LevelDB is supported"} + } + + entries, err := os.ReadDir(dir) + if err != nil { + return &LayoutError{Reason: fmt.Sprintf("cannot list --db directory: %v", err)} + } + var hasCurrent, hasManifest, hasShardSubdir, hasDBSubdir bool + for _, ent := range entries { + name := ent.Name() + switch { + case strings.HasPrefix(name, "OPTIONS"): + return &LayoutError{Reason: "found OPTIONS file: this looks like a pebble database, which is not supported; only the default harmony_db_0 LevelDB is supported"} + case name == "CURRENT": + hasCurrent = true + case strings.HasPrefix(name, "MANIFEST-"): + hasManifest = true + case ent.IsDir() && strings.HasPrefix(name, "harmony_sharddb"): + hasShardSubdir = true + case ent.IsDir() && strings.HasPrefix(name, "harmony_db_"): + hasDBSubdir = true + } + } + if hasShardSubdir { + return &LayoutError{Reason: "directory contains a harmony_sharddb* database; the sharded layout is not supported"} + } + want := fmt.Sprintf("harmony_db_%d", shardID) + if base != want { + reason := fmt.Sprintf("--db must point at the node's %s directory itself (basename is %q)", want, base) + if hasDBSubdir { + reason += fmt.Sprintf("; did you mean the %s subdirectory?", want) + } + return &LayoutError{Reason: reason} + } + if !hasCurrent || !hasManifest { + reason := "not a LevelDB database directory (missing CURRENT/MANIFEST)" + if hasDBSubdir { + reason += "; did you mean to pass the harmony_db_0 subdirectory?" + } + return &LayoutError{Reason: reason} + } + return nil +} diff --git a/internal/recovery/inplace/rodb/open.go b/internal/recovery/inplace/rodb/open.go new file mode 100644 index 0000000000..4365d98577 --- /dev/null +++ b/internal/recovery/inplace/rodb/open.go @@ -0,0 +1,83 @@ +package rodb + +import ( + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +// Options tunes the read-only open. Zero values pick safe defaults. +type Options struct { + // Handles caps goleveldb's open-file cache (OpenFilesCacheCapacity). + Handles int + // BlockCacheMB caps the table block cache, in MiB. + BlockCacheMB int +} + +const ( + // DefaultHandles is the default open-file cache capacity. + DefaultHandles = 512 + // DefaultBlockCacheMB is the default block cache size in MiB. + DefaultBlockCacheMB = 256 +) + +func (o Options) withDefaults() Options { + if o.Handles <= 0 { + o.Handles = DefaultHandles + } + if o.BlockCacheMB <= 0 { + o.BlockCacheMB = DefaultBlockCacheMB + } + return o +} + +// DB is an open, strictly read-only database handle. +type DB struct { + ldb *leveldb.DB + stor *roStorage +} + +// Open opens the database directory read-only via the no-flock storage. +// It never calls leveldb.RecoverFile and never uses geth's leveldb wrapper +// (whose open path calls RecoverFile on ErrCorrupted, dropping ReadOnly and +// rewriting the MANIFEST on disk). +// +// With ReadOnly+ErrorIfMissing: a missing DB errors instead of being +// created, journal recovery is performed in memory only, obsolete-file +// cleanup and compaction are skipped, and every internal write path returns +// an error. +func Open(dir string, opts Options) (*DB, error) { + opts = opts.withDefaults() + stor := newROStorage(dir) + ldb, err := leveldb.Open(stor, &opt.Options{ + ReadOnly: true, + ErrorIfMissing: true, + OpenFilesCacheCapacity: opts.Handles, + BlockCacheCapacity: opts.BlockCacheMB * opt.MiB, + // Fail closed on manifest/journal corruption instead of goleveldb's + // default record-dropping (a silently truncated manifest would make + // missing tables look like key absence - a false FAIL). A cleanly + // torn live-journal tail still reads as graceful EOF; actual + // corruption surfaces as ErrCorrupted and is classified by Classify. + Strict: opt.DefaultStrict | opt.StrictManifest | opt.StrictJournal, + }) + if err != nil { + _ = stor.Close() + return nil, err + } + return &DB{ldb: ldb, stor: stor}, nil +} + +// KV returns the write-refusing ethdb adapter recording read errors into +// latch. +func (d *DB) KV(latch *Latch) *KV { + return &KV{ldb: d.ldb, latch: latch} +} + +// Close closes the goleveldb session and the storage. +func (d *DB) Close() error { + err := d.ldb.Close() + if cerr := d.stor.Close(); err == nil { + err = cerr + } + return err +} diff --git a/internal/recovery/inplace/rodb/probe_other.go b/internal/recovery/inplace/rodb/probe_other.go new file mode 100644 index 0000000000..b81646ac86 --- /dev/null +++ b/internal/recovery/inplace/rodb/probe_other.go @@ -0,0 +1,8 @@ +//go:build !unix + +package rodb + +// ProbeLiveWriter is unsupported on this platform; the result is unknown. +func ProbeLiveWriter(dir string) (running bool, known bool) { + return false, false +} diff --git a/internal/recovery/inplace/rodb/probe_unix.go b/internal/recovery/inplace/rodb/probe_unix.go new file mode 100644 index 0000000000..c60e60cebc --- /dev/null +++ b/internal/recovery/inplace/rodb/probe_unix.go @@ -0,0 +1,36 @@ +//go:build unix + +package rodb + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// ProbeLiveWriter best-effort detects a running node: it opens LOCK +// read-only ONLY if it already exists (never O_CREATE) and tries a +// non-blocking shared flock, which a running node's LOCK_EX blocks. The +// shared lock is released immediately and never held across reads. The +// result is informational only; probe failure is ignored. +func ProbeLiveWriter(dir string) (running bool, known bool) { + path := filepath.Join(dir, "LOCK") + if _, err := os.Lstat(path); err != nil { + // No LOCK file: nothing to probe (and nothing must be created). + return false, false + } + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return false, false + } + defer f.Close() + if err := unix.Flock(int(f.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + if err == unix.EWOULDBLOCK || err == unix.EAGAIN { + return true, true + } + return false, false + } + _ = unix.Flock(int(f.Fd()), unix.LOCK_UN) + return false, true +} diff --git a/internal/recovery/inplace/rodb/retry.go b/internal/recovery/inplace/rodb/retry.go new file mode 100644 index 0000000000..595e3c9b95 --- /dev/null +++ b/internal/recovery/inplace/rodb/retry.go @@ -0,0 +1,130 @@ +package rodb + +import ( + "errors" + "fmt" + "io" +) + +// verificationFailure is the marker interface implemented by check-failure +// errors (see report.Failure). A verification failure computed while the +// latch is clean is a genuine FAIL; one computed while reads were erroring +// is untrustworthy and follows the read-error path instead. +type verificationFailure interface { + VerificationFailure() bool +} + +// IsVerificationFailure reports whether err is a check FAIL rather than an +// I/O problem. +func IsVerificationFailure(err error) bool { + var v verificationFailure + return errors.As(err, &v) && v.VerificationFailure() +} + +// Runner opens the database and runs pipeline stages with the bounded +// reopen-and-retry policy for genuine live-file races. +type Runner struct { + Dir string + Opts Options + // MaxAttempts bounds open+run attempts per stage (default 3). + MaxAttempts int + // Progress, if set, receives human-readable retry notes (stderr). + Progress io.Writer + + // open hooks the database open (tests inject fault wrappers). + open func() (*DB, error) + + db *DB + latch *Latch + reopenCount int +} + +// NewRunner constructs a stage runner for the database directory. +func NewRunner(dir string, opts Options) *Runner { + r := &Runner{Dir: dir, Opts: opts, MaxAttempts: 3} + r.open = func() (*DB, error) { return Open(dir, opts) } + r.latch = &Latch{} + return r +} + +// SetOpenFunc overrides the database open function (test fault injection). +func (r *Runner) SetOpenFunc(open func() (*DB, error)) { r.open = open } + +// ReopenCount returns the total number of reopen attempts performed. +func (r *Runner) ReopenCount() int { return r.reopenCount } + +// Latch returns the shared read-error latch. +func (r *Runner) Latch() *Latch { return r.latch } + +// Close closes the underlying database if open. +func (r *Runner) Close() { + if r.db != nil { + _ = r.db.Close() + r.db = nil + } +} + +func (r *Runner) progressf(format string, args ...interface{}) { + if r.Progress != nil { + fmt.Fprintf(r.Progress, format+"\n", args...) + } +} + +// Stage runs fn against the adapter, classifying failures: +// +// - fn nil + clean latch: stage succeeded +// - verification failure + clean latch: genuine FAIL, returned as-is +// - retryable race (referenced-file ENOENT, journal/manifest turnover): +// close, reopen against the fresh manifest generation, retry the stage; +// bounded by MaxAttempts, exhaustion returns *ReadError +// - immutable-SST corruption: *ReadError immediately, zero retries, +// naming the corrupt table +// - anything else: *ReadError immediately +func (r *Runner) Stage(name string, fn func(kv *KV) error) error { + for attempt := 1; ; attempt++ { + if r.db == nil { + db, err := r.open() + if err != nil { + class, detail := Classify(err) + if class == ClassRetryableRace && attempt < r.MaxAttempts { + r.reopenCount++ + r.progressf("[%s] open hit a live-file race (%v); reopening (attempt %d/%d)", name, err, attempt+1, r.MaxAttempts) + continue + } + return &ReadError{Err: fmt.Errorf("open database: %w", err), Detail: detail, Retries: attempt - 1} + } + r.db = db + } + r.latch.Reset() + err := fn(r.db.KV(r.latch)) + + latched := r.latch.First() + if err == nil && latched == nil { + return nil + } + if err != nil && IsVerificationFailure(err) && latched == nil { + return err + } + // Read-error path. Prefer the latched root cause when present. + cause := err + if latched != nil { + cause = latched + } + class, detail := Classify(cause) + switch class { + case ClassRetryableRace: + if attempt < r.MaxAttempts { + r.reopenCount++ + r.progressf("[%s] read hit a live-file race (%v); reopening and retrying stage (attempt %d/%d)", name, cause, attempt+1, r.MaxAttempts) + r.Close() + continue + } + return &ReadError{Err: cause, Detail: detail, Retries: attempt - 1} + case ClassCorruptTable: + // Immutable-SST corruption: no live-writer race explains it. + return &ReadError{Err: cause, Detail: detail, Retries: 0} + default: + return &ReadError{Err: cause, Detail: detail, Retries: 0} + } + } +} diff --git a/internal/recovery/inplace/rodb/rodb_test.go b/internal/recovery/inplace/rodb/rodb_test.go new file mode 100644 index 0000000000..11325874a9 --- /dev/null +++ b/internal/recovery/inplace/rodb/rodb_test.go @@ -0,0 +1,658 @@ +package rodb + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" + lverrors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/harmony-one/harmony/internal/recovery/inplace/report" +) + +// newTestDB creates a small stopped LevelDB with deterministic content. +func newTestDB(t *testing.T, n int) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "harmony_db_0") + db, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + for i := 0; i < n; i++ { + if err := db.Put(testKey(i), testVal(i), nil); err != nil { + t.Fatal(err) + } + } + if err := db.CompactRange(util.Range{}); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + return dir +} + +func testKey(i int) []byte { return []byte(fmt.Sprintf("key-%06d", i)) } +func testVal(i int) []byte { return []byte(fmt.Sprintf("val-%06d", i)) } + +func TestLayoutGate(t *testing.T) { + t.Run("valid", func(t *testing.T) { + dir := newTestDB(t, 10) + if err := CheckLayout(dir, 0); err != nil { + t.Fatalf("valid layout refused: %v", err) + } + }) + t.Run("missing", func(t *testing.T) { + err := CheckLayout(filepath.Join(t.TempDir(), "nope"), 0) + var le *LayoutError + if !errors.As(err, &le) { + t.Fatalf("want LayoutError, got %v", err) + } + }) + t.Run("sharded-name", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "harmony_sharddb_0") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "sharddb") { + t.Fatalf("sharded layout not refused: %v", err) + } + }) + t.Run("pebble-options", func(t *testing.T) { + dir := newTestDB(t, 1) + if err := os.WriteFile(filepath.Join(dir, "OPTIONS-000003"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "pebble") { + t.Fatalf("pebble layout not refused: %v", err) + } + }) + t.Run("not-a-leveldb", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "harmony_db_0") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "CURRENT") { + t.Fatalf("non-leveldb dir not refused: %v", err) + } + }) + t.Run("wrong-shard-basename", func(t *testing.T) { + // A valid LevelDB named harmony_db_1 must be refused for shard 0 + // (wrong-shard DBs would otherwise FAIL confusingly), and accepted + // when the caller asks for shard 1. + src := newTestDB(t, 5) + dir := filepath.Join(filepath.Dir(src), "harmony_db_1") + if err := os.Rename(src, dir); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "harmony_db_0") { + t.Fatalf("wrong shard basename not refused: %v", err) + } + if err := CheckLayout(dir, 1); err != nil { + t.Fatalf("harmony_db_1 refused for shard 1: %v", err) + } + }) + t.Run("renamed-dir-refused", func(t *testing.T) { + // Even an otherwise valid LevelDB under an arbitrary basename is + // refused: --db must point at the node's harmony_db_0 itself. + src := newTestDB(t, 5) + dir := filepath.Join(filepath.Dir(src), "db-backup") + if err := os.Rename(src, dir); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "harmony_db_0") { + t.Fatalf("renamed dir not refused: %v", err) + } + }) + t.Run("hints-at-subdir", func(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "harmony_db_0"), 0o755); err != nil { + t.Fatal(err) + } + err := CheckLayout(dir, 0) + if err == nil || !strings.Contains(err.Error(), "subdirectory") { + t.Fatalf("no subdir hint: %v", err) + } + }) +} + +func TestROStorageRefusesWrites(t *testing.T) { + stor := newROStorage(newTestDB(t, 5)) + if _, err := stor.Create(storage.FileDesc{Type: storage.TypeTable, Num: 99}); err != ErrWriteRefused { + t.Fatalf("Create: %v", err) + } + if err := stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: 1}); err != ErrWriteRefused { + t.Fatalf("Remove: %v", err) + } + if err := stor.Rename(storage.FileDesc{Type: storage.TypeTable, Num: 1}, storage.FileDesc{Type: storage.TypeTable, Num: 2}); err != ErrWriteRefused { + t.Fatalf("Rename: %v", err) + } + if err := stor.SetMeta(storage.FileDesc{Type: storage.TypeManifest, Num: 9}); err != ErrWriteRefused { + t.Fatalf("SetMeta: %v", err) + } + // GetMeta and List work read-only. + fd, err := stor.GetMeta() + if err != nil || fd.Type != storage.TypeManifest { + t.Fatalf("GetMeta: %v %v", fd, err) + } + fds, err := stor.List(storage.TypeAll) + if err != nil || len(fds) == 0 { + t.Fatalf("List: %v %v", fds, err) + } + // The LOCK file is never part of the descriptor namespace. + for _, fd := range fds { + if !storage.FileDescOk(fd) { + t.Fatalf("bad descriptor %v", fd) + } + } +} + +func TestAdapterRefusesWrites(t *testing.T) { + dir := newTestDB(t, 20) + db, err := Open(dir, Options{}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + latch := &Latch{} + kv := db.KV(latch) + + if err := kv.Put([]byte("k"), []byte("v")); err != ErrWriteRefused { + t.Fatalf("Put: %v", err) + } + if err := kv.Delete([]byte("k")); err != ErrWriteRefused { + t.Fatalf("Delete: %v", err) + } + if err := kv.Compact(nil, nil); err != ErrWriteRefused { + t.Fatalf("Compact: %v", err) + } + for name, b := range map[string]interface { + Put(k, v []byte) error + Delete(k []byte) error + Write() error + }{ + "NewBatch": kv.NewBatch(), + "NewBatchWithSize": kv.NewBatchWithSize(16), + } { + if err := b.Put([]byte("k"), []byte("v")); err != ErrWriteRefused { + t.Fatalf("%s.Put: %v", name, err) + } + if err := b.Delete([]byte("k")); err != ErrWriteRefused { + t.Fatalf("%s.Delete: %v", name, err) + } + if err := b.Write(); err != ErrWriteRefused { + t.Fatalf("%s.Write: %v", name, err) + } + } + if got := latch.WriteAttempts(); got == 0 { + t.Fatal("write attempts not recorded") + } + + // Reads pass through; a missing key is not latched. + val, err := kv.Get(testKey(3)) + if err != nil || !bytes.Equal(val, testVal(3)) { + t.Fatalf("Get: %q %v", val, err) + } + if _, err := kv.Get([]byte("absent")); !IsNotFound(err) { + t.Fatalf("absent key: %v", err) + } + if latch.First() != nil { + t.Fatalf("latch dirtied by not-found: %v", latch.First()) + } + // Iterator and snapshot read paths work. + it := kv.NewIterator([]byte("key-"), nil) + count := 0 + for it.Next() { + count++ + } + it.Release() + if count != 20 || it.Error() != nil { + t.Fatalf("iterator: %d %v", count, it.Error()) + } + snap, err := kv.NewSnapshot() + if err != nil { + t.Fatal(err) + } + defer snap.Release() + if v, err := snap.Get(testKey(1)); err != nil || !bytes.Equal(v, testVal(1)) { + t.Fatalf("snapshot get: %q %v", v, err) + } +} + +func TestClassify(t *testing.T) { + cases := []struct { + name string + err error + want Class + }{ + {"nil", nil, ClassNone}, + {"enoent", &os.PathError{Op: "open", Path: "x", Err: os.ErrNotExist}, ClassRetryableRace}, + {"corrupt-journal", lverrors.NewErrCorrupted(storage.FileDesc{Type: storage.TypeJournal, Num: 3}, errors.New("torn tail")), ClassRetryableRace}, + {"corrupt-manifest", lverrors.NewErrCorrupted(storage.FileDesc{Type: storage.TypeManifest, Num: 2}, errors.New("bad record")), ClassRetryableRace}, + {"corrupt-table", lverrors.NewErrCorrupted(storage.FileDesc{Type: storage.TypeTable, Num: 7}, errors.New("checksum mismatch")), ClassCorruptTable}, + {"corrupt-unattributed", lverrors.NewErrCorrupted(storage.FileDesc{}, errors.New("mystery")), ClassPersistent}, + {"other", errors.New("permission denied"), ClassPersistent}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, detail := Classify(c.err) + if got != c.want { + t.Fatalf("Classify(%v) = %v, want %v", c.err, got, c.want) + } + if c.want == ClassCorruptTable && !strings.Contains(detail, "000007.ldb") { + t.Fatalf("corrupt table detail %q does not name the table", detail) + } + }) + } +} + +func TestRunnerRetryPolicy(t *testing.T) { + dir := newTestDB(t, 10) + + t.Run("open-race-then-success", func(t *testing.T) { + r := NewRunner(dir, Options{}) + defer r.Close() + fails := 1 + r.SetOpenFunc(func() (*DB, error) { + if fails > 0 { + fails-- + return nil, &os.PathError{Op: "open", Path: "000001.ldb", Err: os.ErrNotExist} + } + return Open(dir, Options{}) + }) + err := r.Stage("test", func(kv *KV) error { + _, err := kv.Get(testKey(1)) + return err + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + if r.ReopenCount() != 1 { + t.Fatalf("reopen count %d, want 1", r.ReopenCount()) + } + }) + + t.Run("latched-race-then-success", func(t *testing.T) { + r := NewRunner(dir, Options{}) + defer r.Close() + injected := 1 + err := r.Stage("test", func(kv *KV) error { + if injected > 0 { + injected-- + r.Latch().Record(&os.PathError{Op: "read", Path: "000002.ldb", Err: os.ErrNotExist}) + // A missing-key FAIL computed under a dirty latch must NOT + // surface as a verification failure. + return report.Failf("target_header", "spurious absence") + } + return nil + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + if r.ReopenCount() != 1 { + t.Fatalf("reopen count %d, want 1", r.ReopenCount()) + } + }) + + t.Run("verification-failure-clean-latch", func(t *testing.T) { + r := NewRunner(dir, Options{}) + defer r.Close() + err := r.Stage("test", func(kv *KV) error { + return report.Failf("target_header", "genuinely missing") + }) + var f *report.Failure + if !errors.As(err, &f) { + t.Fatalf("want Failure, got %v", err) + } + if r.ReopenCount() != 0 { + t.Fatalf("reopens on clean FAIL: %d", r.ReopenCount()) + } + }) + + t.Run("corrupt-table-zero-retries", func(t *testing.T) { + r := NewRunner(dir, Options{}) + defer r.Close() + err := r.Stage("test", func(kv *KV) error { + return lverrors.NewErrCorrupted(storage.FileDesc{Type: storage.TypeTable, Num: 5}, errors.New("checksum mismatch")) + }) + var re *ReadError + if !errors.As(err, &re) { + t.Fatalf("want ReadError, got %v", err) + } + if re.Retries != 0 || r.ReopenCount() != 0 { + t.Fatalf("corrupt table must not retry: %+v reopens=%d", re, r.ReopenCount()) + } + if !strings.Contains(re.Detail, "000005.ldb") { + t.Fatalf("detail %q does not name the table", re.Detail) + } + }) + + t.Run("persistent-race-exhausts", func(t *testing.T) { + r := NewRunner(dir, Options{}) + defer r.Close() + err := r.Stage("test", func(kv *KV) error { + return &os.PathError{Op: "read", Path: "000009.ldb", Err: os.ErrNotExist} + }) + var re *ReadError + if !errors.As(err, &re) { + t.Fatalf("want ReadError, got %v", err) + } + if re.Retries != r.MaxAttempts-1 || r.ReopenCount() != r.MaxAttempts-1 { + t.Fatalf("retries %d reopens %d, want %d", re.Retries, r.ReopenCount(), r.MaxAttempts-1) + } + }) +} + +// TestLiveWriterCoexistence: a real goleveldb writer holds LOCK_EX and keeps +// writing/compacting; the lock-free reader opens concurrently and reads +// stable keys; the writer never observes an error. +func TestLiveWriterCoexistence(t *testing.T) { + dir := newTestDB(t, 500) + wdb, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + defer wdb.Close() + + // The writer holds the flock now; probe must see it. + if running, known := ProbeLiveWriter(dir); !known || !running { + t.Fatalf("probe = running:%v known:%v, want running under writer flock", running, known) + } + + stop := make(chan struct{}) + var wg sync.WaitGroup + var writerErr error + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + if err := wdb.Put([]byte(fmt.Sprintf("live-%06d", i)), testVal(i), nil); err != nil { + writerErr = err + return + } + if i%512 == 511 { + if err := wdb.CompactRange(util.Range{}); err != nil { + writerErr = err + return + } + } + } + }() + + // Open lock-free while the writer runs and read the pre-existing keys. + deadline := time.Now().Add(5 * time.Second) + var lastErr error + ok := false + for time.Now().Before(deadline) && !ok { + func() { + db, err := Open(dir, Options{}) + if err != nil { + lastErr = err + time.Sleep(50 * time.Millisecond) + return + } + defer db.Close() + latch := &Latch{} + kv := db.KV(latch) + for i := 0; i < 500; i++ { + val, err := kv.Get(testKey(i)) + if err != nil || !bytes.Equal(val, testVal(i)) { + lastErr = fmt.Errorf("key %d: %q %w", i, val, err) + return + } + } + if latch.First() != nil { + lastErr = latch.First() + return + } + ok = true + }() + } + close(stop) + wg.Wait() + if writerErr != nil { + t.Fatalf("writer observed an error: %v", writerErr) + } + if !ok { + t.Fatalf("reader never completed cleanly against the live writer: %v", lastErr) + } + // The writer still works after our reads. + if err := wdb.Put([]byte("post"), []byte("ok"), nil); err != nil { + t.Fatalf("writer put after coexistence: %v", err) + } +} + +// TestIdleWriterDirectoryUntouched: with an idle writer holding the flock, +// a full read pass leaves the directory byte-identical (no LOCK/LOG +// creation, no manifest rewrite - the geth-wrapper RecoverFile hazard). +func TestIdleWriterDirectoryUntouched(t *testing.T) { + dir := newTestDB(t, 50) + wdb, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + defer wdb.Close() + + snapshot := func() map[string][]byte { + out := map[string][]byte{} + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, ent := range entries { + data, err := os.ReadFile(filepath.Join(dir, ent.Name())) + if err != nil { + t.Fatal(err) + } + out[ent.Name()] = data + } + return out + } + before := snapshot() + + db, err := Open(dir, Options{}) + if err != nil { + t.Fatalf("lock-free open under held flock: %v", err) + } + latch := &Latch{} + kv := db.KV(latch) + it := kv.NewIterator(nil, nil) + for it.Next() { + } + it.Release() + if err := it.Error(); err != nil { + t.Fatal(err) + } + if _, err := kv.Get(testKey(7)); err != nil { + t.Fatal(err) + } + db.Close() + + after := snapshot() + if len(before) != len(after) { + t.Fatalf("file set changed: %d -> %d", len(before), len(after)) + } + for name, data := range before { + if !bytes.Equal(after[name], data) { + t.Fatalf("file %s changed", name) + } + } + if latch.First() != nil { + t.Fatalf("latch: %v", latch.First()) + } +} + +// TestMidScanRelocation: a compaction relocates tables between stages; the +// pinned session either keeps serving (open fds) or the runner reopens - +// stage 2 must succeed either way. +func TestMidScanRelocation(t *testing.T) { + dir := newTestDB(t, 200) + r := NewRunner(dir, Options{}) + defer r.Close() + + if err := r.Stage("stage1", func(kv *KV) error { + _, err := kv.Get(testKey(0)) + return err + }); err != nil { + t.Fatalf("stage1: %v", err) + } + + // Relocate: write + compact through a second (writing) handle. + wdb, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2000; i++ { + if err := wdb.Put([]byte(fmt.Sprintf("fill-%06d", i)), bytes.Repeat([]byte{0xab}, 128), nil); err != nil { + t.Fatal(err) + } + } + if err := wdb.CompactRange(util.Range{}); err != nil { + t.Fatal(err) + } + if err := wdb.Close(); err != nil { + t.Fatal(err) + } + + if err := r.Stage("stage2", func(kv *KV) error { + for i := 0; i < 200; i++ { + if _, err := kv.Get(testKey(i)); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("stage2 after relocation: %v (reopens %d)", err, r.ReopenCount()) + } +} + +// TestRelocationReopenDeterministic pins the successful-relocation path: +// every table file vanishes mid-stage (the extreme compaction race), the +// runner reopens exactly once, and the retried stage reads every key +// correctly after the files return. +func TestRelocationReopenDeterministic(t *testing.T) { + dir := newTestDB(t, 300) + r := NewRunner(dir, Options{}) + defer r.Close() + + stash := t.TempDir() + moveTables := func(from, to string) []string { + entries, err := os.ReadDir(from) + if err != nil { + t.Fatal(err) + } + var moved []string + for _, ent := range entries { + if strings.HasSuffix(ent.Name(), ".ldb") || strings.HasSuffix(ent.Name(), ".sst") { + if err := os.Rename(filepath.Join(from, ent.Name()), filepath.Join(to, ent.Name())); err != nil { + t.Fatal(err) + } + moved = append(moved, ent.Name()) + } + } + if len(moved) == 0 { + t.Fatal("no table files to relocate") + } + return moved + } + + attempt := 0 + err := r.Stage("scan", func(kv *KV) error { + attempt++ + if attempt == 1 { + // Tables vanish before the first read of this session (the + // open-file cache holds nothing yet), producing a genuine + // ENOENT through goleveldb - the retryable race class. + moveTables(dir, stash) + if _, err := kv.Get(testKey(0)); err != nil { + return err + } + return errors.New("read unexpectedly succeeded with tables missing") + } + // Retried attempt: the files are back; every key must read + // correctly through the fresh session. + moveTables(stash, dir) + for i := 0; i < 300; i++ { + val, err := kv.Get(testKey(i)) + if err != nil { + return err + } + if !bytes.Equal(val, testVal(i)) { + return fmt.Errorf("key %d reads %q after reopen", i, val) + } + } + return nil + }) + if err != nil { + t.Fatalf("stage after relocation: %v", err) + } + if attempt != 2 { + t.Fatalf("stage ran %d times, want 2", attempt) + } + if r.ReopenCount() != 1 { + t.Fatalf("reopen count = %d, want exactly 1", r.ReopenCount()) + } +} + +// TestUnlatchedReaderScopesLatch: read errors through the Unlatched view +// must not dirty the shared latch (they back the informational head sample), +// while the same error through the primary adapter must. +func TestUnlatchedReaderScopesLatch(t *testing.T) { + dir := newTestDB(t, 5) + db, err := Open(dir, Options{}) + if err != nil { + t.Fatal(err) + } + latch := &Latch{} + kv := db.KV(latch) + unlatched := kv.Unlatched() + + // Force a real (non-not-found) read error on both views. + if err := db.Close(); err != nil { + t.Fatal(err) + } + if _, err := unlatched.Get(testKey(0)); err == nil { + t.Fatal("read on a closed DB must fail") + } + if latch.First() != nil || latch.Count() != 0 { + t.Fatalf("unlatched read dirtied the shared latch: %v", latch.First()) + } + if _, err := kv.Get(testKey(0)); err == nil { + t.Fatal("read on a closed DB must fail") + } + if latch.First() == nil { + t.Fatal("latched read did not record the error") + } + // The unlatched view still refuses writes. + if w, ok := unlatched.(interface{ Put(k, v []byte) error }); !ok { + t.Fatal("unlatched view lost its type") + } else if err := w.Put([]byte("k"), []byte("v")); err != ErrWriteRefused { + t.Fatalf("unlatched Put: %v", err) + } +} + +func TestOpenMissingDB(t *testing.T) { + _, err := Open(filepath.Join(t.TempDir(), "empty"), Options{}) + if err == nil { + t.Fatal("open of a missing DB must fail (ErrorIfMissing)") + } +} diff --git a/internal/recovery/inplace/rodb/storage.go b/internal/recovery/inplace/rodb/storage.go new file mode 100644 index 0000000000..96dd8d7345 --- /dev/null +++ b/internal/recovery/inplace/rodb/storage.go @@ -0,0 +1,206 @@ +package rodb + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/syndtr/goleveldb/leveldb/storage" +) + +// roStorage implements goleveldb's storage.Storage over an existing DB +// directory using plain file reads only. +// +// Why not storage.OpenFile(dir, readOnly=true): its constructor acquires an +// OS flock on the LOCK file (LOCK_SH for read-only openers) and O_CREATEs +// LOCK if missing. A running validator holds LOCK_EX, so a second process +// cannot open the DB that way at all - and creating LOCK would be a write +// into a live DB directory. The goleveldb session only needs the Storage +// interface's Lock() method, which is an in-process lock; the OS flock lives +// exclusively in storage.OpenFile's constructor. So this implementation: +// +// - never opens, creates, stats or flocks the LOCK file +// - never writes the LOG file (Log is a no-op) +// - returns ErrWriteRefused from Create/Remove/Rename/SetMeta +type roStorage struct { + dir string + + mu sync.Mutex + locked bool + closed bool +} + +var _ storage.Storage = (*roStorage)(nil) + +func newROStorage(dir string) *roStorage { + return &roStorage{dir: dir} +} + +type roLock struct{ s *roStorage } + +func (l *roLock) Unlock() { + l.s.mu.Lock() + defer l.s.mu.Unlock() + l.s.locked = false +} + +// Lock takes the in-process lock the goleveldb session requires. No OS-level +// lock is involved. +func (s *roStorage) Lock() (storage.Locker, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil, storage.ErrClosed + } + if s.locked { + return nil, storage.ErrLocked + } + s.locked = true + return &roLock{s: s}, nil +} + +// Log is a no-op; the file storage implementation would append to LOG. +func (s *roStorage) Log(str string) {} + +// SetMeta would rewrite CURRENT; refused. +func (s *roStorage) SetMeta(fd storage.FileDesc) error { return ErrWriteRefused } + +// GetMeta returns the manifest file descriptor named by CURRENT. +func (s *roStorage) GetMeta() (storage.FileDesc, error) { + if err := s.ok(); err != nil { + return storage.FileDesc{}, err + } + raw, err := os.ReadFile(filepath.Join(s.dir, "CURRENT")) + if err != nil { + return storage.FileDesc{}, err + } + content := strings.TrimRight(string(raw), "\r\n ") + var num int64 + if _, err := fmt.Sscanf(content, "MANIFEST-%d", &num); err != nil || num < 0 { + return storage.FileDesc{}, fmt.Errorf("rodb: malformed CURRENT content %q", content) + } + fd := storage.FileDesc{Type: storage.TypeManifest, Num: num} + if _, err := os.Stat(filepath.Join(s.dir, fsGenName(fd))); err != nil { + return storage.FileDesc{}, err + } + return fd, nil +} + +// List returns the file descriptors in the DB directory matching the type +// mask, following the file storage naming rules (LOCK, LOG, CURRENT* do not +// parse as descriptors and are naturally skipped). +func (s *roStorage) List(ft storage.FileType) ([]storage.FileDesc, error) { + if err := s.ok(); err != nil { + return nil, err + } + entries, err := os.ReadDir(s.dir) + if err != nil { + return nil, err + } + seen := make(map[storage.FileDesc]bool) + var fds []storage.FileDesc + for _, ent := range entries { + if ent.IsDir() { + continue + } + if fd, ok := fsParseName(ent.Name()); ok && fd.Type&ft != 0 && !seen[fd] { + seen[fd] = true + fds = append(fds, fd) + } + } + return fds, nil +} + +// Open opens the named file read-only with plain os.Open. +func (s *roStorage) Open(fd storage.FileDesc) (storage.Reader, error) { + if err := s.ok(); err != nil { + return nil, err + } + if !storage.FileDescOk(fd) { + return nil, storage.ErrInvalidFile + } + f, err := os.Open(filepath.Join(s.dir, fsGenName(fd))) + if os.IsNotExist(err) && fd.Type == storage.TypeTable { + // Tables written by older goleveldb use the .sst suffix. + f, err = os.Open(filepath.Join(s.dir, fsGenOldName(fd))) + } + if err != nil { + return nil, err + } + return f, nil +} + +// Create/Remove/Rename are write paths; refused. +func (s *roStorage) Create(fd storage.FileDesc) (storage.Writer, error) { + return nil, ErrWriteRefused +} + +func (s *roStorage) Remove(fd storage.FileDesc) error { return ErrWriteRefused } + +func (s *roStorage) Rename(oldfd, newfd storage.FileDesc) error { return ErrWriteRefused } + +func (s *roStorage) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} + +func (s *roStorage) ok() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return storage.ErrClosed + } + return nil +} + +// fsGenName/fsGenOldName/fsParseName mirror goleveldb's file storage naming +// rules (leveldb/storage/file_storage.go). +func fsGenName(fd storage.FileDesc) string { + switch fd.Type { + case storage.TypeManifest: + return fmt.Sprintf("MANIFEST-%06d", fd.Num) + case storage.TypeJournal: + return fmt.Sprintf("%06d.log", fd.Num) + case storage.TypeTable: + return fmt.Sprintf("%06d.ldb", fd.Num) + case storage.TypeTemp: + return fmt.Sprintf("%06d.tmp", fd.Num) + default: + panic("rodb: invalid file type") + } +} + +func fsGenOldName(fd storage.FileDesc) string { + if fd.Type == storage.TypeTable { + return fmt.Sprintf("%06d.sst", fd.Num) + } + return fsGenName(fd) +} + +func fsParseName(name string) (fd storage.FileDesc, ok bool) { + var tail string + _, err := fmt.Sscanf(name, "%d.%s", &fd.Num, &tail) + if err == nil { + switch tail { + case "log": + fd.Type = storage.TypeJournal + case "ldb", "sst": + fd.Type = storage.TypeTable + case "tmp": + fd.Type = storage.TypeTemp + default: + return storage.FileDesc{}, false + } + return fd, true + } + n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &fd.Num, &tail) + if n == 1 { + fd.Type = storage.TypeManifest + return fd, true + } + return storage.FileDesc{}, false +} diff --git a/internal/recovery/inplace/statecheck/anomaly.go b/internal/recovery/inplace/statecheck/anomaly.go new file mode 100644 index 0000000000..4f395e0bb1 --- /dev/null +++ b/internal/recovery/inplace/statecheck/anomaly.go @@ -0,0 +1,91 @@ +package statecheck + +import "github.com/harmony-one/harmony/internal/recovery/inplace/report" + +// Anomaly kinds (informational; anomalies never gate). +const ( + // AnomalyFlagDecodedZero: an IsValidatorKey storage leaf exists but its + // decoded value is the zero hash - the account is unflagged (matching + // Object.IsValidator's decode-and-test), the stray leaf is noted. + AnomalyFlagDecodedZero = "flag-decoded-zero" + // AnomalyFlagNonCanonical: the IsValidatorKey leaf decodes non-zero but + // differs from the canonical staking.IsValidator value - flagged, noted. + AnomalyFlagNonCanonical = "flag-noncanonical-value" + // AnomalyCodeMultiLocation: identical code bytes found at more than one + // physical location (c/vc/legacy) - resolved by precedence c > vc > + // legacy, noted. + AnomalyCodeMultiLocation = "code-multiple-locations" + // AnomalyWrapperShapedContract: unflagged account whose code bytes + // RLP-decode as a validator wrapper - stays contract code, noted. + AnomalyWrapperShapedContract = "wrapper-shaped-contract-code" + // AnomalyCodeDualClass: the same code hash referenced as contract code + // by one account and validator code by another - counted per class, + // noted. + AnomalyCodeDualClass = "code-dual-class" +) + +// maxAnomalyExamples bounds the receipt's example list. +const maxAnomalyExamples = 20 + +// AnomalySet keeps full per-kind counters plus the first-seen bounded +// examples, deterministic for a given database. +type AnomalySet struct { + total int + byKind map[string]int + examples []report.Anomaly +} + +// NewAnomalySet builds an empty set. +func NewAnomalySet() *AnomalySet { + return &AnomalySet{byKind: make(map[string]int)} +} + +// Add records one anomaly. +func (s *AnomalySet) Add(kind, detail string) { + s.total++ + s.byKind[kind]++ + if len(s.examples) < maxAnomalyExamples { + s.examples = append(s.examples, report.Anomaly{Kind: kind, Detail: detail}) + } +} + +// AddAll folds src into s preserving src's internal order (used for the +// ordered account fold, keeping examples first-seen deterministic under +// worker parallelism). +func (s *AnomalySet) AddAll(src *AnomalySet) { + if src == nil { + return + } + for _, ex := range src.examples { + s.Add(ex.Kind, ex.Detail) + } + // Examples beyond src's own bound still count. + for kind, n := range src.byKind { + seen := 0 + for _, ex := range src.examples { + if ex.Kind == kind { + seen++ + } + } + for i := seen; i < n; i++ { + s.total++ + s.byKind[kind]++ + } + } +} + +// Report converts to the bounded receipt form. +func (s *AnomalySet) Report() report.Anomalies { + out := report.Anomalies{ + Total: s.total, + Omitted: s.total - len(s.examples), + } + if len(s.byKind) > 0 { + out.ByKind = make(map[string]int, len(s.byKind)) + for k, v := range s.byKind { + out.ByKind[k] = v + } + } + out.Example = append(out.Example, s.examples...) + return out +} diff --git a/internal/recovery/inplace/statecheck/digest.go b/internal/recovery/inplace/statecheck/digest.go new file mode 100644 index 0000000000..5d558ef5a7 --- /dev/null +++ b/internal/recovery/inplace/statecheck/digest.go @@ -0,0 +1,117 @@ +package statecheck + +import ( + "crypto/sha256" + "encoding/binary" + "hash" + + "github.com/ethereum/go-ethereum/common" +) + +// DigestAlgorithm names the logical state digest construction below. +// +// A_i = SHA256("HMY-PF-ACCT-V1" || leafKey || BE64(nonce) || +// BE64(len(bal)) || bal || storageRoot || codeHash || +// H_storage || H_code) +// H_storage = SHA256("HMY-PF-STOR-V1" || (slotKey || BE64(len(value)) || value)*) +// in storage-trie order, or 32 zero bytes for an empty root +// H_code = SHA256("HMY-PF-CODE-V1" || BE64(len(code)) || code), +// or 32 zero bytes for the empty code hash +// digest = SHA256("HMY-PF-STATE-V1" || stateRoot || A_1 || ... || A_n) +// in account-trie order +// +// bal is the minimal big-endian big.Int.Bytes(); value is the decoded +// (logical) storage byte string, making the digest invariant across +// physical database layouts. Identical digests across validators' +// attachments give coordinators a free cross-check; the digest is an +// informational receipt field, never a gate. +const DigestAlgorithm = "preflight_state_digest_v1" + +var zeroHash32 [32]byte + +func be64(n uint64) []byte { + var b [8]byte + binary.BigEndian.PutUint64(b[:], n) + return b[:] +} + +// storageDigest accumulates H_storage in storage-trie order. +type storageDigest struct { + h hash.Hash + empty bool +} + +func newStorageDigest(emptyRoot bool) *storageDigest { + d := &storageDigest{empty: emptyRoot} + if !emptyRoot { + d.h = sha256.New() + d.h.Write([]byte("HMY-PF-STOR-V1")) + } + return d +} + +func (d *storageDigest) addLeaf(slotKey []byte, value []byte) { + d.h.Write(slotKey) + d.h.Write(be64(uint64(len(value)))) + d.h.Write(value) +} + +func (d *storageDigest) sum() [32]byte { + if d.empty { + return zeroHash32 + } + var out [32]byte + copy(out[:], d.h.Sum(nil)) + return out +} + +// codeDigest computes H_code (32 zero bytes for the empty code hash). +func codeDigest(code []byte, emptyCode bool) [32]byte { + if emptyCode { + return zeroHash32 + } + h := sha256.New() + h.Write([]byte("HMY-PF-CODE-V1")) + h.Write(be64(uint64(len(code)))) + h.Write(code) + var out [32]byte + copy(out[:], h.Sum(nil)) + return out +} + +// accountDigest computes A_i. +func accountDigest(leafKey []byte, nonce uint64, bal []byte, storageRoot common.Hash, codeHash []byte, hStorage, hCode [32]byte) [32]byte { + h := sha256.New() + h.Write([]byte("HMY-PF-ACCT-V1")) + h.Write(leafKey) + h.Write(be64(nonce)) + h.Write(be64(uint64(len(bal)))) + h.Write(bal) + h.Write(storageRoot.Bytes()) + h.Write(codeHash) + h.Write(hStorage[:]) + h.Write(hCode[:]) + var out [32]byte + copy(out[:], h.Sum(nil)) + return out +} + +// stateDigest folds A_i in account-trie order. +type stateDigest struct { + h hash.Hash +} + +func newStateDigest(stateRoot common.Hash) *stateDigest { + d := &stateDigest{h: sha256.New()} + d.h.Write([]byte("HMY-PF-STATE-V1")) + d.h.Write(stateRoot.Bytes()) + return d +} + +func (d *stateDigest) addAccount(a [32]byte) { d.h.Write(a[:]) } + +func (d *stateDigest) sum() [32]byte { + var out [32]byte + copy(out[:], d.h.Sum(nil)) + return out +} diff --git a/internal/recovery/inplace/statecheck/walker.go b/internal/recovery/inplace/statecheck/walker.go new file mode 100644 index 0000000000..1eed182925 --- /dev/null +++ b/internal/recovery/inplace/statecheck/walker.go @@ -0,0 +1,535 @@ +// Package statecheck walks the complete target state: every account, every +// storage trie, every code blob, with every standalone trie node +// keccak-authenticated against its hash. +// +// It is a self-contained walker independent of core/state/iterator.go, which +// has two confirmed defects this package must not inherit: +// +// 1. iterator.go:123-126 returns when ContractCode errors, making the +// ValidatorCode fallback unreachable for missing code (validator +// wrappers are stored under the "vc" namespace) - the stock iterator +// hard-fails on every validator account. The walker probes the three +// physical code namespaces (c, vc, legacy bare hash) with raw reads. +// 2. iterator.go:116-119 drops the error of the initial storage-iterator +// step - a storage trie with a missing root is silently treated as +// empty. The walker checks Error() after every Next, including the +// first. +package statecheck + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/big" + "runtime" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" + "github.com/harmony-one/harmony/staking" + staketypes "github.com/harmony-one/harmony/staking/types" +) + +var ( + emptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + emptyCodeHash = crypto.Keccak256Hash(nil) +) + +// Config configures the walk. +type Config struct { + KV ethdb.KeyValueStore + StateRoot common.Hash + TrieCacheMB int + // Workers bounds the storage/code worker pool; results are folded in + // account order so digests, counts and anomaly examples are + // scheduling-independent. Default min(8, NumCPU). + Workers int + Progress io.Writer +} + +// Result is the outcome of a successful walk. +type Result struct { + Counts report.StateCounts + Digest [32]byte + Anomalies *AnomalySet +} + +// Walk runs the completeness walk. The returned error is a *report.Failure +// for integrity FAILs (missing node, authentication mismatch, decode +// failure, classification violation); other errors are read errors for the +// retry runner (transient I/O swallowed by the trie layer is rescued by the +// rodb latch). +func Walk(cfg Config) (*Result, error) { + workers := cfg.Workers + if workers <= 0 { + workers = runtime.NumCPU() + if workers > 8 { + workers = 8 + } + } + trieCfg := &trie.Config{Cache: cfg.TrieCacheMB} + sdb := state.NewDatabaseWithConfig(rawdb.NewDatabase(cfg.KV), trieCfg) + + accountTrie, err := sdb.OpenTrie(cfg.StateRoot) + if err != nil { + return nil, trieFailure("account trie root", cfg.StateRoot, err) + } + + w := &walker{cfg: cfg, sdb: sdb, workers: workers} + return w.run(accountTrie) +} + +// trieFailure converts trie-layer errors into named FAILs. Underlying +// transient I/O does not surface here (the trie layer swallows read errors +// into node absence); the rodb latch records it and the retry runner +// prefers the latched cause over this failure. +func trieFailure(where string, root common.Hash, err error) error { + var missing *trie.MissingNodeError + if errors.As(err, &missing) { + return report.Failf("state_walk", "%s (root %s): missing trie node %s at path %x", where, root.Hex(), missing.NodeHash.Hex(), missing.Path) + } + return report.Failf("state_walk", "%s (root %s): %v", where, root.Hex(), err) +} + +type accountJob struct { + index uint64 + leafKey []byte // 32-byte hashed address (copied) + leafBlob []byte // account RLP (copied) +} + +type codeRef struct { + class string // "contract" | "validator" + hash common.Hash + size uint64 +} + +type accountResult struct { + index uint64 + failure error // *report.Failure or read error + digest [32]byte + code *codeRef + counts report.StateCounts + anomalies *AnomalySet +} + +type walker struct { + cfg Config + sdb state.Database + workers int + + cancel atomic.Bool +} + +func (w *walker) run(accountTrie state.Trie) (*Result, error) { + jobs := make(chan accountJob, w.workers*2) + results := make(chan accountResult, w.workers*2) + + var wg sync.WaitGroup + for i := 0; i < w.workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for job := range jobs { + if w.cancel.Load() { + results <- accountResult{index: job.index, failure: errCanceled} + continue + } + results <- w.processAccount(job) + } + }() + } + + collector := newCollector(w, results) + + // Sequential account-trie pass: authenticate every standalone node, + // dispatch account leaves to the pool in trie order. + var ( + accountTrieNodes uint64 + accounts uint64 + trieFail error + nextIndex uint64 + ) + it := accountTrie.NodeIterator(nil) + for it.Next(true) { + if w.cancel.Load() { + break + } + if it.Hash() != (common.Hash{}) { + accountTrieNodes++ + blob := it.NodeBlob() + if blob == nil { + // resolveBlob failed; surfaced via it.Error() below. + break + } + if got := crypto.Keccak256Hash(blob); got != it.Hash() { + trieFail = report.Failf("state_walk", + "account trie node %s at path %x fails content authentication (blob hashes to %s)", + it.Hash().Hex(), it.Path(), got.Hex()) + break + } + } + if it.Leaf() { + job := accountJob{ + index: nextIndex, + leafKey: append([]byte(nil), it.LeafKey()...), + leafBlob: append([]byte(nil), it.LeafBlob()...), + } + nextIndex++ + accounts++ + jobs <- job + if w.cfg.Progress != nil && accounts%200000 == 0 { + fmt.Fprintf(w.cfg.Progress, "state walk: %d accounts dispatched, %d account-trie nodes authenticated\n", accounts, accountTrieNodes) + } + } + } + if trieFail == nil { + if err := it.Error(); err != nil { + trieFail = trieFailure("account trie walk", w.cfg.StateRoot, err) + } + } + close(jobs) + wg.Wait() + close(results) + collector.wait() + + // Precedence: the earliest failure in walk order wins. Account leaves + // dispatched before the trie-level failure position precede it. + if collector.failure != nil && !errors.Is(collector.failure, errCanceled) { + return nil, collector.failure + } + if trieFail != nil { + return nil, trieFail + } + if collector.failure != nil { + return nil, collector.failure + } + + res := collector.result + res.Counts.Accounts = accounts + res.Counts.AccountTrieNodes = accountTrieNodes + res.Digest = collector.digest.sum() + if w.cfg.Progress != nil { + fmt.Fprintf(w.cfg.Progress, "state walk: complete - %d accounts, %d account-trie nodes, %d storage tries, %d storage nodes, %d storage leaves, %d+%d unique code blobs\n", + res.Counts.Accounts, res.Counts.AccountTrieNodes, res.Counts.StorageTries, + res.Counts.StorageTrieNodes, res.Counts.StorageLeaves, + res.Counts.UniqueCodeContract, res.Counts.UniqueCodeValidator) + } + return &res, nil +} + +var errCanceled = errors.New("statecheck: canceled after earlier failure") + +// collector folds worker results in account order, keeping the digest, +// counts and anomaly examples deterministic regardless of scheduling. +type collector struct { + w *walker + digest *stateDigest + result Result + failure error + done chan struct{} + + uniqueCode map[codeRef]struct{} // (class,hash) pairs seen + classesFor map[common.Hash][2]bool // hash -> [contract, validator] +} + +func newCollector(w *walker, results <-chan accountResult) *collector { + c := &collector{ + w: w, + digest: newStateDigest(w.cfg.StateRoot), + done: make(chan struct{}), + uniqueCode: make(map[codeRef]struct{}), + classesFor: make(map[common.Hash][2]bool), + } + c.result.Anomalies = NewAnomalySet() + go c.loop(results) + return c +} + +func (c *collector) wait() { <-c.done } + +func (c *collector) loop(results <-chan accountResult) { + defer close(c.done) + pending := make(map[uint64]accountResult) + next := uint64(0) + for r := range results { + pending[r.index] = r + for { + rr, ok := pending[next] + if !ok { + break + } + delete(pending, next) + next++ + c.fold(rr) + } + } +} + +func (c *collector) fold(r accountResult) { + if c.failure != nil { + return // draining after the first ordered failure + } + if r.failure != nil { + c.failure = r.failure + c.w.cancel.Store(true) + return + } + c.digest.addAccount(r.digest) + c.result.Counts.StorageTries += r.counts.StorageTries + c.result.Counts.StorageTrieNodes += r.counts.StorageTrieNodes + c.result.Counts.StorageLeaves += r.counts.StorageLeaves + c.result.Anomalies.AddAll(r.anomalies) + if r.code != nil { + key := *r.code + if r.code.class == "validator" { + c.result.Counts.CodeRefsValidator++ + } else { + c.result.Counts.CodeRefsContract++ + } + if _, seen := c.uniqueCode[key]; !seen { + c.uniqueCode[key] = struct{}{} + c.result.Counts.UniqueCodeBytes += r.code.size + if r.code.class == "validator" { + c.result.Counts.UniqueCodeValidator++ + } else { + c.result.Counts.UniqueCodeContract++ + } + classes := c.classesFor[r.code.hash] + if r.code.class == "validator" { + classes[1] = true + } else { + classes[0] = true + } + c.classesFor[r.code.hash] = classes + if classes[0] && classes[1] { + c.result.Anomalies.Add(AnomalyCodeDualClass, + fmt.Sprintf("code hash %s referenced as both contract and validator code", r.code.hash.Hex())) + } + } + } +} + +// processAccount runs the per-account checks: flag classification, full +// storage-trie walk, code resolution and validation, digest contribution. +func (w *walker) processAccount(job accountJob) accountResult { + res := accountResult{index: job.index, anomalies: NewAnomalySet()} + leafKeyHex := common.BytesToHash(job.leafKey).Hex() + + var acct state.Account + if err := rlp.DecodeBytes(job.leafBlob, &acct); err != nil { + res.failure = report.Failf("state_walk", "account leaf %s does not decode: %v", leafKeyHex, err) + return res + } + if acct.Balance == nil { + acct.Balance = new(big.Int) + } + hasStorage := acct.Root != emptyRootHash + emptyCode := bytes.Equal(acct.CodeHash, emptyCodeHash.Bytes()) + addrHash := common.BytesToHash(job.leafKey) + + // Validator flag - for every account, independent of code presence. + // Empty-root accounts are trivially unflagged. The leaf value is + // decoded, not presence-tested, matching Object.IsValidator. + flagged := false + var storageTrie state.Trie + if hasStorage { + var err error + storageTrie, err = w.sdb.OpenStorageTrie(w.cfg.StateRoot, addrHash, acct.Root) + if err != nil { + res.failure = trieFailure(fmt.Sprintf("storage trie open for account %s", leafKeyHex), acct.Root, err) + return res + } + raw, err := storageTrie.TryGet(staking.IsValidatorKey.Bytes()) + if err != nil { + res.failure = trieFailure(fmt.Sprintf("IsValidator flag lookup for account %s", leafKeyHex), acct.Root, err) + return res + } + if len(raw) > 0 { + _, content, _, err := rlp.Split(raw) + if err != nil { + res.failure = report.Failf("state_walk", "account %s IsValidator flag leaf is not an RLP byte string: %v", leafKeyHex, err) + return res + } + value := common.BytesToHash(content) + switch { + case value == (common.Hash{}): + // Stock SetState deletes zero-valued slots; a leaf whose RLP + // decodes to zero is unflagged (decode-and-test) + anomaly. + res.anomalies.Add(AnomalyFlagDecodedZero, + fmt.Sprintf("account %s has an IsValidator flag leaf decoding to zero", leafKeyHex)) + case value == staking.IsValidator: + flagged = true + default: + flagged = true + res.anomalies.Add(AnomalyFlagNonCanonical, + fmt.Sprintf("account %s IsValidator flag value %s differs from canonical", leafKeyHex, value.Hex())) + } + } + } + + // Full storage walk with node content authentication. + hStorage := newStorageDigest(!hasStorage) + if hasStorage { + res.counts.StorageTries++ + if failure := w.walkStorage(storageTrie, acct.Root, leafKeyHex, hStorage, &res); failure != nil { + res.failure = failure + return res + } + } + + // Code across the three namespaces. + var hCode [32]byte + if emptyCode { + if flagged { + res.failure = report.Failf("state_walk", "flagged validator account %s has empty code hash", leafKeyHex) + return res + } + hCode = codeDigest(nil, true) + } else { + codeHash := common.BytesToHash(acct.CodeHash) + code, class, failure := w.resolveCode(codeHash, flagged, job.leafKey, leafKeyHex, &res) + if failure != nil { + res.failure = failure + return res + } + hCode = codeDigest(code, false) + res.code = &codeRef{class: class, hash: codeHash, size: uint64(len(code))} + } + + res.digest = accountDigest(job.leafKey, acct.Nonce, acct.Balance.Bytes(), acct.Root, acct.CodeHash, hStorage.sum(), hCode) + return res +} + +// walkStorage iterates the full storage trie, authenticating every +// standalone node and folding leaves into H_storage in trie order. The +// error status of every Next step is checked, including the very first +// (the stock iterator's defect-2 silently treats a storage trie whose +// initial step fails as empty). +func (w *walker) walkStorage(st state.Trie, root common.Hash, leafKeyHex string, h *storageDigest, res *accountResult) error { + sit := st.NodeIterator(nil) + for sit.Next(true) { + if sit.Hash() != (common.Hash{}) { + res.counts.StorageTrieNodes++ + blob := sit.NodeBlob() + if blob == nil { + break // surfaced via sit.Error() + } + if got := crypto.Keccak256Hash(blob); got != sit.Hash() { + return report.Failf("state_walk", + "storage trie node %s (account %s, path %x) fails content authentication (blob hashes to %s)", + sit.Hash().Hex(), leafKeyHex, sit.Path(), got.Hex()) + } + } + if sit.Leaf() { + blob := sit.LeafBlob() + if len(blob) == 0 { + return report.Failf("state_walk", "storage leaf %x of account %s has an empty value", sit.LeafKey(), leafKeyHex) + } + // Byte and String kinds are both byte strings in RLP (values + // 0x00-0x7f encode as a single byte); lists are not. + kind, content, rest, err := rlp.Split(blob) + if err != nil || (kind != rlp.String && kind != rlp.Byte) || len(rest) != 0 { + return report.Failf("state_walk", "storage leaf %x of account %s is not an RLP byte string", sit.LeafKey(), leafKeyHex) + } + h.addLeaf(append([]byte(nil), sit.LeafKey()...), content) + res.counts.StorageLeaves++ + } + } + if err := sit.Error(); err != nil { + return trieFailure(fmt.Sprintf("storage trie walk for account %s", leafKeyHex), root, err) + } + return nil +} + +// resolveCode probes the raw code keys in order c -> vc -> legacy bare hash +// (physical location does not determine class), requires exactly one +// resolved location (identical bytes at multiple locations is an anomaly +// resolved by precedence; differing bytes FAIL), keccak-authenticates the +// bytes, and classifies from the account's flag: flagged accounts must +// carry a valid address-bound validator wrapper. +func (w *walker) resolveCode(codeHash common.Hash, flagged bool, leafKey []byte, leafKeyHex string, res *accountResult) ([]byte, string, error) { + type loc struct { + name string + key []byte + } + locs := []loc{ + {"c", append([]byte("c"), codeHash.Bytes()...)}, + {"vc", append([]byte("vc"), codeHash.Bytes()...)}, + {"legacy", codeHash.Bytes()}, + } + var ( + found []string + code []byte + ) + for _, l := range locs { + val, ok, err := strictGet(w.cfg.KV, l.key) + if err != nil { + return nil, "", err + } + if !ok { + continue + } + if code == nil { + code = val + } else if !bytes.Equal(code, val) { + return nil, "", report.Failf("state_walk", + "code hash %s resolves to DIFFERENT bytes at locations %v and %s (account %s)", + codeHash.Hex(), found, l.name, leafKeyHex) + } + found = append(found, l.name) + } + if len(found) == 0 { + return nil, "", report.Failf("state_walk", + "code %s for account %s missing from all namespaces (c, vc, legacy)", codeHash.Hex(), leafKeyHex) + } + if len(found) > 1 { + res.anomalies.Add(AnomalyCodeMultiLocation, + fmt.Sprintf("code %s present at %v (identical bytes); precedence %s", codeHash.Hex(), found, found[0])) + } + if got := crypto.Keccak256Hash(code); got != codeHash { + return nil, "", report.Failf("state_walk", + "code at %v for account %s hashes to %s, want %s", found, leafKeyHex, got.Hex(), codeHash.Hex()) + } + + var wrapper staketypes.ValidatorWrapper + wrapperErr := rlp.DecodeBytes(code, &wrapper) + if flagged { + if wrapperErr != nil { + return nil, "", report.Failf("state_walk", + "flagged validator account %s: code %s does not decode as a validator wrapper: %v", + leafKeyHex, codeHash.Hex(), wrapperErr) + } + if crypto.Keccak256Hash(wrapper.Address.Bytes()) != common.BytesToHash(leafKey) { + return nil, "", report.Failf("state_walk", + "flagged validator account %s: wrapper address %s does not bind to the account leaf key", + leafKeyHex, wrapper.Address.Hex()) + } + return code, "validator", nil + } + if wrapperErr == nil { + res.anomalies.Add(AnomalyWrapperShapedContract, + fmt.Sprintf("unflagged account %s carries wrapper-shaped code %s (stays contract)", leafKeyHex, codeHash.Hex())) + } + return code, "contract", nil +} + +func strictGet(kv ethdb.KeyValueReader, key []byte) ([]byte, bool, error) { + val, err := kv.Get(key) + if err != nil { + if rodb.IsNotFound(err) { + return nil, false, nil + } + return nil, false, err + } + return val, true, nil +} diff --git a/internal/recovery/inplace/statecheck/walker_test.go b/internal/recovery/inplace/statecheck/walker_test.go new file mode 100644 index 0000000000..0e471ee163 --- /dev/null +++ b/internal/recovery/inplace/statecheck/walker_test.go @@ -0,0 +1,341 @@ +package statecheck_test + +import ( + "encoding/hex" + "fmt" + "math/big" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/syndtr/goleveldb/leveldb" + + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state" + bls "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" + "github.com/harmony-one/harmony/internal/recovery/inplace/rodb" + "github.com/harmony-one/harmony/internal/recovery/inplace/statecheck" + staketest "github.com/harmony-one/harmony/staking/types/test" +) + +// buildToyState writes the 3-account toy state (EOA, contract with storage, +// vc-namespace validator) and returns its root. tweak lets variants change +// one value before commit. +func buildToyState(t *testing.T, dir string, tweak func(st *state.DB)) common.Hash { + t.Helper() + db, err := rawdb.NewLevelDBDatabase(dir, 16, 64, "", false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + sdb := state.NewDatabase(db) + st, err := state.New(common.Hash{}, sdb, nil) + if err != nil { + t.Fatal(err) + } + + eoa := common.HexToAddress("0x1000000000000000000000000000000000000001") + st.SetBalance(eoa, big.NewInt(12345)) + st.SetNonce(eoa, 7) + + contract := common.HexToAddress("0x2000000000000000000000000000000000000002") + st.SetCode(contract, []byte("toy contract code"), false) + for j := 0; j < 40; j++ { + st.SetState(contract, + crypto.Keccak256Hash([]byte(fmt.Sprintf("k%d", j))), + crypto.Keccak256Hash([]byte(fmt.Sprintf("v%d", j)))) + } + + validator := common.HexToAddress("0x3000000000000000000000000000000000000003") + w := staketest.GetDefaultValidatorWrapperWithAddr(validator, []bls.SerializedPublicKey{{0x0a}}) + if err := st.UpdateValidatorWrapper(validator, &w); err != nil { + t.Fatal(err) + } + st.SetValidatorFlag(validator) + + if tweak != nil { + tweak(st) + } + root, err := st.Commit(false) + if err != nil { + t.Fatal(err) + } + if err := sdb.TrieDB().Commit(root, false); err != nil { + t.Fatal(err) + } + return root +} + +func walkDir(t *testing.T, dir string, root common.Hash, workers int) (*statecheck.Result, error) { + t.Helper() + db, err := rodb.Open(dir, rodb.Options{}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + latch := &rodb.Latch{} + res, err := statecheck.Walk(statecheck.Config{ + KV: db.KV(latch), + StateRoot: root, + Workers: workers, + }) + if latch.First() != nil && err == nil { + t.Fatalf("latch dirty on success: %v", latch.First()) + } + return res, err +} + +func decodeRLP(blob []byte, out interface{}) error { return rlp.DecodeBytes(blob, out) } + +const goldenPath = "../../../../testdata/recovery/preflight/golden/toy_state_digest.txt" + +// TestDigestGoldenVector: the 3-account toy state digest is pinned to a +// committed golden vector (regenerate with UPDATE_GOLDEN=1). +func TestDigestGoldenVector(t *testing.T) { + dir := filepath.Join(t.TempDir(), "toy") + root := buildToyState(t, dir, nil) + res, err := walkDir(t, dir, root, 2) + if err != nil { + t.Fatal(err) + } + got := hex.EncodeToString(res.Digest[:]) + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.MkdirAll(filepath.Dir(goldenPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(goldenPath, []byte(got+"\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Logf("golden updated: %s", got) + return + } + want, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatalf("golden vector missing (run with UPDATE_GOLDEN=1 to create): %v", err) + } + if got != strings.TrimSpace(string(want)) { + t.Fatalf("digest %s != golden %s", got, strings.TrimSpace(string(want))) + } + if res.Counts.Accounts != 3 || res.Counts.StorageTries != 2 || res.Counts.UniqueCodeContract != 1 || res.Counts.UniqueCodeValidator != 1 { + t.Fatalf("toy counts %+v", res.Counts) + } +} + +// TestDigestWorkerInvariance: byte-identical digests across worker counts. +func TestDigestWorkerInvariance(t *testing.T) { + dir := filepath.Join(t.TempDir(), "toy") + root := buildToyState(t, dir, nil) + r1, err := walkDir(t, dir, root, 1) + if err != nil { + t.Fatal(err) + } + r8, err := walkDir(t, dir, root, 8) + if err != nil { + t.Fatal(err) + } + if r1.Digest != r8.Digest || r1.Counts != r8.Counts { + t.Fatalf("worker variance: %x/%x", r1.Digest, r8.Digest) + } +} + +// TestDigestSensitivity: one flipped storage value changes the digest. +func TestDigestSensitivity(t *testing.T) { + dirA := filepath.Join(t.TempDir(), "a") + rootA := buildToyState(t, dirA, nil) + dirB := filepath.Join(t.TempDir(), "b") + rootB := buildToyState(t, dirB, func(st *state.DB) { + st.SetState(common.HexToAddress("0x2000000000000000000000000000000000000002"), + crypto.Keccak256Hash([]byte("k0")), common.HexToHash("0xff")) + }) + if rootA == rootB { + t.Fatal("tweak did not change the root") + } + ra, err := walkDir(t, dirA, rootA, 2) + if err != nil { + t.Fatal(err) + } + rb, err := walkDir(t, dirB, rootB, 2) + if err != nil { + t.Fatal(err) + } + if ra.Digest == rb.Digest { + t.Fatal("digest insensitive to a changed storage value") + } +} + +// TestDigestRematerialization: copying every key into a fresh LevelDB in +// reverse order (different physical layout) leaves the digest unchanged. +func TestDigestRematerialization(t *testing.T) { + dirA := filepath.Join(t.TempDir(), "a") + rootA := buildToyState(t, dirA, nil) + ra, err := walkDir(t, dirA, rootA, 2) + if err != nil { + t.Fatal(err) + } + + // Re-materialize: read all pairs, write them in descending key order. + src, err := leveldb.OpenFile(dirA, nil) + if err != nil { + t.Fatal(err) + } + type kv struct{ k, v []byte } + var pairs []kv + it := src.NewIterator(nil, nil) + for it.Next() { + pairs = append(pairs, kv{ + k: append([]byte(nil), it.Key()...), + v: append([]byte(nil), it.Value()...), + }) + } + it.Release() + src.Close() + + dirB := filepath.Join(t.TempDir(), "b") + dst, err := leveldb.OpenFile(dirB, nil) + if err != nil { + t.Fatal(err) + } + for i := len(pairs) - 1; i >= 0; i-- { + if err := dst.Put(pairs[i].k, pairs[i].v, nil); err != nil { + t.Fatal(err) + } + } + dst.Close() + + rb, err := walkDir(t, dirB, rootA, 4) + if err != nil { + t.Fatal(err) + } + if ra.Digest != rb.Digest || ra.Counts != rb.Counts { + t.Fatalf("digest not layout-invariant: %x vs %x", ra.Digest, rb.Digest) + } +} + +// TestStockIteratorDefectDifferential documents that the stock +// core/state/iterator.go hard-fails on a vc-namespace validator account +// (defect 1: the ValidatorCode fallback is unreachable because +// ContractCode errors on a miss), while this walker passes the same state. +func TestStockIteratorDefectDifferential(t *testing.T) { + dir := filepath.Join(t.TempDir(), "toy") + root := buildToyState(t, dir, nil) + + // Our walker passes. + if _, err := walkDir(t, dir, root, 2); err != nil { + t.Fatalf("walker failed: %v", err) + } + + // The stock iterator fails on the validator account's vc-only code. + db, err := rawdb.NewLevelDBDatabase(dir, 16, 64, "", true) + if err != nil { + t.Fatal(err) + } + defer db.Close() + stDB, err := state.New(root, state.NewDatabase(db), nil) + if err != nil { + t.Fatal(err) + } + it := state.NewNodeIterator(stDB) + for it.Next() { + } + if it.Error == nil { + t.Fatal("stock iterator unexpectedly succeeded on a vc-only validator account (defect 1 fixed upstream? re-evaluate the bypass)") + } + if !strings.Contains(it.Error.Error(), "code") { + t.Fatalf("stock iterator error %q does not look like the missing-code defect", it.Error) + } +} + +// TestWalkerStorageDeletions: unit-level defect-2 geometry - the storage +// root resolves but a child node is missing; the walk must FAIL, never +// silently treat the trie as empty. Companion: deleting the root itself +// fails on the open path. +func TestWalkerStorageDeletions(t *testing.T) { + contractKey := crypto.Keccak256(common.HexToAddress("0x2000000000000000000000000000000000000002").Bytes()) + + enumerate := func(t *testing.T, dir string, root common.Hash) (storageRoot common.Hash, internal []common.Hash) { + db, err := rodb.Open(dir, rodb.Options{}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + sdb := state.NewDatabase(rawdb.NewDatabase(db.KV(&rodb.Latch{}))) + tr, err := sdb.OpenTrie(root) + if err != nil { + t.Fatal(err) + } + it := tr.NodeIterator(nil) + var acct state.Account + for it.Next(true) { + if it.Leaf() && string(it.LeafKey()) == string(contractKey) { + if err := decodeRLP(it.LeafBlob(), &acct); err != nil { + t.Fatal(err) + } + } + } + if err := it.Error(); err != nil { + t.Fatal(err) + } + if acct.Root == (common.Hash{}) { + t.Fatal("contract account not found") + } + stTrie, err := sdb.OpenStorageTrie(root, common.BytesToHash(contractKey), acct.Root) + if err != nil { + t.Fatal(err) + } + sit := stTrie.NodeIterator(nil) + for sit.Next(true) { + if sit.Hash() != (common.Hash{}) && sit.Hash() != acct.Root { + internal = append(internal, sit.Hash()) + } + } + if err := sit.Error(); err != nil { + t.Fatal(err) + } + return acct.Root, internal + } + + t.Run("child-node-deleted", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "toy") + root := buildToyState(t, dir, nil) + _, internal := enumerate(t, dir, root) + if len(internal) == 0 { + t.Fatal("storage trie too small for a child deletion") + } + del, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + if err := del.Delete(internal[0].Bytes(), nil); err != nil { + t.Fatal(err) + } + del.Close() + _, err = walkDir(t, dir, root, 2) + f, ok := err.(*report.Failure) + if !ok || !strings.Contains(f.Reason, "missing trie node") { + t.Fatalf("want missing-node failure, got %v", err) + } + }) + t.Run("root-node-deleted", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "toy") + root := buildToyState(t, dir, nil) + storageRoot, _ := enumerate(t, dir, root) + del, err := leveldb.OpenFile(dir, nil) + if err != nil { + t.Fatal(err) + } + if err := del.Delete(storageRoot.Bytes(), nil); err != nil { + t.Fatal(err) + } + del.Close() + _, err = walkDir(t, dir, root, 2) + f, ok := err.(*report.Failure) + if !ok || !strings.Contains(f.Reason, "missing trie node") { + t.Fatalf("want missing-node failure on the open path, got %v", err) + } + }) +} From c0d13798fd1fbedb686a157204e8301b082b032f Mon Sep 17 00:00:00 2001 From: polymorpher Date: Thu, 13 Aug 2026 14:31:34 -0700 Subject: [PATCH 2/9] Add the harmony-recovery preflight CLI. Provide the cobra root and preflight subcommand, exit-code delivery, dependency guard, and golden/fail-path tests for the eligibility sampler. Co-authored-by: Cursor --- cmd/harmony-recovery/deps_guard_test.go | 77 ++ cmd/harmony-recovery/golden_test.go | 180 +++++ cmd/harmony-recovery/main.go | 107 +++ cmd/harmony-recovery/preflight.go | 303 +++++++ cmd/harmony-recovery/preflight_fail_test.go | 851 ++++++++++++++++++++ cmd/harmony-recovery/preflight_test.go | 216 +++++ cmd/harmony-recovery/rlimit_other.go | 6 + cmd/harmony-recovery/rlimit_unix.go | 29 + 8 files changed, 1769 insertions(+) create mode 100644 cmd/harmony-recovery/deps_guard_test.go create mode 100644 cmd/harmony-recovery/golden_test.go create mode 100644 cmd/harmony-recovery/main.go create mode 100644 cmd/harmony-recovery/preflight.go create mode 100644 cmd/harmony-recovery/preflight_fail_test.go create mode 100644 cmd/harmony-recovery/preflight_test.go create mode 100644 cmd/harmony-recovery/rlimit_other.go create mode 100644 cmd/harmony-recovery/rlimit_unix.go diff --git a/cmd/harmony-recovery/deps_guard_test.go b/cmd/harmony-recovery/deps_guard_test.go new file mode 100644 index 0000000000..29d50f9b6b --- /dev/null +++ b/cmd/harmony-recovery/deps_guard_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "os/exec" + "strings" + "testing" +) + +// forbiddenDeps is the static dependency guard: a compile-time code-hygiene +// boundary (distinct from any approved-build allowlist). The preflight +// binary must never link networking, RPC, consensus-service wiring or BLS +// keystore loading - it reads a database and verifies signatures, nothing +// else. Exact package or prefix matches against `go list -deps`. +var forbiddenDeps = []string{ + // harmony p2p service / host wiring (the libp2p transport stack hangs + // off this package; note the type-only go-libp2p/core/{crypto,peer} + // packages leak in via internal/utils logging helpers and open no + // sockets - the boundary is harmony's own p2p package) + "github.com/harmony-one/harmony/p2p", + // RPC and API services (incl. sync services) + "github.com/harmony-one/harmony/rpc", + "github.com/harmony-one/harmony/api/service", + // node service wiring + "github.com/harmony-one/harmony/node", + // consensus service (the engine's verification-only subpackages + // consensus/engine, consensus/quorum, consensus/signature, + // consensus/votepower are allowed; the service package itself is not) + "github.com/harmony-one/harmony/consensus\x00exact", + // BLS keystore loading (validators' signing keys must never be touched; + // multibls is a pure key-slice type and is allowed) + "github.com/harmony-one/harmony/internal/blsgen", + // libp2p host construction (transport/muxer/swarm - the actual network + // stack, as opposed to the type-only core packages) + "github.com/libp2p/go-libp2p\x00exact", + "github.com/libp2p/go-libp2p/p2p", +} + +// TestDependencyGuard runs `go list -deps ./cmd/harmony-recovery` and fails +// on forbidden imports. +func TestDependencyGuard(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", ".").CombinedOutput() + if err != nil { + t.Fatalf("go list -deps failed: %v\n%s", err, out) + } + deps := strings.Split(strings.TrimSpace(string(out)), "\n") + depSet := make(map[string]bool, len(deps)) + for _, d := range deps { + depSet[strings.TrimSpace(d)] = true + } + var violations []string + for _, rule := range forbiddenDeps { + if exact, ok := strings.CutSuffix(rule, "\x00exact"); ok { + if depSet[exact] { + violations = append(violations, exact) + } + continue + } + for dep := range depSet { + if dep == rule || strings.HasPrefix(dep, rule+"/") { + violations = append(violations, dep) + } + } + } + if len(violations) > 0 { + t.Fatalf("forbidden dependencies linked into harmony-recovery:\n %s", + strings.Join(violations, "\n ")) + } + // Sanity: the audited verification packages ARE expected. + for _, want := range []string{ + "github.com/harmony-one/harmony/internal/chain", + "github.com/harmony-one/harmony/consensus/quorum", + } { + if !depSet[want] { + t.Fatalf("expected dependency %s missing; the dependency guard may be checking the wrong package", want) + } + } +} diff --git a/cmd/harmony-recovery/golden_test.go b/cmd/harmony-recovery/golden_test.go new file mode 100644 index 0000000000..94f43b6a32 --- /dev/null +++ b/cmd/harmony-recovery/golden_test.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/spf13/pflag" + + "github.com/harmony-one/harmony/internal/recovery/inplace/fixture" + "github.com/harmony-one/harmony/internal/recovery/inplace/report" +) + +const goldenDir = "../../testdata/recovery/preflight/golden" + +// normalizeReceipt zeroes the volatile fields (host identity, timing, build +// stamp, machine paths); everything else in the fixture receipts is +// deterministic, including the state digest. +func normalizeReceipt(rec report.Receipt) report.Receipt { + rec.Hostname = "" + rec.DBPath = "" + rec.StartedAt = "