diff --git a/inprocess/genesis.go b/inprocess/genesis.go index b8e5188ece..4ca474abf7 100644 --- a/inprocess/genesis.go +++ b/inprocess/genesis.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/sei-protocol/sei-chain/sei-cosmos/client" "github.com/sei-protocol/sei-chain/sei-cosmos/client/tx" @@ -19,6 +20,7 @@ import ( banktypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/types" "github.com/sei-protocol/sei-chain/sei-cosmos/x/genutil" genutiltypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/genutil/types" + govtypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/gov/types" stakingtypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/staking/types" tmtime "github.com/sei-protocol/sei-chain/sei-tendermint/libs/time" tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" @@ -40,6 +42,9 @@ type genesisBuilder struct { txConfig client.TxConfig chainID string bondDenom string + // govVotingPeriod, when > 0, shortens the gov voting + deposit periods in the base + // genesis so a proposal can pass within a test. + govVotingPeriod time.Duration accounts []authtypes.GenesisAccount balances []banktypes.Balance @@ -152,6 +157,19 @@ func (b *genesisBuilder) writeBaseGenesis(baseState map[string]json.RawMessage, bankGenState.Balances = append(bankGenState.Balances, b.balances...) baseState[banktypes.ModuleName] = b.codec.MustMarshalJSON(&bankGenState) + if b.govVotingPeriod > 0 { + var govGenState govtypes.GenesisState + b.codec.MustUnmarshalJSON(baseState[govtypes.ModuleName], &govGenState) + // Deposit/quorum/threshold keep their defaults — node_admin funds the default + // usei deposit and all validators voting yes clears quorum + threshold. The + // expedited period must stay strictly below the normal one (gov Params + // ValidateBasic rejects >=), so halve it. + govGenState.VotingParams.VotingPeriod = b.govVotingPeriod + govGenState.VotingParams.ExpeditedVotingPeriod = b.govVotingPeriod / 2 + govGenState.DepositParams.MaxDepositPeriod = b.govVotingPeriod + baseState[govtypes.ModuleName] = b.codec.MustMarshalJSON(&govGenState) + } + appStateJSON, err := json.MarshalIndent(baseState, "", " ") if err != nil { return err diff --git a/inprocess/harness.go b/inprocess/harness.go index ccb7b8fee2..0947f64996 100644 --- a/inprocess/harness.go +++ b/inprocess/harness.go @@ -76,6 +76,22 @@ type Options struct { // dominant cadence lever — lower it (e.g. 500ms) for faster tests. TimeoutCommit time.Duration + // SnapshotInterval, when > 0, makes each subprocess validator take a cosmos + // state-sync snapshot every N blocks into /snapshots (the location the + // snapshot/statesync suites check and that a late-joining statesync node + // restores from). 0 disables snapshots. Subprocess backend only — the in-process + // backend does not run `seid start` and ignores it. Keep it small (e.g. 10) for + // a fast test so a snapshot lands inside the window. + SnapshotInterval uint64 + + // GovVotingPeriod, when > 0, shortens the gov voting + deposit periods in genesis + // so a governance proposal can pass within a test. 0 keeps the chain default (far + // too long for a test). Subprocess backend only — used by the upgrade suites, + // which pass a software-upgrade proposal and wait it out. It is coupled to the + // proposal's target height: the height must be far enough ahead that the proposal + // passes before the chain reaches it (see TestSubprocessUpgrade). + GovVotingPeriod time.Duration + // ExtraKeys are non-validator genesis accounts to create + fund. Each key is // written into its target node's home `test` keyring (so a host `seid --home // --keyring-backend test` resolves it) and funded at genesis. This is @@ -192,31 +208,58 @@ func Start(ctx context.Context, opts Options) (_ *Network, retErr error) { } }() + enc, err := net.provision() + if err != nil { + return nil, err + } + + // One network per process (see networkStarted): claim the slot here, right + // before the first app.New (in startNode) — the first point that touches the + // process-global EVM singletons. Every step above (base dir, provisioning, + // genesis) can fail recoverably without burning the slot. Once claimed it is + // never released: app.New's singletons can't re-init. + if !networkStarted.CompareAndSwap(false, true) { + return nil, fmt.Errorf("inprocess: a network was already started in this process; only one is supported (EVM worker pool / metrics printer / Prometheus registries are process-global singletons)") + } + for _, n := range net.nodes { + if err := net.startNode(ctx, n, enc); err != nil { + return nil, fmt.Errorf("start %s: %w", n.moniker, err) + } + } + return net, nil +} + +// provision lays every node's home down on disk — per-node keys, isolated config, +// keyring, the empty-valset genesis, and the gentx-derived peer mesh — the shared +// foundation both backends build on: the in-process backend then starts each node +// in-goroutine (startNode), the subprocess backend writes the remaining on-disk +// config and runs `seid start` (see subprocess.go). It returns the encoding config +// the caller needs to start the nodes. +func (net *Network) provision() (encoding, error) { enc := app.MakeEncodingConfig() gb := &genesisBuilder{ - codec: enc.Marshaler, - txConfig: enc.TxConfig, - chainID: opts.ChainID, - bondDenom: sdk.DefaultBondDenom, + codec: enc.Marshaler, + txConfig: enc.TxConfig, + chainID: net.opts.ChainID, + bondDenom: sdk.DefaultBondDenom, + govVotingPeriod: net.opts.GovVotingPeriod, } - if err := net.provisionNodes(enc, gb); err != nil { - return nil, err + return enc, err } if err := net.provisionExtraKeys(gb); err != nil { - return nil, err + return enc, err } - baseState := app.ModuleBasics.DefaultGenesis(enc.Marshaler) genFiles := make([]string, len(net.nodes)) for i, n := range net.nodes { genFiles[i] = n.tmCfg.GenesisFile() } if err := gb.writeBaseGenesis(baseState, genFiles); err != nil { - return nil, fmt.Errorf("write base genesis: %w", err) + return enc, fmt.Errorf("write base genesis: %w", err) } - if err := gb.collectGentxs(net.nodes, filepath.Join(baseDir, "gentxs")); err != nil { - return nil, fmt.Errorf("collect gentxs: %w", err) + if err := gb.collectGentxs(net.nodes, filepath.Join(net.baseDir, "gentxs")); err != nil { + return enc, fmt.Errorf("collect gentxs: %w", err) } // gentx-derived peer mesh guard: collectGentxs is what populates each node's // PersistentPeers (in place, via GenAppStateFromConfig — see doc.go). For N>=2 @@ -225,28 +268,14 @@ func Start(ctx context.Context, opts Options) (_ *Network, retErr error) { if len(net.nodes) >= 2 { for _, n := range net.nodes { if n.tmCfg.P2P.PersistentPeers == "" { - return nil, fmt.Errorf( + return enc, fmt.Errorf( "inprocess: gentx-derived peer mesh not wired: collectGentxs did not populate PersistentPeers for %s — did a refactor clone or reorder the config?", n.moniker, ) } } } - - // One network per process (see networkStarted): claim the slot here, right - // before the first app.New (in startNode) — the first point that touches the - // process-global EVM singletons. Every step above (base dir, provisioning, - // genesis) can fail recoverably without burning the slot. Once claimed it is - // never released: app.New's singletons can't re-init. - if !networkStarted.CompareAndSwap(false, true) { - return nil, fmt.Errorf("inprocess: a network was already started in this process; only one is supported (EVM worker pool / metrics printer / Prometheus registries are process-global singletons)") - } - for _, n := range net.nodes { - if err := net.startNode(ctx, n, enc); err != nil { - return nil, fmt.Errorf("start %s: %w", n.moniker, err) - } - } - return net, nil + return enc, nil } // provisionNodes runs the first pass: per-node keys, node IDs, gentxs, isolated diff --git a/inprocess/readiness.go b/inprocess/readiness.go index de037e5d4f..5389789262 100644 --- a/inprocess/readiness.go +++ b/inprocess/readiness.go @@ -110,6 +110,7 @@ func latestHeight(ctx context.Context, hc *http.Client, tmRPC string) (int64, bo type syncInfo struct { LatestBlockHeight string `json:"latest_block_height"` + LatestBlockHash string `json:"latest_block_hash"` } // getJSON performs one request and returns the body on HTTP 200, else ok=false diff --git a/inprocess/subprocess.go b/inprocess/subprocess.go new file mode 100644 index 0000000000..41aafebf23 --- /dev/null +++ b/inprocess/subprocess.go @@ -0,0 +1,710 @@ +//go:build inprocess + +package inprocess + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + srvconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + "github.com/sei-protocol/sei-chain/sei-cosmos/x/genutil" + tmconfig "github.com/sei-protocol/sei-chain/sei-tendermint/config" +) + +// statesyncNodeMoniker is the late-joining RPC node's name — the moniker the +// statesync suite targets (node: sei-rpc-node), resolved by the runner's nodeFor. +const statesyncNodeMoniker = "sei-rpc-node" + +// stopWaitTimeout bounds how long a stop/kill waits for one node to be reaped +// before giving up. A process wedged in uninterruptible I/O (a slow pebbledb +// fsync) isn't reaped promptly; without a bound, teardown would block there. It +// also doubles as the graceful-stop grace period before SIGKILL escalation and as +// exec.Cmd.WaitDelay. +const stopWaitTimeout = 5 * time.Second + +// nodeProc is one node's running process and the state the reaper and the control +// verbs (Stop/Kill/Restart/IsRunning) share. Its own methods hold mu; it is never +// copied (SubprocessNetwork.procs is []*nodeProc), so its sync.Mutex is safe. +type nodeProc struct { + node *node // the provisioned node (home/config/ports); set once at build + extraEnv []string // env applied at each (re)start (e.g. UPGRADE_VERSION_LIST); caller-owned, not mu-guarded (the reaper never reads it) + + mu sync.Mutex + cmd *exec.Cmd // current process; replaced on restart + log *os.File // current log sink; closed by the reaper on exit + running bool // true between start and the reaper observing exit + done chan struct{} // closed by the reaper when the current cmd's Wait returns + waitErr error // the current process's exit error (nil = clean exit) +} + +// SubprocessNetwork is the Tier-2 harness: N real `seid` processes on disk, no +// docker. It reuses the in-process provisioning (genesis, keys, gentx-derived peer +// mesh — see provision) but boots each node with `seid start` instead of an +// in-goroutine app. That makes it slower than the in-process backend but able to +// exercise the operational lifecycle the in-process one can't — process +// kill/restart (Stop/Kill/Restart), on-disk WAL recovery, upgrade-by-env. Each node +// is its own OS process, so the process-global-singleton / one-network-per-process +// constraint (see networkStarted) does not apply here. +// +// Not goroutine-safe across calls (mirrors Network): callers drive one +// SubprocessNetwork from a single goroutine. The per-node reaper goroutines are the +// only concurrency, and they touch only their nodeProc under its mutex. +type SubprocessNetwork struct { + net *Network + seidBin string + // ctx parents every spawned process (see startProc) and is reused by Restart, so + // it must outlive the network — cancelling it SIGKILLs every node's group. Stored + // (not passed per-call) because the supervisor's lifetime is the process tree's. + ctx context.Context + procs []*nodeProc // index-aligned with net.nodes + closed bool +} + +// StartSubprocess provisions opts.Validators node homes on disk (reusing the +// in-process provisioning) and boots each as a `seid start` subprocess running +// seidBin. It returns once every process is spawned — NOT once consensus is live; +// call WaitReady for that. On any error mid-bring-up it tears down whatever came up. +// +// ctx bounds the spawned processes: cancelling it SIGKILLs every node's process +// group (see startProc). It is the caller's backstop when Close's defer can't run. +func StartSubprocess(ctx context.Context, opts Options, seidBin string) (_ *SubprocessNetwork, retErr error) { + opts = opts.withDefaults() + // N>=3 only. The in-process backend supports N=1 by pinning the sole validator + // into the genesis doc in memory (see startNode); that pin is never written to + // disk, so a subprocess node — which reads genesis from disk — would see an + // empty valset and stall at height 1. N=2 deadlocks CometBFT block-sync. + if opts.Validators < 3 { + switch opts.Validators { + case 1: + return nil, fmt.Errorf("inprocess: StartSubprocess requires Validators >= 3: N=1 needs the genesis validator pinned on disk, which only the in-process backend does (in memory); a solo subprocess node reads an empty valset and stalls at height 1") + case 2: + return nil, fmt.Errorf("inprocess: StartSubprocess requires Validators >= 3: N=2 deadlocks in CometBFT block-sync (BlockPool.IsCaughtUp requires >1 peer)") + default: + return nil, fmt.Errorf("inprocess: StartSubprocess requires Validators >= 3, got %d", opts.Validators) + } + } + if seidBin == "" { + return nil, fmt.Errorf("inprocess: StartSubprocess requires a seid binary path") + } + + baseDir, ownBaseDir, err := resolveBaseDir(opts.BaseDir) + if err != nil { + return nil, err + } + net := &Network{opts: opts, baseDir: baseDir, ownBaseDir: ownBaseDir} + sn := &SubprocessNetwork{net: net, seidBin: seidBin, ctx: ctx} + defer func() { + if retErr != nil { + sn.Close() + } + }() + + if _, err := net.provision(); err != nil { + return nil, err + } + + // Write the on-disk config `seid start` reads that the in-process path keeps in + // memory (config.toml from tmCfg; the app.toml base + EVM/SeiDB sections) — then + // spawn each node. + for _, n := range net.nodes { + if err := sn.writeNodeConfig(n); err != nil { + return nil, fmt.Errorf("write subprocess config for %s: %w", n.moniker, err) + } + } + sn.procs = make([]*nodeProc, len(net.nodes)) + for i, n := range net.nodes { + sn.procs[i] = &nodeProc{node: n} + } + for _, p := range sn.procs { + if err := sn.startProc(p); err != nil { + return nil, fmt.Errorf("start %s: %w", p.node.moniker, err) + } + } + return sn, nil +} + +// startProc launches (or relaunches) one node's `seid start` in its own process +// group and hands the single Wait to a reaper goroutine. The reaper is the one +// owner of cmd.Wait, so it observes every exit — clean, panic (upgrade suites), or +// our own kill — updating running/waitErr and closing the log; the control verbs +// only signal and wait on done. Setpgid lets cmd.Cancel and the verbs signal the +// whole tree (any children seid spawns); binding cmd to sn.ctx makes cancellation +// the teardown backstop when Close can't run. The log is opened O_APPEND so a +// restart preserves the prior run's output (the upgrade/crash suites read the +// pre-restart panic). +func (sn *SubprocessNetwork) startProc(p *nodeProc) error { + logf, err := os.OpenFile(filepath.Join(p.node.home, "seid.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + cmd := exec.CommandContext(sn.ctx, sn.seidBin, "start", "--home", p.node.home) //nolint:gosec + cmd.Stdout = logf + cmd.Stderr = logf + cmd.Env = append(os.Environ(), p.extraEnv...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // Default CommandContext kills only the leader; -pid signals the whole group so + // Setpgid children aren't orphaned. WaitDelay bounds the post-cancel reaper. + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } + cmd.WaitDelay = stopWaitTimeout + if err := cmd.Start(); err != nil { + _ = logf.Close() + return err + } + + done := make(chan struct{}) + p.mu.Lock() + p.cmd = cmd + p.log = logf + p.running = true + p.waitErr = nil + p.done = done + p.mu.Unlock() + + go func() { + werr := cmd.Wait() + _ = logf.Close() + p.mu.Lock() + // Only record exit if this is still the current process. A restart that gave + // up on a wedged old process (signalAndWait returned !exited, so Restart + // aborts) leaves this reaper live; when the old process finally dies it must + // not clobber a newer process's running/waitErr. + if p.cmd == cmd { + p.running = false + p.waitErr = werr + } + p.mu.Unlock() + close(done) + }() + return nil +} + +// Stop gracefully stops node i (SIGTERM, escalating to SIGKILL after +// stopWaitTimeout) and waits for it to exit. Idempotent: a no-op if the node is +// already down. The node's home/data survive, so a later Restart recovers via WAL. +func (sn *SubprocessNetwork) Stop(i int) error { + _, err := sn.signalAndWait(sn.procs[i], syscall.SIGTERM) + return err +} + +// Kill hard-kills node i (SIGKILL) and waits for it to exit — an ungraceful crash +// for the crash-recovery suites. Idempotent. +func (sn *SubprocessNetwork) Kill(i int) error { + _, err := sn.signalAndWait(sn.procs[i], syscall.SIGKILL) + return err +} + +// Restart stops node i (if running) and starts it again from the same home. WithEnv +// sets the process env for this and subsequent starts (the upgrade suites pass +// UPGRADE_VERSION_LIST to bring a node up "ahead" of the chain). A node that already +// exited on its own (crash-recovery, or an upgrade panic) is simply started again → +// WAL replay. +// +// It refuses to start over a process it could not confirm dead: the restarted node +// reuses the same fixed RPC/P2P/EVM/gRPC ports, so starting while the old process +// still holds them would silently EADDRINUSE-crash the new one. This only happens if +// SIGKILL can't reap the old process within the bound (a rare uninterruptible-I/O +// wedge). +func (sn *SubprocessNetwork) Restart(i int, opts ...RestartOption) error { + var cfg restartConfig + for _, o := range opts { + o(&cfg) + } + p := sn.procs[i] + exited, err := sn.signalAndWait(p, syscall.SIGTERM) + if err != nil { + return err + } + if !exited { + return fmt.Errorf("inprocess: cannot restart %s: previous process did not exit within %s (still holding its ports)", p.node.moniker, stopWaitTimeout) + } + if cfg.env != nil { + p.extraEnv = cfg.env // caller-owned; startProc reads it on this same goroutine + } + return sn.startProc(p) +} + +// IsRunning reports whether node i's process is currently up. It flips to false the +// moment the reaper observes an exit, so the upgrade suites can assert that a +// non-upgraded node panicked at the upgrade height. +func (sn *SubprocessNetwork) IsRunning(i int) bool { + p := sn.procs[i] + p.mu.Lock() + defer p.mu.Unlock() + return p.running +} + +// RestartOption configures a Restart. +type RestartOption func(*restartConfig) + +type restartConfig struct{ env []string } + +// WithEnv sets the process environment (KEY=VALUE strings) applied on the next +// start and retained across later restarts, replacing any previously set env. Used +// by the upgrade suites to set UPGRADE_VERSION_LIST. +func WithEnv(kv ...string) RestartOption { + return func(c *restartConfig) { c.env = kv } +} + +// signalAndWait signals node p's process group and waits for the reaper to observe +// exit, escalating a graceful signal to SIGKILL after stopWaitTimeout. It reports +// whether the process actually exited within the bounds: exited=false means the +// process is still alive and unreaped (a rare uninterruptible-I/O wedge SIGKILL +// can't clear) — the caller must NOT start a replacement over it (see Restart). An +// already-down process (or one that vanished mid-signal, ESRCH) is exited=true. Only +// a genuinely failed signal syscall is an error. +func (sn *SubprocessNetwork) signalAndWait(p *nodeProc, sig syscall.Signal) (exited bool, err error) { + p.mu.Lock() + cmd, done, running := p.cmd, p.done, p.running + p.mu.Unlock() + if !running || cmd == nil || cmd.Process == nil { + return true, nil + } + if err := syscall.Kill(-cmd.Process.Pid, sig); err != nil { + if errors.Is(err, syscall.ESRCH) { + return true, nil // already gone + } + return false, fmt.Errorf("signal %v to %s: %w", sig, p.node.moniker, err) + } + select { + case <-done: + return true, nil + case <-time.After(stopWaitTimeout): + } + if sig != syscall.SIGKILL { + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + return false, fmt.Errorf("SIGKILL to %s: %w", p.node.moniker, err) + } + select { + case <-done: + return true, nil + case <-time.After(stopWaitTimeout): + } + } + return false, nil +} + +// AddStatesyncNode provisions and starts a late-joining, non-validator full node +// (moniker sei-rpc-node) that STATE-SYNCS from the running validators' snapshots +// rather than replaying from genesis. It must be called after the validators have +// produced at least one snapshot (height >= SnapshotInterval — the caller +// sequences this), because a statesync node that finds no snapshot loops forever at +// height 0. It returns the node's handle; the caller waits for it to sync (its TM +// RPC latest_block_height goes > 0). The node joins the network (so Close tears it +// down and the runner can target it by moniker). Requires SnapshotInterval > 0. +func (sn *SubprocessNetwork) AddStatesyncNode(ctx context.Context) (Node, error) { + if sn.net.opts.SnapshotInterval == 0 { + return Node{}, fmt.Errorf("inprocess: AddStatesyncNode requires Options.SnapshotInterval > 0 (validators must produce snapshots to sync from)") + } + if len(sn.net.nodes) < 2 { + return Node{}, fmt.Errorf("inprocess: AddStatesyncNode needs >= 2 validators to peer with, have %d", len(sn.net.nodes)) + } + + // Self-sequence: a joiner that starts before any snapshot exists discovers + // nothing and loops forever at height 0 (no error). Wait for a validator to have + // written one first. + if err := waitValidatorSnapshot(ctx, filepath.Join(sn.net.nodes[0].home, "snapshots")); err != nil { + return Node{}, fmt.Errorf("no validator snapshot to sync from: %w", err) + } + + nodeDir := filepath.Join(sn.net.baseDir, statesyncNodeMoniker, "simd") + if err := os.MkdirAll(filepath.Join(nodeDir, "config"), 0o750); err != nil { + return Node{}, err + } + tmCfg, addrs, err := buildNodeConfig(nodeDir, statesyncNodeMoniker, sn.net.opts.TimeoutCommit) + if err != nil { + return Node{}, err + } + tmCfg.Mode = tmconfig.ModeFull // a plain full node, not in the valset + nodeID, pubKey, err := genutil.InitializeNodeValidatorFiles(tmCfg) + if err != nil { + return Node{}, fmt.Errorf("init node files for %s: %w", statesyncNodeMoniker, err) + } + + // Peer with every validator (the statesync reactor blocks until it has >= 2 + // peers), and anchor the light client to a currently-committed block. + peers := make([]string, len(sn.net.nodes)) + for i, v := range sn.net.nodes { + peers[i] = fmt.Sprintf("%s@%s:%s", v.nodeID, v.p2pHost, v.p2pPort) + } + tmCfg.P2P.PersistentPeers = strings.Join(peers, ",") + + valRPC := sn.Node(0).TendermintRPC() + trustHeight, trustHash, err := fetchTrustBlock(ctx, valRPC) + if err != nil { + return Node{}, fmt.Errorf("read trust block from %s: %w", valRPC, err) + } + // Two DISTINCT validators as light-client providers (primary + witness). More + // robust than the same URL twice, and it survives the upgrade suites bouncing + // node 0 — N>=3 is enforced, so nodes[1] exists. + tmCfg.StateSync.Enable = true + tmCfg.StateSync.RPCServers = []string{sn.net.nodes[0].rpcAddr, sn.net.nodes[1].rpcAddr} + tmCfg.StateSync.TrustHeight = trustHeight + tmCfg.StateSync.TrustHash = trustHash + tmCfg.StateSync.TrustPeriod = 168 * time.Hour + tmCfg.StateSync.DiscoveryTime = 15 * time.Second + + if err := tmconfig.WriteConfigFile(nodeDir, tmCfg); err != nil { + return Node{}, fmt.Errorf("write config.toml: %w", err) + } + // Same chain => identical genesis. State sync verifies the restored app hash + // against headers derived from this genesis, so it must match the validators'. + if err := copyFile(sn.net.nodes[0].tmCfg.GenesisFile(), tmCfg.GenesisFile()); err != nil { + return Node{}, fmt.Errorf("copy genesis: %w", err) + } + // The harness genesis carries an empty valset (CometBFT derives it via InitChain + // on the validators — see the empty-valset invariant). A statesync node SKIPS + // InitChain, so it would build initial state from that empty valset and fail with + // "validatorSet proposer error: nil validator". Inject the live validator set + // (identical to the genesis set — powers don't change) so initial-state creation + // succeeds; statesync then overrides it at the snapshot height. + vals, err := fetchGenesisValidators(ctx, valRPC) + if err != nil { + return Node{}, fmt.Errorf("read validator set from %s: %w", valRPC, err) + } + if err := injectGenesisValidators(tmCfg.GenesisFile(), vals); err != nil { + return Node{}, fmt.Errorf("inject genesis validators: %w", err) + } + appPath := filepath.Join(nodeDir, "config", "app.toml") + // SnapshotInterval 0: the joiner restores snapshots, it need not produce them. + // The SeiDB section (via appendEVMAppConfig) must match the validators or the + // restored app hash won't verify. + if err := writeSubprocessAppBase(appPath, nodeDir, 0); err != nil { + return Node{}, fmt.Errorf("write app.toml base: %w", err) + } + // Match the validators' SeiDB config (incl. the memiavl snapshot cadence) so the + // restored app hash verifies and the joiner's WAL is bounded too. + if err := appendEVMAppConfig(appPath, addrs.httpPort, addrs.wsPort, sn.net.opts.SnapshotInterval); err != nil { + return Node{}, fmt.Errorf("append evm app config: %w", err) + } + if err := writeClientConfig(filepath.Join(nodeDir, "config/client.toml"), sn.net.opts.ChainID, addrs.rpcAddr); err != nil { + return Node{}, fmt.Errorf("write client.toml: %w", err) + } + + n := &node{ + moniker: statesyncNodeMoniker, nodeID: nodeID, pubKey: pubKey, + home: nodeDir, tmCfg: tmCfg, + p2pHost: addrs.p2pHost, p2pPort: addrs.p2pPort, rpcAddr: addrs.rpcAddr, + httpPort: addrs.httpPort, wsPort: addrs.wsPort, + } + // Append only after the process starts, so a start failure doesn't leave a dead + // node in the network for WaitReady to block on. + p := &nodeProc{node: n} + if err := sn.startProc(p); err != nil { + return Node{}, fmt.Errorf("start %s: %w", statesyncNodeMoniker, err) + } + sn.net.nodes = append(sn.net.nodes, n) + sn.procs = append(sn.procs, p) + return Node{n: n}, nil +} + +// fetchTrustBlock reads the light-client trust anchor (latest committed height + +// block hash) from a validator's CometBFT RPC /status. It uses the same dual-shape +// parse as latestHeight (the node's /status may be enveloped or unwrapped — see +// readiness.go) since sync_info carries both the height and the hash. +func fetchTrustBlock(ctx context.Context, tmRPC string) (height int64, hash string, err error) { + body, ok := getJSON(ctx, probeClient, http.MethodGet, tmRPC+"/status", "") + if !ok { + return 0, "", fmt.Errorf("status unreachable at %s", tmRPC) + } + var s struct { + Result *struct { + SyncInfo syncInfo `json:"sync_info"` + } `json:"result,omitempty"` + SyncInfo syncInfo `json:"sync_info"` + } + if err := json.Unmarshal(body, &s); err != nil { + return 0, "", fmt.Errorf("parse /status body: %w", err) + } + si := s.SyncInfo + if s.Result != nil && s.Result.SyncInfo.LatestBlockHeight != "" { + si = s.Result.SyncInfo + } + h, err := strconv.ParseInt(si.LatestBlockHeight, 10, 64) + if err != nil { + return 0, "", fmt.Errorf("parse trust height %q: %w", si.LatestBlockHeight, err) + } + if h == 0 || si.LatestBlockHash == "" { + return 0, "", fmt.Errorf("no committed block yet at %s", tmRPC) + } + return h, si.LatestBlockHash, nil +} + +// genesisValidator is one entry of the CometBFT genesis `validators` array. PubKey +// is passed through verbatim ({"type":...,"value":...}) from the /validators RPC, +// which uses the identical shape. +type genesisValidator struct { + Address string `json:"address"` + PubKey json.RawMessage `json:"pub_key"` + Power string `json:"power"` + Name string `json:"name"` +} + +// fetchGenesisValidators reads a validator's /validators RPC and returns the set in +// genesis-validators shape (voting_power -> power). Dual-shape like fetchTrustBlock. +func fetchGenesisValidators(ctx context.Context, tmRPC string) ([]genesisValidator, error) { + body, ok := getJSON(ctx, probeClient, http.MethodGet, tmRPC+"/validators", "") + if !ok { + return nil, fmt.Errorf("validators unreachable at %s", tmRPC) + } + type rpcVal struct { + Address string `json:"address"` + PubKey json.RawMessage `json:"pub_key"` + VotingPower string `json:"voting_power"` + } + var s struct { + Result *struct { + Validators []rpcVal `json:"validators"` + } `json:"result,omitempty"` + Validators []rpcVal `json:"validators"` + } + if err := json.Unmarshal(body, &s); err != nil { + return nil, fmt.Errorf("parse /validators body: %w", err) + } + vals := s.Validators + if s.Result != nil && len(s.Result.Validators) > 0 { + vals = s.Result.Validators + } + if len(vals) == 0 { + return nil, fmt.Errorf("no validators reported by %s", tmRPC) + } + out := make([]genesisValidator, len(vals)) + for i, v := range vals { + out[i] = genesisValidator{Address: v.Address, PubKey: v.PubKey, Power: v.VotingPower} + } + return out, nil +} + +// injectGenesisValidators rewrites the genesis file's top-level `validators` array, +// leaving every other field byte-identical. +func injectGenesisValidators(genesisPath string, vals []genesisValidator) error { + raw, err := os.ReadFile(genesisPath) + if err != nil { + return err + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return err + } + vb, err := json.Marshal(vals) + if err != nil { + return err + } + doc["validators"] = vb + out, err := json.Marshal(doc) + if err != nil { + return err + } + return os.WriteFile(genesisPath, out, 0o600) +} + +// waitValidatorSnapshot blocks until snapDir holds at least one numeric height +// subdir (a written cosmos state-sync snapshot) or ctx fires. +func waitValidatorSnapshot(ctx context.Context, snapDir string) error { + tick := time.NewTicker(probeInterval) + defer tick.Stop() + for { + if entries, err := os.ReadDir(snapDir); err == nil { + for _, e := range entries { + if !e.IsDir() { + continue + } + if _, err := strconv.Atoi(e.Name()); err == nil { + return nil + } + } + } + select { + case <-ctx.Done(): + return fmt.Errorf("no snapshot under %s: %w", snapDir, ctx.Err()) + case <-tick.C: + } + } +} + +// copyFile copies src to dst (0o600). Used to give the statesync node the +// validators' exact genesis. +func copyFile(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, b, 0o600) +} + +// Node returns a handle to the i-th validator. The handle surface mirrors the +// in-process Network so consumers (the YAML runner) are backend-agnostic. +func (sn *SubprocessNetwork) Node(i int) Node { return Node{n: sn.net.nodes[i]} } + +// Nodes returns handles to every validator in index order. +func (sn *SubprocessNetwork) Nodes() []Node { + out := make([]Node, len(sn.net.nodes)) + for i := range sn.net.nodes { + out[i] = Node{n: sn.net.nodes[i]} + } + return out +} + +// Len is the validator count. +func (sn *SubprocessNetwork) Len() int { return len(sn.net.nodes) } + +// WaitReady blocks until every node is producing blocks and serving EVM, or ctx +// fires — the same readiness probes the in-process backend uses (HTTP against each +// node's own RPC), so the gate is backend-independent. The returned error already +// names the node (Node.WaitReady prepends its moniker). +func (sn *SubprocessNetwork) WaitReady(ctx context.Context) error { + for i := range sn.net.nodes { + if err := sn.Node(i).WaitReady(ctx); err != nil { + return err + } + } + return nil +} + +// Close kills every seid process group and removes the owned temp dir. Idempotent +// and safe on a partial start (nil-guarded). +// +// A completed Close leaves nothing behind — each SIGKILL reaps the node's group and +// the reaper closes its log — but it does NOT defend against the test binary being +// hard-killed: `go test -timeout` sends SIGQUIT (defers don't run) and Setpgid has +// detached each node into its own group, so those groups reparent to PID 1. The ctx +// from StartSubprocess is the backstop there — keep suite time under the outer +// -timeout so this defer runs. +func (sn *SubprocessNetwork) Close() { + if sn.closed { + return + } + sn.closed = true + for _, p := range sn.procs { + if p != nil { + _, _ = sn.signalAndWait(p, syscall.SIGKILL) + } + } + if sn.net.ownBaseDir && sn.net.baseDir != "" { + _ = os.RemoveAll(sn.net.baseDir) + } +} + +// LogTails returns a bounded tail of every node's captured seid.log, for a +// post-mortem when the cluster fails to come up. Best-effort: on Close's own +// SIGKILL a child can't flush, so a node that hung then got killed may have a +// short or stale log. +func (sn *SubprocessNetwork) LogTails() string { + const perNode = 2000 + var b strings.Builder + for _, n := range sn.net.nodes { + fmt.Fprintf(&b, "=== %s seid.log (tail) ===\n", n.moniker) + raw, err := os.ReadFile(filepath.Join(n.home, "seid.log")) + if err != nil { + fmt.Fprintf(&b, "(no log: %v)\n", err) + continue + } + s := string(raw) + if len(s) > perNode { + s = s[len(s)-perNode:] + } + b.WriteString(s) + b.WriteString("\n") + } + return b.String() +} + +// writeNodeConfig writes the on-disk config `seid start` needs that the in-process +// path holds in memory: config.toml (the gentx-derived peer mesh + the loopback +// RPC/P2P listen addrs held in n.tmCfg) and the app.toml base + EVM/SeiDB sections +// (the in-process backend injects the latter via AppOptions and calls +// RegisterLocalServices directly — see appoptions.go / newNodeApp). +func (sn *SubprocessNetwork) writeNodeConfig(n *node) error { + if err := tmconfig.WriteConfigFile(n.home, n.tmCfg); err != nil { + return fmt.Errorf("write config.toml: %w", err) + } + appPath := filepath.Join(n.home, "config", "app.toml") + if err := writeSubprocessAppBase(appPath, n.home, sn.net.opts.SnapshotInterval); err != nil { + return fmt.Errorf("write app.toml base: %w", err) + } + return appendEVMAppConfig(appPath, n.httpPort, n.wsPort, sn.net.opts.SnapshotInterval) +} + +// writeSubprocessAppBase rewrites the cosmos-base app.toml (over provision's +// gRPC-off default) to enable the cosmos gRPC server on a free per-node port, plus +// (when snapshotInterval > 0) cosmos state-sync snapshots into /snapshots. +// +// The gRPC enable is load-bearing: `seid start` constructs the EVM HTTP/WS servers +// (via app.RegisterLocalServices) ONLY when the API or gRPC server is enabled (see +// sei-cosmos server.startInProcess) — with both off, a node reaches consensus but +// never serves EVM. gRPC is the lightest server that trips that gate; no suite +// dials it. API stays off (it would need its own per-node 1317 port). Per-node +// gRPC port so N nodes don't collide. +// +// The snapshot-directory override (validators, when snapshotInterval > 0) is also +// load-bearing: the default is /data/snapshots, but the snapshot suite checks +// /snapshots. The restoring joiner (snapshotInterval 0 here) keeps the default +// — its snapshot store is local restore scratch, unrelated to what it syncs from. +func writeSubprocessAppBase(path, home string, snapshotInterval uint64) error { + grpcPort, err := freePort() + if err != nil { + return err + } + appCfg := srvconfig.DefaultConfig() + appCfg.GRPC.Enable = true + appCfg.GRPC.Address = fmt.Sprintf("127.0.0.1:%d", grpcPort) + // GRPCWeb defaults on (:9091); force off or N nodes collide on that port too. + appCfg.GRPCWeb.Enable = false + appCfg.API.Enable = false + appCfg.Telemetry.Enabled = false + if snapshotInterval > 0 { + appCfg.StateSync.SnapshotInterval = snapshotInterval + appCfg.StateSync.SnapshotKeepRecent = 3 + appCfg.StateSync.SnapshotDirectory = filepath.Join(home, "snapshots") + } + srvconfig.WriteConfigFile(path, appCfg) + return nil +} + +// appendEVMAppConfig appends the sei-chain app sections (EVM enabled on this node's +// ports; SeiDB on) to the cosmos-base app.toml writeSubprocessAppBase just wrote. +// These sections are sei-chain-specific — the cosmos template renders none of them +// — so there is nothing to conflict with. +// +// When scSnapshotInterval > 0 it also sets the memiavl (state-commit) snapshot +// cadence. parseSCConfigs (app/seidb.go) reads sc-snapshot-interval unconditionally, +// so leaving it unset means memiavl never persists a snapshot and never truncates +// its WAL; setting it (to the cosmos state-sync interval) bounds the WAL and matches +// how a production node with snapshots runs — which the statesync path exercises. +func appendEVMAppConfig(path string, httpPort, wsPort int, scSnapshotInterval uint64) error { + stateCommit := "sc-enable = true" + if scSnapshotInterval > 0 { + stateCommit += fmt.Sprintf("\nsc-snapshot-interval = %d\nsc-keep-recent = 2", scSnapshotInterval) + } + extra := fmt.Sprintf(` +[evm] +http_enabled = true +http_port = %d +ws_enabled = true +ws_port = %d + +[state-commit] +%s + +[state-store] +ss-enable = true +ss-backend = "pebbledb" +`, httpPort, wsPort, stateCommit) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = f.WriteString(extra) + return err +} diff --git a/inprocess/subprocess_test.go b/inprocess/subprocess_test.go new file mode 100644 index 0000000000..4d9fb290f5 --- /dev/null +++ b/inprocess/subprocess_test.go @@ -0,0 +1,437 @@ +//go:build inprocess + +package inprocess + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// buildSeid builds the standard seid binary (no build tags — the real production +// start path the subprocess harness runs, distinct from the -tags inprocess app). +func buildSeid(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + root := filepath.Clean(filepath.Join(wd, "..")) // /inprocess -> + bin := filepath.Join(t.TempDir(), "seid") + cmd := exec.Command("go", "build", "-o", bin, "./cmd/seid") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build seid: %v\n%s", err, out) + } + return bin +} + +// TestSubprocessNetwork is the Tier-2 feasibility gate: a cluster of real `seid` +// processes, provisioned by the harness and booted with `seid start`, must reach +// consensus and serve EVM — with no docker. +func TestSubprocessNetwork(t *testing.T) { + seidBin := buildSeid(t) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + sn, err := StartSubprocess(ctx, Options{ + Validators: 3, + ChainID: "sei", + TimeoutCommit: time.Second, + }, seidBin) + if err != nil { + t.Fatalf("StartSubprocess: %v", err) + } + defer sn.Close() + + if err := sn.WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitReady: %v", err) + } + t.Logf("subprocess cluster N=%d reached consensus + serving EVM (no docker)", sn.Len()) +} + +// TestSubprocessCrashRecovery exercises the control lifecycle: on a 4-node cluster +// (which keeps producing with 3/4 online), kill one node, confirm the kill is +// node-scoped and the chain still advances, then restart it and confirm it recovers +// from its on-disk WAL and rejoins — all with no docker. +func TestSubprocessCrashRecovery(t *testing.T) { + seidBin := buildSeid(t) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) + defer cancel() + + sn, err := StartSubprocess(ctx, Options{ + Validators: 4, + ChainID: "sei", + TimeoutCommit: time.Second, + }, seidBin) + if err != nil { + t.Fatalf("StartSubprocess: %v", err) + } + defer sn.Close() + if err := sn.WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitReady: %v", err) + } + for i := 0; i < sn.Len(); i++ { + if !sn.IsRunning(i) { + t.Fatalf("node %d not running after WaitReady", i) + } + } + + // Kill node 3. The kill must be node-scoped: nodes 0-2 stay up. + if err := sn.Kill(3); err != nil { + t.Fatalf("Kill(3): %v", err) + } + if sn.IsRunning(3) { + t.Fatalf("node 3 still running after Kill") + } + for i := 0; i < 3; i++ { + if !sn.IsRunning(i) { + t.Fatalf("Kill(3) also took down node %d", i) + } + } + + // The chain keeps producing with 3/4 validators online. + h0, err := evmBlockHeight(ctx, sn.Node(0).EVMRPC()) + if err != nil { + t.Fatalf("read node0 height: %v", err) + } + if err := waitEVMHeight(ctx, sn.Node(0).EVMRPC(), h0+3); err != nil { + t.Fatalf("chain did not advance with node 3 down: %v", err) + } + + // Restart node 3; it replays its on-disk WAL and rejoins. + if err := sn.Restart(3); err != nil { + t.Fatalf("Restart(3): %v", err) + } + if !sn.IsRunning(3) { + t.Fatalf("node 3 not running after Restart") + } + if err := sn.Node(3).WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("node 3 did not recover after Restart: %v", err) + } + t.Logf("crash-recovery: killed + restarted node 3; it recovered via WAL and rejoined (no docker)") +} + +// TestSubprocessSnapshot verifies the snapshot need of the operational suites: with +// SnapshotInterval set, a validator writes a cosmos state-sync snapshot into +// /snapshots (the location the snapshot suite checks and a statesync joiner +// restores from) — no docker. +func TestSubprocessSnapshot(t *testing.T) { + seidBin := buildSeid(t) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) + defer cancel() + + const interval = 10 + sn, err := StartSubprocess(ctx, Options{ + Validators: 3, + ChainID: "sei", + TimeoutCommit: time.Second, + SnapshotInterval: interval, + }, seidBin) + if err != nil { + t.Fatalf("StartSubprocess: %v", err) + } + defer sn.Close() + if err := sn.WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitReady: %v", err) + } + if err := waitEVMHeight(ctx, sn.Node(0).EVMRPC(), interval+3); err != nil { + t.Fatalf("chain did not reach the snapshot interval: %v", err) + } + + snapDir := filepath.Join(sn.Node(0).Home(), "snapshots") + if err := waitSnapshot(ctx, snapDir); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("no snapshot written under %s: %v", snapDir, err) + } + t.Logf("snapshot: node 0 wrote a state-sync snapshot under %s (no docker)", snapDir) +} + +// TestSubprocessStatesync verifies the statesync need: a late-joining non-validator +// node (sei-rpc-node) state-syncs from the running validators' snapshots to a +// height > 0 — the exact assertion of the docker statesync suite, with no docker. +func TestSubprocessStatesync(t *testing.T) { + seidBin := buildSeid(t) + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + defer cancel() + + const interval = 10 + sn, err := StartSubprocess(ctx, Options{ + Validators: 3, + ChainID: "sei", + TimeoutCommit: time.Second, + SnapshotInterval: interval, + }, seidBin) + if err != nil { + t.Fatalf("StartSubprocess: %v", err) + } + defer sn.Close() + if err := sn.WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitReady: %v", err) + } + + // A snapshot must exist before the joiner starts, or it discovers nothing and + // loops at height 0. + if err := waitEVMHeight(ctx, sn.Node(0).EVMRPC(), interval+3); err != nil { + t.Fatalf("chain did not reach the snapshot interval: %v", err) + } + if err := waitSnapshot(ctx, filepath.Join(sn.Node(0).Home(), "snapshots")); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("no snapshot to sync from: %v", err) + } + + rpc, err := sn.AddStatesyncNode(ctx) + if err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("AddStatesyncNode: %v", err) + } + if err := waitTMHeight(ctx, rpc.TendermintRPC(), 1); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("%s did not state-sync to height > 0: %v", rpc.Name(), err) + } + t.Logf("statesync: %s synced from validator snapshots to height > 0 (no docker)", rpc.Name()) +} + +// waitTMHeight polls a node's CometBFT RPC /status until latest_block_height >= +// target or ctx fires — the statesync suite's own readiness signal. It reuses +// latestHeight's dual-shape parse (the node's /status may be enveloped or not). +func waitTMHeight(ctx context.Context, tmRPC string, target int64) error { + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + if h, ok := latestHeight(ctx, probeClient, tmRPC); ok && h >= target { + return nil + } + } + } +} + +// waitSnapshot polls snapDir until it holds at least one numeric height subdir (a +// real snapshot, not just the dir the store creates at init) or ctx fires. +func waitSnapshot(ctx context.Context, snapDir string) error { + tick := time.NewTicker(500 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + entries, err := os.ReadDir(snapDir) + if err != nil { + continue + } + for _, e := range entries { + if !e.IsDir() { + continue + } + if _, err := strconv.Atoi(e.Name()); err == nil { + return nil + } + } + } + } +} + +// evmBlockHeight returns a node's current block height via eth_blockNumber — a +// liveness probe independent of WaitReady. It reuses the package probe client +// (bounded per-request) + getJSON, matching readiness.go. +func evmBlockHeight(ctx context.Context, url string) (int64, error) { + const body = `{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}` + raw, ok := getJSON(ctx, probeClient, http.MethodPost, url, body) + if !ok { + return 0, fmt.Errorf("eth_blockNumber unreachable at %s", url) + } + var out struct { + Result string `json:"result"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return 0, err + } + if out.Result == "" { + return 0, fmt.Errorf("empty eth_blockNumber result from %s", url) + } + return strconv.ParseInt(strings.TrimPrefix(out.Result, "0x"), 16, 64) +} + +// waitEVMHeight polls url until its block height reaches target or ctx fires. +func waitEVMHeight(ctx context.Context, url string, target int64) error { + tick := time.NewTicker(500 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + if h, err := evmBlockHeight(ctx, url); err == nil && h >= target { + return nil + } + } + } +} + +// TestSubprocessUpgrade exercises the full upgrade path: register the upgrade +// handler on a quorum of nodes via Restart+WithEnv, pass a gov software-upgrade +// proposal, and confirm at the plan height that the upgraded nodes continue while a +// node left behind panics "UPGRADE NEEDED" — then recover it by restarting with the +// upgrade env. No docker. (The docker upgrade YAML suite can't run on the subprocess +// backend: its seid_upgrade.sh does a host-wide `pkill -f "seid start"`, which would +// kill every node — so the mechanism is proven here in Go via the harness gov API.) +func TestSubprocessUpgrade(t *testing.T) { + seidBin := buildSeid(t) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + const upgradeName = "v999.0.0" // valid semver; sorts after every embedded upgrade + sn, err := StartSubprocess(ctx, Options{ + // 4 validators: 3 upgraded keep >2/3 quorum past the upgrade height while the + // 1 left behind panics. + Validators: 4, + ChainID: "sei", + TimeoutCommit: time.Second, + GovVotingPeriod: 30 * time.Second, + }, seidBin) + if err != nil { + t.Fatalf("StartSubprocess: %v", err) + } + defer sn.Close() + if err := sn.WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitReady: %v", err) + } + + // Register the upgrade handler on nodes 0-2 via UPGRADE_VERSION_LIST (exercises + // Restart+WithEnv). Doing the restarts before the proposal removes any race with + // the chain reaching the plan height. + for i := 0; i < 3; i++ { + if err := sn.Restart(i, WithEnv("UPGRADE_VERSION_LIST="+upgradeName)); err != nil { + t.Fatalf("Restart(%d) with upgrade env: %v", i, err) + } + if err := sn.Node(i).WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("node %d not ready after upgrade-env restart: %v", i, err) + } + } + + // Schedule the upgrade a safe margin past the current height. The margin is + // coupled to GovVotingPeriod: at ~1 block/s (TimeoutCommit) it must exceed the + // voting period + tally so the proposal passes (plan scheduled) before the chain + // reaches the height — but not so far the test drags. ~30s vote window, ~70 blocks. + h, err := evmBlockHeight(ctx, sn.Node(0).EVMRPC()) + if err != nil { + t.Fatalf("read height: %v", err) + } + upgradeHeight := h + 70 + + id, err := sn.SubmitUpgradeProposal(ctx, upgradeName, upgradeHeight) + if err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("SubmitUpgradeProposal: %v", err) + } + if err := sn.VoteYes(ctx, id); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("VoteYes: %v", err) + } + if err := sn.WaitProposalPassed(ctx, id); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("WaitProposalPassed: %v", err) + } + + // At the upgrade height: nodes 0-2 apply the upgrade and keep producing (3/4); + // node 3, with no handler, panics "UPGRADE NEEDED". + if err := waitEVMHeight(ctx, sn.Node(0).EVMRPC(), upgradeHeight+2); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("upgraded nodes did not advance past the upgrade height: %v", err) + } + for i := 0; i < 3; i++ { + if !sn.IsRunning(i) { + dumpNodeLogs(t, sn) + t.Fatalf("upgraded node %d stopped at the upgrade height", i) + } + } + if err := waitNodeExited(ctx, sn, 3); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("baseline node 3 did not halt at the upgrade height: %v", err) + } + if !nodeLogContains(sn, 3, "NEEDED at height") { + dumpNodeLogs(t, sn) + t.Fatalf("node 3's halt was not the expected UPGRADE NEEDED panic") + } + + // Recover node 3 by restarting it with the upgrade env — it applies the upgrade on + // replay and rejoins (Restart+WithEnv on the panic-recovery path). + if err := sn.Restart(3, WithEnv("UPGRADE_VERSION_LIST="+upgradeName)); err != nil { + t.Fatalf("Restart(3) to recover: %v", err) + } + if err := sn.Node(3).WaitReady(ctx); err != nil { + dumpNodeLogs(t, sn) + t.Fatalf("node 3 did not recover after upgrade restart: %v", err) + } + t.Logf("upgrade: gov-scheduled software-upgrade at height %d — nodes 0-2 upgraded + continued, node 3 panicked then recovered (no docker)", upgradeHeight) +} + +// waitNodeExited blocks until node i's process has exited (the reaper observed it) or +// ctx fires. +func waitNodeExited(ctx context.Context, sn *SubprocessNetwork, i int) error { + tick := time.NewTicker(probeInterval) + defer tick.Stop() + for { + if !sn.IsRunning(i) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + } + } +} + +// nodeLogContains reports whether node i's seid.log contains substr. +func nodeLogContains(sn *SubprocessNetwork, i int, substr string) bool { + b, err := os.ReadFile(filepath.Join(sn.Node(i).Home(), "seid.log")) + if err != nil { + return false + } + return strings.Contains(string(b), substr) +} + +// dumpNodeLogs tails each node's seid.log to surface a boot failure. +func dumpNodeLogs(t *testing.T, sn *SubprocessNetwork) { + t.Helper() + for _, n := range sn.net.nodes { + if app, err := os.ReadFile(filepath.Join(n.home, "config", "app.toml")); err == nil { + t.Logf("=== %s app.toml [evm] ===", n.moniker) + for _, line := range strings.Split(string(app), "\n") { + if strings.Contains(line, "evm") || strings.Contains(line, "http_") || strings.Contains(line, "ws_") || strings.Contains(line, "[state") || strings.Contains(line, "ss-") || strings.Contains(line, "sc-") { + t.Logf(" %s", line) + } + } + } + b, err := os.ReadFile(filepath.Join(n.home, "seid.log")) + if err != nil { + continue + } + s := string(b) + if len(s) > 8000 { + s = s[len(s)-8000:] + } + t.Logf("=== %s seid.log (tail) ===\n%s", n.moniker, s) + } +} diff --git a/inprocess/upgrade.go b/inprocess/upgrade.go new file mode 100644 index 0000000000..bfdbf239d9 --- /dev/null +++ b/inprocess/upgrade.go @@ -0,0 +1,212 @@ +//go:build inprocess + +package inprocess + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "strconv" + "time" +) + +// Upgrade orchestration for the subprocess backend: submit a gov software-upgrade +// proposal, vote it through, and wait for it to pass, all via the `seid` CLI against +// the running cluster. Paired with Restart(i, WithEnv("UPGRADE_VERSION_LIST=...")), +// this drives the upgrade suites — a node whose UPGRADE_VERSION_LIST registers the +// scheduled upgrade applies it at the plan height; a node without it panics +// "UPGRADE NEEDED" (see app/upgrades.go). Requires Options.GovVotingPeriod > 0 so +// the proposal passes within the test. + +// Deposit and fee are both in usei — sei-chain's bond denom (and gov min-deposit +// denom) is usei, which is what the harness funds each operator with. The deposit +// exceeds the default gov min (10000000usei) so the proposal enters voting +// immediately; the fee clears the chain's usei minimum fee. +const ( + upgradeDeposit = "20000000usei" + upgradeFee = "200000usei" + upgradeGas = "2000000" + + // Repeated seid CLI tokens, hoisted so goconst doesn't trip on the duplicates. + subcmdGov = "gov" + flagOutput = "--output" + outputJSON = "json" + + // proposalPollInterval is WaitProposalPassed's status-poll cadence — slower than + // the in-memory probeInterval because each poll spawns a seid query. + proposalPollInterval = time.Second +) + +// SubmitUpgradeProposal submits a MINOR software-upgrade proposal (named name, +// scheduled at upgradeHeight) from node 0's operator key and returns the new proposal +// id. The proposal enters voting immediately (the deposit clears the gov minimum); +// call VoteYes then WaitProposalPassed to carry it through. +// +// Minor release semantics are what the upgrade suite needs (see sei-cosmos x/upgrade +// BeginBlocker): a node whose binary already registered the handler keeps running +// before the plan height and applies at it, while a node without the handler panics +// "UPGRADE NEEDED" at the height. (A major upgrade instead panics "BINARY UPDATED +// BEFORE TRIGGER" on any node holding the handler early — not modeled here.) +func (sn *SubprocessNetwork) SubmitUpgradeProposal(ctx context.Context, name string, upgradeHeight int64) (string, error) { + before := sn.maxProposalID(ctx) // 0 when none exist yet + + out, err := sn.seidCmd(ctx, 0, + "tx", subcmdGov, "submit-proposal", "software-upgrade", name, + "--title", name, "--description", "harness upgrade test", + "--upgrade-height", strconv.FormatInt(upgradeHeight, 10), + "--upgrade-info", `{"upgradeType":"minor"}`, + "--deposit", upgradeDeposit, + "--from", operatorKeyName, "--keyring-backend", "test", + "--chain-id", sn.net.opts.ChainID, + "--gas", upgradeGas, "--fees", upgradeFee, + "-b", "sync", "-y", flagOutput, outputJSON, + ) + if err != nil { + return "", err + } + if err := checkTxCode(out); err != nil { + return "", fmt.Errorf("submit-proposal: %w", err) + } + + // -b sync returns before inclusion, so poll gov state until the new proposal + // appears (id past the pre-submit max). + tick := time.NewTicker(probeInterval) + defer tick.Stop() + for { + if id := sn.maxProposalID(ctx); id > before { + return strconv.Itoa(id), nil + } + select { + case <-ctx.Done(): + return "", fmt.Errorf("proposal did not appear: %w", ctx.Err()) + case <-tick.C: + } + } +} + +// VoteYes casts a yes vote from every validator's operator key. All validators +// voting clears quorum + threshold. The statesync node (if any) is unbonded and is +// skipped. +func (sn *SubprocessNetwork) VoteYes(ctx context.Context, proposalID string) error { + for i, n := range sn.net.nodes { + if n.moniker == statesyncNodeMoniker { + continue + } + out, err := sn.seidCmd(ctx, i, + "tx", subcmdGov, "vote", proposalID, "yes", + "--from", operatorKeyName, "--keyring-backend", "test", + "--chain-id", sn.net.opts.ChainID, + "--gas", upgradeGas, "--fees", upgradeFee, + "-b", "sync", "-y", flagOutput, outputJSON, + ) + if err != nil { + return fmt.Errorf("vote from %s: %w", n.moniker, err) + } + if err := checkTxCode(out); err != nil { + return fmt.Errorf("vote from %s: %w", n.moniker, err) + } + } + return nil +} + +// WaitProposalPassed polls proposal status until PROPOSAL_STATUS_PASSED, or fails +// fast on a terminal rejected/failed status, or ctx fires. +func (sn *SubprocessNetwork) WaitProposalPassed(ctx context.Context, proposalID string) error { + tick := time.NewTicker(proposalPollInterval) + defer tick.Stop() + for { + if status, ok := sn.proposalStatus(ctx, proposalID); ok { + switch status { + case "PROPOSAL_STATUS_PASSED": + return nil + case "PROPOSAL_STATUS_REJECTED", "PROPOSAL_STATUS_FAILED": + return fmt.Errorf("proposal %s ended %s (not passed)", proposalID, status) + } + } + select { + case <-ctx.Done(): + return fmt.Errorf("proposal %s did not pass before deadline: %w", proposalID, ctx.Err()) + case <-tick.C: + } + } +} + +// maxProposalID returns the highest gov proposal id, or 0 when none exist / the +// query is momentarily unavailable (the caller polls). +func (sn *SubprocessNetwork) maxProposalID(ctx context.Context) int { + out, err := sn.seidCmd(ctx, 0, "query", subcmdGov, "proposals", flagOutput, outputJSON) + if err != nil { + return 0 // "no proposals found" is a non-zero exit; treat as none + } + var r struct { + Proposals []struct { + ID string `json:"proposal_id"` + } `json:"proposals"` + } + if json.Unmarshal([]byte(out), &r) != nil { + return 0 + } + highest := 0 + for _, p := range r.Proposals { + if id, err := strconv.Atoi(p.ID); err == nil && id > highest { + highest = id + } + } + return highest +} + +// proposalStatus reads one proposal's status. ok=false on an unavailable query. +func (sn *SubprocessNetwork) proposalStatus(ctx context.Context, proposalID string) (string, bool) { + out, err := sn.seidCmd(ctx, 0, "query", subcmdGov, "proposal", proposalID, flagOutput, outputJSON) + if err != nil { + return "", false + } + // The status is top-level in this gov version (the docker suite reads .status); + // tolerate a nested {"proposal":{...}} shape too. + var r struct { + Status string `json:"status"` + Proposal *struct { + Status string `json:"status"` + } `json:"proposal"` + } + if json.Unmarshal([]byte(out), &r) != nil { + return "", false + } + if r.Proposal != nil && r.Proposal.Status != "" { + return r.Proposal.Status, true + } + return r.Status, r.Status != "" +} + +// seidCmd runs the seid binary against node i (its home for the keyring, its RPC for +// --node) and returns stdout. On a non-zero exit it returns the captured stderr in +// the error. +func (sn *SubprocessNetwork) seidCmd(ctx context.Context, i int, args ...string) (string, error) { + n := sn.net.nodes[i] + full := append([]string{}, args...) + full = append(full, "--home", n.home, "--node", n.rpcAddr) + out, err := exec.CommandContext(ctx, sn.seidBin, full...).Output() //nolint:gosec + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + return string(out), fmt.Errorf("seid %s: %w: %s", args[0], err, ee.Stderr) + } + return string(out), err + } + return string(out), nil +} + +// checkTxCode fails if a broadcast tx's JSON response reports a non-zero code +// (CheckTx rejection). +func checkTxCode(out string) error { + var resp struct { + Code int `json:"code"` + RawLog string `json:"raw_log"` + } + if json.Unmarshal([]byte(out), &resp) == nil && resp.Code != 0 { + return fmt.Errorf("tx rejected (code %d): %s", resp.Code, resp.RawLog) + } + return nil +} diff --git a/integration_test/chain_operation/snapshot_operation.yaml b/integration_test/chain_operation/snapshot_operation.yaml index 2231ef728b..b2571940a7 100644 --- a/integration_test/chain_operation/snapshot_operation.yaml +++ b/integration_test/chain_operation/snapshot_operation.yaml @@ -1,7 +1,9 @@ - name: Test validators should be able to create snapshot with custom location inputs: - # Check if snapshots are created - - cmd: if [ -d "./build/generated/node_0/snapshots" ]; then echo "true"; else echo "false"; fi + # Check if snapshots are created. SEI_SNAPSHOT_DIR is set by the in-process/ + # subprocess runner to that node's snapshot dir; docker leaves it unset and falls + # back to its container-relative path. + - cmd: if [ -d "${SEI_SNAPSHOT_DIR:-./build/generated/node_0/snapshots}" ]; then echo "true"; else echo "false"; fi env: FOUND node: sei-node-0 verifiers: diff --git a/integration_test/runner/runner_inprocess.go b/integration_test/runner/runner_inprocess.go index 2e05b62ce0..21aa2cff17 100644 --- a/integration_test/runner/runner_inprocess.go +++ b/integration_test/runner/runner_inprocess.go @@ -20,35 +20,54 @@ import ( "github.com/sei-protocol/sei-chain/inprocess" ) -// WithInProcessNetwork selects the in-process backend: commands run on the HOST -// against a real `seid` binary pointed at one of net's in-process nodes, with no -// docker. The Input `node:` field ("sei-node-N", default "sei-node-0") selects -// the node; the command's `seid` invocations are redirected to that node's home -// (and its loopback TM RPC / EVM endpoints) so suites written for the docker +// nodeSource is the backend-agnostic surface the runner needs from a harness +// network: index a validator and count them. Both inprocess.Network (Tier-1, +// in-goroutine) and inprocess.SubprocessNetwork (Tier-2, real `seid` processes) +// satisfy it and return the same inprocess.Node handle, so one execer drives +// either backend — the seid CLIENT commands the suites run don't care whether the +// node they target is a goroutine or a subprocess. +type nodeSource interface { + Node(i int) inprocess.Node + Len() int +} + +// WithInProcessNetwork selects the Tier-1 in-process backend: commands run on the +// HOST against a real `seid` binary pointed at one of net's in-goroutine nodes, +// with no docker. The Input `node:` field ("sei-node-N", default "sei-node-0") +// selects the node; the command's `seid` invocations are redirected to that node's +// home (and its loopback TM RPC / EVM endpoints) so suites written for the docker // cluster run unchanged. -// -// The build tag means this option only exists in an `inprocess` build; docker -// runs (built without the tag) cannot reference it, and so cannot regress. func WithInProcessNetwork(net *inprocess.Network) Option { - return withExecer(newInProcessExecer(net)) + return withExecer(newHostExecer(net)) +} + +// WithSubprocessNetwork selects the Tier-2 subprocess backend: the same host-side +// seid client execer as WithInProcessNetwork, but targeting a cluster of real +// `seid` processes (see inprocess.SubprocessNetwork) instead of in-goroutine +// nodes. Suites run unchanged — the node handle surface is identical — while the +// nodes are now killable/restartable OS processes, which is what the operational +// suites need. +func WithSubprocessNetwork(sn *inprocess.SubprocessNetwork) Option { + return withExecer(newHostExecer(sn)) } -// inProcessExecer runs commands on the host against an inprocess.Network. It -// shims `seid` so opaque sourced helper scripts (which call bare `seid` / -// `$seidbin`) land on the right node: the shim prepends `--home "$SEID_HOME"` -// and `--node "$SEID_NODE"` to every real seid call (explicit home + RPC -// targeting), and the per-node client.toml the harness wrote under that home -// supplies chain-id and the test keyring. -type inProcessExecer struct { - net *inprocess.Network +// hostExecer runs seid client commands on the host against a harness network +// (either backend — see nodeSource). It shims `seid` so opaque sourced helper +// scripts (which call bare `seid` / `$seidbin`) land on the right node: the shim +// prepends `--home "$SEID_HOME"` to every real seid call and `--node "$SEID_NODE"` +// only to the RPC-reading client subcommands (q/query/tx/status — see shimScript), +// and the per-node client.toml the harness wrote under that home supplies chain-id +// and the test keyring. +type hostExecer struct { + net nodeSource once sync.Once binDir string // dir holding the seid shim + real binary, prepended to PATH setup error // first-build error, returned to every run after } -func newInProcessExecer(net *inprocess.Network) *inProcessExecer { - return &inProcessExecer{net: net} +func newHostExecer(net nodeSource) *hostExecer { + return &hostExecer{net: net} } // run resolves node → harness node, sets the per-node targeting env (SEID_HOME @@ -56,7 +75,7 @@ func newInProcessExecer(net *inprocess.Network) *inProcessExecer { // capture env, and runs the command on the host. Non-zero command exit is // reported via stdout (the captured code), matching the docker arm + runner.py // contract; err is reserved for harness-level failures. -func (e *inProcessExecer) run(t *testing.T, cmd, node string, envMap map[string]string, opts Options) (string, error) { +func (e *hostExecer) run(t *testing.T, cmd, node string, envMap map[string]string, opts Options) (string, error) { t.Helper() if err := e.ensureBin(t); err != nil { return "", fmt.Errorf("prepare seid: %w", err) @@ -87,6 +106,10 @@ func (e *inProcessExecer) run(t *testing.T, cmd, node string, envMap map[string] "SEI_EVM_WS="+h.EVMWS(), // Some EVM suites read EVM_RPC; keep parity with SEI_EVM_RPC. "EVM_RPC="+h.EVMRPC(), + // The snapshot suite checks this node's snapshot dir. Docker hardcodes its + // path; this env lets the suite stay backend-portable (the docker arm keeps + // its literal fallback — see snapshot_operation.yaml). + "SEI_SNAPSHOT_DIR="+filepath.Join(h.Home(), "snapshots"), ) out, err := c.Output() @@ -104,30 +127,34 @@ func (e *inProcessExecer) run(t *testing.T, cmd, node string, envMap map[string] // nodeFor maps a "sei-node-N" moniker (the docker container naming the suites // use) to the harness node at index N. An empty string defaults to node 0, the -// suite default (admin's home). -func (e *inProcessExecer) nodeFor(node string) (inprocess.Node, error) { - idx := 0 - if node != "" { - const prefix = "sei-node-" - s, ok := strings.CutPrefix(node, prefix) - if !ok { - return inprocess.Node{}, fmt.Errorf("in-process arm: node %q is not %sN", node, prefix) - } - n, err := strconv.Atoi(s) +// suite default (admin's home). A name without the "sei-node-" prefix is matched +// against node monikers, which resolves the subprocess backend's late-joining +// "sei-rpc-node" (the statesync suite's target). +func (e *hostExecer) nodeFor(node string) (inprocess.Node, error) { + if node == "" { + return e.net.Node(0), nil + } + if s, ok := strings.CutPrefix(node, "sei-node-"); ok { + idx, err := strconv.Atoi(s) if err != nil { - return inprocess.Node{}, fmt.Errorf("in-process arm: node %q has non-numeric index: %w", node, err) + return inprocess.Node{}, fmt.Errorf("host arm: node %q has non-numeric index: %w", node, err) } - idx = n + if idx < 0 || idx >= e.net.Len() { + return inprocess.Node{}, fmt.Errorf("host arm: node index %d out of range [0,%d)", idx, e.net.Len()) + } + return e.net.Node(idx), nil } - if idx < 0 || idx >= e.net.Len() { - return inprocess.Node{}, fmt.Errorf("in-process arm: node index %d out of range [0,%d)", idx, e.net.Len()) + for i := 0; i < e.net.Len(); i++ { + if h := e.net.Node(i); h.Name() == node { + return h, nil + } } - return e.net.Node(idx), nil + return inprocess.Node{}, fmt.Errorf("host arm: no node named %q (not sei-node-N and no matching moniker)", node) } // prepare is the backendPreparer hook: it runs the one-time build (via ensureBin) // against the parent test. See ensureBin for why the parent owns the cleanup. -func (e *inProcessExecer) prepare(t *testing.T) error { +func (e *hostExecer) prepare(t *testing.T) error { t.Helper() return e.ensureBin(t) } @@ -143,7 +170,7 @@ func (e *inProcessExecer) prepare(t *testing.T) error { // t.Cleanup registers on whichever test first triggers the build; prepare makes // that the parent test, so the binary outlives every per-case subtest. Cases run // serially — the unsynchronized binDir read in run is safe only without t.Parallel. -func (e *inProcessExecer) ensureBin(t *testing.T) error { +func (e *hostExecer) ensureBin(t *testing.T) error { e.once.Do(func() { dir, err := os.MkdirTemp("", "sei-inprocess-bin-") if err != nil { diff --git a/integration_test/runner_subprocess/runner_subprocess_test.go b/integration_test/runner_subprocess/runner_subprocess_test.go new file mode 100644 index 0000000000..08ec87b91f --- /dev/null +++ b/integration_test/runner_subprocess/runner_subprocess_test.go @@ -0,0 +1,159 @@ +//go:build inprocess + +// Package runner_subprocess's Tier-2 arm runs the YAML suites against a shared +// cluster of real `seid` PROCESSES (inprocess.SubprocessNetwork) — no docker, no +// in-goroutine node. It proves the same suites the in-process arm runs also pass +// against real seid processes booted with `seid start`, and it is the future home +// for the operational suites (upgrade / crash-recovery / statesync) that need the +// real process boundary the in-process arm cannot provide. +// +// One subprocess cluster is brought up in TestMain and reused across suites, +// mirroring the docker job model and the in-process arm; suites run sequentially +// against it. +// +// go test -tags inprocess -v -timeout 900s ./integration_test/runner_subprocess/ +package runner_subprocess + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + + "github.com/sei-protocol/sei-chain/inprocess" + "github.com/sei-protocol/sei-chain/integration_test/runner" +) + +// chainID is the chain-id the suites sign with (`--chain-id=sei`); the harness and +// the per-node client.toml it writes carry the same id. +const chainID = "sei" + +// adminFunding mirrors the docker step2_genesis admin grant +// (1000000000000000000000usei), matching the in-process arm so the shared +// send-suite YAMLs behave identically. +func adminFunding() sdk.Coins { + amt, ok := sdk.NewIntFromString("1000000000000000000000") + if !ok { + panic("bad admin funding literal") + } + return sdk.NewCoins(sdk.NewCoin("usei", amt)) +} + +// sharedNet is the one subprocess cluster every suite runs against. TestMain owns +// its lifecycle; it is non-nil for the duration of m.Run(). Suites run +// sequentially (the per-node keyring the suites mutate is not concurrent-safe), so +// no suite may call t.Parallel(). +var sharedNet *inprocess.SubprocessNetwork + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} + +// run brings up the shared subprocess cluster, runs the suites, and tears it down. +// It is split from TestMain so the deferred Close/cleanup run before os.Exit (which +// skips defers). +func run(m *testing.M) int { + // go test's own -timeout is the authoritative run bound; this generous ctx is + // the backstop that parents the node processes (StartSubprocess binds each to + // it), so it must outlive m.Run(). + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + + seidBin, binDir, err := buildSeid() + if err != nil { + fmt.Fprintf(os.Stderr, "build seid: %v\n", err) + return 1 + } + defer os.RemoveAll(binDir) + + sn, err := inprocess.StartSubprocess(ctx, inprocess.Options{ + // Three validators — the smallest real multi-node topology (see + // inprocess.Options.Validators). admin is genesis-funded on node 0 (the + // suites' signing key), exactly as the in-process arm funds it. + Validators: 3, + ChainID: chainID, + TimeoutCommit: time.Second, + // Take a state-sync snapshot every 10 blocks so the snapshot suite finds one + // and the statesync joiner has something to restore from. + SnapshotInterval: 10, + ExtraKeys: []inprocess.ExtraKey{ + {Name: "admin", Node: 0, Coins: adminFunding()}, + }, + }, seidBin) + if err != nil { + fmt.Fprintf(os.Stderr, "StartSubprocess: %v\n", err) + return 1 + } + defer sn.Close() + + if err := sn.WaitReady(ctx); err != nil { + fmt.Fprintf(os.Stderr, "WaitReady: %v\n%s", err, sn.LogTails()) + return 1 + } + + // Add the late-joining sei-rpc-node once a snapshot exists (AddStatesyncNode + // waits for one), then wait for it to state-sync and start advancing — so the + // statesync suite finds it at height > 0. + rpc, err := sn.AddStatesyncNode(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "AddStatesyncNode: %v\n%s", err, sn.LogTails()) + return 1 + } + if err := rpc.WaitReady(ctx); err != nil { + fmt.Fprintf(os.Stderr, "statesync node WaitReady: %v\n%s", err, sn.LogTails()) + return 1 + } + sharedNet = sn + return m.Run() +} + +// buildSeid builds the production seid binary (no build tags — the real `seid +// start` path the subprocess nodes run) into a temp dir, returning the binary path +// and its dir for cleanup. +func buildSeid() (bin, dir string, err error) { + wd, err := os.Getwd() + if err != nil { + return "", "", err + } + // go test runs with CWD = the package dir; climb to the module root so `go build + // ./cmd/seid` resolves. + root := filepath.Clean(filepath.Join(wd, "..", "..")) + dir, err = os.MkdirTemp("", "sei-subprocess-node-") + if err != nil { + return "", "", err + } + bin = filepath.Join(dir, "seid") + cmd := exec.Command("go", "build", "-o", bin, "./cmd/seid") + cmd.Dir = root + if out, berr := cmd.CombinedOutput(); berr != nil { + _ = os.RemoveAll(dir) + return "", "", fmt.Errorf("go build seid: %w\n%s", berr, out) + } + return bin, dir, nil +} + +// TestSubprocessBankModule runs the bank send suite against the shared subprocess +// cluster: a genesis-funded `admin` on node 0 drives a real bank tx + historical +// balance queries against a real `seid` process, no docker. +func TestSubprocessBankModule(t *testing.T) { + runner.RunFile(t, "../bank_module/send_funds_test.yaml", runner.WithSubprocessNetwork(sharedNet)) +} + +// TestSubprocessSnapshotModule runs the docker snapshot suite unchanged against the +// subprocess cluster: node 0 (SnapshotInterval set) has written a state-sync +// snapshot, which the suite finds via SEI_SNAPSHOT_DIR — no docker. +func TestSubprocessSnapshotModule(t *testing.T) { + runner.RunFile(t, "../chain_operation/snapshot_operation.yaml", runner.WithSubprocessNetwork(sharedNet)) +} + +// TestSubprocessStatesyncModule runs the docker statesync suite unchanged against +// the subprocess cluster: the late-joining sei-rpc-node (added in TestMain) has +// state-synced from the validators' snapshots to height > 0 — no docker. +func TestSubprocessStatesyncModule(t *testing.T) { + runner.RunFile(t, "../chain_operation/statesync_operation.yaml", runner.WithSubprocessNetwork(sharedNet)) +}