diff --git a/.gitignore b/.gitignore index 22bc8f008..f889e9832 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,4 @@ data *.sqlite-wal **aggkit-001-data** .vscode -debug -bin \ No newline at end of file +debug \ No newline at end of file diff --git a/aggsender/rpcclient/client.go b/aggsender/rpcclient/client.go index 3d711b5df..6c5f15662 100644 --- a/aggsender/rpcclient/client.go +++ b/aggsender/rpcclient/client.go @@ -1,29 +1,35 @@ package rpcclient import ( + "context" "encoding/json" "fmt" + "time" "github.com/0xPolygon/cdk-rpc/rpc" agglayertypes "github.com/agglayer/aggkit/agglayer/types" "github.com/agglayer/aggkit/aggsender/types" ) -var jSONRPCCall = rpc.JSONRPCCall +var jSONRPCCallWithContext = rpc.JSONRPCCallWithContext + +const defaultRequestTimeout = 10 * time.Second // Client wraps all the available endpoints of the data abailability committee node server type Client struct { - url string + url string + requestTimeout time.Duration } func NewClient(url string) *Client { return &Client{ - url: url, + url: url, + requestTimeout: defaultRequestTimeout, } } func (c *Client) GetStatus() (*types.AggsenderInfo, error) { - response, err := jSONRPCCall(c.url, "aggsender_status") + response, err := c.call("aggsender_status") if err != nil { return nil, err } @@ -41,7 +47,7 @@ func (c *Client) GetStatus() (*types.AggsenderInfo, error) { } func (c *Client) GetCertificateHeaderPerHeight(height *uint64) (*types.Certificate, error) { - response, err := jSONRPCCall(c.url, "aggsender_getCertificateHeaderPerHeight", height) + response, err := c.call("aggsender_getCertificateHeaderPerHeight", height) if err != nil { return nil, err } @@ -61,7 +67,7 @@ func (c *Client) GetCertificateHeaderPerHeight(height *uint64) (*types.Certifica // GetCertificateBridgeExits returns the bridge exits for the certificate at the given height. // If height is nil, returns the bridge exits of the last sent certificate. func (c *Client) GetCertificateBridgeExits(height *uint64) ([]*agglayertypes.BridgeExit, error) { - response, err := jSONRPCCall(c.url, "aggsender_getCertificateBridgeExits", height) + response, err := c.call("aggsender_getCertificateBridgeExits", height) if err != nil { return nil, err } @@ -74,3 +80,13 @@ func (c *Client) GetCertificateBridgeExits(height *uint64) ([]*agglayertypes.Bri } return exits, nil } + +func (c *Client) call(method string, params ...interface{}) (rpc.Response, error) { + timeout := c.requestTimeout + if timeout <= 0 { + timeout = defaultRequestTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return jSONRPCCallWithContext(ctx, c.url, method, params...) +} diff --git a/aggsender/rpcclient/client_test.go b/aggsender/rpcclient/client_test.go index 929f9b201..7cb79fd42 100644 --- a/aggsender/rpcclient/client_test.go +++ b/aggsender/rpcclient/client_test.go @@ -1,10 +1,12 @@ package rpcclient import ( + "context" "encoding/json" "fmt" "math/big" "testing" + "time" "github.com/0xPolygon/cdk-rpc/rpc" agglayertypes "github.com/agglayer/aggkit/agglayer/types" @@ -22,7 +24,7 @@ func TestGetCertificateHeaderPerHeight(t *testing.T) { response := rpc.Response{ Result: responseCertJSON, } - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return response, nil } cert, err := sut.GetCertificateHeaderPerHeight(&height) @@ -47,7 +49,7 @@ func TestGetCertificateBridgeExits(t *testing.T) { response := rpc.Response{ Result: responseExitsJSON, } - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return response, nil } exits, err := sut.GetCertificateBridgeExits(&height) @@ -65,7 +67,7 @@ func TestGetStatus(t *testing.T) { response := rpc.Response{ Result: responseDataJSON, } - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return response, nil } result, err := sut.GetStatus() @@ -74,11 +76,33 @@ func TestGetStatus(t *testing.T) { require.Equal(t, responseData, *result) } +func TestClientCallUsesTimeout(t *testing.T) { + sut := NewClient("url") + sut.requestTimeout = 50 * time.Millisecond + + responseData := types.AggsenderInfo{} + responseDataJSON, err := json.Marshal(responseData) + require.NoError(t, err) + response := rpc.Response{ + Result: responseDataJSON, + } + jSONRPCCallWithContext = func(ctx context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.Positive(t, time.Until(deadline)) + require.LessOrEqual(t, time.Until(deadline), 50*time.Millisecond) + return response, nil + } + + _, err = sut.GetStatus() + require.NoError(t, err) +} + func TestGetStatus_Errors(t *testing.T) { sut := NewClient("url") t.Run("rpc call error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{}, fmt.Errorf("network error") } _, err := sut.GetStatus() @@ -86,7 +110,7 @@ func TestGetStatus_Errors(t *testing.T) { }) t.Run("response error field set", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Error: &rpc.ErrorObject{Message: "rpc error"}}, nil } _, err := sut.GetStatus() @@ -95,7 +119,7 @@ func TestGetStatus_Errors(t *testing.T) { }) t.Run("unmarshal error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Result: json.RawMessage("not-json")}, nil } _, err := sut.GetStatus() @@ -109,7 +133,7 @@ func TestGetCertificateHeaderPerHeight_Errors(t *testing.T) { height := uint64(1) t.Run("rpc call error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{}, fmt.Errorf("network error") } _, err := sut.GetCertificateHeaderPerHeight(&height) @@ -117,7 +141,7 @@ func TestGetCertificateHeaderPerHeight_Errors(t *testing.T) { }) t.Run("response error field set", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Error: &rpc.ErrorObject{Message: "rpc error"}}, nil } _, err := sut.GetCertificateHeaderPerHeight(&height) @@ -126,7 +150,7 @@ func TestGetCertificateHeaderPerHeight_Errors(t *testing.T) { }) t.Run("unmarshal error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Result: json.RawMessage("not-json")}, nil } _, err := sut.GetCertificateHeaderPerHeight(&height) @@ -140,7 +164,7 @@ func TestGetCertificateBridgeExits_Errors(t *testing.T) { height := uint64(5) t.Run("rpc call error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{}, fmt.Errorf("network error") } _, err := sut.GetCertificateBridgeExits(&height) @@ -148,7 +172,7 @@ func TestGetCertificateBridgeExits_Errors(t *testing.T) { }) t.Run("response error field set", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Error: &rpc.ErrorObject{Message: "rpc error"}}, nil } _, err := sut.GetCertificateBridgeExits(&height) @@ -157,7 +181,7 @@ func TestGetCertificateBridgeExits_Errors(t *testing.T) { }) t.Run("unmarshal error", func(t *testing.T) { - jSONRPCCall = func(_, _ string, _ ...interface{}) (rpc.Response, error) { + jSONRPCCallWithContext = func(_ context.Context, _, _ string, _ ...interface{}) (rpc.Response, error) { return rpc.Response{Result: json.RawMessage("not-json")}, nil } _, err := sut.GetCertificateBridgeExits(&height) diff --git a/docs/backward_forward_let_runbook.md b/docs/backward_forward_let_runbook.md deleted file mode 100644 index fcddbfeab..000000000 --- a/docs/backward_forward_let_runbook.md +++ /dev/null @@ -1,225 +0,0 @@ -# Backward/Forward LET runbook - -## Introduction - -The Local Exit Tree (LET) on L2 must stay consistent with the Local Exit Root (LER) -settled on L1 through the AggLayer. When they diverge, future certificates can be -rejected until the L2 bridge is reconciled. - -Use `backward-forward-let` for this workflow. The tool already: - -- reads the settled AggLayer state, -- reads the current L2 bridge state, -- queries aggsender for certificate bridge exits, -- finds the divergence point, -- classifies the recovery case, -- prints the recovery plan, -- activates emergency mode when needed, -- executes `BackwardLET` and/or `ForwardLET`, -- verifies deposit count and LER after each step, -- deactivates emergency mode at the end. - -This runbook documents the operator flow. It intentionally avoids manual diagnosis steps -that are already implemented in the tool. - -## When to run this - -Run the tool when the bridge appears out of sync with the last settled AggLayer state, -for example: - -- a certificate is rejected or transitions to `InError`, -- aggsender repeatedly fails to build or send certificates, -- an L2 reorg or aggsender issue is suspected to have settled the wrong LET state. - -The tool determines whether there is actual divergence. Do not manually compare L1 and -L2 state unless you are debugging the tool itself. - -## Prerequisites - -Prepare an aggkit config file that includes the normal chain and AggLayer settings plus -the `BackwardForwardLET` section used by the tool. - -Required config inputs: - -- `Common.L2RPC.URL` -- `BridgeL2Sync.BridgeAddr` -- `AgglayerClient` -- `BackwardForwardLET.BridgeServiceURL` -- `BackwardForwardLET.AggsenderRPCURL` -- `BackwardForwardLET.L2NetworkID` -- `BackwardForwardLET.GERRemoverKey` -- `BackwardForwardLET.EmergencyPauserKey` -- `BackwardForwardLET.EmergencyUnpauserKey` - -Role expectations: - -- `GERRemoverKey` must be allowed to call `backwardLET` and `forwardLET`. -- `EmergencyPauserKey` must be allowed to activate emergency state. -- `EmergencyUnpauserKey` must be allowed to deactivate emergency state. - -The tool handles emergency-mode activation and deactivation itself. There is no separate -manual pause/unpause step in the normal flow. - -For staged malicious-certificate drills used to create divergence intentionally: - -- stop aggkit/aggsender before crafting or sending malicious certificates so normal - certificate production does not race the drill, -- confirm there is no unrelated non-error pending certificate already occupying the next - height before sending the malicious cert, -- if the drill includes genuine L2 bridge creation, wait for bridge-service indexing before - expecting diagnosis or recovery to reason about those bridges, -- restart aggkit/aggsender only after all malicious certificates for that drill have been - submitted. - -Aggsender restart caveat: - -- aggsender intentionally refuses to auto-reconcile if its local DB still points to an - older or different certificate than the one already settled on AggLayer, -- if startup logs that the local certificate state is inconsistent with a further - AggLayer certificate, the operator must wipe the aggsender DB and restart aggsender, -- there is no supported automatic recovery path for that mismatch. - -## Standard procedure - -Run the tool: - -```bash -backward-forward-let --cfg aggkit-config.toml -``` - -For non-interactive execution: - -```bash -backward-forward-let --cfg aggkit-config.toml --yes -``` - -What happens next: - -1. The tool validates connectivity to the bridge service, L2 RPC, AggLayer, and aggsender. -2. It diagnoses the current state and prints one of: - - `NoDivergence` - - a recovery case with the divergence point and affected leaves - - a missing-certificate report if aggsender cannot provide bridge exits -3. If recovery is needed, it prints the exact recovery plan. -4. It asks for confirmation unless `--yes` is set. -5. It executes the required on-chain steps and verifies the resulting deposit count and LER. - -Operational notes from staging: - -- A just-created genuine L2 bridge is not usable by the tool until bridge service has - indexed it. If diagnosis says a deposit is not indexed yet, wait for bridge-service - catch-up instead of improvising a manual recovery. -- In staged Case 3 drills, the state after only the first malicious certificate settles is - still effectively Case 1. Final Case 3 classification only appears after the second - malicious certificate also settles. -- After staged malicious-certificate drills, aggsender may fail its startup consistency - checks because its local DB still points to a pre-drill certificate while AggLayer is - already further ahead. In that case, wipe the aggsender DB and restart it before - expecting honest certificate production to resume. - -Recovery behavior by case: - -- Case 1 and Case 3: `ForwardLET` only. -- Case 2 and Case 4: `BackwardLET`, then `ForwardLET` for divergent settled leaves, then a - second `ForwardLET` when extra real L2 bridges must be replayed. - -## Expected outcomes - -- If the tool reports `NoDivergence`, no action is required. -- If the tool completes recovery successfully, the L2 bridge is reconciled to the settled - AggLayer state and emergency mode is turned off before exit. -- If the tool reports missing certificate bridge exits, stop and use the fallback flow - below. -- For staged Case 2 or Case 4 drills, if recovery replays genuine L2 bridges while - aggsender is still stopped, the first post-recovery rerun may still show divergence. - In that situation, restart aggsender, wait for the honest follow-up certificate(s) to - settle, then rerun until the tool reports `NoDivergence`. - -## Fallback when aggsender bridge exits are unavailable - -If aggsender RPC cannot supply bridge exits for one or more settled certificate heights, -the tool prints an actionable report listing the missing heights and any certificate IDs -it could resolve automatically. - -When aggsender is intentionally stopped for a fallback drill, this missing range may span -the full settled history from height `0` through the latest settled certificate. That is -expected; build an override file for the heights the tool needs and rerun with that data. - -Re-run the tool with an override file once you have the missing bridge exits: - -```bash -backward-forward-let --cfg aggkit-config.toml \ - --cert-exits-file certificate_exits_override.json -``` - -The override file is only a fallback for missing certificate exits. Diagnosis and -recovery still stay tool-driven. - -The same override file can also be supplied to `backward-forward-let craft-cert` when a -later malicious certificate must be crafted while aggsender is still unavailable. - -For the detailed fallback procedure, including AggLayer admin/debug endpoint -prerequisites and override-file handling examples, see -[`tools/backward_forward_let/RECOVERY_PROCEDURE.md`](../tools/backward_forward_let/RECOVERY_PROCEDURE.md). - -### Step 1: fetch missing certificates from the AggLayer admin API - -For each certificate ID reported by the tool, call `admin_getCertificate`: - -```bash -AGGLAYER_ADMIN="http://localhost:4446" -CERT_ID="0xabc123...def456" - -curl -s -X POST "$AGGLAYER_ADMIN" \ - -H "Content-Type: application/json" \ - -d "{\"jsonrpc\":\"2.0\",\"method\":\"admin_getCertificate\",\"params\":[\"$CERT_ID\"],\"id\":1}" -``` - -Use `result[0].bridge_exits` from the response. - -If the tool reports `CertID: UNKNOWN`, the AggLayer admin must first resolve that -certificate ID from AggLayer state before you can fetch its `bridge_exits`. - -### Step 2: build the override file - -The override file must use Go JSON field names for `BridgeExit` objects: - -```json -{ - "network_id": 1, - "description": "Extracted from agglayer admin_getCertificate", - "heights": { - "3": [ - { - "leaf_type": 0, - "token_info": { - "origin_network": 0, - "origin_token_address": "0x0000000000000000000000000000000000000000" - }, - "dest_network": 1, - "dest_address": "0xAbCd...1234", - "amount": "1000000000000000000", - "metadata": null - } - ] - } -} -``` - -Constraints: - -- `network_id` must match the affected L2 network. -- `heights` keys are certificate heights as decimal strings. -- `amount` is a decimal string. -- `metadata` is `null` or base64-encoded bytes. -- Use `dest_network` and `dest_address`, not Rust serde field names. - -### Step 3: rerun the tool - -```bash -backward-forward-let --cfg aggkit-config.toml \ - --cert-exits-file certificate_exits_override.json -``` - -The tool will resume diagnosis using the override data, print the recovery plan, and -execute the same standard recovery flow. diff --git a/test/e2e/backwardforwardlet_test.go b/test/e2e/backwardforwardlet_test.go index f80be6371..e3ad0ce4a 100644 --- a/test/e2e/backwardforwardlet_test.go +++ b/test/e2e/backwardforwardlet_test.go @@ -229,10 +229,11 @@ func TestBackwardForwardLET_Case2(t *testing.T) { defer recoveryCancel() err = bfl.ExecuteRecovery(recoveryCtx, toolEnv, diagnosis) require.NoError(t, err) + waitForAggsenderFollowUpCertificate(ctx, t, toolEnv, "case2-recovery") // Verify: DC should equal DivergencePoint + divergent leaves + extra real bridges. - // For Case2, L2 LER will NOT match L1 settled LER because extra real L2 bridges were - // appended after the fake leaf; the next aggsender cert will advance L1 to match. + // For Case2, recovery appends extra real L2 bridges after the fake leaf. The follow-up + // aggsender certificate above advances L1 to the recovered L2 state before the next test. callOpts := &bind.CallOpts{Context: ctx} expectedDC := diagnosis.DivergencePoint + uint32(len(diagnosis.DivergentLeaves)) + uint32(len(diagnosis.ExtraL2Bridges)) @@ -382,10 +383,11 @@ func TestBackwardForwardLET_Case4(t *testing.T) { defer recoveryCancel() err = bfl.ExecuteRecovery(recoveryCtx, toolEnv, diagnosis) require.NoError(t, err) + waitForAggsenderFollowUpCertificate(ctx, t, toolEnv, "case4-recovery") // Verify: DC should equal DivergencePoint + divergent leaves + extra real bridges. - // For Case4, L2 LER will NOT match L1 settled LER because extra real L2 bridges were - // appended after the fake leaves; the next aggsender cert will advance L1 to match. + // For Case4, recovery appends extra real L2 bridges after the fake leaves. The follow-up + // aggsender certificate above advances L1 to the recovered L2 state before later tests. callOpts := &bind.CallOpts{Context: ctx} expectedDC := diagnosis.DivergencePoint + uint32(len(diagnosis.DivergentLeaves)) + uint32(len(diagnosis.ExtraL2Bridges)) @@ -827,6 +829,74 @@ func waitForCertificateToSettle( require.NoError(t, err, "timeout waiting for certificate at height=%d to settle", expectedHeight) } +func waitForAggsenderFollowUpCertificate(ctx context.Context, t *testing.T, toolEnv *bfl.Env, label string) { + t.Helper() + waitForBridgeServiceSynced(ctx, t) + + callOpts := &bind.CallOpts{Context: ctx} + root, err := toolEnv.L2Bridge.GetRoot(callOpts) + require.NoError(t, err, "get L2 root before aggsender follow-up") + l2LER := common.Hash(root) + dcBig, err := toolEnv.L2Bridge.DepositCount(callOpts) + require.NoError(t, err, "get L2 deposit count before aggsender follow-up") + l2DC := uint32(dcBig.Uint64()) + + info, err := toolEnv.AgglayerClient.GetNetworkInfo(ctx, toolEnv.L2NetworkID) + require.NoError(t, err, "get network info before aggsender follow-up") + if agglayerMatchesL2(info, l2LER, l2DC) { + log.Infof("[waitForAggsenderFollowUpCertificate] %s already reconciled at dc=%d", label, l2DC) + return + } + + log.Infof("[waitForAggsenderFollowUpCertificate] %s triggering aggsender to reconcile dc=%d ler=%s", + label, l2DC, l2LER.Hex()) + triggerAggsenderCertificate(ctx, t) + + err = pollWithBackoff(ctx, bflNoPendingTimeout, backoffInitial, backoffMax, + "aggsender-follow-up-"+label, + func() (bool, error) { + pollInfo, pollErr := toolEnv.AgglayerClient.GetNetworkInfo(ctx, toolEnv.L2NetworkID) + if pollErr != nil { + log.Debugf("[waitForAggsenderFollowUpCertificate] GetNetworkInfo error (retrying): %v", pollErr) + return false, nil + } + + settledH := nilStr + if pollInfo.SettledHeight != nil { + settledH = fmt.Sprintf("%d", *pollInfo.SettledHeight) + } + settledLER := nilStr + if pollInfo.SettledLER != nil { + settledLER = pollInfo.SettledLER.Hex() + } + settledDC := nilStr + if pollInfo.SettledLETLeafCount != nil { + settledDC = fmt.Sprintf("%d", *pollInfo.SettledLETLeafCount) + } + log.Debugf("[waitForAggsenderFollowUpCertificate] settledH=%s settledLER=%s settledDC=%s", + settledH, settledLER, settledDC) + return agglayerMatchesL2(pollInfo, l2LER, l2DC), nil + }, + ) + require.NoError(t, err, "timeout waiting for aggsender follow-up certificate after %s", label) +} + +func agglayerMatchesL2(info agglayertypes.NetworkInfo, l2LER common.Hash, l2DC uint32) bool { + return info.SettledLER != nil && + *info.SettledLER == l2LER && + info.SettledLETLeafCount != nil && + uint32(*info.SettledLETLeafCount) == l2DC +} + +func triggerAggsenderCertificate(ctx context.Context, t *testing.T) { + t.Helper() + response, err := rpc.JSONRPCCallWithContext(ctx, testEnv.AggsenderRPCURL, "aggsender_triggerCertificate") + require.NoError(t, err, "trigger aggsender certificate") + if response.Error != nil { + require.Failf(t, "trigger aggsender certificate", "RPC error: %v", response.Error) + } +} + // loadCertSignerKey loads the sequencer keystore (the agglayer proof signer for PP networks). func loadCertSignerKey(t *testing.T) *ecdsa.PrivateKey { t.Helper() diff --git a/tools/backward_forward_let/RECOVERY_PROCEDURE.md b/tools/backward_forward_let/RECOVERY_PROCEDURE.md index 105f848db..4d5ff084a 100644 --- a/tools/backward_forward_let/RECOVERY_PROCEDURE.md +++ b/tools/backward_forward_let/RECOVERY_PROCEDURE.md @@ -1,301 +1,11 @@ -# Backward/Forward LET — Manual Recovery Procedure +# Backward/Forward LET Recovery Procedure -This document describes the steps for recovering from a backward/forward LET divergence -when the aggsender database is empty or has been wiped. In this situation the tool cannot -fetch bridge exits from the aggsender RPC and instead needs the data extracted directly -from the agglayer node. +The canonical operator workflow now lives in the public runbook: -It also covers the post-drill startup failure where aggsender's local DB still points to -an older or different certificate while AggLayer has already settled a further one. That -startup check is intentionally strict: wipe the aggsender DB and restart aggsender rather -than expecting it to auto-reconcile. +- [Backward/Forward LET Recovery](https://github.com/agglayer/runbooks/blob/main/operations/backward-forward-let-recovery.md) ---- +Use that runbook for diagnosis, missing certificate exits, `export-cert-exits`, +`--cert-exits-file`, recovery execution, verification, and escalation data. -## Prerequisites - -- The agglayer node must have `debug-mode = true` in its configuration. - In the op-pp E2E environment this is already set (`debug-mode = true` in - `test/e2e/envs/op-pp/config/agglayer/config.toml`). -- The agglayer admin JSON-RPC API must be reachable (default port 4446). - The URL is exposed as `agglayer.services.admin_api.external` in `summary.json`. -- In some staging environments the admin API is protected by IAP or another identity - layer rather than being directly reachable on `localhost:4446`. In that case, obtain - the required bearer token first and pass it with the request headers when calling - `admin_getCertificate`. -- `curl` and `jq` must be installed on the operator's machine (`jq` is optional but - makes the JSON manipulation much more convenient). - -If the immediate goal is to recover aggsender itself after a staged malicious-cert drill, -stop here and wipe the aggsender DB first. The mismatch is not repaired in place. - ---- - -## Step 1 — Run the tool to discover missing cert IDs - -```bash -backward-forward-let --cfg aggkit-config.toml -``` - -When the aggsender DB is empty the tool prints an actionable report: - -``` -WARNING: Aggsender RPC returned no bridge exit data for the following certificate heights. -Recovery cannot proceed until this data is provided. - -Missing certificates (2 heights): - Height 3 CertID: 0xabc123...def456 [ID auto-resolved] - Height 2 CertID: UNKNOWN [contact agglayer admin for cert ID] -``` - -- **`[ID auto-resolved]`** — the tool resolved the cert ID from the agglayer gRPC. You can - call `admin_getCertificate` directly in Step 2. -- **`UNKNOWN`** — the cert ID could not be resolved automatically (only the latest settled - height is resolvable via the public gRPC). The agglayer admin must look up - `(network_id, height)` in the `certificate_per_network_cf` column family of the agglayer - state DB and supply the cert ID manually before you can proceed. - ---- - -## Step 2 — Fetch each certificate from the agglayer admin API - -For each cert ID printed by the tool, call `admin_getCertificate`: - -```bash -AGGLAYER_ADMIN="http://localhost:4446" -CERT_ID="0xabc123...def456" - -curl -s -X POST "$AGGLAYER_ADMIN" \ - -H "Content-Type: application/json" \ - -d "{\"jsonrpc\":\"2.0\",\"method\":\"admin_getCertificate\",\"params\":[\"$CERT_ID\"],\"id\":1}" \ - | jq '.' -``` - -The response is a JSON-RPC result where `result` is a two-element array -`[Certificate, CertificateHeader|null]`: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "result": [ - { - "network_id": 1, - "height": 3, - "bridge_exits": [ ... ], - ... - }, - { ... } - ] -} -``` - -You need `result[0].bridge_exits` from each response. - -If the admin API requires a bearer token, include it explicitly: - -```bash -JWT="..." -curl -s -X POST "$AGGLAYER_ADMIN" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $JWT" \ - -d "{\"jsonrpc\":\"2.0\",\"method\":\"admin_getCertificate\",\"params\":[\"$CERT_ID\"],\"id\":1}" \ - | jq '.' -``` - ---- - -## Step 3 — Build the JSON override file - -### Field name note - -The override file uses the **Go `json` tag names** from `agglayertypes.BridgeExit`: - -| Go field | JSON key | -|----------------------|------------------------| -| `LeafType` | `leaf_type` | -| `TokenInfo` | `token_info` | -| `DestinationNetwork` | `dest_network` | -| `DestinationAddress` | `dest_address` | -| `Amount` | `amount` (decimal string) | -| `Metadata` | `metadata` (base64 or null) | - -The agglayer Rust serde may use different field names (e.g., `destination_network` -instead of `dest_network`). **Do not paste the raw `jq` output directly** unless you -have verified the field names match. The safest approach is to let Go do the translation -by using a small helper script (see below). - -### Option A — Shell script (single cert, no Go tooling) - -Verify that the field names in the admin API response match the table above before using -this option. If they do, you can pipe the `bridge_exits` array straight into the file: - -```bash -AGGLAYER_ADMIN="http://localhost:4446" -CERT_ID="0xabc123...def456" -HEIGHT=3 -NETWORK_ID=1 - -BRIDGE_EXITS=$(curl -s -X POST "$AGGLAYER_ADMIN" \ - -H "Content-Type: application/json" \ - -d "{\"jsonrpc\":\"2.0\",\"method\":\"admin_getCertificate\",\"params\":[\"$CERT_ID\"],\"id\":1}" \ - | jq '.result[0].bridge_exits') - -cat > certificate_exits_override.json < certificate_exits_override.json <= height { + printCertStatus(info, cfg.BackwardForwardLET.L2NetworkID, height) + fmt.Printf("Wait complete: height %d is settled.\n", height) + return nil + } + } + if time.Now().After(deadline) { + printCertStatus(info, cfg.BackwardForwardLET.L2NetworkID, c.Uint64("height")) + return fmt.Errorf("timed out after %s waiting for requested certificate status", timeout) + } + select { + case <-c.Done(): + return c.Err() + case <-time.After(certStatusPollInterval): + } + } + } + + info, _, err := getNetworkInfoAllowNotFound(context.Background(), client, cfg.BackwardForwardLET.L2NetworkID) + if err != nil { + return err + } + printCertStatus(info, cfg.BackwardForwardLET.L2NetworkID, c.Uint64("height")) + return nil +} + +func printCertStatus(info agglayertypes.NetworkInfo, networkID uint32, requestedHeight uint64) { + printCertStatusTo(os.Stdout, info, networkID, requestedHeight) +} + +func printCertStatusTo(w io.Writer, info agglayertypes.NetworkInfo, networkID uint32, requestedHeight uint64) { + fmt.Fprintf(w, "Network ID: %d\n", networkID) + if info.SettledHeight == nil { + fmt.Fprintln(w, "Latest settled height: none") + } else { + fmt.Fprintf(w, "Latest settled height: %d\n", *info.SettledHeight) + } + if info.SettledCertificateID != nil { + fmt.Fprintf(w, "Latest settled certificate ID: %s\n", info.SettledCertificateID.Hex()) + } + if info.SettledLER != nil { + fmt.Fprintf(w, "Latest settled LER: %s\n", info.SettledLER.Hex()) + } + if info.SettledLETLeafCount != nil { + fmt.Fprintf(w, "Latest settled deposit count: %d\n", *info.SettledLETLeafCount) + } + + if info.LatestPendingHeight == nil { + fmt.Fprintln(w, "Latest pending certificate: none") + } else { + fmt.Fprintf(w, "Latest pending height: %d\n", *info.LatestPendingHeight) + status := "unknown" + if info.LatestPendingStatus != nil { + status = info.LatestPendingStatus.String() + } + fmt.Fprintf(w, "Latest pending status: %s\n", status) + if info.LatestPendingError != "" { + fmt.Fprintf(w, "Latest pending error: %s\n", info.LatestPendingError) + } + } + + if requestedHeight > 0 || (info.SettledHeight != nil && *info.SettledHeight == 0) { + fmt.Fprintf(w, "Requested height: %d\n", requestedHeight) + switch { + case info.SettledHeight != nil && *info.SettledHeight >= requestedHeight: + fmt.Fprintf(w, "Requested height status: Settled\n") + case info.LatestPendingHeight != nil && + *info.LatestPendingHeight == requestedHeight && + info.LatestPendingStatus != nil: + fmt.Fprintf(w, "Requested height status: %s\n", info.LatestPendingStatus.String()) + default: + fmt.Fprintln(w, "Requested height status: not settled") + } + } +} diff --git a/tools/backward_forward_let/cert_status_test.go b/tools/backward_forward_let/cert_status_test.go new file mode 100644 index 000000000..4140abd60 --- /dev/null +++ b/tools/backward_forward_let/cert_status_test.go @@ -0,0 +1,56 @@ +package backward_forward_let + +import ( + "bytes" + "testing" + + agglayertypes "github.com/agglayer/aggkit/agglayer/types" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +func TestHasOpenPendingAtOrAbove(t *testing.T) { + t.Parallel() + + pendingHeight := uint64(10) + openStatus := agglayertypes.Pending + closedStatus := agglayertypes.Settled + + require.True(t, hasOpenPendingAtOrAbove(agglayertypes.NetworkInfo{ + LatestPendingHeight: &pendingHeight, + LatestPendingStatus: &openStatus, + }, 10)) + require.False(t, hasOpenPendingAtOrAbove(agglayertypes.NetworkInfo{ + LatestPendingHeight: &pendingHeight, + LatestPendingStatus: &closedStatus, + }, 10)) + require.False(t, hasOpenPendingAtOrAbove(agglayertypes.NetworkInfo{ + LatestPendingHeight: &pendingHeight, + LatestPendingStatus: &openStatus, + }, 11)) +} + +func TestPrintCertStatus(t *testing.T) { + t.Parallel() + + settledHeight := uint64(12) + settledID := common.HexToHash("0xabc") + settledLER := common.HexToHash("0xdef") + settledDC := uint64(34) + pendingHeight := uint64(13) + pendingStatus := agglayertypes.Pending + + var buf bytes.Buffer + printCertStatusTo(&buf, agglayertypes.NetworkInfo{ + SettledHeight: &settledHeight, + SettledCertificateID: &settledID, + SettledLER: &settledLER, + SettledLETLeafCount: &settledDC, + LatestPendingHeight: &pendingHeight, + LatestPendingStatus: &pendingStatus, + }, 1, 12) + + output := buf.String() + require.Contains(t, output, "Latest settled height: 12") + require.Contains(t, output, "Requested height status: Settled") +} diff --git a/tools/backward_forward_let/cmd/main.go b/tools/backward_forward_let/cmd/main.go index 52e6d3585..05e1202aa 100644 --- a/tools/backward_forward_let/cmd/main.go +++ b/tools/backward_forward_let/cmd/main.go @@ -25,79 +25,22 @@ func main() { Name: "yes", Usage: "Skip interactive confirmation and execute the recovery plan immediately", }, + &cli.BoolFlag{ + Name: "diagnose-only", + Usage: "Print diagnosis and recovery plan, then stop without prompting or sending recovery transactions", + }, &cli.StringFlag{ Name: "cert-exits-file", Aliases: []string{"f"}, - Usage: "Path to a JSON override file containing pre-extracted bridge exits keyed by certificate height." + - " Use when the aggsender DB is empty and the tool reports missing cert IDs.", + Usage: "Path to a JSON fallback file containing either raw AggLayer certificates or pre-extracted bridge exits keyed by certificate height." + + " Use when the aggsender DB is empty and the tool reports missing certificate exits.", }, } app.Action = backward_forward_let.Run app.Commands = []*cli.Command{ - { - Name: "craft-cert", - Usage: "Build a signed malicious certificate JSON for staging drills", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "signer-key-path", - Usage: "Path to the keystore file used to sign the crafted certificate (overrides AggSender.AggsenderPrivateKey)", - }, - &cli.StringFlag{ - Name: "signer-key-password", - Usage: "Password for the keystore file used to sign the crafted certificate (used with --signer-key-path)", - }, - &cli.StringFlag{ - Name: "out", - Usage: "Write the crafted certificate JSON to this file instead of stdout", - }, - &cli.StringFlag{ - Name: "db-path", - Usage: "Optional path to the aggsender SQLite DB when aggsender RPC is unavailable", - }, - &cli.IntFlag{ - Name: "num-fake-exits", - Usage: "Number of fake bridge exits to include in the crafted certificate", - Value: 1, - }, - &cli.IntFlag{ - Name: "starting-exit-index", - Usage: "Starting index used to derive unique fake destination addresses", - Value: 0, - }, - &cli.StringFlag{ - Name: "nonce", - Usage: "Optional nonce used to derive deterministic fake destination addresses", - }, - &cli.UintFlag{ - Name: "origin-network", - Usage: "Origin network for fake bridge exits", - Value: 0, - }, - &cli.StringFlag{ - Name: "origin-token-address", - Usage: "Origin token address for fake bridge exits", - Value: "0x0000000000000000000000000000000000000000", - }, - &cli.UintFlag{ - Name: "destination-network", - Usage: "Destination network for fake bridge exits", - Value: 0, - }, - &cli.StringFlag{ - Name: "amount", - Usage: "Amount for each fake bridge exit, encoded as a decimal string", - Value: "0", - }, - &cli.BoolFlag{ - Name: "staging-only", - Usage: "Acknowledge that crafted malicious certificates are only for staging drills", - }, - }, - Action: backward_forward_let.RunCraftCert, - }, { Name: "send-cert", - Usage: "Send a certificate to the agglayer and record it in the aggsender DB", + Usage: "Send a certificate to the agglayer and optionally record it in the aggsender DB", Flags: []cli.Flag{ &cli.StringFlag{ Name: "cert-json", @@ -114,7 +57,11 @@ func main() { }, &cli.BoolFlag{ Name: "no-db", - Usage: "Send the certificate to the agglayer without storing it in the aggsender DB", + Usage: "Staging-only: send the certificate without recording it in the aggsender DB", + }, + &cli.BoolFlag{ + Name: "staging-only", + Usage: "Required when using staging-only send modes such as --no-db", }, &cli.StringFlag{ Name: "signer-key-path", @@ -127,6 +74,63 @@ func main() { }, Action: backward_forward_let.RunSendCert, }, + { + Name: "craft-cert", + Usage: "Staging-only: craft a testing certificate for a backward/forward LET drill", + Flags: []cli.Flag{ + &cli.BoolFlag{Name: "staging-only", Usage: "Required safety confirmation for testing certificate crafting"}, + &cli.UintFlag{Name: "num-fake-exits", Usage: "Number of fake bridge exits to include"}, + &cli.StringFlag{Name: "amount", Value: "0", Usage: "Fake bridge exit amount"}, + &cli.UintFlag{Name: "starting-exit-index", Usage: "Starting index for deterministic fake exit uniqueness"}, + &cli.StringFlag{Name: "nonce", Usage: "Optional nonce used to derive fake exit destination addresses"}, + &cli.Uint64Flag{ + Name: "l1-info-tree-leaf-count", + Usage: "Override L1 info tree leaf count when aggsender header data is unavailable", + }, + &cli.Uint64Flag{Name: "signer-index", Usage: "Multisig signer index to write into the crafted certificate"}, + &cli.StringFlag{Name: "out", Usage: "Output path for the crafted certificate JSON", Required: true}, + }, + Action: backward_forward_let.RunCraftCert, + }, + { + Name: "cert-status", + Usage: "Print AggLayer certificate settlement and pending status", + Flags: []cli.Flag{ + &cli.Uint64Flag{Name: "height", Usage: "Certificate height to check"}, + &cli.BoolFlag{Name: "wait-no-pending", Usage: "Wait until AggLayer has no open pending certificate"}, + &cli.BoolFlag{Name: "wait-settled", Usage: "Wait until --height is settled"}, + &cli.DurationFlag{ + Name: "timeout", + Value: backward_forward_let.DefaultCertStatusTimeout, + Usage: "Maximum wait duration", + }, + }, + Action: backward_forward_let.RunCertStatus, + }, + { + Name: "export-cert-exits", + Usage: "Export a certificate-exits override from an authoritative height-to-cert-ID map", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "agglayer-admin-url", Usage: "Read-only AggLayer admin JSON-RPC URL", Required: true}, + &cli.StringFlag{Name: "cert-ids-file", Usage: "JSON file mapping certificate heights to cert IDs", Required: true}, + &cli.StringFlag{Name: "out", Usage: "Output certificate exits override JSON path", Required: true}, + &cli.StringFlag{ + Name: "manifest-out", + Usage: "Output source manifest JSON path (default: .manifest.json)", + }, + &cli.Uint64Flag{ + Name: "max-certs", + Value: backward_forward_let.DefaultExportCertExitsMaxCerts, + Usage: "Maximum certificates to export in one batch", + }, + &cli.DurationFlag{ + Name: "timeout", + Value: backward_forward_let.DefaultExportCertExitsTimeout, + Usage: "Maximum export duration", + }, + }, + Action: backward_forward_let.RunExportCertExits, + }, } if err := app.Run(os.Args); err != nil { diff --git a/tools/backward_forward_let/config.go b/tools/backward_forward_let/config.go index 20a6b77a5..d56516671 100644 --- a/tools/backward_forward_let/config.go +++ b/tools/backward_forward_let/config.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/agglayer/aggkit/agglayer" + aggsenderconfig "github.com/agglayer/aggkit/aggsender/config" "github.com/agglayer/aggkit/bridgesync" aggkitConfig "github.com/agglayer/aggkit/config" ethermanconfig "github.com/agglayer/aggkit/etherman/config" @@ -26,19 +27,13 @@ type Config struct { // AgglayerClient is the AggLayer gRPC client configuration. AgglayerClient agglayer.ClientConfig `mapstructure:"AgglayerClient"` - // AggSender contains the subset of aggsender config reused by craft-cert signer resolution. - AggSender CraftCertAggsenderConfig `mapstructure:"AggSender"` + // AggSender contains the signer config used to craft staging certificates. + AggSender aggsenderconfig.Config `mapstructure:"AggSender"` // BackwardForwardLET contains tool-specific settings. BackwardForwardLET BackwardForwardLETConfig `mapstructure:"BackwardForwardLET"` } -// CraftCertAggsenderConfig contains the aggsender signer settings reused by craft-cert. -type CraftCertAggsenderConfig struct { - // AggsenderPrivateKey is the shared signer config used to sign certificates. - AggsenderPrivateKey signertypes.SignerConfig `mapstructure:"AggsenderPrivateKey"` -} - // BackwardForwardLETConfig contains configuration specific to the backward/forward LET tool. type BackwardForwardLETConfig struct { // GERRemoverKey is the signing key used for GER-removal and bridge admin operations. @@ -59,11 +54,9 @@ type BackwardForwardLETConfig struct { // L2NetworkID is the network ID of the L2 chain. L2NetworkID uint32 `mapstructure:"L2NetworkID"` - // CertificateExitsFile is an optional path to a JSON override file containing - // pre-extracted bridge exits keyed by certificate height. When set, used as a - // fallback if the aggsender RPC cannot supply bridge exits for a height. - // Obtain the file by calling admin_getCertificate on the agglayer for each - // cert ID reported in the tool's missing-cert output. + // CertificateExitsFile is an optional path to a JSON fallback file containing + // raw AggLayer certificates or pre-extracted bridge exits keyed by certificate + // height. When set, used if the aggsender RPC cannot supply bridge exits for a height. CertificateExitsFile string `mapstructure:"CertificateExitsFile"` } diff --git a/tools/backward_forward_let/craft_cert.go b/tools/backward_forward_let/craft_cert.go index ed10f7fd4..a0fa030a9 100644 --- a/tools/backward_forward_let/craft_cert.go +++ b/tools/backward_forward_let/craft_cert.go @@ -3,61 +3,32 @@ package backward_forward_let import ( "context" "encoding/json" - "errors" "fmt" "math/big" "os" "path/filepath" - "strconv" - "strings" "time" agglayertypes "github.com/agglayer/aggkit/agglayer/types" - aggsenderdb "github.com/agglayer/aggkit/aggsender/db" - aggsendertypes "github.com/agglayer/aggkit/aggsender/types" "github.com/agglayer/aggkit/aggsender/validator" bridgetypes "github.com/agglayer/aggkit/bridgesync/types" - aggkitgrpc "github.com/agglayer/aggkit/grpc" "github.com/agglayer/aggkit/log" "github.com/agglayer/go_signer/signer" - signertypes "github.com/agglayer/go_signer/signer/types" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/urfave/cli/v2" - "google.golang.org/grpc/codes" ) -const defaultL1InfoTreeLeafCount uint32 = 1 +const craftCertFileMode os.FileMode = 0o600 -const ( - craftCertFetchMaxAttempts = 6 - craftCertFetchInitialBackoff = 500 * time.Millisecond - craftCertFetchMaxBackoff = 5 * time.Second - craftCertRPCRequestTimeout = 5 * time.Second - craftCertFileMode = 0o600 -) - -type certStoreReader interface { - GetCertificateByHeight(height uint64) (*aggsendertypes.Certificate, error) - GetCertificateHeaderByHeight(height uint64) (*aggsendertypes.CertificateHeader, error) -} - -type craftCertOptions struct { - numFakeExits int - startingExitIndex int - nonce []byte - originNetwork uint32 - originTokenAddr common.Address - destNetwork uint32 - amount *big.Int -} - -// RunCraftCert is the CLI action for the craft-cert subcommand. -// It builds a signed malicious certificate JSON for staging drills. +// RunCraftCert builds a staging-only testing certificate and writes it as JSON. func RunCraftCert(c *cli.Context) error { if !c.Bool("staging-only") { - return fmt.Errorf("craft-cert requires --staging-only acknowledgement") + return fmt.Errorf("craft-cert is dangerous and requires --staging-only") + } + if c.Uint("num-fake-exits") == 0 { + return fmt.Errorf("--num-fake-exits must be greater than zero") } cfg, err := LoadConfig(c) @@ -68,11 +39,6 @@ func RunCraftCert(c *cli.Context) error { cfg.BackwardForwardLET.CertificateExitsFile = f } - opts, err := craftCertOptionsFromCLI(c) - if err != nil { - return err - } - dialCtx, dialCancel := context.WithTimeout(c.Context, dialTimeout) env, err := SetupEnv(dialCtx, cfg) dialCancel() @@ -81,20 +47,17 @@ func RunCraftCert(c *cli.Context) error { } defer env.Close() - var certStore certStoreReader - if dbPath := c.String("db-path"); dbPath != "" { - certStore, err = openCraftCertStorage(dbPath) - if err != nil { - return err - } - } - - certSigner, err := loadCraftCertSigner(c.Context, env, cfg, c) - if err != nil { - return err - } - - cert, err := craftMaliciousCertificate(c.Context, env, certStore, certSigner, opts) + cert, err := craftStagingCertificate(c.Context, env, craftCertOptions{ + NumFakeExits: uint32(c.Uint("num-fake-exits")), + Amount: c.String("amount"), + StartingExitIndex: uint32(c.Uint("starting-exit-index")), + Nonce: c.String("nonce"), + L1InfoTreeLeafCount: uint32(c.Uint64("l1-info-tree-leaf-count")), + SignerIndex: uint32(c.Uint64("signer-index")), + CertificateOutputFile: c.String("out"), + RequireNoOpenCerts: true, + AllowL1InfoCountSource: true, + }) if err != nil { return err } @@ -103,143 +66,112 @@ func RunCraftCert(c *cli.Context) error { if err != nil { return fmt.Errorf("marshal crafted certificate: %w", err) } - data = append(data, '\n') - - outPath := c.String("out") - if outPath == "" { - _, err = os.Stdout.Write(data) - return err + out := filepath.Clean(c.String("out")) + if err := os.WriteFile(out, data, craftCertFileMode); err != nil { + return fmt.Errorf("write crafted certificate %s: %w", out, err) } - if err := os.WriteFile(filepath.Clean(outPath), data, craftCertFileMode); err != nil { - return fmt.Errorf("write crafted certificate to %s: %w", outPath, err) - } - fmt.Printf("Crafted certificate written to %s\n", outPath) + fmt.Println("STAGING ONLY: crafted testing certificate.") + fmt.Printf("Certificate height: %d\n", cert.Height) + fmt.Printf("Previous local exit root: %s\n", cert.PrevLocalExitRoot.Hex()) + fmt.Printf("New local exit root: %s\n", cert.NewLocalExitRoot.Hex()) + fmt.Printf("Fake bridge exits: %d\n", len(cert.BridgeExits)) + fmt.Printf("Certificate file: %s\n", out) + fmt.Printf("Next: backward-forward-let --cfg send-cert --cert-file %s --no-db --staging-only\n", out) return nil } -func craftCertOptionsFromCLI(c *cli.Context) (*craftCertOptions, error) { - numFakeExits := c.Int("num-fake-exits") - if numFakeExits <= 0 { - return nil, fmt.Errorf("--num-fake-exits must be greater than 0") - } - - amount, ok := new(big.Int).SetString(c.String("amount"), decimalBase) - if !ok { - return nil, fmt.Errorf("parse --amount %q as decimal big.Int", c.String("amount")) - } - - originTokenAddr := common.HexToAddress(c.String("origin-token-address")) - - nonce := c.String("nonce") - if nonce == "" { - nonce = strconv.FormatInt(time.Now().UnixNano(), decimalBase) - } - - return &craftCertOptions{ - numFakeExits: numFakeExits, - startingExitIndex: c.Int("starting-exit-index"), - nonce: []byte(nonce), - originNetwork: uint32(c.Uint("origin-network")), - originTokenAddr: originTokenAddr, - destNetwork: uint32(c.Uint("destination-network")), - amount: amount, - }, nil +type craftCertOptions struct { + NumFakeExits uint32 + Amount string + StartingExitIndex uint32 + Nonce string + L1InfoTreeLeafCount uint32 + SignerIndex uint32 + CertificateOutputFile string + RequireNoOpenCerts bool + AllowL1InfoCountSource bool } -func loadCraftCertSigner( +func craftStagingCertificate( ctx context.Context, env *Env, - cfg *Config, - c *cli.Context, -) (signertypes.Signer, error) { - signerCfg, err := resolveCraftCertSignerConfig(cfg, c) - if err != nil { - return nil, err - } - - l2ChainID, err := env.chainIDFn(ctx) - if err != nil { - return nil, fmt.Errorf("get L2 chain ID for craft-cert signer: %w", err) - } - - signingKey, err := signer.NewSigner(ctx, l2ChainID.Uint64(), signerCfg, "craft-cert", log.GetDefaultLogger()) - if err != nil { - return nil, fmt.Errorf("load craft-cert signer: %w", err) - } - - if err := signingKey.Initialize(ctx); err != nil { - return nil, fmt.Errorf("initialize craft-cert signer: %w", err) - } - - return signingKey, nil -} - -func resolveCraftCertSignerConfig(cfg *Config, c *cli.Context) (signertypes.SignerConfig, error) { - if c.String("signer-key-path") != "" { - return signer.NewLocalSignerConfig(c.String("signer-key-path"), c.String("signer-key-password")), nil - } - - if cfg == nil { - return signertypes.SignerConfig{}, fmt.Errorf("craft-cert signer config is required") - } - - if cfg.AggSender.AggsenderPrivateKey.Method == "" { - return signertypes.SignerConfig{}, fmt.Errorf( - "craft-cert signer is not configured; set AggSender.AggsenderPrivateKey in config or pass --signer-key-path") + opts craftCertOptions, +) (*agglayertypes.Certificate, error) { + amount, ok := new(big.Int).SetString(opts.Amount, decimalBase) + if !ok || amount.Sign() < 0 { + return nil, fmt.Errorf("--amount must be a non-negative base-10 integer") } - return cfg.AggSender.AggsenderPrivateKey, nil -} - -func openCraftCertStorage(dbPath string) (certStoreReader, error) { - if dbPath == "" { - return nil, nil - } - storage, err := aggsenderdb.NewAggSenderSQLStorage(log.GetDefaultLogger(), aggsenderdb.AggSenderSQLStorageConfig{ - DBPath: dbPath, - CertificatesDir: filepath.Join(filepath.Dir(dbPath), "certificates"), - }) + info, _, err := getNetworkInfoAllowNotFound(ctx, env.AgglayerClient, env.L2NetworkID) if err != nil { - return nil, fmt.Errorf("open aggsender DB at %s: %w", dbPath, err) - } - return storage, nil -} - -func craftMaliciousCertificate( - ctx context.Context, - env *Env, - certStore certStoreReader, - certSigner signertypes.HashSigner, - opts *craftCertOptions, -) (*agglayertypes.Certificate, error) { - if opts == nil { - return nil, fmt.Errorf("craft certificate options are required") - } - if certSigner == nil { - return nil, fmt.Errorf("craft certificate signer is required") + return nil, err } - fakeBridgeExits := make([]*agglayertypes.BridgeExit, 0, opts.numFakeExits) - for i := 0; i < opts.numFakeExits; i++ { - fakeBridgeExits = append(fakeBridgeExits, makeFakeBridgeExit(opts, opts.startingExitIndex+i)) - } + var certHeight uint64 + var prevLER common.Hash + var existingLeafCount uint32 + l1InfoTreeLeafCount := opts.L1InfoTreeLeafCount - certHeight, prevLER, existingLeafCount, l1InfoTreeLeafCount, err := currentCertBaseState(ctx, env, certStore) - if err != nil { - return nil, err + if info.SettledHeight != nil { + certHeight = *info.SettledHeight + 1 + if info.SettledLER == nil || info.SettledLETLeafCount == nil { + return nil, fmt.Errorf("agglayer settled state is missing LER or LET leaf count") + } + prevLER = *info.SettledLER + existingLeafCount = uint32(*info.SettledLETLeafCount) + if opts.RequireNoOpenCerts && hasOpenPendingAtOrAbove(info, certHeight) { + return nil, fmt.Errorf( + "pending certificate race: latest pending height/status is %s; "+ + "wait for it to settle or enter InError before crafting", + pendingSummary(info), + ) + } + if l1InfoTreeLeafCount == 0 { + count, err := l1InfoTreeLeafCountFromAggsender(env, *info.SettledHeight) + if err != nil { + return nil, fmt.Errorf( + "get L1 info tree leaf count from aggsender for height %d: %w; "+ + "rerun with --l1-info-tree-leaf-count", + *info.SettledHeight, err, + ) + } + l1InfoTreeLeafCount = count + } + } else { + if opts.RequireNoOpenCerts && hasOpenPendingAtOrAbove(info, 0) { + return nil, fmt.Errorf( + "pending certificate race: latest pending height/status is %s; "+ + "wait for it to settle or enter InError before crafting", + pendingSummary(info), + ) + } + callOpts := &bind.CallOpts{Context: ctx} + root, err := env.L2Bridge.GetRoot(callOpts) + if err != nil { + return nil, fmt.Errorf("get initial L2 bridge root: %w", err) + } + prevLER = common.Hash(root) + dcBig, err := env.L2Bridge.DepositCount(callOpts) + if err != nil { + return nil, fmt.Errorf("get initial L2 deposit count: %w", err) + } + existingLeafCount = uint32(dcBig.Uint64()) + if l1InfoTreeLeafCount == 0 { + l1InfoTreeLeafCount = 1 + } } - existingHashes, err := loadExistingLeafHashes(ctx, env, certStore, certHeight, prevLER, existingLeafCount) + existingHashes, err := stagingExistingLeafHashes(ctx, env, info.SettledHeight, existingLeafCount) if err != nil { return nil, err } - newHashes := make([]common.Hash, 0, len(fakeBridgeExits)) - for _, be := range fakeBridgeExits { + fakeExits := makeFakeBridgeExits(opts.NumFakeExits, opts.StartingExitIndex, opts.Nonce, amount) + newHashes := make([]common.Hash, 0, len(fakeExits)) + for _, be := range fakeExits { newHashes = append(newHashes, BridgeExitLeafHash(be)) } - newLER, err := ComputeLERForNewLeaves(existingHashes, newHashes) if err != nil { return nil, fmt.Errorf("compute new local exit root: %w", err) @@ -250,315 +182,128 @@ func craftMaliciousCertificate( Height: certHeight, PrevLocalExitRoot: prevLER, NewLocalExitRoot: newLER, - BridgeExits: fakeBridgeExits, + BridgeExits: fakeExits, + ImportedBridgeExits: nil, L1InfoTreeLeafCount: l1InfoTreeLeafCount, + CustomChainData: nil, + AggchainData: nil, } - - hashToSign, err := validator.HashCertificateToSign(cert) - if err != nil { - return nil, fmt.Errorf("hash crafted certificate to sign: %w", err) - } - sig, err := certSigner.SignHash(ctx, hashToSign) - if err != nil { - return nil, fmt.Errorf("sign crafted certificate: %w", err) - } - - cert.AggchainData = &agglayertypes.AggchainDataMultisig{ - Multisig: &agglayertypes.Multisig{ - Signatures: []agglayertypes.ECDSAMultisigEntry{ - {Index: 0, Signature: sig}, - }, - }, + if err := signStagingCertificate(ctx, env, cert, opts.SignerIndex); err != nil { + return nil, err } - return cert, nil } -func currentCertBaseState( +func stagingExistingLeafHashes( ctx context.Context, env *Env, - certStore certStoreReader, -) (uint64, common.Hash, uint32, uint32, error) { - info, err := env.AgglayerClient.GetNetworkInfo(ctx, env.L2NetworkID) - if err != nil { - var grpcErr aggkitgrpc.GRPCError - if !errors.As(err, &grpcErr) || grpcErr.Code != codes.NotFound { - return 0, common.Hash{}, 0, 0, fmt.Errorf("get network info from agglayer: %w", err) - } - } - if err == nil && info.SettledHeight != nil { - if info.SettledLER == nil || info.SettledLETLeafCount == nil { - return 0, common.Hash{}, 0, 0, fmt.Errorf("agglayer returned incomplete settled state") - } - certHeight := *info.SettledHeight + 1 - existingLeafCount := uint32(*info.SettledLETLeafCount) - l1InfoTreeLeafCount := defaultL1InfoTreeLeafCount - - switch { - case certStore != nil: - header, headerErr := certStore.GetCertificateHeaderByHeight(*info.SettledHeight) - if headerErr == nil && header != nil && header.L1InfoTreeLeafCount > 0 { - l1InfoTreeLeafCount = header.L1InfoTreeLeafCount - } - default: - storedCert, certErr := env.AggsenderRPC.GetCertificateHeaderPerHeight(info.SettledHeight) - if certErr == nil && storedCert != nil && storedCert.Header != nil && storedCert.Header.L1InfoTreeLeafCount > 0 { - l1InfoTreeLeafCount = storedCert.Header.L1InfoTreeLeafCount - } - } - - return certHeight, *info.SettledLER, existingLeafCount, l1InfoTreeLeafCount, nil - } - - callOpts := &bind.CallOpts{Context: ctx} - root, rootErr := env.L2Bridge.GetRoot(callOpts) - if rootErr != nil { - return 0, common.Hash{}, 0, 0, fmt.Errorf("get L2 root for initial certificate: %w", rootErr) - } - - dcBig, dcErr := env.L2Bridge.DepositCount(callOpts) - if dcErr != nil { - return 0, common.Hash{}, 0, 0, fmt.Errorf("get L2 deposit count for initial certificate: %w", dcErr) - } - - return 0, common.Hash(root), uint32(dcBig.Uint64()), defaultL1InfoTreeLeafCount, nil -} - -func loadExistingLeafHashes( - ctx context.Context, - env *Env, - certStore certStoreReader, - certHeight uint64, - settledLER common.Hash, + settledHeight *uint64, existingLeafCount uint32, ) ([]common.Hash, error) { - if certHeight == 0 { - return loadLeafHashesFromBridgeService(ctx, env, existingLeafCount) - } - - bridgeMatchesSettled, err := currentBridgeMatchesSettled(ctx, env, settledLER, existingLeafCount) - if err != nil { - return nil, err - } - if bridgeMatchesSettled { - return loadLeafHashesFromBridgeService(ctx, env, existingLeafCount) + if settledHeight == nil { + return fetchL2LeafHashesUpTo(ctx, env, existingLeafCount) } - - settledHeight := certHeight - 1 hashes := make([]common.Hash, 0, existingLeafCount) - prefixMissing := true - for h := uint64(0); h <= settledHeight; h++ { - exits, err := getStoredBridgeExitsForHeight(env, certStore, h) + for h := uint64(0); h <= *settledHeight; h++ { + exits, err := getBridgeExitsForHeight(env, h) if err != nil { - if !prefixMissing { - return nil, fmt.Errorf("load certificate bridge exits at height %d after later heights already loaded: %w", h, err) - } - continue + return nil, fmt.Errorf("load historical bridge exits for cert height %d: %w", h, err) } - prefixMissing = false for _, be := range exits { hashes = append(hashes, BridgeExitLeafHash(be)) } } - - if uint32(len(hashes)) > existingLeafCount { - return nil, fmt.Errorf( - "loaded %d historical leaf hashes, exceeds expected settled leaf count %d", - len(hashes), existingLeafCount, - ) - } - - missingPrefixLeafCount := existingLeafCount - uint32(len(hashes)) - if missingPrefixLeafCount > 0 { - prefixHashes, err := loadLeafHashesFromBridgeService(ctx, env, missingPrefixLeafCount) - if err != nil { - return nil, fmt.Errorf("reconstruct missing certificate prefix from bridge service: %w", err) - } - hashes = append(prefixHashes, hashes...) - } - - if uint32(len(hashes)) != existingLeafCount { - return nil, fmt.Errorf("reconstructed %d total leaf hashes, expected %d", len(hashes), existingLeafCount) - } - return hashes, nil } -func currentBridgeMatchesSettled( - ctx context.Context, - env *Env, - settledLER common.Hash, - existingLeafCount uint32, -) (bool, error) { - if env == nil || env.L2Bridge == nil { - return false, nil - } - - callOpts := &bind.CallOpts{Context: ctx} - root, err := env.L2Bridge.GetRoot(callOpts) +func l1InfoTreeLeafCountFromAggsender(env *Env, settledHeight uint64) (uint32, error) { + cert, err := env.AggsenderRPC.GetCertificateHeaderPerHeight(&settledHeight) if err != nil { - return false, fmt.Errorf("get L2 root for settled-state comparison: %w", err) + return 0, err } - - dcBig, err := env.L2Bridge.DepositCount(callOpts) - if err != nil { - return false, fmt.Errorf("get L2 deposit count for settled-state comparison: %w", err) - } - - return common.Hash(root) == settledLER && uint32(dcBig.Uint64()) == existingLeafCount, nil -} - -func loadLeafHashesFromBridgeService(ctx context.Context, env *Env, existingLeafCount uint32) ([]common.Hash, error) { - hashes := make([]common.Hash, 0, existingLeafCount) - for dc := uint32(0); dc < existingLeafCount; dc++ { - br, err := env.BridgeService.GetBridgeByDepositCount(ctx, env.L2NetworkID, dc) - if err != nil { - return nil, fmt.Errorf("get bridge service leaf at deposit count %d: %w", dc, err) - } - hashes = append(hashes, BridgeResponseLeafHash(br)) + if cert == nil || cert.Header == nil || cert.Header.L1InfoTreeLeafCount == 0 { + return 0, fmt.Errorf("aggsender returned no L1InfoTreeLeafCount") } - return hashes, nil + return cert.Header.L1InfoTreeLeafCount, nil } -func getStoredBridgeExitsForHeight( - env *Env, - certStore certStoreReader, - height uint64, -) ([]*agglayertypes.BridgeExit, error) { - if env != nil && env.BridgeExitsOverride != nil { - if exits, ok := env.BridgeExitsOverride.GetExits(height); ok { - return exits, nil - } +func signStagingCertificate(ctx context.Context, env *Env, cert *agglayertypes.Certificate, signerIndex uint32) error { + l2ChainID, err := env.chainIDFn(ctx) + if err != nil { + return fmt.Errorf("get L2 chain ID: %w", err) + } + s, err := signer.NewSigner( + ctx, + l2ChainID.Uint64(), + env.Config.AggSender.AggsenderPrivateKey, + "staging-craft-cert", + log.GetDefaultLogger(), + ) + if err != nil { + return fmt.Errorf("load aggsender signer: %w", err) } - - if certStore != nil { - cert, err := certStore.GetCertificateByHeight(height) - if err != nil { - return nil, err - } - if cert == nil { - return nil, fmt.Errorf("certificate not found") - } - if cert.Header != nil && cert.Header.CertSource == aggsendertypes.CertificateSourceAggLayer { - return nil, fmt.Errorf("certificate at height %d has agglayer source and no local bridge exits", height) - } - if cert.SignedCertificate == nil { - return nil, fmt.Errorf("certificate at height %d has no signed certificate payload", height) - } - return parseBridgeExitsFromSignedCertificate(height, *cert.SignedCertificate) + if err := s.Initialize(ctx); err != nil { + return fmt.Errorf("initialize aggsender signer: %w", err) } - - var lastErr error - backoff := craftCertFetchInitialBackoff - for attempt := 1; attempt <= craftCertFetchMaxAttempts; attempt++ { - cert, headerErr := callCraftCertRPCWithTimeout( - func() (*aggsendertypes.Certificate, error) { - return env.AggsenderRPC.GetCertificateHeaderPerHeight(&height) - }, - ) - if headerErr == nil && cert != nil && cert.SignedCertificate != nil { - exits, parseErr := parseBridgeExitsFromSignedCertificate(height, *cert.SignedCertificate) - if parseErr == nil { - return exits, nil - } - } else if headerErr != nil { - lastErr = headerErr - if isRetryableCraftCertFetchError(headerErr) { - if attempt == craftCertFetchMaxAttempts { - break - } - time.Sleep(backoff) - if backoff < craftCertFetchMaxBackoff { - backoff *= 2 - if backoff > craftCertFetchMaxBackoff { - backoff = craftCertFetchMaxBackoff - } - } - continue - } - } - - exits, err := callCraftCertRPCWithTimeout( - func() ([]*agglayertypes.BridgeExit, error) { - return env.AggsenderRPC.GetCertificateBridgeExits(&height) - }, - ) - if err == nil { - return exits, nil - } - lastErr = err - - if !isRetryableCraftCertFetchError(err) && !isRetryableCraftCertFetchError(lastErr) { - return nil, lastErr - } - - if attempt == craftCertFetchMaxAttempts { - break - } - time.Sleep(backoff) - if backoff < craftCertFetchMaxBackoff { - backoff *= 2 - if backoff > craftCertFetchMaxBackoff { - backoff = craftCertFetchMaxBackoff - } - } + hashToSign, err := validator.HashCertificateToSign(cert) + if err != nil { + return fmt.Errorf("hash crafted certificate: %w", err) } - - return nil, lastErr -} - -func callCraftCertRPCWithTimeout[T any](fn func() (T, error)) (T, error) { - type result struct { - value T - err error + sig, err := s.SignHash(ctx, hashToSign) + if err != nil { + return fmt.Errorf("sign crafted certificate with aggsender signer: %w", err) } - - resultCh := make(chan result, 1) - go func() { - value, err := fn() - resultCh <- result{value: value, err: err} - }() - - select { - case result := <-resultCh: - return result.value, result.err - case <-time.After(craftCertRPCRequestTimeout): - var zero T - return zero, fmt.Errorf("aggsender RPC request timed out after %s", craftCertRPCRequestTimeout) + cert.AggchainData = &agglayertypes.AggchainDataMultisig{ + Multisig: &agglayertypes.Multisig{ + Signatures: []agglayertypes.ECDSAMultisigEntry{ + {Index: signerIndex, Signature: sig}, + }, + }, } + return nil } -func parseBridgeExitsFromSignedCertificate(height uint64, signedCert string) ([]*agglayertypes.BridgeExit, error) { - var agglayerCert agglayertypes.Certificate - if err := json.Unmarshal([]byte(signedCert), &agglayerCert); err != nil { - return nil, fmt.Errorf("unmarshal signed certificate at height %d: %w", height, err) +func makeFakeBridgeExits(count, startingIndex uint32, nonce string, amount *big.Int) []*agglayertypes.BridgeExit { + if nonce == "" { + nonce = fmt.Sprintf("%d", time.Now().UnixNano()) + } + exits := make([]*agglayertypes.BridgeExit, 0, count) + for i := uint32(0); i < count; i++ { + exitIndex := startingIndex + i + addrBytes := crypto.Keccak256([]byte(fmt.Sprintf("%s:%d", nonce, exitIndex))) + exits = append(exits, &agglayertypes.BridgeExit{ + LeafType: bridgetypes.LeafTypeAsset, + TokenInfo: &agglayertypes.TokenInfo{ + OriginNetwork: 0, + OriginTokenAddress: common.Address{}, + }, + DestinationNetwork: 0, + DestinationAddress: common.BytesToAddress(addrBytes), + Amount: new(big.Int).Set(amount), + Metadata: nil, + }) } - return agglayerCert.BridgeExits, nil + return exits } -func isRetryableCraftCertFetchError(err error) bool { - if err == nil { +func hasOpenPendingAtOrAbove(info agglayertypes.NetworkInfo, height uint64) bool { + if info.LatestPendingHeight == nil || *info.LatestPendingHeight < height { return false } - msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "found: 429") || - strings.Contains(msg, "too many requests") || - strings.Contains(msg, "connect: connection refused") || - strings.Contains(msg, "no route to host") || - strings.Contains(msg, "timeout") + if info.LatestPendingStatus == nil { + return true + } + return info.LatestPendingStatus.IsOpen() } -func makeFakeBridgeExit(opts *craftCertOptions, exitIndex int) *agglayertypes.BridgeExit { - addrBytes := crypto.Keccak256(append(append([]byte(nil), opts.nonce...), byte(exitIndex))) - return &agglayertypes.BridgeExit{ - LeafType: bridgetypes.LeafTypeAsset, - TokenInfo: &agglayertypes.TokenInfo{ - OriginNetwork: opts.originNetwork, - OriginTokenAddress: opts.originTokenAddr, - }, - DestinationNetwork: opts.destNetwork, - DestinationAddress: common.BytesToAddress(addrBytes), - Amount: new(big.Int).Set(opts.amount), - Metadata: nil, +func pendingSummary(info agglayertypes.NetworkInfo) string { + height := "none" + if info.LatestPendingHeight != nil { + height = fmt.Sprintf("%d", *info.LatestPendingHeight) + } + status := "unknown" + if info.LatestPendingStatus != nil { + status = info.LatestPendingStatus.String() } + return fmt.Sprintf("height=%s status=%s", height, status) } diff --git a/tools/backward_forward_let/craft_cert_test.go b/tools/backward_forward_let/craft_cert_test.go index 757f72248..e91f62db6 100644 --- a/tools/backward_forward_let/craft_cert_test.go +++ b/tools/backward_forward_let/craft_cert_test.go @@ -1,614 +1,28 @@ package backward_forward_let import ( - "context" - "crypto/ecdsa" - "encoding/json" - "errors" - "flag" "math/big" "testing" - "time" - agglayertypes "github.com/agglayer/aggkit/agglayer/types" - aggsendertypes "github.com/agglayer/aggkit/aggsender/types" - bridgeservicetypes "github.com/agglayer/aggkit/bridgeservice/types" - bridgetypes "github.com/agglayer/aggkit/bridgesync/types" - "github.com/agglayer/go_signer/signer" - signertypes "github.com/agglayer/go_signer/signer/types" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" ) -type stubAgglayerClient struct { - info agglayertypes.NetworkInfo - err error -} - -func (s *stubAgglayerClient) SendCertificate(context.Context, *agglayertypes.Certificate) (common.Hash, error) { - return common.Hash{}, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeader(context.Context, common.Hash) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetNetworkInfo(context.Context, uint32) (agglayertypes.NetworkInfo, error) { - return s.info, s.err -} - -func (s *stubAgglayerClient) GetEpochConfiguration(context.Context) (*agglayertypes.ClockConfiguration, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetLatestSettledCertificateHeader(context.Context, uint32) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetLatestPendingCertificateHeader(context.Context, uint32) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeaderByID(context.Context, common.Hash) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeaderByHash(context.Context, common.Hash) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeaderByCertificateID(context.Context, common.Hash) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeaderPerHeight(context.Context, uint32, uint64) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func (s *stubAgglayerClient) GetCertificateHeaderLegacy(context.Context, common.Hash) (*agglayertypes.CertificateHeader, error) { - return nil, errors.New("not implemented") -} - -func TestMakeFakeBridgeExit(t *testing.T) { - t.Parallel() - - opts := &craftCertOptions{ - nonce: []byte("nonce"), - originNetwork: 7, - originTokenAddr: common.HexToAddress("0x1111111111111111111111111111111111111111"), - destNetwork: 9, - amount: big.NewInt(123), - } - - exit0 := makeFakeBridgeExit(opts, 0) - exit1 := makeFakeBridgeExit(opts, 1) - - require.Equal(t, bridgetypes.LeafTypeAsset, exit0.LeafType) - require.Equal(t, uint32(7), exit0.TokenInfo.OriginNetwork) - require.Equal(t, common.HexToAddress("0x1111111111111111111111111111111111111111"), exit0.TokenInfo.OriginTokenAddress) - require.Equal(t, uint32(9), exit0.DestinationNetwork) - require.Equal(t, big.NewInt(123), exit0.Amount) - require.NotEqual(t, exit0.DestinationAddress, exit1.DestinationAddress) -} - -func TestCraftMaliciousCertificate_NoSettledCerts(t *testing.T) { - t.Parallel() - - signerKey, err := crypto.GenerateKey() - require.NoError(t, err) - - bridge := &stubL2Bridge{ - depositCount: big.NewInt(1), - root: [32]byte(common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111")), - } - bridgeSvc := &stubBridgeService{ - bridges: map[uint32]*bridgeservicetypes.BridgeResponse{ - 0: { - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 1, - DestinationAddress: bridgeservicetypes.Address("0x2222222222222222222222222222222222222222"), - Amount: bridgeservicetypes.BigIntString("5"), - }, - }, - } - env := &Env{ - L2Bridge: bridge, - BridgeService: bridgeSvc, - AgglayerClient: &stubAgglayerClient{}, - L2NetworkID: 1, - } - - opts := &craftCertOptions{ - numFakeExits: 1, - startingExitIndex: 0, - nonce: []byte("run-a"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - } - - cert, err := craftMaliciousCertificate(context.Background(), env, nil, &stubHashSigner{key: signerKey}, opts) - require.NoError(t, err) - require.Equal(t, uint64(0), cert.Height) - require.Equal(t, common.Hash(bridge.root), cert.PrevLocalExitRoot) - require.Len(t, cert.BridgeExits, 1) - require.Equal(t, uint32(1), cert.L1InfoTreeLeafCount) - - expectedLER, err := ComputeLERForNewLeaves( - []common.Hash{BridgeResponseLeafHash(bridgeSvc.bridges[0])}, - []common.Hash{BridgeExitLeafHash(cert.BridgeExits[0])}, - ) - require.NoError(t, err) - require.Equal(t, expectedLER, cert.NewLocalExitRoot) - - multisig, ok := cert.AggchainData.(*agglayertypes.AggchainDataMultisig) - require.True(t, ok) - require.Len(t, multisig.Multisig.Signatures, 1) -} - -func TestCraftMaliciousCertificate_SettledCertsFromAggsenderRPC(t *testing.T) { - t.Parallel() - - signerKey, err := crypto.GenerateKey() - require.NoError(t, err) - - settledHeight := uint64(1) - settledLER := common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - settledLeafCount := uint64(2) - info := agglayertypes.NetworkInfo{ - SettledHeight: &settledHeight, - SettledLER: &settledLER, - SettledLETLeafCount: &settledLeafCount, - } - - exit0 := makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("existing-0"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0) - exit1 := makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("existing-1"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0) - - rpc := &stubAggsenderRPC{ - exitsByHeight: map[uint64][]*agglayertypes.BridgeExit{ - 0: {exit0}, - 1: {exit1}, - }, - failHeights: map[uint64]bool{}, - } - rpcHeader := &aggsendertypes.Certificate{ - Header: &aggsendertypes.CertificateHeader{L1InfoTreeLeafCount: 7}, - } - rpcWithHeader := &stubCraftAggsenderRPC{stubAggsenderRPC: rpc, certByHeight: map[uint64]*aggsendertypes.Certificate{1: rpcHeader}} - - env := &Env{ - AgglayerClient: &stubAgglayerClient{info: info}, - AggsenderRPC: rpcWithHeader, - L2NetworkID: 1, - } - - opts := &craftCertOptions{ - numFakeExits: 1, - startingExitIndex: 5, - nonce: []byte("new"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - } - - cert, err := craftMaliciousCertificate(context.Background(), env, nil, &stubHashSigner{key: signerKey}, opts) - require.NoError(t, err) - require.Equal(t, uint64(2), cert.Height) - require.Equal(t, settledLER, cert.PrevLocalExitRoot) - require.Equal(t, uint32(7), cert.L1InfoTreeLeafCount) - - expectedLER, err := ComputeLERForNewLeaves( - []common.Hash{BridgeExitLeafHash(exit0), BridgeExitLeafHash(exit1)}, - []common.Hash{BridgeExitLeafHash(cert.BridgeExits[0])}, - ) - require.NoError(t, err) - require.Equal(t, expectedLER, cert.NewLocalExitRoot) -} - -func TestLoadExistingLeafHashes_ReconstructsMissingPrefixFromBridgeService(t *testing.T) { - t.Parallel() - - exit2 := makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("existing-2"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0) - exit3 := makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("existing-3"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0) - - bridge0 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x1111111111111111111111111111111111111111"), - Amount: bridgeservicetypes.BigIntString("1"), - } - bridge1 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x2222222222222222222222222222222222222222"), - Amount: bridgeservicetypes.BigIntString("2"), - } - - env := &Env{ - L2NetworkID: 1, - BridgeService: &stubBridgeService{ - bridges: map[uint32]*bridgeservicetypes.BridgeResponse{ - 0: bridge0, - 1: bridge1, - }, - }, - AggsenderRPC: &stubAggsenderRPC{ - exitsByHeight: map[uint64][]*agglayertypes.BridgeExit{ - 2: {exit2}, - 3: {exit3}, - }, - failHeights: map[uint64]bool{ - 0: true, - 1: true, - }, - }, - } - - hashes, err := loadExistingLeafHashes(context.Background(), env, nil, 4, common.Hash{}, 4) - require.NoError(t, err) - require.Equal(t, []common.Hash{ - BridgeResponseLeafHash(bridge0), - BridgeResponseLeafHash(bridge1), - BridgeExitLeafHash(exit2), - BridgeExitLeafHash(exit3), - }, hashes) -} - -func TestLoadExistingLeafHashes_AllHistoricalHeightsMissingFallsBackToBridgeService(t *testing.T) { - t.Parallel() - - bridge0 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x3333333333333333333333333333333333333333"), - Amount: bridgeservicetypes.BigIntString("3"), - } - bridge1 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x4444444444444444444444444444444444444444"), - Amount: bridgeservicetypes.BigIntString("4"), - } - - env := &Env{ - L2NetworkID: 1, - BridgeService: &stubBridgeService{ - bridges: map[uint32]*bridgeservicetypes.BridgeResponse{ - 0: bridge0, - 1: bridge1, - }, - }, - AggsenderRPC: &stubAggsenderRPC{ - failHeights: map[uint64]bool{ - 0: true, - 1: true, - }, - }, - } - - hashes, err := loadExistingLeafHashes(context.Background(), env, nil, 2, common.Hash{}, 2) - require.NoError(t, err) - require.Equal(t, []common.Hash{ - BridgeResponseLeafHash(bridge0), - BridgeResponseLeafHash(bridge1), - }, hashes) -} - -func TestLoadExistingLeafHashes_UsesBridgeServiceWhenCurrentBridgeMatchesSettled(t *testing.T) { - t.Parallel() - - bridge0 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x5555555555555555555555555555555555555555"), - Amount: bridgeservicetypes.BigIntString("5"), - } - bridge1 := &bridgeservicetypes.BridgeResponse{ - LeafType: bridgetypes.LeafTypeAsset.Uint8(), - OriginNetwork: 0, - OriginAddress: bridgeservicetypes.Address("0x0000000000000000000000000000000000000000"), - DestinationNetwork: 0, - DestinationAddress: bridgeservicetypes.Address("0x6666666666666666666666666666666666666666"), - Amount: bridgeservicetypes.BigIntString("6"), - } - - settledLER, err := ComputeLERForNewLeaves( - []common.Hash{BridgeResponseLeafHash(bridge0)}, - []common.Hash{BridgeResponseLeafHash(bridge1)}, - ) - require.NoError(t, err) - - env := &Env{ - L2NetworkID: 1, - L2Bridge: &stubL2Bridge{ - depositCount: big.NewInt(2), - root: [32]byte(settledLER), - }, - BridgeService: &stubBridgeService{ - bridges: map[uint32]*bridgeservicetypes.BridgeResponse{ - 0: bridge0, - 1: bridge1, - }, - }, - AggsenderRPC: &stubAggsenderRPC{ - failHeights: map[uint64]bool{ - 0: true, - 1: true, - }, - }, - } - - hashes, err := loadExistingLeafHashes(context.Background(), env, nil, 2, settledLER, 2) - require.NoError(t, err) - require.Equal(t, []common.Hash{ - BridgeResponseLeafHash(bridge0), - BridgeResponseLeafHash(bridge1), - }, hashes) -} - -func TestGetStoredBridgeExitsForHeight_FromDB(t *testing.T) { - t.Parallel() - - payload := &agglayertypes.Certificate{ - BridgeExits: []*agglayertypes.BridgeExit{ - makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("db"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0), - }, - } - raw, err := json.Marshal(payload) - require.NoError(t, err) - - store := &stubCraftCertStore{ - certs: map[uint64]*aggsendertypes.Certificate{ - 0: {SignedCertificate: ptrString(string(raw)), Header: &aggsendertypes.CertificateHeader{}}, - }, - } - - exits, err := getStoredBridgeExitsForHeight(&Env{}, store, 0) - require.NoError(t, err) - require.Len(t, exits, 1) -} - -func TestGetStoredBridgeExitsForHeight_FromOverride(t *testing.T) { - t.Parallel() - - overrideExit := makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("override"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0) - - exits, err := getStoredBridgeExitsForHeight(&Env{ - BridgeExitsOverride: &BridgeExitsOverride{ - parsed: map[uint64][]*agglayertypes.BridgeExit{ - 7: {overrideExit}, - }, - }, - }, nil, 7) - require.NoError(t, err) - require.Equal(t, []*agglayertypes.BridgeExit{overrideExit}, exits) -} - -func TestGetStoredBridgeExitsForHeight_FromAggsenderHeaderFallback(t *testing.T) { - t.Parallel() - - payload := &agglayertypes.Certificate{ - BridgeExits: []*agglayertypes.BridgeExit{ - makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("rpc-header"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0), - }, - } - raw, err := json.Marshal(payload) - require.NoError(t, err) - - rpc := &stubCraftAggsenderRPC{ - stubAggsenderRPC: &stubAggsenderRPC{ - failHeights: map[uint64]bool{0: true}, - }, - certByHeight: map[uint64]*aggsendertypes.Certificate{ - 0: {SignedCertificate: ptrString(string(raw))}, - }, - } - - exits, err := getStoredBridgeExitsForHeight(&Env{AggsenderRPC: rpc}, nil, 0) - require.NoError(t, err) - require.Len(t, exits, 1) -} - -func TestGetStoredBridgeExitsForHeight_Retries429OnHeaderPath(t *testing.T) { - t.Parallel() - - payload := &agglayertypes.Certificate{ - BridgeExits: []*agglayertypes.BridgeExit{ - makeFakeBridgeExit(&craftCertOptions{ - nonce: []byte("rpc-retry"), - originNetwork: 0, - originTokenAddr: common.Address{}, - destNetwork: 0, - amount: big.NewInt(0), - }, 0), - }, - } - raw, err := json.Marshal(payload) - require.NoError(t, err) - - rpc := &stubCraftAggsenderRPC{ - stubAggsenderRPC: &stubAggsenderRPC{ - failHeights: map[uint64]bool{0: true}, - }, - certByHeight: map[uint64]*aggsendertypes.Certificate{ - 0: {SignedCertificate: ptrString(string(raw))}, - }, - headerErrsRemaining: map[uint64]int{0: 2}, - } - - exits, err := getStoredBridgeExitsForHeight(&Env{AggsenderRPC: rpc}, nil, 0) - require.NoError(t, err) - require.Len(t, exits, 1) -} - -func TestCallCraftCertRPCWithTimeout_ReturnsResult(t *testing.T) { +func TestMakeFakeBridgeExits(t *testing.T) { t.Parallel() - value, err := callCraftCertRPCWithTimeout(func() (int, error) { - return 7, nil - }) - require.NoError(t, err) - require.Equal(t, 7, value) -} - -func TestCallCraftCertRPCWithTimeout_TimesOut(t *testing.T) { - t.Parallel() - - start := time.Now() - _, err := callCraftCertRPCWithTimeout(func() (int, error) { - time.Sleep(craftCertRPCRequestTimeout + 200*time.Millisecond) - return 0, nil - }) - require.ErrorContains(t, err, "aggsender RPC request timed out") - require.Less(t, time.Since(start), craftCertRPCRequestTimeout+time.Second) -} - -type stubCraftAggsenderRPC struct { - *stubAggsenderRPC - certByHeight map[uint64]*aggsendertypes.Certificate - headerErrsRemaining map[uint64]int -} - -func (s *stubCraftAggsenderRPC) GetCertificateHeaderPerHeight(height *uint64) (*aggsendertypes.Certificate, error) { - if s.headerErrsRemaining != nil && s.headerErrsRemaining[*height] > 0 { - s.headerErrsRemaining[*height]-- - return nil, errors.New("invalid status code, expected: 200, found: 429") - } - return s.certByHeight[*height], nil -} - -type stubCraftCertStore struct { - certs map[uint64]*aggsendertypes.Certificate - headers map[uint64]*aggsendertypes.CertificateHeader -} - -func (s *stubCraftCertStore) GetCertificateByHeight(height uint64) (*aggsendertypes.Certificate, error) { - return s.certs[height], nil -} - -func (s *stubCraftCertStore) GetCertificateHeaderByHeight(height uint64) (*aggsendertypes.CertificateHeader, error) { - return s.headers[height], nil -} - -func ptrString(v string) *string { return &v } + exits := makeFakeBridgeExits(2, 7, "test-nonce", big.NewInt(42)) -type stubHashSigner struct { - key *ecdsa.PrivateKey + require.Len(t, exits, 2) + require.Equal(t, big.NewInt(42), exits[0].Amount) + require.NotEqual(t, exits[0].DestinationAddress, exits[1].DestinationAddress) + require.Equal(t, exits[0].DestinationNetwork, exits[1].DestinationNetwork) } -func (s *stubHashSigner) SignHash(_ context.Context, hash common.Hash) ([]byte, error) { - return crypto.Sign(hash.Bytes(), s.key) -} - -func TestResolveCraftCertSignerConfig_FromCLI(t *testing.T) { - t.Parallel() - - app := cli.NewApp() - set := flagSetForCraftCert(t, - "--signer-key-path", "/tmp/sequencer.keystore", - "--signer-key-password", "secret", - ) - ctx := cli.NewContext(app, set, nil) - - cfg, err := resolveCraftCertSignerConfig(&Config{}, ctx) - require.NoError(t, err) - require.Equal(t, signer.NewLocalSignerConfig("/tmp/sequencer.keystore", "secret"), cfg) -} - -func TestResolveCraftCertSignerConfig_FromAggsenderConfig(t *testing.T) { +func TestMakeFakeBridgeExits_DeterministicWithNonce(t *testing.T) { t.Parallel() - app := cli.NewApp() - ctx := cli.NewContext(app, flagSetForCraftCert(t), nil) - expected := signertypes.SignerConfig{ - Method: signertypes.MethodGCPKMS, - Config: map[string]any{"KeyName": "projects/p/locations/l/keyRings/r/cryptoKeys/k/cryptoKeyVersions/1"}, - } - - cfg, err := resolveCraftCertSignerConfig(&Config{ - AggSender: CraftCertAggsenderConfig{ - AggsenderPrivateKey: expected, - }, - }, ctx) - require.NoError(t, err) - require.Equal(t, expected, cfg) -} - -func TestResolveCraftCertSignerConfig_Missing(t *testing.T) { - t.Parallel() - - app := cli.NewApp() - ctx := cli.NewContext(app, flagSetForCraftCert(t), nil) - - _, err := resolveCraftCertSignerConfig(&Config{}, ctx) - require.Error(t, err) - require.Contains(t, err.Error(), "AggSender.AggsenderPrivateKey") -} - -func flagSetForCraftCert(t *testing.T, args ...string) *flag.FlagSet { - t.Helper() + a := makeFakeBridgeExits(1, 1, "same", big.NewInt(0)) + b := makeFakeBridgeExits(1, 1, "same", big.NewInt(0)) - set := flag.NewFlagSet("craft-cert", flag.ContinueOnError) - set.String("signer-key-path", "", "") - set.String("signer-key-password", "", "") - require.NoError(t, set.Parse(args)) - return set + require.Equal(t, a[0].DestinationAddress, b[0].DestinationAddress) } diff --git a/tools/backward_forward_let/diagnosis.go b/tools/backward_forward_let/diagnosis.go index d1cc8f105..730ef1b16 100644 --- a/tools/backward_forward_let/diagnosis.go +++ b/tools/backward_forward_let/diagnosis.go @@ -12,10 +12,8 @@ import ( bridgeservice "github.com/agglayer/aggkit/bridgeservice/client" bridgeservicetypes "github.com/agglayer/aggkit/bridgeservice/types" "github.com/agglayer/aggkit/bridgesync" - aggkitgrpc "github.com/agglayer/aggkit/grpc" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "google.golang.org/grpc/codes" ) // aggsenderRPCClient is the subset of rpcclient.Client used by the tool and its tests. @@ -37,15 +35,14 @@ func Diagnose(ctx context.Context, env *Env) (*DiagnosisResult, error) { result := &DiagnosisResult{Case: NoDivergence} // Step 1 — Query AggLayer settled state. - info, err := env.AgglayerClient.GetNetworkInfo(ctx, env.L2NetworkID) + info, notFound, err := getNetworkInfoAllowNotFound(ctx, env.AgglayerClient, env.L2NetworkID) if err != nil { + return nil, err + } + if notFound { // A NotFound response means the network is not yet known to the agglayer // (no certificates have been settled), so there is no divergence. - var grpcErr aggkitgrpc.GRPCError - if errors.As(err, &grpcErr) && grpcErr.Code == codes.NotFound { - return result, nil - } - return nil, fmt.Errorf("get network info from agglayer: %w", err) + return result, nil } if info.SettledHeight == nil { // Agglayer has no settled certificates for this network. @@ -139,7 +136,8 @@ type missingCertsError struct { // getBridgeExitsForHeight fetches bridge exits for a certificate height using a // two-source fallback chain: // 1. Aggsender RPC (primary) — works when the aggsender DB is intact. -// 2. JSON override file (secondary) — operator-supplied pre-extracted data. +// 2. JSON fallback file (secondary) — operator-supplied AggLayer certificate +// data or pre-extracted bridge exits. // // An error is returned only when both sources fail or the override has no entry // for the given height. @@ -285,9 +283,13 @@ func collectExtraL2Bridges( br, err := env.BridgeService.GetBridgeByDepositCount(ctx, env.L2NetworkID, dc) if err != nil { if isNotFound(err) { - return nil, fmt.Errorf("get L2 bridge at DC=%d: not indexed yet", dc) + return nil, fmt.Errorf( + "bridge service data not ready for recovery: missing L2 bridge at DC=%d; "+ + "wait for bridge-service indexing and rerun diagnosis", + dc, + ) } - return nil, fmt.Errorf("get L2 bridge at DC=%d: %w", dc, err) + return nil, fmt.Errorf("bridge service data not ready for recovery: get L2 bridge at DC=%d: %w", dc, err) } extra = append(extra, BridgeResponseToLeafData(br)) } @@ -368,7 +370,7 @@ func PrintDiagnosis(w io.Writer, result *DiagnosisResult) { return } - fmt.Fprintf(w, "Case: %s\n", caseDescription(result.Case)) + fmt.Fprintf(w, "Status: Recovery required - %s\n", recoveryDescription(result.Case)) fmt.Fprintf(w, "Divergence Point (matching leaf count): %d\n", result.DivergencePoint) fmt.Fprintln(w) @@ -433,9 +435,10 @@ func PrintDiagnosis(w io.Writer, result *DiagnosisResult) { // printMissingCertReport prints actionable, copy-pasteable instructions when one // or more certificate heights had no bridge exit data from any source. // It lists each missing height with its cert ID (or UNKNOWN), explains how to call -// admin_getCertificate on the agglayer, shows the override file template with the -// actual heights, and prints the re-run command. +// admin_getCertificate on the agglayer, shows the certificate export template +// with the actual heights, and prints the re-run command. func printMissingCertReport(w io.Writer, result *DiagnosisResult) { + fmt.Fprintln(w, "Status: Missing certificate exits - recovery cannot continue yet.") fmt.Fprintln(w, "WARNING: Aggsender RPC returned no bridge exit data for the following certificate heights.") fmt.Fprintln(w, "Recovery cannot proceed until this data is provided.") fmt.Fprintln(w) @@ -460,6 +463,14 @@ func printMissingCertReport(w io.Writer, result *DiagnosisResult) { } fmt.Fprintln(w) + if n > 1 { + fmt.Fprintln(w, "NOTE: After an aggsender DB wipe, this missing range may span the full settled history") + fmt.Fprintln(w, " (for example heights 0..latest). This is expected for the fallback path.") + fmt.Fprintln(w, " Do not fetch large ranges one-by-one manually; use a script or ask the agglayer admin") + fmt.Fprintln(w, " for a batch export of cert IDs / bridge exits when many heights are missing.") + fmt.Fprintln(w) + } + if hasUnknown { fmt.Fprintln(w, "NOTE: For heights with UNKNOWN cert IDs, ask the agglayer admin to look up") fmt.Fprintln(w, " (network_id, height) in the agglayer's certificate_per_network_cf column family,") @@ -467,46 +478,62 @@ func printMissingCertReport(w io.Writer, result *DiagnosisResult) { fmt.Fprintln(w) } - fmt.Fprintln(w, "To extract bridge exits for each KNOWN cert ID:") + fmt.Fprintln(w, "Preferred batch export path:") + fmt.Fprintln(w, " 1. Ask the agglayer admin owner to resolve an authoritative cert ID map") + fmt.Fprintln(w, " from agglayer state, then fetch raw admin_getCertificate responses:") + fmt.Fprintln(w, " {") + fmt.Fprintln(w, ` "network_id": ,`) + fmt.Fprintln(w, ` "certificates": {`) + for i, mc := range result.MissingCerts { + suffix := "," + if i == n-1 { + suffix = "" + } + certID := "" + if mc.CertIDResolved { + certID = mc.CertID.Hex() + } + fmt.Fprintf(w, " \"%d\": \"%s\"%s\n", mc.Height, certID, suffix) + } + fmt.Fprintln(w, " }") + fmt.Fprintln(w, " }") + fmt.Fprintln(w, " 2. Store the raw agglayer responses in an agglayer certificate file:") + fmt.Fprintln(w, " {") + fmt.Fprintln(w, ` "network_id": ,`) + fmt.Fprintln(w, ` "certificates": {`) + fmt.Fprintln(w, ` "": {"jsonrpc":"2.0","result":[, ]}`) + fmt.Fprintln(w, " }") + fmt.Fprintln(w, " }") + fmt.Fprintln(w) + + fmt.Fprintln(w, "The --cert-exits-file loader accepts either this raw agglayer certificate file") + fmt.Fprintln(w, "or the Aggkit-native heights-to-bridge_exits override format.") + fmt.Fprintln(w) + + fmt.Fprintln(w, "Manual admin API shape for each KNOWN cert ID:") fmt.Fprintln(w, " POST http:///") fmt.Fprintln(w, " Content-Type: application/json") fmt.Fprintln(w) fmt.Fprintln(w, ` {"jsonrpc":"2.0","method":"admin_getCertificate","params":[""],"id":1}`) fmt.Fprintln(w) fmt.Fprintln(w, " The response is [Certificate, CertificateHeader|null].") - fmt.Fprintln(w, ` Extract the "bridge_exits" field from the Certificate object.`) - fmt.Fprintln(w) - - fmt.Fprintln(w, "Build a JSON override file in this format:") - fmt.Fprintln(w, " {") - fmt.Fprintln(w, ` "network_id": ,`) - fmt.Fprintln(w, ` "heights": {`) - for i, mc := range result.MissingCerts { - suffix := "," - if i == n-1 { - suffix = "" - } - fmt.Fprintf(w, " \"%d\": [ ...bridge_exits from admin_getCertificate response... ]%s\n", - mc.Height, suffix) - } - fmt.Fprintln(w, " }") - fmt.Fprintln(w, " }") + fmt.Fprintln(w, " It can be stored directly under the matching height key in --cert-exits-file.") fmt.Fprintln(w) fmt.Fprintln(w, "Re-run the tool with:") - fmt.Fprintln(w, " backward-forward-let --cfg --cert-exits-file ") + fmt.Fprintln(w, " backward-forward-let --cfg --cert-exits-file ") } -func caseDescription(c RecoveryCase) string { +func recoveryDescription(c RecoveryCase) string { switch c { case Case1: - return "Case1 — ForwardLET only: single divergent leaf batch, no extra L2 bridges" + return "ForwardLET recovery required for divergent settled bridge exits" case Case2: - return "Case2 — BackwardLET + ForwardLET: single divergent leaf + extra real L2 bridges" + return "BackwardLET and ForwardLET recovery required, including replay of real L2 bridges" case Case3: - return "Case3 — ForwardLET only: multiple divergent leaf batches, no extra L2 bridges" + return "ForwardLET recovery required for multiple divergent settled bridge exits" case Case4: - return "Case4 — BackwardLET + ForwardLET: multiple divergent leaves + extra real L2 bridges" + return "BackwardLET and ForwardLET recovery required, including multiple divergent exits and real L2 bridge replay" default: return string(c) } diff --git a/tools/backward_forward_let/diagnosis_test.go b/tools/backward_forward_let/diagnosis_test.go index 2b02307f8..cb9da809a 100644 --- a/tools/backward_forward_let/diagnosis_test.go +++ b/tools/backward_forward_let/diagnosis_test.go @@ -223,7 +223,8 @@ func TestPrintDiagnosis(t *testing.T) { PrintDiagnosis(&buf, result) output := buf.String() - require.Contains(t, output, "Case3") + require.Contains(t, output, "Status: Recovery required") + require.NotContains(t, output, "Case3") require.Contains(t, output, ler.Hex()) require.Contains(t, output, tokenA.Hex()) require.Contains(t, output, "500") @@ -247,7 +248,6 @@ func TestDiagnosisResult_IsCompleteNoDivergence(t *testing.T) { require.True(t, (&DiagnosisResult{Case: NoDivergence}).IsCompleteNoDivergence()) require.False(t, (&DiagnosisResult{Case: NoDivergence, AggsenderAPIFailed: true}).IsCompleteNoDivergence()) require.False(t, (&DiagnosisResult{Case: Case1}).IsCompleteNoDivergence()) - require.False(t, (*DiagnosisResult)(nil).IsCompleteNoDivergence()) } // TestPrintDiagnosis_AggsenderAPIFailed verifies the actionable missing-cert output @@ -257,7 +257,7 @@ func TestPrintDiagnosis_AggsenderAPIFailed(t *testing.T) { certID := common.HexToHash("0xDEAD") result := &DiagnosisResult{ - Case: Case1, + Case: NoDivergence, AggsenderAPIFailed: true, MissingCerts: []MissingCertInfo{ {Height: 7, CertID: certID, CertIDResolved: true}, @@ -277,35 +277,12 @@ func TestPrintDiagnosis_AggsenderAPIFailed(t *testing.T) { require.Contains(t, output, "admin_getCertificate") require.Contains(t, output, `"7":`) require.Contains(t, output, "--cert-exits-file") + require.NotContains(t, output, "Case: NoDivergence") // No UNKNOWN note when all cert IDs are resolved. require.NotContains(t, output, "UNKNOWN") require.NotContains(t, output, "certificate_per_network_cf") } -func TestPrintDiagnosis_AggsenderAPIFailed_PartialResultDoesNotPrintNoDivergence(t *testing.T) { - t.Parallel() - - result := &DiagnosisResult{ - Case: NoDivergence, - AggsenderAPIFailed: true, - L1SettledLER: common.HexToHash("0x1111"), - L1SettledDepositCount: 4, - L2CurrentLER: common.HexToHash("0x2222"), - L2CurrentDepositCount: 3, - MissingCerts: []MissingCertInfo{ - {Height: 1150, CertID: common.HexToHash("0x3333"), CertIDResolved: true}, - }, - } - - var buf bytes.Buffer - PrintDiagnosis(&buf, result) - output := buf.String() - - require.Contains(t, output, "Aggsender RPC returned no bridge exit data") - require.NotContains(t, output, "Case: NoDivergence") - require.NotContains(t, output, "Nothing to do") -} - // TestPrintDiagnosis_AggsenderAPIFailed_WithUnknownCertID verifies that the extra // UNKNOWN note is printed when one or more cert IDs could not be resolved. func TestPrintDiagnosis_AggsenderAPIFailed_WithUnknownCertID(t *testing.T) { @@ -585,26 +562,29 @@ func TestIsNotFound(t *testing.T) { require.False(t, isNotFound(errors.New("some other error"))) } -// TestCaseDescription verifies caseDescription returns the correct string for each case. -func TestCaseDescription(t *testing.T) { +// TestRecoveryDescription verifies recoveryDescription returns public-facing strings without case labels. +func TestRecoveryDescription(t *testing.T) { t.Parallel() tests := []struct { c RecoveryCase want string }{ - {Case1, "Case1"}, - {Case2, "Case2"}, - {Case3, "Case3"}, - {Case4, "Case4"}, + {Case1, "ForwardLET recovery required"}, + {Case2, "BackwardLET and ForwardLET recovery required"}, + {Case3, "ForwardLET recovery required"}, + {Case4, "BackwardLET and ForwardLET recovery required"}, {NoDivergence, string(NoDivergence)}, // default branch } for _, tc := range tests { t.Run(string(tc.c), func(t *testing.T) { t.Parallel() - got := caseDescription(tc.c) + got := recoveryDescription(tc.c) require.Contains(t, got, tc.want) + if tc.c != NoDivergence { + require.NotContains(t, got, string(tc.c)) + } }) } } @@ -788,7 +768,7 @@ func TestCollectExtraL2Bridges_HappyPath(t *testing.T) { require.Len(t, extra, 2) } -// TestCollectExtraL2Bridges_NotFound verifies that missing bridge-service entries fail fast. +// TestCollectExtraL2Bridges_NotFound verifies that NotFound entries are safe stops. func TestCollectExtraL2Bridges_NotFound(t *testing.T) { t.Parallel() @@ -799,7 +779,7 @@ func TestCollectExtraL2Bridges_NotFound(t *testing.T) { BridgeService: &stubBridgeService{ bridges: map[uint32]*bridgeservicetypes.BridgeResponse{ 3: br3, - // DC 4 is absent → returns ErrNotFound → fail fast + // DC 4 is absent and must stop recovery planning. }, }, L2NetworkID: 1, @@ -807,8 +787,8 @@ func TestCollectExtraL2Bridges_NotFound(t *testing.T) { _, err := collectExtraL2Bridges(context.Background(), env, 3, 5) require.Error(t, err) + require.Contains(t, err.Error(), "bridge service data not ready") require.Contains(t, err.Error(), "DC=4") - require.Contains(t, err.Error(), "not indexed yet") } // TestCollectExtraL2Bridges_ServiceError verifies a non-NotFound error is propagated. @@ -908,7 +888,8 @@ func TestPrintDiagnosis_WithExtraL2Bridges(t *testing.T) { require.Contains(t, output, "Extra Real L2 Bridges") require.Contains(t, output, "200") - require.Contains(t, output, "Case2") + require.Contains(t, output, "Status: Recovery required") + require.NotContains(t, output, "Case2") } // TestFindDivergencePoint_NonMatchingExits verifies the path where exits from a cert diff --git a/tools/backward_forward_let/export_cert_exits.go b/tools/backward_forward_let/export_cert_exits.go new file mode 100644 index 000000000..6446997f5 --- /dev/null +++ b/tools/backward_forward_let/export_cert_exits.go @@ -0,0 +1,259 @@ +package backward_forward_let + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/0xPolygon/cdk-rpc/rpc" + agglayertypes "github.com/agglayer/aggkit/agglayer/types" + "github.com/ethereum/go-ethereum/common" + "github.com/urfave/cli/v2" +) + +const ( + DefaultExportCertExitsMaxCerts uint64 = 2000 + DefaultExportCertExitsTimeout = 30 * time.Minute + exportCertExitsFileMode = 0o600 +) + +var fetchAdminCertificate = fetchAgglayerAdminCertificate + +type certIDsFileJSON struct { + NetworkID uint32 `json:"network_id"` + Certificates map[string]string `json:"certificates"` +} + +type exportCertExitsManifest struct { + NetworkID uint32 `json:"network_id"` + Source string `json:"source"` + AgglayerAdminURL string `json:"agglayer_admin_url"` + OverrideFile string `json:"override_file"` + Certificates []exportCertExitsEntry `json:"certificates"` +} + +type exportCertExitsEntry struct { + Height uint64 `json:"height"` + CertificateID string `json:"certificate_id"` + BridgeExitCount int `json:"bridge_exit_count"` +} + +// RunExportCertExits exports an override JSON from a read-only AggLayer admin source. +func RunExportCertExits(c *cli.Context) error { + cfg, err := LoadConfig(c) + if err != nil { + return err + } + + certIDs, err := loadCertIDsFile(c.String("cert-ids-file"), cfg.BackwardForwardLET.L2NetworkID) + if err != nil { + return err + } + if len(certIDs) == 0 { + return fmt.Errorf("cert IDs file contains no certificates") + } + if uint64(len(certIDs)) > c.Uint64("max-certs") { + return fmt.Errorf("refusing to export %d certificates; increase --max-certs if this batch is intended", len(certIDs)) + } + + timeout := c.Duration("timeout") + if timeout <= 0 { + timeout = DefaultExportCertExitsTimeout + } + ctx, cancel := context.WithTimeout(c.Context, timeout) + defer cancel() + + outPath := filepath.Clean(c.String("out")) + manifestPath := c.String("manifest-out") + if manifestPath == "" { + manifestPath = outPath + ".manifest.json" + } + manifestPath = filepath.Clean(manifestPath) + + heights := sortedCertHeights(certIDs) + override := overrideFileJSON{ + NetworkID: cfg.BackwardForwardLET.L2NetworkID, + Description: "generated by backward-forward-let export-cert-exits from agglayer admin_getCertificate", + Heights: make(map[string][]*agglayertypes.BridgeExit, len(heights)), + } + manifest := exportCertExitsManifest{ + NetworkID: cfg.BackwardForwardLET.L2NetworkID, + Source: "agglayer admin_getCertificate", + AgglayerAdminURL: sanitizeAdminURL(c.String("agglayer-admin-url")), + OverrideFile: outPath, + Certificates: make([]exportCertExitsEntry, 0, len(heights)), + } + + for _, height := range heights { + certID := certIDs[height] + cert, err := fetchAdminCertificate(ctx, c.String("agglayer-admin-url"), certID) + if err != nil { + return fmt.Errorf("fetch admin_getCertificate height=%d certID=%s: %w", height, certID.Hex(), err) + } + if err := validateAdminCertificate(cert, cfg.BackwardForwardLET.L2NetworkID, height, certID); err != nil { + return fmt.Errorf("validate admin certificate height=%d certID=%s: %w", height, certID.Hex(), err) + } + exits := cert.BridgeExits + if exits == nil { + exits = []*agglayertypes.BridgeExit{} + } + override.Heights[strconv.FormatUint(height, 10)] = exits + manifest.Certificates = append(manifest.Certificates, exportCertExitsEntry{ + Height: height, + CertificateID: certID.Hex(), + BridgeExitCount: len(exits), + }) + } + + if err := writeJSONFile(outPath, override); err != nil { + return err + } + if err := writeJSONFile(manifestPath, manifest); err != nil { + return err + } + + fmt.Println("Exported certificate exits override.") + fmt.Printf("Network ID: %d\n", cfg.BackwardForwardLET.L2NetworkID) + fmt.Printf("Certificates exported: %d\n", len(heights)) + fmt.Printf("Override file: %s\n", outPath) + fmt.Printf("Source manifest: %s\n", manifestPath) + fmt.Println("Next:") + fmt.Printf(" backward-forward-let --cfg --cert-exits-file %s --diagnose-only\n", outPath) + fmt.Printf(" backward-forward-let --cfg --cert-exits-file %s\n", outPath) + return nil +} + +func loadCertIDsFile(filePath string, expectedNetworkID uint32) (map[uint64]common.Hash, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("read cert IDs file %s: %w", filePath, err) + } + var raw certIDsFileJSON + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("parse cert IDs file %s: %w", filePath, err) + } + if raw.NetworkID != 0 && raw.NetworkID != expectedNetworkID { + return nil, fmt.Errorf("cert IDs file %s: network_id %d does not match config L2NetworkID %d", + filePath, raw.NetworkID, expectedNetworkID) + } + if raw.Certificates == nil { + return nil, fmt.Errorf("cert IDs file %s: certificates map is missing", filePath) + } + + parsed := make(map[uint64]common.Hash, len(raw.Certificates)) + for key, value := range raw.Certificates { + height, err := strconv.ParseUint(key, 10, 64) + if err != nil { + return nil, fmt.Errorf("cert IDs file %s: non-numeric height key %q: %w", filePath, key, err) + } + certID, err := parseCertificateID(value) + if err != nil { + return nil, fmt.Errorf("cert IDs file %s: height %d has invalid cert ID: %w", filePath, height, err) + } + parsed[height] = certID + } + return parsed, nil +} + +func parseCertificateID(value string) (common.Hash, error) { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, "0x") || len(value) != 66 { + return common.Hash{}, fmt.Errorf("must be a 32-byte 0x-prefixed hex string") + } + if _, err := hex.DecodeString(value[2:]); err != nil { + return common.Hash{}, err + } + return common.HexToHash(value), nil +} + +func sortedCertHeights(certIDs map[uint64]common.Hash) []uint64 { + heights := make([]uint64, 0, len(certIDs)) + for height := range certIDs { + heights = append(heights, height) + } + sort.Slice(heights, func(i, j int) bool { return heights[i] < heights[j] }) + return heights +} + +func fetchAgglayerAdminCertificate( + ctx context.Context, + adminURL string, + certID common.Hash, +) (*agglayertypes.Certificate, error) { + response, err := rpc.JSONRPCCallWithContext(ctx, adminURL, "admin_getCertificate", certID) + if err != nil { + return nil, err + } + if response.Error != nil { + return nil, fmt.Errorf("admin_getCertificate returned error: %v", response.Error) + } + var pair [2]json.RawMessage + if err := json.Unmarshal(response.Result, &pair); err != nil { + return nil, fmt.Errorf( + "unmarshal admin_getCertificate result as [Certificate, CertificateHeader|null]: %w", + err, + ) + } + if len(pair[0]) == 0 || string(pair[0]) == "null" { + return nil, fmt.Errorf("admin_getCertificate returned nil certificate") + } + var cert agglayertypes.Certificate + if err := json.Unmarshal(pair[0], &cert); err != nil { + return nil, fmt.Errorf("unmarshal Certificate from admin_getCertificate result: %w", err) + } + return &cert, nil +} + +func validateAdminCertificate( + cert *agglayertypes.Certificate, + networkID uint32, + height uint64, + certID common.Hash, +) error { + if cert == nil { + return fmt.Errorf("certificate is nil") + } + if cert.NetworkID != networkID { + return fmt.Errorf("network_id %d does not match expected %d", cert.NetworkID, networkID) + } + if cert.Height != height { + return fmt.Errorf("height %d does not match expected %d", cert.Height, height) + } + if calculated := cert.CertificateID(); calculated != certID { + return fmt.Errorf( + "calculated certificate ID %s does not match expected %s", + calculated.Hex(), certID.Hex(), + ) + } + return nil +} + +func writeJSONFile(filePath string, value interface{}) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("marshal %s: %w", filePath, err) + } + if err := os.WriteFile(filePath, append(data, '\n'), exportCertExitsFileMode); err != nil { + return fmt.Errorf("write %s: %w", filePath, err) + } + return nil +} + +func sanitizeAdminURL(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + parsed.User = nil + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String() +} diff --git a/tools/backward_forward_let/export_cert_exits_test.go b/tools/backward_forward_let/export_cert_exits_test.go new file mode 100644 index 000000000..0d1186780 --- /dev/null +++ b/tools/backward_forward_let/export_cert_exits_test.go @@ -0,0 +1,214 @@ +package backward_forward_let + +import ( + "context" + "encoding/json" + "flag" + "math/big" + "os" + "path/filepath" + "testing" + + agglayertypes "github.com/agglayer/aggkit/agglayer/types" + bridgetypes "github.com/agglayer/aggkit/bridgesync/types" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" +) + +func TestLoadCertIDsFile(t *testing.T) { + t.Parallel() + + certID := common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") + path := filepath.Join(t.TempDir(), "cert-ids.json") + require.NoError(t, os.WriteFile(path, []byte(`{ + "network_id": 7, + "certificates": { + "42": "`+certID.Hex()+`" + } + }`), 0o600)) + + got, err := loadCertIDsFile(path, 7) + require.NoError(t, err) + require.Equal(t, certID, got[42]) +} + +func TestLoadCertIDsFileRejectsWrongNetwork(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "cert-ids.json") + require.NoError(t, os.WriteFile(path, []byte(`{"network_id":8,"certificates":{}}`), 0o600)) + + _, err := loadCertIDsFile(path, 7) + require.Error(t, err) + require.Contains(t, err.Error(), "does not match config L2NetworkID") +} + +func TestParseCertificateIDRejectsShortHex(t *testing.T) { + t.Parallel() + + _, err := parseCertificateID("0xdead") + require.Error(t, err) + require.Contains(t, err.Error(), "32-byte") +} + +func TestValidateAdminCertificate(t *testing.T) { + t.Parallel() + + cert := exportTestCertificate(7, 42, nil) + certID := cert.CertificateID() + + require.NoError(t, validateAdminCertificate(cert, 7, 42, certID)) + require.ErrorContains(t, validateAdminCertificate(cert, 8, 42, certID), "network_id") + require.ErrorContains(t, validateAdminCertificate(cert, 7, 43, certID), "height") + require.ErrorContains(t, validateAdminCertificate(cert, 7, 42, common.HexToHash("0x01")), "certificate ID") +} + +func TestRunExportCertExitsWritesOverrideAndManifest(t *testing.T) { + cert42 := exportTestCertificate(7, 42, []*agglayertypes.BridgeExit{ + exportTestBridgeExit(1), + }) + cert43 := exportTestCertificate(7, 43, nil) + cert42ID := cert42.CertificateID() + cert43ID := cert43.CertificateID() + + oldFetch := fetchAdminCertificate + fetchAdminCertificate = func(_ context.Context, adminURL string, certID common.Hash) (*agglayertypes.Certificate, error) { + require.Equal(t, "http://example.test/admin?debug=true", adminURL) + switch certID { + case cert42ID: + return cert42, nil + case cert43ID: + return cert43, nil + default: + t.Fatalf("unexpected cert ID: %s", certID.Hex()) + return nil, nil + } + } + t.Cleanup(func() { fetchAdminCertificate = oldFetch }) + + tmpDir := t.TempDir() + cfgPath := filepath.Join(tmpDir, "config.toml") + require.NoError(t, os.WriteFile(cfgPath, []byte("[BackwardForwardLET]\nL2NetworkID = 7\n"), 0o600)) + certIDsPath := filepath.Join(tmpDir, "cert-ids.json") + require.NoError(t, os.WriteFile(certIDsPath, []byte(`{ + "network_id": 7, + "certificates": { + "42": "`+cert42ID.Hex()+`", + "43": "`+cert43ID.Hex()+`" + } + }`), 0o600)) + + outPath := filepath.Join(tmpDir, "override.json") + manifestPath := filepath.Join(tmpDir, "manifest.json") + ctx := newExportCertExitsCLIContext(t, cfgPath, map[string]string{ + "agglayer-admin-url": "http://example.test/admin?debug=true", + "cert-ids-file": certIDsPath, + "out": outPath, + "manifest-out": manifestPath, + "max-certs": "10", + "timeout": "1m", + }) + + require.NoError(t, RunExportCertExits(ctx)) + + overrideData, err := os.ReadFile(outPath) + require.NoError(t, err) + var override overrideFileJSON + require.NoError(t, json.Unmarshal(overrideData, &override)) + require.Equal(t, uint32(7), override.NetworkID) + require.Len(t, override.Heights["42"], 1) + require.Empty(t, override.Heights["43"]) + + manifestData, err := os.ReadFile(manifestPath) + require.NoError(t, err) + var manifest exportCertExitsManifest + require.NoError(t, json.Unmarshal(manifestData, &manifest)) + require.Equal(t, "http://example.test/admin", manifest.AgglayerAdminURL) + require.Len(t, manifest.Certificates, 2) + require.Equal(t, uint64(42), manifest.Certificates[0].Height) + require.Equal(t, 1, manifest.Certificates[0].BridgeExitCount) + require.Equal(t, uint64(43), manifest.Certificates[1].Height) + require.Equal(t, 0, manifest.Certificates[1].BridgeExitCount) +} + +func TestRunExportCertExitsRejectsOverMaxCerts(t *testing.T) { + tmpDir := t.TempDir() + cfgPath := filepath.Join(tmpDir, "config.toml") + require.NoError(t, os.WriteFile(cfgPath, []byte("[BackwardForwardLET]\nL2NetworkID = 7\n"), 0o600)) + certID := common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") + certIDsPath := filepath.Join(tmpDir, "cert-ids.json") + require.NoError(t, os.WriteFile(certIDsPath, []byte(`{ + "network_id": 7, + "certificates": { + "1": "`+certID.Hex()+`", + "2": "`+certID.Hex()+`" + } + }`), 0o600)) + + ctx := newExportCertExitsCLIContext(t, cfgPath, map[string]string{ + "agglayer-admin-url": "http://example.test/admin", + "cert-ids-file": certIDsPath, + "out": filepath.Join(tmpDir, "override.json"), + "max-certs": "1", + "timeout": "1m", + }) + + err := RunExportCertExits(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "refusing to export 2 certificates") +} + +func newExportCertExitsCLIContext(t *testing.T, configPath string, flags map[string]string) *cli.Context { + t.Helper() + app := cli.NewApp() + app.Flags = []cli.Flag{ + &cli.StringSliceFlag{Name: "cfg", Aliases: []string{"c"}}, + } + parentSet := flag.NewFlagSet("app", flag.ContinueOnError) + for _, f := range app.Flags { + require.NoError(t, f.Apply(parentSet)) + } + require.NoError(t, parentSet.Parse([]string{"--cfg", configPath})) + parentCtx := cli.NewContext(app, parentSet, nil) + + commandSet := flag.NewFlagSet("export-cert-exits", flag.ContinueOnError) + commandSet.String("agglayer-admin-url", "", "") + commandSet.String("cert-ids-file", "", "") + commandSet.String("out", "", "") + commandSet.String("manifest-out", "", "") + commandSet.Uint64("max-certs", DefaultExportCertExitsMaxCerts, "") + commandSet.Duration("timeout", DefaultExportCertExitsTimeout, "") + for name, value := range flags { + require.NoError(t, commandSet.Set(name, value)) + } + return cli.NewContext(app, commandSet, parentCtx) +} + +func exportTestCertificate(networkID uint32, height uint64, exits []*agglayertypes.BridgeExit) *agglayertypes.Certificate { + if exits == nil { + exits = []*agglayertypes.BridgeExit{} + } + return &agglayertypes.Certificate{ + NetworkID: networkID, + Height: height, + PrevLocalExitRoot: common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + NewLocalExitRoot: common.HexToHash("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + BridgeExits: exits, + ImportedBridgeExits: []*agglayertypes.ImportedBridgeExit{}, + } +} + +func exportTestBridgeExit(index byte) *agglayertypes.BridgeExit { + return &agglayertypes.BridgeExit{ + LeafType: bridgetypes.LeafTypeAsset, + TokenInfo: &agglayertypes.TokenInfo{ + OriginNetwork: 0, + OriginTokenAddress: common.Address{}, + }, + DestinationNetwork: 7, + DestinationAddress: common.BytesToAddress([]byte{index}), + Amount: big.NewInt(int64(index)), + Metadata: nil, + } +} diff --git a/tools/backward_forward_let/helpers.go b/tools/backward_forward_let/helpers.go index 620492fb9..624fbc5b2 100644 --- a/tools/backward_forward_let/helpers.go +++ b/tools/backward_forward_let/helpers.go @@ -132,7 +132,8 @@ func makeZeroHashes() []common.Hash { // set by any leaf insertion. This matches the contract's initial storage state and is required // by _checkValidSubtreeFrontier, which rejects non-zero values in unused positions. func computeFrontier(leafHashes []common.Hash, targetIndex uint32) ([32]common.Hash, error) { - if uint32(len(leafHashes)) < targetIndex { + target := int(targetIndex) + if len(leafHashes) < target { return [32]common.Hash{}, fmt.Errorf( "insufficient leaf hashes: need %d, got %d", targetIndex, len(leafHashes), ) @@ -143,10 +144,11 @@ func computeFrontier(leafHashes []common.Hash, targetIndex uint32) ([32]common.H // contract's initial _branch storage state before any leaves are inserted. var frontier [32]common.Hash - for i := uint32(0); i < targetIndex; i++ { - node := leafHashes[i] + for i := 0; i < target; i++ { + node := leafHashes[i] //nolint:gosec // i is bounded by len(leafHashes) through target. + leafIndex := uint32(i) for h := range 32 { - if (i>>h)&1 == 0 { + if (leafIndex>>h)&1 == 0 { // Left child: cache node at this height, propagate up with zero sibling. frontier[h] = node node = crypto.Keccak256Hash(node.Bytes(), zeros[h].Bytes()) diff --git a/tools/backward_forward_let/network_info.go b/tools/backward_forward_let/network_info.go new file mode 100644 index 000000000..6d537f997 --- /dev/null +++ b/tools/backward_forward_let/network_info.go @@ -0,0 +1,32 @@ +package backward_forward_let + +import ( + "context" + "errors" + "fmt" + + agglayertypes "github.com/agglayer/aggkit/agglayer/types" + aggkitgrpc "github.com/agglayer/aggkit/grpc" + "google.golang.org/grpc/codes" +) + +type networkInfoClient interface { + GetNetworkInfo(ctx context.Context, networkID uint32) (agglayertypes.NetworkInfo, error) +} + +func getNetworkInfoAllowNotFound( + ctx context.Context, + client networkInfoClient, + networkID uint32, +) (agglayertypes.NetworkInfo, bool, error) { + info, err := client.GetNetworkInfo(ctx, networkID) + if err == nil { + return info, false, nil + } + + var grpcErr aggkitgrpc.GRPCError + if errors.As(err, &grpcErr) && grpcErr.Code == codes.NotFound { + return agglayertypes.NetworkInfo{}, true, nil + } + return agglayertypes.NetworkInfo{}, false, fmt.Errorf("get network info from agglayer: %w", err) +} diff --git a/tools/backward_forward_let/network_info_test.go b/tools/backward_forward_let/network_info_test.go new file mode 100644 index 000000000..d3084c819 --- /dev/null +++ b/tools/backward_forward_let/network_info_test.go @@ -0,0 +1,47 @@ +package backward_forward_let + +import ( + "context" + "errors" + "testing" + + agglayertypes "github.com/agglayer/aggkit/agglayer/types" + aggkitgrpc "github.com/agglayer/aggkit/grpc" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +type stubNetworkInfoClient struct { + info agglayertypes.NetworkInfo + err error +} + +func (s stubNetworkInfoClient) GetNetworkInfo( + _ context.Context, + _ uint32, +) (agglayertypes.NetworkInfo, error) { + return s.info, s.err +} + +func TestGetNetworkInfoAllowNotFound(t *testing.T) { + t.Parallel() + + info, notFound, err := getNetworkInfoAllowNotFound(context.Background(), stubNetworkInfoClient{ + err: aggkitgrpc.GRPCError{Code: codes.NotFound, Message: "not found"}, + }, 1) + + require.NoError(t, err) + require.True(t, notFound) + require.Empty(t, info) +} + +func TestGetNetworkInfoAllowNotFound_OtherError(t *testing.T) { + t.Parallel() + + _, notFound, err := getNetworkInfoAllowNotFound(context.Background(), stubNetworkInfoClient{ + err: errors.New("boom"), + }, 1) + + require.Error(t, err) + require.False(t, notFound) +} diff --git a/tools/backward_forward_let/override.go b/tools/backward_forward_let/override.go index 393c3d750..29276bafb 100644 --- a/tools/backward_forward_let/override.go +++ b/tools/backward_forward_let/override.go @@ -11,12 +11,6 @@ import ( // BridgeExitsOverride holds pre-extracted certificate bridge exits keyed by height. // Load via LoadBridgeExitsOverride. Use GetExits to retrieve exits for a specific height. -// -// NOTE: the JSON field names follow the Go agglayertypes.BridgeExit json tags -// (e.g., "dest_network", "dest_address"). The agglayer Rust serde may use different -// names (e.g., "destination_network"); if so, build the file by marshaling the -// Certificate.BridgeExits value obtained via json.Unmarshal from the admin API response, -// not from the raw Rust JSON text. type BridgeExitsOverride struct { NetworkID uint32 Description string @@ -38,10 +32,24 @@ type overrideFileJSON struct { Heights map[string][]*agglayertypes.BridgeExit `json:"heights"` } -// LoadBridgeExitsOverride reads and validates a JSON override file containing -// pre-extracted certificate bridge exits keyed by certificate height. +type bridgeExitsOverrideEnvelope struct { + NetworkID uint32 `json:"network_id"` + Description string `json:"description"` + Heights json.RawMessage `json:"heights"` + Certificates json.RawMessage `json:"certificates"` +} + +type agglayerCertificatesFileJSON struct { + NetworkID uint32 `json:"network_id"` + Description string `json:"description"` + Certificates map[string]json.RawMessage `json:"certificates"` +} + +// LoadBridgeExitsOverride reads and validates a JSON fallback file containing +// either raw AggLayer certificates or pre-extracted bridge exits keyed by +// certificate height. // -// Expected file format (heights are string-keyed; amount is a decimal string): +// Preferred Aggkit override file format (heights are string-keyed; amount is a decimal string): // // { // "network_id": 1, @@ -61,27 +69,57 @@ type overrideFileJSON struct { // } // } // +// AggLayer admin export format is also accepted. Each certificate value may be +// either a raw Certificate object, the raw admin_getCertificate JSON-RPC response, +// or the admin_getCertificate result pair [Certificate, CertificateHeader|null]: +// +// { +// "network_id": 1, +// "description": "optional description", +// "certificates": { +// "42": { +// "jsonrpc": "2.0", +// "id": 1, +// "result": [{ "network_id": 1, "height": 42, "bridge_exits": [] }, null] +// } +// } +// } +// // Returns an error when: // - the file cannot be read // - the JSON is malformed // - network_id is zero -// - the heights map is absent +// - neither heights nor certificates is present // - any height key is not a non-negative integer +// - an AggLayer certificate entry has a mismatched network_id or height func LoadBridgeExitsOverride(filePath string) (*BridgeExitsOverride, error) { data, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("read override file %s: %w", filePath, err) } + var envelope bridgeExitsOverrideEnvelope + if err := json.Unmarshal(data, &envelope); err != nil { + return nil, fmt.Errorf("parse override file %s: %w", filePath, err) + } + + if len(envelope.Heights) > 0 { + return loadAggkitBridgeExitsOverride(filePath, data) + } + if len(envelope.Certificates) > 0 { + return loadAgglayerCertificatesOverride(filePath, data) + } + return nil, fmt.Errorf("override file %s: heights map is missing and certificates map is missing", filePath) +} + +func loadAggkitBridgeExitsOverride(filePath string, data []byte) (*BridgeExitsOverride, error) { var raw overrideFileJSON if err := json.Unmarshal(data, &raw); err != nil { return nil, fmt.Errorf("parse override file %s: %w", filePath, err) } - if raw.NetworkID == 0 { return nil, fmt.Errorf("override file %s: network_id must be non-zero", filePath) } - if raw.Heights == nil { return nil, fmt.Errorf("override file %s: heights map is missing", filePath) } @@ -101,3 +139,87 @@ func LoadBridgeExitsOverride(filePath string) (*BridgeExitsOverride, error) { parsed: parsed, }, nil } + +func loadAgglayerCertificatesOverride(filePath string, data []byte) (*BridgeExitsOverride, error) { + var raw agglayerCertificatesFileJSON + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("parse agglayer certificates file %s: %w", filePath, err) + } + if raw.NetworkID == 0 { + return nil, fmt.Errorf("agglayer certificates file %s: network_id must be non-zero", filePath) + } + if raw.Certificates == nil { + return nil, fmt.Errorf("agglayer certificates file %s: certificates map is missing", filePath) + } + + parsed := make(map[uint64][]*agglayertypes.BridgeExit, len(raw.Certificates)) + for key, certRaw := range raw.Certificates { + height, parseErr := strconv.ParseUint(key, 10, 64) + if parseErr != nil { + return nil, fmt.Errorf("agglayer certificates file %s: non-numeric height key %q: %w", filePath, key, parseErr) + } + cert, err := extractAgglayerCertificate(certRaw) + if err != nil { + return nil, fmt.Errorf("agglayer certificates file %s: height %d: %w", filePath, height, err) + } + if cert.NetworkID != raw.NetworkID { + return nil, fmt.Errorf("agglayer certificates file %s: height %d certificate network_id %d does not match file network_id %d", + filePath, height, cert.NetworkID, raw.NetworkID) + } + if cert.Height != height { + return nil, fmt.Errorf("agglayer certificates file %s: height key %d does not match certificate height %d", + filePath, height, cert.Height) + } + exits := cert.BridgeExits + if exits == nil { + exits = []*agglayertypes.BridgeExit{} + } + parsed[height] = exits + } + + description := raw.Description + if description == "" { + description = "generated from agglayer admin_getCertificate responses" + } + return &BridgeExitsOverride{ + NetworkID: raw.NetworkID, + Description: description, + parsed: parsed, + }, nil +} + +func extractAgglayerCertificate(data json.RawMessage) (*agglayertypes.Certificate, error) { + var rpcResponse struct { + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` + } + if err := json.Unmarshal(data, &rpcResponse); err == nil { + if len(rpcResponse.Error) > 0 && string(rpcResponse.Error) != "null" { + return nil, fmt.Errorf("admin_getCertificate response contains error: %s", string(rpcResponse.Error)) + } + if len(rpcResponse.Result) > 0 { + return extractAgglayerCertificate(rpcResponse.Result) + } + } + + var pair [2]json.RawMessage + if err := json.Unmarshal(data, &pair); err == nil && len(pair[0]) > 0 && string(pair[0]) != "null" { + return extractAgglayerCertificate(pair[0]) + } + + var wrapped struct { + Certificate json.RawMessage `json:"certificate"` + } + if err := json.Unmarshal(data, &wrapped); err == nil && len(wrapped.Certificate) > 0 { + return extractAgglayerCertificate(wrapped.Certificate) + } + + var cert agglayertypes.Certificate + if err := json.Unmarshal(data, &cert); err != nil { + return nil, fmt.Errorf("parse certificate: %w", err) + } + if cert.NetworkID == 0 { + return nil, fmt.Errorf("certificate network_id must be non-zero") + } + return &cert, nil +} diff --git a/tools/backward_forward_let/override_test.go b/tools/backward_forward_let/override_test.go index f95c2de77..3bfd51605 100644 --- a/tools/backward_forward_let/override_test.go +++ b/tools/backward_forward_let/override_test.go @@ -169,6 +169,120 @@ func TestLoadBridgeExitsOverride_RoundTrip(t *testing.T) { require.Nil(t, exits[0].Metadata) } +func TestLoadBridgeExitsOverride_AgglayerCertificateObject(t *testing.T) { + t.Parallel() + + originAddr := "0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + destAddr := "0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" + path := writeOverrideFile(t, `{ + "network_id": 7, + "description": "raw agglayer certificates", + "certificates": { + "42": { + "network_id": 7, + "height": 42, + "prev_local_exit_root": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "new_local_exit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "bridge_exits": [ + { + "leaf_type": 0, + "token_info": { + "origin_network": 1, + "origin_token_address": "`+originAddr+`" + }, + "dest_network": 2, + "dest_address": "`+destAddr+`", + "amount": "100", + "metadata": null + } + ], + "imported_bridge_exits": [], + "metadata": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "43": { + "network_id": 7, + "height": 43, + "prev_local_exit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "new_local_exit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "bridge_exits": [], + "imported_bridge_exits": [], + "metadata": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + } + }`) + + result, err := LoadBridgeExitsOverride(path) + require.NoError(t, err) + require.Equal(t, uint32(7), result.NetworkID) + require.Equal(t, "raw agglayer certificates", result.Description) + + exits42, ok := result.GetExits(42) + require.True(t, ok) + require.Len(t, exits42, 1) + require.Equal(t, uint32(2), exits42[0].DestinationNetwork) + require.Equal(t, common.HexToAddress(destAddr), exits42[0].DestinationAddress) + require.Equal(t, big.NewInt(100), exits42[0].Amount) + + exits43, ok := result.GetExits(43) + require.True(t, ok) + require.Empty(t, exits43) +} + +func TestLoadBridgeExitsOverride_AgglayerAdminGetCertificateResponse(t *testing.T) { + t.Parallel() + + path := writeOverrideFile(t, `{ + "network_id": 7, + "certificates": { + "42": { + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "network_id": 7, + "height": 42, + "prev_local_exit_root": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "new_local_exit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "bridge_exits": [], + "imported_bridge_exits": [], + "metadata": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + null + ] + } + } + }`) + + result, err := LoadBridgeExitsOverride(path) + require.NoError(t, err) + require.Equal(t, "generated from agglayer admin_getCertificate responses", result.Description) + exits, ok := result.GetExits(42) + require.True(t, ok) + require.Empty(t, exits) +} + +func TestLoadBridgeExitsOverride_AgglayerCertificateRejectsMismatchedHeight(t *testing.T) { + t.Parallel() + + path := writeOverrideFile(t, `{ + "network_id": 7, + "certificates": { + "42": { + "network_id": 7, + "height": 43, + "prev_local_exit_root": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "new_local_exit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "bridge_exits": [], + "imported_bridge_exits": [], + "metadata": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + } + }`) + + _, err := LoadBridgeExitsOverride(path) + require.ErrorContains(t, err, "height key 42 does not match certificate height 43") +} + // TestLoadBridgeExitsOverride_NonNumericKey verifies that a non-numeric height key // causes an error. func TestLoadBridgeExitsOverride_NonNumericKey(t *testing.T) { diff --git a/tools/backward_forward_let/recovery.go b/tools/backward_forward_let/recovery.go index c47f39a43..16cbecf51 100644 --- a/tools/backward_forward_let/recovery.go +++ b/tools/backward_forward_let/recovery.go @@ -8,6 +8,7 @@ import ( "github.com/0xPolygon/cdk-contracts-tooling/contracts/aggchain-multisig/agglayerbridgel2" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + gethTypes "github.com/ethereum/go-ethereum/core/types" ) // ExecuteRecovery performs the on-chain recovery steps for the given diagnosis. @@ -78,6 +79,10 @@ func ExecuteRecovery(ctx context.Context, env *Env, diagnosis *DiagnosisResult) } } + if err := printFinalVerification(callOpts, env, diagnosis); err != nil { + return fmt.Errorf("final verification: %w", err) + } + return nil } @@ -94,6 +99,7 @@ func stepActivateEmergency( if err != nil { return fmt.Errorf("send ActivateEmergencyState tx: %w", err) } + printTxSent("ActivateEmergencyState", tx) receipt, err := env.waitReceiptFn(ctx, tx) if err != nil { @@ -102,6 +108,7 @@ func stepActivateEmergency( if receipt.Status != 1 { return fmt.Errorf("ActivateEmergencyState tx failed (status=%d)", receipt.Status) } + printTxConfirmed("ActivateEmergencyState", receipt) active, err := env.L2Bridge.IsEmergencyState(callOpts) if err != nil { @@ -128,6 +135,7 @@ func stepDeactivateEmergency( if err != nil { return fmt.Errorf("send DeactivateEmergencyState tx: %w", err) } + printTxSent("DeactivateEmergencyState", tx) receipt, err := env.waitReceiptFn(ctx, tx) if err != nil { @@ -136,6 +144,7 @@ func stepDeactivateEmergency( if receipt.Status != 1 { return fmt.Errorf("DeactivateEmergencyState tx failed (status=%d)", receipt.Status) } + printTxConfirmed("DeactivateEmergencyState", receipt) active, err := env.L2Bridge.IsEmergencyState(callOpts) if err != nil { @@ -170,12 +179,12 @@ func stepBackwardLET( } var frontierBytes [32][32]byte - for i, h := range frontier { - frontierBytes[i] = [32]byte(h) + for i := 0; i < len(frontier); i++ { + frontierBytes[i] = [32]byte(frontier[i]) } var proofBytes [32][32]byte - for i, h := range proof { - proofBytes[i] = [32]byte(h) + for i := 0; i < len(proof); i++ { + proofBytes[i] = [32]byte(proof[i]) } tx, err := env.L2Bridge.BackwardLET( @@ -188,6 +197,7 @@ func stepBackwardLET( if err != nil { return fmt.Errorf("send BackwardLET tx: %w", err) } + printTxSent("BackwardLET", tx) receipt, err := env.waitReceiptFn(ctx, tx) if err != nil { @@ -196,6 +206,7 @@ func stepBackwardLET( if receipt.Status != 1 { return fmt.Errorf("BackwardLET tx failed (status=%d)", receipt.Status) } + printTxConfirmed("BackwardLET", receipt) dcBig, err := env.L2Bridge.DepositCount(callOpts) if err != nil { @@ -255,6 +266,7 @@ func stepForwardLETDivergentLeaves( if err != nil { return fmt.Errorf("send ForwardLET (divergent leaves) tx: %w", err) } + printTxSent("ForwardLET (divergent leaves)", tx) receipt, err := env.waitReceiptFn(ctx, tx) if err != nil { @@ -263,6 +275,7 @@ func stepForwardLETDivergentLeaves( if receipt.Status != 1 { return fmt.Errorf("ForwardLET (divergent leaves) tx failed (status=%d)", receipt.Status) } + printTxConfirmed("ForwardLET (divergent leaves)", receipt) expectedDC := diagnosis.DivergencePoint + uint32(len(diagnosis.DivergentLeaves)) @@ -343,6 +356,7 @@ func stepForwardLETExtraL2Bridges( if err != nil { return fmt.Errorf("send ForwardLET (extra L2 bridges) tx: %w", err) } + printTxSent("ForwardLET (extra L2 bridges)", tx) receipt, err := env.waitReceiptFn(ctx, tx) if err != nil { @@ -351,6 +365,7 @@ func stepForwardLETExtraL2Bridges( if receipt.Status != 1 { return fmt.Errorf("ForwardLET (extra L2 bridges) tx failed (status=%d)", receipt.Status) } + printTxConfirmed("ForwardLET (extra L2 bridges)", receipt) expectedDC := afterDivergentCount + uint32(len(diagnosis.ExtraL2Bridges)) @@ -375,3 +390,57 @@ func stepForwardLETExtraL2Bridges( fmt.Printf("[step] ForwardLET (extra L2 bridges) complete. DC=%d, LER=%s\n", expectedDC, expectedLER.Hex()) return nil } + +func printTxSent(name string, tx *gethTypes.Transaction) { + hash, ok := txHashHex(tx) + if !ok { + fmt.Printf("[tx] %s sent.\n", name) + return + } + fmt.Printf("[tx] %s sent: %s\n", name, hash) +} + +func txHashHex(tx *gethTypes.Transaction) (hash string, ok bool) { + if tx == nil { + return "", false + } + defer func() { + if recover() != nil { + hash = "" + ok = false + } + }() + return tx.Hash().Hex(), true +} + +func printTxConfirmed(name string, receipt *gethTypes.Receipt) { + if receipt == nil || receipt.BlockNumber == nil { + fmt.Printf("[tx] %s confirmed.\n", name) + return + } + fmt.Printf("[tx] %s confirmed in block %s.\n", name, receipt.BlockNumber.String()) +} + +func printFinalVerification(callOpts *bind.CallOpts, env *Env, diagnosis *DiagnosisResult) error { + dcBig, err := env.L2Bridge.DepositCount(callOpts) + if err != nil { + return fmt.Errorf("get final deposit count: %w", err) + } + root32, err := env.L2Bridge.GetRoot(callOpts) + if err != nil { + return fmt.Errorf("get final LER: %w", err) + } + + finalDC := uint32(dcBig.Uint64()) + finalLER := common.Hash(root32) + fmt.Printf("[verify] Final L2 state: DC=%d, LER=%s\n", finalDC, finalLER.Hex()) + if finalLER == diagnosis.L1SettledLER && finalDC == diagnosis.L1SettledDepositCount { + fmt.Println("[verify] Final L2 state matches L1 settled state.") + } else { + fmt.Println( + "[verify] Final L2 state includes replayed L2 bridge data; " + + "rerun diagnosis after aggsender settles the follow-up certificate.", + ) + } + return nil +} diff --git a/tools/backward_forward_let/run.go b/tools/backward_forward_let/run.go index c4f0ebcb9..36082d748 100644 --- a/tools/backward_forward_let/run.go +++ b/tools/backward_forward_let/run.go @@ -174,14 +174,19 @@ func Run(c *cli.Context) error { PrintDiagnosis(os.Stdout, diagnosis) + if diagnosis.AggsenderAPIFailed { + fmt.Printf("\nNo recovery transactions were sent.\n") + fmt.Printf("Provide the missing certificate exits with --cert-exits-file, then rerun diagnosis.\n") + return nil + } + if diagnosis.IsCompleteNoDivergence() { fmt.Println("Nothing to do: L1 settled state and L2 on-chain state are in sync.") return nil } - if diagnosis.AggsenderAPIFailed { - fmt.Printf("\nAggsender RPC was unreachable. Cannot proceed with recovery.\n") - fmt.Printf("Contact your AggLayer admin with the failed certificate details above.\n") + if c.Bool("diagnose-only") { + fmt.Println("Diagnose-only mode: no recovery transactions were sent.") return nil } diff --git a/tools/backward_forward_let/send_cert.go b/tools/backward_forward_let/send_cert.go index dc6a22f87..1c94c2c99 100644 --- a/tools/backward_forward_let/send_cert.go +++ b/tools/backward_forward_let/send_cert.go @@ -34,7 +34,7 @@ type certStorager interface { // RunSendCert is the CLI action for the send-cert subcommand. // It reads a certificate from JSON (--cert-json or --cert-file), sends it to the agglayer, -// and optionally stores it in the aggsender SQLite DB. +// and stores it in the aggsender SQLite DB. func RunSendCert(c *cli.Context) error { // Load config. cfg, err := LoadConfig(c) @@ -52,6 +52,14 @@ func RunSendCert(c *cli.Context) error { return fmt.Errorf("parse certificate JSON: %w", err) } + noDB := c.Bool("no-db") + if noDB && !c.Bool("staging-only") { + return fmt.Errorf("--no-db requires --staging-only") + } + if noDB && c.String("db-path") != "" { + return fmt.Errorf("--no-db and --db-path are mutually exclusive") + } + // Create agglayer client. logger := log.GetDefaultLogger() agglayerClient, err := agglayer.NewAgglayerClient(cfg.AgglayerClient, logger) @@ -59,13 +67,15 @@ func RunSendCert(c *cli.Context) error { return fmt.Errorf("create agglayer client: %w", err) } - var storage certStorager - if !c.Bool("no-db") { - dbPath := c.String("db-path") - storage, err = openAggsenderStorage(logger, dbPath) - if err != nil { - return err - } + // Open aggsender DB. + if noDB { + return sendCertificate(c.Context, cert, certJSON, agglayerClient, nil) + } + + dbPath := c.String("db-path") + storage, err := openAggsenderStorage(logger, dbPath) + if err != nil { + return err } return sendCertificate(c.Context, cert, certJSON, agglayerClient, storage) @@ -85,10 +95,12 @@ func sendCertificate( if err != nil { return fmt.Errorf("send certificate to agglayer: %w", err) } - fmt.Printf("Certificate sent. Hash: %s\n", certHash.Hex()) + fmt.Printf("Certificate ID: %s\n", certHash.Hex()) + fmt.Printf("Certificate height: %d\n", cert.Height) if storage == nil { - fmt.Println("Skipping aggsender DB storage (--no-db).") + fmt.Println("Aggsender DB storage skipped (--no-db).") + fmt.Printf("Next: backward-forward-let --cfg cert-status --wait-settled --height %d\n", cert.Height) return nil } @@ -127,9 +139,11 @@ func sendCertificate( // Store in DB. if err := storage.SaveLastSentCertificate(ctx, record); err != nil { - return fmt.Errorf("store certificate in aggsender DB: %w", err) + return fmt.Errorf("certificate sent with hash %s at height %d, but storing in aggsender DB failed: %w", + certHash.Hex(), cert.Height, err) } fmt.Printf("Certificate stored in aggsender DB at height %d.\n", cert.Height) + fmt.Printf("Next: backward-forward-let --cfg cert-status --wait-settled --height %d\n", cert.Height) return nil } diff --git a/tools/backward_forward_let/send_cert_test.go b/tools/backward_forward_let/send_cert_test.go index 3a08c645b..a553d73fb 100644 --- a/tools/backward_forward_let/send_cert_test.go +++ b/tools/backward_forward_let/send_cert_test.go @@ -144,6 +144,20 @@ func TestSendCertificate_HappyPath(t *testing.T) { require.Equal(t, certJSON, *storage.saved.SignedCertificate) } +func TestSendCertificate_NoDB(t *testing.T) { + t.Parallel() + + expectedHash := common.HexToHash("0xbeef") + sender := &stubAgglayerSender{hash: expectedHash} + + certJSON := minimalCertJSON(9) + var cert agglayertypes.Certificate + require.NoError(t, cert.UnmarshalJSON([]byte(certJSON))) + + err := sendCertificate(context.Background(), cert, certJSON, sender, nil) + require.NoError(t, err) +} + func TestSendCertificate_AgglayerError(t *testing.T) { t.Parallel() @@ -162,20 +176,6 @@ func TestSendCertificate_AgglayerError(t *testing.T) { require.Nil(t, storage.saved) } -func TestSendCertificate_NoDB(t *testing.T) { - t.Parallel() - - expectedHash := common.HexToHash("0xbeef") - sender := &stubAgglayerSender{hash: expectedHash} - - certJSON := minimalCertJSON(4) - var cert agglayertypes.Certificate - require.NoError(t, cert.UnmarshalJSON([]byte(certJSON))) - - err := sendCertificate(context.Background(), cert, certJSON, sender, nil) - require.NoError(t, err) -} - func TestSendCertificate_DBError(t *testing.T) { t.Parallel() @@ -302,6 +302,7 @@ func newSendCertCLIContext(flags map[string]string) *cli.Context { fs.String("cert-file", "", "") fs.String("db-path", "", "") fs.Bool("no-db", false, "") + fs.Bool("staging-only", false, "") for name, val := range flags { _ = fs.Set(name, val) } @@ -328,7 +329,6 @@ func TestRunSendCert_LoadConfigError(t *testing.T) { &cli.StringFlag{Name: "cert-json"}, &cli.StringFlag{Name: "cert-file"}, &cli.StringFlag{Name: "db-path"}, - &cli.BoolFlag{Name: "no-db"}, }, }, } @@ -363,3 +363,29 @@ func TestRunSendCert_InvalidCertJSON(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "parse certificate JSON") } + +func TestRunSendCert_NoDBRequiresStagingOnly(t *testing.T) { + t.Parallel() + + ctx := newSendCertCLIContext(map[string]string{ + "cert-json": minimalCertJSON(1), + "no-db": "true", + }) + err := RunSendCert(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "--no-db requires --staging-only") +} + +func TestRunSendCert_NoDBRejectsDBPath(t *testing.T) { + t.Parallel() + + ctx := newSendCertCLIContext(map[string]string{ + "cert-json": minimalCertJSON(1), + "db-path": "/tmp/test.sqlite", + "no-db": "true", + "staging-only": "true", + }) + err := RunSendCert(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "--no-db and --db-path are mutually exclusive") +}