diff --git a/.github/workflows/socket-basics.yml b/.github/workflows/socket-basics.yml new file mode 100644 index 000000000..11e92e2c9 --- /dev/null +++ b/.github/workflows/socket-basics.yml @@ -0,0 +1,49 @@ +# Socket Basics security scan for stellar-horizon. +# Upstream: https://github.com/SocketDev/socket-basics +# Scanner settings live in .socket-basics.json; SAST path exclusions live in +# .semgrepignore. +# +# This workflow does SAST through OpenGrep, secret scanning through TruffleHog, +# Dockerfile misconfiguration scanning through Trivy, and submits results to +# Socket.dev + +name: Socket Basics security scan + +on: + workflow_dispatch: + schedule: + - cron: "47 14 * * 6" + +permissions: + contents: read + +jobs: + socket-basics: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # Remove the GitHub token from the workspace after checkout, so it is + # not mounted into the Socket Basics container, which doesn't need it. + persist-credentials: false + + - name: Run Socket Basics + env: + SOCKET_SECURITY_API_TOKEN: ${{ secrets.SOCKET_SECURITY_API_TOKEN }} + SOCKET_ORG: stellar + run: | + # Socket Basics is pinned by digest since rule tuning is calibrated + # to the exact ruleset contained in this image. Automatic updates + # could introduce new rules and a sudden increase in false + # positives, as these tools are generally noisy without tuning. + docker run --rm \ + -v "$PWD:/github/workspace" \ + -w /github/workspace \ + -e SOCKET_SECURITY_API_TOKEN \ + -e SOCKET_ORG \ + -e GITHUB_REPOSITORY \ + -e GITHUB_REF_NAME \ + -e GITHUB_SHA \ + ghcr.io/socketdev/socket-basics@sha256:d463bae84f21d0240d5e197acb81e95144a966a20ab7cd05cca3a70e1796a4bb \ + --config .socket-basics.json diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 000000000..c81d9f136 --- /dev/null +++ b/.semgrepignore @@ -0,0 +1,111 @@ +# This file excludes scan paths from Socket Basics SAST scanning (through OpenGrep). +# It replaces the built-in ignore list for scanning with a more thorough list of paths. + +# --- third-party / generated --- +.git/ +node_modules/ +vendor/ +third_party/ +thirdparty/ +.devcontainer/ +dist/ +build/ +target/ +.venv/ +venv/ +__pycache__/ +.yarn/ +generated/ +*.min.js + +# --- test / example / mock code --- +__fixtures__/ +__mocks__/ +__snapshots__/ +__tests__/ +acceptance-test/ +acceptance-tests/ +acceptance_test/ +acceptance_tests/ +benches/ +browser-test/ +browser-tests/ +browser_test/ +browser_tests/ +e2e/ +e2e-test/ +e2e-tests/ +e2e_test/ +e2e_tests/ +example/ +examples/ +fixtures/ +functional-test/ +functional-tests/ +functional_test/ +functional_tests/ +integration-test/ +integration-tests/ +integration_test/ +integration_tests/ +integrationtest/ +integrationtests/ +mock/ +mock-dapp/ +mocks/ +perf-test/ +perf-tests/ +perf_test/ +perf_tests/ +performance-test/ +performance-tests/ +performance_test/ +performance_tests/ +regression-test/ +regression-tests/ +regression_test/ +regression_tests/ +smoke-test/ +smoke-tests/ +smoke_test/ +smoke_tests/ +spec/ +specs/ +test/ +test-data/ +test-fixtures/ +testFixtures/ +testdata/ +testfixtures/ +tests/ +unit-test/ +unit-tests/ +unit_test/ +unit_tests/ +*.test.js +*.test.jsx +*.test.ts +*.test.tsx +*.test.mjs +*.spec.js +*.spec.jsx +*.spec.ts +*.spec.tsx +*_test.go +*_test.py +*_test.rb +*_test.exs +test_*.py +*Test.java +*Tests.java +*Test.kt +*Tests.kt +*Test.scala +*Test.cs +*Tests.cs +tests.rs +test.rs + +# --- repo-specific third-party (PER_REPO_VENDORED + +# git submodules); the generic patterns above cannot know these --- +sac_test/ diff --git a/.socket-basics.json b/.socket-basics.json new file mode 100644 index 000000000..e7dffa450 --- /dev/null +++ b/.socket-basics.json @@ -0,0 +1,13 @@ +{ + "workspace": ".", + "go_sast_enabled": true, + "rust_sast_enabled": true, + "go_disabled_rules": "go-error-not-checked", + "sast_ignore_overrides": "go-command-execution:internal/scripts/build_release_artifacts/main.go,go-file-path-traversal:internal/flags.go,go-file-path-traversal:internal/ingest/main.go,go-file-path-traversal:internal/ingest/processor_runner.go,go-decompression-bomb:internal/db2/schema/bindata.go,go-profiling-endpoint-exposed:cmd/ingest.go,go-weak-random:internal/ingest/loadtest.go,rust-panic-in-production:internal/integration/contracts/bulk/src/lib.rs,rust-panic-in-production:internal/integration/contracts/increment/src/lib.rs", + "secret_scanning_enabled": true, + "console_tabular_enabled": true, + "dockerfiles": "docker/Dockerfile,docker/Dockerfile.core-testing,docker/Dockerfile.dev,docker/Dockerfile.stable,docker/verify-range/Dockerfile,internal/scripts/check_release_hash/Dockerfile", + "socket_tier_1_enabled": false, + "trivy_vuln_enabled": false, + "trufflehog_exclude_dir": "vendor,target,.git,sac_test" +} diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 000000000..2857632dc --- /dev/null +++ b/.trivyignore @@ -0,0 +1,27 @@ +# This file excludes non-security rules from Socket Basics Dockerfile scanning (through Trivy). +# +# Trivy reads this file from the working directory. socket-basics has no config +# key that can disable a Trivy rule, so this is the only lever. +# +# Rules that ARE kept: DS-0002 (no USER - runs as root), DS-0001 (:latest tag), +# DS-0017 (package-manager update alone, which can install from a stale index). + +# Excluded rules: + +# 'apt-get' missing '--no-install-recommends' +DS-0029 + +# 'RUN cd ...' instead of WORKDIR +DS-0013 + +# Deprecated MAINTAINER instruction +DS-0022 + +# 'apk add' missing '--no-cache' +DS-0025 + +# WORKDIR path not absolute +DS-0009 + +# 'apt-get' missing '-y' +DS-0021 diff --git a/CHANGELOG.md b/CHANGELOG.md index fc9a3e529..b8ef4ac95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,24 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Added Protocol 30 support and bumped `MaxSupportedProtocolVersion` to 30 ([#237](https://github.com/stellar/stellar-horizon/pull/237)). A Stellar Asset Contract `transfer` or `mint` to a muxed contract (`W...`) gives the `contract_credited` effect `contract_muxed` and `contract_muxed_id` fields. ### Fixed +- The asset ingestion filter now matches rules on ledger entry changes and on Soroban data in the envelope, in addition to operation bodies. These transactions are now ingested when they involve a whitelisted asset: + - path payments routed through the asset + - trustline flag changes and sponsorship revocations of trustlines + - calls to the Stellar Asset Contract of the asset, including the contract for native lumens, and deployments of that contract + - `ExtendFootprintTTL` and `RestoreFootprint` on entries of that contract + - claims and clawbacks of claimable balances, when the transaction succeeds + - liquidity pool deposits and withdrawals, when the transaction succeeds + - A failed claim, clawback, deposit or withdrawal is still dropped. The operation body carries only a balance id or a pool id, and a failed transaction writes no operation changes. A failed payment or offer in the same asset is kept. So `include_failed` history for an asset depends on the operation type. +- The admin API rejects malformed asset whitelist entries that are not already stored. It stores each valid entry in canonical form. The response lists every rejected entry. +- Operators running with an asset filter can re-run historical range ingestion to back-fill transactions that earlier versions skipped. `db reingest range` first deletes all history in the range, then ingests it again through the filter config stored now. Do this only when the filter config has not changed since the range was first ingested. Otherwise transactions the current config does not keep are deleted, and `db detect-gaps` does not report them. - The asset ingestion filter now matches rules on the inner operations of fee-bump transactions. Earlier versions collected operations only from TxV0 and Tx envelopes, so a fee-bump transaction was not ingested even when its inner operations referenced a whitelisted asset. +- The contract balances behind the `contracts` fields of `/assets` are tracked by ledger key, so each stored balance follows the latest value of its ledger entry, and the contract stats are adjusted from the stored balances. Expired contract balances and asset contracts are removed whenever their expiration ledger has passed, rather than only in the ledger right after it. + +### Updated +- SSE streams are now compressed for clients that send `Accept-Encoding: gzip` or `deflate`, like other responses. A 200-event stream is about 80-90% smaller; clients that don't accept compression get the same response as before. +- Responses are now gzipped for axios 0.x (js-stellar-sdk before 11.0) and Guzzle (PHP SDK) clients even though they send no `Accept-Encoding`, since both always decode gzip. Their large list pages are about 90% smaller. +- The `method` label of the `horizon_http_requests_duration_seconds`, `horizon_http_requests_in_flight` and `horizon_http_requests_received` metrics reports standard HTTP methods as before and groups any other method under `OTHER`. +- State verification compares the asset contracts, contract balances and contract stats behind the `contracts` fields of `/assets` with the history archive, only for Stellar Asset Contracts, and a difference marks the ingestion state invalid, like a difference in any other ingested data. ## 29.0.0 diff --git a/go.mod b/go.mod index 5a91d1dee..de9bc32db 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/squirrel v1.5.4 github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go v1.45.27 // indirect github.com/djherbis/fscache v0.10.1 // indirect diff --git a/go.sum b/go.sum index 13f8a9365..fa581e0a9 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,6 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315 h1:zje9aPr1kQ5nKwjO5MC0S/jehRtNrjfYuLfFRWZH6kY= -github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315/go.mod h1:4U522XvlkqOY2AVBUM7ISHODDb6tdB+KAXfGaBDsWts= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= diff --git a/internal/actions/filter_rules.go b/internal/actions/filter_rules.go index dbaf9dac4..02d3b3f31 100644 --- a/internal/actions/filter_rules.go +++ b/internal/actions/filter_rules.go @@ -4,11 +4,15 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" + "strings" hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon" + "github.com/stellar/go-stellar-sdk/support/collections/set" "github.com/stellar/go-stellar-sdk/support/render/problem" horizonContext "github.com/stellar/stellar-horizon/internal/context" "github.com/stellar/stellar-horizon/internal/db2/history" + "github.com/stellar/stellar-horizon/internal/ingest/filters" ) // these admin HTTP endpoints are documented in internal/httpx/static/admin_oapi.yml @@ -76,6 +80,7 @@ func (handler FilterConfigHandler) UpdateAccountConfig(w http.ResponseWriter, r config, err := historyQ.UpdateAccountFilterConfig(r.Context(), filterConfig) if err != nil { problem.Render(r.Context(), w, err) + return } responsePayload := handler.accountConfigResource(config) @@ -98,13 +103,26 @@ func (handler FilterConfigHandler) UpdateAssetConfig(w http.ResponseWriter, r *h return } + stored, err := historyQ.GetAssetFilterConfig(r.Context()) + if err != nil { + problem.Render(r.Context(), w, err) + return + } + + whitelist, err := canonicalAssetWhitelist(filterRequest.Whitelist, stored.Whitelist) + if err != nil { + problem.Render(r.Context(), w, err) + return + } + filterConfig := history.AssetFilterConfig{} filterConfig.Enabled = *filterRequest.Enabled - filterConfig.Whitelist = filterRequest.Whitelist + filterConfig.Whitelist = whitelist config, err := historyQ.UpdateAssetFilterConfig(r.Context(), filterConfig) if err != nil { problem.Render(r.Context(), w, err) + return } responsePayload := handler.assetConfigResource(config) @@ -114,6 +132,40 @@ func (handler FilterConfigHandler) UpdateAssetConfig(w http.ResponseWriter, r *h } } +// canonicalAssetWhitelist parses each whitelist entry as a SEP-11 asset +// ("CODE:ISSUER" or "native") and returns the canonical form the ingestion +// filter compares against. A new entry that does not parse is rejected, so an +// entry that could never match is not stored. An entry that is already stored +// is kept as it is, even when it does not parse: earlier versions stored +// entries without validation, and a client that reads the config, flips +// "enabled", and writes it back must not be blocked by such an entry. +func canonicalAssetWhitelist(entries []string, stored []string) ([]string, error) { + storedSet := set.NewSet[string](len(stored)) + storedSet.AddSlice(stored) + + whitelist := make([]string, 0, len(entries)) + var invalid []string + for _, entry := range entries { + asset, err := filters.ParseWhitelistAsset(entry) + switch { + case err == nil: + whitelist = append(whitelist, asset.StringCanonical()) + case storedSet.Contains(entry): + whitelist = append(whitelist, entry) + default: + invalid = append(invalid, strconv.Quote(entry)) + } + } + if len(invalid) > 0 { + return nil, problem.NewProblemWithInvalidField( + problem.BadRequest, + "whitelist", + fmt.Errorf("%s: not a valid asset, expected CODE:ISSUER or native", strings.Join(invalid, ", ")), + ) + } + return whitelist, nil +} + func (handler FilterConfigHandler) assetFilterResource(r *http.Request) (hProtocol.AssetFilterConfig, error) { var filterRequest hProtocol.AssetFilterConfig dec := json.NewDecoder(r.Body) diff --git a/internal/actions/filter_rules_test.go b/internal/actions/filter_rules_test.go index 04a0d810a..7fb5312d9 100644 --- a/internal/actions/filter_rules_test.go +++ b/internal/actions/filter_rules_test.go @@ -9,6 +9,7 @@ import ( "testing" hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-horizon/internal/db2/history" "github.com/stellar/stellar-horizon/internal/test" ) @@ -179,7 +180,10 @@ func TestUpdateAssetFilterConfig(t *testing.T) { request.Body = ioutil.NopCloser(strings.NewReader(` { - "whitelist": ["4","5","6"], + "whitelist": [ + "USDC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL", + "EURC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL" + ], "enabled": true }`)) @@ -200,7 +204,134 @@ func TestUpdateAssetFilterConfig(t *testing.T) { tt.Assert.Equal(*filterCfgResource.Enabled, true) tt.Assert.True(filterCfgResource.LastModified > 0) - tt.Assert.ElementsMatch(filterCfgResource.Whitelist, []string{"4", "5", "6"}) + tt.Assert.ElementsMatch(filterCfgResource.Whitelist, []string{ + "USDC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL", + "EURC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL", + }) +} + +func TestUpdateAssetFilterConfigStoresCanonicalWhitelistEntries(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + + q := &history.Q{SessionInterface: tt.HorizonSession()} + + handler := &FilterConfigHandler{} + recorder := httptest.NewRecorder() + request := makeRequest( + t, + map[string]string{}, + map[string]string{}, + q, + ) + + request.Body = ioutil.NopCloser(strings.NewReader(` + { + "whitelist": ["NATIVE"], + "enabled": true + }`)) + + handler.UpdateAssetConfig( + recorder, + request, + ) + + resp := recorder.Result() + tt.Assert.Equal(http.StatusOK, resp.StatusCode) + + raw, err := ioutil.ReadAll(resp.Body) + tt.Assert.NoError(err) + + var filterCfgResource hProtocol.AssetFilterConfig + tt.Assert.NoError(json.Unmarshal(raw, &filterCfgResource)) + tt.Assert.Equal([]string{xdr.MustNewNativeAsset().StringCanonical()}, filterCfgResource.Whitelist) +} + +func TestUpdateAssetFilterConfigRejectsMalformedWhitelistEntry(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + + q := &history.Q{SessionInterface: tt.HorizonSession()} + + for _, entry := range []string{ + "USDC-GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL", + "USDC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL ", + "USDC:not-an-issuer", + "USDC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL,EURC:GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL", + "", + } { + handler := &FilterConfigHandler{} + recorder := httptest.NewRecorder() + request := makeRequest( + t, + map[string]string{}, + map[string]string{}, + q, + ) + body, err := json.Marshal(map[string]interface{}{"whitelist": []string{entry}, "enabled": true}) + tt.Assert.NoError(err) + request.Body = ioutil.NopCloser(strings.NewReader(string(body))) + + handler.UpdateAssetConfig(recorder, request) + + resp := recorder.Result() + tt.Assert.Equal(http.StatusBadRequest, resp.StatusCode, "entry %q", entry) + } + + config, err := q.GetAssetFilterConfig(tt.Ctx) + tt.Assert.NoError(err) + tt.Assert.Empty(config.Whitelist) +} + +func TestUpdateAssetFilterConfigKeepsStoredLegacyWhitelistEntry(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + + q := &history.Q{SessionInterface: tt.HorizonSession()} + // Earlier versions stored entries without validation. + _, err := q.UpdateAssetFilterConfig(tt.Ctx, history.AssetFilterConfig{ + Whitelist: []string{"usdc:1234"}, + Enabled: true, + }) + tt.Assert.NoError(err) + + // A client reads the config, flips "enabled", and writes it back. + handler := &FilterConfigHandler{} + recorder := httptest.NewRecorder() + request := makeRequest(t, map[string]string{}, map[string]string{}, q) + request.Body = ioutil.NopCloser(strings.NewReader(`{"whitelist": ["usdc:1234"], "enabled": false}`)) + handler.UpdateAssetConfig(recorder, request) + + resp := recorder.Result() + tt.Assert.Equal(http.StatusOK, resp.StatusCode) + raw, err := ioutil.ReadAll(resp.Body) + tt.Assert.NoError(err) + var filterCfgResource hProtocol.AssetFilterConfig + tt.Assert.NoError(json.Unmarshal(raw, &filterCfgResource)) + tt.Assert.Equal([]string{"usdc:1234"}, filterCfgResource.Whitelist) + tt.Assert.False(*filterCfgResource.Enabled) +} + +func TestUpdateAssetFilterConfigReportsEveryMalformedWhitelistEntry(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + + q := &history.Q{SessionInterface: tt.HorizonSession()} + handler := &FilterConfigHandler{} + recorder := httptest.NewRecorder() + request := makeRequest(t, map[string]string{}, map[string]string{}, q) + request.Body = ioutil.NopCloser(strings.NewReader(`{"whitelist": ["native", "usdc:1234", "eurc:5678"], "enabled": true}`)) + handler.UpdateAssetConfig(recorder, request) + + resp := recorder.Result() + tt.Assert.Equal(http.StatusBadRequest, resp.StatusCode) + raw, err := ioutil.ReadAll(resp.Body) + tt.Assert.NoError(err) + tt.Assert.Contains(string(raw), `\"usdc:1234\", \"eurc:5678\"`) } func TestUpdateAccountFilterConfig(t *testing.T) { diff --git a/internal/db2/history/asset_stats.go b/internal/db2/history/asset_stats.go index 6b4b1498a..13ade5328 100644 --- a/internal/db2/history/asset_stats.go +++ b/internal/db2/history/asset_stats.go @@ -124,11 +124,11 @@ func (q *Q) UpdateAssetContractExpirations(ctx context.Context, keys []xdr.Hash, return q.updateExpirations(ctx, "asset_contracts", keys, expirationLedgers) } -// DeleteAssetContractsExpiringAt deletes all contract asset contract rows which are active -// at `ledger` and expired at `ledger+1` -func (q *Q) DeleteAssetContractsExpiringAt(ctx context.Context, ledger uint32) (int64, error) { +// DeleteAssetContractsExpiredBefore deletes all asset contract rows whose expiration +// ledger is less than `ledger`, i.e. rows which are no longer active at `ledger`. +func (q *Q) DeleteAssetContractsExpiredBefore(ctx context.Context, ledger uint32) (int64, error) { sql := sq.Delete("asset_contracts"). - Where(map[string]interface{}{"expiration_ledger": ledger}) + Where(sq.Lt{"expiration_ledger": ledger}) result, err := q.Exec(ctx, sql) if err != nil { return 0, err @@ -170,42 +170,36 @@ func (q *Q) InsertContractAssetBalances(ctx context.Context, rows []ContractAsse return nil } -const maxUpdateBatchSize = 30000 +// maxQueryBatchSize keeps the number of query parameters in a single query +// below the postgres limit of 65535. +const maxQueryBatchSize = 10000 -// UpdateContractAssetBalanceAmounts will update the expiration ledgers for the given list of keys -// (if they exist in the db). -func (q *Q) UpdateContractAssetBalanceAmounts(ctx context.Context, keys []xdr.Hash, amounts []string) error { - for len(keys) > 0 { - var args []interface{} - var values []string - - for i := 0; len(keys) > 0 && i < maxUpdateBatchSize; i++ { - args = append(args, keys[0][:], amounts[0]) - values = append(values, "(cast(? as bytea), cast(? as numeric))") - keys = keys[1:] - amounts = amounts[1:] - } +// UpsertContractAssetBalances inserts the given list of rows into the contract_asset_balances +// table, replacing any row which already exists for the same key. +func (q *Q) UpsertContractAssetBalances(ctx context.Context, rows []ContractAssetBalance) error { + for len(rows) > 0 { + batch := rows[:min(len(rows), maxQueryBatchSize)] + rows = rows[len(batch):] - sql := fmt.Sprintf(` - UPDATE contract_asset_balances - SET - amount = myvalues.amount - FROM ( - VALUES - %s - ) AS myvalues (key_hash, amount) - WHERE contract_asset_balances.key_hash = myvalues.key_hash`, - strings.Join(values, ","), + sql := sq.Insert("contract_asset_balances").Columns( + "key_hash", "asset_contract_id", "amount", "expiration_ledger", ) - - _, err := q.ExecRaw(ctx, sql, args...) - if err != nil { - return err + for _, row := range batch { + sql = sql.Values(row.KeyHash, row.ContractID, row.Amount, row.ExpirationLedger) + } + sql = sql.Suffix("ON CONFLICT (key_hash) DO UPDATE SET " + + "asset_contract_id = excluded.asset_contract_id, " + + "amount = excluded.amount, " + + "expiration_ledger = excluded.expiration_ledger") + if _, err := q.Exec(ctx, sql); err != nil { + return errors.Wrap(err, "could not upsert contract asset balances") } } return nil } +const maxUpdateBatchSize = 30000 + // UpdateContractAssetBalanceExpirations will update the expiration ledgers for the given list of keys // (if they exist in the db). func (q *Q) UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xdr.Hash, expirationLedgers []uint32) error { @@ -247,11 +241,12 @@ func (q *Q) updateExpirations(ctx context.Context, table string, keys []xdr.Hash return nil } -// DeleteContractAssetBalancesExpiringAt deletes and returns all contract asset balances which are active -// at `ledger` and expired at `ledger+1` -func (q *Q) DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) { +// DeleteContractAssetBalancesExpiredBefore deletes and returns all contract asset balances +// whose expiration ledger is less than `ledger`, i.e. balances which are no longer active +// at `ledger`. +func (q *Q) DeleteContractAssetBalancesExpiredBefore(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) { sql := sq.Delete("contract_asset_balances"). - Where(map[string]interface{}{"expiration_ledger": ledger}).Suffix("RETURNING *") + Where(sq.Lt{"expiration_ledger": ledger}).Suffix("RETURNING *") var balances []ContractAssetBalance err := q.Select(ctx, &balances, sql) return balances, err @@ -260,15 +255,24 @@ func (q *Q) DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger ui // GetContractAssetBalances fetches all contract_asset_balances rows for the // given list of key hashes. func (q *Q) GetContractAssetBalances(ctx context.Context, keys []xdr.Hash) ([]ContractAssetBalance, error) { - keyBytes := make([][]byte, len(keys)) - for i := range keys { - keyBytes[i] = keys[i][:] - } - sql := sq.Select("contract_asset_balances.*").From("contract_asset_balances"). - Where(map[string]interface{}{"key_hash": keyBytes}) var balances []ContractAssetBalance - err := q.Select(ctx, &balances, sql) - return balances, err + for len(keys) > 0 { + batch := keys[:min(len(keys), maxQueryBatchSize)] + keys = keys[len(batch):] + + keyBytes := make([][]byte, len(batch)) + for i := range batch { + keyBytes[i] = batch[i][:] + } + sql := sq.Select("contract_asset_balances.*").From("contract_asset_balances"). + Where(map[string]interface{}{"key_hash": keyBytes}) + var rows []ContractAssetBalance + if err := q.Select(ctx, &rows, sql); err != nil { + return nil, err + } + balances = append(balances, rows...) + } + return balances, nil } // RemoveContractAssetBalances removes rows from the contract_asset_balances table diff --git a/internal/db2/history/asset_stats_test.go b/internal/db2/history/asset_stats_test.go index 63caa3ac3..f4ee80967 100644 --- a/internal/db2/history/asset_stats_test.go +++ b/internal/db2/history/asset_stats_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/binary" "sort" "testing" @@ -558,7 +559,7 @@ func TestAssetContractExpiration(t *testing.T) { }, results, ) - rowsDeleted, err := q.DeleteAssetContractsExpiringAt(tt.Ctx, 4) + rowsDeleted, err := q.DeleteAssetContractsExpiredBefore(tt.Ctx, 5) tt.Assert.Zero(rowsDeleted) tt.Assert.NoError(err) @@ -572,7 +573,7 @@ func TestAssetContractExpiration(t *testing.T) { }, results, ) - rowsDeleted, err = q.DeleteAssetContractsExpiringAt(tt.Ctx, 5) + rowsDeleted, err = q.DeleteAssetContractsExpiredBefore(tt.Ctx, 6) tt.Assert.Equal(int64(1), rowsDeleted) tt.Assert.NoError(err) @@ -600,7 +601,7 @@ func TestAssetContractExpiration(t *testing.T) { }, results, ) - rowsDeleted, err = q.DeleteAssetContractsExpiringAt(tt.Ctx, 10) + rowsDeleted, err = q.DeleteAssetContractsExpiredBefore(tt.Ctx, 11) tt.Assert.Equal(int64(1), rowsDeleted) tt.Assert.NoError(err) @@ -1170,7 +1171,7 @@ func TestRemoveContractAssetBalances(t *testing.T) { tt.Assert.NoError(q.Rollback()) } -func TestUpdateContractAssetBalanceAmounts(t *testing.T) { +func TestUpsertContractAssetBalances(t *testing.T) { tt := test.Start(t) defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) @@ -1200,22 +1201,53 @@ func TestUpdateContractAssetBalanceAmounts(t *testing.T) { q.InsertContractAssetBalances(context.Background(), []ContractAssetBalance{balance, otherBalance}), ) - nonExistantKeyHash := xdr.Hash{4} + newKeyHash := [32]byte{4} + newBalance := ContractAssetBalance{ + KeyHash: newKeyHash[:], + ContractID: contractID[:], + Amount: "170141183460469231731687303715884105727", + ExpirationLedger: 12, + } + balance.Amount = "2" + balance.ExpirationLedger = 20 + otherBalance.Amount = "1" tt.Assert.NoError( - q.UpdateContractAssetBalanceAmounts( + q.UpsertContractAssetBalances( context.Background(), - []xdr.Hash{otherKeyHash, keyHash, nonExistantKeyHash}, - []string{"1", "2", "3"}, + []ContractAssetBalance{otherBalance, balance, newBalance}, ), ) - balances, err := q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash}) + balances, err := q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash, newKeyHash}) tt.Assert.NoError(err) + assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance, newBalance}) - balance.Amount = "2" - otherBalance.Amount = "1" - assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance}) + tt.Assert.NoError(q.Rollback()) +} + +// TestGetContractAssetBalancesWithManyKeys looks up more keys than the number of +// parameters postgres accepts in a single query. +func TestGetContractAssetBalancesWithManyKeys(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + + q := &Q{tt.HorizonSession()} + tt.Assert.NoError(q.Begin(context.Background())) + + keys := make([]xdr.Hash, 7*maxQueryBatchSize) + for i := range keys { + binary.BigEndian.PutUint32(keys[i][:], uint32(i)) + } + contractID := [32]byte{1} + first := ContractAssetBalance{KeyHash: keys[0][:], ContractID: contractID[:], Amount: "1", ExpirationLedger: 10} + last := ContractAssetBalance{KeyHash: keys[len(keys)-1][:], ContractID: contractID[:], Amount: "2", ExpirationLedger: 11} + tt.Assert.NoError(q.InsertContractAssetBalances(context.Background(), []ContractAssetBalance{first, last})) + + balances, err := q.GetContractAssetBalances(context.Background(), keys) + tt.Assert.NoError(err) + assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{first, last}) tt.Assert.NoError(q.Rollback()) } @@ -1250,11 +1282,15 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) { q.InsertContractAssetBalances(context.Background(), []ContractAssetBalance{balance, otherBalance}), ) - balances, err := q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10) + balances, err := q.DeleteContractAssetBalancesExpiredBefore(context.Background(), 10) + tt.Assert.NoError(err) + tt.Assert.Empty(balances) + + balances, err = q.DeleteContractAssetBalancesExpiredBefore(context.Background(), 11) tt.Assert.NoError(err) assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance}) - balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 11) + balances, err = q.DeleteContractAssetBalancesExpiredBefore(context.Background(), 12) tt.Assert.NoError(err) assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{otherBalance}) @@ -1276,7 +1312,7 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Empty(balances) - balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10) + balances, err = q.DeleteContractAssetBalancesExpiredBefore(context.Background(), 500) tt.Assert.NoError(err) assert.Empty(t, balances) diff --git a/internal/db2/history/main.go b/internal/db2/history/main.go index c9e78d2e4..cbe939dec 100644 --- a/internal/db2/history/main.go +++ b/internal/db2/history/main.go @@ -573,13 +573,13 @@ func (e *ExpAssetStatBalances) Scan(src interface{}) error { type QAssetStats interface { InsertAssetContracts(ctx context.Context, rows []AssetContract) error UpdateAssetContractExpirations(ctx context.Context, keys []xdr.Hash, expirationLedgers []uint32) error - DeleteAssetContractsExpiringAt(ctx context.Context, ledger uint32) (int64, error) + DeleteAssetContractsExpiredBefore(ctx context.Context, ledger uint32) (int64, error) InsertContractAssetBalances(ctx context.Context, rows []ContractAssetBalance) error + UpsertContractAssetBalances(ctx context.Context, rows []ContractAssetBalance) error RemoveContractAssetBalances(ctx context.Context, keys []xdr.Hash) error - UpdateContractAssetBalanceAmounts(ctx context.Context, keys []xdr.Hash, amounts []string) error UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xdr.Hash, expirationLedgers []uint32) error GetContractAssetBalances(ctx context.Context, keys []xdr.Hash) ([]ContractAssetBalance, error) - DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) + DeleteContractAssetBalancesExpiredBefore(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) InsertAssetStats(ctx context.Context, stats []ExpAssetStat) error InsertContractAssetStats(ctx context.Context, rows []ContractAssetStatRow) error InsertAssetStat(ctx context.Context, stat ExpAssetStat) (int64, error) diff --git a/internal/db2/history/mock_q_asset_stats.go b/internal/db2/history/mock_q_asset_stats.go index 8e7d93767..d8e18ba94 100644 --- a/internal/db2/history/mock_q_asset_stats.go +++ b/internal/db2/history/mock_q_asset_stats.go @@ -24,7 +24,7 @@ func (m *MockQAssetStats) UpdateAssetContractExpirations(ctx context.Context, ke return a.Error(0) } -func (m *MockQAssetStats) DeleteAssetContractsExpiringAt(ctx context.Context, ledger uint32) (int64, error) { +func (m *MockQAssetStats) DeleteAssetContractsExpiredBefore(ctx context.Context, ledger uint32) (int64, error) { a := m.Called(ctx, ledger) return a.Get(0).(int64), a.Error(1) } @@ -39,8 +39,8 @@ func (m *MockQAssetStats) RemoveContractAssetBalances(ctx context.Context, keys return a.Error(0) } -func (m *MockQAssetStats) UpdateContractAssetBalanceAmounts(ctx context.Context, keys []xdr.Hash, amounts []string) error { - a := m.Called(ctx, keys, amounts) +func (m *MockQAssetStats) UpsertContractAssetBalances(ctx context.Context, rows []ContractAssetBalance) error { + a := m.Called(ctx, rows) return a.Error(0) } @@ -54,7 +54,7 @@ func (m *MockQAssetStats) GetContractAssetBalances(ctx context.Context, keys []x return a.Get(0).([]ContractAssetBalance), a.Error(1) } -func (m *MockQAssetStats) DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) { +func (m *MockQAssetStats) DeleteContractAssetBalancesExpiredBefore(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) { a := m.Called(ctx, ledger) return a.Get(0).([]ContractAssetBalance), a.Error(1) } diff --git a/internal/httpx/handler.go b/internal/httpx/handler.go index 2f8155ce1..8f139237d 100644 --- a/internal/httpx/handler.go +++ b/internal/httpx/handler.go @@ -38,7 +38,7 @@ func (handler ObjectActionHandler) ServeHTTP( r *http.Request, ) { switch render.Negotiate(r) { - case render.MimeHal, render.MimeJSON: + case render.MimeHal: response, err := handler.Action.GetResource(w, r) if err != nil { problem.Render(r.Context(), w, err) @@ -81,7 +81,7 @@ func (handler streamableObjectActionHandler) ServeHTTP( r *http.Request, ) { switch render.Negotiate(r) { - case render.MimeHal, render.MimeJSON: + case render.MimeHal: response, err := handler.action.GetResource(w, r) if err != nil { problem.Render(r.Context(), w, err) @@ -275,7 +275,7 @@ func (handler pageActionHandler) renderStream(w http.ResponseWriter, r *http.Req func (handler pageActionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch render.Negotiate(r) { - case render.MimeHal, render.MimeJSON: + case render.MimeHal: handler.renderPage(w, r) return case render.MimeEventStream: diff --git a/internal/httpx/middleware.go b/internal/httpx/middleware.go index ffce93dbb..fe3ce2ff8 100644 --- a/internal/httpx/middleware.go +++ b/internal/httpx/middleware.go @@ -64,7 +64,38 @@ func newWrapResponseWriter(w http.ResponseWriter, r *http.Request) middleware.Wr return mw } -// loggerMiddleware logs http requests and resposnes to the logging subsytem of horizon. +// methodLabel returns the value of the "method" label in the HTTP request +// metrics. Standard HTTP methods are reported as is and any other method is +// reported as "OTHER", so the label has a fixed set of values. +func methodLabel(method string) string { + switch method { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodConnect, + http.MethodOptions, http.MethodTrace: + return method + default: + return "OTHER" + } +} + +// gzipForDecodingClients has the compressor gzip responses for clients that send +// no Accept-Encoding but always decode gzip anyway: axios 0.x (js-stellar-sdk +// before 11.0) and Guzzle (the PHP SDK). Without it they download large list pages +// uncompressed. +func gzipForDecodingClients(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Only when the header is absent: an empty Accept-Encoding asks for no encoding. + if _, ok := r.Header["Accept-Encoding"]; !ok { + ua := r.Header.Get("User-Agent") + if strings.HasPrefix(ua, "axios/0.") || strings.HasPrefix(ua, "GuzzleHttp/") { + r.Header.Set("Accept-Encoding", "gzip") + } + } + next.ServeHTTP(w, r) + }) +} + +// loggerMiddleware logs http requests and responses to the logging subsystem of horizon. func loggerMiddleware(serverMetrics *ServerMetrics) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -76,14 +107,13 @@ func loggerMiddleware(serverMetrics *ServerMetrics) func(next http.Handler) http // Checking `Accept` header from user request because if the streaming connection // is reset before sending the first event no Content-Type header is sent in a response. - acceptHeader := r.Header.Get("Accept") - streaming := strings.Contains(acceptHeader, render.MimeEventStream) + streaming := render.Negotiate(r) == render.MimeEventStream route := supportHttp.GetChiRoutePattern(r) requestLabels := prometheus.Labels{ "route": route, "streaming": strconv.FormatBool(streaming), - "method": r.Method, + "method": methodLabel(r.Method), } serverMetrics.RequestsInFlightGauge.With(requestLabels).Inc() defer serverMetrics.RequestsInFlightGauge.With(requestLabels).Dec() @@ -172,7 +202,7 @@ func logEndOfRequest(ctx context.Context, r *http.Request, route string, request "status": strconv.FormatInt(int64(mw.Status()), 10), "route": route, "streaming": strconv.FormatBool(streaming), - "method": r.Method, + "method": methodLabel(r.Method), }).Observe(float64(duration.Seconds())) } diff --git a/internal/httpx/middleware_test.go b/internal/httpx/middleware_test.go new file mode 100644 index 000000000..a6890f181 --- /dev/null +++ b/internal/httpx/middleware_test.go @@ -0,0 +1,48 @@ +package httpx + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMethodLabel(t *testing.T) { + for _, method := range []string{ + http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, + http.MethodDelete, http.MethodConnect, http.MethodOptions, http.MethodTrace, + } { + assert.Equal(t, method, methodLabel(method)) + } + for _, method := range []string{"get", "Post", "PROPFIND", "PURGE", "X"} { + assert.Equal(t, "OTHER", methodLabel(method)) + } +} + +func TestGzipForDecodingClients(t *testing.T) { + for _, tc := range []struct { + userAgent string + header []string // Accept-Encoding values sent, nil when the header is absent + want []string // Accept-Encoding values the next handler sees + }{ + {"axios/0.21.4", nil, []string{"gzip"}}, + {"GuzzleHttp/7", nil, []string{"gzip"}}, + {"axios/0.21.4", []string{""}, []string{""}}, + {"GuzzleHttp/7", []string{"br"}, []string{"br"}}, + {"axios/1.6.0", nil, nil}, + {"Go-http-client/1.1", nil, nil}, + } { + r, err := http.NewRequest(http.MethodGet, "/", nil) + assert.NoError(t, err) + r.Header.Set("User-Agent", tc.userAgent) + if tc.header != nil { + r.Header["Accept-Encoding"] = tc.header + } + var got []string + gzipForDecodingClients(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header["Accept-Encoding"] + })).ServeHTTP(httptest.NewRecorder(), r) + assert.Equal(t, tc.want, got, "%s %q", tc.userAgent, tc.header) + } +} diff --git a/internal/httpx/router.go b/internal/httpx/router.go index 5290acaa5..cd791ec95 100644 --- a/internal/httpx/router.go +++ b/internal/httpx/router.go @@ -84,6 +84,11 @@ func NewRouter(config *RouterConfig, serverMetrics *ServerMetrics, ledgerState * return &result, nil } +// compressedContentTypes are the response types gzipped or deflated for clients +// that accept it. SSE streams flush after every event, and the compressor flushes +// with them, so compressing streams doesn't hold events back. +var compressedContentTypes = []string{"application/hal+json", "text/event-stream"} + func (r *Router) addMiddleware(config *RouterConfig, rateLimitter *throttled.HTTPRateLimiter, serverMetrics *ServerMetrics) { @@ -108,7 +113,8 @@ func (r *Router) addMiddleware(config *RouterConfig, }) } r.Use(recoverMiddleware) - r.Use(chimiddleware.Compress(flate.DefaultCompression, "application/hal+json")) + r.Use(gzipForDecodingClients) + r.Use(chimiddleware.Compress(flate.DefaultCompression, compressedContentTypes...)) c := cors.New(cors.Options{ AllowedOrigins: []string{}, diff --git a/internal/httpx/server.go b/internal/httpx/server.go index b7a1038ff..b42899db8 100644 --- a/internal/httpx/server.go +++ b/internal/httpx/server.go @@ -51,9 +51,14 @@ const ( // defaultWriteTimeout applies when the connection timeout is not set. defaultWriteTimeout = 60 * time.Second + + // maxHeaderBytes bounds the size of request headers, which net/http + // otherwise allows up to 1MB. Requests to Horizon need far less. + maxHeaderBytes = 64 * 1024 ) -// newHTTPServer builds a listener bounded on read, write, and idle timeouts. +// newHTTPServer builds a listener bounded on read, write, and idle timeouts, +// and on request header size. // // The write timeout is derived from the connection timeout rather than fixed. // --connection-timeout defaults to 55s, but operators raise it behind load @@ -72,11 +77,12 @@ func newHTTPServer(addr string, handler http.Handler, connectionTimeout time.Dur writeTimeout = connectionTimeout + writeTimeoutBuffer } return &http.Server{ - Addr: addr, - Handler: handler, - ReadTimeout: serverReadTimeout, - WriteTimeout: writeTimeout, - IdleTimeout: serverIdleTimeout, + Addr: addr, + Handler: handler, + ReadTimeout: serverReadTimeout, + WriteTimeout: writeTimeout, + IdleTimeout: serverIdleTimeout, + MaxHeaderBytes: maxHeaderBytes, } } diff --git a/internal/httpx/server_timeouts_test.go b/internal/httpx/server_timeouts_test.go index aa2b44aa7..7cc5872f9 100644 --- a/internal/httpx/server_timeouts_test.go +++ b/internal/httpx/server_timeouts_test.go @@ -20,3 +20,8 @@ func TestNewHTTPServerFallsBackWhenConnectionTimeoutUnset(t *testing.T) { srv := newHTTPServer(":0", http.NewServeMux(), 0) assert.Equal(t, defaultWriteTimeout, srv.WriteTimeout) } + +func TestNewHTTPServerLimitsRequestHeaderSize(t *testing.T) { + srv := newHTTPServer(":0", http.NewServeMux(), 0) + assert.Equal(t, maxHeaderBytes, srv.MaxHeaderBytes) +} diff --git a/internal/httpx/static/admin_oapi.yml b/internal/httpx/static/admin_oapi.yml index 2919af59e..aeed58f58 100644 --- a/internal/httpx/static/admin_oapi.yml +++ b/internal/httpx/static/admin_oapi.yml @@ -143,11 +143,11 @@ components: items: type: string description: |- - a list of canonical asset ids which the asset filter will inspect ledger transactions, if any transaction operations reference the asset id, then the transaction is ingested to local horizon history database, otherwise it will be skipped. + a list of canonical asset ids, each in the form CODE:ISSUER or the string native. A transaction is ingested to the local horizon history database when one of its operations or its Soroban footprint names a listed asset. It is also ingested when one of its ledger entry changes touches a listed asset. Otherwise it is skipped. A failed transaction has no ledger entry changes, so a failed claimable balance claim, liquidity pool deposit or similar operation that names the asset only through an id is skipped. A new entry that is not a valid asset id is rejected with a 400 response that lists every invalid entry. An entry that is already stored is kept as it is. example: - - 'usdc:1234' - - 'dotx:1234' - - 'abdc:1234' + - 'USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN' + - 'EURC:GDHU6WRG4IEQXM5NZ4BMPKOXHW76MZM4Y2IEMFDVXBSDP6SJY4ITNPP2' + - 'native' enabled: type: boolean description: |- diff --git a/internal/ingest/asset_stats_contract_data_test.go b/internal/ingest/asset_stats_contract_data_test.go new file mode 100644 index 000000000..71ff1fb9b --- /dev/null +++ b/internal/ingest/asset_stats_contract_data_test.go @@ -0,0 +1,664 @@ +package ingest + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + ingestsdk "github.com/stellar/go-stellar-sdk/ingest" + "github.com/stellar/go-stellar-sdk/ingest/sac" + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/support/db" + "github.com/stellar/go-stellar-sdk/support/errors" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-horizon/internal/db2" + "github.com/stellar/stellar-horizon/internal/db2/history" + "github.com/stellar/stellar-horizon/internal/ingest/processors" + "github.com/stellar/stellar-horizon/internal/test" +) + +const contractDataTestPassphrase = "contract data test network" + +// contractDataTest ingests ledgers of contract data changes the way live ingestion +// does, while keeping a model of the contract data and ttl ledger entries, including +// entries which expired and were not restored. +type contractDataTest struct { + t *testing.T + ctx context.Context + q *history.Q + rand *rand.Rand + ledger uint32 + data map[xdr.Hash]xdr.LedgerEntryData + ttl map[xdr.Hash]uint32 + changes []ingestsdk.Change + + assetContracts []xdr.ContractId + instances map[xdr.ContractId]xdr.LedgerEntryData + otherContracts []xdr.ContractId + holders [][32]byte +} + +func newContractDataTest(t *testing.T, seed int64) (*contractDataTest, func()) { + tt := test.Start(t) + test.ResetHorizonDB(t, tt.HorizonDB) + + c := &contractDataTest{ + t: t, + ctx: tt.Ctx, + q: &history.Q{SessionInterface: &db.Session{DB: tt.HorizonDB}}, + rand: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded so that failures are reproducible + ledger: 100, + data: map[xdr.Hash]xdr.LedgerEntryData{}, + ttl: map[xdr.Hash]uint32{}, + instances: map[xdr.ContractId]xdr.LedgerEntryData{}, + holders: [][32]byte{{1}, {2}, {3}, {4}}, + } + issuer := keypair.MustRandom().Address() + for _, code := range []string{"EUR", "USDOLLAR"} { + contractID, err := xdr.MustNewCreditAsset(code, issuer).ContractID(contractDataTestPassphrase) + require.NoError(t, err) + instance, err := sac.AssetToContractData(false, code, issuer, contractID) + require.NoError(t, err) + c.assetContracts = append(c.assetContracts, contractID) + c.instances[contractID] = instance + } + c.otherContracts = []xdr.ContractId{{0xAA}, {0xBB}} + return c, tt.Finish +} + +func contractDataKeyHash(t *testing.T, data xdr.LedgerEntryData) xdr.Hash { + entry := xdr.LedgerEntry{Data: data} + key, err := entry.LedgerKey() + require.NoError(t, err) + bin, err := key.MarshalBinary() + require.NoError(t, err) + return sha256.Sum256(bin) +} + +func ttlEntryData(keyHash xdr.Hash, liveUntil uint32) xdr.LedgerEntryData { + return xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeTtl, + Ttl: &xdr.TtlEntry{KeyHash: keyHash, LiveUntilLedgerSeq: xdr.Uint32(liveUntil)}, + } +} + +func ledgerEntryChange(changeType xdr.LedgerEntryChangeType, pre, post *xdr.LedgerEntryData) ingestsdk.Change { + change := ingestsdk.Change{ChangeType: changeType} + if pre != nil { + change.Type = pre.Type + change.Pre = &xdr.LedgerEntry{Data: *pre} + } + if post != nil { + change.Type = post.Type + change.Post = &xdr.LedgerEntry{Data: *post} + } + return change +} + +// withValue returns a copy of a contract data entry with a different value. +func withValue(data xdr.LedgerEntryData, val xdr.ScVal) xdr.LedgerEntryData { + contractData := *data.ContractData + contractData.Val = val + return xdr.LedgerEntryData{Type: xdr.LedgerEntryTypeContractData, ContractData: &contractData} +} + +func (c *contractDataTest) create(data xdr.LedgerEntryData, liveUntil uint32) { + keyHash := contractDataKeyHash(c.t, data) + ttl := ttlEntryData(keyHash, liveUntil) + c.data[keyHash] = data + c.ttl[keyHash] = liveUntil + c.changes = append(c.changes, + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryCreated, nil, &data), + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryCreated, nil, &ttl), + ) +} + +func (c *contractDataTest) update(keyHash xdr.Hash, data xdr.LedgerEntryData) { + pre := c.data[keyHash] + c.data[keyHash] = data + c.changes = append(c.changes, ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryUpdated, &pre, &data)) +} + +func (c *contractDataTest) extend(keyHash xdr.Hash, liveUntil uint32) { + pre := ttlEntryData(keyHash, c.ttl[keyHash]) + post := ttlEntryData(keyHash, liveUntil) + c.ttl[keyHash] = liveUntil + c.changes = append(c.changes, ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryUpdated, &pre, &post)) +} + +func (c *contractDataTest) remove(keyHash xdr.Hash) { + data := c.data[keyHash] + ttl := ttlEntryData(keyHash, c.ttl[keyHash]) + delete(c.data, keyHash) + delete(c.ttl, keyHash) + c.changes = append(c.changes, + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryRemoved, &data, nil), + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryRemoved, &ttl, nil), + ) +} + +func (c *contractDataTest) restore(keyHash xdr.Hash, liveUntil uint32) { + data := c.data[keyHash] + ttl := ttlEntryData(keyHash, liveUntil) + c.ttl[keyHash] = liveUntil + c.changes = append(c.changes, + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryRestored, nil, &data), + ledgerEntryChange(xdr.LedgerEntryChangeTypeLedgerEntryRestored, nil, &ttl), + ) +} + +// closeLedger ingests the changes of the current ledger with the same change +// compactor configuration and processor as live ingestion, in one db transaction. +func (c *contractDataTest) closeLedger() { + compactor := ingestsdk.NewChangeCompactor(ingestsdk.ChangeCompactorConfig{SuppressRemoveAfterRestoreChange: true}) + for _, change := range c.changes { + require.NoError(c.t, compactor.AddChange(change)) + } + + require.NoError(c.t, c.q.BeginTx(c.ctx, &sql.TxOptions{})) + processor := processors.NewAssetStatsProcessor(c.q, contractDataTestPassphrase, false, c.ledger) + for _, change := range compactor.GetChanges() { + require.NoError(c.t, processor.ProcessChange(c.ctx, change)) + } + require.NoError(c.t, processor.Commit(c.ctx)) + require.NoError(c.t, c.q.Commit()) + + c.changes = nil + c.ledger++ +} + +func (c *contractDataTest) liveUntil() uint32 { + return c.ledger + c.rand.Uint32()%40 +} + +// randomValue returns the value of a balance entry. Stellar Asset Contracts always +// write balances, while other contracts can write anything. +func (c *contractDataTest) randomValue(contractID xdr.ContractId, holder [32]byte, assetContract bool) xdr.LedgerEntryData { + balance := sac.BalanceToContractData(contractID, holder, c.rand.Uint64()%1000) + if assetContract { + return balance + } + switch c.rand.Intn(4) { + case 0: + val := xdr.Uint32(c.rand.Uint32() % 10) + return withValue(balance, xdr.ScVal{Type: xdr.ScValTypeScvU32, U32: &val}) + case 1: + return sac.BalanceInt128ToContractData(contractID, holder, xdr.Int128Parts{Hi: -1, Lo: 0}) + default: + return balance + } +} + +// randomLedger changes a few random balance entries and asset contract instances. +func (c *contractDataTest) randomLedger() { + contracts := append(append([]xdr.ContractId{}, c.assetContracts...), c.otherContracts...) + changed := map[xdr.Hash]bool{} + for range 1 + c.rand.Intn(4) { + contractIndex := c.rand.Intn(len(contracts)) + contractID := contracts[contractIndex] + holder := c.holders[c.rand.Intn(len(c.holders))] + keyHash := contractDataKeyHash(c.t, sac.BalanceToContractData(contractID, holder, 0)) + if !changed[keyHash] { + changed[keyHash] = true + c.randomBalanceChange(keyHash, contractID, holder, contractIndex < len(c.assetContracts)) + } + } + c.randomInstanceChanges() +} + +func (c *contractDataTest) randomBalanceChange(keyHash xdr.Hash, contractID xdr.ContractId, holder [32]byte, assetContract bool) { + _, exists := c.data[keyHash] + switch { + case !exists: + c.create(c.randomValue(contractID, holder, assetContract), c.liveUntil()) + case c.ttl[keyHash] < c.ledger: + // archived entries can only be restored, sometimes along with an update + if c.rand.Intn(2) == 0 { + c.restore(keyHash, c.liveUntil()) + if c.rand.Intn(2) == 0 { + c.update(keyHash, c.randomValue(contractID, holder, assetContract)) + } + } + default: + switch c.rand.Intn(5) { + case 0: + c.update(keyHash, c.randomValue(contractID, holder, assetContract)) + case 1: + c.extend(keyHash, c.ttl[keyHash]+1+c.rand.Uint32()%20) + case 2: + c.update(keyHash, c.randomValue(contractID, holder, assetContract)) + c.extend(keyHash, c.ttl[keyHash]+1+c.rand.Uint32()%20) + case 3: + c.remove(keyHash) + } + } +} + +func (c *contractDataTest) randomInstanceChanges() { + for _, contractID := range c.assetContracts { + keyHash := contractDataKeyHash(c.t, c.instances[contractID]) + switch _, exists := c.data[keyHash]; { + case !exists: + c.create(c.instances[contractID], c.liveUntil()) + case c.ttl[keyHash] < c.ledger: + if c.rand.Intn(10) == 0 { + c.restore(keyHash, c.liveUntil()) + } + case c.rand.Intn(10) == 0: + c.extend(keyHash, c.ttl[keyHash]+1+c.rand.Uint32()%20) + } + } +} + +// checkAssetContractBalances checks that the stored balances of the Stellar Asset +// Contracts match the balance entries which are active in the given ledger. +func (c *contractDataTest) checkAssetContractBalances(ledger uint32) { + var keys []xdr.Hash + expected := map[xdr.Hash]history.ContractAssetBalance{} + for _, contractID := range c.assetContracts { + for _, holder := range c.holders { + keyHash := contractDataKeyHash(c.t, sac.BalanceToContractData(contractID, holder, 0)) + keys = append(keys, keyHash) + data, ok := c.data[keyHash] + if !ok || c.ttl[keyHash] < ledger { + continue + } + _, amount, ok := sac.ContractBalanceFromContractData(xdr.LedgerEntry{Data: data}, contractDataTestPassphrase) + require.True(c.t, ok) + expected[keyHash] = history.ContractAssetBalance{ + KeyHash: keyHash[:], + ContractID: contractID[:], + Amount: amount.String(), + ExpirationLedger: c.ttl[keyHash], + } + } + } + + rows, err := c.q.GetContractAssetBalances(c.ctx, keys) + require.NoError(c.t, err) + actual := map[xdr.Hash]history.ContractAssetBalance{} + for _, row := range rows { + var keyHash xdr.Hash + copy(keyHash[:], row.KeyHash) + actual[keyHash] = row + } + require.Equal(c.t, expected, actual, "contract balances of asset contracts at ledger %d", ledger) +} + +// newSystem returns a system which last ingested the given ledger, and whose history +// archive holds the ledger entries of the model at that ledger. +func (c *contractDataTest) newSystem(ledger uint32) (*system, *mockHistoryArchiveAdapter) { + reader := &memoryChangeReader{} + addEntry := func(entry xdr.LedgerEntry) { + reader.changes = append(reader.changes, xdr.LedgerEntryChange{ + Type: xdr.LedgerEntryChangeTypeLedgerEntryState, + State: &entry, + }) + } + for _, data := range c.data { + if entry := (xdr.LedgerEntry{Data: data}); ledgerEntryFilter(contractDataTestPassphrase, entry) { + addEntry(entry) + } + } + for keyHash, liveUntil := range c.ttl { + addEntry(xdr.LedgerEntry{Data: ttlEntryData(keyHash, liveUntil)}) + } + + historyAdapter := &mockHistoryArchiveAdapter{} + historyAdapter.On("GetLatestLedgerSequence").Return(ledger, nil).Maybe() + historyAdapter.On("GetState", mock.Anything, ledger).Return(reader, nil).Once() + + require.NoError(c.t, c.q.UpdateLastLedgerIngest(c.ctx, ledger)) + sys := &system{ + ctx: c.ctx, + historyQ: c.q, + historyAdapter: historyAdapter, + runStateVerificationOnLedger: func(uint32) bool { return true }, + config: Config{NetworkPassphrase: contractDataTestPassphrase}, + } + sys.initMetrics() + return sys, historyAdapter +} + +// verifyState runs the state verification of a system whose history archive holds the +// ledger entries of the model at the given ledger. +func (c *contractDataTest) verifyState(ledger uint32) error { + sys, historyAdapter := c.newSystem(ledger) + err := sys.verifyState(false, ledger, xdr.Hash{}) + historyAdapter.AssertExpectations(c.t) + return err +} + +// checkAssetStats runs the check of asset stats of the state verification, in the +// current transaction of the test, with the ledger entries in the model. +func (c *contractDataTest) checkAssetStats(ledger uint32) error { + set := processors.NewContractAssetStatSet(c.q, contractDataTestPassphrase, ledger) + for keyHash, liveUntil := range c.ttl { + entry := xdr.LedgerEntry{Data: ttlEntryData(keyHash, liveUntil)} + if err := set.AddTTL(ingestsdk.Change{Type: xdr.LedgerEntryTypeTtl, Post: &entry}); err != nil { + return err + } + } + for _, data := range c.data { + if err := set.AddContractData(ingestsdk.Change{ + Type: xdr.LedgerEntryTypeContractData, + Post: &xdr.LedgerEntry{Data: data}, + }); err != nil { + return err + } + } + return checkAssetStats(c.ctx, processors.NewAssetStatSet(), set, c.q) +} + +// TestContractDataIngestionMatchesVerification ingests ledgers of random changes to +// contract balance entries and asset contract instances. The values written by +// contracts which are not Stellar Asset Contracts change between balances and other +// values. After every ledger the stored balances of the Stellar Asset Contracts must +// match the ledger, and every 10 ledgers the state verification must pass. +func TestContractDataIngestionMatchesVerification(t *testing.T) { + for _, seed := range []int64{1, 2, 3} { + t.Run(fmt.Sprintf("seed %d", seed), func(t *testing.T) { + c, finish := newContractDataTest(t, seed) + defer finish() + + for range 300 { + c.randomLedger() + c.closeLedger() + ledger := c.ledger - 1 + c.checkAssetContractBalances(ledger) + if ledger%10 == 0 { + require.NoError(t, c.verifyState(ledger), "state verification at ledger %d", ledger) + } + } + }) + } +} + +func (c *contractDataTest) balanceRow( + contractID xdr.ContractId, holder [32]byte, amount string, expirationLedger uint32, +) history.ContractAssetBalance { + keyHash := contractDataKeyHash(c.t, sac.BalanceToContractData(contractID, holder, 0)) + return history.ContractAssetBalance{ + KeyHash: keyHash[:], + ContractID: contractID[:], + Amount: amount, + ExpirationLedger: expirationLedger, + } +} + +type contractDataMismatchCase struct { + name string + change func() error + expected []string +} + +// contractDataMismatchCases returns changes to the stored contract data of a test whose +// first asset contract holds a balance of 10 for its first holder and of 20 for its +// second holder, along with parts of the mismatches each change must produce. +func contractDataMismatchCases( + c *contractDataTest, liveUntil uint32, extraAssetContract history.AssetContract, +) []contractDataMismatchCase { + assetContractID := c.assetContracts[0] + upsertBalance := func(contractID xdr.ContractId, holder [32]byte, amount string, expirationLedger uint32) error { + return c.q.UpsertContractAssetBalances(c.ctx, []history.ContractAssetBalance{ + c.balanceRow(contractID, holder, amount, expirationLedger), + }) + } + return []contractDataMismatchCase{ + { + name: "balance amount", + change: func() error { return upsertBalance(assetContractID, c.holders[0], "11", liveUntil) }, + expected: []string{"has amount 10 in HAS but 11 in db"}, + }, + { + name: "balance expiration", + change: func() error { return upsertBalance(assetContractID, c.holders[0], "10", liveUntil+1) }, + expected: []string{fmt.Sprintf("has expiration %d in HAS but %d in db", liveUntil, liveUntil+1)}, + }, + { + name: "missing balance", + change: func() error { + keyHash := contractDataKeyHash(c.t, sac.BalanceToContractData(assetContractID, c.holders[1], 0)) + return c.q.RemoveContractAssetBalances(c.ctx, []xdr.Hash{keyHash}) + }, + expected: []string{"is missing from db"}, + }, + { + name: "contract stats", + change: func() error { + _, err := c.q.UpdateContractAssetStat(c.ctx, history.ContractAssetStatRow{ + ContractID: assetContractID[:], + Stat: history.ContractStat{ActiveBalance: "31", ActiveHolders: 2}, + }) + return err + }, + expected: []string{"in HAS but {ActiveBalance:31 ActiveHolders:2} in db"}, + }, + { + name: "asset contract type", + change: func() error { + _, err := c.q.ExecRaw(c.ctx, "UPDATE asset_contracts SET asset_type = ? WHERE contract_id = ?", + xdr.AssetTypeAssetTypeCreditAlphanum12, assetContractID[:]) + return err + }, + expected: []string{"has type AssetTypeAssetTypeCreditAlphanum4 in HAS but AssetTypeAssetTypeCreditAlphanum12 in db"}, + }, + { + name: "missing asset contract", + change: func() error { + _, err := c.q.DeleteAssetContractsExpiredBefore(c.ctx, liveUntil+1) + return err + }, + expected: []string{"HAS contains 1 asset contracts which are missing from db"}, + }, + { + name: "extra asset contract", + change: func() error { return c.q.InsertAssetContracts(c.ctx, []history.AssetContract{extraAssetContract}) }, + expected: []string{"db contains asset contract of USDOLLAR:"}, + }, + { + name: "balance of a contract which is not an asset contract", + change: func() error { return upsertBalance(c.otherContracts[0], c.holders[0], "31", liveUntil) }, + }, + } +} + +// requireStateError checks that err is a StateError which mentions each expected string. +func requireStateError(t *testing.T, err error, expected ...string) { + _, isStateError := errors.Cause(err).(ingestsdk.StateError) + require.True(t, isStateError, "expected a StateError, got %v", err) + for _, e := range expected { + require.Contains(t, err.Error(), e) + } +} + +// TestCheckAssetStatsReportsContractDataMismatches changes the stored contract data in +// different ways, and checks that the state verification of asset stats reports each +// difference as a StateError. Differences in the balances of contracts which are not +// Stellar Asset Contracts are not reported. It then checks that the mismatch marks the +// ingestion state invalid. +func TestCheckAssetStatsReportsContractDataMismatches(t *testing.T) { + c, finish := newContractDataTest(t, 1) + defer finish() + + assetContractID := c.assetContracts[0] + liveUntil := c.ledger + 100 + c.create(c.instances[assetContractID], liveUntil) + c.create(sac.BalanceToContractData(assetContractID, c.holders[0], 10), liveUntil) + c.create(sac.BalanceToContractData(assetContractID, c.holders[1], 20), liveUntil) + c.create(sac.BalanceToContractData(c.otherContracts[0], c.holders[0], 30), liveUntil) + c.closeLedger() + ledger := c.ledger - 1 + require.NoError(t, c.checkAssetStats(ledger)) + + // an asset contract whose instance is not in the ledger + instance := c.instances[c.assetContracts[1]] + asset, ok := sac.AssetFromContractData(xdr.LedgerEntry{Data: instance}, contractDataTestPassphrase) + require.True(t, ok) + instanceKeyHash := contractDataKeyHash(t, instance) + extraAssetContract := history.AssetContract{ + KeyHash: instanceKeyHash[:], + ContractID: c.assetContracts[1][:], + ExpirationLedger: liveUntil, + } + require.NoError(t, asset.Extract( + &extraAssetContract.AssetType, &extraAssetContract.AssetCode, &extraAssetContract.AssetIssuer, + )) + + for _, testCase := range contractDataMismatchCases(c, liveUntil, extraAssetContract) { + t.Run(testCase.name, func(t *testing.T) { + require.NoError(t, c.q.Begin(c.ctx)) + defer func() { + require.NoError(t, c.q.Rollback()) + }() + require.NoError(t, testCase.change()) + + err := c.checkAssetStats(ledger) + if len(testCase.expected) == 0 { + require.NoError(t, err) + return + } + requireStateError(t, err, testCase.expected...) + }) + } + + c.checkContractDataMismatchInvalidatesState(ledger, liveUntil) +} + +// checkContractDataMismatchInvalidatesState changes the stored balance of the first +// holder of the first asset contract, and checks that the state verification reports it +// as a StateError and that, run from the ingestion system, it marks the state invalid. +func (c *contractDataTest) checkContractDataMismatchInvalidatesState(ledger, liveUntil uint32) { + t := c.t + require.NoError(t, c.verifyState(ledger)) + require.NoError(t, c.q.UpsertContractAssetBalances(c.ctx, []history.ContractAssetBalance{ + c.balanceRow(c.assetContracts[0], c.holders[0], "11", liveUntil), + })) + requireStateError(t, c.verifyState(ledger), "has amount 10 in HAS but 11 in db") + + sys, historyAdapter := c.newSystem(ledger) + sys.maybeVerifyState(ledger, xdr.Hash{}) + sys.wg.Wait() + historyAdapter.AssertExpectations(t) + invalid, err := c.q.GetExpStateInvalid(c.ctx) + require.NoError(t, err) + require.True(t, invalid) +} + +// TestCheckAssetStatsRejectsClassicAssetStatsWithoutAccounts checks that a classic +// asset stat row without accounts, which ingestion never stores, is a StateError, +// whether or not its asset has an asset contract. +func TestCheckAssetStatsRejectsClassicAssetStatsWithoutAccounts(t *testing.T) { + c, finish := newContractDataTest(t, 1) + defer finish() + + assetContractID := c.assetContracts[0] + c.create(c.instances[assetContractID], c.ledger+100) + c.closeLedger() + ledger := c.ledger - 1 + require.NoError(t, c.checkAssetStats(ledger)) + + asset, ok := sac.AssetFromContractData(xdr.LedgerEntry{Data: c.instances[assetContractID]}, contractDataTestPassphrase) + require.True(t, ok) + withAssetContract := history.ExpAssetStat{Balances: emptyClassicBalances} + withAssetContract.Balances.Authorized = "5" + require.NoError(t, asset.Extract( + &withAssetContract.AssetType, &withAssetContract.AssetCode, &withAssetContract.AssetIssuer, + )) + withoutAssetContract := history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetCode: "GBP", + AssetIssuer: keypair.MustRandom().Address(), + Balances: emptyClassicBalances, + } + + for _, assetStat := range []history.ExpAssetStat{withAssetContract, withoutAssetContract} { + t.Run(assetStat.AssetCode, func(t *testing.T) { + require.NoError(t, c.q.Begin(c.ctx)) + defer func() { + require.NoError(t, c.q.Rollback()) + }() + _, err := c.q.InsertAssetStat(c.ctx, assetStat) + require.NoError(t, err) + _, isStateError := errors.Cause(c.checkAssetStats(ledger)).(ingestsdk.StateError) + require.True(t, isStateError) + }) + } +} + +// TestCheckAssetStatsMatchesClassicAssetStatsByAssetType checks that an asset contract +// whose asset type differs from the classic asset stat of its asset is reported as a +// StateError, whichever of the two db rows of the asset comes first. +func TestCheckAssetStatsMatchesClassicAssetStatsByAssetType(t *testing.T) { + ctx := context.Background() + issuer := keypair.MustRandom().Address() + asset := xdr.MustNewCreditAsset("EUR", issuer) + contractID, err := asset.ContractID(contractDataTestPassphrase) + require.NoError(t, err) + instance, err := sac.AssetToContractData(false, "EUR", issuer, contractID) + require.NoError(t, err) + + set := processors.NewAssetStatSet() + require.NoError(t, set.AddTrustline(ingestsdk.Change{ + Type: xdr.LedgerEntryTypeTrustline, + Post: &xdr.LedgerEntry{Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: &xdr.TrustLineEntry{ + AccountId: xdr.MustAddress(keypair.MustRandom().Address()), + Balance: 123, + Asset: asset.ToTrustLineAsset(), + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), + }, + }}, + })) + contractSet := processors.NewContractAssetStatSet(&mockDBQ{}, contractDataTestPassphrase, 100) + require.NoError(t, contractSet.AddContractData(ingestsdk.Change{ + Type: xdr.LedgerEntryTypeContractData, + Post: &xdr.LedgerEntry{Data: instance}, + })) + ttl := xdr.LedgerEntry{Data: ttlEntryData(contractDataKeyHash(t, instance), 200)} + require.NoError(t, contractSet.AddTTL(ingestsdk.Change{Type: xdr.LedgerEntryTypeTtl, Post: &ttl})) + + // the join of GetAssetStats returns separate rows for the classic asset stat and + // for an asset contract with a different asset type + classicRow := history.AssetAndContractStat{ExpAssetStat: set.All()[0]} + contractIDBytes := contractID[:] + contractRow := history.AssetAndContractStat{ + ExpAssetStat: history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum12, + AssetCode: "EUR", + AssetIssuer: issuer, + Balances: emptyClassicBalances, + }, + ContractID: &contractIDBytes, + Contracts: history.ContractStat{ActiveBalance: "0", ActiveHolders: 0}, + } + + for name, rows := range map[string][]history.AssetAndContractStat{ + "classic row first": {classicRow, contractRow}, + "contract row first": {contractRow, classicRow}, + } { + t.Run(name, func(t *testing.T) { + q := &mockDBQ{} + q.MockQAssetStats.On("GetAssetStats", ctx, "", "", db2.PageQuery{ + Order: "asc", + Limit: assetStatsBatchSize, + }).Return(rows, nil).Once() + q.MockQAssetStats.On("GetAssetStats", ctx, "", "", db2.PageQuery{ + Cursor: rows[1].PagingToken(), + Order: "asc", + Limit: assetStatsBatchSize, + }).Return([]history.AssetAndContractStat{}, nil).Maybe() + requireStateError(t, checkAssetStats(ctx, set, contractSet, q), + "has type AssetTypeAssetTypeCreditAlphanum4 in HAS but AssetTypeAssetTypeCreditAlphanum12 in db") + q.MockQAssetStats.AssertExpectations(t) + }) + } +} diff --git a/internal/ingest/core_lcm_test.go b/internal/ingest/core_lcm_test.go index ab38f53d0..c5bb7cf8f 100644 --- a/internal/ingest/core_lcm_test.go +++ b/internal/ingest/core_lcm_test.go @@ -83,7 +83,7 @@ func ingestCoreLCMFile(t *testing.T, path string) *history.Q { }, historyQ: historyQ, session: historyQ, - filters: filters.NewFilters(), + filters: filters.NewFilters(coreTestNetworkPassphrase), } // Run the full pipeline (change + transaction processors) on diff --git a/internal/ingest/filters/asset.go b/internal/ingest/filters/asset.go index 1bffcfe3c..6278ddba3 100644 --- a/internal/ingest/filters/asset.go +++ b/internal/ingest/filters/asset.go @@ -2,6 +2,7 @@ package filters import ( "context" + "fmt" "github.com/stellar/go-stellar-sdk/ingest" "github.com/stellar/go-stellar-sdk/support/collections/set" @@ -18,9 +19,53 @@ var ( ) type assetFilter struct { - canonicalAssetsLookup set.Set[string] - lastModified int64 - enabled bool + networkPassphrase string + whitelistedAssets map[assetKey]struct{} + // Stellar Asset Contract ids of the whitelisted assets. Rebuilt in + // RefreshAssetFilter so FilterTransaction never hashes per transaction. + contractIDs set.Set[xdr.ContractId] + // Number of entries in the stored whitelist, including entries that do not + // parse. The filter is enabled only when the stored whitelist is not empty. + whitelistLen int + lastModified int64 + enabled bool +} + +// assetKey identifies an asset with plain values, so it can be a map key. +// xdr.Asset holds pointers and cannot be compared. Building an assetKey +// copies a few bytes. Building the canonical string, as +// xdr.Asset.StringCanonical does, encodes the issuer to strkey and allocates. +// The filter builds a key for every asset in every transaction, so the key +// keeps that cost low. +// +// The key matches the canonical string rules: +// - A native asset is the zero key. A credit asset always has an issuer. +// - The code is padded with zero bytes to 12 bytes. So "USDC" stored as a +// 4-byte code and "USDC" stored as a 12-byte code give the same key, the +// same way both give the string "USDC:". +type assetKey struct { + code [12]byte + issuer xdr.Uint256 +} + +func newAssetKey(asset xdr.Asset) assetKey { + return alphaNumKey(asset.Type, asset.AlphaNum4, asset.AlphaNum12) +} + +// alphaNumKey builds the key from the pointers the XDR unions already hold, so +// xdr.Asset, xdr.TrustLineAsset and xdr.ChangeTrustAsset share one builder +// and no value is copied to the heap. +func alphaNumKey(assetType xdr.AssetType, alphaNum4 *xdr.AlphaNum4, alphaNum12 *xdr.AlphaNum12) assetKey { + var key assetKey + switch assetType { + case xdr.AssetTypeAssetTypeCreditAlphanum4: + copy(key.code[:], alphaNum4.AssetCode[:]) + key.issuer = *alphaNum4.Issuer.Ed25519 + case xdr.AssetTypeAssetTypeCreditAlphanum12: + copy(key.code[:], alphaNum12.AssetCode[:]) + key.issuer = *alphaNum12.Issuer.Ed25519 + } + return key } type AssetFilter interface { @@ -28,10 +73,25 @@ type AssetFilter interface { RefreshAssetFilter(filterConfig *history.AssetFilterConfig) error } -func NewAssetFilter() AssetFilter { +func NewAssetFilter(networkPassphrase string) AssetFilter { return &assetFilter{ - canonicalAssetsLookup: set.Set[string]{}, + networkPassphrase: networkPassphrase, + whitelistedAssets: map[assetKey]struct{}{}, + contractIDs: set.NewSet[xdr.ContractId](0), + } +} + +// ParseWhitelistAsset parses one asset whitelist entry in the SEP-11 form, +// "CODE:ISSUER" or "native". +func ParseWhitelistAsset(entry string) (xdr.Asset, error) { + assets, err := xdr.BuildAssets(entry) + if err != nil { + return xdr.Asset{}, err } + if len(assets) != 1 { + return xdr.Asset{}, fmt.Errorf("%q is not a single asset", entry) + } + return assets[0], nil } func (f *assetFilter) Name() string { @@ -44,19 +104,56 @@ func (f *assetFilter) RefreshAssetFilter(filterConfig *history.AssetFilterConfig if filterConfig.LastModified > f.lastModified { logger.Infof("New Asset Filter config detected, reloading new config %v ", *filterConfig) f.enabled = filterConfig.Enabled - f.canonicalAssetsLookup = listToSet(filterConfig.Whitelist) + f.whitelistedAssets, f.contractIDs = f.lookupsForWhitelist(filterConfig.Whitelist) + f.whitelistLen = len(filterConfig.Whitelist) f.lastModified = filterConfig.LastModified } return nil } +// lookupsForWhitelist parses each entry, so an entry such as "NATIVE", saved +// before the admin API validated entries, still matches. An entry that does +// not parse never matches. It still counts in whitelistLen, so the filter +// stays enabled and keeps dropping unmatched transactions, as before. +func (f *assetFilter) lookupsForWhitelist(whitelist []string) (map[assetKey]struct{}, set.Set[xdr.ContractId]) { + assets := make(map[assetKey]struct{}, len(whitelist)) + contractIDs := set.NewSet[xdr.ContractId](len(whitelist)) + for _, entry := range whitelist { + asset, err := ParseWhitelistAsset(entry) + if err != nil { + logger.Warnf("asset filter whitelist entry %q is not a canonical asset, it will never match", entry) + continue + } + assets[newAssetKey(asset)] = struct{}{} + id, err := asset.ContractID(f.networkPassphrase) + if err != nil { + logger.Warnf("could not derive contract id for asset filter whitelist entry %q: %v", entry, err) + continue + } + contractIDs.Add(xdr.ContractId(id)) + } + return assets, contractIDs +} + +// FilterTransaction keeps a transaction when a whitelisted asset is named by +// its envelope or touched by one of its ledger entry changes. func (f *assetFilter) FilterTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (bool, bool, error) { if !f.isEnabled() { return false, true, nil } - if f.filterOperationsMatchedOnRules(transaction.Envelope.Operations()) { + // The envelope comes first because it is cheaper to read. It is also the + // only source for a failed transaction, whose meta carries no operation + // changes. + if f.matchEnvelope(transaction) { + return true, true, nil + } + matched, err := f.matchMeta(transaction.UnsafeMeta) + if err != nil { + return true, false, err + } + if matched { return true, true, nil } @@ -64,77 +161,275 @@ func (f *assetFilter) FilterTransaction(ctx context.Context, transaction ingest. return true, false, nil } -func (f assetFilter) filterOperationsMatchedOnRules(operations []xdr.Operation) bool { - for _, operation := range operations { - switch operation.Body.Type { - case xdr.OperationTypeChangeTrust: - if f.filterChangeTrustMatched(operation) { - return true - } - case xdr.OperationTypeManageSellOffer: - if f.assetMatchedFilter(&operation.Body.ManageSellOfferOp.Buying) || f.assetMatchedFilter(&operation.Body.ManageSellOfferOp.Selling) { - return true - } - case xdr.OperationTypeManageBuyOffer: - if f.assetMatchedFilter(&operation.Body.ManageBuyOfferOp.Buying) || f.assetMatchedFilter(&operation.Body.ManageBuyOfferOp.Selling) { - return true - } - case xdr.OperationTypeCreateClaimableBalance: - if f.assetMatchedFilter(&operation.Body.CreateClaimableBalanceOp.Asset) { - return true - } - case xdr.OperationTypeCreatePassiveSellOffer: - if f.assetMatchedFilter(&operation.Body.CreatePassiveSellOfferOp.Buying) || f.assetMatchedFilter(&operation.Body.CreatePassiveSellOfferOp.Selling) { - return true - } - case xdr.OperationTypeClawback: - if f.assetMatchedFilter(&operation.Body.ClawbackOp.Asset) { - return true - } - case xdr.OperationTypePayment: - if f.assetMatchedFilter(&operation.Body.PaymentOp.Asset) { - return true - } - case xdr.OperationTypePathPaymentStrictReceive: - if f.assetMatchedFilter(&operation.Body.PathPaymentStrictReceiveOp.DestAsset) || f.assetMatchedFilter(&operation.Body.PathPaymentStrictReceiveOp.SendAsset) { - return true - } - case xdr.OperationTypePathPaymentStrictSend: - if f.assetMatchedFilter(&operation.Body.PathPaymentStrictSendOp.DestAsset) || f.assetMatchedFilter(&operation.Body.PathPaymentStrictSendOp.SendAsset) { - return true - } +func (f *assetFilter) matchEnvelope(transaction ingest.LedgerTransaction) bool { + for _, operation := range transaction.Envelope.Operations() { + if f.matchOperation(transaction, operation) { + return true + } + } + return f.matchFootprint(transaction.Envelope) +} + +// matchOperation reads every asset an operation body names. +func (f *assetFilter) matchOperation(transaction ingest.LedgerTransaction, operation xdr.Operation) bool { + body := operation.Body + switch body.Type { + case xdr.OperationTypeChangeTrust: + return f.matchChangeTrustAsset(&body.ChangeTrustOp.Line) + case xdr.OperationTypeManageSellOffer: + return f.matchAsset(&body.ManageSellOfferOp.Buying) || f.matchAsset(&body.ManageSellOfferOp.Selling) + case xdr.OperationTypeManageBuyOffer: + return f.matchAsset(&body.ManageBuyOfferOp.Buying) || f.matchAsset(&body.ManageBuyOfferOp.Selling) + case xdr.OperationTypeCreatePassiveSellOffer: + return f.matchAsset(&body.CreatePassiveSellOfferOp.Buying) || f.matchAsset(&body.CreatePassiveSellOfferOp.Selling) + case xdr.OperationTypeCreateClaimableBalance: + return f.matchAsset(&body.CreateClaimableBalanceOp.Asset) + case xdr.OperationTypeClawback: + return f.matchAsset(&body.ClawbackOp.Asset) + case xdr.OperationTypePayment: + return f.matchAsset(&body.PaymentOp.Asset) + case xdr.OperationTypePathPaymentStrictReceive: + op := body.PathPaymentStrictReceiveOp + return f.matchAsset(&op.SendAsset) || f.matchAsset(&op.DestAsset) || f.matchAnyAsset(op.Path) + case xdr.OperationTypePathPaymentStrictSend: + op := body.PathPaymentStrictSendOp + return f.matchAsset(&op.SendAsset) || f.matchAsset(&op.DestAsset) || f.matchAnyAsset(op.Path) + case xdr.OperationTypeSetTrustLineFlags: + return f.matchAsset(&body.SetTrustLineFlagsOp.Asset) + case xdr.OperationTypeAllowTrust: + return f.matchAllowTrust(transaction, operation) + case xdr.OperationTypeRevokeSponsorship: + return f.matchRevokeSponsorship(body.RevokeSponsorshipOp) + case xdr.OperationTypeInvokeHostFunction: + return f.matchHostFunction(&body.InvokeHostFunctionOp.HostFunction) + } + return false +} + +// matchAllowTrust rebuilds the asset from the code in the operation. The +// issuer is the operation source, or the transaction source when the +// operation has none. For a fee-bump, Envelope.SourceAccount is the inner +// transaction source. +func (f *assetFilter) matchAllowTrust(transaction ingest.LedgerTransaction, operation xdr.Operation) bool { + issuer := transaction.Envelope.SourceAccount() + if operation.SourceAccount != nil { + issuer = *operation.SourceAccount + } + var key assetKey + switch code := operation.Body.AllowTrustOp.Asset; code.Type { + case xdr.AssetTypeAssetTypeCreditAlphanum4: + copy(key.code[:], code.AssetCode4[:]) + case xdr.AssetTypeAssetTypeCreditAlphanum12: + copy(key.code[:], code.AssetCode12[:]) + default: + return false + } + key.issuer = ed25519Of(issuer) + _, ok := f.whitelistedAssets[key] + return ok +} + +func ed25519Of(account xdr.MuxedAccount) xdr.Uint256 { + if account.Type == xdr.CryptoKeyTypeKeyTypeMuxedEd25519 { + return account.Med25519.Ed25519 + } + return *account.Ed25519 +} + +// matchRevokeSponsorship matches nothing for a pool share trustline. The key +// carries only the pool id, and the revoke does not change the pool entry. +func (f *assetFilter) matchRevokeSponsorship(op *xdr.RevokeSponsorshipOp) bool { + if op.LedgerKey == nil || op.LedgerKey.Type != xdr.LedgerEntryTypeTrustline { + return false + } + return f.matchTrustLineAsset(&op.LedgerKey.TrustLine.Asset) +} + +// matchHostFunction matches a call into the Stellar Asset Contract of a +// whitelisted asset, and the deployment of that contract. A failed call has +// no operation meta, so this is the only source that can match it. +func (f *assetFilter) matchHostFunction(fn *xdr.HostFunction) bool { + switch fn.Type { + case xdr.HostFunctionTypeHostFunctionTypeInvokeContract: + return f.matchContractAddress(&fn.InvokeContract.ContractAddress) + case xdr.HostFunctionTypeHostFunctionTypeCreateContract: + return f.matchContractPreimage(&fn.CreateContract.ContractIdPreimage) + case xdr.HostFunctionTypeHostFunctionTypeCreateContractV2: + return f.matchContractPreimage(&fn.CreateContractV2.ContractIdPreimage) + } + return false +} + +func (f *assetFilter) matchContractPreimage(preimage *xdr.ContractIdPreimage) bool { + return preimage.FromAsset != nil && f.matchAsset(preimage.FromAsset) +} + +// matchFootprint reads the Soroban footprint of the envelope. It names every +// contract data entry a Soroban transaction reads or writes, so it matches +// ExtendFootprintTTL and RestoreFootprint on the entries of a whitelisted +// Stellar Asset Contract. Those operations write only TTL entries to the meta +// before protocol 23, and a TTL entry carries only a key hash. +func (f *assetFilter) matchFootprint(envelope xdr.TransactionEnvelope) bool { + var ext *xdr.TransactionExt + switch envelope.Type { + case xdr.EnvelopeTypeEnvelopeTypeTx: + ext = &envelope.V1.Tx.Ext + case xdr.EnvelopeTypeEnvelopeTypeTxFeeBump: + ext = &envelope.FeeBump.Tx.InnerTx.V1.Tx.Ext + default: + return false + } + if ext.SorobanData == nil { + return false + } + footprint := &ext.SorobanData.Resources.Footprint + return f.matchLedgerKeys(footprint.ReadOnly) || f.matchLedgerKeys(footprint.ReadWrite) +} + +func (f *assetFilter) matchLedgerKeys(keys []xdr.LedgerKey) bool { + for i := range keys { + if keys[i].Type == xdr.LedgerEntryTypeContractData && f.matchContractAddress(&keys[i].ContractData.Contract) { + return true } } return false } -func (f assetFilter) filterChangeTrustMatched(operation xdr.Operation) bool { - if pool, ok := operation.Body.ChangeTrustOp.Line.GetLiquidityPool(); ok { - if f.assetMatchedFilter(&pool.ConstantProduct.AssetA) || f.assetMatchedFilter(&pool.ConstantProduct.AssetB) { +// matchMeta reads the ledger entry changes straight from the transaction meta, +// in order: changes before the operations, the changes of each operation, and +// the changes after the operations. Fee changes are not in the meta and are +// not read. ingest.LedgerTransaction.GetChanges is not used: it rejects meta +// V0, and it sorts every change, which costs time on each transaction the +// filter drops. +func (f *assetFilter) matchMeta(meta xdr.TransactionMeta) (bool, error) { + switch meta.V { + case 0: + // GetOperations would dereference a nil pointer on an empty V0 meta. + return meta.Operations != nil && f.matchOperationChanges(*meta.Operations), nil + case 1: + return f.matchChanges(meta.V1.TxChanges) || f.matchOperationChanges(meta.V1.Operations), nil + case 2: + return f.matchChanges(meta.V2.TxChangesBefore) || + f.matchOperationChanges(meta.V2.Operations) || + f.matchChanges(meta.V2.TxChangesAfter), nil + case 3: + return f.matchChanges(meta.V3.TxChangesBefore) || + f.matchOperationChanges(meta.V3.Operations) || + f.matchChanges(meta.V3.TxChangesAfter), nil + case 4: + return f.matchChanges(meta.V4.TxChangesBefore) || + f.matchOperationChangesV2(meta.V4.Operations) || + f.matchChanges(meta.V4.TxChangesAfter), nil + default: + return false, fmt.Errorf("unsupported transaction meta version %d", meta.V) + } +} + +func (f *assetFilter) matchOperationChanges(operations []xdr.OperationMeta) bool { + for i := range operations { + if f.matchChanges(operations[i].Changes) { return true } - } else { - asset := operation.Body.ChangeTrustOp.Line.ToAsset() - if f.assetMatchedFilter(&asset) { + } + return false +} + +func (f *assetFilter) matchOperationChangesV2(operations []xdr.OperationMetaV2) bool { + for i := range operations { + if f.matchChanges(operations[i].Changes) { return true } } return false } -func (f *assetFilter) assetMatchedFilter(asset *xdr.Asset) bool { - return f.canonicalAssetsLookup.Contains(asset.StringCanonical()) +// matchChanges skips removals. A removal carries only the ledger key, and core +// writes the state of the removed entry just before it. +func (f *assetFilter) matchChanges(changes xdr.LedgerEntryChanges) bool { + for i := range changes { + var entry *xdr.LedgerEntry + switch change := &changes[i]; change.Type { + case xdr.LedgerEntryChangeTypeLedgerEntryCreated: + entry = change.Created + case xdr.LedgerEntryChangeTypeLedgerEntryUpdated: + entry = change.Updated + case xdr.LedgerEntryChangeTypeLedgerEntryState: + entry = change.State + case xdr.LedgerEntryChangeTypeLedgerEntryRestored: + entry = change.Restored + } + if entry != nil && f.matchEntry(entry) { + return true + } + } + return false +} + +// matchEntry reads every asset a ledger entry names. Account entries hold +// only lumens and are not read. +func (f *assetFilter) matchEntry(entry *xdr.LedgerEntry) bool { + switch data := &entry.Data; data.Type { + case xdr.LedgerEntryTypeTrustline: + return f.matchTrustLineAsset(&data.TrustLine.Asset) + case xdr.LedgerEntryTypeOffer: + return f.matchAsset(&data.Offer.Selling) || f.matchAsset(&data.Offer.Buying) + case xdr.LedgerEntryTypeClaimableBalance: + return f.matchAsset(&data.ClaimableBalance.Asset) + case xdr.LedgerEntryTypeLiquidityPool: + pool := data.LiquidityPool.Body.ConstantProduct + return pool != nil && (f.matchAsset(&pool.Params.AssetA) || f.matchAsset(&pool.Params.AssetB)) + case xdr.LedgerEntryTypeContractData: + // Every entry stored under the Stellar Asset Contract of a whitelisted + // asset matches: a holder balance, the contract instance written on + // deployment, an allowance. The contract id is a hash of the asset and + // the network passphrase, so no other contract can store entries under + // it. For native lumens this covers holders with a contract address. + // A holder with an account address keeps lumens in its account entry, + // which is not read; matchHostFunction covers that transfer. + return f.matchContractAddress(&data.ContractData.Contract) + } + return false +} + +func (f *assetFilter) matchContractAddress(address *xdr.ScAddress) bool { + return address.ContractId != nil && f.contractIDs.Contains(*address.ContractId) } -func listToSet(list []string) set.Set[string] { - set := set.NewSet[string](len(list)) - for i := 0; i < len(list); i++ { - set.Add(list[i]) +// matchTrustLineAsset skips pool share trustlines. They carry a pool id, not +// an asset. A deposit or a withdrawal also changes the pool entry, which +// names the assets. +func (f *assetFilter) matchTrustLineAsset(asset *xdr.TrustLineAsset) bool { + if asset.Type == xdr.AssetTypeAssetTypePoolShare { + return false } - return set + _, ok := f.whitelistedAssets[alphaNumKey(asset.Type, asset.AlphaNum4, asset.AlphaNum12)] + return ok +} + +func (f *assetFilter) matchChangeTrustAsset(asset *xdr.ChangeTrustAsset) bool { + if asset.Type == xdr.AssetTypeAssetTypePoolShare { + pool := asset.LiquidityPool.ConstantProduct + return pool != nil && (f.matchAsset(&pool.AssetA) || f.matchAsset(&pool.AssetB)) + } + _, ok := f.whitelistedAssets[alphaNumKey(asset.Type, asset.AlphaNum4, asset.AlphaNum12)] + return ok +} + +func (f *assetFilter) matchAnyAsset(assets []xdr.Asset) bool { + for i := range assets { + if f.matchAsset(&assets[i]) { + return true + } + } + return false +} + +func (f *assetFilter) matchAsset(asset *xdr.Asset) bool { + _, ok := f.whitelistedAssets[alphaNumKey(asset.Type, asset.AlphaNum4, asset.AlphaNum12)] + return ok } -func (f assetFilter) isEnabled() bool { +func (f *assetFilter) isEnabled() bool { // filtering is disabled if the whitelist is empty for now as that is the only filter rule - return len(f.canonicalAssetsLookup) >= 1 && f.enabled + return f.whitelistLen >= 1 && f.enabled } diff --git a/internal/ingest/filters/asset_bench_test.go b/internal/ingest/filters/asset_bench_test.go new file mode 100644 index 000000000..ace6105b0 --- /dev/null +++ b/internal/ingest/filters/asset_bench_test.go @@ -0,0 +1,56 @@ +package filters + +import ( + "context" + "testing" + + "github.com/stellar/go-stellar-sdk/xdr" +) + +// The common case on a filtered node is a dropped transaction, so the benchmark +// measures a credit/credit offer that crosses three offers and matches nothing. +func BenchmarkAssetFilterDropsCrossedOffer(b *testing.B) { + issuer := testIssuer + selling := xdr.MustNewCreditAsset("EURC", issuer) + buying := xdr.MustNewCreditAsset("GBPC", issuer) + filter := newTestAssetFilter(b, xdr.MustNewCreditAsset("USDC", issuer)) + + offer := func(id xdr.Int64, amount xdr.Int64) xdr.LedgerEntry { + return xdr.LedgerEntry{Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeOffer, + Offer: &xdr.OfferEntry{ + SellerId: xdr.MustAddress(testHolder), + OfferId: id, + Selling: buying, + Buying: selling, + Amount: amount, + Price: xdr.Price{N: 1, D: 1}, + }, + }} + } + var changes xdr.LedgerEntryChanges + for id := xdr.Int64(1); id <= 3; id++ { + changes = append(changes, stateAndUpdated(offer(id, 100), offer(id, 50))...) + changes = append(changes, stateAndUpdated(holderTrustLineEntry(selling, 100), holderTrustLineEntry(selling, 150))...) + changes = append(changes, stateAndUpdated(holderTrustLineEntry(buying, 100), holderTrustLineEntry(buying, 50))...) + } + tx := successfulTxWithMetaV3(testHolder, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypeManageSellOffer, + ManageSellOfferOp: &xdr.ManageSellOfferOp{ + Selling: selling, + Buying: buying, + Amount: 150, + Price: xdr.Price{N: 1, D: 1}, + }, + }}}, changes) + ctx := context.Background() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, include, err := filter.FilterTransaction(ctx, tx) + if err != nil || include { + b.Fatal("expected the transaction to be dropped") + } + } +} diff --git a/internal/ingest/filters/asset_test.go b/internal/ingest/filters/asset_test.go index 44b1e8773..53b0ea7ac 100644 --- a/internal/ingest/filters/asset_test.go +++ b/internal/ingest/filters/asset_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stellar/go-stellar-sdk/ingest" + "github.com/stellar/go-stellar-sdk/ingest/sac" "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-horizon/internal/db2/history" ) @@ -21,7 +22,7 @@ func TestAssetFilterAllowsOnMatch(t *testing.T) { Enabled: true, LastModified: 1, } - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) err := filter.RefreshAssetFilter(filterConfig) tt.NoError(err) @@ -45,7 +46,7 @@ func TestAssetFilterAllowsWhenEmptyWhitelist(t *testing.T) { Enabled: true, LastModified: 1, } - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) err := filter.RefreshAssetFilter(filterConfig) tt.NoError(err) @@ -69,7 +70,7 @@ func TestAssetFilterAllowsWhenDisabled(t *testing.T) { Enabled: false, LastModified: 1, } - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) err := filter.RefreshAssetFilter(filterConfig) tt.NoError(err) @@ -90,7 +91,7 @@ func TestAssetFilterDoesNotAllowV1WhenNoMatch(t *testing.T) { LastModified: 1, } - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) err := filter.RefreshAssetFilter(filterConfig) tt.NoError(err) @@ -209,7 +210,7 @@ func TestAssetFilterMatchesFeeBumpInnerOperations(t *testing.T) { ctx := context.Background() issuer := "GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL" - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) tt.NoError(filter.RefreshAssetFilter(&history.AssetFilterConfig{ Whitelist: []string{"USDC:" + issuer}, Enabled: true, @@ -227,7 +228,7 @@ func TestAssetFilterDoesNotAllowFeeBumpWhenNoMatch(t *testing.T) { ctx := context.Background() issuer := "GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL" - filter := NewAssetFilter() + filter := NewAssetFilter(testNetworkPassphrase) tt.NoError(filter.RefreshAssetFilter(&history.AssetFilterConfig{ Whitelist: []string{"USDX:" + issuer}, Enabled: true, @@ -239,3 +240,711 @@ func TestAssetFilterDoesNotAllowFeeBumpWhenNoMatch(t *testing.T) { tt.True(isEnabled) tt.False(result) } + +func TestAssetFilterMatchesPathPaymentIntermediateAsset(t *testing.T) { + tt := assert.New(t) + whitelisted := xdr.MustNewCreditAsset("USDC", testIssuer) + other1 := xdr.MustNewCreditAsset("EURC", testIssuer) + other2 := xdr.MustNewCreditAsset("GBPC", testIssuer) + filter := newTestAssetFilter(t, whitelisted) + + pathPayments := func(path []xdr.Asset) map[string]xdr.Operation { + return map[string]xdr.Operation{ + "strict send": {Body: xdr.OperationBody{ + Type: xdr.OperationTypePathPaymentStrictSend, + PathPaymentStrictSendOp: &xdr.PathPaymentStrictSendOp{ + SendAsset: other1, + SendAmount: 100, + Destination: xdr.MustMuxedAddress(testHolder), + DestAsset: other2, + DestMin: 1, + Path: path, + }, + }}, + "strict receive": {Body: xdr.OperationBody{ + Type: xdr.OperationTypePathPaymentStrictReceive, + PathPaymentStrictReceiveOp: &xdr.PathPaymentStrictReceiveOp{ + SendAsset: other1, + SendMax: 100, + Destination: xdr.MustMuxedAddress(testHolder), + DestAsset: other2, + DestAmount: 1, + Path: path, + }, + }}, + } + } + + for name, op := range pathPayments([]xdr.Asset{whitelisted}) { + isEnabled, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, []xdr.Operation{op}, nil)) + tt.NoError(err, name) + tt.True(isEnabled, name) + tt.True(include, name) + } + for name, op := range pathPayments([]xdr.Asset{xdr.MustNewNativeAsset()}) { + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, []xdr.Operation{op}, nil)) + tt.NoError(err, name) + tt.False(include, name) + } +} + +func newTestAssetFilter(t testing.TB, whitelist ...xdr.Asset) AssetFilter { + canonical := make([]string, 0, len(whitelist)) + for _, asset := range whitelist { + canonical = append(canonical, asset.StringCanonical()) + } + filter := NewAssetFilter(testNetworkPassphrase) + require.NoError(t, filter.RefreshAssetFilter(&history.AssetFilterConfig{ + Whitelist: canonical, + Enabled: true, + LastModified: 1, + })) + return filter +} + +func successfulTxWithMetaV3(source string, ops []xdr.Operation, changes xdr.LedgerEntryChanges) ingest.LedgerTransaction { + return ingest.LedgerTransaction{ + Result: xdr.TransactionResultPair{ + Result: xdr.TransactionResult{ + Result: xdr.TransactionResultResult{Code: xdr.TransactionResultCodeTxSuccess}, + }, + }, + Envelope: xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress(source), + Operations: ops, + }, + }, + }, + UnsafeMeta: xdr.TransactionMeta{ + V: 3, + V3: &xdr.TransactionMetaV3{ + Operations: []xdr.OperationMeta{{Changes: changes}}, + }, + }, + } +} + +func stateAndRemoved(entry xdr.LedgerEntry) xdr.LedgerEntryChanges { + key, err := entry.LedgerKey() + if err != nil { + panic(err) + } + return xdr.LedgerEntryChanges{ + {Type: xdr.LedgerEntryChangeTypeLedgerEntryState, State: &entry}, + {Type: xdr.LedgerEntryChangeTypeLedgerEntryRemoved, Removed: &key}, + } +} + +func stateAndUpdated(before, after xdr.LedgerEntry) xdr.LedgerEntryChanges { + return xdr.LedgerEntryChanges{ + {Type: xdr.LedgerEntryChangeTypeLedgerEntryState, State: &before}, + {Type: xdr.LedgerEntryChangeTypeLedgerEntryUpdated, Updated: &after}, + } +} + +func created(entry xdr.LedgerEntry) xdr.LedgerEntryChanges { + return xdr.LedgerEntryChanges{ + {Type: xdr.LedgerEntryChangeTypeLedgerEntryCreated, Created: &entry}, + } +} + +func TestAssetFilterMatchesSetTrustLineFlags(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + setFlags := func(asset xdr.Asset) ingest.LedgerTransaction { + return successfulTxWithMetaV3(issuer, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypeSetTrustLineFlags, + SetTrustLineFlagsOp: &xdr.SetTrustLineFlagsOp{ + Trustor: xdr.MustAddress(issuer), + Asset: asset, + SetFlags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), + }, + }}}, nil) + } + + isEnabled, include, err := filter.FilterTransaction(context.Background(), setFlags(whitelisted)) + tt.NoError(err) + tt.True(isEnabled) + tt.True(include) + + _, include, err = filter.FilterTransaction(context.Background(), setFlags(xdr.MustNewCreditAsset("EURC", issuer))) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterMatchesAllowTrustUsingIssuerAsSource(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + other := testHolder + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + code, codeErr := xdr.NewAssetCodeFromString("USDC") + tt.NoError(codeErr) + allowTrust := func(opSource *xdr.MuxedAccount) xdr.Operation { + return xdr.Operation{ + SourceAccount: opSource, + Body: xdr.OperationBody{ + Type: xdr.OperationTypeAllowTrust, + AllowTrustOp: &xdr.AllowTrustOp{ + Trustor: xdr.MustAddress(other), + Asset: code, + Authorize: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), + }, + }, + } + } + issuerMuxed := xdr.MustMuxedAddress(issuer) + + for name, tx := range map[string]ingest.LedgerTransaction{ + "tx source is issuer": successfulTxWithMetaV3(issuer, []xdr.Operation{allowTrust(nil)}, nil), + "op source is issuer": successfulTxWithMetaV3(other, []xdr.Operation{allowTrust(&issuerMuxed)}, nil), + } { + isEnabled, include, err := filter.FilterTransaction(context.Background(), tx) + tt.NoError(err, name) + tt.True(isEnabled, name) + tt.True(include, name) + } + + feeBump := wrapInFeeBump(successfulTxWithMetaV3(issuer, []xdr.Operation{allowTrust(nil)}, nil)) + feeBump.Envelope.FeeBump.Tx.FeeSource = xdr.MustMuxedAddress(other) + _, include, err := filter.FilterTransaction(context.Background(), feeBump) + tt.NoError(err) + tt.True(include, "fee-bump: the issuer is the inner transaction source, not the fee source") + + feeSourceIsIssuer := wrapInFeeBump(successfulTxWithMetaV3(other, []xdr.Operation{allowTrust(nil)}, nil)) + feeSourceIsIssuer.Envelope.FeeBump.Tx.FeeSource = issuerMuxed + _, include, err = filter.FilterTransaction(context.Background(), feeSourceIsIssuer) + tt.NoError(err) + tt.False(include, "fee-bump: the fee source is not the issuer of an AllowTrust") + + notIssuer := successfulTxWithMetaV3(other, []xdr.Operation{allowTrust(nil)}, nil) + _, include, err = filter.FilterTransaction(context.Background(), notIssuer) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterMatchesRevokeSponsorshipOfTrustline(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + other := testHolder + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + revoke := func(asset xdr.Asset) ingest.LedgerTransaction { + key := xdr.LedgerKey{ + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: &xdr.LedgerKeyTrustLine{ + AccountId: xdr.MustAddress(other), + Asset: asset.ToTrustLineAsset(), + }, + } + return successfulTxWithMetaV3(issuer, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypeRevokeSponsorship, + RevokeSponsorshipOp: &xdr.RevokeSponsorshipOp{ + Type: xdr.RevokeSponsorshipTypeRevokeSponsorshipLedgerEntry, + LedgerKey: &key, + }, + }}}, nil) + } + + isEnabled, include, err := filter.FilterTransaction(context.Background(), revoke(whitelisted)) + tt.NoError(err) + tt.True(isEnabled) + tt.True(include) + + _, include, err = filter.FilterTransaction(context.Background(), revoke(xdr.MustNewCreditAsset("EURC", issuer))) + tt.NoError(err) + tt.False(include) +} + +func claimableBalanceEntry(asset xdr.Asset) xdr.LedgerEntry { + return xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &xdr.ClaimableBalanceEntry{ + BalanceId: xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1}, + }, + Asset: asset, + Amount: 100, + }, + }, + } +} + +func TestAssetFilterMatchesClaimableBalanceClaimAndClawback(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + other := xdr.MustNewCreditAsset("EURC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1}, + } + for name, op := range map[string]xdr.Operation{ + "claim": {Body: xdr.OperationBody{ + Type: xdr.OperationTypeClaimClaimableBalance, + ClaimClaimableBalanceOp: &xdr.ClaimClaimableBalanceOp{BalanceId: balanceID}, + }}, + "clawback": {Body: xdr.OperationBody{ + Type: xdr.OperationTypeClawbackClaimableBalance, + ClawbackClaimableBalanceOp: &xdr.ClawbackClaimableBalanceOp{BalanceId: balanceID}, + }}, + } { + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(issuer, []xdr.Operation{op}, stateAndRemoved(claimableBalanceEntry(whitelisted)))) + tt.NoError(err, name) + tt.True(include, name) + + _, include, err = filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(issuer, []xdr.Operation{op}, stateAndRemoved(claimableBalanceEntry(other)))) + tt.NoError(err, name) + tt.False(include, name) + } +} + +func liquidityPoolEntry(assetA, assetB xdr.Asset, reserveA xdr.Int64) xdr.LedgerEntry { + return xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeLiquidityPool, + LiquidityPool: &xdr.LiquidityPoolEntry{ + LiquidityPoolId: xdr.PoolId{2}, + Body: xdr.LiquidityPoolEntryBody{ + Type: xdr.LiquidityPoolTypeLiquidityPoolConstantProduct, + ConstantProduct: &xdr.LiquidityPoolEntryConstantProduct{ + Params: xdr.LiquidityPoolConstantProductParameters{ + AssetA: assetA, + AssetB: assetB, + Fee: 30, + }, + ReserveA: reserveA, + ReserveB: reserveA, + TotalPoolShares: reserveA, + }, + }, + }, + }, + } +} + +func TestAssetFilterMatchesLiquidityPoolDepositAndWithdraw(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + other := xdr.MustNewCreditAsset("EURC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + poolChanges := stateAndUpdated(liquidityPoolEntry(other, whitelisted, 100), liquidityPoolEntry(other, whitelisted, 200)) + native := xdr.MustNewNativeAsset() + unrelatedPoolChanges := stateAndUpdated(liquidityPoolEntry(other, native, 100), liquidityPoolEntry(other, native, 200)) + + for name, op := range map[string]xdr.Operation{ + "deposit": {Body: xdr.OperationBody{ + Type: xdr.OperationTypeLiquidityPoolDeposit, + LiquidityPoolDepositOp: &xdr.LiquidityPoolDepositOp{ + LiquidityPoolId: xdr.PoolId{2}, + MaxAmountA: 100, + MaxAmountB: 100, + MinPrice: xdr.Price{N: 1, D: 1}, + MaxPrice: xdr.Price{N: 1, D: 1}, + }, + }}, + "withdraw": {Body: xdr.OperationBody{ + Type: xdr.OperationTypeLiquidityPoolWithdraw, + LiquidityPoolWithdrawOp: &xdr.LiquidityPoolWithdrawOp{ + LiquidityPoolId: xdr.PoolId{2}, + Amount: 100, + MinAmountA: 1, + MinAmountB: 1, + }, + }}, + } { + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(issuer, []xdr.Operation{op}, poolChanges)) + tt.NoError(err, name) + tt.True(include, name) + + _, include, err = filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(issuer, []xdr.Operation{op}, unrelatedPoolChanges)) + tt.NoError(err, name) + tt.False(include, name) + } +} + +func holderTrustLineEntry(asset xdr.Asset, balance xdr.Int64) xdr.LedgerEntry { + return xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: &xdr.TrustLineEntry{ + AccountId: xdr.MustAddress(testHolder), + Asset: asset.ToTrustLineAsset(), + Balance: balance, + Limit: 1000, + }, + }, + } +} + +func TestAssetFilterMatchesTrustlineBalanceChange(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + holder := testHolder + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + other := xdr.MustNewCreditAsset("EURC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(holder, nil, + stateAndUpdated(holderTrustLineEntry(whitelisted, 100), holderTrustLineEntry(whitelisted, 200)))) + tt.NoError(err) + tt.True(include) + + _, include, err = filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(holder, nil, + stateAndUpdated(holderTrustLineEntry(other, 100), holderTrustLineEntry(other, 200)))) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterMatchesFailedTransactionOnOperationBody(t *testing.T) { + tt := assert.New(t) + issuer := testIssuer + whitelisted := xdr.MustNewCreditAsset("USDC", issuer) + filter := newTestAssetFilter(t, whitelisted) + + tx := successfulTxWithMetaV3(issuer, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypePayment, + PaymentOp: &xdr.PaymentOp{ + Destination: xdr.MustMuxedAddress(issuer), + Asset: whitelisted, + Amount: 100, + }, + }}}, nil) + tx.Result.Result.Result.Code = xdr.TransactionResultCodeTxFailed + tx.UnsafeMeta.V3.Operations = nil + + _, include, err := filter.FilterTransaction(context.Background(), tx) + tt.NoError(err) + tt.True(include) +} + +func sacContractID(t testing.TB, asset xdr.Asset) xdr.ContractId { + id, err := asset.ContractID(testNetworkPassphrase) + require.NoError(t, err) + return xdr.ContractId(id) +} + +func sacBalanceEntry(contractID xdr.ContractId) xdr.LedgerEntry { + holder := [32]byte{7} + return xdr.LedgerEntry{Data: sac.BalanceToContractData(contractID, holder, 100)} +} + +func TestAssetFilterMatchesSACContractBalanceChange(t *testing.T) { + tt := assert.New(t) + native := xdr.MustNewNativeAsset() + usdc := xdr.MustNewCreditAsset("USDC", testIssuer) + eurc := xdr.MustNewCreditAsset("EURC", testIssuer) + + for _, tc := range []struct { + name string + whitelist xdr.Asset + balanceOf xdr.Asset + include bool + }{ + {"usdc balance, usdc whitelisted", usdc, usdc, true}, + {"eurc balance, usdc whitelisted", usdc, eurc, false}, + {"native balance, native whitelisted", native, native, true}, + {"usdc balance, native whitelisted", native, usdc, false}, + } { + filter := newTestAssetFilter(t, tc.whitelist) + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, nil, created(sacBalanceEntry(sacContractID(t, tc.balanceOf))))) + tt.NoError(err, tc.name) + tt.Equal(tc.include, include, tc.name) + } +} + +func invokeContract(contractID xdr.ContractId) xdr.Operation { + return xdr.Operation{Body: xdr.OperationBody{ + Type: xdr.OperationTypeInvokeHostFunction, + InvokeHostFunctionOp: &xdr.InvokeHostFunctionOp{ + HostFunction: xdr.HostFunction{ + Type: xdr.HostFunctionTypeHostFunctionTypeInvokeContract, + InvokeContract: &xdr.InvokeContractArgs{ + ContractAddress: xdr.ScAddress{ + Type: xdr.ScAddressTypeScAddressTypeContract, + ContractId: &contractID, + }, + FunctionName: "transfer", + }, + }, + }, + }} +} + +func accountEntry(address string, balance xdr.Int64) xdr.LedgerEntry { + return xdr.LedgerEntry{Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeAccount, + Account: &xdr.AccountEntry{ + AccountId: xdr.MustAddress(address), + Balance: balance, + }, + }} +} + +func TestAssetFilterMatchesSACInvocationOnOperationBody(t *testing.T) { + tt := assert.New(t) + native := xdr.MustNewNativeAsset() + usdc := xdr.MustNewCreditAsset("USDC", testIssuer) + eurc := xdr.MustNewCreditAsset("EURC", testIssuer) + + // A failed call writes no operation meta. Only the operation body can match. + failedInvoke := func(contractID xdr.ContractId) ingest.LedgerTransaction { + tx := successfulTxWithMetaV3(testHolder, []xdr.Operation{invokeContract(contractID)}, nil) + tx.Result.Result.Result.Code = xdr.TransactionResultCodeTxFailed + tx.UnsafeMeta.V3.Operations = nil + return tx + } + filter := newTestAssetFilter(t, usdc) + _, include, err := filter.FilterTransaction(context.Background(), failedInvoke(sacContractID(t, usdc))) + tt.NoError(err) + tt.True(include, "failed call to the whitelisted SAC") + _, include, err = filter.FilterTransaction(context.Background(), failedInvoke(sacContractID(t, eurc))) + tt.NoError(err) + tt.False(include, "failed call to another SAC") + + // A native transfer between two account addresses changes only account + // entries, which the filter does not read. The operation body matches. + nativeTransfer := successfulTxWithMetaV3(testHolder, []xdr.Operation{invokeContract(sacContractID(t, native))}, + append(stateAndUpdated(accountEntry(testHolder, 200), accountEntry(testHolder, 100)), + stateAndUpdated(accountEntry(testIssuer, 100), accountEntry(testIssuer, 200))...)) + _, include, err = newTestAssetFilter(t, native).FilterTransaction(context.Background(), nativeTransfer) + tt.NoError(err) + tt.True(include, "native SAC transfer between two G accounts") + _, include, err = filter.FilterTransaction(context.Background(), nativeTransfer) + tt.NoError(err) + tt.False(include, "native SAC transfer when only USDC is whitelisted") +} + +func TestAssetFilterMatchesSACDeploymentOnOperationBody(t *testing.T) { + tt := assert.New(t) + usdc := xdr.MustNewCreditAsset("USDC", testIssuer) + eurc := xdr.MustNewCreditAsset("EURC", testIssuer) + filter := newTestAssetFilter(t, usdc) + + deploy := func(asset xdr.Asset) ingest.LedgerTransaction { + fn := xdr.HostFunction{ + Type: xdr.HostFunctionTypeHostFunctionTypeCreateContract, + CreateContract: &xdr.CreateContractArgs{ + ContractIdPreimage: xdr.ContractIdPreimage{ + Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAsset, + FromAsset: &asset, + }, + Executable: xdr.ContractExecutable{Type: xdr.ContractExecutableTypeContractExecutableStellarAsset}, + }, + } + tx := successfulTxWithMetaV3(testHolder, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypeInvokeHostFunction, + InvokeHostFunctionOp: &xdr.InvokeHostFunctionOp{HostFunction: fn}, + }}}, nil) + tx.Result.Result.Result.Code = xdr.TransactionResultCodeTxFailed + tx.UnsafeMeta.V3.Operations = nil + return tx + } + + _, include, err := filter.FilterTransaction(context.Background(), deploy(usdc)) + tt.NoError(err) + tt.True(include) + _, include, err = filter.FilterTransaction(context.Background(), deploy(eurc)) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterMatchesSorobanFootprint(t *testing.T) { + tt := assert.New(t) + usdc := xdr.MustNewCreditAsset("USDC", testIssuer) + eurc := xdr.MustNewCreditAsset("EURC", testIssuer) + filter := newTestAssetFilter(t, usdc) + + // ExtendFootprintTTL names the entries only in the footprint. The meta + // holds TTL entries, which carry a key hash and no contract id. + extendTTL := func(contractID xdr.ContractId) ingest.LedgerTransaction { + entry := sacBalanceEntry(contractID) + key, err := entry.LedgerKey() + require.NoError(t, err) + tx := successfulTxWithMetaV3(testHolder, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypeExtendFootprintTtl, + ExtendFootprintTtlOp: &xdr.ExtendFootprintTtlOp{ExtendTo: 1000}, + }}}, nil) + tx.Envelope.V1.Tx.Ext = xdr.TransactionExt{ + V: 1, + SorobanData: &xdr.SorobanTransactionData{ + Resources: xdr.SorobanResources{ + Footprint: xdr.LedgerFootprint{ReadOnly: []xdr.LedgerKey{key}}, + }, + }, + } + return tx + } + + _, include, err := filter.FilterTransaction(context.Background(), extendTTL(sacContractID(t, usdc))) + tt.NoError(err) + tt.True(include) + _, include, err = filter.FilterTransaction(context.Background(), wrapInFeeBump(extendTTL(sacContractID(t, usdc)))) + tt.NoError(err) + tt.True(include, "fee-bump") + _, include, err = filter.FilterTransaction(context.Background(), extendTTL(sacContractID(t, eurc))) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterRejectsUnknownMetaVersion(t *testing.T) { + tt := assert.New(t) + filter := newTestAssetFilter(t, xdr.MustNewCreditAsset("USDC", testIssuer)) + + tx := successfulTxWithMetaV3(testHolder, nil, nil) + tx.UnsafeMeta = xdr.TransactionMeta{V: 5} + _, include, err := filter.FilterTransaction(context.Background(), tx) + tt.EqualError(err, "unsupported transaction meta version 5") + tt.False(include) +} + +func TestAssetFilterMatchesSACContractDeployment(t *testing.T) { + tt := assert.New(t) + whitelisted := xdr.MustNewCreditAsset("USDC", testIssuer) + filter := newTestAssetFilter(t, whitelisted) + + metadataCreated := func(code string) xdr.LedgerEntryChanges { + asset := xdr.MustNewCreditAsset(code, testIssuer) + contractID, err := asset.ContractID(testNetworkPassphrase) + require.NoError(t, err) + data, err := sac.AssetToContractData(false, code, testIssuer, contractID) + require.NoError(t, err) + return created(xdr.LedgerEntry{Data: data}) + } + + _, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, nil, metadataCreated("USDC"))) + tt.NoError(err) + tt.True(include) + + _, include, err = filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, nil, metadataCreated("EURC"))) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterMatchesWhitelistEntryStoredInNonCanonicalForm(t *testing.T) { + tt := assert.New(t) + filter := NewAssetFilter(testNetworkPassphrase) + tt.NoError(filter.RefreshAssetFilter(&history.AssetFilterConfig{ + Whitelist: []string{"NATIVE"}, + Enabled: true, + LastModified: 1, + })) + + payment := func(asset xdr.Asset) ingest.LedgerTransaction { + return successfulTxWithMetaV3(testIssuer, []xdr.Operation{{Body: xdr.OperationBody{ + Type: xdr.OperationTypePayment, + PaymentOp: &xdr.PaymentOp{ + Destination: xdr.MustMuxedAddress(testHolder), + Asset: asset, + Amount: 100, + }, + }}}, nil) + } + + _, include, err := filter.FilterTransaction(context.Background(), payment(xdr.MustNewNativeAsset())) + tt.NoError(err) + tt.True(include) + + _, include, err = filter.FilterTransaction(context.Background(), payment(xdr.MustNewCreditAsset("USDC", testIssuer))) + tt.NoError(err) + tt.False(include) +} + +func TestAssetFilterKeepsMalformedWhitelistEntryEnabled(t *testing.T) { + tt := assert.New(t) + filter := NewAssetFilter(testNetworkPassphrase) + tt.NoError(filter.RefreshAssetFilter(&history.AssetFilterConfig{ + Whitelist: []string{"not-an-asset"}, + Enabled: true, + LastModified: 1, + })) + + isEnabled, include, err := filter.FilterTransaction(context.Background(), + successfulTxWithMetaV3(testIssuer, nil, nil)) + tt.NoError(err) + tt.True(isEnabled) + tt.False(include) +} + +func TestAssetFilterMatchesChangesInEveryMetaVersion(t *testing.T) { + tt := assert.New(t) + whitelisted := xdr.MustNewCreditAsset("USDC", testIssuer) + other := xdr.MustNewCreditAsset("EURC", testIssuer) + filter := newTestAssetFilter(t, whitelisted) + + metas := func(asset xdr.Asset) map[string]xdr.TransactionMeta { + changes := stateAndUpdated(holderTrustLineEntry(asset, 100), holderTrustLineEntry(asset, 200)) + v0Operations := []xdr.OperationMeta{{Changes: changes}} + return map[string]xdr.TransactionMeta{ + "v0": {V: 0, Operations: &v0Operations}, + "v1": {V: 1, V1: &xdr.TransactionMetaV1{Operations: []xdr.OperationMeta{{Changes: changes}}}}, + "v2 before": {V: 2, V2: &xdr.TransactionMetaV2{TxChangesBefore: changes}}, + "v3 after": {V: 3, V3: &xdr.TransactionMetaV3{TxChangesAfter: changes}}, + "v4": {V: 4, V4: &xdr.TransactionMetaV4{Operations: []xdr.OperationMetaV2{{Changes: changes}}}}, + } + } + + for name, meta := range metas(whitelisted) { + tx := successfulTxWithMetaV3(testHolder, nil, nil) + tx.UnsafeMeta = meta + _, include, err := filter.FilterTransaction(context.Background(), tx) + tt.NoError(err, name) + tt.True(include, name) + } + for name, meta := range metas(other) { + tx := successfulTxWithMetaV3(testHolder, nil, nil) + tx.UnsafeMeta = meta + _, include, err := filter.FilterTransaction(context.Background(), tx) + tt.NoError(err, name) + tt.False(include, name) + } +} + +func TestAssetKeyAgreesWithCanonicalString(t *testing.T) { + tt := assert.New(t) + var usdc12Code [12]byte + copy(usdc12Code[:], "USDC") + issuer := xdr.MustAddress(testIssuer) + assets := []xdr.Asset{ + xdr.MustNewNativeAsset(), + xdr.MustNewCreditAsset("USDC", testIssuer), + xdr.MustNewCreditAsset("USDC", testHolder), + xdr.MustNewCreditAsset("USD", testIssuer), + xdr.MustNewCreditAsset("USDCOIN", testIssuer), + { + Type: xdr.AssetTypeAssetTypeCreditAlphanum12, + AlphaNum12: &xdr.AlphaNum12{AssetCode: usdc12Code, Issuer: issuer}, + }, + } + for _, a := range assets { + for _, b := range assets { + tt.Equal(a.StringCanonical() == b.StringCanonical(), newAssetKey(a) == newAssetKey(b), + "%s vs %s", a.StringCanonical(), b.StringCanonical()) + } + } +} diff --git a/internal/ingest/filters/main.go b/internal/ingest/filters/main.go index 24c552542..28fca8b0c 100644 --- a/internal/ingest/filters/main.go +++ b/internal/ingest/filters/main.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/stellar/go-stellar-sdk/support/collections/set" "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/stellar-horizon/internal/db2/history" "github.com/stellar/stellar-horizon/internal/ingest/processors" @@ -46,9 +47,9 @@ type Filters interface { GetFilters(filterQ history.QFilter, ctx context.Context) []processors.LedgerTransactionFilterer } -func NewFilters() Filters { +func NewFilters(networkPassphrase string) Filters { return &filtersCache{ - assetFilter: NewAssetFilter(), + assetFilter: NewAssetFilter(networkPassphrase), accountFilter: NewAccountFilter(), } } @@ -87,3 +88,11 @@ func (f *filtersCache) GetFilters(filterQ history.QFilter, ctx context.Context) func (f *filtersCache) convertCacheToList() []processors.LedgerTransactionFilterer { return []processors.LedgerTransactionFilterer{f.assetFilter, f.accountFilter} } + +func listToSet(list []string) set.Set[string] { + set := set.NewSet[string](len(list)) + for i := 0; i < len(list); i++ { + set.Add(list[i]) + } + return set +} diff --git a/internal/ingest/filters/main_test.go b/internal/ingest/filters/main_test.go index 295294889..c79cfb7bb 100644 --- a/internal/ingest/filters/main_test.go +++ b/internal/ingest/filters/main_test.go @@ -13,7 +13,7 @@ func TestItGetsFilters(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} - filtersService := NewFilters() + filtersService := NewFilters(testNetworkPassphrase) ingestFilters := filtersService.GetFilters(q, tt.Ctx) diff --git a/internal/ingest/filters/testutils_test.go b/internal/ingest/filters/testutils_test.go new file mode 100644 index 000000000..a9a505e29 --- /dev/null +++ b/internal/ingest/filters/testutils_test.go @@ -0,0 +1,7 @@ +package filters + +const ( + testNetworkPassphrase = "Test SDF Network ; September 2015" + testIssuer = "GD6WNNTW664WH7FXC5RUMUTF7P5QSURC2IT36VOQEEGFZ4UWUEQGECAL" + testHolder = "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A" +) diff --git a/internal/ingest/main.go b/internal/ingest/main.go index 24e9dfb45..e52ed08ff 100644 --- a/internal/ingest/main.go +++ b/internal/ingest/main.go @@ -354,7 +354,7 @@ func NewSystem(config Config) (System, error) { historyQ := &history.Q{config.HistorySession.Clone()} historyAdapter := newHistoryArchiveAdapter(archive, config.NetworkPassphrase) - filters := filters.NewFilters() + filters := filters.NewFilters(config.NetworkPassphrase) loadtestSnapshot := &loadTestSnapshot{HistoryQ: historyQ} maxLedgersPerFlush := config.MaxLedgerPerFlush diff --git a/internal/ingest/processor_runner_test.go b/internal/ingest/processor_runner_test.go index 2ac6616ad..bdee79700 100644 --- a/internal/ingest/processor_runner_test.go +++ b/internal/ingest/processor_runner_test.go @@ -273,22 +273,14 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { defer mock.AssertExpectationsForObjects(t, mockBatchInsertBuilder) - q.MockQAssetStats.On("InsertAssetContracts", ctx, []history.AssetContract(nil)). - Return(nil).Once() q.MockQAssetStats.On("UpdateAssetContractExpirations", ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - q.MockQAssetStats.On("DeleteAssetContractsExpiringAt", ctx, uint32(22)). + q.MockQAssetStats.On("DeleteAssetContractsExpiredBefore", ctx, uint32(23)). Return(int64(0), nil).Once() - q.MockQAssetStats.On("RemoveContractAssetBalances", ctx, []xdr.Hash(nil)). - Return(nil).Once() - q.MockQAssetStats.On("UpdateContractAssetBalanceAmounts", ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - q.MockQAssetStats.On("InsertContractAssetBalances", ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() q.MockQAssetStats.On("UpdateContractAssetBalanceExpirations", ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - q.MockQAssetStats.On("DeleteContractAssetBalancesExpiringAt", ctx, uint32(22)). + q.MockQAssetStats.On("DeleteContractAssetBalancesExpiredBefore", ctx, uint32(23)). Return([]history.ContractAssetBalance{}, nil).Once() runner := ProcessorRunner{ @@ -375,14 +367,10 @@ func TestProcessorRunnerRunTransactionsProcessorsOnLedgers(t *testing.T) { q.MockQAssetStats.On("RemoveContractAssetBalances", ctx, []xdr.Hash(nil)). Return(nil).Once() - q.MockQAssetStats.On("UpdateContractAssetBalanceAmounts", ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() q.MockQAssetStats.On("InsertContractAssetBalances", ctx, []history.ContractAssetBalance(nil)). Return(nil).Once() q.MockQAssetStats.On("UpdateContractAssetBalanceExpirations", ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - q.MockQAssetStats.On("DeleteContractAssetBalancesExpiringAt", ctx, uint32(22)). - Return([]history.ContractAssetBalance{}, nil).Once() runner := ProcessorRunner{ ctx: ctx, @@ -494,22 +482,14 @@ func TestProcessorRunnerRunAllProcessorsOnLedgerProtocolVersionNotSupportedButAl defer mock.AssertExpectationsForObjects(t, mockBatchInsertBuilder) - q.MockQAssetStats.On("InsertAssetContracts", ctx, []history.AssetContract(nil)). - Return(nil).Once() q.MockQAssetStats.On("UpdateAssetContractExpirations", ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - q.MockQAssetStats.On("DeleteAssetContractsExpiringAt", ctx, uint32(22)). + q.MockQAssetStats.On("DeleteAssetContractsExpiredBefore", ctx, uint32(23)). Return(int64(0), nil).Once() - q.MockQAssetStats.On("RemoveContractAssetBalances", ctx, []xdr.Hash(nil)). - Return(nil).Once() - q.MockQAssetStats.On("UpdateContractAssetBalanceAmounts", ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - q.MockQAssetStats.On("InsertContractAssetBalances", ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() q.MockQAssetStats.On("UpdateContractAssetBalanceExpirations", ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - q.MockQAssetStats.On("DeleteContractAssetBalancesExpiringAt", ctx, uint32(22)). + q.MockQAssetStats.On("DeleteContractAssetBalancesExpiredBefore", ctx, uint32(23)). Return([]history.ContractAssetBalance{}, nil).Once() runner := ProcessorRunner{ diff --git a/internal/ingest/processors/asset_stats_processor.go b/internal/ingest/processors/asset_stats_processor.go index ca0f4ed57..d3e828a50 100644 --- a/internal/ingest/processors/asset_stats_processor.go +++ b/internal/ingest/processors/asset_stats_processor.go @@ -6,7 +6,6 @@ import ( "math/big" "github.com/stellar/go-stellar-sdk/ingest" - "github.com/stellar/go-stellar-sdk/ingest/sac" "github.com/stellar/go-stellar-sdk/support/errors" "github.com/stellar/go-stellar-sdk/xdr" @@ -17,12 +16,8 @@ type AssetStatsProcessor struct { assetStatsQ history.QAssetStats currentLedger uint32 assetStatSet AssetStatSet - contractDataChanges []ingest.Change - removedExpirationEntries map[xdr.Hash]uint32 - createdExpirationEntries map[xdr.Hash]uint32 - updatedExpirationEntries map[xdr.Hash][2]uint32 + contractAssetStatSet *ContractAssetStatSet ingestFromHistoryArchive bool - networkPassphrase string } // NewAssetStatsProcessor constructs a new AssetStatsProcessor instance. @@ -36,12 +31,8 @@ func NewAssetStatsProcessor( currentLedger: currentLedger, assetStatsQ: assetStatsQ, ingestFromHistoryArchive: ingestFromHistoryArchive, - networkPassphrase: networkPassphrase, assetStatSet: NewAssetStatSet(), - contractDataChanges: []ingest.Change{}, - removedExpirationEntries: map[xdr.Hash]uint32{}, - createdExpirationEntries: map[xdr.Hash]uint32{}, - updatedExpirationEntries: map[xdr.Hash][2]uint32{}, + contractAssetStatSet: NewContractAssetStatSet(assetStatsQ, networkPassphrase, currentLedger), } return p } @@ -68,35 +59,9 @@ func (p *AssetStatsProcessor) ProcessChange(ctx context.Context, change ingest.C case xdr.LedgerEntryTypeTrustline: err = p.assetStatSet.AddTrustline(change) case xdr.LedgerEntryTypeContractData: - // Only ingest contract data entries which could be relevant to - // asset stats. - // - // We care about SAC activity exclusively, but balance provenance cannot - // be determined from the entry alone. Thus we propagate balance-looking - // changes and validate SACs when possible. - if change.Post == nil { // removal - if change.Pre == nil { // defensive - return nil - } - // Keys are immutable so we check that over the value on removal. - ledgerKey, keyErr := change.Pre.LedgerKey() - if keyErr != nil { - return errors.Wrap(keyErr, "could not extract ledger key") - } - - if !sac.ValidContractBalanceLedgerKey(ledgerKey) { - return nil - } - } else { // create/update - _, assetFound := sac.AssetFromContractData(*change.Post, p.networkPassphrase) - _, _, balanceFound := sac.ContractBalanceFromContractData(*change.Post, p.networkPassphrase) - if !assetFound && !balanceFound { // neither balance nor asset init - return nil - } - } - p.contractDataChanges = append(p.contractDataChanges, change) + err = p.contractAssetStatSet.AddContractData(change) case xdr.LedgerEntryTypeTtl: - err = p.addExpirationChange(change) + err = p.contractAssetStatSet.AddTTL(change) default: return errors.Errorf("Change type %v is unexpected", change.Type) } @@ -104,103 +69,7 @@ func (p *AssetStatsProcessor) ProcessChange(ctx context.Context, change ingest.C return err } -// AssetStatsProcessor requires that the ttl changes it ingests are already compacted -// because the TTL change semantics in CAP-63 (see -// https://github.com/stellar/stellar-protocol/blob/master/core/cap-0063.md#ttl-ledger-change-semantics ) -// are encapsulated in the ChangeCompactor -func (p *AssetStatsProcessor) checkTTLChangeIsCompacted(change ingest.Change) error { - var keyHash xdr.Hash - if change.Pre != nil { - keyHash = change.Pre.Data.MustTtl().KeyHash - } else { - keyHash = change.Post.Data.MustTtl().KeyHash - } - if _, ok := p.removedExpirationEntries[keyHash]; ok { - return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) - } - if _, ok := p.createdExpirationEntries[keyHash]; ok { - return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) - } - if _, ok := p.updatedExpirationEntries[keyHash]; ok { - return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) - } - return nil -} - -func (p *AssetStatsProcessor) addExpirationChange(change ingest.Change) error { - if err := p.checkTTLChangeIsCompacted(change); err != nil { - return err - } - - switch { - case change.Pre == nil && change.Post != nil: // created - post := change.Post.Data.MustTtl() - p.createdExpirationEntries[post.KeyHash] = uint32(post.LiveUntilLedgerSeq) - case change.Pre != nil && change.Post == nil: // removed - pre := change.Pre.Data.MustTtl() - p.removedExpirationEntries[pre.KeyHash] = uint32(pre.LiveUntilLedgerSeq) - case change.Pre != nil && change.Post != nil: // updated - pre := change.Pre.Data.MustTtl() - post := change.Post.Data.MustTtl() - // it's unclear if core could emit a ledger entry change where the - // expiration ledger remains the same - if pre.LiveUntilLedgerSeq == post.LiveUntilLedgerSeq { - return nil - } - - // if a TTL was created, restored, or updated at ledger N - // the LiveUntilLedgerSeq must be >= current ledger - if uint32(post.LiveUntilLedgerSeq) < p.currentLedger { - return errors.Errorf( - "post TTL %v is less than current ledger %v", - post.LiveUntilLedgerSeq, - p.currentLedger, - ) - } - - // The previous expiration ledger must always be greater than or equal to the current ledger - // because if the previous expiration ledger is less than the current ledger then it implies - // the ledger entry was archived. However, an archived ledger entry cannot be updated without - // first being restored. - // Alternatively, the TTL can be updated without being restored if it is a temporary ledger - // entry. However, in that case, we can ignore the ledger entry entirely because SAC - // ledger entries are always kept in persistent storage. - if uint32(pre.LiveUntilLedgerSeq) < p.currentLedger { - return nil - } - p.updatedExpirationEntries[pre.KeyHash] = [2]uint32{ - uint32(pre.LiveUntilLedgerSeq), - uint32(post.LiveUntilLedgerSeq), - } - default: - return errors.Errorf("unexpected change Pre: %v Post: %v", change.Pre, change.Post) - } - - return nil -} - func (p *AssetStatsProcessor) Commit(ctx context.Context) error { - contractAssetStatSet := NewContractAssetStatSet( - p.assetStatsQ, - p.networkPassphrase, - p.removedExpirationEntries, - p.createdExpirationEntries, - p.updatedExpirationEntries, - p.currentLedger, - ) - for _, change := range p.contractDataChanges { - if err := contractAssetStatSet.AddContractData(change); err != nil { - return errors.Wrap(err, "Error ingesting contract data") - } - } - - return p.updateDB(ctx, contractAssetStatSet) -} - -func (p *AssetStatsProcessor) updateDB( - ctx context.Context, - contractAssetStatSet *ContractAssetStatSet, -) error { if p.ingestFromHistoryArchive { // When ingesting from the history archives we can take advantage of the fact // that there are only created ledger entries. We don't need to execute any @@ -213,19 +82,20 @@ func (p *AssetStatsProcessor) updateDB( } } - if rows := contractAssetStatSet.GetContractStats(); len(rows) > 0 { - if err := p.assetStatsQ.InsertContractAssetStats(ctx, rows); err != nil { + balances, statRows := p.contractAssetStatSet.GetCreatedBalancesAndStats() + if len(statRows) > 0 { + if err := p.assetStatsQ.InsertContractAssetStats(ctx, statRows); err != nil { return errors.Wrap(err, "Error inserting asset contract stats") } } - if len(contractAssetStatSet.createdBalances) > 0 { - if err := p.assetStatsQ.InsertContractAssetBalances(ctx, contractAssetStatSet.createdBalances); err != nil { - return errors.Wrap(err, "Error inserting asset contract stats") + if len(balances) > 0 { + if err := p.assetStatsQ.InsertContractAssetBalances(ctx, balances); err != nil { + return errors.Wrap(err, "Error inserting contract asset balances") } } - rows, err := contractAssetStatSet.GetCreatedAssetContracts() + rows, err := p.contractAssetStatSet.GetCreatedAssetContracts() if err != nil { return errors.Wrap(err, "Error getting created asset contracts") } @@ -243,68 +113,57 @@ func (p *AssetStatsProcessor) updateDB( return err } - // reads the rows RemoveContractAssetBalances is about to delete - if err := contractAssetStatSet.ingestRemovedBalances(ctx); err != nil { - return err - } - - if err := p.assetStatsQ.RemoveContractAssetBalances(ctx, contractAssetStatSet.removedBalances); err != nil { - return errors.Wrap(err, "Error removing contract asset balances") - } - - if err := p.updateContractAssetBalanceAmounts(ctx, contractAssetStatSet.updatedBalances); err != nil { - return err - } - + // Apply the ttl changes first, so that rows whose expiration was extended in this + // ledger are not reaped below. if err := p.updateContractDataExpirations(ctx); err != nil { return err } - // Reap expired rows before inserting. An entry archived at ledger E can be restored - // in E+1, which stellar-core emits as LEDGER_ENTRY_RESTORED (Change.Pre == nil) and - // we ingest as a creation. The stale row is only reaped when currentLedger-1 == E, - // i.e. this same ledger, so inserting first would collide with it on the primary key - // and abort ingestion of the ledger (see processor_runner.go SuppressRemoveAfterRestoreChange). - if err := contractAssetStatSet.ingestExpiredBalances(ctx); err != nil { + // Reap the rows which are no longer active before reading the stored rows of the + // changed balances. An entry archived at ledger E can be restored in E+1, which + // stellar-core emits as LEDGER_ENTRY_RESTORED (Change.Pre == nil), and its expired + // row must be removed from the stats once, when it is reaped. + if err := p.contractAssetStatSet.ingestExpiredBalances(ctx); err != nil { return err } - if _, err := p.assetStatsQ.DeleteAssetContractsExpiringAt(ctx, p.currentLedger-1); err != nil { + if _, err := p.assetStatsQ.DeleteAssetContractsExpiredBefore(ctx, p.currentLedger); err != nil { return errors.Wrap(err, "Error deleting expired asset contracts") } - assetContractRows, err := contractAssetStatSet.GetCreatedAssetContracts() + upserts, removals, err := p.contractAssetStatSet.reconcileBalances(ctx) if err != nil { - return errors.Wrap(err, "Error getting created asset contracts") + return err } - if err = p.assetStatsQ.InsertAssetContracts(ctx, assetContractRows); err != nil { - return errors.Wrap(err, "Error inserting asset contracts") + if len(removals) > 0 { + if err = p.assetStatsQ.RemoveContractAssetBalances(ctx, removals); err != nil { + return errors.Wrap(err, "Error removing contract asset balances") + } } - - if err := p.assetStatsQ.InsertContractAssetBalances(ctx, contractAssetStatSet.createdBalances); err != nil { - return errors.Wrap(err, "Error inserting contract asset balances") + if len(upserts) > 0 { + if err = p.assetStatsQ.UpsertContractAssetBalances(ctx, upserts); err != nil { + return errors.Wrap(err, "Error upserting contract asset balances") + } } - return p.updateContractAssetStats(ctx, contractAssetStatSet.contractAssetStats) -} - -func (p *AssetStatsProcessor) updateContractAssetBalanceAmounts(ctx context.Context, updatedBalances map[xdr.Hash]*big.Int) error { - keys := make([]xdr.Hash, 0, len(updatedBalances)) - amounts := make([]string, 0, len(updatedBalances)) - for key, amount := range updatedBalances { - keys = append(keys, key) - amounts = append(amounts, amount.String()) + assetContractRows, err := p.contractAssetStatSet.GetCreatedAssetContracts() + if err != nil { + return errors.Wrap(err, "Error getting created asset contracts") } - if err := p.assetStatsQ.UpdateContractAssetBalanceAmounts(ctx, keys, amounts); err != nil { - return errors.Wrap(err, "Error updating contract asset balance amounts") + if len(assetContractRows) > 0 { + if err = p.assetStatsQ.InsertAssetContracts(ctx, assetContractRows); err != nil { + return errors.Wrap(err, "Error inserting asset contracts") + } } - return nil + + return p.updateContractAssetStats(ctx) } func (p *AssetStatsProcessor) updateContractDataExpirations(ctx context.Context) error { - keys := make([]xdr.Hash, 0, len(p.updatedExpirationEntries)) - expirationLedgers := make([]uint32, 0, len(p.updatedExpirationEntries)) - for key, update := range p.updatedExpirationEntries { + updatedExpirationEntries := p.contractAssetStatSet.updatedExpirationEntries + keys := make([]xdr.Hash, 0, len(updatedExpirationEntries)) + expirationLedgers := make([]uint32, 0, len(updatedExpirationEntries)) + for key, update := range updatedExpirationEntries { keys = append(keys, key) expirationLedgers = append(expirationLedgers, update[1]) } @@ -453,11 +312,8 @@ func (p *AssetStatsProcessor) addContractAssetStat(contractAssetStat assetContra return nil } -func (p *AssetStatsProcessor) updateContractAssetStats( - ctx context.Context, - contractAssetStats map[xdr.ContractId]assetContractStatValue, -) error { - for contractID, contractAssetStat := range contractAssetStats { +func (p *AssetStatsProcessor) updateContractAssetStats(ctx context.Context) error { + for contractID, contractAssetStat := range p.contractAssetStatSet.contractAssetStats { if err := p.updateAssetContractStats(ctx, contractID, contractAssetStat); err != nil { return err } diff --git a/internal/ingest/processors/asset_stats_processor_contract_data_test.go b/internal/ingest/processors/asset_stats_processor_contract_data_test.go index 308a45ace..f3cf0baec 100644 --- a/internal/ingest/processors/asset_stats_processor_contract_data_test.go +++ b/internal/ingest/processors/asset_stats_processor_contract_data_test.go @@ -16,22 +16,19 @@ import ( ) // runContractDataLedger ingests changes for a single ledger the way live -// ingestion does, with one AssetStatsProcessor per ledger, and reports the key -// hashes the processor asked the db to insert into and remove from -// contract_asset_balances, plus the contract asset stats it wrote. storedRows is -// what contract_asset_balances already holds for the keys the ledger removes. +// ingestion does, with one AssetStatsProcessor per ledger, and reports the rows +// the processor asked the db to upsert into contract_asset_balances, the key +// hashes it asked the db to remove, plus the contract asset stats it wrote. +// storedRows is what contract_asset_balances already holds for the keys the +// ledger changes. func runContractDataLedger( t *testing.T, ledger uint32, storedRows []history.ContractAssetBalance, changes ...ingest.Change, -) (inserted []xdr.Hash, removed []xdr.Hash, statInserts []history.ContractAssetStatRow) { +) (upserted []history.ContractAssetBalance, removed []xdr.Hash, statInserts []history.ContractAssetStatRow) { ctx := context.Background() q := &history.MockQAssetStats{} - q.On("InsertContractAssetBalances", ctx, mock.Anything).Run(func(args mock.Arguments) { - for _, row := range args.Get(1).([]history.ContractAssetBalance) { - var keyHash xdr.Hash - copy(keyHash[:], row.KeyHash) - inserted = append(inserted, keyHash) - } + q.On("UpsertContractAssetBalances", ctx, mock.Anything).Run(func(args mock.Arguments) { + upserted = append(upserted, args.Get(1).([]history.ContractAssetBalance)...) }).Return(nil).Maybe() q.On("RemoveContractAssetBalances", ctx, mock.Anything).Run(func(args mock.Arguments) { removed = append(removed, args.Get(1).([]xdr.Hash)...) @@ -40,12 +37,11 @@ func runContractDataLedger( q.On("GetContractAssetBalances", ctx, mock.Anything).Return(storedRows, nil).Maybe() q.On("InsertAssetContracts", ctx, mock.Anything).Return(nil).Maybe() - q.On("UpdateContractAssetBalanceAmounts", ctx, mock.Anything, mock.Anything).Return(nil).Maybe() q.On("UpdateContractAssetBalanceExpirations", ctx, mock.Anything, mock.Anything).Return(nil).Maybe() q.On("UpdateAssetContractExpirations", ctx, mock.Anything, mock.Anything).Return(nil).Maybe() - q.On("DeleteContractAssetBalancesExpiringAt", ctx, mock.Anything). + q.On("DeleteContractAssetBalancesExpiredBefore", ctx, ledger). Return([]history.ContractAssetBalance{}, nil).Maybe() - q.On("DeleteAssetContractsExpiringAt", ctx, mock.Anything).Return(int64(0), nil).Maybe() + q.On("DeleteAssetContractsExpiredBefore", ctx, ledger).Return(int64(0), nil).Maybe() q.On("GetContractAssetStat", ctx, mock.Anything). Return(history.ContractAssetStatRow{}, sql.ErrNoRows).Maybe() q.On("InsertContractAssetStat", ctx, mock.Anything).Run(func(args mock.Arguments) { @@ -58,7 +54,7 @@ func runContractDataLedger( } assert.NoError(t, p.Commit(ctx)) - return inserted, removed, statInserts + return upserted, removed, statInserts } func contractDataChange(pre, post *xdr.LedgerEntryData) ingest.Change { @@ -72,29 +68,50 @@ func contractDataChange(pre, post *xdr.LedgerEntryData) ingest.Change { return change } -func ttlCreate(keyHash xdr.Hash, liveUntil uint32) ingest.Change { - return ingest.Change{ +func ttlEntry(keyHash xdr.Hash, liveUntil uint32) *xdr.LedgerEntry { + return &xdr.LedgerEntry{Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTtl, - Post: &xdr.LedgerEntry{Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeTtl, - Ttl: &xdr.TtlEntry{ - KeyHash: keyHash, - LiveUntilLedgerSeq: xdr.Uint32(liveUntil), - }, - }}, - } + Ttl: &xdr.TtlEntry{ + KeyHash: keyHash, + LiveUntilLedgerSeq: xdr.Uint32(liveUntil), + }, + }} +} + +func ttlCreate(keyHash xdr.Hash, liveUntil uint32) ingest.Change { + return ingest.Change{Type: xdr.LedgerEntryTypeTtl, Post: ttlEntry(keyHash, liveUntil)} +} + +func ttlUpdate(keyHash xdr.Hash, from, to uint32) ingest.Change { + return ingest.Change{Type: xdr.LedgerEntryTypeTtl, Pre: ttlEntry(keyHash, from), Post: ttlEntry(keyHash, to)} } func ttlRemove(keyHash xdr.Hash, liveUntil uint32) ingest.Change { - return ingest.Change{ - Type: xdr.LedgerEntryTypeTtl, - Pre: &xdr.LedgerEntry{Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeTtl, - Ttl: &xdr.TtlEntry{ - KeyHash: keyHash, - LiveUntilLedgerSeq: xdr.Uint32(liveUntil), - }, - }}, + return ingest.Change{Type: xdr.LedgerEntryTypeTtl, Pre: ttlEntry(keyHash, liveUntil)} +} + +// rewrittenBalance returns a balance entry whose value was rewritten, at the same +// ledger key, into something which is not a Stellar Asset Contract balance. +func rewrittenBalance(t *testing.T, balance xdr.LedgerEntryData) xdr.LedgerEntryData { + rewritten := *balance.ContractData + val := xdr.Uint32(1) + rewritten.Val = xdr.ScVal{Type: xdr.ScValTypeScvU32, U32: &val} + rewrittenData := xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeContractData, + ContractData: &rewritten, + } + _, _, ok := sac.ContractBalanceFromContractData(xdr.LedgerEntry{Data: rewrittenData}, "passphrase") + assert.False(t, ok, "rewritten entry must no longer resemble a balance") + return rewrittenData +} + +// storedBalance returns the row of a balance of 1000 which expires at ledger 5000. +func storedBalance(keyHash xdr.Hash, assetContractID [32]byte) history.ContractAssetBalance { + return history.ContractAssetBalance{ + KeyHash: keyHash[:], + ContractID: assetContractID[:], + Amount: "1000", + ExpirationLedger: 5000, } } @@ -109,31 +126,75 @@ func TestContractDataRemovalFilteredByLedgerKey(t *testing.T) { balance := sac.BalanceToContractData(assetContractID, holderID, 1000) keyHash := getKeyHashForBalance(t, assetContractID, holderID) + rewrittenData := rewrittenBalance(t, balance) + stored := storedBalance(keyHash, assetContractID) - // value rewritten, ledger key unchanged - rewritten := *balance.ContractData - val := xdr.Uint32(1) - rewritten.Val = xdr.ScVal{Type: xdr.ScValTypeScvU32, U32: &val} - rewrittenData := xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeContractData, - ContractData: &rewritten, - } - - _, _, ok := sac.ContractBalanceFromContractData( - xdr.LedgerEntry{Data: rewrittenData}, "passphrase", - ) - assert.False(t, ok, "rewritten entry must no longer resemble a balance") - - inserted, _, _ := runContractDataLedger(t, 100, nil, + upserted, _, _ := runContractDataLedger(t, 100, nil, contractDataChange(nil, &balance), ttlCreate(keyHash, 5000), ) - assert.Equal(t, []xdr.Hash{keyHash}, inserted) + assert.Equal(t, []history.ContractAssetBalance{stored}, upserted) - _, removed, _ := runContractDataLedger(t, 101, nil, contractDataChange(&rewrittenData, nil)) + _, removed, _ := runContractDataLedger(t, 101, []history.ContractAssetBalance{stored}, + contractDataChange(&rewrittenData, nil), + ) assert.Equal(t, []xdr.Hash{keyHash}, removed) } +// TestContractDataUpdateFollowsValue covers updates which rewrite the value of a +// balance entry into something which is not a balance and back. The row follows +// the latest value of the entry: it is removed while the value is not a balance, +// and stored again when the value turns back into a balance in a ledger which also +// changes its ttl, since the expiration ledger was removed along with the row. +func TestContractDataUpdateFollowsValue(t *testing.T) { + assetContractID := [32]byte{0xAA} + holderID := [32]byte{0xBB} + + balance := sac.BalanceToContractData(assetContractID, holderID, 1000) + keyHash := getKeyHashForBalance(t, assetContractID, holderID) + rewrittenData := rewrittenBalance(t, balance) + updatedBalance := sac.BalanceToContractData(assetContractID, holderID, 5000) + + upserted, removed, statInserts := runContractDataLedger(t, 101, + []history.ContractAssetBalance{storedBalance(keyHash, assetContractID)}, + contractDataChange(&balance, &rewrittenData), + ) + assert.Empty(t, upserted) + assert.Equal(t, []xdr.Hash{keyHash}, removed) + assert.Equal(t, []history.ContractAssetStatRow{ + { + ContractID: assetContractID[:], + Stat: history.ContractStat{ + ActiveBalance: "-1000", + ActiveHolders: -1, + }, + }, + }, statInserts) + + upserted, removed, statInserts = runContractDataLedger(t, 102, nil, + contractDataChange(&rewrittenData, &updatedBalance), + ttlUpdate(keyHash, 5000, 6000), + ) + assert.Empty(t, removed) + assert.Equal(t, []history.ContractAssetBalance{ + { + KeyHash: keyHash[:], + ContractID: assetContractID[:], + Amount: "5000", + ExpirationLedger: 6000, + }, + }, upserted) + assert.Equal(t, []history.ContractAssetStatRow{ + { + ContractID: assetContractID[:], + Stat: history.ContractStat{ + ActiveBalance: "5000", + ActiveHolders: 1, + }, + }, + }, statInserts) +} + // TestContractDataRemovalOfUnrelatedKeyIgnored covers the other side of that // filter: a removal whose ledger key could never have identified a row in // contract_asset_balances is still skipped. @@ -167,26 +228,10 @@ func TestContractDataRemovalUsesStoredAmount(t *testing.T) { balance := sac.BalanceToContractData(assetContractID, holderID, 1000) keyHash := getKeyHashForBalance(t, assetContractID, holderID) + rewrittenData := rewrittenBalance(t, balance) - // value rewritten, ledger key unchanged - rewritten := *balance.ContractData - val := xdr.Uint32(1) - rewritten.Val = xdr.ScVal{Type: xdr.ScValTypeScvU32, U32: &val} - rewrittenData := xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeContractData, - ContractData: &rewritten, - } - - storedRows := []history.ContractAssetBalance{ - { - KeyHash: keyHash[:], - ContractID: assetContractID[:], - Amount: "1000", - ExpirationLedger: 5000, - }, - } - - _, removed, statInserts := runContractDataLedger(t, 101, storedRows, + _, removed, statInserts := runContractDataLedger(t, 101, + []history.ContractAssetBalance{storedBalance(keyHash, assetContractID)}, contractDataChange(&rewrittenData, nil), ttlRemove(keyHash, 5000), ) @@ -204,9 +249,9 @@ func TestContractDataRemovalUsesStoredAmount(t *testing.T) { } // TestContractDataRemovalWithoutStoredRowLeavesStats covers a removal whose entry -// does resemble a balance but which was never ingested as one, because it only -// came to resemble a balance after it was created. With no stored row there is no -// stat to adjust, whatever the removed entry's value looks like. +// does resemble a balance but which was never ingested as one. With no stored row +// there is nothing to remove and no stat to adjust, whatever the removed entry's +// value looks like. func TestContractDataRemovalWithoutStoredRowLeavesStats(t *testing.T) { assetContractID := [32]byte{0xAA} holderID := [32]byte{0xBB} @@ -219,6 +264,6 @@ func TestContractDataRemovalWithoutStoredRowLeavesStats(t *testing.T) { ttlRemove(keyHash, 5000), ) - assert.Equal(t, []xdr.Hash{keyHash}, removed) + assert.Empty(t, removed) assert.Empty(t, statInserts) } diff --git a/internal/ingest/processors/asset_stats_processor_test.go b/internal/ingest/processors/asset_stats_processor_test.go index 81b9a91e4..b5f9f9c57 100644 --- a/internal/ingest/processors/asset_stats_processor_test.go +++ b/internal/ingest/processors/asset_stats_processor_test.go @@ -338,22 +338,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertClaimableBalance() { }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -508,22 +500,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertTrustLine() { }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -663,18 +647,12 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertAssetContract() { }).Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -697,22 +675,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertAssetContractMissingTTL() }) s.Assert().NoError(err) - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -750,22 +720,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertExpiredAssetContract() { }, })) - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -815,28 +777,35 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRestoreReapsBeforeInsert() { KeyHash: balanceKeyHash[:], ContractID: usdID[:], Amount: "200", ExpirationLedger: reapLedger, } - var inserted bool + // The stale balance is reaped before the stored rows of the restored balance are + // read, so that it is removed from the stats once: when it is reaped. The restored + // balance then adds it back, which leaves the stats unchanged. + var reaped, upserted bool reapFirst := func(mock.Arguments) { - s.Assert().False(inserted, "rows must be reaped before restores are inserted") + s.Assert().False(upserted, "rows must be reaped before restores are upserted") + reaped = true + } + readAfterReap := func(mock.Arguments) { + s.Assert().True(reaped, "stored rows must be read after rows are reaped") } - markInsert := func(mock.Arguments) { inserted = true } + markUpsert := func(mock.Arguments) { upserted = true } - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)).Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}).Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}).Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}).Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, reapLedger).Run(reapFirst). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, s.processor.currentLedger).Run(reapFirst). Return([]history.ContractAssetBalance{staleBalance}, nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, reapLedger).Run(reapFirst). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, s.processor.currentLedger).Run(reapFirst). Return(int64(1), nil).Once() + s.mockQ.On("GetContractAssetBalances", s.ctx, []xdr.Hash{balanceKeyHash}).Run(readAfterReap). + Return([]history.ContractAssetBalance{}, nil).Once() s.mockQ.On("InsertAssetContracts", s.ctx, mock.MatchedBy(func(rows []history.AssetContract) bool { return len(rows) == 1 && bytes.Equal(rows[0].KeyHash, usdKeyHash[:]) && rows[0].ExpirationLedger == uint32(restoredTTL) - })).Run(markInsert).Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, mock.MatchedBy(func(rows []history.ContractAssetBalance) bool { + })).Run(markUpsert).Return(nil).Once() + s.mockQ.On("UpsertContractAssetBalances", s.ctx, mock.MatchedBy(func(rows []history.ContractAssetBalance) bool { return len(rows) == 1 && bytes.Equal(rows[0].KeyHash, balanceKeyHash[:]) && rows[0].ExpirationLedger == uint32(restoredTTL) - })).Run(markInsert).Return(nil).Once() + })).Run(markUpsert).Return(nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) } @@ -884,18 +853,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertContractBalance() { usdAssetContractStat.Stat == row.Stat })).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance{ + s.mockQ.On("GetContractAssetBalances", s.ctx, []xdr.Hash{keyHash}). + Return([]history.ContractAssetBalance{}, nil).Once() + s.mockQ.On("UpsertContractAssetBalances", s.ctx, []history.ContractAssetBalance{ { KeyHash: keyHash[:], ContractID: usdID[:], @@ -905,7 +870,7 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertContractBalance() { }).Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -945,22 +910,21 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateContractBalance() { usdAssetContractStat.Stat == row.Stat })).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{keyHash}, []string{"300"}). - Return(nil).Once() + s.mockQ.On("GetContractAssetBalances", s.ctx, []xdr.Hash{keyHash}). + Return([]history.ContractAssetBalance{ + {KeyHash: keyHash[:], ContractID: usdID[:], Amount: "100", ExpirationLedger: 2234}, + }, nil).Once() + s.mockQ.On("UpsertContractAssetBalances", s.ctx, []history.ContractAssetBalance{ + {KeyHash: keyHash[:], ContractID: usdID[:], Amount: "300", ExpirationLedger: 2234}, + }).Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1004,11 +968,9 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveContractBalance() { s.mockQ.On("GetContractAssetStat", s.ctx, usdID[:]). Return(usdAssetContractStat, nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() usdAssetContractStat.Stat.ActiveHolders = 0 @@ -1020,13 +982,9 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveContractBalance() { }, nil).Once() s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash{keyHash}). Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1205,14 +1163,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertAssetContractWithBalance( }).Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance{ + // the usd balance has no ttl in this ledger and no stored row, so it gets no row + usdBalanceKeyHash := getKeyHashForBalance(s.T(), usdID, [32]byte{1}) + s.mockQ.On("GetContractAssetBalances", s.ctx, []xdr.Hash{usdBalanceKeyHash, keyHash}). + Return([]history.ContractAssetBalance{}, nil).Once() + s.mockQ.On("UpsertContractAssetBalances", s.ctx, []history.ContractAssetBalance{ { KeyHash: keyHash[:], ContractID: btcID[:], @@ -1222,7 +1180,7 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertAssetContractWithBalance( }).Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1325,22 +1283,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertClaimableBalanceAndTrustl }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1377,22 +1327,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestExpirationLedgerDecreases() { }, })) - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{keyHash}, []uint32{2234}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{keyHash}, []uint32{2234}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1429,22 +1371,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestExpirationLedgerIgnoredIfLessTh }, })) - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1549,22 +1483,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLine() { }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1775,22 +1701,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLineAuthorization() }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -1897,22 +1815,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveClaimableBalance() { }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -2005,22 +1915,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveTrustLine() { trustLineIssuer.Address(), ).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) @@ -2095,22 +1997,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestProcessUpgradeChange() { }, }).Return(int64(1), nil).Once() - s.mockQ.On("InsertAssetContracts", s.ctx, []history.AssetContract(nil)). - Return(nil).Once() s.mockQ.On("UpdateAssetContractExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteAssetContractsExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteAssetContractsExpiredBefore", s.ctx, uint32(1235)). Return(int64(0), nil).Once() - s.mockQ.On("RemoveContractAssetBalances", s.ctx, []xdr.Hash(nil)). - Return(nil).Once() - s.mockQ.On("UpdateContractAssetBalanceAmounts", s.ctx, []xdr.Hash{}, []string{}). - Return(nil).Once() - s.mockQ.On("InsertContractAssetBalances", s.ctx, []history.ContractAssetBalance(nil)). - Return(nil).Once() s.mockQ.On("UpdateContractAssetBalanceExpirations", s.ctx, []xdr.Hash{}, []uint32{}). Return(nil).Once() - s.mockQ.On("DeleteContractAssetBalancesExpiringAt", s.ctx, uint32(1234)). + s.mockQ.On("DeleteContractAssetBalancesExpiredBefore", s.ctx, uint32(1235)). Return([]history.ContractAssetBalance{}, nil).Once() s.Assert().NoError(s.processor.Commit(s.ctx)) diff --git a/internal/ingest/processors/contract_asset_stats.go b/internal/ingest/processors/contract_asset_stats.go index 0123e4c07..d4be52f0c 100644 --- a/internal/ingest/processors/contract_asset_stats.go +++ b/internal/ingest/processors/contract_asset_stats.go @@ -1,6 +1,7 @@ package processors import ( + "bytes" "context" "crypto/sha256" "math/big" @@ -29,19 +30,71 @@ func (v assetContractStatValue) ConvertToHistoryObject() history.ContractAssetSt } } +// contractStats holds the stats of contract balances, or the changes to them, by contract. +type contractStats map[xdr.ContractId]assetContractStatValue + +// add adds holders and amount to the stats of a contract. +func (c contractStats) add(contractID xdr.ContractId, holders int32, amount *big.Int) { + stat, ok := c[contractID] + if !ok { + stat = assetContractStatValue{contractID: contractID, activeBalance: big.NewInt(0)} + } + stat.activeHolders += holders + stat.activeBalance.Add(stat.activeBalance, amount) + if stat.activeHolders == 0 && stat.activeBalance.Sign() == 0 { + delete(c, contractID) + } else { + c[contractID] = stat + } +} + +// removeRow removes a contract balance row stored in the db from the stats. +func (c contractStats) removeRow(row history.ContractAssetBalance) error { + amount, ok := new(big.Int).SetString(row.Amount, 10) + if !ok { + return errors.Errorf( + "contract balance %v has invalid amount: %v", + row.KeyHash, + row.Amount, + ) + } + + var contractID xdr.ContractId + copy(contractID[:], row.ContractID) + c.add(contractID, -1, amount.Neg(amount)) + return nil +} + +func (c contractStats) rows() []history.ContractAssetStatRow { + var rows []history.ContractAssetStatRow + for _, stat := range c { + rows = append(rows, stat.ConvertToHistoryObject()) + } + return rows +} + type contractAssetBalancesQ interface { GetContractAssetBalances(ctx context.Context, keys []xdr.Hash) ([]history.ContractAssetBalance, error) - DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]history.ContractAssetBalance, error) + DeleteContractAssetBalancesExpiredBefore(ctx context.Context, ledger uint32) ([]history.ContractAssetBalance, error) +} + +// contractBalance is the latest value of a contract balance entry. amount is nil if the +// entry was removed or if its value is not a Stellar Asset Contract balance. +type contractBalance struct { + contractID xdr.ContractId + amount *big.Int } // ContractAssetStatSet represents a collection of asset stats for // contract asset holders type ContractAssetStatSet struct { - createdAssetContracts []xdr.Asset - contractAssetStats map[xdr.ContractId]assetContractStatValue - createdBalances []history.ContractAssetBalance - removedBalances []xdr.Hash - updatedBalances map[xdr.Hash]*big.Int + createdAssetContracts []xdr.Asset + // contractAssetStats holds the change to the stats of each contract. + contractAssetStats contractStats + // balances holds the latest value of the changed contract balance entries by key + // hash, and balanceKeys holds their key hashes in the order they were added. + balances map[xdr.Hash]contractBalance + balanceKeys []xdr.Hash removedExpirationEntries map[xdr.Hash]uint32 createdExpirationEntries map[xdr.Hash]uint32 updatedExpirationEntries map[xdr.Hash][2]uint32 @@ -50,300 +103,327 @@ type ContractAssetStatSet struct { currentLedger uint32 } -// NewContractAssetStatSet constructs a new ContractAssetStatSet instance +// NewContractAssetStatSet constructs a new ContractAssetStatSet instance. The ttl +// changes of a ledger are only read when the rows of the set are built, so they can be +// added before or after its contract data changes. func NewContractAssetStatSet( assetStatsQ contractAssetBalancesQ, networkPassphrase string, - removedExpirationEntries map[xdr.Hash]uint32, - createdExpirationEntries map[xdr.Hash]uint32, - updatedExpirationEntries map[xdr.Hash][2]uint32, currentLedger uint32, ) *ContractAssetStatSet { return &ContractAssetStatSet{ createdAssetContracts: []xdr.Asset{}, - contractAssetStats: map[xdr.ContractId]assetContractStatValue{}, + contractAssetStats: contractStats{}, + balances: map[xdr.Hash]contractBalance{}, + removedExpirationEntries: map[xdr.Hash]uint32{}, + createdExpirationEntries: map[xdr.Hash]uint32{}, + updatedExpirationEntries: map[xdr.Hash][2]uint32{}, networkPassphrase: networkPassphrase, assetStatsQ: assetStatsQ, - removedExpirationEntries: removedExpirationEntries, - createdExpirationEntries: createdExpirationEntries, - updatedExpirationEntries: updatedExpirationEntries, currentLedger: currentLedger, - updatedBalances: map[xdr.Hash]*big.Int{}, } } // AddContractData updates the set to account for how a given contract data entry has changed. // change must be a xdr.LedgerEntryTypeContractData type. func (s *ContractAssetStatSet) AddContractData(change ingest.Change) error { - // skip ingestion of contract asset balances if we find an asset contract metadata entry - // because a ledger entry cannot be both an asset contract metadata entry and a - // contract asset balance. - if found, err := s.ingestAssetContractMetadata(change); err != nil { - return err - } else if found { + // a ledger entry cannot be both an asset contract instance and a contract balance + if s.addAssetContract(change) { return nil } - return s.ingestContractAssetBalance(change) + return s.addContractBalance(change) } -func (s *ContractAssetStatSet) GetCreatedAssetContracts() ([]history.AssetContract, error) { - var rows []history.AssetContract - for _, asset := range s.createdAssetContracts { - contractID, err := asset.ContractID(s.networkPassphrase) - if err != nil { - return nil, err - } - row := history.AssetContract{ - ContractID: contractID[:], +// ContractAssetStatSet requires that the ttl changes it ingests are already compacted +// because the TTL change semantics in CAP-63 (see +// https://github.com/stellar/stellar-protocol/blob/master/core/cap-0063.md#ttl-ledger-change-semantics ) +// are encapsulated in the ChangeCompactor +func (s *ContractAssetStatSet) checkTTLChangeIsCompacted(change ingest.Change) error { + var keyHash xdr.Hash + if change.Pre != nil { + keyHash = change.Pre.Data.MustTtl().KeyHash + } else { + keyHash = change.Post.Data.MustTtl().KeyHash + } + if _, ok := s.removedExpirationEntries[keyHash]; ok { + return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) + } + if _, ok := s.createdExpirationEntries[keyHash]; ok { + return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) + } + if _, ok := s.updatedExpirationEntries[keyHash]; ok { + return errors.Errorf("ttl change is not compacted Pre: %v Post: %v", change.Pre, change.Post) + } + return nil +} + +// AddTTL updates the set to account for how a given ttl entry has changed. +// change must be a xdr.LedgerEntryTypeTtl type. +func (s *ContractAssetStatSet) AddTTL(change ingest.Change) error { + if err := s.checkTTLChangeIsCompacted(change); err != nil { + return err + } + + switch { + case change.Pre == nil && change.Post != nil: // created + post := change.Post.Data.MustTtl() + s.createdExpirationEntries[post.KeyHash] = uint32(post.LiveUntilLedgerSeq) + case change.Pre != nil && change.Post == nil: // removed + pre := change.Pre.Data.MustTtl() + s.removedExpirationEntries[pre.KeyHash] = uint32(pre.LiveUntilLedgerSeq) + case change.Pre != nil && change.Post != nil: // updated + pre := change.Pre.Data.MustTtl() + post := change.Post.Data.MustTtl() + // it's unclear if core could emit a ledger entry change where the + // expiration ledger remains the same + if pre.LiveUntilLedgerSeq == post.LiveUntilLedgerSeq { + return nil } - if err = asset.Extract(&row.AssetType, &row.AssetCode, &row.AssetIssuer); err != nil { - return nil, errors.Wrap(err, "could not extract asset info from asset") + + // if a TTL was created, restored, or updated at ledger N + // the LiveUntilLedgerSeq must be >= current ledger + if uint32(post.LiveUntilLedgerSeq) < s.currentLedger { + return errors.Errorf( + "post TTL %v is less than current ledger %v", + post.LiveUntilLedgerSeq, + s.currentLedger, + ) } - ledgerKey := sac.AssetToContractDataLedgerKey(contractID) - bin, err := ledgerKey.MarshalBinary() - if err != nil { - return nil, errors.Wrap(err, "could not marshal key") + // The previous expiration ledger must always be greater than or equal to the current ledger + // because if the previous expiration ledger is less than the current ledger then it implies + // the ledger entry was archived. However, an archived ledger entry cannot be updated without + // first being restored. + // Alternatively, the TTL can be updated without being restored if it is a temporary ledger + // entry. However, in that case, we can ignore the ledger entry entirely because SAC + // ledger entries are always kept in persistent storage. + if uint32(pre.LiveUntilLedgerSeq) < s.currentLedger { + return nil } - keyHash := sha256.Sum256(bin) - row.KeyHash = keyHash[:] - var ok bool - row.ExpirationLedger, ok = s.createdExpirationEntries[keyHash] - if !ok { - return nil, errors.Errorf("could not find expiration ledger entry for asset contract %d", contractID) + s.updatedExpirationEntries[pre.KeyHash] = [2]uint32{ + uint32(pre.LiveUntilLedgerSeq), + uint32(post.LiveUntilLedgerSeq), } - rows = append(rows, row) - } - - return rows, nil -} - -func (s *ContractAssetStatSet) GetContractStats() []history.ContractAssetStatRow { - var contractStats []history.ContractAssetStatRow - for _, contractStat := range s.contractAssetStats { - contractStats = append(contractStats, contractStat.ConvertToHistoryObject()) + default: + return errors.Errorf("unexpected change Pre: %v Post: %v", change.Pre, change.Post) } - return contractStats -} -func (s *ContractAssetStatSet) GetCreatedBalances() []history.ContractAssetBalance { - return s.createdBalances + return nil } -func (s *ContractAssetStatSet) ingestAssetContractMetadata(change ingest.Change) (bool, error) { +// addAssetContract records the asset of a created asset contract instance, and reports +// whether the change created one. Asset contract instances are recognized by their +// value, which names the asset that their contract id is derived from. +func (s *ContractAssetStatSet) addAssetContract(change ingest.Change) bool { if change.Pre != nil || change.Post == nil { - return false, nil + return false } asset, found := sac.AssetFromContractData(*change.Post, s.networkPassphrase) - if !found { - return false, nil - } - keyHash, err := getKeyHash(*change.Post) - if err != nil { - return false, err - } - expirationLedger, ok := s.createdExpirationEntries[keyHash] - if !ok || expirationLedger < s.currentLedger { - return false, nil - } - if pContactID := change.Post.Data.MustContractData().Contract.ContractId; pContactID != nil { + if found { s.createdAssetContracts = append(s.createdAssetContracts, asset) - return true, nil } - return false, nil + return found } -func getKeyHash(ledgerEntry xdr.LedgerEntry) (xdr.Hash, error) { - lk, err := ledgerEntry.LedgerKey() - if err != nil { - return xdr.Hash{}, errors.Wrap(err, "could not extract ledger key") - } - bin, err := lk.MarshalBinary() - if err != nil { - return xdr.Hash{}, errors.Wrap(err, "could not marshal key") - } - return sha256.Sum256(bin), nil -} - -func (s *ContractAssetStatSet) ingestContractAssetBalance(change ingest.Change) error { - switch { - case change.Pre == nil && change.Post != nil: // created or restored - pContractID := change.Post.Data.MustContractData().Contract.ContractId - if pContractID == nil { - return nil - } - - _, postAmt, postOk := sac.ContractBalanceFromContractData(*change.Post, s.networkPassphrase) - // we only ingest created ledger entries if we determine that they resemble the shape of - // a Stellar Asset Contract balance ledger entry - if !postOk { - return nil +// GetCreatedAssetContracts returns the rows of the created asset contract instances +// which are active in the current ledger. +func (s *ContractAssetStatSet) GetCreatedAssetContracts() ([]history.AssetContract, error) { + var rows []history.AssetContract + for _, asset := range s.createdAssetContracts { + contractID, err := asset.ContractID(s.networkPassphrase) + if err != nil { + return nil, err } - - keyHash, err := getKeyHash(*change.Post) + keyHash, err := getLedgerKeyHash(sac.AssetToContractDataLedgerKey(contractID)) if err != nil { - return err + return nil, err } expirationLedger, ok := s.createdExpirationEntries[keyHash] if !ok || expirationLedger < s.currentLedger { - return nil + continue } - s.createdBalances = append(s.createdBalances, history.ContractAssetBalance{ + row := history.AssetContract{ KeyHash: keyHash[:], - ContractID: (*pContractID)[:], - Amount: postAmt.String(), + ContractID: contractID[:], ExpirationLedger: expirationLedger, - }) - - stat := s.getContractAssetStat(*pContractID) - stat.activeHolders++ - stat.activeBalance.Add(stat.activeBalance, postAmt) - s.maybeAddContractAssetStat(*pContractID, stat) - case change.Pre != nil && change.Post == nil: // removed - pContractID := change.Pre.Data.MustContractData().Contract.ContractId - if pContractID == nil { - return nil } - - keyHash, err := getKeyHash(*change.Pre) - if err != nil { - return err - } - // The key hash is recorded whether or not the entry still resembles a - // balance, since rows are keyed on the ledger key. The stat delta is - // applied by ingestRemovedBalances from the stored row, because the - // value here is not a reliable record of what was counted. - s.removedBalances = append(s.removedBalances, keyHash) - case change.Pre != nil && change.Post != nil: // updated - pContractID := change.Pre.Data.MustContractData().Contract.ContractId - if pContractID == nil { - return nil - } - - holder, amt, ok := sac.ContractBalanceFromContractData(*change.Pre, s.networkPassphrase) - if !ok { - return nil - } - - // if the updated ledger entry is not in the expected format then this - // cannot be emitted by the stellar asset contract, so ignore it - postHolder, postAmt, postOk := sac.ContractBalanceFromContractData(*change.Post, s.networkPassphrase) - if !postOk || postHolder != holder { - return nil + if err = asset.Extract(&row.AssetType, &row.AssetCode, &row.AssetIssuer); err != nil { + return nil, errors.Wrap(err, "could not extract asset info from asset") } + rows = append(rows, row) + } - amtDelta := new(big.Int).Sub(postAmt, amt) - if amtDelta.Cmp(big.NewInt(0)) == 0 { - return nil - } + return rows, nil +} - keyHash, err := getKeyHash(*change.Post) - if err != nil { - return err +// GetCreatedBalancesAndStats returns the rows of the contract balances added to the set, +// assuming none of them is stored yet, along with their stats grouped by contract. This +// is the case when ingesting state from a history archive, where every change creates a +// ledger entry. +func (s *ContractAssetStatSet) GetCreatedBalancesAndStats() ([]history.ContractAssetBalance, []history.ContractAssetStatRow) { + var rows []history.ContractAssetBalance + stats := contractStats{} + for _, keyHash := range s.balanceKeys { + balance := s.balances[keyHash] + if row, ok := s.balanceRow(keyHash, balance, nil); ok { + rows = append(rows, row) + stats.add(balance.contractID, 1, balance.amount) } + } + return rows, stats.rows() +} - s.updatedBalances[keyHash] = postAmt - stat := s.getContractAssetStat(*pContractID) - stat.activeBalance.Add(stat.activeBalance, amtDelta) - s.maybeAddContractAssetStat(*pContractID, stat) - default: - return errors.Errorf("unexpected change Pre: %v Post: %v", change.Pre, change.Post) +func getLedgerKeyHash(ledgerKey xdr.LedgerKey) (xdr.Hash, error) { + bin, err := ledgerKey.MarshalBinary() + if err != nil { + return xdr.Hash{}, errors.Wrap(err, "could not marshal key") } - return nil + return sha256.Sum256(bin), nil } -// ingestRemovedBalances applies the stat delta for balances whose ledger entry -// was removed, taking the amount from the stored row rather than from the removed -// entry. A removed key with no stored row was either never ingested as a balance -// or was already accounted for when its row was reaped on expiry, and needs no -// adjustment either way. -// -// This must run before RemoveContractAssetBalances, which deletes the rows it -// reads. -func (s *ContractAssetStatSet) ingestRemovedBalances(ctx context.Context) error { - if len(s.removedBalances) == 0 { +// addContractBalance records the latest value of a ledger entry whose key has the form +// of a Stellar Asset Contract balance key. Entries are selected by their key, which +// never changes, rather than by their value, which any contract can set to anything for +// its own entries. A value which is not a balance removes the row of the key, the same +// way a removed entry does. +func (s *ContractAssetStatSet) addContractBalance(change ingest.Change) error { + if change.Pre == nil && change.Post == nil { + return errors.Errorf("unexpected change Pre: %v Post: %v", change.Pre, change.Post) + } + ledgerKey, err := change.LedgerKey() + if err != nil { + return errors.Wrap(err, "could not extract ledger key") + } + if !sac.ValidContractBalanceLedgerKey(ledgerKey) { return nil } - - rows, err := s.assetStatsQ.GetContractAssetBalances(ctx, s.removedBalances) + contractID := ledgerKey.ContractData.Contract.ContractId + if contractID == nil { + return nil + } + keyHash, err := getLedgerKeyHash(ledgerKey) if err != nil { - return errors.Wrap(err, "Error fetching removed contract asset balances") + return err } - - for _, row := range rows { - amt, ok := new(big.Int).SetString(row.Amount, 10) - if !ok { - return errors.Errorf( - "contract balance %v has invalid amount: %v", - row.KeyHash, - row.Amount, - ) + balance := contractBalance{contractID: *contractID} + if change.Post != nil { + if _, amount, valid := sac.ContractBalanceFromContractData(*change.Post, s.networkPassphrase); valid { + balance.amount = amount } + } + if _, seen := s.balances[keyHash]; !seen { + s.balanceKeys = append(s.balanceKeys, keyHash) + } + s.balances[keyHash] = balance + return nil +} - var contractID xdr.ContractId - copy(contractID[:], row.ContractID) - stat := s.getContractAssetStat(contractID) - stat.activeHolders-- - stat.activeBalance.Sub(stat.activeBalance, amt) - s.maybeAddContractAssetStat(contractID, stat) +// balanceRow returns the row to store for the latest value of a contract balance entry, +// or false if the entry should have no row: when it was removed, when its value is not +// a Stellar Asset Contract balance, or when it is not active in the current ledger. +// stored is the row in the db, if any. +func (s *ContractAssetStatSet) balanceRow( + keyHash xdr.Hash, + balance contractBalance, + stored *history.ContractAssetBalance, +) (history.ContractAssetBalance, bool) { + if balance.amount == nil { + return history.ContractAssetBalance{}, false } + expirationLedger, ok := s.expirationLedger(keyHash, stored) + if !ok || expirationLedger < s.currentLedger { + return history.ContractAssetBalance{}, false + } + return history.ContractAssetBalance{ + KeyHash: keyHash[:], + ContractID: balance.contractID[:], + Amount: balance.amount.String(), + ExpirationLedger: expirationLedger, + }, true +} - return nil +// expirationLedger returns the expiration ledger of a contract balance entry. A ttl +// change in this ledger takes precedence over the stored row. +// +// Neither is known when a value which is not a balance removed the row of an entry, and +// a later ledger turns the value back into a balance without changing its ttl. Stellar +// Asset Contracts only write balances, so from protocol 23, where restoring an entry +// also emits the entry and its ttl, this only happens to the entries of other +// contracts, whose balances are not served. Such an entry is left without a row. +func (s *ContractAssetStatSet) expirationLedger(keyHash xdr.Hash, stored *history.ContractAssetBalance) (uint32, bool) { + if expirationLedger, ok := s.createdExpirationEntries[keyHash]; ok { + return expirationLedger, true + } + if update, ok := s.updatedExpirationEntries[keyHash]; ok { + return update[1], true + } + if stored != nil { + return stored.ExpirationLedger, true + } + return 0, false } +// ingestExpiredBalances deletes the rows of contract balances which are no longer +// active in the current ledger and removes them from the contract stats. func (s *ContractAssetStatSet) ingestExpiredBalances(ctx context.Context) error { - rows, err := s.assetStatsQ.DeleteContractAssetBalancesExpiringAt(ctx, s.currentLedger-1) + rows, err := s.assetStatsQ.DeleteContractAssetBalancesExpiredBefore(ctx, s.currentLedger) if err != nil { - return errors.Wrap(err, "Error fetching contract asset balances") + return errors.Wrap(err, "Error deleting expired contract asset balances") } for _, row := range rows { - var keyHash xdr.Hash - copy(keyHash[:], row.KeyHash) - - if _, ok := s.updatedExpirationEntries[keyHash]; ok { - // the expiration of this contract balance was bumped, so we can - // skip this contract balance since it is still active - continue - } - - var contractID xdr.ContractId - copy(contractID[:], row.ContractID) - stat := s.getContractAssetStat(contractID) - amt, ok := new(big.Int).SetString(row.Amount, 10) - if !ok { - return errors.Errorf( - "contract balance %v has invalid amount: %v", - row.KeyHash, - row.Amount, - ) + if err = s.contractAssetStats.removeRow(row); err != nil { + return err } - - stat.activeHolders-- - stat.activeBalance.Sub(stat.activeBalance, amt) - s.maybeAddContractAssetStat(contractID, stat) } - return nil } -func (s *ContractAssetStatSet) maybeAddContractAssetStat(contractID xdr.ContractId, stat assetContractStatValue) { - if stat.activeHolders == 0 && - stat.activeBalance.Cmp(big.NewInt(0)) == 0 { - delete(s.contractAssetStats, contractID) - } else { - s.contractAssetStats[contractID] = stat +// reconcileBalances compares the row which the latest value of every changed contract +// balance entry calls for with the row stored in the db. It returns the rows to upsert +// and the keys of the rows to remove, and adjusts the contract stats by the difference +// between the stored and the new rows. +// +// It must run after ingestExpiredBalances: an entry can be restored in the ledger right +// after it expired, and its expired row must be removed from the stats only once. +func (s *ContractAssetStatSet) reconcileBalances(ctx context.Context) ([]history.ContractAssetBalance, []xdr.Hash, error) { + if len(s.balanceKeys) == 0 { + return nil, nil, nil } -} -func (s *ContractAssetStatSet) getContractAssetStat(contractID xdr.ContractId) assetContractStatValue { - stat, ok := s.contractAssetStats[contractID] - if !ok { - stat = assetContractStatValue{ - contractID: contractID, - activeBalance: big.NewInt(0), - activeHolders: 0, + rows, err := s.assetStatsQ.GetContractAssetBalances(ctx, s.balanceKeys) + if err != nil { + return nil, nil, errors.Wrap(err, "Error fetching contract asset balances") + } + stored := make(map[xdr.Hash]*history.ContractAssetBalance, len(rows)) + for i := range rows { + var keyHash xdr.Hash + copy(keyHash[:], rows[i].KeyHash) + stored[keyHash] = &rows[i] + } + + var upserts []history.ContractAssetBalance + var removals []xdr.Hash + for _, keyHash := range s.balanceKeys { + balance := s.balances[keyHash] + storedRow := stored[keyHash] + row, hasRow := s.balanceRow(keyHash, balance, storedRow) + if storedRow != nil { + if hasRow && row.Amount == storedRow.Amount && row.ExpirationLedger == storedRow.ExpirationLedger && + bytes.Equal(row.ContractID, storedRow.ContractID) { + continue + } + if err = s.contractAssetStats.removeRow(*storedRow); err != nil { + return nil, nil, err + } + if !hasRow { + removals = append(removals, keyHash) + } + } + if hasRow { + s.contractAssetStats.add(balance.contractID, 1, balance.amount) + upserts = append(upserts, row) } } - return stat + return upserts, removals, nil } diff --git a/internal/ingest/processors/contract_asset_stats_test.go b/internal/ingest/processors/contract_asset_stats_test.go index b8f9c5fa3..3f9254e0d 100644 --- a/internal/ingest/processors/contract_asset_stats_test.go +++ b/internal/ingest/processors/contract_asset_stats_test.go @@ -2,7 +2,6 @@ package processors import ( "context" - "crypto/sha256" "testing" "github.com/stretchr/testify/assert" @@ -15,10 +14,17 @@ import ( ) func getKeyHashForBalance(t *testing.T, assetContractId, holderID [32]byte) xdr.Hash { - ledgerKey := sac.ContractBalanceLedgerKey(assetContractId, holderID) - bin, err := ledgerKey.MarshalBinary() + keyHash, err := getLedgerKeyHash(sac.ContractBalanceLedgerKey(assetContractId, holderID)) assert.NoError(t, err) - return sha256.Sum256(bin) + return keyHash +} + +func getKeyHash(ledgerEntry xdr.LedgerEntry) (xdr.Hash, error) { + ledgerKey, err := ledgerEntry.LedgerKey() + if err != nil { + return xdr.Hash{}, err + } + return getLedgerKeyHash(ledgerKey) } func TestAddContractData(t *testing.T) { @@ -39,9 +45,6 @@ func TestAddContractData(t *testing.T) { set := NewContractAssetStatSet( &history.MockQAssetStats{}, "passphrase", - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, 150, ) @@ -147,10 +150,12 @@ func TestAddContractData(t *testing.T) { assert.NoError(t, err) // negative balances will be ignored + negativeBalanceKeyHash := getKeyHashForBalance(t, etherID, [32]byte{3}) + set.createdExpirationEntries[negativeBalanceKeyHash] = 150 err = set.AddContractData(ingest.Change{ Type: xdr.LedgerEntryTypeContractData, Post: &xdr.LedgerEntry{ - Data: sac.BalanceInt128ToContractData(etherID, [32]byte{1}, xdr.Int128Parts{Hi: -1, Lo: 0}), + Data: sac.BalanceInt128ToContractData(etherID, [32]byte{3}, xdr.Int128Parts{Hi: -1, Lo: 0}), }, }) assert.NoError(t, err) @@ -167,8 +172,6 @@ func TestAddContractData(t *testing.T) { }) assert.NoError(t, err) - assert.Empty(t, set.updatedBalances) - assert.Empty(t, set.removedBalances) createdAssetContracts, err := set.GetCreatedAssetContracts() assert.NoError(t, err) assert.Equal(t, []history.AssetContract{ @@ -189,6 +192,7 @@ func TestAddContractData(t *testing.T) { ExpirationLedger: 150, }, }, createdAssetContracts) + createdBalances, statRows := set.GetCreatedBalancesAndStats() assert.Equal(t, []history.ContractAssetBalance{ { KeyHash: uniBalanceKeyHash[:], @@ -202,8 +206,8 @@ func TestAddContractData(t *testing.T) { Amount: "150", ExpirationLedger: 150, }, - }, set.createdBalances) - assert.ElementsMatch(t, set.GetContractStats(), []history.ContractAssetStatRow{ + }, createdBalances) + assert.ElementsMatch(t, statRows, []history.ContractAssetStatRow{ { ContractID: uniID[:], Stat: history.ContractStat{ @@ -221,7 +225,7 @@ func TestAddContractData(t *testing.T) { }) } -func TestGetCreatedAssetContractsRequiresExpiration(t *testing.T) { +func TestGetCreatedAssetContractsSkipsMissingExpiration(t *testing.T) { usdcIssuer := keypair.MustRandom().Address() usdcAsset := xdr.MustNewCreditAsset("USDC", usdcIssuer) usdcID, err := usdcAsset.ContractID("passphrase") @@ -230,9 +234,6 @@ func TestGetCreatedAssetContractsRequiresExpiration(t *testing.T) { set := NewContractAssetStatSet( &history.MockQAssetStats{}, "passphrase", - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, 150, ) @@ -251,163 +252,217 @@ func TestGetCreatedAssetContractsRequiresExpiration(t *testing.T) { }) assert.NoError(t, err) + rows, err := set.GetCreatedAssetContracts() + assert.NoError(t, err) + assert.Len(t, rows, 1) + delete(set.createdExpirationEntries, usdcAssetKeyHash) - _, err = set.GetCreatedAssetContracts() - assert.ErrorContains(t, err, "could not find expiration ledger entry for asset contract") + rows, err = set.GetCreatedAssetContracts() + assert.NoError(t, err) + assert.Empty(t, rows) } -func TestUpdateContractBalance(t *testing.T) { - usdcIssuer := keypair.MustRandom().Address() - usdcAsset := xdr.MustNewCreditAsset("USDC", usdcIssuer) - usdcID, err := usdcAsset.ContractID("passphrase") - assert.NoError(t, err) - etherIssuer := keypair.MustRandom().Address() - etherAsset := xdr.MustNewCreditAsset("ETHER", etherIssuer) - etherID, err := etherAsset.ContractID("passphrase") - assert.NoError(t, err) - btcAsset := xdr.MustNewCreditAsset("BTC", etherIssuer) - btcID, err := btcAsset.ContractID("passphrase") - assert.NoError(t, err) - uniAsset := xdr.MustNewCreditAsset("UNI", etherIssuer) - uniID, err := uniAsset.ContractID("passphrase") +func assetContractID(t *testing.T, code, issuer string) xdr.ContractId { + contractID, err := xdr.MustNewCreditAsset(code, issuer).ContractID("passphrase") assert.NoError(t, err) + return contractID +} +// reconcileTest adds updates of contract balance entries to a ContractAssetStatSet, +// along with the rows the db holds for them. +type reconcileTest struct { + t *testing.T + mockQ *history.MockQAssetStats + set *ContractAssetStatSet + keys []xdr.Hash + stored []history.ContractAssetBalance +} + +func newReconcileTest(t *testing.T) *reconcileTest { mockQ := &history.MockQAssetStats{} - set := NewContractAssetStatSet( - mockQ, - "passphrase", - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, - 150, - ) - expectedBalances := map[xdr.Hash]string{} + return &reconcileTest{ + t: t, + mockQ: mockQ, + set: NewContractAssetStatSet( + mockQ, + "passphrase", + 150, + ), + } +} - keyHash := getKeyHashForBalance(t, usdcID, [32]byte{}) - set.updatedExpirationEntries[keyHash] = [2]uint32{160, 170} - expectedBalances[keyHash] = "100" - err = set.AddContractData(ingest.Change{ +func (r *reconcileTest) update(contractID xdr.ContractId, holder [32]byte, pre, post xdr.LedgerEntryData) xdr.Hash { + keyHash := getKeyHashForBalance(r.t, contractID, holder) + r.keys = append(r.keys, keyHash) + assert.NoError(r.t, r.set.AddContractData(ingest.Change{ Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(usdcID, [32]byte{}, 50), - }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(usdcID, [32]byte{}, 100), - }, - }) - assert.NoError(t, err) + Pre: &xdr.LedgerEntry{Data: pre}, + Post: &xdr.LedgerEntry{Data: post}, + })) + return keyHash +} - keyHash = getKeyHashForBalance(t, usdcID, [32]byte{2}) - expectedBalances[keyHash] = "100" - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(usdcID, [32]byte{2}, 30), - }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(usdcID, [32]byte{2}, 100), - }, +func (r *reconcileTest) store(keyHash xdr.Hash, contractID xdr.ContractId, amount string, expirationLedger uint32) { + r.stored = append(r.stored, history.ContractAssetBalance{ + KeyHash: keyHash[:], + ContractID: contractID[:], + Amount: amount, + ExpirationLedger: expirationLedger, }) - assert.NoError(t, err) +} - keyHash = getKeyHashForBalance(t, etherID, [32]byte{}) - expectedBalances[keyHash] = "50" - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(etherID, [32]byte{}, 200), - }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(etherID, [32]byte{}, 50), - }, - }) - assert.NoError(t, err) +func (r *reconcileTest) reconcile() ([]history.ContractAssetBalance, []xdr.Hash) { + ctx := context.Background() + r.mockQ.On("GetContractAssetBalances", ctx, r.keys).Return(r.stored, nil).Once() + upserts, removals, err := r.set.reconcileBalances(ctx) + assert.NoError(r.t, err) + assert.Empty(r.t, r.set.createdAssetContracts) + r.mockQ.AssertExpectations(r.t) + return upserts, removals +} - // negative balances will be ignored - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(etherID, [32]byte{}, 200), - }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceInt128ToContractData(etherID, [32]byte{1}, xdr.Int128Parts{Hi: -1, Lo: 0}), - }, - }) - assert.NoError(t, err) +func TestReconcileBalanceAmounts(t *testing.T) { + usdcID := assetContractID(t, "USDC", keypair.MustRandom().Address()) + etherIssuer := keypair.MustRandom().Address() + etherID := assetContractID(t, "ETHER", etherIssuer) + btcID := assetContractID(t, "BTC", etherIssuer) + uniID := assetContractID(t, "UNI", etherIssuer) + r := newReconcileTest(t) + + // the amount and ttl change + usdcKeyHash := r.update(usdcID, [32]byte{}, + sac.BalanceToContractData(usdcID, [32]byte{}, 50), + sac.BalanceToContractData(usdcID, [32]byte{}, 100), + ) + r.set.updatedExpirationEntries[usdcKeyHash] = [2]uint32{160, 170} + r.store(usdcKeyHash, usdcID, "50", 160) - // negative balances will be ignored - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceInt128ToContractData(etherID, [32]byte{1}, xdr.Int128Parts{Hi: -1, Lo: 0}), - }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(etherID, [32]byte{}, 200), - }, - }) - assert.NoError(t, err) + // the amount changes and the ttl is kept from the stored row + usdcKeyHash2 := r.update(usdcID, [32]byte{2}, + sac.BalanceToContractData(usdcID, [32]byte{2}, 30), + sac.BalanceToContractData(usdcID, [32]byte{2}, 100), + ) + r.store(usdcKeyHash2, usdcID, "30", 165) - // balances where the amount doesn't change will be ignored - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(btcID, [32]byte{2}, 300), + etherKeyHash := r.update(etherID, [32]byte{}, + sac.BalanceToContractData(etherID, [32]byte{}, 200), + sac.BalanceToContractData(etherID, [32]byte{}, 50), + ) + r.store(etherKeyHash, etherID, "200", 165) + + // balances where the amount and ttl don't change are ignored + btcKeyHash := r.update(btcID, [32]byte{2}, + sac.BalanceToContractData(btcID, [32]byte{2}, 300), + sac.BalanceToContractData(btcID, [32]byte{2}, 300), + ) + r.store(btcKeyHash, btcID, "300", 165) + + uniKeyHash := r.update(uniID, [32]byte{4}, + sac.BalanceToContractData(uniID, [32]byte{4}, 50), + sac.BalanceToContractData(uniID, [32]byte{4}, 75), + ) + r.set.updatedExpirationEntries[uniKeyHash] = [2]uint32{150, 170} + r.store(uniKeyHash, uniID, "50", 150) + + upserts, removals := r.reconcile() + assert.Empty(t, removals) + assert.Equal(t, []history.ContractAssetBalance{ + {KeyHash: usdcKeyHash[:], ContractID: usdcID[:], Amount: "100", ExpirationLedger: 170}, + {KeyHash: usdcKeyHash2[:], ContractID: usdcID[:], Amount: "100", ExpirationLedger: 165}, + {KeyHash: etherKeyHash[:], ContractID: etherID[:], Amount: "50", ExpirationLedger: 165}, + {KeyHash: uniKeyHash[:], ContractID: uniID[:], Amount: "75", ExpirationLedger: 170}, + }, upserts) + assert.ElementsMatch(t, r.set.contractAssetStats.rows(), []history.ContractAssetStatRow{ + { + ContractID: usdcID[:], + Stat: history.ContractStat{ActiveBalance: "120", ActiveHolders: 0}, }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(btcID, [32]byte{2}, 300), + { + ContractID: etherID[:], + Stat: history.ContractStat{ActiveBalance: "-150", ActiveHolders: 0}, + }, + { + ContractID: uniID[:], + Stat: history.ContractStat{ActiveBalance: "25", ActiveHolders: 0}, }, }) - assert.NoError(t, err) +} - keyHash = getKeyHashForBalance(t, uniID, [32]byte{4}) - set.updatedExpirationEntries[keyHash] = [2]uint32{150, 170} - expectedBalances[keyHash] = "75" - err = set.AddContractData(ingest.Change{ - Type: xdr.LedgerEntryTypeContractData, - Pre: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(uniID, [32]byte{4}, 50), +// TestReconcileBalanceValues covers updates which change whether the value of an +// entry is a balance. The row follows the latest value, whatever the previous one was. +func TestReconcileBalanceValues(t *testing.T) { + etherIssuer := keypair.MustRandom().Address() + etherID := assetContractID(t, "ETHER", etherIssuer) + btcID := assetContractID(t, "BTC", etherIssuer) + negative := xdr.Int128Parts{Hi: -1, Lo: 0} + r := newReconcileTest(t) + + // a value which is not a valid balance removes the row + etherKeyHash := r.update(etherID, [32]byte{1}, + sac.BalanceToContractData(etherID, [32]byte{1}, 40), + sac.BalanceInt128ToContractData(etherID, [32]byte{1}, negative), + ) + r.store(etherKeyHash, etherID, "40", 165) + + // a valid balance replacing a value which was not one is stored when its ttl is known + btcKeyHash := r.update(btcID, [32]byte{5}, + sac.BalanceInt128ToContractData(btcID, [32]byte{5}, negative), + sac.BalanceToContractData(btcID, [32]byte{5}, 10), + ) + r.set.updatedExpirationEntries[btcKeyHash] = [2]uint32{160, 180} + + // and skipped when it is not + r.update(btcID, [32]byte{6}, + sac.BalanceInt128ToContractData(btcID, [32]byte{6}, negative), + sac.BalanceToContractData(btcID, [32]byte{6}, 10), + ) + + upserts, removals := r.reconcile() + assert.Equal(t, []xdr.Hash{etherKeyHash}, removals) + assert.Equal(t, []history.ContractAssetBalance{ + {KeyHash: btcKeyHash[:], ContractID: btcID[:], Amount: "10", ExpirationLedger: 180}, + }, upserts) + assert.ElementsMatch(t, r.set.contractAssetStats.rows(), []history.ContractAssetStatRow{ + { + ContractID: etherID[:], + Stat: history.ContractStat{ActiveBalance: "-40", ActiveHolders: -1}, }, - Post: &xdr.LedgerEntry{ - Data: sac.BalanceToContractData(uniID, [32]byte{4}, 75), + { + ContractID: btcID[:], + Stat: history.ContractStat{ActiveBalance: "10", ActiveHolders: 1}, }, }) - assert.NoError(t, err) +} - assert.Empty(t, set.createdAssetContracts) - assert.Empty(t, set.removedBalances) - assert.Empty(t, set.createdExpirationEntries) - for key, amt := range set.updatedBalances { - assert.Equal(t, expectedBalances[key], amt.String()) - delete(expectedBalances, key) - } - assert.Empty(t, expectedBalances) +// TestReconcileBalanceContract covers a stored row whose contract is not the contract +// of its ledger key. The row is rewritten, and its balance moves to the stats of the +// contract of the key. +func TestReconcileBalanceContract(t *testing.T) { + usdcID := assetContractID(t, "USDC", keypair.MustRandom().Address()) + otherID := assetContractID(t, "EURC", keypair.MustRandom().Address()) + r := newReconcileTest(t) + + keyHash := r.update(usdcID, [32]byte{1}, + sac.BalanceToContractData(usdcID, [32]byte{1}, 50), + sac.BalanceToContractData(usdcID, [32]byte{1}, 50), + ) + r.store(keyHash, otherID, "50", 160) - result := set.GetContractStats() - assert.ElementsMatch(t, result, []history.ContractAssetStatRow{ + upserts, removals := r.reconcile() + assert.Empty(t, removals) + assert.Equal(t, []history.ContractAssetBalance{ + {KeyHash: keyHash[:], ContractID: usdcID[:], Amount: "50", ExpirationLedger: 160}, + }, upserts) + assert.ElementsMatch(t, r.set.contractAssetStats.rows(), []history.ContractAssetStatRow{ { ContractID: usdcID[:], - Stat: history.ContractStat{ - ActiveBalance: "120", - ActiveHolders: 0, - }, + Stat: history.ContractStat{ActiveBalance: "50", ActiveHolders: 1}, }, { - ContractID: etherID[:], - Stat: history.ContractStat{ - ActiveBalance: "-150", - ActiveHolders: 0, - }, - }, - { - ContractID: uniID[:], - Stat: history.ContractStat{ - ActiveBalance: "25", - ActiveHolders: 0, - }, + ContractID: otherID[:], + Stat: history.ContractStat{ActiveBalance: "-50", ActiveHolders: -1}, }, }) - - mockQ.AssertExpectations(t) } func TestRemoveContractData(t *testing.T) { @@ -420,9 +475,6 @@ func TestRemoveContractData(t *testing.T) { set := NewContractAssetStatSet( mockQ, "passphrase", - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, 150, ) @@ -452,8 +504,6 @@ func TestRemoveContractData(t *testing.T) { }, }) assert.NoError(t, err) - - assert.Equal(t, []xdr.Hash{keyHash, keyHash1, keyHash2}, set.removedBalances) assert.Empty(t, set.createdAssetContracts) // keyHash1 has no stored row, so it contributes nothing to the stats @@ -463,9 +513,12 @@ func TestRemoveContractData(t *testing.T) { {KeyHash: keyHash[:], ContractID: usdcID[:], Amount: "50", ExpirationLedger: 170}, {KeyHash: keyHash2[:], ContractID: usdcID[:], Amount: "34", ExpirationLedger: 170}, }, nil).Once() - assert.NoError(t, set.ingestRemovedBalances(ctx)) + upserts, removals, err := set.reconcileBalances(ctx) + assert.NoError(t, err) + assert.Empty(t, upserts) + assert.Equal(t, []xdr.Hash{keyHash, keyHash2}, removals) - assert.ElementsMatch(t, set.GetContractStats(), []history.ContractAssetStatRow{ + assert.ElementsMatch(t, set.contractAssetStats.rows(), []history.ContractAssetStatRow{ { ContractID: usdcID[:], Stat: history.ContractStat{ @@ -492,19 +545,14 @@ func TestIngestExpiredBalances(t *testing.T) { set := NewContractAssetStatSet( mockQ, "passphrase", - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, 150, ) usdcKeyHash := getKeyHashForBalance(t, usdcID, [32]byte{}) usdcKeyHash1 := getKeyHashForBalance(t, usdcID, [32]byte{1}) ethKeyHash := getKeyHashForBalance(t, etherID, [32]byte{}) - ethKeyHash1 := getKeyHashForBalance(t, etherID, [32]byte{1}) - set.updatedExpirationEntries[ethKeyHash1] = [2]uint32{149, 180} ctx := context.Background() - mockQ.On("DeleteContractAssetBalancesExpiringAt", ctx, set.currentLedger-1). + mockQ.On("DeleteContractAssetBalancesExpiredBefore", ctx, set.currentLedger). Return([]history.ContractAssetBalance{ { KeyHash: usdcKeyHash[:], @@ -516,7 +564,7 @@ func TestIngestExpiredBalances(t *testing.T) { KeyHash: usdcKeyHash1[:], ContractID: usdcID[:], Amount: "200", - ExpirationLedger: 149, + ExpirationLedger: 120, }, { KeyHash: ethKeyHash[:], @@ -524,16 +572,10 @@ func TestIngestExpiredBalances(t *testing.T) { Amount: "8", ExpirationLedger: 149, }, - { - KeyHash: ethKeyHash1[:], - ContractID: etherID[:], - Amount: "67", - ExpirationLedger: 149, - }, }, nil).Once() assert.NoError(t, set.ingestExpiredBalances(ctx)) - assert.ElementsMatch(t, set.GetContractStats(), []history.ContractAssetStatRow{ + assert.ElementsMatch(t, set.contractAssetStats.rows(), []history.ContractAssetStatRow{ { ContractID: usdcID[:], Stat: history.ContractStat{ diff --git a/internal/ingest/verify.go b/internal/ingest/verify.go index 3418843d4..9c24a1029 100644 --- a/internal/ingest/verify.go +++ b/internal/ingest/verify.go @@ -354,8 +354,7 @@ func (s *system) verifyState(verifyAgainstLatestCheckpoint bool, checkpointSeque verifier := NewStateVerifier(stateReader) assetStats := processors.NewAssetStatSet() - createdExpirationEntries := map[xdr.Hash]uint32{} - var contractDataEntries []xdr.LedgerEntry + contractAssetStatSet := processors.NewContractAssetStatSet(historyQ, s.config.NetworkPassphrase, ledgerSequence) total := int64(0) for { var entries []xdr.LedgerEntry @@ -409,7 +408,12 @@ func (s *system) verifyState(verifyAgainstLatestCheckpoint bool, checkpointSeque if err = verifier.Write(entry); err != nil { return err } - contractDataEntries = append(contractDataEntries, entry) + if err = contractAssetStatSet.AddContractData(ingestsdk.Change{ + Type: xdr.LedgerEntryTypeContractData, + Post: &entry, + }); err != nil { + return errors.Wrap(err, "Error ingesting contract data") + } totalByType["contract_data"]++ case xdr.LedgerEntryTypeTtl: // we don't store all expiration entries in the db, @@ -418,8 +422,12 @@ func (s *system) verifyState(verifyAgainstLatestCheckpoint bool, checkpointSeque return err } totalByType["ttl"]++ - ttl := entry.Data.MustTtl() - createdExpirationEntries[ttl.KeyHash] = uint32(ttl.LiveUntilLedgerSeq) + if err = contractAssetStatSet.AddTTL(ingestsdk.Change{ + Type: xdr.LedgerEntryTypeTtl, + Post: &entry, + }); err != nil { + return errors.Wrap(err, "Error ingesting ttl") + } totalByType["expiration"]++ default: return errors.New("GetLedgerEntries return unexpected type") @@ -460,24 +468,6 @@ func (s *system) verifyState(verifyAgainstLatestCheckpoint bool, checkpointSeque localLog.WithField("total", total).Info("Batch added to StateVerifier") } - contractAssetStatSet := processors.NewContractAssetStatSet( - historyQ, - s.config.NetworkPassphrase, - map[xdr.Hash]uint32{}, - createdExpirationEntries, - map[xdr.Hash][2]uint32{}, - ledgerSequence, - ) - for i := range contractDataEntries { - entry := contractDataEntries[i] - if err = contractAssetStatSet.AddContractData(ingestsdk.Change{ - Type: xdr.LedgerEntryTypeContractData, - Post: &entry, - }); err != nil { - return errors.Wrap(err, "Error ingesting contract data") - } - } - localLog.WithField("total", total).Info("Finished writing to StateVerifier") countAccounts, err := historyQ.CountAccounts(ctx) @@ -528,23 +518,35 @@ func (s *system) verifyState(verifyAgainstLatestCheckpoint bool, checkpointSeque return nil } +// checkAssetStats compares the asset stats in the db with the ones computed from the +// history archive. Any difference, in the classic asset stats or in the data derived +// from Stellar Asset Contract ledger entries (asset contracts, contract balances and +// contract stats), is a StateError. Only the balances of Stellar Asset Contracts are +// compared, because any contract can write entries which look like contract balances. func checkAssetStats( ctx context.Context, set processors.AssetStatSet, contractAssetStatSet *processors.ContractAssetStatSet, q history.IngestionQ, ) error { - all, balances, err := extractAssetStatsAndBalances(set, contractAssetStatSet) + classicStats := map[string]history.ExpAssetStat{} + for _, assetStat := range set.All() { + // no need to handle the native asset because asset stats only + // include non-native assets. + classicStats[classicAssetStatKey(assetStat)] = assetStat + } + contractStats, balances, err := extractContractAssetStatsAndBalances(contractAssetStatSet) if err != nil { return err } + var assetStats []history.AssetAndContractStat page := db2.PageQuery{ Order: "asc", Limit: assetStatsBatchSize, } for { - assetStats, err := q.GetAssetStats(ctx, "", "", page) + assetStats, err = q.GetAssetStats(ctx, "", "", page) if err != nil { return errors.Wrap(err, "could not fetch asset stats from db") } @@ -554,185 +556,225 @@ func checkAssetStats( for _, assetStat := range assetStats { key := assetStat.AssetCode + ":" + assetStat.AssetIssuer - fromSet, ok := all[key] - if !ok { - return ingestsdk.NewStateError( - fmt.Errorf( - "db contains asset stat with code %s issuer %s which is missing from HAS", - assetStat.AssetCode, assetStat.AssetIssuer, - ), - ) + if err = checkClassicAssetStat(classicStats, assetStat); err != nil { + return err } - delete(all, key) - - if !fromSet.Equals(assetStat) { - return ingestsdk.NewStateError( - fmt.Errorf( - "db asset stat with code %s issuer %s does not match asset stat from HAS: expected=%v actual=%v", - assetStat.AssetCode, assetStat.AssetIssuer, fromSet, assetStat, - ), - ) + if err = checkContractAssetStat(contractStats, key, assetStat); err != nil { + return err } } page.Cursor = assetStats[len(assetStats)-1].PagingToken() } - if len(all) > 0 { + if len(classicStats) > 0 { return ingestsdk.NewStateError( fmt.Errorf( "HAS contains %d more asset stats than db", - len(all), + len(classicStats), ), ) } + if len(contractStats) > 0 { + return ingestsdk.NewStateError( + fmt.Errorf("HAS contains %d asset contracts which are missing from db", len(contractStats)), + ) + } + return checkContractBalances(ctx, balances, q) +} - if err := checkContractBalances(ctx, balances, q); err != nil { - return err +// emptyClassicBalances are the classic balances of the db row of an asset which only +// has an asset contract, and no classic asset stat row. +var emptyClassicBalances = history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + ClaimableBalances: "0", + LiquidityPools: "0", + Unauthorized: "0", +} + +// classicAssetStatKey returns the key of a classic asset stat. It includes the asset type, +// like the join of GetAssetStats, so that the db row of an asset contract whose asset +// type differs from the classic asset stat of its asset is never matched with it. +func classicAssetStatKey(assetStat history.ExpAssetStat) string { + return fmt.Sprintf("%d:%s:%s", assetStat.AssetType, assetStat.AssetCode, assetStat.AssetIssuer) +} + +// checkClassicAssetStat compares the classic asset stat in a db row with the one +// computed from the history archive, and removes the latter from classicStats. +func checkClassicAssetStat( + classicStats map[string]history.ExpAssetStat, + assetStat history.AssetAndContractStat, +) error { + key := classicAssetStatKey(assetStat.ExpAssetStat) + fromSet, ok := classicStats[key] + if !ok { + if assetStat.ContractID != nil && assetStat.Accounts.IsZero() && assetStat.Balances == emptyClassicBalances { + return nil + } + return ingestsdk.NewStateError( + fmt.Errorf( + "db contains asset stat with code %s issuer %s which is missing from HAS", + assetStat.AssetCode, assetStat.AssetIssuer, + ), + ) + } + delete(classicStats, key) + + if !fromSet.Equals(assetStat.ExpAssetStat) { + return ingestsdk.NewStateError( + fmt.Errorf( + "db asset stat with code %s issuer %s does not match asset stat from HAS: expected=%v actual=%v", + assetStat.AssetCode, assetStat.AssetIssuer, fromSet, assetStat.ExpAssetStat, + ), + ) } return nil } -func extractAssetStatsAndBalances(set processors.AssetStatSet, contractAssetStatSet *processors.ContractAssetStatSet) (map[string]history.AssetAndContractStat, []history.ContractAssetBalance, error) { - all := map[string]history.AssetAndContractStat{} - for _, assetStat := range set.All() { - // no need to handle the native asset because asset stats only - // include non-native assets. - all[assetStat.AssetCode+":"+assetStat.AssetIssuer] = history.AssetAndContractStat{ - ExpAssetStat: assetStat, - Contracts: history.ContractStat{ - ActiveBalance: "0", - ActiveHolders: 0, - }, - } +// checkContractAssetStat compares the asset contract and contract stats in a db row +// with the ones computed from the history archive, and removes the latter from +// contractStats. +func checkContractAssetStat( + contractStats map[string]expectedContractAssetStat, + key string, + assetStat history.AssetAndContractStat, +) error { + if assetStat.ContractID == nil { + // an asset contract which is missing from the db is reported once + // all the rows in the db have been checked + return nil + } + fromSet, ok := contractStats[key] + if !ok { + return ingestsdk.NewStateError(errors.Errorf("db contains asset contract of %s which is missing from HAS", key)) + } + delete(contractStats, key) + + if !bytes.Equal(fromSet.contractID, *assetStat.ContractID) { + return ingestsdk.NewStateError(errors.Errorf( + "asset contract of %s is %x in HAS but %x in db", key, fromSet.contractID, *assetStat.ContractID, + )) + } + if fromSet.assetType != assetStat.AssetType { + return ingestsdk.NewStateError(errors.Errorf( + "asset contract of %s has type %v in HAS but %v in db", key, fromSet.assetType, assetStat.AssetType, + )) + } + if fromSet.stat != assetStat.Contracts { + return ingestsdk.NewStateError(errors.Errorf( + "contract stats of %s are %+v in HAS but %+v in db", key, fromSet.stat, assetStat.Contracts, + )) } + return nil +} - contractToStats := map[xdr.ContractId]history.ContractAssetStatRow{} - for _, row := range contractAssetStatSet.GetContractStats() { +// expectedContractAssetStat is the asset contract, asset type and contract stats of an +// asset, computed from the history archive. +type expectedContractAssetStat struct { + contractID []byte + assetType xdr.AssetType + stat history.ContractStat +} + +// extractContractAssetStatsAndBalances returns, by asset, the asset contract and +// contract stats computed from the history archive, along with the balances held in +// those asset contracts. Balances of other contracts are left out: any contract can +// write entries which look like Stellar Asset Contract balances, and horizon does not +// serve them. +func extractContractAssetStatsAndBalances( + contractAssetStatSet *processors.ContractAssetStatSet, +) (map[string]expectedContractAssetStat, []history.ContractAssetBalance, error) { + createdBalances, contractStatRows := contractAssetStatSet.GetCreatedBalancesAndStats() + contractToStats := map[xdr.ContractId]history.ContractStat{} + for _, row := range contractStatRows { var contractID xdr.ContractId copy(contractID[:], row.ContractID) - contractToStats[contractID] = row + contractToStats[contractID] = row.Stat } assetContracts, err := contractAssetStatSet.GetCreatedAssetContracts() if err != nil { return nil, nil, errors.Wrap(err, "Error getting created asset contracts") } + contractStats := map[string]expectedContractAssetStat{} + assetContractIDs := map[xdr.ContractId]bool{} for _, assetContract := range assetContracts { - key := assetContract.AssetCode + ":" + assetContract.AssetIssuer - entry, ok := all[key] - if !ok { - assetType := xdr.AssetTypeAssetTypeCreditAlphanum4 - if len(assetContract.AssetCode) > 4 { - assetType = xdr.AssetTypeAssetTypeCreditAlphanum12 - } - entry = history.AssetAndContractStat{ - ExpAssetStat: history.ExpAssetStat{ - AssetType: assetType, - AssetCode: assetContract.AssetCode, - AssetIssuer: assetContract.AssetIssuer, - Accounts: history.ExpAssetStatAccounts{}, - Balances: history.ExpAssetStatBalances{ - Authorized: "0", - AuthorizedToMaintainLiabilities: "0", - ClaimableBalances: "0", - LiquidityPools: "0", - Unauthorized: "0", - }, - }, - } - } - contractID := assetContract.ContractID - entry.ContractID = &contractID - var contractIDHash xdr.ContractId - copy(contractIDHash[:], assetContract.ContractID) - contractStats, ok := contractToStats[contractIDHash] + var contractID xdr.ContractId + copy(contractID[:], assetContract.ContractID) + assetContractIDs[contractID] = true + + stat, ok := contractToStats[contractID] if !ok { - entry.Contracts = history.ContractStat{ + stat = history.ContractStat{ ActiveBalance: "0", ActiveHolders: 0, } - } else { - entry.Contracts = contractStats.Stat } - all[key] = entry + contractStats[assetContract.AssetCode+":"+assetContract.AssetIssuer] = expectedContractAssetStat{ + contractID: assetContract.ContractID, + assetType: assetContract.AssetType, + stat: stat, + } } - // only check contract asset balances which belong to stellar asset contracts - // because other balances may be forged. - var filteredBalances []history.ContractAssetBalance - for _, balance := range contractAssetStatSet.GetCreatedBalances() { + var balances []history.ContractAssetBalance + for _, balance := range createdBalances { var contractID xdr.ContractId copy(contractID[:], balance.ContractID) - if _, ok := contractToStats[contractID]; ok { - filteredBalances = append(filteredBalances, balance) + if assetContractIDs[contractID] { + balances = append(balances, balance) } } - return all, filteredBalances, nil + return contractStats, balances, nil } +// checkContractBalances compares the given balances, computed from the history archive, +// with the rows in the db, and returns the first difference. func checkContractBalances( ctx context.Context, balances []history.ContractAssetBalance, q history.IngestionQ, ) error { for i := 0; i < len(balances); { - end := i + assetStatsBatchSize - if end > len(balances) { - end = len(balances) - } - + end := min(i+assetStatsBatchSize, len(balances)) subset := balances[i:end] - var keys []xdr.Hash - byKey := map[xdr.Hash]history.ContractAssetBalance{} + keys := make([]xdr.Hash, 0, len(subset)) for _, balance := range subset { var key xdr.Hash copy(key[:], balance.KeyHash) keys = append(keys, key) - byKey[key] = balance } rows, err := q.GetContractAssetBalances(ctx, keys) if err != nil { return err } - + byKey := make(map[xdr.Hash]history.ContractAssetBalance, len(rows)) for _, row := range rows { var key xdr.Hash copy(key[:], row.KeyHash) - expected := byKey[key] - - if !bytes.Equal(row.ContractID, expected.ContractID) { - return ingestsdk.NewStateError( - fmt.Errorf( - "contract balance %v has contract %v in HAS but is %v in db", - key, - expected.ContractID, - row.ContractID, - ), - ) - } - - if row.ExpirationLedger != expected.ExpirationLedger { - return ingestsdk.NewStateError( - fmt.Errorf( - "contract balance %v has expiration %v in HAS but is %v in db", - key, - expected.ExpirationLedger, - row.ExpirationLedger, - ), - ) - } + byKey[key] = row + } - if row.Amount != expected.Amount { - return ingestsdk.NewStateError( - fmt.Errorf( - "contract balance %v has amount %v in HAS but is %v in db", - key, - expected.Amount, - row.Amount, - ), - ) + for j, expected := range subset { + key := keys[j] + row, ok := byKey[key] + switch { + case !ok: + return ingestsdk.NewStateError(errors.Errorf("contract balance %x is missing from db", key)) + case !bytes.Equal(row.ContractID, expected.ContractID): + return ingestsdk.NewStateError(errors.Errorf( + "contract balance %x has contract %x in HAS but %x in db", key, expected.ContractID, row.ContractID, + )) + case row.ExpirationLedger != expected.ExpirationLedger: + return ingestsdk.NewStateError(errors.Errorf( + "contract balance %x has expiration %v in HAS but %v in db", key, expected.ExpirationLedger, row.ExpirationLedger, + )) + case row.Amount != expected.Amount: + return ingestsdk.NewStateError(errors.Errorf( + "contract balance %x has amount %v in HAS but %v in db", key, expected.Amount, row.Amount, + )) } } diff --git a/internal/ingest/verify_range_state_test.go b/internal/ingest/verify_range_state_test.go index 814f8eddd..3694df3a1 100644 --- a/internal/ingest/verify_range_state_test.go +++ b/internal/ingest/verify_range_state_test.go @@ -619,9 +619,6 @@ func (s *VerifyRangeStateTestSuite) TestVerifyFailsWhenAssetStatsMismatch() { contractAssetStatsSet := processors.NewContractAssetStatSet( s.historyQ, s.system.config.NetworkPassphrase, - map[xdr.Hash]uint32{}, - map[xdr.Hash]uint32{}, - map[xdr.Hash][2]uint32{}, 100, ) diff --git a/internal/integration/ingestion_filtering_test.go b/internal/integration/ingestion_filtering_test.go index d96c6647e..65678169e 100644 --- a/internal/integration/ingestion_filtering_test.go +++ b/internal/integration/ingestion_filtering_test.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/go-stellar-sdk/clients/horizonclient" hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon" "github.com/stellar/go-stellar-sdk/txnbuild" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-horizon/internal/ingest/filters" "github.com/stellar/stellar-horizon/internal/test/integration" ) @@ -168,6 +169,51 @@ func TestFilteringAssetWhiteList(t *testing.T) { ) _, err = itest.Client().TransactionDetail(txResp.Hash) tt.NoError(err) + + // Offers so a path payment can route SEK -> PTS -> XLM, and SEK to spend. + itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), + &txnbuild.ManageSellOffer{ + Selling: whitelistedAsset, + Buying: nonWhitelistedAsset, + Amount: "100", + Price: xdr.Price{N: 1, D: 1}, + }, + &txnbuild.ManageSellOffer{ + Selling: txnbuild.NativeAsset{}, + Buying: whitelistedAsset, + Amount: "100", + Price: xdr.Price{N: 1, D: 1}, + }, + &txnbuild.Payment{ + Destination: defaultAllowedAccount.GetAccountID(), + Amount: "50", + Asset: nonWhitelistedAsset, + }, + ) + + // A path payment that only routes through the whitelisted asset is stored. + txResp = itest.MustSubmitOperations(defaultAllowedAccount, defaultAllowedAccountKey, + &txnbuild.PathPaymentStrictSend{ + SendAsset: nonWhitelistedAsset, + SendAmount: "10", + Destination: itest.Master().Address(), + DestAsset: txnbuild.NativeAsset{}, + DestMin: "1", + Path: []txnbuild.Asset{whitelistedAsset}, + }, + ) + _, err = itest.Client().TransactionDetail(txResp.Hash) + tt.NoError(err) + + // A claim of a claimable balance denominated in the whitelisted asset is + // stored, even though the claim operation only carries the balance id. + claim := itest.MustCreateClaimableBalance(itest.Master(), whitelistedAsset, "10", + txnbuild.NewClaimant(defaultAllowedAccount.GetAccountID(), nil)) + txResp = itest.MustSubmitOperations(defaultAllowedAccount, defaultAllowedAccountKey, + &txnbuild.ClaimClaimableBalance{BalanceID: claim.BalanceID}, + ) + _, err = itest.Client().TransactionDetail(txResp.Hash) + tt.NoError(err) } func TestFilteringAssetAndAccountFilters(t *testing.T) { diff --git a/internal/render/main.go b/internal/render/main.go index 163cf3742..19c5e9264 100644 --- a/internal/render/main.go +++ b/internal/render/main.go @@ -2,28 +2,23 @@ package render import ( "net/http" - - "github.com/adjust/goautoneg" - "github.com/stellar/go-stellar-sdk/support/log" + "strings" ) -// Negotiate inspects the Accept header of the provided request and determines -// what the most appropriate response type should be. Defaults to HAL. +// Negotiate returns the media type to respond to the provided request with: +// the first media type listed in its Accept header that Horizon serves +// (MimeEventStream, MimeRaw, or MimeHal, which also covers application/json), +// or MimeHal if it lists none. Media range parameters, including q-values, are +// ignored. func Negotiate(r *http.Request) string { - ctx := r.Context() - alternatives := []string{MimeHal, MimeJSON, MimeEventStream, MimeRaw} - accept := r.Header.Get("Accept") - - if accept == "" { - return MimeHal + for mediaRange := range strings.SplitSeq(r.Header.Get("Accept"), ",") { + mediaType, _, _ := strings.Cut(mediaRange, ";") + switch mediaType = strings.TrimSpace(mediaType); mediaType { + case MimeEventStream, MimeRaw: + return mediaType + case MimeHal, "application/json": + return MimeHal + } } - - result := goautoneg.Negotiate(r.Header.Get("Accept"), alternatives) - - log.Ctx(ctx).WithFields(log.F{ - "content_type": result, - "accept": accept, - }).Debug("Negotiated content type") - - return result + return MimeHal } diff --git a/internal/render/main_test.go b/internal/render/main_test.go index b9205b585..5abf9b16c 100644 --- a/internal/render/main_test.go +++ b/internal/render/main_test.go @@ -17,14 +17,25 @@ func TestNegotiate(t *testing.T) { Header string ExpectedResponseType string }{ - // Obeys the Accept header's prioritization - {"application/hal+json", MimeHal}, + // Responds with the first media type listed that Horizon serves + {"text/event-stream", MimeEventStream}, {"text/event-stream,application/hal+json", MimeEventStream}, + {"application/hal+json, text/event-stream", MimeHal}, + {"application/json, text/event-stream", MimeHal}, + {"*/*, text/event-stream", MimeEventStream}, + {"application/octet-stream", MimeRaw}, + // Ignores media range parameters, including q-values + {"text/event-stream; charset=utf-8", MimeEventStream}, + {"text/event-stream;q=0.5, application/hal+json", MimeEventStream}, // Defaults to HAL - {"text/event-stream;q=0.5,application/hal+json", MimeHal}, + {"application/hal+json", MimeHal}, + {"application/json, text/plain, */*", MimeHal}, + {"text/plain", MimeHal}, {"", MimeHal}, - // Returns empty string for invalid type - {"text/plain", ""}, + // Matches whole media types only + {"text/event-streaming", MimeHal}, + {"application/octet-stream-custom", MimeHal}, + {"application/json;profile=text/event-stream", MimeHal}, } for _, tc := range testCases { t.Run("", func(t *testing.T) { diff --git a/internal/render/mime.go b/internal/render/mime.go index a4b80f5c5..ec81df119 100644 --- a/internal/render/mime.go +++ b/internal/render/mime.go @@ -6,8 +6,6 @@ const ( MimeEventStream = "text/event-stream" //MimeHal is the mime type for "application/hal+json" MimeHal = "application/hal+json" - //MimeJSON is the mime type for "application/json" - MimeJSON = "application/json" //MimeRaw is the mime type for "application/octet-stream" MimeRaw = "application/octet-stream" )