Merge main into protocol-next

.github/workflows/socket-basics.yml

@@ -0,0 +1,49 @@
+# Socket Basics security scan for stellar-horizon.
+# Upstream: https://github.com/SocketDev/socket-basics
+# Scanner settings live in .socket-basics.json; SAST path exclusions live in
+# .semgrepignore.
+#
+# This workflow does SAST through OpenGrep, secret scanning through TruffleHog,
+# Dockerfile misconfiguration scanning through Trivy, and submits results to
+# Socket.dev
+
+name: Socket Basics security scan
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "47 14 * * 6"
+
+permissions:
+  contents: read
+
+jobs:
+  socket-basics:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          # Remove the GitHub token from the workspace after checkout, so it is
+          # not mounted into the Socket Basics container, which doesn't need it.
+          persist-credentials: false
+
+      - name: Run Socket Basics
+        env:
+          SOCKET_SECURITY_API_TOKEN: ${{ secrets.SOCKET_SECURITY_API_TOKEN }}
+          SOCKET_ORG: stellar
+        run: |
+          # Socket Basics is pinned by digest since rule tuning is calibrated
+          # to the exact ruleset contained in this image. Automatic updates
+          # could introduce new rules and a sudden increase in false
+          # positives, as these tools are generally noisy without tuning.
+          docker run --rm \
+            -v "$PWD:/github/workspace" \
+            -w /github/workspace \
+            -e SOCKET_SECURITY_API_TOKEN \
+            -e SOCKET_ORG \
+            -e GITHUB_REPOSITORY \
+            -e GITHUB_REF_NAME \
+            -e GITHUB_SHA \
+            ghcr.io/socketdev/socket-basics@sha256:d463bae84f21d0240d5e197acb81e95144a966a20ab7cd05cca3a70e1796a4bb \
+            --config .socket-basics.json

.semgrepignore

@@ -0,0 +1,111 @@
+# This file excludes scan paths from Socket Basics SAST scanning (through OpenGrep).
+# It replaces the built-in ignore list for scanning with a more thorough list of paths.
+
+# --- third-party / generated ---
+.git/
+node_modules/
+vendor/
+third_party/
+thirdparty/
+.devcontainer/
+dist/
+build/
+target/
+.venv/
+venv/
+__pycache__/
+.yarn/
+generated/
+*.min.js
+
+# --- test / example / mock code ---
+__fixtures__/
+__mocks__/
+__snapshots__/
+__tests__/
+acceptance-test/
+acceptance-tests/
+acceptance_test/
+acceptance_tests/
+benches/
+browser-test/
+browser-tests/
+browser_test/
+browser_tests/
+e2e/
+e2e-test/
+e2e-tests/
+e2e_test/
+e2e_tests/
+example/
+examples/
+fixtures/
+functional-test/
+functional-tests/
+functional_test/
+functional_tests/
+integration-test/
+integration-tests/
+integration_test/
+integration_tests/
+integrationtest/
+integrationtests/
+mock/
+mock-dapp/
+mocks/
+perf-test/
+perf-tests/
+perf_test/
+perf_tests/
+performance-test/
+performance-tests/
+performance_test/
+performance_tests/
+regression-test/
+regression-tests/
+regression_test/
+regression_tests/
+smoke-test/
+smoke-tests/
+smoke_test/
+smoke_tests/
+spec/
+specs/
+test/
+test-data/
+test-fixtures/
+testFixtures/
+testdata/
+testfixtures/
+tests/
+unit-test/
+unit-tests/
+unit_test/
+unit_tests/
+*.test.js
+*.test.jsx
+*.test.ts
+*.test.tsx
+*.test.mjs
+*.spec.js
+*.spec.jsx
+*.spec.ts
+*.spec.tsx
+*_test.go
+*_test.py
+*_test.rb
+*_test.exs
+test_*.py
+*Test.java
+*Tests.java
+*Test.kt
+*Tests.kt
+*Test.scala
+*Test.cs
+*Tests.cs
+tests.rs
+test.rs
+
+# --- repo-specific third-party (PER_REPO_VENDORED +
+# git submodules); the generic patterns above cannot know these ---
+sac_test/

.socket-basics.json

@@ -0,0 +1,13 @@
+{
+  "workspace": ".",
+  "go_sast_enabled": true,
+  "rust_sast_enabled": true,
+  "go_disabled_rules": "go-error-not-checked",
+  "sast_ignore_overrides": "go-command-execution:internal/scripts/build_release_artifacts/main.go,go-file-path-traversal:internal/flags.go,go-file-path-traversal:internal/ingest/main.go,go-file-path-traversal:internal/ingest/processor_runner.go,go-decompression-bomb:internal/db2/schema/bindata.go,go-profiling-endpoint-exposed:cmd/ingest.go,go-weak-random:internal/ingest/loadtest.go,rust-panic-in-production:internal/integration/contracts/bulk/src/lib.rs,rust-panic-in-production:internal/integration/contracts/increment/src/lib.rs",
+  "secret_scanning_enabled": true,
+  "console_tabular_enabled": true,
+  "dockerfiles": "docker/Dockerfile,docker/Dockerfile.core-testing,docker/Dockerfile.dev,docker/Dockerfile.stable,docker/verify-range/Dockerfile,internal/scripts/check_release_hash/Dockerfile",
+  "socket_tier_1_enabled": false,
+  "trivy_vuln_enabled": false,
+  "trufflehog_exclude_dir": "vendor,target,.git,sac_test"
+}

.trivyignore

@@ -0,0 +1,27 @@
+# This file excludes non-security rules from Socket Basics Dockerfile scanning (through Trivy).
+#
+# Trivy reads this file from the working directory. socket-basics has no config
+# key that can disable a Trivy rule, so this is the only lever.
+#
+# Rules that ARE kept: DS-0002 (no USER - runs as root), DS-0001 (:latest tag),
+# DS-0017 (package-manager update alone, which can install from a stale index).
+
+# Excluded rules:
+
+# 'apt-get' missing '--no-install-recommends'
+DS-0029
+
+# 'RUN cd ...' instead of WORKDIR
+DS-0013
+
+# Deprecated MAINTAINER instruction
+DS-0022
+
+# 'apk add' missing '--no-cache'
+DS-0025
+
+# WORKDIR path not absolute
+DS-0009
+
+# 'apt-get' missing '-y'
+DS-0021

CHANGELOG.md

@@ -9,7 +9,24 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).
 - Added Protocol 30 support and bumped `MaxSupportedProtocolVersion` to 30 ([#237](https://github.com/stellar/stellar-horizon/pull/237)). A Stellar Asset Contract `transfer` or `mint` to a muxed contract (`W...`) gives the `contract_credited` effect `contract_muxed` and `contract_muxed_id` fields.
 
 ### Fixed
+- The asset ingestion filter now matches rules on ledger entry changes and on Soroban data in the envelope, in addition to operation bodies. These transactions are now ingested when they involve a whitelisted asset:
+  - path payments routed through the asset
+  - trustline flag changes and sponsorship revocations of trustlines
+  - calls to the Stellar Asset Contract of the asset, including the contract for native lumens, and deployments of that contract
+  - `ExtendFootprintTTL` and `RestoreFootprint` on entries of that contract
+  - claims and clawbacks of claimable balances, when the transaction succeeds
+  - liquidity pool deposits and withdrawals, when the transaction succeeds
+  - A failed claim, clawback, deposit or withdrawal is still dropped. The operation body carries only a balance id or a pool id, and a failed transaction writes no operation changes. A failed payment or offer in the same asset is kept. So `include_failed` history for an asset depends on the operation type.
+- The admin API rejects malformed asset whitelist entries that are not already stored. It stores each valid entry in canonical form. The response lists every rejected entry.
+- Operators running with an asset filter can re-run historical range ingestion to back-fill transactions that earlier versions skipped. `db reingest range` first deletes all history in the range, then ingests it again through the filter config stored now. Do this only when the filter config has not changed since the range was first ingested. Otherwise transactions the current config does not keep are deleted, and `db detect-gaps` does not report them.
 - The asset ingestion filter now matches rules on the inner operations of fee-bump transactions. Earlier versions collected operations only from TxV0 and Tx envelopes, so a fee-bump transaction was not ingested even when its inner operations referenced a whitelisted asset.
+- The contract balances behind the `contracts` fields of `/assets` are tracked by ledger key, so each stored balance follows the latest value of its ledger entry, and the contract stats are adjusted from the stored balances. Expired contract balances and asset contracts are removed whenever their expiration ledger has passed, rather than only in the ledger right after it.
+
+### Updated
+- SSE streams are now compressed for clients that send `Accept-Encoding: gzip` or `deflate`, like other responses. A 200-event stream is about 80-90% smaller; clients that don't accept compression get the same response as before.
+- Responses are now gzipped for axios 0.x (js-stellar-sdk before 11.0) and Guzzle (PHP SDK) clients even though they send no `Accept-Encoding`, since both always decode gzip. Their large list pages are about 90% smaller.
+- The `method` label of the `horizon_http_requests_duration_seconds`, `horizon_http_requests_in_flight` and `horizon_http_requests_received` metrics reports standard HTTP methods as before and groups any other method under `OTHER`.
+- State verification compares the asset contracts, contract balances and contract stats behind the `contracts` fields of `/assets` with the history archive, only for Stellar Asset Contracts, and a difference marks the ingestion state invalid, like a difference in any other ingested data.
 
 ## 29.0.0
 

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/Masterminds/squirrel v1.5.4
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/aws/aws-sdk-go v1.45.27 // indirect
 	github.com/djherbis/fscache v0.10.1 // indirect

go.sum

@@ -44,8 +44,6 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315 h1:zje9aPr1kQ5nKwjO5MC0S/jehRtNrjfYuLfFRWZH6kY=
-github.com/adjust/goautoneg v0.0.0-20150426214442-d788f35a0315/go.mod h1:4U522XvlkqOY2AVBUM7ISHODDb6tdB+KAXfGaBDsWts=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=

internal/actions/filter_rules.go

@@ -4,11 +4,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
 
 	hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon"
+	"github.com/stellar/go-stellar-sdk/support/collections/set"
 	"github.com/stellar/go-stellar-sdk/support/render/problem"
 	horizonContext "github.com/stellar/stellar-horizon/internal/context"
 	"github.com/stellar/stellar-horizon/internal/db2/history"
+	"github.com/stellar/stellar-horizon/internal/ingest/filters"
 )
 
 // these admin HTTP endpoints are documented in internal/httpx/static/admin_oapi.yml
@@ -76,6 +80,7 @@ func (handler FilterConfigHandler) UpdateAccountConfig(w http.ResponseWriter, r
 	config, err := historyQ.UpdateAccountFilterConfig(r.Context(), filterConfig)
 	if err != nil {
 		problem.Render(r.Context(), w, err)
+		return
 	}
 
 	responsePayload := handler.accountConfigResource(config)
@@ -98,13 +103,26 @@ func (handler FilterConfigHandler) UpdateAssetConfig(w http.ResponseWriter, r *h
 		return
 	}
 
+	stored, err := historyQ.GetAssetFilterConfig(r.Context())
+	if err != nil {
+		problem.Render(r.Context(), w, err)
+		return
+	}
+
+	whitelist, err := canonicalAssetWhitelist(filterRequest.Whitelist, stored.Whitelist)
+	if err != nil {
+		problem.Render(r.Context(), w, err)
+		return
+	}
+
 	filterConfig := history.AssetFilterConfig{}
 	filterConfig.Enabled = *filterRequest.Enabled
-	filterConfig.Whitelist = filterRequest.Whitelist
+	filterConfig.Whitelist = whitelist
 
 	config, err := historyQ.UpdateAssetFilterConfig(r.Context(), filterConfig)
 	if err != nil {
 		problem.Render(r.Context(), w, err)
+		return
 	}
 
 	responsePayload := handler.assetConfigResource(config)
@@ -114,6 +132,40 @@ func (handler FilterConfigHandler) UpdateAssetConfig(w http.ResponseWriter, r *h
 	}
 }
 
+// canonicalAssetWhitelist parses each whitelist entry as a SEP-11 asset
+// ("CODE:ISSUER" or "native") and returns the canonical form the ingestion
+// filter compares against. A new entry that does not parse is rejected, so an
+// entry that could never match is not stored. An entry that is already stored
+// is kept as it is, even when it does not parse: earlier versions stored
+// entries without validation, and a client that reads the config, flips
+// "enabled", and writes it back must not be blocked by such an entry.
+func canonicalAssetWhitelist(entries []string, stored []string) ([]string, error) {
+	storedSet := set.NewSet[string](len(stored))
+	storedSet.AddSlice(stored)
+
+	whitelist := make([]string, 0, len(entries))
+	var invalid []string
+	for _, entry := range entries {
+		asset, err := filters.ParseWhitelistAsset(entry)
+		switch {
+		case err == nil:
+			whitelist = append(whitelist, asset.StringCanonical())
+		case storedSet.Contains(entry):
+			whitelist = append(whitelist, entry)
+		default:
+			invalid = append(invalid, strconv.Quote(entry))
+		}
+	}
+	if len(invalid) > 0 {
+		return nil, problem.NewProblemWithInvalidField(
+			problem.BadRequest,
+			"whitelist",
+			fmt.Errorf("%s: not a valid asset, expected CODE:ISSUER or native", strings.Join(invalid, ", ")),
+		)
+	}
+	return whitelist, nil
+}
+
 func (handler FilterConfigHandler) assetFilterResource(r *http.Request) (hProtocol.AssetFilterConfig, error) {
 	var filterRequest hProtocol.AssetFilterConfig
 	dec := json.NewDecoder(r.Body)