From e2fb8b068b2e73219f3e2496567547186c293dc5 Mon Sep 17 00:00:00 2001 From: jxom <7336481+jxom@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:47:41 +1000 Subject: [PATCH 1/3] feat(clickhouse): add dex_order_events + dex_fills_enriched for point-in-time fill state Decode OrderPlaced+OrderFlipped into a positioned order-state stream (dex_order_events) and ASOF-join each fill to the latest preceding state in a refreshable MV (dex_fills_enriched), materializing token/isBid/tick, at_peg, price, quote amount and taker-oriented source/destination tokens. Lets Cadent's swaps feed restore at-peg and source/destination-token filters as SQL column reads instead of per-request raw-log replay. Amp-Thread-ID: https://ampcode.com/threads/T-019eb863-5051-7439-a545-3f40e9e65a55 --- .changelog/dex-enriched-fills.md | 5 + db/clickhouse/dex_fills_enriched.sql | 109 ++++++++++++++++++++++ db/clickhouse/dex_order_events.sql | 30 ++++++ db/clickhouse/dex_order_events_select.sql | 46 +++++++++ src/clickhouse_schema/dex.rs | 97 ++++++++++++++++++- 5 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 .changelog/dex-enriched-fills.md create mode 100644 db/clickhouse/dex_fills_enriched.sql create mode 100644 db/clickhouse/dex_order_events.sql create mode 100644 db/clickhouse/dex_order_events_select.sql diff --git a/.changelog/dex-enriched-fills.md b/.changelog/dex-enriched-fills.md new file mode 100644 index 0000000..03c7730 --- /dev/null +++ b/.changelog/dex-enriched-fills.md @@ -0,0 +1,5 @@ +--- +tidx: minor +--- + +Added `dex_order_events` and `dex_fills_enriched`, two ClickHouse objects that resolve each DEX fill's point-in-time order state server-side. `dex_order_events` decodes the union of `OrderPlaced` and `OrderFlipped` into one positioned state stream (the existing `dex_orders` captures `OrderPlaced` only, so it is stale for flip orders). `dex_fills_enriched` is a refreshable materialized view that ASOF-joins each fill to the latest order-state event strictly before it, materializing `token`/`isBid`/`tick`, `at_peg`, `price`, the quote-side amount, and taker-oriented `source_*`/`destination_*` tokens. This lets the swaps feed restore its at-peg and source/destination-token filters as plain SQL aggregates instead of replaying the raw `logs` order-state stream and assembling routes per request. diff --git a/db/clickhouse/dex_fills_enriched.sql b/db/clickhouse/dex_fills_enriched.sql new file mode 100644 index 0000000..06413f8 --- /dev/null +++ b/db/clickhouse/dex_fills_enriched.sql @@ -0,0 +1,109 @@ +-- Point-in-time enriched `OrderFilled` stream, refreshed on a schedule. +-- +-- `OrderFilled` carries no `(token, isBid, tick)` — those are properties of the +-- resting order at the moment of the fill, and for flip orders they change over +-- the order's life (see `dex_order_events`). Consumers (the swaps feed, OHLC, +-- at-peg / token filters) therefore had to replay the raw `logs` order-state +-- stream per request and re-derive each fill's price and orientation in memory. +-- +-- This view resolves that state once, server-side: each fill is ASOF-joined to +-- the latest `dex_order_events` row strictly before its `(block_num, log_idx)` +-- position, exactly mirroring the app's "latest state event before the fill" +-- rule. It then materializes, per fill: +-- * `token` / `isBid` / `tick` — the order state at fill time (flip-correct) +-- * `at_peg` — 1 iff the fill executed at tick 0 (rate == 1) +-- * `price` — quote-per-base implied by the tick: (priceScale + tick) / +-- priceScale, side-independent (priceScale = 100000) +-- * `quote_amount` — the quote-side amount: intDiv(amountFilled * +-- (priceScale + tick), priceScale) +-- * taker-oriented `source_*` / `destination_*` — the token the taker sent +-- and received in this fill. For a maker bid (taker sells base) +-- source = base, destination = quote; for an ask, reversed. +-- These let a swap's route ends and at-peg status be computed +-- as plain SQL aggregates over a `(block_num, tx_hash, taker)` +-- group instead of an app-side chain assembly. +-- +-- Quote resolution joins each book base to its `dex_pairs` row. Genesis system +-- books predate the indexer's `PairCreated` decode and have no `dex_pairs` row, +-- so their `quote_token` (and the side that references it) resolves to '' — +-- best-effort, matching that those books are absent from `dex_pairs`. +-- +-- Why refreshable (full recompute) rather than an insert-time MV: the ASOF join +-- must see the complete order-events stream, but insert-time MV cascade order +-- between sibling views on a single block is not guaranteed, so a flip emitted +-- in the same block as a later fill could be missed. A periodic full recompute +-- always sees a consistent snapshot and, having no incremental state, is +-- reorg-correct by construction (the same reasoning as `dex_ohlc_1m` / +-- `token_balances_snapshot`). Recompute cost and table size are bounded by the +-- rolling retention window below; older fills fall back to scanning `dex_fills` +-- directly. `FINAL` collapses the `ReplacingMergeTree` sources so duplicate +-- pre-merge parts can't double-count. +-- +-- Requires `allow_experimental_refreshable_materialized_view` at creation time; +-- the sink sets it when applying this DDL. +CREATE MATERIALIZED VIEW IF NOT EXISTS dex_fills_enriched +REFRESH EVERY 1 MINUTE +ENGINE = MergeTree +PARTITION BY toYYYYMM(block_timestamp) +ORDER BY (block_num, log_idx) +AS +SELECT + f.block_num AS block_num, + f.block_timestamp AS block_timestamp, + f.tx_idx AS tx_idx, + f.log_idx AS log_idx, + f.tx_hash AS tx_hash, + f.orderId AS orderId, + f.maker AS maker, + f.taker AS taker, + f.amountFilled AS amountFilled, + f.partialFill AS partialFill, + e.token AS token, + e.isBid AS isBid, + e.tick AS tick, + toUInt8(e.tick = 0) AS at_peg, + (toFloat64(100000) + e.tick) / 100000 AS price, + intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000) AS quote_amount, + p.quote AS quote_token, + if(e.isBid = 1, e.token, p.quote) AS source_token, + if( + e.isBid = 1, + f.amountFilled, + intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000) + ) AS source_amount, + if(e.isBid = 1, p.quote, e.token) AS destination_token, + if( + e.isBid = 1, + intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000), + f.amountFilled + ) AS destination_amount +FROM +( + SELECT + block_num, + block_timestamp, + tx_idx, + log_idx, + tx_hash, + orderId, + maker, + taker, + amountFilled, + partialFill, + toUInt64(block_num) * 4294967296 + toUInt64(log_idx) AS pos + FROM dex_fills FINAL + WHERE block_timestamp >= now() - INTERVAL 30 DAY +) AS f +ASOF INNER JOIN +( + SELECT + orderId, + token, + isBid, + tick, + toUInt64(block_num) * 4294967296 + toUInt64(log_idx) AS pos + FROM dex_order_events FINAL +) AS e + ON f.orderId = e.orderId AND f.pos > e.pos +LEFT JOIN dex_pairs AS p FINAL ON p.base = e.token +SETTINGS join_use_nulls = 0 diff --git a/db/clickhouse/dex_order_events.sql b/db/clickhouse/dex_order_events.sql new file mode 100644 index 0000000..80d1b5d --- /dev/null +++ b/db/clickhouse/dex_order_events.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS dex_order_events ( + block_num Int64, + block_timestamp DateTime64(3, 'UTC'), + tx_idx Int32, + log_idx Int32, + tx_hash String, + address String, + orderId UInt256, + maker String, + token String, + amount UInt256, + isBid UInt8, + tick Int16, + eventType String, + + INDEX idx_order_id orderId TYPE bloom_filter GRANULARITY 1, + INDEX idx_maker maker TYPE bloom_filter GRANULARITY 1 +) ENGINE = ReplacingMergeTree() +PARTITION BY toYYYYMM(block_timestamp) +ORDER BY (orderId, block_num, log_idx) +-- Point-in-time order-state stream: the union of `OrderPlaced` and +-- `OrderFlipped` events decoded from the raw `logs` stream, one positioned +-- `(orderId, block_num, log_idx) -> (token, isBid, tick)` row per state change. +-- +-- The decoded `dex_orders` table captures `OrderPlaced` only, so it is stale +-- for flip orders: a T5+ flip order keeps its `orderId` but mutates +-- `(isBid, tick)` on every fill and emits only `OrderFlipped` (no second +-- `OrderPlaced`). Resolving a fill's true price therefore requires the latest +-- state event *before* the fill, which this table makes available as a plain +-- ASOF join (see `dex_fills_enriched`) instead of an app-side replay of `logs`. diff --git a/db/clickhouse/dex_order_events_select.sql b/db/clickhouse/dex_order_events_select.sql new file mode 100644 index 0000000..5d1d5f4 --- /dev/null +++ b/db/clickhouse/dex_order_events_select.sql @@ -0,0 +1,46 @@ +-- Decodes the union of `OrderPlaced(uint128 indexed orderId, +-- address indexed maker, address indexed token, uint128 amount, bool isBid, +-- int16 tick, bool isFlipOrder, int16 flipTick)` (selector 0xc200d837…) and +-- `OrderFlipped(uint128 indexed orderId, address indexed maker, +-- address indexed token, uint128 amount, bool isBid, int16 tick, +-- int16 flipTick)` (selector 0x37a42d10…) from the raw `logs` stream. +-- +-- Both events share the layout of the words read here: orderId/maker/token are +-- indexed topics; `data` word0 is `amount` (low 16 bytes, hex 35..66), word1 is +-- `isBid` (last byte, hex 129..130), word2 is `tick` (int16, sign-extended, +-- last 2 bytes, hex 191..194). int16s read the trailing 2 bytes little-endian +-- so two's-complement negatives decode correctly; the `OrderPlaced`-only tail +-- words (`isFlipOrder`, `flipTick`) are not read, so the shorter `OrderFlipped` +-- payload (4 data words) decodes through the same projection as `OrderPlaced`. +SELECT + block_num, + block_timestamp, + tx_idx, + log_idx, + tx_hash, + address, + reinterpretAsUInt256(reverse(unhex(substring(topic1, 3, 64)))) AS orderId, + concat('0x', lower(substring(topic2, 27))) AS maker, + concat('0x', lower(substring(topic3, 27))) AS token, + reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))) AS amount, + reinterpretAsUInt8(unhex(substring(data, 129, 2))) AS isBid, + reinterpretAsInt16(reverse(unhex(substring(data, 191, 4)))) AS tick, + if( + selector = '0xc200d837816d02c5ee9bf081cba1a32ab1482de7a738b41c0b357186b0b998cd', + 'placed', + 'flipped' + ) AS eventType +FROM logs +WHERE + selector IN ( + '0xc200d837816d02c5ee9bf081cba1a32ab1482de7a738b41c0b357186b0b998cd', + '0x37a42d10bbce3e94e109a6a44e4479f0ee45dd6ecc6ca902168ea58e01ba32fe' + ) + AND address = '0xdec0000000000000000000000000000000000000' + AND topic1 IS NOT NULL + AND topic2 IS NOT NULL + AND topic3 IS NOT NULL + AND length(topic1) >= 66 + AND length(topic2) >= 66 + AND length(topic3) >= 66 + AND length(data) >= 194 diff --git a/src/clickhouse_schema/dex.rs b/src/clickhouse_schema/dex.rs index 442b412..faa6978 100644 --- a/src/clickhouse_schema/dex.rs +++ b/src/clickhouse_schema/dex.rs @@ -6,6 +6,10 @@ const DEX_ORDERS_SCHEMA: &str = include_str!("../../db/clickhouse/dex_orders.sql const DEX_ORDERS_SELECT: &str = include_str!("../../db/clickhouse/dex_orders_select.sql"); const DEX_FILLS_SCHEMA: &str = include_str!("../../db/clickhouse/dex_fills.sql"); const DEX_FILLS_SELECT: &str = include_str!("../../db/clickhouse/dex_fills_select.sql"); +const DEX_ORDER_EVENTS_SCHEMA: &str = include_str!("../../db/clickhouse/dex_order_events.sql"); +const DEX_ORDER_EVENTS_SELECT: &str = + include_str!("../../db/clickhouse/dex_order_events_select.sql"); +const DEX_FILLS_ENRICHED: &str = include_str!("../../db/clickhouse/dex_fills_enriched.sql"); const DEX_OHLC_1M: &str = include_str!("../../db/clickhouse/dex_ohlc_1m.sql"); const DEX_PAIR_LIQUIDITY: &str = include_str!("../../db/clickhouse/dex_pair_liquidity.sql"); @@ -82,6 +86,27 @@ pub const OBJECTS: &[ClickHouseObject] = &[ block_column: None, backfill: None, }, + ClickHouseObject { + name: "dex_order_events", + kind: ClickHouseObjectKind::Table(DEX_ORDER_EVENTS_SCHEMA), + depends_on: &["logs"], + public_query: true, + block_column: Some("block_num"), + backfill: Some(BackfillPolicy::Ranged { + select_sql: DEX_ORDER_EVENTS_SELECT, + }), + }, + ClickHouseObject { + name: "dex_order_events_mv", + kind: ClickHouseObjectKind::MaterializedView { + target_table: "dex_order_events", + select_sql: DEX_ORDER_EVENTS_SELECT, + }, + depends_on: &["logs", "dex_order_events"], + public_query: false, + block_column: None, + backfill: None, + }, ClickHouseObject { name: "dex_pair_liquidity", kind: ClickHouseObjectKind::View(DEX_PAIR_LIQUIDITY), @@ -101,6 +126,17 @@ pub const OBJECTS: &[ClickHouseObject] = &[ block_column: None, backfill: None, }, + ClickHouseObject { + name: "dex_fills_enriched", + kind: ClickHouseObjectKind::RefreshableMaterializedView(DEX_FILLS_ENRICHED), + depends_on: &["dex_fills", "dex_order_events", "dex_pairs"], + public_query: true, + // Self-storing refreshable MV: ASOF-joins each fill to its point-in-time + // order state over a rolling window, so it's fully replaced each refresh + // — reorg-correct by construction, and reorg cleanup skips it. + block_column: None, + backfill: None, + }, ]; #[cfg(test)] @@ -113,7 +149,7 @@ mod tests { #[test] fn decoded_tables_are_block_scoped_and_public() { - for name in ["dex_pairs", "dex_orders", "dex_fills"] { + for name in ["dex_pairs", "dex_orders", "dex_fills", "dex_order_events"] { let table = object(name); assert!(table.is_table(), "{name} should be a table"); assert!(table.public_query, "{name} should be public"); @@ -187,6 +223,65 @@ mod tests { ); } + #[test] + fn order_events_decode_placed_and_flipped_from_logs() { + let mv = object("dex_order_events_mv"); + let ddl = mv.ddl(); + assert!(ddl.starts_with("CREATE MATERIALIZED VIEW IF NOT EXISTS dex_order_events_mv")); + assert!(ddl.contains("TO dex_order_events AS\n")); + assert!(ddl.contains("FROM logs")); + assert!( + ddl.contains("address = '0xdec0000000000000000000000000000000000000'"), + "should only decode DEX precompile logs" + ); + // Unions OrderPlaced (0xc200d837…) and OrderFlipped (0x37a42d10…) so + // flip orders' mutated (isBid, tick) are captured — `dex_orders` misses + // them. Asserted here so an accidental edit can't drop a selector. + for selector in [ + "0xc200d837816d02c5ee9bf081cba1a32ab1482de7a738b41c0b357186b0b998cd", + "0x37a42d10bbce3e94e109a6a44e4479f0ee45dd6ecc6ca902168ea58e01ba32fe", + ] { + assert!(ddl.contains(selector), "should decode selector {selector}"); + } + // Both events share the data-word layout read here (isBid word1, tick + // word2 as a sign-extended int16). + assert!(ddl.contains("reinterpretAsUInt8(unhex(substring(data, 129, 2))) AS isBid")); + assert!( + ddl.contains("reinterpretAsInt16(reverse(unhex(substring(data, 191, 4)))) AS tick") + ); + } + + #[test] + fn enriched_fills_are_reorg_safe_point_in_time_join() { + let enriched = object("dex_fills_enriched"); + assert!(enriched.is_refreshable_materialized_view()); + // Public so Cadent reads flip-correct, pre-oriented fills instead of + // replaying the raw order-state stream; self-storing, so no block_column. + assert!(enriched.public_query); + assert!(enriched.block_column.is_none()); + + let ddl = enriched.ddl(); + assert!(ddl.contains("CREATE MATERIALIZED VIEW IF NOT EXISTS dex_fills_enriched")); + assert!(ddl.contains("REFRESH EVERY")); + // ASOF join resolves each fill's latest state strictly before its + // position, against the placed+flipped events stream. + assert!(ddl.contains("ASOF INNER JOIN")); + assert!(ddl.contains("FROM dex_fills FINAL")); + assert!(ddl.contains("dex_order_events FINAL")); + assert!(ddl.contains("f.orderId = e.orderId AND f.pos > e.pos")); + // Quote side resolved from dex_pairs; genesis books fall back to ''. + assert!(ddl.contains("LEFT JOIN dex_pairs AS p FINAL")); + // Materializes at-peg + price/quote so the filters become column reads. + assert!(ddl.contains("toUInt8(e.tick = 0) AS at_peg")); + assert!(ddl.contains("AS source_token")); + assert!(ddl.contains("AS destination_token")); + assert!(ddl.contains("100000")); + assert_eq!( + enriched.drop_sql().as_deref(), + Some("DROP VIEW IF EXISTS dex_fills_enriched") + ); + } + #[test] fn pair_liquidity_joins_pairs_to_dex_escrow_balances() { let view = object("dex_pair_liquidity"); From a350ea75c8f65bcab66031266c6d429ba16e05b4 Mon Sep 17 00:00:00 2001 From: jxom <7336481+jxom@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:55:57 +1000 Subject: [PATCH 2/3] refactor(clickhouse): make dex_fills_enriched a realtime query-time view Resolve the fill->order-state ASOF join at query time over the already decoded, sort-keyed dex_order_events / dex_fills tables instead of in a refreshable MV. A plain view is realtime (no refresh lag), correct for same-block flips and reorgs (it reads the complete live state), and still far cheaper than the app's per-request raw-logs replay. Validated against ClickHouse 26.5 incl. a same-block flip+fill that an insert-time MV would mis-resolve. Amp-Thread-ID: https://ampcode.com/threads/T-019eb863-5051-7439-a545-3f40e9e65a55 --- .changelog/dex-enriched-fills.md | 2 +- db/clickhouse/dex_fills_enriched.sql | 56 ++++++++++++++-------------- src/clickhouse_schema/dex.rs | 26 ++++++++----- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/.changelog/dex-enriched-fills.md b/.changelog/dex-enriched-fills.md index 03c7730..f2bef7b 100644 --- a/.changelog/dex-enriched-fills.md +++ b/.changelog/dex-enriched-fills.md @@ -2,4 +2,4 @@ tidx: minor --- -Added `dex_order_events` and `dex_fills_enriched`, two ClickHouse objects that resolve each DEX fill's point-in-time order state server-side. `dex_order_events` decodes the union of `OrderPlaced` and `OrderFlipped` into one positioned state stream (the existing `dex_orders` captures `OrderPlaced` only, so it is stale for flip orders). `dex_fills_enriched` is a refreshable materialized view that ASOF-joins each fill to the latest order-state event strictly before it, materializing `token`/`isBid`/`tick`, `at_peg`, `price`, the quote-side amount, and taker-oriented `source_*`/`destination_*` tokens. This lets the swaps feed restore its at-peg and source/destination-token filters as plain SQL aggregates instead of replaying the raw `logs` order-state stream and assembling routes per request. +Added `dex_order_events` and `dex_fills_enriched`, two ClickHouse objects that resolve each DEX fill's point-in-time order state. `dex_order_events` decodes the union of `OrderPlaced` and `OrderFlipped` into one positioned state stream (the existing `dex_orders` captures `OrderPlaced` only, so it is stale for flip orders). `dex_fills_enriched` is a plain view that ASOF-joins each fill to the latest order-state event strictly before it, exposing `token`/`isBid`/`tick`, `at_peg`, `price`, the quote-side amount, and taker-oriented `source_*`/`destination_*` tokens. Resolving the join at query time over the already-decoded, sort-keyed source tables keeps it realtime (no refresh lag) and correct for same-block flips and reorgs, while letting the swaps feed restore its at-peg and source/destination-token filters as plain SQL predicates instead of replaying the raw `logs` order-state stream per request. diff --git a/db/clickhouse/dex_fills_enriched.sql b/db/clickhouse/dex_fills_enriched.sql index 06413f8..b842e1a 100644 --- a/db/clickhouse/dex_fills_enriched.sql +++ b/db/clickhouse/dex_fills_enriched.sql @@ -1,15 +1,15 @@ --- Point-in-time enriched `OrderFilled` stream, refreshed on a schedule. +-- Point-in-time enriched `OrderFilled` stream, resolved at query time. -- -- `OrderFilled` carries no `(token, isBid, tick)` — those are properties of the -- resting order at the moment of the fill, and for flip orders they change over --- the order's life (see `dex_order_events`). Consumers (the swaps feed, OHLC, --- at-peg / token filters) therefore had to replay the raw `logs` order-state --- stream per request and re-derive each fill's price and orientation in memory. +-- the order's life (see `dex_order_events`). Consumers (the swaps feed, at-peg / +-- token filters) otherwise had to replay the raw `logs` order-state stream per +-- request and re-derive each fill's price and orientation in memory. -- --- This view resolves that state once, server-side: each fill is ASOF-joined to --- the latest `dex_order_events` row strictly before its `(block_num, log_idx)` +-- This view resolves that state in ClickHouse: each fill is ASOF-joined to the +-- latest `dex_order_events` row strictly before its `(block_num, log_idx)` -- position, exactly mirroring the app's "latest state event before the fill" --- rule. It then materializes, per fill: +-- rule. It then exposes, per fill: -- * `token` / `isBid` / `tick` — the order state at fill time (flip-correct) -- * `at_peg` — 1 iff the fill executed at tick 0 (rate == 1) -- * `price` — quote-per-base implied by the tick: (priceScale + tick) / @@ -19,34 +19,33 @@ -- * taker-oriented `source_*` / `destination_*` — the token the taker sent -- and received in this fill. For a maker bid (taker sells base) -- source = base, destination = quote; for an ask, reversed. --- These let a swap's route ends and at-peg status be computed --- as plain SQL aggregates over a `(block_num, tx_hash, taker)` --- group instead of an app-side chain assembly. +-- These let a swap's route ends and at-peg status be expressed +-- as plain SQL predicates instead of an app-side chain assembly. -- -- Quote resolution joins each book base to its `dex_pairs` row. Genesis system -- books predate the indexer's `PairCreated` decode and have no `dex_pairs` row, -- so their `quote_token` (and the side that references it) resolves to '' — -- best-effort, matching that those books are absent from `dex_pairs`. -- --- Why refreshable (full recompute) rather than an insert-time MV: the ASOF join --- must see the complete order-events stream, but insert-time MV cascade order --- between sibling views on a single block is not guaranteed, so a flip emitted --- in the same block as a later fill could be missed. A periodic full recompute --- always sees a consistent snapshot and, having no incremental state, is --- reorg-correct by construction (the same reasoning as `dex_ohlc_1m` / --- `token_balances_snapshot`). Recompute cost and table size are bounded by the --- rolling retention window below; older fills fall back to scanning `dex_fills` --- directly. `FINAL` collapses the `ReplacingMergeTree` sources so duplicate --- pre-merge parts can't double-count. +-- Why a plain view (query-time join) rather than a materialized view: +-- * Realtime. A fill is filterable the instant it lands; there is no refresh +-- lag and no rolling-window cutoff, because the view reads the live tables. +-- * Correct. The join sees the complete, ordered, reorg-corrected state at +-- query time, so same-block flips and late/out-of-order events resolve +-- right. An insert-time MV can't: sibling MVs (`dex_fills_mv`, +-- `dex_order_events_mv`) off one `logs` insert have no guaranteed execution +-- order, so a flip emitted in the same block as a later fill could be +-- joined before it exists. A refreshable MV is correct but only as fresh as +-- its last recompute. +-- * Cheap enough. The heavy raw-log decode is already materialized and +-- sort-keyed in `dex_order_events` (by `orderId`) and `dex_fills`, so the +-- per-query cost is an ASOF join over decoded, indexed rows — far below the +-- app's current per-request raw-`logs` replay. Callers scope cost with the +-- usual time / token / pagination predicates, applied in SQL before LIMIT. -- --- Requires `allow_experimental_refreshable_materialized_view` at creation time; --- the sink sets it when applying this DDL. -CREATE MATERIALIZED VIEW IF NOT EXISTS dex_fills_enriched -REFRESH EVERY 1 MINUTE -ENGINE = MergeTree -PARTITION BY toYYYYMM(block_timestamp) -ORDER BY (block_num, log_idx) -AS +-- `FINAL` collapses the `ReplacingMergeTree` sources so reorg-duplicated +-- pre-merge parts can't double-count. +CREATE VIEW IF NOT EXISTS dex_fills_enriched AS SELECT f.block_num AS block_num, f.block_timestamp AS block_timestamp, @@ -92,7 +91,6 @@ FROM partialFill, toUInt64(block_num) * 4294967296 + toUInt64(log_idx) AS pos FROM dex_fills FINAL - WHERE block_timestamp >= now() - INTERVAL 30 DAY ) AS f ASOF INNER JOIN ( diff --git a/src/clickhouse_schema/dex.rs b/src/clickhouse_schema/dex.rs index faa6978..75c33eb 100644 --- a/src/clickhouse_schema/dex.rs +++ b/src/clickhouse_schema/dex.rs @@ -128,12 +128,13 @@ pub const OBJECTS: &[ClickHouseObject] = &[ }, ClickHouseObject { name: "dex_fills_enriched", - kind: ClickHouseObjectKind::RefreshableMaterializedView(DEX_FILLS_ENRICHED), + kind: ClickHouseObjectKind::View(DEX_FILLS_ENRICHED), depends_on: &["dex_fills", "dex_order_events", "dex_pairs"], public_query: true, - // Self-storing refreshable MV: ASOF-joins each fill to its point-in-time - // order state over a rolling window, so it's fully replaced each refresh - // — reorg-correct by construction, and reorg cleanup skips it. + // Plain view: ASOF-joins each fill to its point-in-time order state at + // query time over the already-decoded, sort-keyed source tables. Stores + // nothing, so it's realtime (no refresh lag), reorg-correct (reads live + // tables) and carries no block_column. block_column: None, backfill: None, }, @@ -252,17 +253,24 @@ mod tests { } #[test] - fn enriched_fills_are_reorg_safe_point_in_time_join() { + fn enriched_fills_are_realtime_point_in_time_join() { let enriched = object("dex_fills_enriched"); - assert!(enriched.is_refreshable_materialized_view()); + // Plain view, not a materialized view: the join runs at query time so a + // fill is filterable the instant it lands, and the join always sees the + // complete, reorg-corrected source state (no insert-time ordering hazard, + // no refresh lag). + assert!(enriched.is_view()); // Public so Cadent reads flip-correct, pre-oriented fills instead of - // replaying the raw order-state stream; self-storing, so no block_column. + // replaying the raw order-state stream; stores nothing, so no block_column. assert!(enriched.public_query); assert!(enriched.block_column.is_none()); let ddl = enriched.ddl(); - assert!(ddl.contains("CREATE MATERIALIZED VIEW IF NOT EXISTS dex_fills_enriched")); - assert!(ddl.contains("REFRESH EVERY")); + assert!(ddl.contains("CREATE VIEW IF NOT EXISTS dex_fills_enriched")); + // A plain view stores nothing, so it must not carry materialization or + // refresh clauses. + assert!(!ddl.contains("MATERIALIZED VIEW")); + assert!(!ddl.contains("REFRESH")); // ASOF join resolves each fill's latest state strictly before its // position, against the placed+flipped events stream. assert!(ddl.contains("ASOF INNER JOIN")); From b6be098ddc1499b65fa20be611f6ab2c909d6d33 Mon Sep 17 00:00:00 2001 From: jxom <7336481+jxom@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:18:30 +1000 Subject: [PATCH 3/3] refactor(clickhouse): make dex_fills_enriched book-native Drop the per-fill taker-oriented source_*/destination_* columns. A fill is natively one taker filling one resting maker order on one book, so the view exposes token/quote_token, isBid, tick, at_peg, price and the base/quote amounts (amountFilled/quote_amount). Taker source->destination is a swap-level (route) notion derived during getSwaps assembly, not a per-fill property; removing it also drops the redundant quote_amount/quote_token duplication. Amp-Thread-ID: https://ampcode.com/threads/T-019eb863-5051-7439-a545-3f40e9e65a55 --- .changelog/dex-enriched-fills.md | 2 +- db/clickhouse/dex_fills_enriched.sql | 51 ++++++++++++---------------- src/clickhouse_schema/dex.rs | 16 ++++++--- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/.changelog/dex-enriched-fills.md b/.changelog/dex-enriched-fills.md index f2bef7b..6a0d60f 100644 --- a/.changelog/dex-enriched-fills.md +++ b/.changelog/dex-enriched-fills.md @@ -2,4 +2,4 @@ tidx: minor --- -Added `dex_order_events` and `dex_fills_enriched`, two ClickHouse objects that resolve each DEX fill's point-in-time order state. `dex_order_events` decodes the union of `OrderPlaced` and `OrderFlipped` into one positioned state stream (the existing `dex_orders` captures `OrderPlaced` only, so it is stale for flip orders). `dex_fills_enriched` is a plain view that ASOF-joins each fill to the latest order-state event strictly before it, exposing `token`/`isBid`/`tick`, `at_peg`, `price`, the quote-side amount, and taker-oriented `source_*`/`destination_*` tokens. Resolving the join at query time over the already-decoded, sort-keyed source tables keeps it realtime (no refresh lag) and correct for same-block flips and reorgs, while letting the swaps feed restore its at-peg and source/destination-token filters as plain SQL predicates instead of replaying the raw `logs` order-state stream per request. +Added `dex_order_events` and `dex_fills_enriched`, two ClickHouse objects that resolve each DEX fill's point-in-time order state. `dex_order_events` decodes the union of `OrderPlaced` and `OrderFlipped` into one positioned state stream (the existing `dex_orders` captures `OrderPlaced` only, so it is stale for flip orders). `dex_fills_enriched` is a plain view that ASOF-joins each fill to the latest order-state event strictly before it, exposing each fill book-natively: `token`/`quote_token`, `isBid`, `tick`, `at_peg`, `price`, and the base/quote amounts (`amountFilled`/`quote_amount`). Resolving the join at query time over the already-decoded, sort-keyed source tables keeps it realtime (no refresh lag) and correct for same-block flips and reorgs, letting the swaps feed restore its at-peg and token filters as plain SQL predicates instead of replaying the raw `logs` order-state stream per request. Taker source→destination orientation is left to swap assembly in the API, since it is a swap-level (route) notion rather than a per-fill property. diff --git a/db/clickhouse/dex_fills_enriched.sql b/db/clickhouse/dex_fills_enriched.sql index b842e1a..d880d02 100644 --- a/db/clickhouse/dex_fills_enriched.sql +++ b/db/clickhouse/dex_fills_enriched.sql @@ -4,28 +4,33 @@ -- resting order at the moment of the fill, and for flip orders they change over -- the order's life (see `dex_order_events`). Consumers (the swaps feed, at-peg / -- token filters) otherwise had to replay the raw `logs` order-state stream per --- request and re-derive each fill's price and orientation in memory. +-- request and re-derive each fill's price in memory. -- -- This view resolves that state in ClickHouse: each fill is ASOF-joined to the -- latest `dex_order_events` row strictly before its `(block_num, log_idx)` -- position, exactly mirroring the app's "latest state event before the fill" --- rule. It then exposes, per fill: --- * `token` / `isBid` / `tick` — the order state at fill time (flip-correct) --- * `at_peg` — 1 iff the fill executed at tick 0 (rate == 1) --- * `price` — quote-per-base implied by the tick: (priceScale + tick) / --- priceScale, side-independent (priceScale = 100000) --- * `quote_amount` — the quote-side amount: intDiv(amountFilled * --- (priceScale + tick), priceScale) --- * taker-oriented `source_*` / `destination_*` — the token the taker sent --- and received in this fill. For a maker bid (taker sells base) --- source = base, destination = quote; for an ask, reversed. --- These let a swap's route ends and at-peg status be expressed --- as plain SQL predicates instead of an app-side chain assembly. +-- rule. It exposes each fill the way a fill is natively represented — one taker +-- filling one resting maker order on one book — point-in-time and flip-correct: +-- * `token` / `quote_token` — the book's base and quote tokens +-- * `isBid` — order side: a maker bid pays quote for base, an ask sells base +-- * `tick` — the order tick at fill time +-- * `at_peg` — 1 iff the fill executed at tick 0 (rate == 1) +-- * `price` — quote-per-base implied by the tick: (priceScale + tick) / +-- priceScale, side-independent (priceScale = 100000) +-- * `amountFilled` / `quote_amount` — the base-side and quote-side amounts of +-- the fill (quote = intDiv(amountFilled * (priceScale + tick), +-- priceScale)) +-- +-- Taker source -> destination orientation is intentionally NOT materialized +-- here: it is a swap-level (route) notion, not a fill property. `getSwaps` +-- groups these fills by `(tx, taker)`, orders by `logIndex`, and derives each +-- hop's source/destination from `isBid` (bid: taker sends base, receives quote; +-- ask: reversed) plus the swap's route ends and rate during assembly. -- -- Quote resolution joins each book base to its `dex_pairs` row. Genesis system -- books predate the indexer's `PairCreated` decode and have no `dex_pairs` row, --- so their `quote_token` (and the side that references it) resolves to '' — --- best-effort, matching that those books are absent from `dex_pairs`. +-- so their `quote_token` resolves to '' — best-effort, matching that those +-- books are absent from `dex_pairs`. -- -- Why a plain view (query-time join) rather than a materialized view: -- * Realtime. A fill is filterable the instant it lands; there is no refresh @@ -58,24 +63,12 @@ SELECT f.amountFilled AS amountFilled, f.partialFill AS partialFill, e.token AS token, + p.quote AS quote_token, e.isBid AS isBid, e.tick AS tick, toUInt8(e.tick = 0) AS at_peg, (toFloat64(100000) + e.tick) / 100000 AS price, - intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000) AS quote_amount, - p.quote AS quote_token, - if(e.isBid = 1, e.token, p.quote) AS source_token, - if( - e.isBid = 1, - f.amountFilled, - intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000) - ) AS source_amount, - if(e.isBid = 1, p.quote, e.token) AS destination_token, - if( - e.isBid = 1, - intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000), - f.amountFilled - ) AS destination_amount + intDiv(f.amountFilled * toUInt256(toInt64(100000) + e.tick), 100000) AS quote_amount FROM ( SELECT diff --git a/src/clickhouse_schema/dex.rs b/src/clickhouse_schema/dex.rs index 75c33eb..6bd5632 100644 --- a/src/clickhouse_schema/dex.rs +++ b/src/clickhouse_schema/dex.rs @@ -260,7 +260,7 @@ mod tests { // complete, reorg-corrected source state (no insert-time ordering hazard, // no refresh lag). assert!(enriched.is_view()); - // Public so Cadent reads flip-correct, pre-oriented fills instead of + // Public so Cadent reads flip-correct, book-resolved fills instead of // replaying the raw order-state stream; stores nothing, so no block_column. assert!(enriched.public_query); assert!(enriched.block_column.is_none()); @@ -277,12 +277,18 @@ mod tests { assert!(ddl.contains("FROM dex_fills FINAL")); assert!(ddl.contains("dex_order_events FINAL")); assert!(ddl.contains("f.orderId = e.orderId AND f.pos > e.pos")); - // Quote side resolved from dex_pairs; genesis books fall back to ''. + // Quote token resolved from dex_pairs; genesis books fall back to ''. assert!(ddl.contains("LEFT JOIN dex_pairs AS p FINAL")); - // Materializes at-peg + price/quote so the filters become column reads. + // Book-native fill columns: base/quote tokens, side, at-peg, price and + // the quote-side amount become column reads. Taker source/destination is + // a swap-level (route) notion derived during assembly, not stored here. + assert!(ddl.contains("e.token AS token")); + assert!(ddl.contains("p.quote AS quote_token")); + assert!(ddl.contains("e.isBid AS isBid")); assert!(ddl.contains("toUInt8(e.tick = 0) AS at_peg")); - assert!(ddl.contains("AS source_token")); - assert!(ddl.contains("AS destination_token")); + assert!(ddl.contains("AS quote_amount")); + assert!(!ddl.contains("source_token")); + assert!(!ddl.contains("destination_token")); assert!(ddl.contains("100000")); assert_eq!( enriched.drop_sql().as_deref(),