From 77d1da062584d7137410edf74c22766f622fc9fc Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:27:28 +0700 Subject: [PATCH 1/6] Fix Codex token count saturation --- rust/src/codex_costs.rs | 42 ++------- rust/src/core/cost_cache_budget.rs | 16 ++-- rust/src/core/cost_pricing.rs | 27 ++---- rust/src/core/jsonl_scanner.rs | 65 +++++++++---- rust/src/core/jsonl_scanner/codex/helpers.rs | 99 +++++++------------- rust/src/core/jsonl_scanner/codex/parser.rs | 18 ++-- rust/src/core/jsonl_scanner/tests.rs | 75 +++++++++++++++ rust/src/cost_scanner/tests.rs | 4 +- 8 files changed, 192 insertions(+), 154 deletions(-) diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index 4474c30b2b..82e6425e48 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -60,7 +60,7 @@ pub(crate) fn add_codex_records_to_summary( /// Merge billable records into a day→model→`[input,cached,output]` map. pub(crate) fn merge_codex_records_into_days( - days: &mut std::collections::HashMap>>, + days: &mut std::collections::HashMap>>, records: &[CodexUsageRecord], ) { for record in records { @@ -77,7 +77,7 @@ pub(crate) fn merge_codex_records_into_days( pub(crate) fn add_codex_packed_tokens_to_summary( summary: &mut CostSummary, model: &str, - packed: &[i32], + packed: &[i64], pricing_day: Option, ) -> Option { let input = packed.first().copied().unwrap_or(0); @@ -99,7 +99,7 @@ pub(crate) fn add_codex_packed_tokens_to_summary( /// Returns `(session_cost, has_tokens)` — caller adds cost to `total_cost_usd`. pub(crate) fn add_codex_days_map_to_summary( summary: &mut CostSummary, - days: &std::collections::HashMap>>, + days: &std::collections::HashMap>>, range: &CostUsageDayRange, ) -> (f64, bool) { let mut total_cost = 0.0; @@ -153,12 +153,12 @@ struct CodexTokenCounts { } impl CodexTokenCounts { - fn from_values(input: i32, cached: i32, output: i32) -> Self { - let input = input.max(0) as u64; + fn from_values(input: i64, cached: i64, output: i64) -> Self { + let input = u64::try_from(input.max(0)).unwrap_or(0); Self { input, - cached: (cached.max(0) as u64).min(input), - output: output.max(0) as u64, + cached: u64::try_from(cached.max(0)).unwrap_or(0).min(input), + output: u64::try_from(output.max(0)).unwrap_or(0), reasoning: None, } } @@ -404,35 +404,11 @@ fn codex_cost_usd_for_day( let normalized = CostUsagePricing::normalize_codex_model(model); if normalized.contains("fast") || normalized.contains("priority") { - // Fast pricing takes i32 token counts; usage-record counts fit far - // below i32::MAX, and the callee re-checks the long-context threshold - // against the original u64 magnitude. - #[allow( - clippy::cast_possible_truncation, - reason = "token counts from usage records fit i32" - )] - #[allow( - clippy::cast_possible_wrap, - reason = "token counts are non-negative; wrapping is impossible" - )] let fast = pricing_day .and_then(|day| { - CostUsagePricing::codex_fast_cost_usd_at_date( - model, - input as i32, - cached as i32, - output as i32, - day, - ) + CostUsagePricing::codex_fast_cost_usd_at_date(model, input, cached, output, day) }) - .or_else(|| { - CostUsagePricing::codex_fast_cost_usd( - model, - input as i32, - cached as i32, - output as i32, - ) - }); + .or_else(|| CostUsagePricing::codex_fast_cost_usd(model, input, cached, output)); if let Some(cost) = fast { return cost; } diff --git a/rust/src/core/cost_cache_budget.rs b/rust/src/core/cost_cache_budget.rs index 81b40681f7..ca40ed3332 100644 --- a/rust/src/core/cost_cache_budget.rs +++ b/rust/src/core/cost_cache_budget.rs @@ -93,7 +93,7 @@ fn estimated_entry_bytes(entry: &CostUsageFileUsage) -> usize { /// Conservative estimate of the encoded artifact size. pub fn estimated_cache_bytes( files: &HashMap, - days: &HashMap>>, + days: &HashMap>>, ) -> usize { let mut bytes = 4096; bytes += files.len() * 160; @@ -121,7 +121,7 @@ pub fn estimated_cache_bytes( /// shape (no fork lineages, no discovery/lookback state). pub fn prune_out_of_window_for_budget( files: &mut HashMap, - days: &mut HashMap>>, + days: &mut HashMap>>, scan_since_key: Option<&str>, scan_until_key: Option<&str>, force: bool, @@ -171,7 +171,7 @@ pub fn prune_out_of_window_for_budget( /// cache shape. pub fn trim_in_window_for_budget( files: &mut HashMap, - days: &mut HashMap>>, + days: &mut HashMap>>, scan_since_key: Option<&str>, scan_until_key: Option<&str>, max_bytes: usize, @@ -248,8 +248,8 @@ pub fn trim_in_window_for_budget( /// of the scanner's `rebuild_cache_days` accumulation), so pruned entries do /// not inflate totals. fn subtract_entry_days( - days: &mut HashMap>>, - entry_days: &HashMap>>, + days: &mut HashMap>>, + entry_days: &HashMap>>, ) { let mut empty_days = Vec::new(); for (day, models) in entry_days { @@ -315,7 +315,7 @@ mod tests { use super::*; fn entry(days: &[&str], parsed: Option, size: i64) -> CostUsageFileUsage { - let mut day_map: HashMap>> = HashMap::new(); + let mut day_map: HashMap>> = HashMap::new(); for day in days { day_map.insert( (*day).to_string(), @@ -342,12 +342,12 @@ mod tests { type TestCache = ( HashMap, - HashMap>>, + HashMap>>, ); fn cache(files: &[(&str, CostUsageFileUsage)]) -> TestCache { let mut file_map = HashMap::new(); - let mut days: HashMap>> = HashMap::new(); + let mut days: HashMap>> = HashMap::new(); for (key, entry) in files { for (day, models) in &entry.days { let day_entry = days.entry(day.clone()).or_default(); diff --git a/rust/src/core/cost_pricing.rs b/rust/src/core/cost_pricing.rs index e019647a36..d220c6b8f8 100755 --- a/rust/src/core/cost_pricing.rs +++ b/rust/src/core/cost_pricing.rs @@ -703,22 +703,17 @@ impl CostUsagePricing { /// suffixes), then applies the Fast multiplier. Returns `None` when the /// model has no Fast lane or when a model without Astra's published /// long-context Fast rates exceeds the 272 000 threshold. - pub fn codex_fast_cost_usd(model: &str, input: i32, cached: i32, output: i32) -> Option { + pub fn codex_fast_cost_usd(model: &str, input: u64, cached: u64, output: u64) -> Option { let multiplier = Self::codex_api_fast_multiplier(model)?; // Older models do not offer Fast for long-context requests. Astra // publishes a Fast rate for the same whole-request long-context tier. - if (input.max(0) as u64) > codex_pricing::CODEX_LONG_CONTEXT_THRESHOLD + if input > codex_pricing::CODEX_LONG_CONTEXT_THRESHOLD && !codex_pricing::codex_fast_allows_long_context(model) { return None; } let base = Self::codex_fast_base_model(model); - let base_cost = Self::codex_cost_usd( - &base, - input.max(0) as u64, - cached.max(0) as u64, - output.max(0) as u64, - )?; + let base_cost = Self::codex_cost_usd(&base, input, cached, output)?; Some(base_cost * multiplier) } @@ -782,25 +777,19 @@ impl CostUsagePricing { pub fn codex_fast_cost_usd_at_date( model: &str, - input: i32, - cached: i32, - output: i32, + input: u64, + cached: u64, + output: u64, pricing_date: NaiveDate, ) -> Option { let multiplier = Self::codex_api_fast_multiplier(model)?; - if (input.max(0) as u64) > codex_pricing::CODEX_LONG_CONTEXT_THRESHOLD + if input > codex_pricing::CODEX_LONG_CONTEXT_THRESHOLD && !codex_pricing::codex_fast_allows_long_context(model) { return None; } let base = Self::codex_fast_base_model(model); - let base_cost = Self::codex_cost_usd_at_date( - &base, - input.max(0) as u64, - cached.max(0) as u64, - output.max(0) as u64, - pricing_date, - )?; + let base_cost = Self::codex_cost_usd_at_date(&base, input, cached, output, pricing_date)?; Some(base_cost * multiplier) } diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index d86ca9d720..e4ef6df84b 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -22,6 +22,8 @@ use std::io::{BufReader, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; +const CODEX_CACHE_SCHEMA_VERSION: u32 = 1; + #[derive(Debug, Clone, Default)] pub struct CachedCostReadStatus { pub has_days: bool, @@ -31,6 +33,8 @@ pub struct CachedCostReadStatus { #[derive(Deserialize, Default)] struct CachedCostReadStatusProjection { + #[serde(default)] + codex_cache_schema_version: u32, #[serde( default, rename = "days", @@ -182,12 +186,15 @@ pub enum CodexScanPauseReason { /// Cache for scanned file data #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct CostUsageCache { + /// Codex cache schema. Version 0 is any pre-64-bit cache and must be rebuilt. + #[serde(default)] + pub codex_cache_schema_version: u32, /// Last scan timestamp in milliseconds pub last_scan_unix_ms: i64, /// Per-file usage data pub files: HashMap, /// Aggregated daily data: day_key -> model -> [input, cached, output, reasoning?] - pub days: HashMap>>, + pub days: HashMap>>, /// Inclusive range covered by the last successful full inspection. #[serde(default, skip_serializing_if = "Option::is_none")] pub scan_since_key: Option, @@ -244,7 +251,7 @@ pub struct CostUsageFileUsage { #[serde(default, skip_serializing_if = "Option::is_none")] pub codex_file_identity: Option, /// Daily usage data extracted from this file - pub days: HashMap>>, + pub days: HashMap>>, /// Bytes parsed so far (for incremental parsing) pub parsed_bytes: Option, /// Frozen logical end of the scan target. A growing rollout may have a @@ -293,11 +300,11 @@ pub(crate) struct CodexSessionMetadata { /// Running totals for Codex token counting #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CodexTotals { - pub input: i32, - pub cached: i32, - pub output: i32, + pub input: i64, + pub cached: i64, + pub output: i64, #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning: Option, + pub reasoning: Option, } /// Snapshot of the last validated cost report, persisted so spend surfaces keep @@ -309,14 +316,14 @@ pub struct CachedCostReport { /// Total cost in USD for the reported window. pub total_cost_usd: f64, /// Total input tokens. - pub input_tokens: i32, + pub input_tokens: i64, /// Total cached tokens. - pub cached_tokens: i32, + pub cached_tokens: i64, /// Total output tokens. - pub output_tokens: i32, + pub output_tokens: i64, /// Total reasoning output tokens when every contributing packed row knows it. #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_tokens: Option, + pub reasoning_tokens: Option, /// Number of sessions contributing. pub sessions_count: i32, /// ISO 8601 timestamp when this report was generated. @@ -360,10 +367,10 @@ pub struct CodexParseResult { pub struct CodexUsageRecord { pub day_key: String, pub model: String, - pub input: i32, - pub cached: i32, - pub output: i32, - pub reasoning: Option, + pub input: i64, + pub cached: i64, + pub output: i64, + pub reasoning: Option, } /// Day range for scanning @@ -440,7 +447,17 @@ impl JsonlScanner { if let Ok(contents) = fs::read_to_string(&cache_path) && let Ok(mut cache) = serde_json::from_str::(&contents) { - cache.loaded_stamp = Some(Some(CacheStamp::from_bytes(contents.as_bytes()))); + let stamp = CacheStamp::from_bytes(contents.as_bytes()); + if provider == ProviderId::Codex + && cache.codex_cache_schema_version != CODEX_CACHE_SCHEMA_VERSION + { + return CostUsageCache { + codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, + loaded_stamp: Some(Some(stamp)), + ..CostUsageCache::default() + }; + } + cache.loaded_stamp = Some(Some(stamp)); return cache; } @@ -480,6 +497,11 @@ impl JsonlScanner { else { return CachedCostReadStatus::default(); }; + if provider == ProviderId::Codex + && projection.codex_cache_schema_version != CODEX_CACHE_SCHEMA_VERSION + { + return CachedCostReadStatus::default(); + } CachedCostReadStatus { has_days: projection.has_days, previous_report: projection.previous_report, @@ -488,10 +510,10 @@ impl JsonlScanner { } pub(crate) fn cached_cost_report_from_days(cache: &CostUsageCache) -> CachedCostReport { let mut total_cost_usd = 0.0; - let mut input_tokens = 0_i32; - let mut cached_tokens = 0_i32; - let mut output_tokens = 0_i32; - let mut reasoning_tokens = 0_i32; + let mut input_tokens = 0_i64; + let mut cached_tokens = 0_i64; + let mut output_tokens = 0_i64; + let mut reasoning_tokens = 0_i64; let mut reasoning_known = true; let mut partial = false; @@ -569,7 +591,7 @@ impl JsonlScanner { /// Merge one Codex record into a packed day/model row. A three-slot row is /// deliberately treated as reasoning-unknown, including when a known row /// is merged into an existing legacy row. - pub(crate) fn merge_codex_record_into_packed(packed: &mut Vec, record: &CodexUsageRecord) { + pub(crate) fn merge_codex_record_into_packed(packed: &mut Vec, record: &CodexUsageRecord) { let was_empty = packed.is_empty(); if packed.len() < 3 { packed.resize(3, 0); @@ -627,6 +649,9 @@ impl JsonlScanner { { return; } + if provider == ProviderId::Codex { + cache.codex_cache_schema_version = CODEX_CACHE_SCHEMA_VERSION; + } let Some(parent) = cache_path.parent() else { return; diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index e01a4764ee..27a27fa9f9 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -31,15 +31,15 @@ pub(super) struct CodexFastPayload<'a> { #[serde(default, borrow)] pub(super) info: Option>, #[serde(default)] - pub(super) input_tokens: Option, + pub(super) input_tokens: Option, #[serde(default)] - pub(super) cached_input_tokens: Option, + pub(super) cached_input_tokens: Option, #[serde(default)] - pub(super) cache_read_input_tokens: Option, + pub(super) cache_read_input_tokens: Option, #[serde(default)] - pub(super) output_tokens: Option, + pub(super) output_tokens: Option, #[serde(default)] - pub(super) reasoning_output_tokens: Option, + pub(super) reasoning_output_tokens: Option, } #[derive(Debug, Deserialize)] @@ -57,15 +57,15 @@ pub(super) struct CodexFastInfo<'a> { #[derive(Debug, Clone, Copy, Deserialize)] pub(super) struct CodexFastTotals { #[serde(default)] - pub(super) input_tokens: i32, + pub(super) input_tokens: i64, #[serde(default)] - pub(super) cached_input_tokens: Option, + pub(super) cached_input_tokens: Option, #[serde(default)] - pub(super) cache_read_input_tokens: Option, + pub(super) cache_read_input_tokens: Option, #[serde(default)] - pub(super) output_tokens: i32, + pub(super) output_tokens: i64, #[serde(default)] - pub(super) reasoning_output_tokens: Option, + pub(super) reasoning_output_tokens: Option, } pub(super) enum CodexFastEvent<'a> { @@ -103,7 +103,7 @@ pub(super) fn contained_total_delta( reasoning: None, }); - let component = |water: i32, counted: i32, current: i32| -> i32 { + let component = |water: i64, counted: i64, current: i64| -> i64 { if current >= water { // Only growth above the historical high watermark counts. (current - water.max(counted)).max(0) @@ -128,9 +128,9 @@ pub(super) fn contained_total_delta( pub(super) fn cumulative_reasoning_delta( previous: Option<&CodexTotals>, - current: Option, - output_delta: i32, -) -> Option { + current: Option, + output_delta: i64, +) -> Option { let current = current?; let previous = match previous { Some(previous) => previous.reasoning?, @@ -498,31 +498,19 @@ pub(super) fn bare_usage_totals(obj: &Value) -> Option<(CodexTotals, Option Option<(CodexTotals, Option Option<&Value> { } pub(super) fn read_token_totals(value: &Value) -> CodexTotals { - // Token counts come from Codex usage records and fit within i32, which is + // Token counts come from Codex usage records and use i64, which is // the canonical storage type of the totals table. - #[allow( - clippy::cast_possible_truncation, - reason = "token counts from usage records fit i32" - )] let cached = value .get("cached_input_tokens") .and_then(|v| v.as_i64()) @@ -584,14 +568,14 @@ pub(super) fn read_token_totals(value: &Value) -> CodexTotals { .get("cache_read_input_tokens") .and_then(|v| v.as_i64()) .unwrap_or(0), - ) as i32; + ); CodexTotals { - input: token_i32(value, "input_tokens"), + input: token_i64(value, "input_tokens"), cached, - output: token_i32(value, "output_tokens"), + output: token_i64(value, "output_tokens"), reasoning: clamp_reasoning( - optional_token_i32(value, "reasoning_output_tokens"), - token_i32(value, "output_tokens"), + optional_token_i64(value, "reasoning_output_tokens"), + token_i64(value, "output_tokens"), ), } } @@ -623,33 +607,22 @@ pub(super) fn fast_totals_from_payload(value: &CodexFastPayload<'_>) -> CodexTot } } -fn token_i32(value: &Value, key: &str) -> i32 { - // Token counts from usage records fit i32, the canonical totals storage type. - #[allow( - clippy::cast_possible_truncation, - reason = "token counts from usage records fit i32" - )] - let tokens = value.get(key).and_then(|v| v.as_i64()).unwrap_or(0) as i32; +fn token_i64(value: &Value, key: &str) -> i64 { + // Token counts from usage records use i64, the canonical totals storage type. + let tokens = value.get(key).and_then(|v| v.as_i64()).unwrap_or(0); tokens } -fn optional_token_i32(value: &Value, key: &str) -> Option { - // Token counts from usage records fit i32, the canonical storage type. - #[allow( - clippy::cast_possible_truncation, - reason = "token counts from usage records fit i32" - )] - value - .get(key) - .and_then(Value::as_i64) - .map(|tokens| tokens as i32) +fn optional_token_i64(value: &Value, key: &str) -> Option { + // Token counts from usage records use i64, the canonical storage type. + value.get(key).and_then(Value::as_i64) } -pub(super) fn clamp_reasoning(reasoning: Option, output: i32) -> Option { +pub(super) fn clamp_reasoning(reasoning: Option, output: i64) -> Option { reasoning.map(|tokens| tokens.max(0).min(output.max(0))) } -pub(super) fn last_usage_delta(last: &Value) -> (i32, i32, i32, Option) { +pub(super) fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { let totals = read_token_totals(last); ( totals.input.max(0), @@ -659,7 +632,7 @@ pub(super) fn last_usage_delta(last: &Value) -> (i32, i32, i32, Option) { ) } -pub(super) fn fast_last_usage_delta(last: CodexFastTotals) -> (i32, i32, i32, Option) { +pub(super) fn fast_last_usage_delta(last: CodexFastTotals) -> (i64, i64, i64, Option) { let totals = codex_totals_from_fast(last); ( totals.input.max(0), diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index fee86ecbff..99c214f3cb 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -287,10 +287,10 @@ impl CodexParserState { range: &CostUsageDayRange, day_key: String, model: &str, - input: i32, - cached: i32, - output: i32, - reasoning: Option, + input: i64, + cached: i64, + output: i64, + reasoning: Option, ) { if !CostUsageDayRange::is_in_range(&day_key, &range.since_key, &range.until_key) { return; @@ -320,7 +320,7 @@ impl CodexParserState { .to_string() } - fn token_deltas(&mut self, payload: &Value) -> Option<(i32, i32, i32, Option)> { + fn token_deltas(&mut self, payload: &Value) -> Option<(i64, i64, i64, Option)> { let info = payload.get("info"); if let Some(total) = info.and_then(|i| i.get("total_token_usage")) { return Some(self.total_usage_delta(total)); @@ -342,7 +342,7 @@ impl CodexParserState { fn fast_token_deltas( &mut self, payload: &CodexFastPayload<'_>, - ) -> Option<(i32, i32, i32, Option)> { + ) -> Option<(i64, i64, i64, Option)> { if let Some(total) = payload .info .as_ref() @@ -364,12 +364,12 @@ impl CodexParserState { )) } - pub(super) fn total_usage_delta(&mut self, total: &Value) -> (i32, i32, i32, Option) { + pub(super) fn total_usage_delta(&mut self, total: &Value) -> (i64, i64, i64, Option) { let totals = read_token_totals(total); self.apply_totals_delta(totals) } - fn fast_total_usage_delta(&mut self, total: CodexFastTotals) -> (i32, i32, i32, Option) { + fn fast_total_usage_delta(&mut self, total: CodexFastTotals) -> (i64, i64, i64, Option) { let totals = codex_totals_from_fast(total); self.apply_totals_delta(totals) } @@ -377,7 +377,7 @@ impl CodexParserState { pub(super) fn apply_totals_delta( &mut self, totals: CodexTotals, - ) -> (i32, i32, i32, Option) { + ) -> (i64, i64, i64, Option) { self.latch_if_below_watermark(&totals); let delta = if self.saw_interleaved_totals { diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 3e056bbc95..4cb7499cb6 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -119,6 +119,44 @@ fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { assert!(!state.fork_baseline_ambiguous); } +#[test] +fn codex_token_pipeline_preserves_counts_above_i32_max() { + let parsed = read_token_totals(&serde_json::json!({ + "input_tokens": 3_000_000_000_i64, + "cached_input_tokens": 2_800_000_000_i64, + "output_tokens": 200, + })); + assert_eq!(parsed.input, 3_000_000_000); + assert_eq!(parsed.cached, 2_800_000_000); + assert_eq!(parsed.output, 200); + + let mut packed = Vec::new(); + for _ in 0..2 { + JsonlScanner::merge_codex_record_into_packed( + &mut packed, + &CodexUsageRecord { + day_key: "2026-09-09".to_string(), + model: "gpt-5.6-luna".to_string(), + input: 1_500_000_000, + cached: 1_400_000_000, + output: 100, + reasoning: None, + }, + ); + } + assert_eq!(packed, vec![3_000_000_000, 2_800_000_000, 200]); + + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), packed)]), + ); + let report = JsonlScanner::cached_cost_report_from_days(&cache); + assert_eq!(report.input_tokens, 3_000_000_000); + assert_eq!(report.cached_tokens, 2_800_000_000); + assert_eq!(report.output_tokens, 200); +} + #[test] fn legacy_packed_rows_remain_three_slots_and_report_reasoning_is_unknown() { let record = CodexUsageRecord { @@ -1108,6 +1146,43 @@ fn catch_up_snapshot_preserves_established_codex_cost_and_tokens() { assert!(report.updated_at.is_some()); } +#[test] +fn codex_cache_round_trip_preserves_64_bit_counts_and_rebuilds_legacy_schema() { + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path(); + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-09-09".to_string(), + HashMap::from([( + "gpt-5.6-luna".to_string(), + vec![3_000_000_000, 2_800_000_000, 200], + )]), + ); + + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(cache_root)); + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + assert_eq!( + loaded.days["2026-09-09"]["gpt-5.6-luna"], + vec![3_000_000_000, 2_800_000_000, 200] + ); + + let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(cache_root)); + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&cache_path).unwrap()).unwrap(); + legacy + .as_object_mut() + .unwrap() + .remove("codex_cache_schema_version"); + std::fs::write(&cache_path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + + let invalidated = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + assert!(invalidated.days.is_empty()); + assert!(invalidated.files.is_empty()); + let status = JsonlScanner::load_cache_status(ProviderId::Codex, Some(cache_root)); + assert!(!status.has_days); + assert!(status.previous_report.is_none()); +} + #[test] fn save_cache_persists_small_codex_artifact() { // F19 integration: a normal-sized Codex cache is persisted and diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 3b5bdc5263..dda00f2a27 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -554,7 +554,7 @@ fn write_codex_session_fixture_with_inputs( path } -fn cached_usage_with_packed(day: &str, model: &str, packed: Vec) -> CostUsageFileUsage { +fn cached_usage_with_packed(day: &str, model: &str, packed: Vec) -> CostUsageFileUsage { CostUsageFileUsage { mtime_unix_ms: 0, size: 1, @@ -600,7 +600,7 @@ fn rebuild_cache_days_preserves_known_reasoning() { #[test] fn rebuild_cache_days_reasoning_unknown_is_order_independent() { - let run = |first: Vec, second: Vec| { + let run = |first: Vec, second: Vec| { let day = Local::now().format("%Y-%m-%d").to_string(); let mut cache = CostUsageCache { files: HashMap::from([ From 903897b5398356a642d29d9858e3c20824921372 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:46:10 +0700 Subject: [PATCH 2/6] Fix widened Codex scanner checks --- rust/src/core/jsonl_scanner/codex/helpers.rs | 7 +++++-- rust/src/core/jsonl_scanner/tests.rs | 14 +++++++------- rust/src/cost_scanner/tests.rs | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index 27a27fa9f9..fce9e0fd23 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -68,6 +68,10 @@ pub(super) struct CodexFastTotals { pub(super) reasoning_output_tokens: Option, } +#[allow( + clippy::large_enum_variant, + reason = "keeping the borrowed fast payload inline avoids heap allocation in the JSONL scan hot path" +)] pub(super) enum CodexFastEvent<'a> { TurnContext { model: Option<&'a str>, @@ -609,8 +613,7 @@ pub(super) fn fast_totals_from_payload(value: &CodexFastPayload<'_>) -> CodexTot fn token_i64(value: &Value, key: &str) -> i64 { // Token counts from usage records use i64, the canonical totals storage type. - let tokens = value.get(key).and_then(|v| v.as_i64()).unwrap_or(0); - tokens + value.get(key).and_then(Value::as_i64).unwrap_or(0) } fn optional_token_i64(value: &Value, key: &str) -> Option { diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 4cb7499cb6..4045dc9fea 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -506,7 +506,7 @@ fn codex_append_timestamp_state_is_output_equivalent_and_boundary_only() { JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None).expect("parse prefix"); assert_eq!(prefix.token_timestamps_monotonic, Some(true)); assert_eq!(prefix.token_timestamp_comparisons, 1); - let prefix_input: i32 = prefix.records.iter().map(|record| record.input).sum(); + let prefix_input: i64 = prefix.records.iter().map(|record| record.input).sum(); writeln!( file, @@ -533,8 +533,8 @@ fn codex_append_timestamp_state_is_output_equivalent_and_boundary_only() { let full = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None) .expect("parse complete file"); - let full_input: i32 = full.records.iter().map(|record| record.input).sum(); - let appended_input: i32 = appended.records.iter().map(|record| record.input).sum(); + let full_input: i64 = full.records.iter().map(|record| record.input).sum(); + let appended_input: i64 = appended.records.iter().map(|record| record.input).sum(); assert_eq!(prefix_input + appended_input, full_input); assert_eq!(full_input, 30); } @@ -891,8 +891,8 @@ fn interleaved_lineage_totals_never_exceed_high_watermark_growth() { &range, ); - let total_input: i32 = parser.records.iter().map(|r| r.input).sum(); - let total_output: i32 = parser.records.iter().map(|r| r.output).sum(); + let total_input: i64 = parser.records.iter().map(|r| r.input).sum(); + let total_output: i64 = parser.records.iter().map(|r| r.output).sum(); assert!( total_input <= 101, "input inflated to {total_input}, expected <= 101" @@ -921,8 +921,8 @@ fn interleaved_lineage_mid_range_climb_below_watermark_does_not_readd() { ); } - let total_input: i32 = parser.records.iter().map(|r| r.input).sum(); - let total_output: i32 = parser.records.iter().map(|r| r.output).sum(); + let total_input: i64 = parser.records.iter().map(|r| r.input).sum(); + let total_output: i64 = parser.records.iter().map(|r| r.output).sum(); assert!( total_input <= 101, "mid-range climb re-added input to {total_input}, expected <= 101" diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index dda00f2a27..4280332f1d 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -800,7 +800,7 @@ fn write_codex_fork_session_fixture( path } -fn cached_input_total(usage: &CostUsageFileUsage) -> i32 { +fn cached_input_total(usage: &CostUsageFileUsage) -> i64 { usage .days .values() From a7a3d8b2abaf7855459b403c1c4706681fb3dfd6 Mon Sep 17 00:00:00 2001 From: Finesssee Date: Sat, 12 Sep 2026 13:04:38 +0700 Subject: [PATCH 3/6] Clamp negative Codex totals and widen packed byte estimate --- rust/src/core/cost_cache_budget.rs | 49 ++++++++++++++++- rust/src/core/jsonl_scanner/codex.rs | 5 +- rust/src/core/jsonl_scanner/codex/helpers.rs | 57 +++++++++++--------- rust/src/core/jsonl_scanner/tests.rs | 55 +++++++++++++++++++ 4 files changed, 137 insertions(+), 29 deletions(-) diff --git a/rust/src/core/cost_cache_budget.rs b/rust/src/core/cost_cache_budget.rs index ca40ed3332..b71f92e90f 100644 --- a/rust/src/core/cost_cache_budget.rs +++ b/rust/src/core/cost_cache_budget.rs @@ -71,6 +71,12 @@ fn touches_window(entry: &CostUsageFileUsage, since_key: &str, until_key: &str) .any(|day| CostUsageDayRange::is_in_range(day, since_key, until_key)) } +/// Conservative JSON byte width of one packed token value. The totals table was +/// widened from `i32` to `i64`, so a single value can serialize to up to 20 +/// bytes (19 digits plus a sign); budgeting 10 bytes would under-estimate the +/// artifact and let pruning admit a cache that is already over budget. +const PACKED_VALUE_BYTES: usize = 20; + /// Cheap per-entry byte estimate (conservative overhead) so the save path can /// decide whether to prune *before* materializing the encoded document. Mirrors /// upstream `estimatedCodexCacheBytes`'s per-entry shape; it deliberately @@ -84,7 +90,7 @@ fn estimated_entry_bytes(entry: &CostUsageFileUsage) -> usize { for (day, models) in &entry.days { bytes += day.len() + 32; for (model, packed) in models { - bytes += model.len() + 40 + packed.len() * 10; + bytes += model.len() + 40 + packed.len() * PACKED_VALUE_BYTES; } } bytes @@ -103,7 +109,7 @@ pub fn estimated_cache_bytes( for (day, models) in days { bytes += day.len() + 32; for (model, packed) in models { - bytes += model.len() + 40 + packed.len() * 10; + bytes += model.len() + 40 + packed.len() * PACKED_VALUE_BYTES; } } bytes @@ -526,6 +532,45 @@ mod tests { assert!(files.contains_key("c"), "newest kept"); } + #[test] + fn estimated_entry_bytes_reserves_twenty_bytes_per_packed_i64_value() { + let day = "2026-01-09"; + let model = "gpt-5.6-sol"; + let three = entry(&[day], None, 100); + let mut four = entry(&[day], None, 100); + four.days + .get_mut(day) + .unwrap() + .insert(model.to_string(), vec![1, 2, 3, 4]); + + assert_eq!( + estimated_entry_bytes(&four) - estimated_entry_bytes(&three), + 20, + "each packed i64 slot must reserve its full 20-byte JSON width" + ); + } + + #[test] + fn estimated_cache_bytes_reserves_twenty_bytes_per_packed_i64_value_in_days() { + let day = "2026-01-09"; + let model = "gpt-5.6-sol"; + let files: HashMap = HashMap::new(); + let three = HashMap::from([( + day.to_string(), + HashMap::from([(model.to_string(), vec![1_i64, 2, 3])]), + )]); + let four = HashMap::from([( + day.to_string(), + HashMap::from([(model.to_string(), vec![1_i64, 2, 3, 4])]), + )]); + + assert_eq!( + estimated_cache_bytes(&files, &four) - estimated_cache_bytes(&files, &three), + 20, + "each packed i64 slot in the aggregate day map must reserve 20 bytes" + ); + } + #[test] fn is_unpriced_codex_routing_model_flags_auto_review_and_unattributed() { assert!(is_unpriced_codex_routing_model("codex-auto-review")); diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 4bf8b07584..8fccc32590 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -11,8 +11,9 @@ use parser::CodexParserState; #[cfg(test)] use helpers::{ - CodexFastTotals, bare_usage_totals, codex_timestamp_day_key, codex_totals_from_fast, - last_usage_delta, parse_codex_timestamp, read_token_totals, + CodexFastPayload, CodexFastTotals, bare_usage_totals, codex_timestamp_day_key, + codex_totals_from_fast, fast_totals_from_payload, last_usage_delta, parse_codex_timestamp, + read_token_totals, }; impl JsonlScanner { diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index fce9e0fd23..e559700cbb 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -562,7 +562,9 @@ pub(super) fn token_count_payload(obj: &Value) -> Option<&Value> { pub(super) fn read_token_totals(value: &Value) -> CodexTotals { // Token counts come from Codex usage records and use i64, which is - // the canonical storage type of the totals table. + // the canonical storage type of the totals table. Malformed negative + // counts are clamped at the source so they can never lower the high + // watermark and inflate a later `apply_totals_delta`. let cached = value .get("cached_input_tokens") .and_then(|v| v.as_i64()) @@ -572,42 +574,47 @@ pub(super) fn read_token_totals(value: &Value) -> CodexTotals { .get("cache_read_input_tokens") .and_then(|v| v.as_i64()) .unwrap_or(0), - ); + ) + .max(0); + let input = token_i64(value, "input_tokens").max(0); + let output = token_i64(value, "output_tokens").max(0); CodexTotals { - input: token_i64(value, "input_tokens"), + input, cached, - output: token_i64(value, "output_tokens"), - reasoning: clamp_reasoning( - optional_token_i64(value, "reasoning_output_tokens"), - token_i64(value, "output_tokens"), - ), + output, + reasoning: clamp_reasoning(optional_token_i64(value, "reasoning_output_tokens"), output), } } pub(super) fn codex_totals_from_fast(value: CodexFastTotals) -> CodexTotals { + let input = value.input_tokens.max(0); + let cached = value + .cached_input_tokens + .unwrap_or(0) + .max(value.cache_read_input_tokens.unwrap_or(0)) + .max(0); + let output = value.output_tokens.max(0); CodexTotals { - input: value.input_tokens, - cached: value - .cached_input_tokens - .unwrap_or(0) - .max(value.cache_read_input_tokens.unwrap_or(0)), - output: value.output_tokens, - reasoning: clamp_reasoning(value.reasoning_output_tokens, value.output_tokens), + input, + cached, + output, + reasoning: clamp_reasoning(value.reasoning_output_tokens, output), } } pub(super) fn fast_totals_from_payload(value: &CodexFastPayload<'_>) -> CodexTotals { + let input = value.input_tokens.unwrap_or(0).max(0); + let cached = value + .cached_input_tokens + .unwrap_or(0) + .max(value.cache_read_input_tokens.unwrap_or(0)) + .max(0); + let output = value.output_tokens.unwrap_or(0).max(0); CodexTotals { - input: value.input_tokens.unwrap_or(0), - cached: value - .cached_input_tokens - .unwrap_or(0) - .max(value.cache_read_input_tokens.unwrap_or(0)), - output: value.output_tokens.unwrap_or(0), - reasoning: clamp_reasoning( - value.reasoning_output_tokens, - value.output_tokens.unwrap_or(0), - ), + input, + cached, + output, + reasoning: clamp_reasoning(value.reasoning_output_tokens, output), } } diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 4045dc9fea..aa7bf76c18 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -157,6 +157,61 @@ fn codex_token_pipeline_preserves_counts_above_i32_max() { assert_eq!(report.output_tokens, 200); } +#[test] +fn negative_cumulative_components_are_clamped_at_the_source() { + let value = serde_json::json!({ + "input_tokens": -5, + "cached_input_tokens": -9, + "cache_read_input_tokens": -3, + "output_tokens": -2, + "reasoning_output_tokens": -1, + }); + let totals = read_token_totals(&value); + assert_eq!(totals.input, 0); + assert_eq!(totals.cached, 0); + assert_eq!(totals.output, 0); + assert_eq!(totals.reasoning, Some(0)); + + let fast: CodexFastTotals = serde_json::from_value(value.clone()).unwrap(); + let fast_totals = codex_totals_from_fast(fast); + assert_eq!(fast_totals.input, 0); + assert_eq!(fast_totals.cached, 0); + assert_eq!(fast_totals.output, 0); + assert_eq!(fast_totals.reasoning, Some(0)); + + // The payload borrows `&str` fields, so deserialize from a str rather than + // an owned `Value`. + let payload_json = value.to_string(); + let payload: CodexFastPayload<'_> = serde_json::from_str(&payload_json).unwrap(); + let payload_totals = fast_totals_from_payload(&payload); + assert_eq!(payload_totals.input, 0); + assert_eq!(payload_totals.cached, 0); + assert_eq!(payload_totals.output, 0); + assert_eq!(payload_totals.reasoning, Some(0)); +} + +#[test] +fn negative_cumulative_totals_do_not_inflate_later_deltas() { + let mut state = CodexParserState::new(None, None); + // A malformed cumulative record with negative counts must be clamped so it + // cannot lower the high watermark below zero. + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "input_tokens": -5, + "output_tokens": -2, + })), + (0, 0, 0, None) + ); + // A later normal climb only counts its true growth above the clamped zero. + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "input_tokens": 3, + "output_tokens": 1, + })), + (3, 0, 1, None) + ); +} + #[test] fn legacy_packed_rows_remain_three_slots_and_report_reasoning_is_unknown() { let record = CodexUsageRecord { From dd2db3b604c0908e73ebda6b94fd1732bbfd3547 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:46:34 +0700 Subject: [PATCH 4/6] Fix reasoning totals when pruning Codex cache --- rust/src/core/cost_cache_budget.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/rust/src/core/cost_cache_budget.rs b/rust/src/core/cost_cache_budget.rs index b71f92e90f..e6523df3a7 100644 --- a/rust/src/core/cost_cache_budget.rs +++ b/rust/src/core/cost_cache_budget.rs @@ -267,7 +267,7 @@ fn subtract_entry_days( let Some(dest) = day_entry.get_mut(model) else { continue; }; - for (i, value) in packed.iter().take(3).enumerate() { + for (i, value) in packed.iter().take(4).enumerate() { if i < dest.len() { dest[i] = dest[i].saturating_sub(*value); } @@ -373,6 +373,24 @@ mod tests { (file_map, days) } + #[test] + fn subtract_entry_days_removes_reasoning_slot() { + let day = "2026-01-10".to_string(); + let model = "gpt-5.6-sol".to_string(); + let mut days = HashMap::from([( + day.clone(), + HashMap::from([(model.clone(), vec![30, 12, 9, 6])]), + )]); + let entry_days = HashMap::from([( + day.clone(), + HashMap::from([(model.clone(), vec![10, 4, 3, 2])]), + )]); + + subtract_entry_days(&mut days, &entry_days); + + assert_eq!(days[&day][&model], vec![20, 8, 6, 4]); + } + #[test] fn out_of_window_entries_are_pruned_and_days_subtracted() { let (mut files, mut days) = cache(&[ From cc167c3dea65bd67a63565063e389f7b71febf7e Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:09:46 +0700 Subject: [PATCH 5/6] test(codex): cover large cross-file retained totals --- rust/src/cost_scanner/tests.rs | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 4280332f1d..67f05d5062 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -646,6 +646,72 @@ fn rebuild_cache_days_zero_row_does_not_poison_reasoning() { assert_eq!(cache.days[&day]["gpt-5"], vec![10, 0, 4, 3]); } +#[test] +fn rebuild_cache_days_aggregates_multiple_files_above_i32_max() { + let day = Local::now().format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), + ), + ( + "b".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), + ), + ]), + ..CostUsageCache::default() + }; + + rebuild_cache_days(&mut cache); + + assert_eq!( + cache.days[&day]["gpt-5"], + vec![3_000_000_000, 2_800_000_000, 200] + ); +} + +#[test] +fn retained_report_sums_multiple_days_above_i32_max() { + let day_a = "2026-09-08"; + let day_b = "2026-09-09"; + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed( + day_a, + "gpt-5.6-sol", + vec![1_500_000_000, 1_400_000_000, 1_000_000], + ), + ), + ( + "b".to_string(), + cached_usage_with_packed( + day_b, + "gpt-5.6-sol", + vec![1_500_000_000, 1_400_000_000, 1_000_000], + ), + ), + ]), + ..CostUsageCache::default() + }; + rebuild_cache_days(&mut cache); + + let report = JsonlScanner::cached_cost_report_from_days(&cache); + assert_eq!(report.input_tokens, 3_000_000_000); + assert_eq!(report.cached_tokens, 2_800_000_000); + assert_eq!(report.output_tokens, 2_000_000); + + let start = chrono::NaiveDate::from_ymd_opt(2026, 9, 8).unwrap(); + let end = chrono::NaiveDate::from_ymd_opt(2026, 9, 9).unwrap(); + let summary = summary_from_cached_report(&report, start, end); + assert_eq!(summary.input_tokens, 3_000_000_000); + assert_eq!(summary.cached_tokens, 2_800_000_000); + assert_eq!(summary.output_tokens, 2_000_000); + assert_eq!(summary.sessions_count, 2); +} + #[test] fn reasoning_survives_scan_rebuild_and_cache_reload() { let root = tempfile::tempdir().unwrap(); From 01935c10fc2f4267c1b83a6898d430ed23287599 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:20:50 +0700 Subject: [PATCH 6/6] Move Codex cache schema policy into scanner codex module --- rust/src/core/jsonl_scanner.rs | 16 +++------- rust/src/core/jsonl_scanner/codex.rs | 37 +++++++++++++++++++++++ rust/src/core/jsonl_scanner/tests.rs | 45 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index e4ef6df84b..0d476992ac 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -22,8 +22,6 @@ use std::io::{BufReader, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -const CODEX_CACHE_SCHEMA_VERSION: u32 = 1; - #[derive(Debug, Clone, Default)] pub struct CachedCostReadStatus { pub has_days: bool, @@ -448,14 +446,8 @@ impl JsonlScanner { && let Ok(mut cache) = serde_json::from_str::(&contents) { let stamp = CacheStamp::from_bytes(contents.as_bytes()); - if provider == ProviderId::Codex - && cache.codex_cache_schema_version != CODEX_CACHE_SCHEMA_VERSION - { - return CostUsageCache { - codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, - loaded_stamp: Some(Some(stamp)), - ..CostUsageCache::default() - }; + if provider == ProviderId::Codex { + return codex::codex_cache_apply_load_policy(cache, stamp); } cache.loaded_stamp = Some(Some(stamp)); return cache; @@ -498,7 +490,7 @@ impl JsonlScanner { return CachedCostReadStatus::default(); }; if provider == ProviderId::Codex - && projection.codex_cache_schema_version != CODEX_CACHE_SCHEMA_VERSION + && !codex::codex_cache_schema_is_current(projection.codex_cache_schema_version) { return CachedCostReadStatus::default(); } @@ -650,7 +642,7 @@ impl JsonlScanner { return; } if provider == ProviderId::Codex { - cache.codex_cache_schema_version = CODEX_CACHE_SCHEMA_VERSION; + codex::codex_cache_stamp_schema_version(cache); } let Some(parent) = cache_path.parent() else { diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 8fccc32590..9901ee6296 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -9,6 +9,43 @@ use helpers::{ }; use parser::CodexParserState; +/// Persisted Codex cache schema version. Version 0 is any pre-64-bit cache +/// and must be rebuilt from source logs. +pub(crate) const CODEX_CACHE_SCHEMA_VERSION: u32 = 1; + +/// Whether a persisted Codex cache artifact matches the current schema. +/// A mismatched artifact (e.g. a pre-64-bit cache from an older release) is +/// invalid and must be rebuilt rather than deserialized into wider fields. +pub(crate) fn codex_cache_schema_is_current(schema_version: u32) -> bool { + schema_version == CODEX_CACHE_SCHEMA_VERSION +} + +/// Apply the Codex cache schema version policy to a freshly decoded artifact. +/// +/// A mismatched artifact is invalidated: a fresh, current-version cache is +/// returned with the decoded baseline stamp retained so the caller stays +/// authoritative over the artifact it just read. A matching artifact keeps its +/// contents and receives the same stamp. +pub(crate) fn codex_cache_apply_load_policy( + mut cache: CostUsageCache, + stamp: CacheStamp, +) -> CostUsageCache { + if !codex_cache_schema_is_current(cache.codex_cache_schema_version) { + return CostUsageCache { + codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, + loaded_stamp: Some(Some(stamp)), + ..CostUsageCache::default() + }; + } + cache.loaded_stamp = Some(Some(stamp)); + cache +} + +/// Stamp the current schema version before a Codex cache is persisted. +pub(crate) fn codex_cache_stamp_schema_version(cache: &mut CostUsageCache) { + cache.codex_cache_schema_version = CODEX_CACHE_SCHEMA_VERSION; +} + #[cfg(test)] use helpers::{ CodexFastPayload, CodexFastTotals, bare_usage_totals, codex_timestamp_day_key, diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index aa7bf76c18..17f2c21c4e 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -1238,6 +1238,51 @@ fn codex_cache_round_trip_preserves_64_bit_counts_and_rebuilds_legacy_schema() { assert!(status.previous_report.is_none()); } +#[test] +fn codex_cache_schema_policy_helpers_rebuild_mismatched_load() { + let stamp = CacheStamp::from_bytes(b"baseline"); + + let legacy = CostUsageCache { + codex_cache_schema_version: 0, + days: HashMap::from([( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), + )]), + ..CostUsageCache::default() + }; + let rebuilt = codex_cache_apply_load_policy(legacy, stamp.clone()); + assert_eq!( + rebuilt.codex_cache_schema_version, + CODEX_CACHE_SCHEMA_VERSION + ); + assert!(rebuilt.days.is_empty()); + assert!(rebuilt.files.is_empty()); + assert!(rebuilt.loaded_stamp.is_some()); + + let current = CostUsageCache { + codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, + days: HashMap::from([( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), + )]), + ..CostUsageCache::default() + }; + let kept = codex_cache_apply_load_policy(current, stamp); + assert_eq!(kept.codex_cache_schema_version, CODEX_CACHE_SCHEMA_VERSION); + assert_eq!(kept.days["2026-09-09"]["gpt-5.6-luna"], vec![1, 2, 3]); + assert!(kept.loaded_stamp.is_some()); + + assert!(codex_cache_schema_is_current(CODEX_CACHE_SCHEMA_VERSION)); + assert!(!codex_cache_schema_is_current(0)); + + let mut stamped = CostUsageCache::default(); + codex_cache_stamp_schema_version(&mut stamped); + assert_eq!( + stamped.codex_cache_schema_version, + CODEX_CACHE_SCHEMA_VERSION + ); +} + #[test] fn save_cache_persists_small_codex_artifact() { // F19 integration: a normal-sized Codex cache is persisted and