From 730d7ac798d631414960b582278d2982b97734f2 Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 02:59:06 -0400 Subject: [PATCH 1/6] Cache NY pinned tax-benefit systems instead of cloning per formula call The NY EITC and pre-2024 CTC formulas deep-cloned the entire tax-benefit system (parameter tree + variable registry) on every call that hit the decoupled/pre-TCJA branch, driving the NY credits test memory to ~12 GB and requiring a dedicated quarantined CI runner. Build each pinned system (pre-ARPA 2020 EITC, pre-TCJA 2017 CTC) once per process in a module-level cache keyed on the baseline system's identity, so layered reforms still rebuild against their own base. Measured on the credits test files: ny_ctc_pre_2024.yaml 373s -> 57s, ny_eitc.yaml 106s -> 53s, peak RSS 4.8 -> 2.8 GB, outputs identical. Fixes #8114 Co-Authored-By: Claude Fable 5 --- policyengine_us/tools/pinned_tbs.py | 69 +++++++++++++++++++ .../tax/income/credits/ctc/ny_ctc_pre_2024.py | 19 ++--- .../credits/ctc/ny_ctc_pre_2024_eligible.py | 20 ++---- .../states/ny/tax/income/credits/ny_eitc.py | 18 ++--- 4 files changed, 85 insertions(+), 41 deletions(-) create mode 100644 policyengine_us/tools/pinned_tbs.py diff --git a/policyengine_us/tools/pinned_tbs.py b/policyengine_us/tools/pinned_tbs.py new file mode 100644 index 00000000000..2e9c1318836 --- /dev/null +++ b/policyengine_us/tools/pinned_tbs.py @@ -0,0 +1,69 @@ +"""Module-level cache of pinned tax-benefit systems (issue #8114). + +The NY EITC/CTC formulas recompute the federal EITC/CTC with parameters +pinned to an earlier vintage (pre-ARPA 2020, pre-TCJA 2017). Cloning the +full tax-benefit system inside the formula deep-copies the entire +parameter tree and variable registry on every call; this cache builds each +pinned system once per process and reuses it across simulations. + +The pinned system must be treated as read-only by all callers. +""" + +import weakref + +from policyengine_core.parameters import Parameter +from policyengine_core.periods import instant + +# pin name -> (weakref to base TBS, pinned clone). +# The clone is held strongly so it survives across simulations; the +# weakref to the base is used to detect that the base system changed +# (e.g. a reformed system in another test), in which case the entry is +# rebuilt. Using a weakref (rather than id() alone) also guards against +# id reuse after the base system is garbage-collected. +_PINNED_TBS_CACHE = {} + + +def _get_pinned_tbs(base_tbs, pin_name, pin_fn): + entry = _PINNED_TBS_CACHE.get(pin_name) + if entry is not None and entry[0]() is base_tbs: + return entry[1] + pinned = base_tbs.clone() + pin_fn(pinned) + _PINNED_TBS_CACHE[pin_name] = (weakref.ref(base_tbs), pinned) + return pinned + + +def _pin_pre_arpa_eitc(tbs): + # NY decoupled from post-March 2020 IRC amendments for TY 2021: + # pin the federal EITC parameters to their 2020 (pre-ARPA) values. + pin_date = instant("2020-01-01") + start = instant("2021-01-01") + stop = instant("2021-12-31") + for param in tbs.parameters.gov.irs.credits.eitc.get_descendants(): + if isinstance(param, Parameter): + try: + value = param(pin_date) + param.update(start=start, stop=stop, value=value) + except Exception: + pass + + +def _pin_pre_tcja_ctc(tbs): + # Pin the federal CTC parameters to their 2017 (pre-TCJA) values. + for ctc_parameter in tbs.parameters.gov.irs.credits.ctc.get_descendants(): + if isinstance(ctc_parameter, Parameter): + ctc_parameter.update( + start=instant("2017-01-01"), + stop=instant("2035-01-01"), + value=ctc_parameter("2017-01-01"), + ) + + +def get_pre_arpa_eitc_tbs(base_tbs): + """Pre-ARPA (2020-pinned) EITC system for NY's TY2021 decoupling.""" + return _get_pinned_tbs(base_tbs, "ny_pre_arpa_eitc", _pin_pre_arpa_eitc) + + +def get_pre_tcja_ctc_tbs(base_tbs): + """Pre-TCJA (2017-pinned) CTC system for the NY Empire State Child Credit.""" + return _get_pinned_tbs(base_tbs, "ny_pre_tcja_ctc", _pin_pre_tcja_ctc) diff --git a/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024.py b/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024.py index f80cfcc1a0c..0688b6389bb 100644 --- a/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024.py +++ b/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024.py @@ -1,5 +1,5 @@ from policyengine_us.model_api import * -from policyengine_core.periods import instant +from policyengine_us.tools.pinned_tbs import get_pre_tcja_ctc_tbs class ny_ctc_pre_2024(Variable): @@ -26,21 +26,14 @@ def formula(tax_unit, period, parameters): gov = parameters(period).gov qualifies_for_federal_ctc = person("ctc_qualifying_child", period) else: - # Initialize pre-TCJA CTC branch and parameters. + # Initialize pre-TCJA CTC branch with cached pinned parameters + # (one clone per process; see tools/pinned_tbs.py, issue #8114). simulation = tax_unit.simulation pre_tcja_ctc = simulation.get_branch("pre_tcja_ctc") - pre_tcja_ctc.tax_benefit_system = simulation.tax_benefit_system.clone() + pre_tcja_ctc.tax_benefit_system = get_pre_tcja_ctc_tbs( + simulation.tax_benefit_system + ) branch_parameters = pre_tcja_ctc.tax_benefit_system.parameters - # Update parameters to pre-TCJA values. - for ( - ctc_parameter - ) in branch_parameters.gov.irs.credits.ctc.get_descendants(): - if isinstance(ctc_parameter, Parameter): - ctc_parameter.update( - start=instant("2017-01-01"), - stop=instant("2035-01-01"), - value=ctc_parameter("2017-01-01"), - ) # Delete all arrays from pre-TCJA CTC branch. for variable in pre_tcja_ctc.tax_benefit_system.variables: if "ctc" in variable: diff --git a/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024_eligible.py b/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024_eligible.py index bde566efda0..3364c58fd18 100644 --- a/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024_eligible.py +++ b/policyengine_us/variables/gov/states/ny/tax/income/credits/ctc/ny_ctc_pre_2024_eligible.py @@ -1,5 +1,5 @@ from policyengine_us.model_api import * -from policyengine_core.periods import instant +from policyengine_us.tools.pinned_tbs import get_pre_tcja_ctc_tbs class ny_ctc_pre_2024_eligible(Variable): @@ -25,21 +25,13 @@ def formula(tax_unit, period, parameters): if not p.pre_tcja: qualifies_for_federal_ctc = person("ctc_qualifying_child", period) else: - # Initialize pre-TCJA CTC branch for eligibility check + # Initialize pre-TCJA CTC branch for eligibility check with + # cached pinned parameters (see tools/pinned_tbs.py, issue #8114). simulation = tax_unit.simulation pre_tcja_ctc = simulation.get_branch("pre_tcja_ctc") - pre_tcja_ctc.tax_benefit_system = simulation.tax_benefit_system.clone() - branch_parameters = pre_tcja_ctc.tax_benefit_system.parameters - # Update parameters to pre-TCJA values. - for ( - ctc_parameter - ) in branch_parameters.gov.irs.credits.ctc.get_descendants(): - if isinstance(ctc_parameter, Parameter): - ctc_parameter.update( - start=instant("2017-01-01"), - stop=instant("2035-01-01"), - value=ctc_parameter("2017-01-01"), - ) + pre_tcja_ctc.tax_benefit_system = get_pre_tcja_ctc_tbs( + simulation.tax_benefit_system + ) # Delete all arrays from pre-TCJA CTC branch. for variable in pre_tcja_ctc.tax_benefit_system.variables: if "ctc" in variable: diff --git a/policyengine_us/variables/gov/states/ny/tax/income/credits/ny_eitc.py b/policyengine_us/variables/gov/states/ny/tax/income/credits/ny_eitc.py index 694160355b8..a45b7d86eb2 100644 --- a/policyengine_us/variables/gov/states/ny/tax/income/credits/ny_eitc.py +++ b/policyengine_us/variables/gov/states/ny/tax/income/credits/ny_eitc.py @@ -1,5 +1,5 @@ from policyengine_us.model_api import * -from policyengine_core.periods import instant +from policyengine_us.tools.pinned_tbs import get_pre_arpa_eitc_tbs class ny_eitc(Variable): @@ -24,19 +24,9 @@ def formula(tax_unit, period, parameters): # pre-ARPA (2020) parameter values. simulation = tax_unit.simulation branch = simulation.get_branch("ny_pre_arpa_eitc") - branch.tax_benefit_system = simulation.tax_benefit_system.clone() - branch_params = branch.tax_benefit_system.parameters - pin_date = instant("2020-01-01") - start = instant("2021-01-01") - stop = instant("2021-12-31") - eitc_node = branch_params.gov.irs.credits.eitc - for param in eitc_node.get_descendants(): - if isinstance(param, Parameter): - try: - value = param(pin_date) - param.update(start=start, stop=stop, value=value) - except Exception: - pass + branch.tax_benefit_system = get_pre_arpa_eitc_tbs( + simulation.tax_benefit_system + ) for variable in branch.tax_benefit_system.variables: if "eitc" in variable: branch.delete_arrays(variable) From 103a2c712bf8b3a8dc3166493659b245dbac5ebc Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 02:59:06 -0400 Subject: [PATCH 2/6] Add concurrent worker pool and peak-RSS reporting to test_batched.py New --workers N flag (default 1 = unchanged sequential behavior) runs batch subprocesses concurrently, longest-first by YAML file count, with per-batch output buffered and printed atomically plus a 60s heartbeat. Each batch now reports its subprocess peak RSS (VmHWM from /proc on Linux) so CI logs show real per-batch memory. Grace sleep after pytest completion reduced from 5s to 1s. Co-Authored-By: Claude Fable 5 --- policyengine_us/tests/test_batched.py | 266 ++++++++++++++++++++++---- 1 file changed, 232 insertions(+), 34 deletions(-) diff --git a/policyengine_us/tests/test_batched.py b/policyengine_us/tests/test_batched.py index a33714b3d6b..5bf9a29f37d 100644 --- a/policyengine_us/tests/test_batched.py +++ b/policyengine_us/tests/test_batched.py @@ -12,8 +12,11 @@ import argparse import re import select +import threading +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor +from concurrent.futures import wait as futures_wait from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional def count_yaml_files(directory: Path) -> int: @@ -23,6 +26,34 @@ def count_yaml_files(directory: Path) -> int: return len(list(directory.rglob("*.yaml"))) +def batch_cost(batch_paths: List[str]) -> int: + """Estimated batch cost: total YAML files across the batch's paths.""" + return sum( + count_yaml_files(Path(p)) if Path(p).is_dir() else 1 for p in batch_paths + ) + + +def _read_vm_hwm_mb(pid: int) -> Optional[float]: + """Peak RSS (VmHWM high-water mark) of a live process, in MB. + + Linux-only: /proc does not exist on macOS, and the entry disappears + once the pid is reaped — both cases return None. + """ + try: + with open(f"/proc/{pid}/status") as status_file: + for line in status_file: + if line.startswith("VmHWM:"): + # Format: "VmHWM: 123456 kB" + return int(line.split()[1]) / 1024 + except (OSError, ValueError, IndexError): + return None + return None + + +def _format_rss(peak_rss_mb: Optional[float]) -> str: + return f"{peak_rss_mb:.0f} MB" if peak_rss_mb is not None else "n/a" + + def split_into_batches( base_path: Path, num_batches: int, @@ -303,15 +334,31 @@ def collect_gov_paths(folder_names): return batches -def run_batch(test_paths: List[str], batch_name: str) -> Dict: - """Run a batch of tests in an isolated subprocess.""" +def run_batch(test_paths: List[str], batch_name: str, stream: bool = True) -> Dict: + """Run a batch of tests in an isolated subprocess. + + With stream=True output is echoed to stdout in real time (sequential + mode). With stream=False output is buffered and returned under the + "output" key so a concurrent caller can print the whole batch + atomically once it finishes. + """ python_exe = sys.executable + buf: List[str] = [] + + def emit(text: str, end: str = "\n") -> None: + if stream: + print(text, end=end, flush=True) + else: + buf.append(text + end) + start_time = time.time() last_output_time = start_time last_heartbeat_time = start_time heartbeat_interval = 30 + peak_rss_mb = None + last_rss_sample = 0.0 # Build command - direct policyengine-core with timeout protection cmd = ( @@ -325,9 +372,9 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: + ["-c", "policyengine_us"] ) - print(f" Running {batch_name}...") - print(f" Paths: {len(test_paths)} items") - print() + emit(f" Running {batch_name}...") + emit(f" Paths: {len(test_paths)} items") + emit("") # Use Popen for more control over process lifecycle process = subprocess.Popen( @@ -337,6 +384,15 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: bufsize=0, ) + def sample_rss() -> None: + # VmHWM is monotonic, so keeping only the latest reading preserves + # the peak; readings stop (None) once the pid is reaped or off-Linux. + nonlocal peak_rss_mb, last_rss_sample + rss = _read_vm_hwm_mb(process.pid) + if rss is not None: + peak_rss_mb = rss + last_rss_sample = time.time() + try: test_completed = False test_passed = False @@ -347,12 +403,17 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: while True: ready, _, _ = select.select([process.stdout], [], [], 0.1) if not ready: + # Sample before poll(): poll() reaps the child, after which + # /proc//status is gone. + sample_rss() poll_result = process.poll() if poll_result is not None: # Process terminated break now = time.time() - if now - last_heartbeat_time >= heartbeat_interval: + # In buffered (concurrent) mode the shared heartbeat in + # run_batches_concurrently reports liveness instead. + if stream and now - last_heartbeat_time >= heartbeat_interval: elapsed = now - start_time quiet_for = now - last_output_time print( @@ -367,6 +428,7 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: chunk = os.read(process.stdout.fileno(), 4096) if not chunk: + sample_rss() if process.poll() is not None: break continue @@ -374,11 +436,13 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: # Print output in real-time, including partial pytest progress # lines such as dots that do not end with newlines. line = chunk.decode(errors="replace") - print(line, end="", flush=True) + emit(line, end="") last_output_time = time.time() last_heartbeat_time = last_output_time output_lines.append(line) output_text += line + if last_output_time - last_rss_sample >= 1.0: + sample_rss() # Detect pytest completion # Look for patterns like "====== 5638 passed in 491.24s ======" @@ -397,10 +461,13 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: # No "X failed" in line means all passed test_passed = True - print(f"\n Tests completed, terminating process...") + emit(f"\n Tests completed, terminating process...") - # Give 5 seconds grace period for cleanup - time.sleep(5) + # Give 1 second grace period for cleanup + time.sleep(1) + # Final sample while the pid still exists — captures the + # end-of-run high-water mark. + sample_rss() # Terminate the process process.terminate() @@ -408,7 +475,7 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: process.wait(timeout=5) except subprocess.TimeoutExpired: # Force kill if it won't terminate - print(f" Force killing process...") + emit(f" Force killing process...") process.kill() process.wait() break @@ -421,7 +488,8 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: remaining_timeout = max(1800 - elapsed, 1) process.wait(timeout=remaining_timeout) except subprocess.TimeoutExpired: - print(f"\n ⏱️ Timeout - terminating process...") + emit(f"\n ⏱️ Timeout - terminating process...") + sample_rss() process.terminate() try: process.wait(timeout=5) @@ -433,28 +501,40 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: return { "elapsed": elapsed, "status": "timeout", + "peak_rss_mb": peak_rss_mb, + "output": "".join(buf), } elapsed = time.time() - start_time if test_completed: - print(f"\n Batch completed in {elapsed:.1f}s") + emit( + f"\n Batch completed in {elapsed:.1f}s " + f"(peak RSS: {_format_rss(peak_rss_mb)})" + ) return { "elapsed": elapsed, "status": "passed" if test_passed else "failed", + "peak_rss_mb": peak_rss_mb, + "output": "".join(buf), } else: # Process ended without detecting test completion returncode = process.poll() - print(f"\n Batch completed in {elapsed:.1f}s") + emit( + f"\n Batch completed in {elapsed:.1f}s " + f"(peak RSS: {_format_rss(peak_rss_mb)})" + ) return { "elapsed": elapsed, "status": "passed" if returncode == 0 else "failed", + "peak_rss_mb": peak_rss_mb, + "output": "".join(buf), } except Exception as e: elapsed = time.time() - start_time - print(f"\n ❌ Error: {str(e)[:100]}") + emit(f"\n ❌ Error: {str(e)[:100]}") # Clean up process if still running try: @@ -470,9 +550,107 @@ def run_batch(test_paths: List[str], batch_name: str) -> Dict: return { "elapsed": elapsed, "status": "error", + "peak_rss_mb": peak_rss_mb, + "output": "".join(buf), } +def run_batches_concurrently(batches: List[List[str]], workers: int) -> List: + """Run batches in a thread pool, longest-first. + + Each batch is still its own policyengine_command subprocess; the + threads only monitor them. Output is buffered per batch and printed + atomically (header/body/footer) when the batch finishes — only this + (main) thread prints, so concurrent logs never interleave. A shared + heartbeat line reports in-flight batches every ~60s. + + Returns [(batch_name, result_dict), ...] ordered by batch index. + """ + total = len(batches) + # Longest-first (estimated by YAML file count) so the biggest batches + # start immediately and cannot serialize at the tail of the run. + order = sorted( + enumerate(batches, 1), + key=lambda indexed: batch_cost(indexed[1]), + reverse=True, + ) + + in_flight: Dict[str, float] = {} + in_flight_lock = threading.Lock() + + def run_one(idx: int, batch_paths: List[str]) -> Dict: + name = f"Batch {idx}" + with in_flight_lock: + in_flight[name] = time.time() + try: + return run_batch(batch_paths, name, stream=False) + finally: + with in_flight_lock: + in_flight.pop(name, None) + + print( + f"\nRunning {total} batches with {workers} workers " + f"(longest first; output buffered per batch)", + flush=True, + ) + + results: Dict[int, Dict] = {} + heartbeat_interval = 60 + last_heartbeat = time.time() + with ThreadPoolExecutor(max_workers=workers) as executor: + future_info = { + executor.submit(run_one, idx, batch_paths): (idx, batch_paths) + for idx, batch_paths in order + } + pending = set(future_info) + while pending: + done, pending = futures_wait( + pending, timeout=5, return_when=FIRST_COMPLETED + ) + for future in done: + idx, batch_paths = future_info[future] + try: + result = future.result() + except Exception as e: + # run_batch catches its own errors; this guards the + # thin wrapper so one crashed worker cannot hang the + # run or skip the summary. + result = { + "elapsed": 0.0, + "status": "error", + "peak_rss_mb": None, + "output": f"Worker crashed: {str(e)[:200]}\n", + } + results[idx] = result + print(f"\n[Batch {idx}/{total}] {' '.join(batch_paths)}") + print("-" * 60) + print(result.get("output", ""), end="") + print( + f"\n[Batch {idx}/{total}] finished in " + f"{result['elapsed']:.1f}s | peak RSS: " + f"{_format_rss(result.get('peak_rss_mb'))} | " + f"{result['status']}", + flush=True, + ) + now = time.time() + if pending and now - last_heartbeat >= heartbeat_interval: + with in_flight_lock: + running = ", ".join( + f"{name} ({now - started:.0f}s)" + for name, started in sorted( + in_flight.items(), key=lambda item: item[1] + ) + ) + print( + f"[heartbeat] {len(pending)} batches outstanding; " + f"in flight: {running or 'none'}", + flush=True, + ) + last_heartbeat = now + + return [(f"Batch {idx}", results[idx]) for idx in sorted(results)] + + def main(): """Main entry point for the generic test runner.""" @@ -504,6 +682,12 @@ def main(): default="auto", help="Batching mode. 'per-subdir' = each immediate subdir is its own batch; 'per-file' = each yaml is its own batch.", ) + parser.add_argument( + "--workers", + type=int, + default=1, + help="Number of batches to run concurrently (default: 1 = sequential)", + ) args = parser.parse_args() exclude_list = [x.strip() for x in args.exclude.split(",") if x.strip()] @@ -519,6 +703,10 @@ def main(): print(f"Error: --shard I must satisfy 1 <= I <= N (got {args.shard!r})") sys.exit(1) + if args.workers < 1: + print(f"Error: --workers must be >= 1 (got {args.workers})") + sys.exit(1) + if not os.path.exists("policyengine_us"): print("Error: Must run from PolicyEngine US root directory") sys.exit(1) @@ -532,6 +720,8 @@ def main(): print("=" * 60) print(f"Test path: {test_path}") print(f"Requested batches: {args.batches}") + if args.workers > 1: + print(f"Workers: {args.workers}") if exclude_list: print(f"Excluding: {', '.join(exclude_list)}") @@ -586,35 +776,43 @@ def _is_ri(batch): print("=" * 60) # Run batches - all_failed = False - total_elapsed = 0 - - for i, batch_paths in enumerate(batches, 1): - print(f"\n[Batch {i}/{len(batches)}]") + run_start = time.time() + if args.workers > 1: + results = run_batches_concurrently(batches, args.workers) + else: + results = [] + for i, batch_paths in enumerate(batches, 1): + print(f"\n[Batch {i}/{len(batches)}]") - # Show what's in this batch - batch_test_count = sum( - count_yaml_files(Path(p)) if Path(p).is_dir() else 1 for p in batch_paths - ) - print(f" Test files: ~{batch_test_count}") - print("-" * 60) + # Show what's in this batch + print(f" Test files: ~{batch_cost(batch_paths)}") + print("-" * 60) - result = run_batch(batch_paths, f"Batch {i}") - total_elapsed += result["elapsed"] + result = run_batch(batch_paths, f"Batch {i}") + results.append((f"Batch {i}", result)) - if result["status"] != "passed": - all_failed = True + # Memory cleanup after each batch + print(" Cleaning up memory...") + gc.collect() - # Memory cleanup after each batch - print(" Cleaning up memory...") - gc.collect() + all_failed = any(result["status"] != "passed" for _, result in results) + total_elapsed = sum(result["elapsed"] for _, result in results) # Final summary print("\n" + "=" * 60) print("TEST SUMMARY") print("=" * 60) print(f"Total test files: {total_tests}") + for name, result in results: + print( + f" {name}: {result['elapsed']:.1f}s | " + f"peak RSS: {_format_rss(result.get('peak_rss_mb'))} | " + f"{result['status']}" + ) + print(f"Workers: {args.workers}") print(f"Total time: {total_elapsed:.1f}s") + if args.workers > 1: + print(f"Wall time: {time.time() - run_start:.1f}s") # Exit code sys.exit(1 if all_failed else 0) From a5caceddae501103a1a4158a4595518224c971c8 Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 02:59:06 -0400 Subject: [PATCH 3/6] Rebalance CI from 17 to 14 runners with concurrent batches With the NY clone fix, states-ny folds back into the states matrix (8 batches, 2 shards). Reform and hhs merge into the baseline-contrib runner, usda moves to the ssa runner, contrib other-shards 1+3 merge, and the partners job fans analytics_coverage/edge_cases out per topic/state folder (invocation-only; partner files untouched). Heavy targets run batches 2-wide, the light rest job 3-wide. All test jobs set PYTEST_ADDOPTS to print the 25 slowest cases and skip pytest's unraisableexception gc sweep (~11s per subprocess). Co-Authored-By: Claude Fable 5 --- .github/workflows/pr.yaml | 73 +++++++------------ .github/workflows/push.yaml | 77 +++++++------------- Makefile | 114 +++++++++++++++++++----------- changelog.d/ci-speedup.changed.md | 1 + 4 files changed, 123 insertions(+), 142 deletions(-) create mode 100644 changelog.d/ci-speedup.changed.md diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 95dd039b4fc..3532b3d43d7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -103,6 +103,7 @@ jobs: env: SELECTIVE_TEST_MAX_TARGETS: 25 SELECTIVE_TEST_MAX_FILES: 250 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo uses: actions/checkout@v6 @@ -174,24 +175,25 @@ jobs: name: Full Suite - Baseline (${{ matrix.group }}) runs-on: ubuntu-latest timeout-minutes: 60 + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" strategy: fail-fast: false matrix: include: - - group: states-non-ny-shard-1 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 6 --exclude ny --shard 1/2 - - group: states-non-ny-shard-2 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 6 --exclude ny --shard 2/2 - - group: states-ny - cmd: make test-yaml-no-structural-states-ny + - group: states-shard-1 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 1/2 + - group: states-shard-2 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 2/2 - group: irs-household cmd: make test-yaml-no-structural-other-irs-household - - group: ssa - cmd: make test-yaml-no-structural-other-ssa + - group: ssa-usda + cmd: make test-yaml-no-structural-other-ssa-usda - group: rest cmd: make test-yaml-no-structural-other-rest - - group: contrib - cmd: make test-yaml-no-structural-other-contrib + - group: contrib-reform-hhs + cmd: make test-yaml-contrib-reform-hhs steps: - name: Checkout repo uses: actions/checkout@v6 @@ -207,13 +209,14 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Baseline YAML tests (${{ matrix.group }}) - env: - PYTHONUNBUFFERED: 1 run: uv run ${{ matrix.cmd }} HouseholdAPIPartners: name: Household API Partners runs-on: ubuntu-latest timeout-minutes: 60 + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo uses: actions/checkout@v6 @@ -229,13 +232,14 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Household API Partners YAML tests - env: - PYTHONUNBUFFERED: 1 run: uv run make test-yaml-no-structural-other-partners Contrib: name: Full Suite - Contrib (${{ matrix.group }}) runs-on: ubuntu-latest timeout-minutes: 60 + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" strategy: fail-fast: false matrix: @@ -246,12 +250,10 @@ jobs: target: test-yaml-structural-heavy-shard-2 - group: states-shard-3 target: test-yaml-structural-heavy-shard-3 - - group: other-shard-1 - target: test-yaml-structural-other + - group: other-shards-1-3 + target: test-yaml-structural-other-shards-1-3 - group: other-shard-2 target: test-yaml-structural-other-shard-2 - - group: other-shard-3 - target: test-yaml-structural-other-shard-3 - group: congress target: test-yaml-structural-congress steps: @@ -269,18 +271,17 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Contrib YAML tests (${{ matrix.group }}) - env: - PYTHONUNBUFFERED: 1 run: uv run make ${{ matrix.target }} - name: Run Contrib Python tests - if: ${{ matrix.group == 'other-shard-1' }} - env: - PYTHONUNBUFFERED: 1 + if: ${{ matrix.group == 'other-shards-1-3' }} run: uv run make test-policy-contrib-python Rest: name: Full Suite - Rest (Python + variables/ YAMLs) runs-on: ubuntu-latest timeout-minutes: 60 + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo uses: actions/checkout@v6 @@ -296,33 +297,7 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Python tests (code_health, core, microsimulation, utilities, top-level) - env: - PYTHONUNBUFFERED: 1 run: uv run make test-other - name: Run tests/variables YAML tests if: always() - env: - PYTHONUNBUFFERED: 1 run: uv run make test-yaml-variables - Reform: - name: Full Suite - Reform (per-file) - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout repo - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.14 - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - - name: Install dependencies - run: uv sync --extra dev - - name: Turn off default branching - shell: bash - run: bash ./update_itemization.sh - - name: Run tests/policy/reform YAML tests (one batch per file) - env: - PYTHONUNBUFFERED: 1 - run: uv run make test-yaml-reform diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index f2c010598c5..38039185b73 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -63,24 +63,25 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" strategy: fail-fast: false matrix: include: - - group: states-non-ny-shard-1 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 6 --exclude ny --shard 1/2 - - group: states-non-ny-shard-2 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 6 --exclude ny --shard 2/2 - - group: states-ny - cmd: make test-yaml-no-structural-states-ny + - group: states-shard-1 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 1/2 + - group: states-shard-2 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 2/2 - group: irs-household cmd: make test-yaml-no-structural-other-irs-household - - group: ssa - cmd: make test-yaml-no-structural-other-ssa + - group: ssa-usda + cmd: make test-yaml-no-structural-other-ssa-usda - group: rest cmd: make test-yaml-no-structural-other-rest - - group: contrib - cmd: make test-yaml-no-structural-other-contrib + - group: contrib-reform-hhs + cmd: make test-yaml-contrib-reform-hhs steps: - name: Checkout repo uses: actions/checkout@v6 @@ -96,8 +97,6 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Baseline YAML tests (${{ matrix.group }}) - env: - PYTHONUNBUFFERED: 1 run: uv run ${{ matrix.cmd }} HouseholdAPIPartners: name: Household API Partners @@ -106,6 +105,9 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo uses: actions/checkout@v6 @@ -121,8 +123,6 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Household API Partners YAML tests - env: - PYTHONUNBUFFERED: 1 run: uv run make test-yaml-no-structural-other-partners Contrib: name: Full Suite - Contrib (${{ matrix.group }}) @@ -131,6 +131,9 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" strategy: fail-fast: false matrix: @@ -141,12 +144,10 @@ jobs: target: test-yaml-structural-heavy-shard-2 - group: states-shard-3 target: test-yaml-structural-heavy-shard-3 - - group: other-shard-1 - target: test-yaml-structural-other + - group: other-shards-1-3 + target: test-yaml-structural-other-shards-1-3 - group: other-shard-2 target: test-yaml-structural-other-shard-2 - - group: other-shard-3 - target: test-yaml-structural-other-shard-3 - group: congress target: test-yaml-structural-congress steps: @@ -164,13 +165,9 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Contrib YAML tests (${{ matrix.group }}) - env: - PYTHONUNBUFFERED: 1 run: uv run make ${{ matrix.target }} - name: Run Contrib Python tests - if: ${{ matrix.group == 'other-shard-1' }} - env: - PYTHONUNBUFFERED: 1 + if: ${{ matrix.group == 'other-shards-1-3' }} run: uv run make test-policy-contrib-python Rest: name: Full Suite - Rest (Python + variables/ YAMLs) @@ -179,6 +176,9 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo uses: actions/checkout@v6 @@ -194,45 +194,16 @@ jobs: shell: bash run: bash ./update_itemization.sh - name: Run Python tests (code_health, core, microsimulation, utilities, top-level) - env: - PYTHONUNBUFFERED: 1 run: uv run make test-other - name: Run tests/variables YAML tests if: always() - env: - PYTHONUNBUFFERED: 1 run: uv run make test-yaml-variables - Reform: - name: Full Suite - Reform (per-file) - runs-on: ubuntu-latest - timeout-minutes: 60 - if: | - (github.repository == 'PolicyEngine/policyengine-us') - && (github.event.head_commit.message == 'Update PolicyEngine US') - steps: - - name: Checkout repo - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.14 - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - - name: Install dependencies - run: uv sync --extra dev - - name: Turn off default branching - shell: bash - run: bash ./update_itemization.sh - - name: Run tests/policy/reform YAML tests (one batch per file) - env: - PYTHONUNBUFFERED: 1 - run: uv run make test-yaml-reform Publish: runs-on: ubuntu-latest if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') - needs: [Baseline, HouseholdAPIPartners, Contrib, Rest, Reform] + needs: [Baseline, HouseholdAPIPartners, Contrib, Rest] steps: - name: Checkout repo uses: actions/checkout@v6 diff --git a/Makefile b/Makefile index 8bffebf5135..7da4d397cc6 100644 --- a/Makefile +++ b/Makefile @@ -23,21 +23,25 @@ test-yaml-structural: test-yaml-structural-heavy: $(BATCH) $(TESTS)/policy/contrib/states --batches 1 test-yaml-structural-heavy-shard-1: - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 1/3 + # --workers 2 runs two state batches concurrently (~5 GB worst-case each, + # well under the 16 GB runner) instead of leaving the 4-vCPU runner idle. + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 1/3 --workers 2 test-yaml-structural-heavy-shard-2: - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 2/3 + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 2/3 --workers 2 test-yaml-structural-heavy-shard-3: - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 3/3 + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 3/3 --workers 2 test-yaml-structural-other: # Per-subdir so every remaining contrib folder runs in its own subprocess, # instead of stacking ~20 light files into one ~13-min catch-all batch that # risked the 30-min per-batch timeout on slow runners. ssa is excluded (it - # has no YAML tests — only a pytest .py run elsewhere). - $(BATCH) $(TESTS)/policy/contrib --exclude states,ctc,ubi_center,federal,harris,treasury,crfb,congress,refundable_credit_conversion,ssa --mode per-subdir + # has no YAML tests — only a pytest .py run elsewhere). Light folders + # (~1-3 GB peaks) run two at a time. + $(BATCH) $(TESTS)/policy/contrib --exclude states,ctc,ubi_center,federal,harris,treasury,crfb,congress,refundable_credit_conversion,ssa --mode per-subdir --workers 2 test-yaml-structural-other-shard-2: - # ctc + crfb are microsim-heavy: per-file isolation keeps RAM under the cap. - $(BATCH) $(TESTS)/policy/contrib/ctc --mode per-file - $(BATCH) $(TESTS)/policy/contrib/crfb --mode per-file + # ctc + crfb are microsim-heavy: per-file isolation keeps RAM under the + # cap; two ~5 GB per-file peaks fit side by side on the 16 GB runner. + $(BATCH) $(TESTS)/policy/contrib/ctc --mode per-file --workers 2 + $(BATCH) $(TESTS)/policy/contrib/crfb --mode per-file --workers 2 $(BATCH) $(TESTS)/policy/contrib/ubi_center --batches 1 $(BATCH) $(TESTS)/policy/contrib/federal --batches 1 $(BATCH) $(TESTS)/policy/contrib/harris --batches 1 @@ -45,60 +49,90 @@ test-yaml-structural-other-shard-2: test-yaml-structural-other-shard-3: # refundable_credit_conversion force-applies a reform per case; each distinct # gov.contrib.* combination clones the full tax-benefit system (~5 GB peak/ - # file). Per-file isolation frees each peak between files; run on its own - # shard so its ~27-min sweep no longer stacks onto other-shard-1. - $(BATCH) $(TESTS)/policy/contrib/refundable_credit_conversion --mode per-file + # file). Per-file isolation frees each peak between files; two ~5 GB peaks + # fit concurrently under the 16 GB runner. + $(BATCH) $(TESTS)/policy/contrib/refundable_credit_conversion --mode per-file --workers 2 +# other-shard-1 + other-shard-3 share one CI runner: with --workers 2 each +# shard alone no longer fills a 60-min job, so merging frees a runner. +test-yaml-structural-other-shards-1-3: test-yaml-structural-other test-yaml-structural-other-shard-3 test-yaml-structural-congress: - # One subprocess per congress proposal; new proposals auto-route. - $(BATCH) $(TESTS)/policy/contrib/congress --mode per-subdir + # One subprocess per congress proposal; new proposals auto-route and run + # two at a time (~3-5 GB peaks). + $(BATCH) $(TESTS)/policy/contrib/congress --mode per-subdir --workers 2 test-yaml-variables: $(BATCH) $(TESTS)/variables --batches 1 test-yaml-no-structural-states: - $(BATCH) $(TESTS)/policy/baseline/gov/states --batches 4 --exclude ny - $(MAKE) test-yaml-no-structural-states-ny -test-yaml-no-structural-states-ny: - # NY credits clone the tax_benefit_system per scenario (~12 GB) — - # split explicitly. Everything else under ny/ auto-fans out. - $(BATCH) $(TESTS)/policy/baseline/gov/states/ny/tax/income/credits --batches 3 - $(BATCH) $(TESTS)/policy/baseline/gov/states/ny/tax/income --exclude credits --mode per-subdir - $(BATCH) $(TESTS)/policy/baseline/gov/states/ny --exclude tax --mode per-subdir + # NY folds back in now that its credits reuse one cached pinned system + # (#8114) — it peaks ~3-5 GB like other big states. 8 batches (was 6) + # keep each batch small enough that two co-scheduled ones (--workers 2) + # stay well under the 16 GB runner. + $(BATCH) $(TESTS)/policy/baseline/gov/states --batches 8 --workers 2 test-yaml-no-structural-other: $(BATCH) $(TESTS)/policy/baseline --batches 2 --exclude states $(BATCH) $(TESTS)/policy/baseline/household --batches 1 $(BATCH) $(TESTS)/policy/baseline/contrib --batches 1 $(BATCH) $(TESTS)/policy/reform --mode per-file test-yaml-no-structural-other-irs: - # One subprocess per irs subfolder + trailing batch for loose yamls. - $(BATCH) $(TESTS)/policy/baseline/gov/irs --mode per-subdir + # One subprocess per irs subfolder (run two at a time) + trailing batch + # for loose yamls. + $(BATCH) $(TESTS)/policy/baseline/gov/irs --mode per-subdir --workers 2 test-yaml-no-structural-other-household: - $(BATCH) $(TESTS)/policy/baseline/household --batches 2 + # Two batches run concurrently instead of back to back. + $(BATCH) $(TESTS)/policy/baseline/household --batches 2 --workers 2 test-yaml-no-structural-other-irs-household: test-yaml-no-structural-other-irs test-yaml-no-structural-other-household test-yaml-no-structural-other-contrib: - # ubi_center is microsim-heavy → per-file. Other contrib subdirs - # (biden, states, + any future folder) auto-fan out. - $(BATCH) $(TESTS)/policy/baseline/contrib/ubi_center --mode per-file - $(BATCH) $(TESTS)/policy/baseline/contrib --exclude ubi_center --mode per-subdir + # ubi_center is microsim-heavy → per-file, two ~4 GB peaks at a time. + # Other contrib subdirs (biden, states, + any future folder) auto-fan out. + $(BATCH) $(TESTS)/policy/baseline/contrib/ubi_center --mode per-file --workers 2 + $(BATCH) $(TESTS)/policy/baseline/contrib --exclude ubi_center --mode per-subdir --workers 2 test-yaml-reform: # Reforms are force-applied and deepcopy the full parameter tree # (~5.5 GB peak/file for ctc_linear_phase_out and winship, measured). # Running all files in one subprocess stacks past the 16 GB runner cap # → "runner received a shutdown signal". One batch per file frees each - # peak between files; new reform files auto-route. - $(BATCH) $(TESTS)/policy/reform --mode per-file + # peak between files (two ~5.5 GB peaks fit concurrently); new reform + # files auto-route. + $(BATCH) $(TESTS)/policy/reform --mode per-file --workers 2 +test-yaml-no-structural-other-hhs: + # hhs (~2.7 GB peak) moved out of the rest job to ride along with the + # baseline-contrib + reform runner, which has spare headroom. + $(BATCH) $(TESTS)/policy/baseline/gov/hhs --batches 1 +# baseline contrib + policy/reform + gov/hhs share one CI runner: each piece +# alone is far below a full job, so merging frees two runners. +test-yaml-contrib-reform-hhs: test-yaml-no-structural-other-contrib test-yaml-reform test-yaml-no-structural-other-hhs test-yaml-no-structural-other-ssa: - # revenue is heavy enough to need its own 2-batch split; others auto-fan. - $(BATCH) $(TESTS)/policy/baseline/gov/ssa/revenue --batches 2 - $(BATCH) $(TESTS)/policy/baseline/gov/ssa --exclude revenue --mode per-subdir + # revenue is heavy enough to need its own 2-batch split (run both at + # once); other ssa subfolders auto-fan two at a time. + $(BATCH) $(TESTS)/policy/baseline/gov/ssa/revenue --batches 2 --workers 2 + $(BATCH) $(TESTS)/policy/baseline/gov/ssa --exclude revenue --mode per-subdir --workers 2 +test-yaml-no-structural-other-usda: + # usda (~3 GB peak, ~7 min) moved out of the rest job to ride along + # with ssa, rebalancing the two runners. + $(BATCH) $(TESTS)/policy/baseline/gov/usda --batches 1 +test-yaml-no-structural-other-ssa-usda: test-yaml-no-structural-other-ssa test-yaml-no-structural-other-usda test-yaml-no-structural-other-rest: - # All remaining gov/ subdirs + any new ones auto-route here. - $(BATCH) $(TESTS)/policy/baseline/gov --exclude states,irs,ssa --mode per-subdir - # All top-level baseline/ subdirs except gov/household/contrib/partners - # (calcfunctions, income, parameters + any new folder) auto-route here. - $(BATCH) $(TESTS)/policy/baseline --exclude gov,household,contrib,partners --mode per-subdir + # All remaining gov/ subdirs + any new ones auto-route here (usda and + # hhs moved to the ssa-usda and contrib-reform-hhs jobs). Every batch + # peaks <=3.6 GB, so three run concurrently. + $(BATCH) $(TESTS)/policy/baseline/gov --exclude states,irs,ssa,usda,hhs --mode per-subdir --workers 3 + # calcfunctions, income, parameters + any new top-level baseline/ folder + # are all light (<2.3 GB peak) — group them into one subprocess instead + # of paying the ~33s interpreter+system-build startup per folder. + $(BATCH) $(TESTS)/policy/baseline --exclude gov,household,contrib,partners --batches 1 test-yaml-no-structural-other-partners: # Customer/API partner fixtures mirrored from policyengine-household-api. - # One subprocess per partner; new partners auto-route. - $(BATCH) $(TESTS)/policy/baseline/partners --mode per-subdir + # analytics_coverage/edge_cases is ~90% of this job's time as a single + # batch — fan it out per topic/state folder and run two at a time. Only + # the invocations change here; partner files themselves are untouched. + $(BATCH) $(TESTS)/policy/baseline/partners/analytics_coverage/edge_cases/federal --mode per-subdir --workers 2 + $(BATCH) $(TESTS)/policy/baseline/partners/analytics_coverage/edge_cases/state --mode per-subdir --workers 2 + # Safety net: anything added directly under edge_cases/ besides federal + # and state (currently produces zero batches). + $(BATCH) $(TESTS)/policy/baseline/partners/analytics_coverage/edge_cases --exclude federal,state --batches 1 + # signatures + anything new under analytics_coverage/ as one batch. + $(BATCH) $(TESTS)/policy/baseline/partners/analytics_coverage --exclude edge_cases --batches 1 + # amplifi + impactica + my_friend_ben (+ any new partner) in one light batch. + $(BATCH) $(TESTS)/policy/baseline/partners --exclude analytics_coverage --batches 1 test-other: pytest policyengine_us/tests/ --maxfail=0 --ignore=$(TESTS)/policy/contrib test-policy-contrib-python: diff --git a/changelog.d/ci-speedup.changed.md b/changelog.d/ci-speedup.changed.md new file mode 100644 index 00000000000..f474df48626 --- /dev/null +++ b/changelog.d/ci-speedup.changed.md @@ -0,0 +1 @@ +Speed up CI: cache New York's pinned EITC/CTC tax-benefit systems (#8114), run test batches concurrently within each runner, rebalance CI jobs from 17 to 14 runners, and report per-case durations and per-batch peak memory in CI logs. From 1082d24abc83f8a1ee266cdb868421b716a705be Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 02:59:06 -0400 Subject: [PATCH 4/6] Halve microsimulation test builds and subsample heavy LSR/CG tests test_microsim.py now builds one Microsimulation per dataset via a module-scoped fixture shared across the parametrized years (2 full builds instead of 4; ~17 min of the Rest CI job). The RUN_HEAVY_TESTS LSR/CG interaction tests (still skipped by default) subsample to 10,000 households and reuse a single baseline, so opting in is cheaper. Co-Authored-By: Claude Fable 5 --- .../test_behavioral_structural.py | 35 +++++++++++++------ .../test_lsr_cg_interaction.py | 32 ++++++++++------- .../tests/microsimulation/test_microsim.py | 18 +++++++--- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/policyengine_us/tests/microsimulation/test_behavioral_structural.py b/policyengine_us/tests/microsimulation/test_behavioral_structural.py index 5109059309f..65c4ba25e7e 100644 --- a/policyengine_us/tests/microsimulation/test_behavioral_structural.py +++ b/policyengine_us/tests/microsimulation/test_behavioral_structural.py @@ -20,6 +20,14 @@ TEST_YEAR = 2026 DATE_RANGE = "2020-01-01.2100-12-31" +# Subsample size for the default dataset. Larger than the 1,000 used in +# test_microsim.py because the surtax and CG reforms here mostly affect +# high-AGI / high-capital-gains households, which a smaller sample can +# miss entirely. subsample() seeds from the dataset name, so every +# simulation in this module draws the same households and the impacts +# stay comparable; the assertions are ratio-based, so they are +# subsample-scale-free. +SUBSAMPLE_SIZE = 10_000 def _make_structural_surtax_reform(): @@ -82,17 +90,18 @@ def _make_cg_reform(): ) -def _tax_impact(reform_tuple): - sim_bl = Microsimulation() - sim_rf = Microsimulation(reform=reform_tuple) - t_bl = sim_bl.calculate("income_tax", period=TEST_YEAR, map_to="household") - t_rf = sim_rf.calculate("income_tax", period=TEST_YEAR, map_to="household") - return float((t_rf - t_bl).sum()) +def _tax(reform_tuple=None): + """Total federal income tax on a fixed subsample, optionally reformed.""" + sim = Microsimulation(reform=reform_tuple) + sim.subsample(SUBSAMPLE_SIZE) + return float( + sim.calculate("income_tax", period=TEST_YEAR, map_to="household").sum() + ) @pytest.mark.skipif( os.environ.get("RUN_HEAVY_TESTS") != "1", - reason="Requires ~80min; set RUN_HEAVY_TESTS=1", + reason="Heavy: builds 4 default-dataset microsimulations; set RUN_HEAVY_TESTS=1", ) def test_structural_reform_with_lsr_and_cg(): """Structural reform + LSR + CG should produce bounded results.""" @@ -100,9 +109,12 @@ def test_structural_reform_with_lsr_and_cg(): lsr = _make_lsr_reform() cg = _make_cg_reform() - impact_lsr = _tax_impact(structural + (lsr,)) - impact_cg = _tax_impact(structural + (cg,)) - impact_both = _tax_impact(structural + (lsr, cg)) + # The deterministic subsample makes the baseline identical for all + # three impacts, so compute it once. + tax_baseline = _tax() + impact_lsr = _tax(structural + (lsr,)) - tax_baseline + impact_cg = _tax(structural + (cg,)) - tax_baseline + impact_both = _tax(structural + (lsr, cg)) - tax_baseline individual_sum = abs(impact_lsr) + abs(impact_cg) assert abs(impact_both) < 3 * individual_sum, ( @@ -113,7 +125,7 @@ def test_structural_reform_with_lsr_and_cg(): @pytest.mark.skipif( os.environ.get("RUN_HEAVY_TESTS") != "1", - reason="Requires ~40min; set RUN_HEAVY_TESTS=1", + reason="Heavy: builds a default-dataset microsimulation; set RUN_HEAVY_TESTS=1", ) def test_structural_reform_cg_response_bounded(): """CG response with structural reform should not produce extreme values. @@ -125,6 +137,7 @@ def test_structural_reform_cg_response_bounded(): cg = _make_cg_reform() sim = Microsimulation(reform=structural + (cg,)) + sim.subsample(SUBSAMPLE_SIZE) cg_resp = np.array( sim.calculate("capital_gains_behavioral_response", period=TEST_YEAR) ) diff --git a/policyengine_us/tests/microsimulation/test_lsr_cg_interaction.py b/policyengine_us/tests/microsimulation/test_lsr_cg_interaction.py index dd6a2d83d64..3909a8a453b 100644 --- a/policyengine_us/tests/microsimulation/test_lsr_cg_interaction.py +++ b/policyengine_us/tests/microsimulation/test_lsr_cg_interaction.py @@ -15,15 +15,20 @@ YEAR = 2026 DATE_RANGE = "2020-01-01.2100-12-31" +# Subsample size for the default dataset. Larger than the 1,000 used in +# test_microsim.py because the reforms here mostly affect top-bracket and +# high-capital-gains households, which a smaller sample can miss entirely. +# subsample() seeds from the dataset name, so every simulation in this +# module draws the same households and the impacts stay comparable; the +# assertions are ratio-based, so they are subsample-scale-free. +SUBSAMPLE_SIZE = 10_000 -def _federal_tax_impact(reform_tuple): - """Calculate federal tax revenue impact for a reform.""" - sim_baseline = Microsimulation() - sim_reform = Microsimulation(reform=reform_tuple) - tax_baseline = sim_baseline.calculate("income_tax", period=YEAR, map_to="household") - tax_reform = sim_reform.calculate("income_tax", period=YEAR, map_to="household") - return float((tax_reform - tax_baseline).sum()) +def _federal_tax(reform_tuple=None): + """Total federal income tax on a fixed subsample, optionally reformed.""" + sim = Microsimulation(reform=reform_tuple) + sim.subsample(SUBSAMPLE_SIZE) + return float(sim.calculate("income_tax", period=YEAR, map_to="household").sum()) def _make_tax_reform(): @@ -89,9 +94,12 @@ def _assert_combined_bounded(reform_tuple, label): lsr = _make_lsr_reform() cg = _make_cg_reform() - impact_lsr = _federal_tax_impact(reform_tuple + (lsr,)) - impact_cg = _federal_tax_impact(reform_tuple + (cg,)) - impact_both = _federal_tax_impact(reform_tuple + (lsr, cg)) + # The deterministic subsample makes the baseline identical for all + # three impacts, so compute it once. + tax_baseline = _federal_tax() + impact_lsr = _federal_tax(reform_tuple + (lsr,)) - tax_baseline + impact_cg = _federal_tax(reform_tuple + (cg,)) - tax_baseline + impact_both = _federal_tax(reform_tuple + (lsr, cg)) - tax_baseline individual_sum = abs(impact_lsr) + abs(impact_cg) assert abs(impact_both) < 3 * individual_sum, ( @@ -105,7 +113,7 @@ def _assert_combined_bounded(reform_tuple, label): @pytest.mark.skipif( os.environ.get("RUN_HEAVY_TESTS") != "1", - reason="Requires ~60min for 3 microsimulations; set RUN_HEAVY_TESTS=1", + reason="Heavy: builds 4 default-dataset microsimulations; set RUN_HEAVY_TESTS=1", ) def test_combined_lsr_cg_parametric_reform(): """Parameter-only reform: combined LSR+CG should be bounded.""" @@ -114,7 +122,7 @@ def test_combined_lsr_cg_parametric_reform(): @pytest.mark.skipif( os.environ.get("RUN_HEAVY_TESTS") != "1", - reason="Requires ~80min for 3 microsimulations; set RUN_HEAVY_TESTS=1", + reason="Heavy: builds 4 default-dataset microsimulations; set RUN_HEAVY_TESTS=1", ) def test_combined_lsr_cg_structural_reform(): """Structural reform (WATCA): combined LSR+CG should be bounded. diff --git a/policyengine_us/tests/microsimulation/test_microsim.py b/policyengine_us/tests/microsimulation/test_microsim.py index ee065929468..331fc436bf5 100644 --- a/policyengine_us/tests/microsimulation/test_microsim.py +++ b/policyengine_us/tests/microsimulation/test_microsim.py @@ -8,14 +8,22 @@ YEARS = list(range(2024, 2026)) -@pytest.mark.parametrize("dataset", DATASETS) -@pytest.mark.parametrize("year", YEARS) -def test_microsim_runs(dataset: str, year: int): - import numpy as np +@pytest.fixture(scope="module", params=DATASETS) +def dataset_sim(request): + """One subsampled Microsimulation per dataset, shared across the + parametrized years (the tests only read via calc, so sharing is safe).""" from policyengine_us import Microsimulation - sim = Microsimulation(dataset=dataset) + sim = Microsimulation(dataset=request.param) sim.subsample(1_000) + return sim + + +@pytest.mark.parametrize("year", YEARS) +def test_microsim_runs(dataset_sim, year: int): + import numpy as np + + sim = dataset_sim hnet = sim.calc("household_net_income", period=year) assert not hnet.isna().any(), "Some households have NaN net income." # Deciles are 1-10, with -1 for negative income. From 23c75ebec7b19480c906031a8aeb90442e7dca10 Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 11:12:35 -0400 Subject: [PATCH 5/6] Re-batch CI from measured peak RSS: single-worker batches, 23 runners Round-1's --workers 2 co-scheduling OOM-killed four runners: the new per-batch RSS logging showed single batches already peak far higher on Linux than local estimates suggested (ri/ctc_reform_test.yaml 14.6 GB alone, contrib/states/oh 12.4 GB, baseline/contrib/states 12.2 GB, gov/simulation 12.1 GB, half of baseline/household 11.8 GB). Headroom-first re-layout: every heavy target runs one batch at a time (--workers 2 kept only for partners, max batch 2.5 GB), oversized batches are split finer (states --batches 16 over 4 shards, household --batches 4, gov/simulation and baseline/contrib/states per-file, oh joins ri in PER_FILE_STATES), the two files too big for any batching are split by reform combo (RI CTC into 3, CRFB payroll into 2, cases byte-identical), and the microsimulation pytest run gets its own job since the combined process was OOM-killed at 94%. Every subprocess now stays at or below ~8 GB, leaving half the 16 GB runner free for future test growth. Quick Feedback's selective cap drops to 100 files (one subprocess with 250 yaml files exceeds runner memory; over-cap safely defers to the full suites). Co-Authored-By: Claude Fable 5 --- .github/workflows/pr.yaml | 83 +++- .github/workflows/push.yaml | 81 +++- Makefile | 158 ++++--- changelog.d/ci-speedup.changed.md | 2 +- ...employer_payroll_tax_percentage_basic.yaml | 75 ++++ ...employer_payroll_tax_percentage_edge.yaml} | 76 ---- .../ri/ctc_reform_refundability_test.yaml | 178 ++++++++ .../ri/ctc_reform_stepped_phaseout_test.yaml | 67 +++ .../contrib/states/ri/ctc_reform_test.yaml | 390 ------------------ .../ri/ctc_reform_young_child_boost_test.yaml | 143 +++++++ policyengine_us/tests/test_batched.py | 5 +- 11 files changed, 697 insertions(+), 561 deletions(-) create mode 100644 policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_basic.yaml rename policyengine_us/tests/policy/contrib/crfb/{tax_employer_payroll_tax_percentage.yaml => tax_employer_payroll_tax_percentage_edge.yaml} (54%) create mode 100644 policyengine_us/tests/policy/contrib/states/ri/ctc_reform_refundability_test.yaml create mode 100644 policyengine_us/tests/policy/contrib/states/ri/ctc_reform_stepped_phaseout_test.yaml delete mode 100644 policyengine_us/tests/policy/contrib/states/ri/ctc_reform_test.yaml create mode 100644 policyengine_us/tests/policy/contrib/states/ri/ctc_reform_young_child_boost_test.yaml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3532b3d43d7..93cfc73200a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -102,7 +102,12 @@ jobs: runs-on: ubuntu-latest env: SELECTIVE_TEST_MAX_TARGETS: 25 - SELECTIVE_TEST_MAX_FILES: 250 + # 100 (was 250): selective runs pack every matched yaml into ONE + # policyengine-core subprocess, and ~250 yaml files retained ~15 GB + # and OOM'd the 16 GB runner on run 28698452678. When the limit is + # exceeded run_selective_tests.py safely narrows to directly changed + # test files (or skips, deferring to the full suite jobs). + SELECTIVE_TEST_MAX_FILES: 100 PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" steps: - name: Checkout repo @@ -182,18 +187,30 @@ jobs: fail-fast: false matrix: include: + # 16 batches / 4 shards / --workers 1: the 8-batch 2-wide shards + # OOM'd on run 28698452678; see the Makefile memory-layout notes. - group: states-shard-1 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 1/2 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 1/4 - group: states-shard-2 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 2/2 - - group: irs-household - cmd: make test-yaml-no-structural-other-irs-household + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 2/4 + - group: states-shard-3 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 3/4 + - group: states-shard-4 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 4/4 + - group: irs + cmd: make test-yaml-no-structural-other-irs + - group: household + cmd: make test-yaml-no-structural-other-household - group: ssa-usda cmd: make test-yaml-no-structural-other-ssa-usda - - group: rest - cmd: make test-yaml-no-structural-other-rest - - group: contrib-reform-hhs - cmd: make test-yaml-contrib-reform-hhs + - group: rest-a + cmd: make test-yaml-no-structural-other-rest-a + - group: rest-b + cmd: make test-yaml-no-structural-other-rest-b + - group: contrib-hhs + cmd: make test-yaml-contrib-hhs + - group: reform + cmd: make test-yaml-reform steps: - name: Checkout repo uses: actions/checkout@v6 @@ -250,10 +267,16 @@ jobs: target: test-yaml-structural-heavy-shard-2 - group: states-shard-3 target: test-yaml-structural-heavy-shard-3 - - group: other-shards-1-3 - target: test-yaml-structural-other-shards-1-3 - - group: other-shard-2 - target: test-yaml-structural-other-shard-2 + - group: states-shard-4 + target: test-yaml-structural-heavy-shard-4 + - group: other-shard-1 + target: test-yaml-structural-other + - group: other-shard-2a + target: test-yaml-structural-other-shard-2a + - group: other-shard-2b + target: test-yaml-structural-other-shard-2b + - group: other-shard-3 + target: test-yaml-structural-other-shard-3 - group: congress target: test-yaml-structural-congress steps: @@ -273,10 +296,10 @@ jobs: - name: Run Contrib YAML tests (${{ matrix.group }}) run: uv run make ${{ matrix.target }} - name: Run Contrib Python tests - if: ${{ matrix.group == 'other-shards-1-3' }} + if: ${{ matrix.group == 'other-shard-1' }} run: uv run make test-policy-contrib-python Rest: - name: Full Suite - Rest (Python + variables/ YAMLs) + name: Full Suite - Rest (Python + variables) runs-on: ubuntu-latest timeout-minutes: 60 env: @@ -296,8 +319,34 @@ jobs: - name: Turn off default branching shell: bash run: bash ./update_itemization.sh - - name: Run Python tests (code_health, core, microsimulation, utilities, top-level) - run: uv run make test-other + - name: Run Python tests (code_health, core, utilities, top-level) + run: uv run make test-other-python - name: Run tests/variables YAML tests if: always() run: uv run make test-yaml-variables + Microsimulation: + name: Full Suite - Microsimulation + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.14 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + - name: Install dependencies + run: uv sync --extra dev + - name: Turn off default branching + shell: bash + run: bash ./update_itemization.sh + # Split out of the Rest job: the combined single-process pytest run + # was OOM-killed at ~94% on run 28698452678 once microsimulation + # peaks stacked on the accumulated python-test RSS. + - name: Run microsimulation tests + run: uv run make test-microsimulation diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 38039185b73..1a5363d47f9 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -70,18 +70,30 @@ jobs: fail-fast: false matrix: include: + # 16 batches / 4 shards / --workers 1: the 8-batch 2-wide shards + # OOM'd on run 28698452678; see the Makefile memory-layout notes. - group: states-shard-1 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 1/2 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 1/4 - group: states-shard-2 - cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 8 --workers 2 --shard 2/2 - - group: irs-household - cmd: make test-yaml-no-structural-other-irs-household + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 2/4 + - group: states-shard-3 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 3/4 + - group: states-shard-4 + cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --batches 16 --workers 1 --shard 4/4 + - group: irs + cmd: make test-yaml-no-structural-other-irs + - group: household + cmd: make test-yaml-no-structural-other-household - group: ssa-usda cmd: make test-yaml-no-structural-other-ssa-usda - - group: rest - cmd: make test-yaml-no-structural-other-rest - - group: contrib-reform-hhs - cmd: make test-yaml-contrib-reform-hhs + - group: rest-a + cmd: make test-yaml-no-structural-other-rest-a + - group: rest-b + cmd: make test-yaml-no-structural-other-rest-b + - group: contrib-hhs + cmd: make test-yaml-contrib-hhs + - group: reform + cmd: make test-yaml-reform steps: - name: Checkout repo uses: actions/checkout@v6 @@ -144,10 +156,16 @@ jobs: target: test-yaml-structural-heavy-shard-2 - group: states-shard-3 target: test-yaml-structural-heavy-shard-3 - - group: other-shards-1-3 - target: test-yaml-structural-other-shards-1-3 - - group: other-shard-2 - target: test-yaml-structural-other-shard-2 + - group: states-shard-4 + target: test-yaml-structural-heavy-shard-4 + - group: other-shard-1 + target: test-yaml-structural-other + - group: other-shard-2a + target: test-yaml-structural-other-shard-2a + - group: other-shard-2b + target: test-yaml-structural-other-shard-2b + - group: other-shard-3 + target: test-yaml-structural-other-shard-3 - group: congress target: test-yaml-structural-congress steps: @@ -167,10 +185,10 @@ jobs: - name: Run Contrib YAML tests (${{ matrix.group }}) run: uv run make ${{ matrix.target }} - name: Run Contrib Python tests - if: ${{ matrix.group == 'other-shards-1-3' }} + if: ${{ matrix.group == 'other-shard-1' }} run: uv run make test-policy-contrib-python Rest: - name: Full Suite - Rest (Python + variables/ YAMLs) + name: Full Suite - Rest (Python + variables) runs-on: ubuntu-latest timeout-minutes: 60 if: | @@ -193,17 +211,46 @@ jobs: - name: Turn off default branching shell: bash run: bash ./update_itemization.sh - - name: Run Python tests (code_health, core, microsimulation, utilities, top-level) - run: uv run make test-other + - name: Run Python tests (code_health, core, utilities, top-level) + run: uv run make test-other-python - name: Run tests/variables YAML tests if: always() run: uv run make test-yaml-variables + Microsimulation: + name: Full Suite - Microsimulation + runs-on: ubuntu-latest + timeout-minutes: 60 + if: | + (github.repository == 'PolicyEngine/policyengine-us') + && (github.event.head_commit.message == 'Update PolicyEngine US') + env: + PYTHONUNBUFFERED: 1 + PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.14 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + - name: Install dependencies + run: uv sync --extra dev + - name: Turn off default branching + shell: bash + run: bash ./update_itemization.sh + # Split out of the Rest job: the combined single-process pytest run + # was OOM-killed at ~94% on run 28698452678 once microsimulation + # peaks stacked on the accumulated python-test RSS. + - name: Run microsimulation tests + run: uv run make test-microsimulation Publish: runs-on: ubuntu-latest if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') - needs: [Baseline, HouseholdAPIPartners, Contrib, Rest] + needs: [Baseline, HouseholdAPIPartners, Contrib, Rest, Microsimulation] steps: - name: Checkout repo uses: actions/checkout@v6 diff --git a/Makefile b/Makefile index 7da4d397cc6..44a67a07202 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,12 @@ # --mode per-file = each yaml runs in its own subprocess. Used for # microsim-heavy folders where one file per subprocess # is needed to keep peak RAM under the 16 GB runner. +# +# Memory layout (round 2), calibrated against measured Linux per-batch +# peak RSS from CI run 28698452678 on 16 GB ubuntu-latest runners: +# every batch targets <= ~8 GB peak so >= ~7 GB stays free for future +# test growth, and --workers 1 everywhere except the partners target +# (its largest batch measured 2.5 GB, so two-wide stays trivially safe). BATCH := python policyengine_us/tests/test_batched.py TESTS := policyengine_us/tests @@ -22,99 +28,127 @@ test-yaml-structural: $(BATCH) $(TESTS)/policy/contrib --exclude states test-yaml-structural-heavy: $(BATCH) $(TESTS)/policy/contrib/states --batches 1 +# Contrib states: 4 shards (was 3), one batch at a time. states-shard-1 +# OOM'd two-wide on CI run 28698452678 — oh's folder alone peaked +# 12.4 GB before joining test_batched.py's PER_FILE_STATES set, and ri's +# ctc reform file peaked 14.6 GB before it was split into three files. +# With oh/ri per-file plus the ri split, the worst subprocess is ~6-7 GB; +# --workers 1 keeps only one such peak resident. test-yaml-structural-heavy-shard-1: - # --workers 2 runs two state batches concurrently (~5 GB worst-case each, - # well under the 16 GB runner) instead of leaving the 4-vCPU runner idle. - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 1/3 --workers 2 + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 1/4 --workers 1 test-yaml-structural-heavy-shard-2: - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 2/3 --workers 2 + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 2/4 --workers 1 test-yaml-structural-heavy-shard-3: - $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 3/3 --workers 2 + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 3/4 --workers 1 +test-yaml-structural-heavy-shard-4: + $(BATCH) $(TESTS)/policy/contrib/states --batches 1 --shard 4/4 --workers 1 test-yaml-structural-other: # Per-subdir so every remaining contrib folder runs in its own subprocess, # instead of stacking ~20 light files into one ~13-min catch-all batch that # risked the 30-min per-batch timeout on slow runners. ssa is excluded (it - # has no YAML tests — only a pytest .py run elsewhere). Light folders - # (~1-3 GB peaks) run two at a time. - $(BATCH) $(TESTS)/policy/contrib --exclude states,ctc,ubi_center,federal,harris,treasury,crfb,congress,refundable_credit_conversion,ssa --mode per-subdir --workers 2 -test-yaml-structural-other-shard-2: - # ctc + crfb are microsim-heavy: per-file isolation keeps RAM under the - # cap; two ~5 GB per-file peaks fit side by side on the 16 GB runner. - $(BATCH) $(TESTS)/policy/contrib/ctc --mode per-file --workers 2 - $(BATCH) $(TESTS)/policy/contrib/crfb --mode per-file --workers 2 + # has no YAML tests — only a pytest .py run elsewhere). --workers 1 so a + # single ~1-5 GB peak is resident at a time, leaving headroom for growth. + $(BATCH) $(TESTS)/policy/contrib --exclude states,ctc,ubi_center,federal,harris,treasury,crfb,congress,refundable_credit_conversion,ssa --mode per-subdir --workers 1 +test-yaml-structural-other-shard-2a: + # ctc is microsim-heavy: per-file isolation frees each ~5 GB peak between + # files, and --workers 1 keeps only one peak resident at a time. + $(BATCH) $(TESTS)/policy/contrib/ctc --mode per-file --workers 1 $(BATCH) $(TESTS)/policy/contrib/ubi_center --batches 1 $(BATCH) $(TESTS)/policy/contrib/federal --batches 1 +test-yaml-structural-other-shard-2b: + # crfb runs per-file: tax_employer_payroll_tax_percentage peaked 9.6 GB as + # a single file on CI run 28698452678 and is now split into two ~3-case + # files so each subprocess stays <= ~5 GB; --workers 1 keeps one peak + # resident at a time. + $(BATCH) $(TESTS)/policy/contrib/crfb --mode per-file --workers 1 $(BATCH) $(TESTS)/policy/contrib/harris --batches 1 $(BATCH) $(TESTS)/policy/contrib/treasury --batches 1 test-yaml-structural-other-shard-3: # refundable_credit_conversion force-applies a reform per case; each distinct # gov.contrib.* combination clones the full tax-benefit system (~5 GB peak/ - # file). Per-file isolation frees each peak between files; two ~5 GB peaks - # fit concurrently under the 16 GB runner. - $(BATCH) $(TESTS)/policy/contrib/refundable_credit_conversion --mode per-file --workers 2 -# other-shard-1 + other-shard-3 share one CI runner: with --workers 2 each -# shard alone no longer fills a 60-min job, so merging frees a runner. -test-yaml-structural-other-shards-1-3: test-yaml-structural-other test-yaml-structural-other-shard-3 + # file). Per-file isolation frees each peak between files; --workers 1 keeps + # one peak resident at a time. + $(BATCH) $(TESTS)/policy/contrib/refundable_credit_conversion --mode per-file --workers 1 test-yaml-structural-congress: - # One subprocess per congress proposal; new proposals auto-route and run - # two at a time (~3-5 GB peaks). - $(BATCH) $(TESTS)/policy/contrib/congress --mode per-subdir --workers 2 + # One subprocess per congress proposal; new proposals auto-route. + # --workers 1: congress OOM'd two-wide on CI run 28698452678 — the + # romney batch alone peaked 7.1 GB. + $(BATCH) $(TESTS)/policy/contrib/congress --mode per-subdir --workers 1 test-yaml-variables: $(BATCH) $(TESTS)/variables --batches 1 test-yaml-no-structural-states: - # NY folds back in now that its credits reuse one cached pinned system - # (#8114) — it peaks ~3-5 GB like other big states. 8 batches (was 6) - # keep each batch small enough that two co-scheduled ones (--workers 2) - # stay well under the 16 GB runner. - $(BATCH) $(TESTS)/policy/baseline/gov/states --batches 8 --workers 2 + # 16 batches (was 8) ~= ~3 states per batch. Both 8-batch state shard + # jobs OOM'd two-wide on CI run 28698452678 (each ~6-state batch peaks + # ~8+ GB); halving the batch size at --workers 1 keeps each subprocess + # <= ~6 GB with >= ~10 GB free for future state growth. + $(BATCH) $(TESTS)/policy/baseline/gov/states --batches 16 --workers 1 test-yaml-no-structural-other: $(BATCH) $(TESTS)/policy/baseline --batches 2 --exclude states $(BATCH) $(TESTS)/policy/baseline/household --batches 1 $(BATCH) $(TESTS)/policy/baseline/contrib --batches 1 $(BATCH) $(TESTS)/policy/reform --mode per-file test-yaml-no-structural-other-irs: - # One subprocess per irs subfolder (run two at a time) + trailing batch - # for loose yamls. - $(BATCH) $(TESTS)/policy/baseline/gov/irs --mode per-subdir --workers 2 + # One subprocess per irs subfolder + trailing batch for loose yamls. + # --workers 1: the irs/tax batch alone peaked 7.6 GB on CI run + # 28698452678 — co-scheduling a second batch leaves no headroom. + $(BATCH) $(TESTS)/policy/baseline/gov/irs --mode per-subdir --workers 1 test-yaml-no-structural-other-household: - # Two batches run concurrently instead of back to back. - $(BATCH) $(TESTS)/policy/baseline/household --batches 2 --workers 2 -test-yaml-no-structural-other-irs-household: test-yaml-no-structural-other-irs test-yaml-no-structural-other-household + # 4 batches (was 2), --workers 1: a 2-batch half of this folder (incl. + # the weights/MTR files) peaked 11.8 GB on CI run 28698452678. Quarter + # batches keep each subprocess <= ~6 GB. + $(BATCH) $(TESTS)/policy/baseline/household --batches 4 --workers 1 test-yaml-no-structural-other-contrib: - # ubi_center is microsim-heavy → per-file, two ~4 GB peaks at a time. - # Other contrib subdirs (biden, states, + any future folder) auto-fan out. - $(BATCH) $(TESTS)/policy/baseline/contrib/ubi_center --mode per-file --workers 2 - $(BATCH) $(TESTS)/policy/baseline/contrib --exclude ubi_center --mode per-subdir --workers 2 + # ubi_center is microsim-heavy → per-file, one ~4 GB peak at a time. + $(BATCH) $(TESTS)/policy/baseline/contrib/ubi_center --mode per-file --workers 1 + # baseline/contrib/states peaked 12.2 GB as a single per-subdir batch on + # CI run 28698452678, and its yamls sit two levels deep (states/ri/*/), + # so per-subdir here would still yield one big "ri" batch — per-file + # gives each reform yaml its own subprocess instead. + $(BATCH) $(TESTS)/policy/baseline/contrib/states --mode per-file --workers 1 + # Other contrib subdirs (biden + any future folder) auto-fan out. + $(BATCH) $(TESTS)/policy/baseline/contrib --exclude ubi_center,states --mode per-subdir --workers 1 test-yaml-reform: # Reforms are force-applied and deepcopy the full parameter tree # (~5.5 GB peak/file for ctc_linear_phase_out and winship, measured). # Running all files in one subprocess stacks past the 16 GB runner cap # → "runner received a shutdown signal". One batch per file frees each - # peak between files (two ~5.5 GB peaks fit concurrently); new reform - # files auto-route. - $(BATCH) $(TESTS)/policy/reform --mode per-file --workers 2 + # peak between files; --workers 1 keeps a single peak resident (even an + # ~8 GB file runs solo with ~8 GB spare); new reform files auto-route. + $(BATCH) $(TESTS)/policy/reform --mode per-file --workers 1 test-yaml-no-structural-other-hhs: - # hhs (~2.7 GB peak) moved out of the rest job to ride along with the - # baseline-contrib + reform runner, which has spare headroom. + # hhs (~2.7 GB peak) rides along with the baseline-contrib runner, + # which has spare headroom. $(BATCH) $(TESTS)/policy/baseline/gov/hhs --batches 1 -# baseline contrib + policy/reform + gov/hhs share one CI runner: each piece -# alone is far below a full job, so merging frees two runners. -test-yaml-contrib-reform-hhs: test-yaml-no-structural-other-contrib test-yaml-reform test-yaml-no-structural-other-hhs +# baseline contrib + gov/hhs share one CI runner. policy/reform no longer +# rides along: its per-file peaks (~5.5-8 GB) plus this pair filled a job +# past the safety margin, so reform gets its own runner. +test-yaml-contrib-hhs: test-yaml-no-structural-other-contrib test-yaml-no-structural-other-hhs test-yaml-no-structural-other-ssa: - # revenue is heavy enough to need its own 2-batch split (run both at - # once); other ssa subfolders auto-fan two at a time. - $(BATCH) $(TESTS)/policy/baseline/gov/ssa/revenue --batches 2 --workers 2 - $(BATCH) $(TESTS)/policy/baseline/gov/ssa --exclude revenue --mode per-subdir --workers 2 + # revenue is heavy enough to need its own 2-batch split. --workers 1: + # revenue batch 1 peaked 8.5 GB on CI run 28698452678, so it must run + # solo; other ssa subfolders auto-fan one at a time. + $(BATCH) $(TESTS)/policy/baseline/gov/ssa/revenue --batches 2 --workers 1 + $(BATCH) $(TESTS)/policy/baseline/gov/ssa --exclude revenue --mode per-subdir --workers 1 test-yaml-no-structural-other-usda: - # usda (~3 GB peak, ~7 min) moved out of the rest job to ride along - # with ssa, rebalancing the two runners. + # usda (~3 GB peak, ~7 min) rides along with ssa, rebalancing runners. $(BATCH) $(TESTS)/policy/baseline/gov/usda --batches 1 test-yaml-no-structural-other-ssa-usda: test-yaml-no-structural-other-ssa test-yaml-no-structural-other-usda -test-yaml-no-structural-other-rest: - # All remaining gov/ subdirs + any new ones auto-route here (usda and - # hhs moved to the ssa-usda and contrib-reform-hhs jobs). Every batch - # peaks <=3.6 GB, so three run concurrently. - $(BATCH) $(TESTS)/policy/baseline/gov --exclude states,irs,ssa,usda,hhs --mode per-subdir --workers 3 +test-yaml-no-structural-other-rest-a: + # First half of the old "rest" job: four independent folders, one + # single-batch subprocess each (local/aca/fcc/hud measured <= ~5 GB + # per batch on CI run 28698452678). + $(BATCH) $(TESTS)/policy/baseline/gov/local --batches 1 + $(BATCH) $(TESTS)/policy/baseline/gov/aca --batches 1 + $(BATCH) $(TESTS)/policy/baseline/gov/fcc --batches 1 + $(BATCH) $(TESTS)/policy/baseline/gov/hud --batches 1 +test-yaml-no-structural-other-rest-b: + # gov/simulation's 5 files together peaked 12.1 GB on CI run + # 28698452678 — per-file frees each peak between files. + $(BATCH) $(TESTS)/policy/baseline/gov/simulation --mode per-file --workers 1 + # All remaining gov/ subdirs (cbo, doe, ed, tax, territories) + any new + # ones auto-route here, one subprocess each (<= ~5 GB measured); loose + # gov/*.yaml files get the trailing per-subdir batch. + $(BATCH) $(TESTS)/policy/baseline/gov --exclude states,irs,ssa,usda,hhs,local,aca,fcc,hud,simulation --mode per-subdir --workers 1 # calcfunctions, income, parameters + any new top-level baseline/ folder # are all light (<2.3 GB peak) — group them into one subprocess instead # of paying the ~33s interpreter+system-build startup per folder. @@ -133,8 +167,16 @@ test-yaml-no-structural-other-partners: $(BATCH) $(TESTS)/policy/baseline/partners/analytics_coverage --exclude edge_cases --batches 1 # amplifi + impactica + my_friend_ben (+ any new partner) in one light batch. $(BATCH) $(TESTS)/policy/baseline/partners --exclude analytics_coverage --batches 1 -test-other: - pytest policyengine_us/tests/ --maxfail=0 --ignore=$(TESTS)/policy/contrib +# The old single-process test-other run was OOM-killed at ~94% on CI run +# 28698452678: cumulative RSS from the python tests plus the +# microsimulation suite in one pytest process exceeded the 16 GB runner. +# Two pytest processes (run as two CI jobs) bound each peak separately. +test-other-python: + pytest policyengine_us/tests/ --maxfail=0 --ignore=$(TESTS)/policy/contrib --ignore=$(TESTS)/microsimulation +test-microsimulation: + pytest $(TESTS)/microsimulation --maxfail=0 +# Local convenience: run both halves back to back. +test-other: test-other-python test-microsimulation test-policy-contrib-python: pytest $$(find $(TESTS)/policy/contrib -name 'test*.py' -print) --maxfail=0 coverage: diff --git a/changelog.d/ci-speedup.changed.md b/changelog.d/ci-speedup.changed.md index f474df48626..2e8a227f667 100644 --- a/changelog.d/ci-speedup.changed.md +++ b/changelog.d/ci-speedup.changed.md @@ -1 +1 @@ -Speed up CI: cache New York's pinned EITC/CTC tax-benefit systems (#8114), run test batches concurrently within each runner, rebalance CI jobs from 17 to 14 runners, and report per-case durations and per-batch peak memory in CI logs. +Speed up and memory-harden CI: cache New York's pinned EITC/CTC tax-benefit systems (#8114), report per-case durations and per-batch peak memory in CI logs, and re-batch the test suite from measured per-batch peak RSS — finer single-worker batches across more runners so every subprocess stays at or below roughly half of the 16 GB runner memory, splitting the two heaviest contrib reform test files (RI CTC, CRFB employer payroll tax percentage) and the microsimulation pytest run into their own subprocesses. diff --git a/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_basic.yaml b/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_basic.yaml new file mode 100644 index 00000000000..bc881903dfb --- /dev/null +++ b/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_basic.yaml @@ -0,0 +1,75 @@ +- name: Full employer payroll tax inclusion (100%) + period: 2024 + reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object + input: + gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true + gov.contrib.crfb.tax_employer_payroll_tax.percentage: 1.0 + people: + person1: + employment_income: 50_000 + tax_units: + tax_unit: + members: [person1] + households: + household: + members: [person1] + state_code: CA + output: + # With $50,000 income: + # Employer SS tax: $50,000 * 0.062 = $3,100 + # Employer Medicare tax: $50,000 * 0.0145 = $725 + # Total employer tax: $3,825 + employer_social_security_tax: [3_100] + employer_medicare_tax: [725] + # IRS gross income should include 100% of employer taxes + irs_gross_income: [53_825] # $50,000 + $3,825 + +- name: Half employer payroll tax inclusion (50%) + period: 2024 + reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object + input: + gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true + gov.contrib.crfb.tax_employer_payroll_tax.percentage: 0.5 + people: + person1: + employment_income: 50_000 + tax_units: + tax_unit: + members: [person1] + households: + household: + members: [person1] + state_code: CA + output: + employer_social_security_tax: [3_100] + employer_medicare_tax: [725] + # IRS gross income should include 50% of employer taxes + # $50,000 + ($3,825 * 0.5) = $50,000 + $1,912.50 + irs_gross_income: [51_912.5] + +- name: Quarter employer payroll tax inclusion (25%) + period: 2024 + reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object + input: + gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true + gov.contrib.crfb.tax_employer_payroll_tax.percentage: 0.25 + people: + person1: + employment_income: 100_000 + tax_units: + tax_unit: + members: [person1] + households: + household: + members: [person1] + state_code: CA + output: + # With $100,000 income: + # Employer SS tax: $100,000 * 0.062 = $6,200 + # Employer Medicare tax: $100,000 * 0.0145 = $1,450 + # Total employer tax: $7,650 + employer_social_security_tax: [6_200] + employer_medicare_tax: [1_450] + # IRS gross income should include 25% of employer taxes + # $100,000 + ($7,650 * 0.25) = $100,000 + $1,912.50 + irs_gross_income: [101_912.5] diff --git a/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage.yaml b/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_edge.yaml similarity index 54% rename from policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage.yaml rename to policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_edge.yaml index 1f8963fa0cc..52018a4f373 100644 --- a/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage.yaml +++ b/policyengine_us/tests/policy/contrib/crfb/tax_employer_payroll_tax_percentage_edge.yaml @@ -1,79 +1,3 @@ -- name: Full employer payroll tax inclusion (100%) - period: 2024 - reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object - input: - gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true - gov.contrib.crfb.tax_employer_payroll_tax.percentage: 1.0 - people: - person1: - employment_income: 50_000 - tax_units: - tax_unit: - members: [person1] - households: - household: - members: [person1] - state_code: CA - output: - # With $50,000 income: - # Employer SS tax: $50,000 * 0.062 = $3,100 - # Employer Medicare tax: $50,000 * 0.0145 = $725 - # Total employer tax: $3,825 - employer_social_security_tax: [3_100] - employer_medicare_tax: [725] - # IRS gross income should include 100% of employer taxes - irs_gross_income: [53_825] # $50,000 + $3,825 - -- name: Half employer payroll tax inclusion (50%) - period: 2024 - reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object - input: - gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true - gov.contrib.crfb.tax_employer_payroll_tax.percentage: 0.5 - people: - person1: - employment_income: 50_000 - tax_units: - tax_unit: - members: [person1] - households: - household: - members: [person1] - state_code: CA - output: - employer_social_security_tax: [3_100] - employer_medicare_tax: [725] - # IRS gross income should include 50% of employer taxes - # $50,000 + ($3,825 * 0.5) = $50,000 + $1,912.50 - irs_gross_income: [51_912.5] - -- name: Quarter employer payroll tax inclusion (25%) - period: 2024 - reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object - input: - gov.contrib.crfb.tax_employer_payroll_tax.in_effect: true - gov.contrib.crfb.tax_employer_payroll_tax.percentage: 0.25 - people: - person1: - employment_income: 100_000 - tax_units: - tax_unit: - members: [person1] - households: - household: - members: [person1] - state_code: CA - output: - # With $100,000 income: - # Employer SS tax: $100,000 * 0.062 = $6,200 - # Employer Medicare tax: $100,000 * 0.0145 = $1,450 - # Total employer tax: $7,650 - employer_social_security_tax: [6_200] - employer_medicare_tax: [1_450] - # IRS gross income should include 25% of employer taxes - # $100,000 + ($7,650 * 0.25) = $100,000 + $1,912.50 - irs_gross_income: [101_912.5] - - name: No employer payroll tax inclusion (0%) period: 2024 reforms: policyengine_us.reforms.crfb.tax_employer_payroll_tax.tax_employer_payroll_tax_reform_object diff --git a/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_refundability_test.yaml b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_refundability_test.yaml new file mode 100644 index 00000000000..6344f984336 --- /dev/null +++ b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_refundability_test.yaml @@ -0,0 +1,178 @@ +- name: RI CTC - fully refundable with AGI phaseout + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 # High cap = fully refundable, amount.yaml controls max + gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 + gov.contrib.states.ri.ctc.phaseout.rate: 0.05 + people: + parent: + age: 35 + employment_income: 50_000 + child1: + age: 5 + is_tax_unit_dependent: true + child2: + age: 10 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2] + state_name: RI + output: + ri_ctc_eligible_children: 2 + ri_ctc_maximum: 2_000 # 2 children × $1_000 amount + ri_ctc_phaseout: 0 + ri_total_ctc: 2_000 + ri_refundable_ctc: 2_000 # min(2_000 total credit, 999_999 cap) = 2_000 (amount controls) + ri_non_refundable_ctc: 0 + ri_refundable_credits: 2_246.32 # 2_000 (CTC) + 246.32 (EITC) + ri_non_refundable_credits: 0 + +- name: RI CTC - partially refundable + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_500 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.refundability.cap: 1_000 # Cap < total credit = partially refundable + gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 + gov.contrib.states.ri.ctc.phaseout.rate: 0.05 + people: + parent: + age: 35 + employment_income: 30_000 + child1: + age: 5 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1] + filing_status: SINGLE + households: + household: + members: [parent, child1] + state_name: RI + output: + ri_total_ctc: 1_500 + ri_refundable_ctc: 1_000 # min(1_500 total credit, 1_000 cap) = 1_000 + ri_non_refundable_ctc: 500 # 1_500 - 1_000 = 500 + ri_refundable_credits: 1_522.45 # 1_000 (CTC refundable) + 522.45 (EITC) + ri_income_tax_before_non_refundable_credits: 333.75 + ri_non_refundable_credits: 333.75 # applied amount capped by remaining tax liability + +- name: RI CTC - nonrefundable with no phaseout + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 500 + gov.contrib.states.ri.ctc.age_limit: 6 + gov.contrib.states.ri.ctc.refundability.cap: 0 # Cap = 0, credit is nonrefundable + people: + parent: + age: 35 + employment_income: 40_000 + child1: + age: 3 + is_tax_unit_dependent: true + child2: + age: 10 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2] + state_name: RI + output: + ri_ctc_eligible_children: 1 + ri_total_ctc: 500 + ri_refundable_ctc: 0 # min(500 total credit, 0 cap) = 0 + ri_non_refundable_ctc: 500 # 500 - 0 = 500 + ri_refundable_credits: 583.28 # 0 (CTC refundable) + 583.28 (EITC) + ri_non_refundable_credits: 500 # 500 (CTC non-refundable) + +- name: RI CTC - AGI phaseout reduces credit + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 # Fully refundable + gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 + gov.contrib.states.ri.ctc.phaseout.rate: 0.05 + people: + parent: + age: 35 + employment_income: 85_000 + child1: + age: 5 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1] + filing_status: SINGLE + households: + household: + members: [parent, child1] + state_name: RI + output: + ri_ctc_maximum: 1_000 # 1 child × $1_000 amount + ri_ctc_phaseout: 500 # (85_000 - 75_000) * 0.05 = 500 + ri_total_ctc: 500 # 1_000 - 500 = 500 + ri_refundable_ctc: 500 # min(500 total credit, 999_999 cap) = 500 (total credit controls) + ri_non_refundable_ctc: 0 + ri_refundable_credits: 500 # 500 (CTC refundable) + 0 (EITC - income too high) + ri_non_refundable_credits: 0 + +- name: RI CTC - cap exceeds amount but amount controls max + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 500 # Only $500 per child + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.refundability.cap: 10_000 # Cap is much higher than possible credit + people: + parent: + age: 35 + employment_income: 40_000 + child1: + age: 5 + is_tax_unit_dependent: true + child2: + age: 10 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2] + state_name: RI + output: + ri_ctc_eligible_children: 2 + ri_ctc_maximum: 1_000 # 2 children × $500 amount (NOT $10_000!) + ri_total_ctc: 1_000 # amount.yaml controls the maximum + ri_refundable_ctc: 1_000 # min(1_000 total credit, 10_000 cap) = 1_000 + ri_non_refundable_ctc: 0 + ri_refundable_credits: 1_583.28 # 1_000 (CTC refundable) + 583.28 (EITC) + ri_non_refundable_credits: 0 diff --git a/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_stepped_phaseout_test.yaml b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_stepped_phaseout_test.yaml new file mode 100644 index 00000000000..cfcdee0f7e9 --- /dev/null +++ b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_stepped_phaseout_test.yaml @@ -0,0 +1,67 @@ +- name: RI CTC - stepped phaseout uses single filer threshold. + absolute_error_margin: 0.1 + period: 2027 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 330 + gov.contrib.states.ri.ctc.age_limit: 19 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + gov.contrib.states.ri.ctc.stepped_phaseout.threshold.SINGLE: 88_500 + gov.contrib.states.ri.ctc.stepped_phaseout.increment.SINGLE: 2_875 + gov.contrib.states.ri.ctc.stepped_phaseout.rate_per_step: 0.2 + people: + person1: + age: 35 + employment_income: 100_000 + person2: + age: 5 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [person1, person2] + filing_status: SINGLE + households: + household: + members: [person1, person2] + state_name: RI + output: + ri_ctc_eligible_children: 1 + ri_ctc_maximum: 330 + ri_ctc_phaseout: 264 + ri_total_ctc: 66 + +- name: RI CTC - stepped phaseout uses joint filer threshold and increment. + absolute_error_margin: 0.1 + period: 2027 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 330 + gov.contrib.states.ri.ctc.age_limit: 19 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + gov.contrib.states.ri.ctc.stepped_phaseout.threshold.JOINT: 110_640 + gov.contrib.states.ri.ctc.stepped_phaseout.increment.JOINT: 3_590 + gov.contrib.states.ri.ctc.stepped_phaseout.rate_per_step: 0.2 + people: + person1: + age: 35 + employment_income: 117_640 + person2: + age: 35 + person3: + age: 5 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [person1, person2, person3] + filing_status: JOINT + households: + household: + members: [person1, person2, person3] + state_name: RI + output: + ri_ctc_eligible_children: 1 + ri_ctc_maximum: 330 + ri_ctc_phaseout: 132 + ri_total_ctc: 198 diff --git a/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_test.yaml b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_test.yaml deleted file mode 100644 index 2b95b2920ed..00000000000 --- a/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_test.yaml +++ /dev/null @@ -1,390 +0,0 @@ -- name: RI CTC - fully refundable with AGI phaseout - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 # High cap = fully refundable, amount.yaml controls max - gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 - gov.contrib.states.ri.ctc.phaseout.rate: 0.05 - people: - parent: - age: 35 - employment_income: 50_000 - child1: - age: 5 - is_tax_unit_dependent: true - child2: - age: 10 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2] - state_name: RI - output: - ri_ctc_eligible_children: 2 - ri_ctc_maximum: 2_000 # 2 children × $1_000 amount - ri_ctc_phaseout: 0 - ri_total_ctc: 2_000 - ri_refundable_ctc: 2_000 # min(2_000 total credit, 999_999 cap) = 2_000 (amount controls) - ri_non_refundable_ctc: 0 - ri_refundable_credits: 2_246.32 # 2_000 (CTC) + 246.32 (EITC) - ri_non_refundable_credits: 0 - -- name: RI CTC - partially refundable - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_500 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.refundability.cap: 1_000 # Cap < total credit = partially refundable - gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 - gov.contrib.states.ri.ctc.phaseout.rate: 0.05 - people: - parent: - age: 35 - employment_income: 30_000 - child1: - age: 5 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1] - filing_status: SINGLE - households: - household: - members: [parent, child1] - state_name: RI - output: - ri_total_ctc: 1_500 - ri_refundable_ctc: 1_000 # min(1_500 total credit, 1_000 cap) = 1_000 - ri_non_refundable_ctc: 500 # 1_500 - 1_000 = 500 - ri_refundable_credits: 1_522.45 # 1_000 (CTC refundable) + 522.45 (EITC) - ri_income_tax_before_non_refundable_credits: 333.75 - ri_non_refundable_credits: 333.75 # applied amount capped by remaining tax liability - -- name: RI CTC - nonrefundable with no phaseout - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 500 - gov.contrib.states.ri.ctc.age_limit: 6 - gov.contrib.states.ri.ctc.refundability.cap: 0 # Cap = 0, credit is nonrefundable - people: - parent: - age: 35 - employment_income: 40_000 - child1: - age: 3 - is_tax_unit_dependent: true - child2: - age: 10 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2] - state_name: RI - output: - ri_ctc_eligible_children: 1 - ri_total_ctc: 500 - ri_refundable_ctc: 0 # min(500 total credit, 0 cap) = 0 - ri_non_refundable_ctc: 500 # 500 - 0 = 500 - ri_refundable_credits: 583.28 # 0 (CTC refundable) + 583.28 (EITC) - ri_non_refundable_credits: 500 # 500 (CTC non-refundable) - -- name: RI CTC - AGI phaseout reduces credit - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 # Fully refundable - gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 75_000 - gov.contrib.states.ri.ctc.phaseout.rate: 0.05 - people: - parent: - age: 35 - employment_income: 85_000 - child1: - age: 5 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1] - filing_status: SINGLE - households: - household: - members: [parent, child1] - state_name: RI - output: - ri_ctc_maximum: 1_000 # 1 child × $1_000 amount - ri_ctc_phaseout: 500 # (85_000 - 75_000) * 0.05 = 500 - ri_total_ctc: 500 # 1_000 - 500 = 500 - ri_refundable_ctc: 500 # min(500 total credit, 999_999 cap) = 500 (total credit controls) - ri_non_refundable_ctc: 0 - ri_refundable_credits: 500 # 500 (CTC refundable) + 0 (EITC - income too high) - ri_non_refundable_credits: 0 - -- name: RI CTC - cap exceeds amount but amount controls max - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 500 # Only $500 per child - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.refundability.cap: 10_000 # Cap is much higher than possible credit - people: - parent: - age: 35 - employment_income: 40_000 - child1: - age: 5 - is_tax_unit_dependent: true - child2: - age: 10 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2] - state_name: RI - output: - ri_ctc_eligible_children: 2 - ri_ctc_maximum: 1_000 # 2 children × $500 amount (NOT $10_000!) - ri_total_ctc: 1_000 # amount.yaml controls the maximum - ri_refundable_ctc: 1_000 # min(1_000 total credit, 10_000 cap) = 1_000 - ri_non_refundable_ctc: 0 - ri_refundable_credits: 1_583.28 # 1_000 (CTC refundable) + 583.28 (EITC) - ri_non_refundable_credits: 0 - -- name: RI CTC - young child boost applies to children under age 6 - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.young_child_boost.amount: 500 - gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 100_000 - gov.contrib.states.ri.ctc.phaseout.rate: 0.05 - people: - parent: - age: 35 - employment_income: 50_000 - child1: - age: 3 - is_tax_unit_dependent: true - child2: - age: 10 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2] - state_name: RI - output: - ri_ctc_eligible_children: 2 - ri_ctc_young_child_boost: 500 # 1 child under 6 × $500 = $500 - ri_ctc_maximum: 2_500 # Base: 2 × $1_000 = $2_000; Boost: $500; Total = $2_500 - ri_total_ctc: 2_500 - ri_refundable_ctc: 2_500 - ri_non_refundable_ctc: 0 - -- name: RI CTC - young child boost boundary at age 6 - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.young_child_boost.amount: 500 - gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - people: - parent: - age: 35 - employment_income: 40_000 - child1: - age: 5 - is_tax_unit_dependent: true - child2: - age: 6 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2] - state_name: RI - output: - ri_ctc_eligible_children: 2 - ri_ctc_young_child_boost: 500 # Only child age 5 qualifies (age 6 does not) - ri_ctc_maximum: 2_500 # Base: 2 × $1_000 = $2_000; Boost: $500 (only age 5 child); Total = $2_500 - ri_total_ctc: 2_500 - -- name: RI CTC - young child boost with multiple young children - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.young_child_boost.amount: 500 - gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - people: - parent: - age: 35 - employment_income: 50_000 - child1: - age: 2 - is_tax_unit_dependent: true - child2: - age: 4 - is_tax_unit_dependent: true - child3: - age: 10 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1, child2, child3] - filing_status: SINGLE - households: - household: - members: [parent, child1, child2, child3] - state_name: RI - output: - ri_ctc_eligible_children: 3 - ri_ctc_young_child_boost: 1_000 # 2 children under 6 × $500 = $1_000 - ri_ctc_maximum: 4_000 # Base: 3 × $1_000 = $3_000; Boost: $1_000; Total = $4_000 - ri_total_ctc: 4_000 - -- name: RI CTC - young child boost with no boost amount (zero) - absolute_error_margin: 0.1 - period: 2025 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 1_000 - gov.contrib.states.ri.ctc.age_limit: 18 - gov.contrib.states.ri.ctc.young_child_boost.amount: 0 # No boost - gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - people: - parent: - age: 35 - employment_income: 50_000 - child1: - age: 3 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [parent, child1] - filing_status: SINGLE - households: - household: - members: [parent, child1] - state_name: RI - output: - ri_ctc_eligible_children: 1 - ri_ctc_young_child_boost: 0 # 1 child under 6 × $0 = $0 - ri_ctc_maximum: 1_000 # Base: 1 × $1_000 = $1_000; Boost: $0; Total = $1_000 - ri_total_ctc: 1_000 - -- name: RI CTC - stepped phaseout uses single filer threshold. - absolute_error_margin: 0.1 - period: 2027 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 330 - gov.contrib.states.ri.ctc.age_limit: 19 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - gov.contrib.states.ri.ctc.stepped_phaseout.threshold.SINGLE: 88_500 - gov.contrib.states.ri.ctc.stepped_phaseout.increment.SINGLE: 2_875 - gov.contrib.states.ri.ctc.stepped_phaseout.rate_per_step: 0.2 - people: - person1: - age: 35 - employment_income: 100_000 - person2: - age: 5 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [person1, person2] - filing_status: SINGLE - households: - household: - members: [person1, person2] - state_name: RI - output: - ri_ctc_eligible_children: 1 - ri_ctc_maximum: 330 - ri_ctc_phaseout: 264 - ri_total_ctc: 66 - -- name: RI CTC - stepped phaseout uses joint filer threshold and increment. - absolute_error_margin: 0.1 - period: 2027 - reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc - input: - gov.contrib.states.ri.ctc.in_effect: true - gov.contrib.states.ri.ctc.amount: 330 - gov.contrib.states.ri.ctc.age_limit: 19 - gov.contrib.states.ri.ctc.refundability.cap: 999_999 - gov.contrib.states.ri.ctc.stepped_phaseout.threshold.JOINT: 110_640 - gov.contrib.states.ri.ctc.stepped_phaseout.increment.JOINT: 3_590 - gov.contrib.states.ri.ctc.stepped_phaseout.rate_per_step: 0.2 - people: - person1: - age: 35 - employment_income: 117_640 - person2: - age: 35 - person3: - age: 5 - is_tax_unit_dependent: true - tax_units: - tax_unit: - members: [person1, person2, person3] - filing_status: JOINT - households: - household: - members: [person1, person2, person3] - state_name: RI - output: - ri_ctc_eligible_children: 1 - ri_ctc_maximum: 330 - ri_ctc_phaseout: 132 - ri_total_ctc: 198 diff --git a/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_young_child_boost_test.yaml b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_young_child_boost_test.yaml new file mode 100644 index 00000000000..89488cff47c --- /dev/null +++ b/policyengine_us/tests/policy/contrib/states/ri/ctc_reform_young_child_boost_test.yaml @@ -0,0 +1,143 @@ +- name: RI CTC - young child boost applies to children under age 6 + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.young_child_boost.amount: 500 + gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + gov.contrib.states.ri.ctc.phaseout.threshold.SINGLE: 100_000 + gov.contrib.states.ri.ctc.phaseout.rate: 0.05 + people: + parent: + age: 35 + employment_income: 50_000 + child1: + age: 3 + is_tax_unit_dependent: true + child2: + age: 10 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2] + state_name: RI + output: + ri_ctc_eligible_children: 2 + ri_ctc_young_child_boost: 500 # 1 child under 6 × $500 = $500 + ri_ctc_maximum: 2_500 # Base: 2 × $1_000 = $2_000; Boost: $500; Total = $2_500 + ri_total_ctc: 2_500 + ri_refundable_ctc: 2_500 + ri_non_refundable_ctc: 0 + +- name: RI CTC - young child boost boundary at age 6 + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.young_child_boost.amount: 500 + gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + people: + parent: + age: 35 + employment_income: 40_000 + child1: + age: 5 + is_tax_unit_dependent: true + child2: + age: 6 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2] + state_name: RI + output: + ri_ctc_eligible_children: 2 + ri_ctc_young_child_boost: 500 # Only child age 5 qualifies (age 6 does not) + ri_ctc_maximum: 2_500 # Base: 2 × $1_000 = $2_000; Boost: $500 (only age 5 child); Total = $2_500 + ri_total_ctc: 2_500 + +- name: RI CTC - young child boost with multiple young children + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.young_child_boost.amount: 500 + gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + people: + parent: + age: 35 + employment_income: 50_000 + child1: + age: 2 + is_tax_unit_dependent: true + child2: + age: 4 + is_tax_unit_dependent: true + child3: + age: 10 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1, child2, child3] + filing_status: SINGLE + households: + household: + members: [parent, child1, child2, child3] + state_name: RI + output: + ri_ctc_eligible_children: 3 + ri_ctc_young_child_boost: 1_000 # 2 children under 6 × $500 = $1_000 + ri_ctc_maximum: 4_000 # Base: 3 × $1_000 = $3_000; Boost: $1_000; Total = $4_000 + ri_total_ctc: 4_000 + +- name: RI CTC - young child boost with no boost amount (zero) + absolute_error_margin: 0.1 + period: 2025 + reforms: policyengine_us.reforms.states.ri.ctc.ri_ctc_reform.ri_ctc + input: + gov.contrib.states.ri.ctc.in_effect: true + gov.contrib.states.ri.ctc.amount: 1_000 + gov.contrib.states.ri.ctc.age_limit: 18 + gov.contrib.states.ri.ctc.young_child_boost.amount: 0 # No boost + gov.contrib.states.ri.ctc.young_child_boost.age_limit: 6 + gov.contrib.states.ri.ctc.refundability.cap: 999_999 + people: + parent: + age: 35 + employment_income: 50_000 + child1: + age: 3 + is_tax_unit_dependent: true + tax_units: + tax_unit: + members: [parent, child1] + filing_status: SINGLE + households: + household: + members: [parent, child1] + state_name: RI + output: + ri_ctc_eligible_children: 1 + ri_ctc_young_child_boost: 0 # 1 child under 6 × $0 = $0 + ri_ctc_maximum: 1_000 # Base: 1 × $1_000 = $1_000; Boost: $0; Total = $1_000 + ri_total_ctc: 1_000 diff --git a/policyengine_us/tests/test_batched.py b/policyengine_us/tests/test_batched.py index 5bf9a29f37d..c29d6d8de43 100644 --- a/policyengine_us/tests/test_batched.py +++ b/policyengine_us/tests/test_batched.py @@ -167,8 +167,9 @@ def split_into_batches( # peak/case, see #8559); isolating per-file frees each peak between # files. RI is the worst offender — its ctc_reform_test.yaml sweeps # ~66 reform combinations and previously OOMed shard-2 mid-run when it - # shared a subprocess with exemption_reform_test.yaml. - PER_FILE_STATES = {"ri"} + # shared a subprocess with exemption_reform_test.yaml. OH's folder + # measured 12.4 GB as one batch on CI (run 28698452678). + PER_FILE_STATES = {"oh", "ri"} subdirs = sorted([item for item in base_path.iterdir() if item.is_dir()]) batches = [] From 685408a5eab227575cf3d4d52ebc594d285983cc Mon Sep 17 00:00:00 2001 From: Ziming Date: Sat, 4 Jul 2026 11:49:25 -0400 Subject: [PATCH 6/6] Run each selective-test location in its own subprocess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quick Feedback OOM'd twice: run_selective_tests packed every selected location into one process (this PR selected the whole NY folder plus all RI and CRFB reform files), where reformed tax-benefit systems and per-case simulations accumulate for the process lifetime — exceeding runner memory under coverage even though each location alone is fine. One subprocess per location bounds memory at the heaviest single location; coverage -a appends results across invocations unchanged. Co-Authored-By: Claude Fable 5 --- policyengine_us/tests/run_selective_tests.py | 31 ++++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/policyengine_us/tests/run_selective_tests.py b/policyengine_us/tests/run_selective_tests.py index 1e9c4799359..e3fbf8c784f 100644 --- a/policyengine_us/tests/run_selective_tests.py +++ b/policyengine_us/tests/run_selective_tests.py @@ -476,7 +476,7 @@ def run_tests( ) if with_coverage: - pytest_args = [ + base_cmd = [ sys.executable, "-m", "coverage", @@ -488,9 +488,9 @@ def run_tests( # Add --include flag to only track relevant files if include_patterns: include_pattern = ",".join(include_patterns) - pytest_args.extend(["--include", include_pattern]) + base_cmd.extend(["--include", include_pattern]) - pytest_args.extend( + base_cmd.extend( [ "-m", "policyengine_core.scripts.policyengine_command", @@ -500,7 +500,7 @@ def run_tests( ] ) else: - pytest_args = [ + base_cmd = [ sys.executable, "-m", "policyengine_core.scripts.policyengine_command", @@ -509,15 +509,20 @@ def run_tests( "policyengine_us", ] - # Add test paths - pytest_args.extend(sorted(test_paths)) - - print(f"\nRunning command: {' '.join(pytest_args)}") - - # Run pytest - result = subprocess.run(pytest_args) - - return result.returncode + # Run each location in its own subprocess: reformed tax-benefit + # systems and per-case simulations accumulate for the life of a + # process, so a single process running every selected location can + # exceed runner memory even when each location alone is fine. + # Coverage's -a flag appends results across the invocations. + worst_returncode = 0 + for test_path in sorted(test_paths): + cmd = base_cmd + [test_path] + print(f"\nRunning command: {' '.join(cmd)}") + result = subprocess.run(cmd) + if result.returncode != 0: + worst_returncode = result.returncode + + return worst_returncode def run_all_tests(self) -> int: """Run all tests (fallback option)."""