add unit tests for deterministic logic

README.md

@@ -24,10 +24,72 @@ mpiexec -np <num_processes> -bind-to hwthread python -u runscript.py <path/to/co
 
 Note: `-bind-to hwthread` is optional but expected to give the best performance if enough threads are available.
 
+### Tests
+
+The deterministic logic (kinematics, position binning, transition models,
+utils, config loader, synthetic source) has unit tests under `tests/`.
+These do not require MPI or any acquisition hardware.
+
+```
+pip install -e .[test]
+pytest -q
+```
+
+### Running without acquisition hardware (synthetic data source)
+
+You can run the full MPI pipeline against an in-process synthetic data
+generator — no Trodes, SpikeGLX, or other acquisition rig required. This
+is useful for smoke-testing an install, developing on a laptop, and CI.
+
+```
+mpiexec -np 5 python -u runscript.py config/demo_synthetic.yml
+```
+
+Selection is driven by the top-level `datasource` config key
+(`trodes` by default; `synthetic` to use the generator). The synthetic
+source produces Poisson spikes with Gaussian mark vectors, walks the
+synthetic animal back and forth along a single linear segment, and
+auto-terminates after `synthetic.run_duration_s` seconds. See
+`realtime_decoder/synthetic.py` for the full set of tunables.
+
 # Configuration
 
 Please see the example configuration file in the `example_config` folder. Options are described in more detail below.
 
+## Defaults and inheritance
+
+Configs can extend a shared base file by declaring `_extends:` at the top:
+
+```yaml
+_extends: defaults.yml
+
+# only the keys that actually differ from defaults.yml go here
+rank:
+  supervisor: [0]
+  ...
+```
+
+`config/defaults.yml` ships with values shared across the SC* / fred / ginny
+configs (sampling rates, ripple filter, GUI, MUA, kinematics smoothing
+filter, display intervals, process monitor). Per-animal configs only need
+to specify what's actually different — typically `rank`, `trode_selection`,
+`decoder_assignment`, `files`, `encoder.position`, `kinematics.scale_factor`,
+and `stimulation`. Existing configs without `_extends` continue to work
+unchanged.
+
+Relative paths in `_extends` resolve next to the file that declares them.
+You may pass a single path or a list (parents merged in order).
+
+## Validation
+
+At startup the loader checks the resolved config against a minimal schema
+(required ranks, known algorithm, known datasource, encoder dimensions
+match the synthetic source, every decoder rank has an assignment, etc.).
+Missing or malformed keys produce a single readable error before the MPI
+processes spawn workers, instead of an `IndexError` deep inside a rank.
+
+
+
 ## `rank`
 
 Describes which MPI rank should be assigned to each process type.

config/defaults.yml

@@ -0,0 +1,101 @@
+---
+# Shared defaults for realtime_decoder configs.
+#
+# Per-animal / per-experiment YAMLs should declare:
+#     _extends: defaults.yml
+# at the top, then override only what's actually different. Anything not
+# overridden inherits from here. Lists and scalars are replaced as a
+# whole; nested dicts merge key-by-key.
+#
+# This file is intentionally conservative: it only contains values that
+# have been stable across the SC* / fred / ginny configs in this repo,
+# plus a few sane defaults for new fields. Per-animal kinematics,
+# stimulation, trode selection, and file paths must still live in the
+# child config.
+
+algorithm: "clusterless_decoder"
+datasource: "trodes"
+num_setup_messages: 100
+preloaded_model: false
+frozen_model: false
+
+sampling_rate:
+  spikes: 30000
+  lfp: 1500
+  position: 30
+
+ripples:
+  max_ripple_samples: 450
+  vel_thresh: 10
+  freeze_stats: false
+  timings_bufsize: 1000000
+  filter:
+    type: 'iir'
+    order: 2
+    crit_freqs: [150, 250]
+    kwargs:
+      btype: 'bandpass'
+      ftype: 'butter'
+  smoothing_filter:
+    num_taps: 15
+    band_edges: [50, 55]
+    desired: [1, 0]
+  threshold:
+    standard: 3.5
+    conditioning: 3.75
+    content: 4
+    end: 0
+
+decoder:
+  cred_int_bufsize: 10
+  time_bin:
+    samples: 180          # 6 ms at 30 kHz spike clock
+    delay_samples: 180
+
+clusterless_decoder:
+  state_labels: ['state']
+  transmat_bias: 1
+
+gui:
+  colormap: 'rocket'
+  send_interval: 0
+  refresh_rate: 25
+  trace_length: 2
+  state_colors: ['#4c72b0', '#dd8452', '#55a868']
+  num_xticks: 5
+
+mua:
+  threshold:
+    trigger: 4
+    end: 0
+  freeze_stats: false
+  moving_avg_window: 5
+
+cred_interval:
+  val: 0.5
+  max_num: 5
+
+kinematics:
+  smooth_x: true
+  smooth_y: true
+  smooth_speed: false
+  smoothing_filter: [0.31, 0.29, 0.25, 0.15]
+
+display:
+  stim_decider:
+    position: 150
+    decoding_bins: 2000
+  ripples:
+    lfp: 100000
+  encoder:
+    encoding_spikes: 5000
+    total_spikes: 50000
+    occupancy: 5000
+    position: 5000
+  decoder:
+    total_spikes: 50000
+    occupancy: 100
+
+process_monitor:
+  interval: 15
+  timeout: 3

config/demo_synthetic.yml

@@ -0,0 +1,129 @@
+---
+# Demo config: runs the full MPI pipeline against the in-process
+# synthetic data source. No Trodes / acquisition hardware required.
+#
+# Run:
+#   mpiexec -np 5 python -u runscript.py config/demo_synthetic.yml
+#
+# Everything not set here inherits from defaults.yml via `_extends`.
+
+_extends: defaults.yml
+
+datasource: "synthetic"
+
+rank:
+  supervisor: [0]
+  ripples: [3]
+  decoders: [1]
+  encoders: [4]
+  gui: [2]
+rank_settings:
+  enable_rec: [0, 1, 3, 4]
+trode_selection:
+  ripples: [1]
+  decoding: [1]
+decoder_assignment:
+  1: [1]
+
+files:
+  output_dir: '/tmp/realtime_decoder_demo'
+  prefix: 'demo'
+  rec_postfix: 'bin_rec'
+  timing_postfix: 'timing'
+
+# --- synthetic-source parameters (all optional, defaults documented in
+# realtime_decoder/synthetic.py)
+synthetic:
+  spike_rate_hz: 30
+  mark_dim: 4              # must equal encoder.mark_dim below
+  mark_amplitude_uv: 120
+  track_length_cm: 40
+  walk_speed_cm_s: 20
+  startup_delay_s: 1.0
+  run_duration_s: 30
+  voltage_scaling_factor: 0.195
+
+# Lower-volume timings so the demo doesn't waste memory
+ripples:
+  timings_bufsize: 100000
+
+encoder:
+  spk_amp: 60
+  use_channel_dist_from_max_amp: 2
+  mark_dim: 4
+  bufsize: 5000
+  timings_bufsize: 5000
+  vel_thresh: 5
+  num_pos_points: 30
+  position:
+    lower: 0
+    upper: 41
+    num_bins: 41
+    arm_ids: [0]
+    arm_coords: [[0, 40]]
+  mark_kernel:
+    mean: 0
+    std: 20
+    use_filter: false
+    n_std: 1
+    n_marks_min: 10
+
+decoder:
+  decoder_to_message: 1
+  bufsize: 2000
+  timings_bufsize: 10000
+  starting_arm1_bin: 10
+  starting_arm2_bin: 30
+  num_pos_points: 30
+
+stimulation:
+  instructive: false
+  shortcut_msg_on: false
+  automatic_threshold_update: false
+  num_each_arm_per_minute: 1.1
+  num_pos_points: 30
+  center_well_loc: [100, 100]
+  max_center_well_dist: 50
+  replay:
+    enabled: false
+    method: "posterior"
+    target_arm: 0
+    event_lockout: 0.2
+    sliding_window: 5
+    primary_arm_threshold: 0.4
+    secondary_arm_threshold: 0.4
+    other_arm_threshold: 0.3
+    max_arm_repeats: 8
+    instr_max_repeats: 3
+    min_unique_trodes: 1
+  ripples:
+    enabled: false
+    type: "standard"
+    method: "multichannel"
+    event_lockout: 0
+    num_above_thresh: 1
+    suprathreshold_period: 0.1
+  head_direction:
+    enabled: false
+    rotate_180: false
+    event_lockout: 10
+    min_duration: 2
+    well_angle_range: 6
+    within_angle_range: 6
+    well_loc: [[100, 100], [200, 200]]
+
+kinematics:
+  scale_factor: 0.2644
+
+display:
+  stim_decider:
+    decoding_bins: 200
+  ripples:
+    lfp: 10000
+  encoder:
+    encoding_spikes: 500
+    total_spikes: 5000
+    occupancy: 500
+    position: 500
+  decoder:
+    total_spikes: 5000