diff --git a/lampOS/Cargo.lock b/lampOS/Cargo.lock index 3ed1c6343b..34a68efa05 100644 --- a/lampOS/Cargo.lock +++ b/lampOS/Cargo.lock @@ -1811,7 +1811,10 @@ dependencies = [ "lamp-bus", "lamp-ipc", "lamp-msgs", + "rustfft", "serde_json", + "sha2", + "tract-onnx", ] [[package]] diff --git a/lampOS/crates/ears/Cargo.toml b/lampOS/crates/ears/Cargo.toml index 5818c800fb..506a85c7e2 100644 --- a/lampOS/crates/ears/Cargo.toml +++ b/lampOS/crates/ears/Cargo.toml @@ -11,6 +11,11 @@ serde_json.workspace = true hound.workspace = true lamp-msgs = { path = "../msgs" } lamp-bus = { path = "../bus" } +# The optional room sound classes (`src/classes.rs`, #1669), pinned as in +# lamp-turn-end and lamp-same-voice. +tract-onnx = { version = "=0.23.8", default-features = false } +rustfft = "=6.4.1" +sha2.workspace = true # The room microphone's reader (`src/main.rs`). [target.'cfg(target_os = "linux")'.dependencies] diff --git a/lampOS/crates/ears/src/classes.rs b/lampOS/crates/ears/src/classes.rs new file mode 100644 index 0000000000..481de703ce --- /dev/null +++ b/lampOS/crates/ears/src/classes.rs @@ -0,0 +1,704 @@ +//! Room sound classes on `ears/classes` (#1669): speech, music and laughter +//! scores from YAMNet over mic1. Off unless `LAMP_EARS_CLASSES=on`, and only +//! with `LAMP_PRIVACY_CONTROLS=on` (docs/sound-events.md). +//! +//! Capture hands each admitted 20 ms block to a bounded queue and never waits: +//! a full queue drops the block and marks the window overloaded. One worker +//! thread keeps at most one patch of one uninterrupted capture era (under two +//! seconds), scores it every 500 ms and purges it on any gap, mute, epoch or +//! fence change, overflow or age past two seconds. Its results go to the Bus +//! pump, which checks current authority again before publishing. +//! +//! The frontend and patch bounds are the offline probe's (Sixth, 19 October), +//! checked there against an independent NumPy specification. +use crate::{ + BLOCK, RATE, + pcm::CaptureStamp, + privacy::{Permit, PublicationGate}, +}; +use lamp_msgs::{ + OwnVoiceEvidence, SOUND_RAW_AGE_US, SoundClasses, SoundScores, SoundUnavailable, SoundWindow, +}; +use rustfft::{Fft, FftPlanner, num_complex::Complex}; +use sha2::{Digest, Sha256}; +use std::{ + collections::VecDeque, + io::Read, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError}, + }, + time::Duration, +}; +use tract_onnx::{model::ParseResult, prelude::*}; + +/// Only an exact `on` selects it, as with the other Ears switches. +pub fn enabled(classes: Option<&str>) -> bool { + classes == Some("on") +} + +/// Qualcomm AI Hub's YAMNet ONNX float export, release v0.59.0 +/// (`yamnet-onnx-float.zip`): the graph and its external weights beside it. +pub const MODEL_ID: &str = "yamnet-qai-v0.59.0"; +const GRAPH_BYTES: u64 = 25_789; +const GRAPH_SHA256: &str = "970ea63f4dc1aa33c86ef7d61d854319fdfe6bdd078f5b3d674ee1db5dd96471"; +/// The graph names its weights by this file name, beside it. +pub const DATA_NAME: &str = "yamnet.data"; +const DATA_BYTES: u64 = 14_915_108; +const DATA_SHA256: &str = "1d6dff488da4712190e10f1c841252aa58011381132d0126dcd5ee56a4e7ed31"; +/// AudioSet indices in the bundled label map: Speech, Music, Laughter. +const SPEECH: usize = 0; +const MUSIC: usize = 132; +const LAUGHTER: usize = 13; +const CLASSES: usize = 521; + +pub const FFT: usize = 512; +const WINDOW: usize = 400; +pub const HOP: usize = 160; +pub const FRAMES: usize = 96; +pub const MELS: usize = 64; +/// 50 frames: a patch every 500 ms. +pub const PATCH_HOP: usize = 50; +/// Blocks the queue holds: 500 ms. Longer inference overflows and purges. +pub const QUEUE: usize = 25; +const BLOCK_US: u64 = 20_000; +/// No block for this long and the window is gone: closed or a device gap. +const SILENT_US: u64 = 100_000; +/// The contract's retention bound, in samples: two seconds. +const RETAIN: usize = 2 * RATE as usize; + +/// One admitted block, owned. Neither Clone nor Debug; zeroed on drop with the +/// same best-effort limits as [`crate::pcm::Pcm`]. +pub struct Block { + samples: [i16; BLOCK], + stamp: CaptureStamp, +} + +impl Drop for Block { + fn drop(&mut self) { + self.samples.fill(0); + } +} + +/// The capture side. `offer` copies 640 bytes into a preallocated bounded +/// queue and never blocks or allocates. +pub struct Handoff { + sender: SyncSender, + overflowed: Arc, +} + +/// The worker side of the queue. +pub struct Intake { + receiver: Receiver, + overflowed: Arc, +} + +pub fn handoff() -> (Handoff, Intake) { + let (sender, receiver) = mpsc::sync_channel(QUEUE); + let overflowed = Arc::new(AtomicBool::new(false)); + ( + Handoff { + sender, + overflowed: overflowed.clone(), + }, + Intake { + receiver, + overflowed, + }, + ) +} + +impl Handoff { + /// Only a block whose own permit and fence still hold is copied. A full + /// queue drops it and marks the worker's window overloaded. + pub fn offer( + &self, + samples: &[i16; BLOCK], + gate: &PublicationGate, + stamp: CaptureStamp, + now_us: u64, + ) -> bool { + if stamp.captured_at_us > now_us || !gate.allows(stamp.permit, stamp.fence, now_us) { + return false; + } + match self.sender.try_send(Block { + samples: *samples, + stamp, + }) { + Ok(()) => true, + Err(TrySendError::Full(_)) => { + self.overflowed.store(true, Ordering::Release); + false + } + Err(TrySendError::Disconnected(_)) => false, + } + } +} + +/// A patch's samples, by index from the start of its capture era. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Bounds { + pub first_frame: usize, + pub start_sample: usize, + pub end_sample: usize, +} + +/// The full centered 512-sample FFT extent of 96 frames 160 samples apart. +/// The first patch of an era reflects at its start (15456 samples); later +/// ones take 15712 real samples. No end is ever padded. +pub fn bounds(first_frame: usize) -> Bounds { + Bounds { + first_frame, + start_sample: (first_frame * HOP).saturating_sub(FFT / 2), + end_sample: (first_frame + FRAMES - 1) * HOP + FFT / 2, + } +} + +/// The torch_audioset proxy's log-mel frontend, as pinned by the release: +/// periodic Hann 400 centered in FFT 512, hop 160, centered reflect padding, +/// magnitude, 64 HTK mel bands 125-7500 Hz with slopes linear in Hz, +/// natural log(mel + 0.001). +pub struct Frontend { + fft: Arc>, + window: [f32; FFT], + mel: Vec<[f32; MELS]>, +} + +impl Default for Frontend { + fn default() -> Self { + Self::new() + } +} + +impl Frontend { + pub fn new() -> Self { + let mut window = [0.0; FFT]; + // torch.hann_window defaults to periodic; torch.stft centers this + // shorter window inside its n_fft window (56 zeros each side). + for n in 0..WINDOW { + window[(FFT - WINDOW) / 2 + n] = + 0.5 - 0.5 * (2.0 * std::f32::consts::PI * n as f32 / WINDOW as f32).cos(); + } + let hz_to_mel = |hz: f64| 2595.0 * (1.0 + hz / 700.0).log10(); + let low = hz_to_mel(125.0); + let high = hz_to_mel(7500.0); + // torch.linspace makes float32 mel points, converted back in float32. + let edges: Vec = (0..MELS + 2) + .map(|i| { + let mel = (low + (high - low) * i as f64 / (MELS + 1) as f64) as f32; + 700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0) + }) + .collect(); + let mel = (0..=FFT / 2) + .map(|bin| { + let hz = bin as f32 * RATE as f32 / FFT as f32; + std::array::from_fn(|band| { + let lower = (hz - edges[band]) / (edges[band + 1] - edges[band]); + let upper = (edges[band + 2] - hz) / (edges[band + 2] - edges[band + 1]); + lower.min(upper).max(0.0) + }) + }) + .collect(); + Self { + fft: FftPlanner::new().plan_fft_forward(FFT), + window, + mel, + } + } + + /// 96x64 features for `bounds`; `pcm[0]` is the era's sample `base`. + pub fn features(&self, pcm: &[f32], base: usize, bounds: Bounds) -> Result, String> { + if bounds != self::bounds(bounds.first_frame) + || bounds.start_sample < base + || bounds.end_sample > base + pcm.len() + || (bounds.start_sample == 0 && base != 0) + { + return Err("patch lacks its centered FFT extent".into()); + } + let mut features = Vec::with_capacity(FRAMES * MELS); + let mut spectrum = vec![Complex::new(0.0, 0.0); FFT]; + let mut scratch = vec![Complex::new(0.0, 0.0); self.fft.get_inplace_scratch_len()]; + for frame in bounds.first_frame..bounds.first_frame + FRAMES { + let center = frame * HOP; + for (n, value) in spectrum.iter_mut().enumerate() { + // Reflection excludes sample zero itself: -1 is sample 1. + let index = (center as isize + n as isize - (FFT / 2) as isize).unsigned_abs(); + *value = Complex::new(pcm[index - base] * self.window[n], 0.0); + } + self.fft.process_with_scratch(&mut spectrum, &mut scratch); + let mut energies = [0.0_f32; MELS]; + for (bin, value) in spectrum[..=FFT / 2].iter().enumerate() { + let magnitude = value.norm_sqr().sqrt(); + for (energy, weight) in energies.iter_mut().zip(self.mel[bin]) { + *energy += magnitude * weight; + } + } + features.extend(energies.map(|energy| (energy + 0.001).ln())); + } + // What the FFT made of the samples goes as they do (the reviewer on + // #1923): zeroed before it is freed, best effort as theirs. + spectrum.fill(Complex::new(0.0, 0.0)); + scratch.fill(Complex::new(0.0, 0.0)); + Ok(features) + } +} + +/// Logits get independent sigmoids: scores are multi-label, never a softmax. +pub fn sigmoid(logit: f32) -> f32 { + if logit >= 0.0 { + 1.0 / (1.0 + (-logit).exp()) + } else { + let x = logit.exp(); + x / (1.0 + x) + } +} + +fn read_pinned(path: &Path, bytes: u64, sha256: &str) -> Result, String> { + let said = |error: std::io::Error| format!("{}: {error}", path.display()); + let file = std::fs::File::open(path).map_err(said)?; + if file.metadata().map_err(said)?.len() != bytes { + return Err(format!("{} has an unexpected size", path.display())); + } + let mut contents = Vec::with_capacity(bytes as usize); + file.take(bytes + 1) + .read_to_end(&mut contents) + .map_err(said)?; + let digest = Sha256::digest(&contents) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + if contents.len() as u64 != bytes || digest != sha256 { + return Err(format!( + "{} does not match the pinned SHA256", + path.display() + )); + } + Ok(contents) +} + +/// The parser reads weights only from the verified bytes in memory, never by +/// path again, so the files cannot change between the check and the load. +struct PinnedData(Vec); + +const VIRTUAL_DIR: &str = "pinned"; + +impl tract_onnx::data_resolver::ModelDataResolver for PinnedData { + fn read_bytes_from_path( + &self, + buf: &mut Vec, + p: &Path, + offset: usize, + length: Option, + ) -> TractResult<()> { + if p != Path::new(VIRTUAL_DIR).join(DATA_NAME) { + return Err(TractError::msg("the graph names other weights")); + } + let end = match length { + Some(length) => offset.checked_add(length), + None => Some(self.0.len()), + }; + let Some(range) = end.filter(|end| offset <= *end && *end <= self.0.len()) else { + return Err(TractError::msg("external data range is out of bounds")); + }; + buf.extend_from_slice(&self.0[offset..range]); + Ok(()) + } +} + +pub struct Model { + plan: Arc, +} + +impl Model { + /// `graph` is `yamnet.onnx`; its weights are `yamnet.data` beside it. + /// Both must match their pinned size and SHA256 before parsing. + pub fn load(graph: &Path) -> Result { + let graph_bytes = read_pinned(graph, GRAPH_BYTES, GRAPH_SHA256)?; + let data = read_pinned(&graph.with_file_name(DATA_NAME), DATA_BYTES, DATA_SHA256)?; + let said = |error: TractError| format!("{}: {error}", graph.display()); + let onnx = tract_onnx::Onnx { + provider: Arc::new(PinnedData(data)), + ..tract_onnx::onnx() + }; + let proto = onnx + .proto_model_for_read(&mut graph_bytes.as_slice()) + .map_err(said)?; + let ParseResult { + model, + unresolved_inputs, + .. + } = onnx.parse(&proto, Some(VIRTUAL_DIR)).map_err(said)?; + if !unresolved_inputs.is_empty() { + return Err(format!("{}: unresolved inputs", graph.display())); + } + let plan = model + .into_optimized() + .and_then(|model| model.into_runnable()) + .map_err(said)?; + Ok(Self { plan }) + } + + pub fn score(&self, features: &[f32]) -> Result { + let input = Tensor::from_shape(&[1, 1, FRAMES, MELS], features) + .map_err(|error| error.to_string())?; + let output = self + .plan + .run(tvec!(input.into())) + .map_err(|error| error.to_string())?; + let view = output[0] + .to_plain_array_view::() + .map_err(|error| error.to_string())?; + if view.shape() != [1, CLASSES] || !view.iter().all(|x| x.is_finite()) { + return Err("invalid classifier output".into()); + } + let logits = view + .as_slice() + .ok_or("classifier output is not contiguous")?; + Ok(SoundScores { + speech: sigmoid(logits[SPEECH]), + music: sigmoid(logits[MUSIC]), + laughter: sigmoid(logits[LAUGHTER]), + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Era { + epoch: u64, + fence: u64, + next_sequence: u64, +} + +/// A patch ready to score, with its source window on the capture clock. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Patch { + pub bounds: Bounds, + pub start_us: u64, + pub end_us: u64, + pub capture_epoch: u64, + pub fence: u64, +} + +/// The samples of one uninterrupted capture era, from the next patch's start. +/// Whole blocks are kept, so a block's read-completion time stays with it. +#[derive(Default)] +pub struct Window { + samples: Vec, + times: VecDeque, + base: usize, + first_frame: usize, + era: Option, +} + +impl Drop for Window { + fn drop(&mut self) { + self.purge(); + } +} + +fn current(gate: &PublicationGate, epoch: u64, fence: u64, now_us: u64) -> bool { + // Same epoch and fence with a live lease now: no gap, mute or expiry has + // closed this era since its samples were admitted. + gate.allows( + Permit { + epoch, + until_us: u64::MAX, + }, + fence, + now_us, + ) +} + +impl Window { + pub fn purge(&mut self) -> bool { + self.samples.fill(0.0); + self.samples.clear(); + self.times.clear(); + self.base = 0; + self.first_frame = 0; + self.era.take().is_some() + } + + pub fn held(&self) -> usize { + self.samples.len() + } + + /// Warming up: an era began and has not yet completed its first patch. + pub fn warming(&self) -> bool { + self.era.is_some() && self.first_frame == 0 + } + + /// Takes a block; returns why an earlier window was purged, if one was. + /// A block after a gap begins a new era; it never extends the old one. + pub fn push( + &mut self, + block: &Block, + gate: &PublicationGate, + now_us: u64, + ) -> Option { + let stamp = block.stamp; + let mut purged = self.maintain(gate, now_us); + if let Some(era) = self.era { + let reason = if era.epoch != stamp.permit.epoch || era.fence != stamp.fence { + Some(SoundUnavailable::Privacy) + } else if era.next_sequence != stamp.sequence { + Some(SoundUnavailable::DeviceGap) + } else { + None + }; + if reason.is_some() { + self.purge(); + purged = purged.or(reason); + } + } + if stamp.captured_at_us > now_us || !current(gate, stamp.permit.epoch, stamp.fence, now_us) + { + self.purge(); + return purged.or(Some(SoundUnavailable::Privacy)); + } + let Some(next_sequence) = stamp.sequence.checked_add(1) else { + self.purge(); + return purged.or(Some(SoundUnavailable::DeviceGap)); + }; + self.era = Some(Era { + epoch: stamp.permit.epoch, + fence: stamp.fence, + next_sequence, + }); + self.samples + .extend(block.samples.iter().map(|&x| f32::from(x) / 32768.0)); + self.times.push_back(stamp.captured_at_us); + if self.samples.len() > RETAIN { + self.purge(); + return purged.or(Some(SoundUnavailable::Overloaded)); + } + purged + } + + /// Purges a window whose authority lapsed or whose oldest sample passed + /// the two-second raw age bound. Run on every worker wake. + pub fn maintain(&mut self, gate: &PublicationGate, now_us: u64) -> Option { + let era = self.era?; + let reason = if !current(gate, era.epoch, era.fence, now_us) { + SoundUnavailable::Privacy + } else if self.times.front().is_some_and(|&at| { + now_us.saturating_sub(at.saturating_sub(BLOCK_US)) > SOUND_RAW_AGE_US + }) { + SoundUnavailable::Overloaded + } else { + return None; + }; + self.purge(); + Some(reason) + } + + pub fn ready(&self) -> Option { + let era = self.era?; + let bounds = bounds(self.first_frame); + if bounds.end_sample > self.base + self.samples.len() { + return None; + } + let first = (bounds.start_sample - self.base) / BLOCK; + let last = (bounds.end_sample - 1 - self.base) / BLOCK; + Some(Patch { + bounds, + start_us: self.times[first].saturating_sub(BLOCK_US), + end_us: self.times[last], + capture_epoch: era.epoch, + fence: era.fence, + }) + } + + pub fn features(&self, frontend: &Frontend, patch: &Patch) -> Result, String> { + frontend.features(&self.samples, self.base, patch.bounds) + } + + /// Drops every whole block before the next patch's start. + pub fn advance(&mut self) { + self.first_frame += PATCH_HOP; + let start = bounds(self.first_frame).start_sample; + let blocks = (start.saturating_sub(self.base) / BLOCK).min(self.times.len()); + self.samples[..blocks * BLOCK].fill(0.0); + self.samples.drain(..blocks * BLOCK); + self.times.drain(..blocks); + self.base += blocks * BLOCK; + } +} + +/// A message for the pump, with the fence its scores were admitted under. +pub struct Classified { + pub classes: SoundClasses, + pub fence: u64, +} + +impl Classified { + /// The pump's last check before publishing: scores leave only while their + /// era still holds authority now and inside the two-second age bound. + pub fn publishable(&self, gate: &PublicationGate, now_us: u64) -> bool { + match &self.classes { + SoundClasses::Unavailable { .. } => true, + SoundClasses::Scored { window, .. } => { + current(gate, window.capture_epoch, self.fence, now_us) + && self.classes.check_at(now_us).is_ok() + } + } + } +} + +/// When to say the topic is unavailable: at once after scores, otherwise +/// every 500 ms (the topic's period), and at most every 100 ms. +struct Status { + reason: SoundUnavailable, + sent_us: Option, + scored: bool, +} + +impl Status { + fn since(&self, now_us: u64) -> u64 { + self.sent_us + .map_or(u64::MAX, |at| now_us.saturating_sub(at)) + } + + fn send(&mut self, now_us: u64, send: &mut impl FnMut(Classified)) { + self.sent_us = Some(now_us); + self.scored = false; + send(Classified { + classes: SoundClasses::Unavailable { + reason: self.reason, + }, + fence: 0, + }); + } + + fn purged(&mut self, reason: SoundUnavailable, now_us: u64, send: &mut impl FnMut(Classified)) { + self.reason = reason; + if self.scored || self.since(now_us) >= 100_000 { + self.send(now_us, send); + } + } + + fn idle(&mut self, now_us: u64, send: &mut impl FnMut(Classified)) { + // A scored era overdue by a whole patch is behind: overloaded. + let due = if self.scored { 1_000_000 } else { 500_000 }; + if self.since(now_us) >= due { + if self.scored { + self.reason = SoundUnavailable::Overloaded; + } + self.send(now_us, send); + } + } +} + +/// The worker thread's loop. `score` is `None` for a model that failed its +/// pins: then no block is retained and the topic says `invalid_model`. +/// Ends on `stop` or when capture drops its [`Handoff`]. +pub fn serve( + intake: Intake, + gate: &PublicationGate, + stop: &AtomicBool, + mut now: impl FnMut() -> u64, + mut score: Option Result>, + mut send: impl FnMut(Classified), +) { + let frontend = Frontend::new(); + let mut window = Window::default(); + let mut status = Status { + reason: if score.is_some() { + SoundUnavailable::WarmingUp + } else { + SoundUnavailable::InvalidModel + }, + sent_us: None, + scored: false, + }; + let mut heard_us = now(); + while !stop.load(Ordering::Acquire) { + let received = intake.receiver.recv_timeout(Duration::from_millis(20)); + let at = now(); + match received { + Ok(block) if score.is_some() => { + heard_us = at; + if intake.overflowed.swap(false, Ordering::AcqRel) && window.purge() { + status.purged(SoundUnavailable::Overloaded, at, &mut send); + } + if let Some(reason) = window.push(&block, gate, at) { + status.purged(reason, at, &mut send); + } + if window.warming() { + status.reason = SoundUnavailable::WarmingUp; + } + } + Ok(_) => {} + Err(RecvTimeoutError::Timeout) => { + if at.saturating_sub(heard_us) >= SILENT_US { + // No blocks and no live lease: closed for privacy, as + // for a lamp muted from the start (Third on #1923). + let reason = window.maintain(gate, at).unwrap_or(if gate.live(at) { + SoundUnavailable::DeviceGap + } else { + SoundUnavailable::Privacy + }); + if window.purge() || reason != SoundUnavailable::DeviceGap { + status.purged(reason, at, &mut send); + } + } + } + Err(RecvTimeoutError::Disconnected) => break, + } + if let Some(reason) = window.maintain(gate, at) { + status.purged(reason, at, &mut send); + } + if let (Some(score), Some(patch)) = (score.as_mut(), window.ready()) { + let scores = window.features(&frontend, &patch).and_then(|mut features| { + // A log-mel patch is about a second of the room, roughly + // invertible: zeroed once scored (the reviewer on #1923). + let scored = score(&features); + features.fill(0.0); + scored + }); + window.advance(); + let completed_us = now(); + match scores { + Ok(scores) => { + status.sent_us = Some(completed_us); + status.scored = true; + send(Classified { + classes: SoundClasses::Scored { + model_id: MODEL_ID.into(), + window: SoundWindow { + start_us: patch.start_us, + end_us: patch.end_us, + completed_us, + capture_epoch: patch.capture_epoch, + }, + scores, + // Ears' speaking flag maps a stale voice state to + // false, so it is not evidence of a quiet lamp. + own_voice: OwnVoiceEvidence::Unknown, + }, + fence: patch.fence, + }); + } + Err(_) => { + window.purge(); + status.purged(SoundUnavailable::InferenceFailed, completed_us, &mut send); + } + } + } + status.idle(now(), &mut send); + } +} + +/// The pinned model at `LAMP_EARS_CLASSES_MODEL`, or why there is none. +pub fn model_from(path: Option<&str>) -> Result { + let path = path + .filter(|path| Path::new(path).is_absolute()) + .map(PathBuf::from) + .ok_or("LAMP_EARS_CLASSES_MODEL is not an absolute path to yamnet.onnx")?; + Model::load(&path) +} + +#[cfg(test)] +mod tests; diff --git a/lampOS/crates/ears/src/classes/tests.rs b/lampOS/crates/ears/src/classes/tests.rs new file mode 100644 index 0000000000..2966db52f9 --- /dev/null +++ b/lampOS/crates/ears/src/classes/tests.rs @@ -0,0 +1,435 @@ +use super::*; +use std::{sync::mpsc::TryRecvError, thread, time::Instant}; + +const FAR: u64 = u64::MAX / 2; + +fn gate(epoch: u64) -> PublicationGate { + let gate = PublicationGate::default(); + gate.update( + Some(Permit { + epoch, + until_us: FAR, + }), + 0, + ); + gate +} + +fn stamp(epoch: u64, sequence: u64, captured_at_us: u64) -> CaptureStamp { + CaptureStamp { + permit: Permit { + epoch, + until_us: FAR, + }, + fence: 0, + sequence, + captured_at_us, + } +} + +/// A 440 Hz tone with a slow swell: a WAV-like 16 kHz mono PCM16 signal. +fn tone(blocks: usize) -> Vec { + (0..blocks * BLOCK) + .map(|i| { + let t = i as f32 / RATE as f32; + let swell = 0.2 + 0.1 * (2.0 * std::f32::consts::PI * 0.5 * t).sin(); + (swell * (2.0 * std::f32::consts::PI * 440.0 * t).sin() * 32767.0) as i16 + }) + .collect() +} + +fn block(signal: &[i16], index: usize, stamp: CaptureStamp) -> Block { + Block { + samples: signal[index * BLOCK..(index + 1) * BLOCK] + .try_into() + .unwrap(), + stamp, + } +} + +#[test] +fn the_switch_is_exact_on_only() { + for value in [None, Some(""), Some("off"), Some("ON"), Some("yes")] { + assert!(!enabled(value)); + } + assert!(enabled(Some("on"))); +} + +#[test] +fn bounds_include_future_fft_samples_and_never_pad_an_incomplete_tail() { + assert_eq!( + bounds(0), + Bounds { + first_frame: 0, + start_sample: 0, + end_sample: 15_456 + } + ); + assert_eq!( + bounds(50), + Bounds { + first_frame: 50, + start_sample: 7_744, + end_sample: 23_456 + } + ); + let frontend = Frontend::new(); + assert!(frontend.features(&[0.0; 15_455], 0, bounds(0)).is_err()); + // A later patch never reflects at a buffer's trimmed start. + assert!(frontend.features(&[0.0; 16_000], 7_744, bounds(0)).is_err()); +} + +#[test] +fn silence_is_log_offset_and_tone_is_finite_and_not_silence() { + let frontend = Frontend::new(); + let silence = frontend + .features(&vec![0.0; RATE as usize], 0, bounds(0)) + .unwrap(); + assert_eq!(silence.len(), FRAMES * MELS); + assert!(silence.iter().all(|x| (*x - 0.001_f32.ln()).abs() < 1e-6)); + let tone: Vec<_> = (0..RATE as usize) + .map(|i| 0.25 * (2.0 * std::f32::consts::PI * 440.0 * i as f32 / RATE as f32).sin()) + .collect(); + let features = frontend.features(&tone, 0, bounds(0)).unwrap(); + assert!(features.iter().all(|x| x.is_finite())); + // 440 Hz falls in the low mel bands, not at either edge. + let middle = &features[40 * MELS..41 * MELS]; + let peak = middle + .iter() + .enumerate() + .max_by(|a, b| a.1.total_cmp(b.1)) + .unwrap() + .0; + assert!((5..=15).contains(&peak), "peak band {peak}"); +} + +#[test] +fn stable_sigmoid_is_multilabel_and_bounded() { + assert_eq!(sigmoid(0.0), 0.5); + assert_eq!(sigmoid(1000.0), 1.0); + assert_eq!(sigmoid(-1000.0), 0.0); + assert!((sigmoid(2.0) + sigmoid(-2.0) - 1.0).abs() < 1e-6); +} + +#[test] +fn capture_handoff_is_bounded_and_never_waits_for_a_slow_classifier() { + let (handoff, intake) = handoff(); + let gate = gate(1); + let samples = [1; BLOCK]; + let mut times = Vec::new(); + for sequence in 0..1000 { + let started = Instant::now(); + assert!(handoff.offer(&samples, &gate, stamp(1, sequence, 1), 2)); + times.push(started.elapsed().as_nanos()); + assert!(intake.receiver.try_recv().is_ok()); + } + // A stalled worker: the queue fills, then offers drop and mark overload. + let mut stalled = Vec::new(); + for sequence in 0..(QUEUE as u64 + 100) { + let started = Instant::now(); + let kept = handoff.offer(&samples, &gate, stamp(1, sequence, 1), 2); + stalled.push(started.elapsed().as_nanos()); + assert_eq!(kept, sequence < QUEUE as u64); + } + assert!(intake.overflowed.load(Ordering::Acquire)); + times.sort_unstable(); + stalled.sort_unstable(); + eprintln!( + "ears classes host handoff per 20 ms block: p50={}ns p95={}ns max={}ns (1000 blocks); with the worker stalled: p50={}ns max={}ns; excludes ALSA and target hardware", + times[499], + times[949], + times[999], + stalled[stalled.len() / 2], + stalled[stalled.len() - 1] + ); + // Far under the 1 ms target; a median keeps a busy CI host from flaking. + assert!(times[499] < 1_000_000 && stalled[stalled.len() / 2] < 1_000_000); +} + +#[test] +fn unauthorized_or_future_blocks_are_never_copied() { + let (handoff, intake) = handoff(); + let gate = gate(1); + assert!(!handoff.offer(&[1; BLOCK], &gate, stamp(2, 0, 1), 2)); + assert!(!handoff.offer(&[1; BLOCK], &gate, stamp(1, 0, 3), 2)); + gate.fence(); + assert!(!handoff.offer(&[1; BLOCK], &gate, stamp(1, 0, 1), 2)); + assert!(matches!( + intake.receiver.try_recv(), + Err(TryRecvError::Empty) + )); +} + +#[test] +fn a_streamed_window_scores_every_500_ms_with_the_offline_features_and_stays_bounded() { + let signal = tone(200); + let all: Vec = signal.iter().map(|&x| f32::from(x) / 32768.0).collect(); + let frontend = Frontend::new(); + let gate = gate(1); + let mut window = Window::default(); + let mut patches = Vec::new(); + for index in 0..200 { + let at = 1_000_000 + 20_000 * (index as u64 + 1); + assert_eq!( + window.push( + &block(&signal, index, stamp(1, index as u64, at)), + &gate, + at + ), + None + ); + assert!(window.held() <= RETAIN); + if let Some(patch) = window.ready() { + let streamed = window.features(&frontend, &patch).unwrap(); + assert_eq!(streamed, frontend.features(&all, 0, patch.bounds).unwrap()); + patches.push((index, patch)); + window.advance(); + } + } + // 15456 samples is 48.3 blocks: the first patch with block 49 (index 48), + // then one every 25 blocks (500 ms). + let at: Vec = patches.iter().map(|(index, _)| *index).collect(); + assert_eq!(at, vec![48, 73, 98, 123, 148, 173, 198]); + let (_, first) = patches[0]; + assert_eq!( + (first.start_us, first.end_us, first.capture_epoch), + (1_000_000, 1_000_000 + 20_000 * 49, 1) + ); + let (_, second) = patches[1]; + // Sample 7744 is in block 24, which began at 1.48 s; 23455 in block 73. + assert_eq!( + (second.start_us, second.end_us), + (1_000_000 + 20_000 * 24, 1_000_000 + 20_000 * 74) + ); + assert!(second.end_us - second.start_us < SOUND_RAW_AGE_US); +} + +#[test] +fn a_gap_mute_epoch_or_age_purges_and_a_later_block_starts_a_new_era() { + let signal = tone(60); + let push_run = |window: &mut Window, gate: &PublicationGate, epoch, from: u64, count| { + for sequence in from..from + count { + let at = 1_000_000 + 20_000 * sequence; + let block = block( + &signal, + (sequence % 60) as usize, + stamp(epoch, sequence, at), + ); + window.push(&block, gate, at); + } + }; + let gate1 = gate(1); + let mut window = Window::default(); + push_run(&mut window, &gate1, 1, 0, 40); + // A dropped block: the old samples go and cannot complete a patch. + let at = 1_000_000 + 20_000 * 41; + assert_eq!( + window.push(&block(&signal, 41, stamp(1, 41, at)), &gate1, at), + Some(SoundUnavailable::DeviceGap) + ); + assert!(window.warming()); + assert_eq!(window.held(), BLOCK); + push_run(&mut window, &gate1, 1, 42, 47); + assert!(window.ready().is_none()); + push_run(&mut window, &gate1, 1, 89, 1); + assert!(window.ready().is_some()); + + // Mute: the gate closes; maintain purges without waiting for a block. + let mut window = Window::default(); + push_run(&mut window, &gate1, 1, 0, 40); + gate1.update(None, 0); + assert_eq!( + window.maintain(&gate1, 1_900_000), + Some(SoundUnavailable::Privacy) + ); + assert_eq!(window.held(), 0); + + // A new epoch (renewed after a gap) cannot extend the old window. + let gate2 = gate(2); + let mut window = Window::default(); + push_run(&mut window, &gate2, 1, 0, 40); + assert_eq!(window.held(), 0); + push_run(&mut window, &gate2, 2, 40, 40); + assert_eq!(window.held(), 40 * BLOCK); + assert!(window.ready().is_none()); + + // A fence revokes the era even with the same epoch. + let fence = gate2.fence(); + gate2.update( + Some(Permit { + epoch: 2, + until_us: FAR, + }), + fence, + ); + assert_eq!( + window.maintain(&gate2, 1_900_000), + Some(SoundUnavailable::Privacy) + ); + + // A stalled worker: samples older than two seconds are purged. + let gate3 = gate(3); + let mut window = Window::default(); + push_run(&mut window, &gate3, 3, 0, 10); + // The first block's samples began at 0.98 s. + assert_eq!(window.maintain(&gate3, 980_000 + 2_000_000), None); + assert_eq!( + window.maintain(&gate3, 980_000 + 2_000_001), + Some(SoundUnavailable::Overloaded) + ); + assert_eq!(window.held(), 0); +} + +#[test] +fn the_worker_publishes_scores_then_unavailable_and_the_pump_rechecks_authority() { + let (handoff, intake) = handoff(); + let gate = Arc::new(gate(1)); + let stop = Arc::new(AtomicBool::new(false)); + let (sent, received) = mpsc::channel(); + let worker = { + let (gate, stop) = (gate.clone(), stop.clone()); + thread::spawn(move || { + serve( + intake, + &gate, + &stop, + lamp_ipc::monotonic_us, + Some(|_: &[f32]| { + Ok(SoundScores { + speech: 0.1, + music: 0.9, + laughter: 0.0, + }) + }), + |classified| sent.send(classified).unwrap(), + ) + }) + }; + let signal = tone(74); + for index in 0..74 { + let at = lamp_ipc::monotonic_us(); + let samples: &[i16; BLOCK] = signal[index * BLOCK..(index + 1) * BLOCK] + .try_into() + .unwrap(); + assert!(handoff.offer(samples, &gate, stamp(1, index as u64, at), at)); + thread::sleep(Duration::from_millis(1)); + } + let mut scored = Vec::new(); + while scored.len() < 2 { + let classified = received.recv_timeout(Duration::from_secs(5)).unwrap(); + if let SoundClasses::Scored { window, .. } = &classified.classes { + assert_eq!(window.capture_epoch, 1); + assert!(classified.publishable(&gate, lamp_ipc::monotonic_us())); + scored.push(classified); + } + } + // Mute: a held result may no longer leave, and the worker says privacy. + gate.update(None, 0); + assert!(!scored[1].publishable(&gate, lamp_ipc::monotonic_us())); + let reason = loop { + if let SoundClasses::Unavailable { reason } = received + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .classes + { + break reason; + } + }; + assert_eq!(reason, SoundUnavailable::Privacy); + stop.store(true, Ordering::Release); + worker.join().unwrap(); +} + +#[test] +fn a_model_that_fails_its_pins_is_refused_before_parsing() { + let dir = std::env::temp_dir().join(format!("ears-classes-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let graph = dir.join("yamnet.onnx"); + std::fs::write(&graph, vec![0u8; GRAPH_BYTES as usize]).unwrap(); + let error = Model::load(&graph).err().unwrap(); + assert!(error.contains("pinned SHA256"), "{error}"); + std::fs::write(&graph, b"short").unwrap(); + assert!(Model::load(&graph).is_err()); + assert!(model_from(Some("relative/yamnet.onnx")).is_err()); + assert!(model_from(None).is_err()); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// Runs only with the pinned files present: +/// `LAMP_EARS_CLASSES_TEST_MODEL=/abs/yamnet.onnx cargo test -p lamp-ears -- --nocapture`. +#[test] +fn the_pinned_model_scores_a_patch_in_time() { + let Some(path) = std::env::var_os("LAMP_EARS_CLASSES_TEST_MODEL") else { + eprintln!("skipped: LAMP_EARS_CLASSES_TEST_MODEL is unset"); + return; + }; + if !Path::new(&path).exists() { + eprintln!("skipped: {} is absent", Path::new(&path).display()); + return; + } + let started = Instant::now(); + let model = Model::load(Path::new(&path)).unwrap(); + let load = started.elapsed(); + let frontend = Frontend::new(); + let signal: Vec = tone(200).iter().map(|&x| f32::from(x) / 32768.0).collect(); + let mut times = Vec::new(); + for first_frame in (0..).step_by(PATCH_HOP).take(20) { + let bounds = bounds(first_frame % 300); + let started = Instant::now(); + let features = frontend.features(&signal, 0, bounds).unwrap(); + let scores = model.score(&features).unwrap(); + times.push(started.elapsed().as_micros()); + for score in [scores.speech, scores.music, scores.laughter] { + assert!(score.is_finite() && (0.0..=1.0).contains(&score)); + } + } + let silence = model.score(&vec![0.001_f32.ln(); FRAMES * MELS]).unwrap(); + times.sort_unstable(); + eprintln!( + "ears classes model load {:?}; per patch (frontend + inference) p50={}us max={}us over 20; silence scores {silence:?}; host only", + load, times[10], times[19] + ); + // A patch every 500 ms; scoring must keep well inside it. + assert!(times[10] < 250_000); +} + +/// Third on #1923 (P3): a lamp muted from the start has no lease and sends +/// no blocks; after 100 ms of none the worker says `privacy`, not +/// `warming_up` or `device_gap`. +#[test] +fn a_lamp_muted_from_the_start_says_privacy() { + let (_handoff, intake) = handoff(); + let gate = Arc::new(PublicationGate::default()); + let stop = Arc::new(AtomicBool::new(false)); + let (sent, received) = mpsc::channel(); + let worker = { + let (gate, stop) = (gate.clone(), stop.clone()); + thread::spawn(move || { + serve( + intake, + &gate, + &stop, + lamp_ipc::monotonic_us, + Some(|_: &[f32]| Err::("never scored".to_owned())), + |classified| sent.send(classified).unwrap(), + ) + }) + }; + // Its first word, at start, is still `warming_up`; within a second it + // is `privacy`. + let deadline = Instant::now() + Duration::from_secs(1); + let said = std::iter::from_fn(|| { + received + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .ok() + }) + .find_map(|classified| match classified.classes { + SoundClasses::Unavailable { reason } if reason != SoundUnavailable::WarmingUp => { + Some(reason) + } + _ => None, + }); + assert_eq!(said, Some(SoundUnavailable::Privacy)); + stop.store(true, Ordering::Release); + worker.join().unwrap(); +} diff --git a/lampOS/crates/ears/src/lib.rs b/lampOS/crates/ears/src/lib.rs index e0d0c8ca42..65137bd346 100644 --- a/lampOS/crates/ears/src/lib.rs +++ b/lampOS/crates/ears/src/lib.rs @@ -5,6 +5,7 @@ use lamp_msgs::{EARS_SILENCE_DB, RoomSound, SuddenSound}; use std::collections::VecDeque; +pub mod classes; pub mod pcm; pub mod privacy; diff --git a/lampOS/crates/ears/src/main.rs b/lampOS/crates/ears/src/main.rs index eb1cba71e3..bace29313d 100644 --- a/lampOS/crates/ears/src/main.rs +++ b/lampOS/crates/ears/src/main.rs @@ -22,6 +22,9 @@ While the voice says it is speaking, and half a second after, nothing is sudden. By default it keeps no sound and writes none down; only metadata leaves it. LAMP_BUS_AUDIO=on exports raw mic1/audio (16 kHz mono S16LE), only with LAMP_PRIVACY_CONTROLS=on. Missing privacy enablement refuses startup. +LAMP_EARS_CLASSES=on scores speech, music and laughter on ears/classes with the +pinned YAMNet at LAMP_EARS_CLASSES_MODEL (yamnet.onnx, yamnet.data beside it), +only with LAMP_PRIVACY_CONTROLS=on; without it no classifier runs. Ears remains the only microphone reader; an explicit bus recorder may retain PCM. lamp-ears check FILE... runs the same listening over files of sound (16 kHz, one channel, @@ -200,6 +203,9 @@ mod serve { if lamp_ears::privacy::enabled(privacy.as_deref()) { return crate::privacy_runtime::run(arguments); } + if lamp_ears::classes::enabled(std::env::var("LAMP_EARS_CLASSES").ok().as_deref()) { + log("LAMP_EARS_CLASSES=on needs LAMP_PRIVACY_CONTROLS=on; no classifier runs"); + } if let Some(value) = privacy.filter(|value| !value.is_empty() && value != "off") { eprintln!( "lamp-ears: LAMP_PRIVACY_CONTROLS={value:?} is not \"on\"; the room microphone ignores the privacy switch" diff --git a/lampOS/crates/ears/src/privacy.rs b/lampOS/crates/ears/src/privacy.rs index 7b76a91788..033a3135c6 100644 --- a/lampOS/crates/ears/src/privacy.rs +++ b/lampOS/crates/ears/src/privacy.rs @@ -73,6 +73,12 @@ impl PublicationGate { } } + /// Whether any lease is live at `now`: none while muted, lapsed or + /// before the first grant. + pub fn live(&self, now: u64) -> bool { + now < self.until_us.load(Ordering::Acquire) + } + pub fn allows(&self, permit: Permit, generation: u64, now: u64) -> bool { generation != u64::MAX && generation == self.generation() diff --git a/lampOS/crates/ears/src/privacy_runtime.rs b/lampOS/crates/ears/src/privacy_runtime.rs index 85383569d1..3dca16e64a 100644 --- a/lampOS/crates/ears/src/privacy_runtime.rs +++ b/lampOS/crates/ears/src/privacy_runtime.rs @@ -8,10 +8,12 @@ use alsa::{ use lamp_bus::{Bus, Message, Notice, Subscriber}; use lamp_ears::{ BLOCK, RATE, Settings, + classes::Classified, privacy::{Capture, CaptureOwner, Emission, LeaseGate, PublicationGate}, }; use lamp_msgs::{ - Audio, Diagnostic, Level, MicrophonePrivacy, RoomSound, SuddenSound, Voice, VoiceState, + Audio, Diagnostic, Level, MicrophonePrivacy, RoomSound, SoundClasses, SuddenSound, Voice, + VoiceState, }; use std::{ io, @@ -200,6 +202,7 @@ fn connected( gate: &mut LeaseGate, sender: &SyncSender, output: &Receiver, + classes: Option<&Receiver>, shared: &Shared, ) -> io::Result<()> { let mut privacy = bus.subscribe_typed( @@ -211,6 +214,13 @@ fn connected( let mut sudden = bus.publisher_declared(SuddenSound::TOPIC, &SuddenSound::declaration(), NAME)?; let mut health = bus.publisher_declared(Diagnostic::TOPIC, &Diagnostic::declaration(), NAME)?; + let mut classes = match classes { + Some(receiver) => Some(( + receiver, + bus.publisher_declared(SoundClasses::TOPIC, &SoundClasses::declaration(), NAME)?, + )), + None => None, + }; let mut audio = None; let mut audio_retry = lamp_ears::pcm::ExportRetry::default(); let mut looked = 0; @@ -352,6 +362,23 @@ fn connected( Err(TryRecvError::Disconnected) => return Ok(()), } } + // Scores leave only while their capture era still holds authority + // now; the header keeps the audio's end, never the publish time. + if let Some((receiver, publisher)) = classes.as_mut() { + for _ in 0..BATCH { + let Ok(classified) = receiver.try_recv() else { + break; + }; + let now = lamp_ipc::monotonic_us(); + if classified.publishable(&shared.publication, now) { + let at_us = match &classified.classes { + SoundClasses::Scored { window, .. } => window.end_us, + SoundClasses::Unavailable { .. } => now, + }; + publisher.publish_json(at_us, &classified.classes)?; + } + } + } let now = lamp_ipc::monotonic_us(); if now.saturating_sub(looked) >= 100_000 { looked = now; @@ -374,13 +401,22 @@ fn pump( clock_id: String, sender: SyncSender, output: Receiver, + classes: Option>, shared: Arc, ) { let mut gate = LeaseGate::new(clock_id.clone()); while !shared.stop.load(Ordering::Acquire) { gate.close(); shared.revoke(); - let _ = connected(&bus, &clock_id, &mut gate, &sender, &output, &shared); + let _ = connected( + &bus, + &clock_id, + &mut gate, + &sender, + &output, + classes.as_ref(), + &shared, + ); shared.revoke(); shared.speaking.store(false, Ordering::Release); if !shared.stop.load(Ordering::Acquire) { @@ -389,6 +425,83 @@ fn pump( } } +/// Room sound classes (#1669): model loading and inference run on their own +/// thread, never on capture or the pump; capture only offers blocks. Off +/// unless `LAMP_EARS_CLASSES` is exactly `on`: then no thread, no model, no +/// handoff for capture and no publisher for the pump. +fn start_classes( + switch: Option<&str>, + model: Option, + shared: &Arc, +) -> io::Result<( + Option, + Option>, +)> { + if !lamp_ears::classes::enabled(switch) { + return Ok((None, None)); + } + let (handoff, intake) = lamp_ears::classes::handoff(); + let (send_classes, receive_classes) = mpsc::sync_channel(4); + let worker_shared = shared.clone(); + thread::Builder::new() + .name("ears-classes".into()) + .spawn(move || { + let model = lamp_ears::classes::model_from(model.as_deref()) + .inspect_err(|error| log(&format!("ears/classes says invalid_model: {error}"))) + .ok(); + lamp_ears::classes::serve( + intake, + &worker_shared.publication, + &worker_shared.stop, + lamp_ipc::monotonic_us, + model + .as_ref() + .map(|model| |features: &[f32]| model.score(features)), + |classified| { + // A stalled pump drops results; they would go stale. + let _ = send_classes.try_send(classified); + }, + ); + })?; + log( + "LAMP_EARS_CLASSES=on: ears/classes scores mic1 locally; at most one patch of one capture era (under 2 s) is held in memory, none written", + ); + Ok((Some(handoff), Some(receive_classes))) +} + +/// How long the route may go with no permission from Voice before it is +/// said, once. +const NO_PERMISSION_SAID_AFTER_US: u64 = 10_000_000; + +/// The once-only line for a route that never opens (Third's yes on #1722, +/// 04:37): with no lease from Voice for 10 s since the start or the last +/// one, it is said once in the process's life, so a lamp whose Controls has +/// no privacy source (no slide in its pins) says why it hears nothing, +/// instead of waiting silently. +#[derive(Default)] +struct Unpermitted { + since_us: Option, + said: bool, +} + +impl Unpermitted { + fn note(&mut self, permitted: bool, now_us: u64) -> Option<&'static str> { + if permitted { + self.since_us = None; + return None; + } + let since = *self.since_us.get_or_insert(now_us); + if self.said || now_us.saturating_sub(since) < NO_PERMISSION_SAID_AFTER_US { + return None; + } + self.said = true; + Some( + "no permission from Voice for 10 s: the microphone stays closed until one comes (muted, \ + or Controls has no privacy source, as with no privacy slide in its pins)", + ) + } +} + pub(super) fn run(arguments: &Arguments) -> io::Result<()> { let bus = match &arguments.bus { Some(root) => Bus::new(root)?, @@ -423,10 +536,24 @@ pub(super) fn run(arguments: &Arguments) -> io::Result<()> { }); let (send_input, input) = mpsc::sync_channel(CAPACITY); let (output, receive_output) = mpsc::sync_channel(CAPACITY); + let (classes, receive_classes) = start_classes( + std::env::var("LAMP_EARS_CLASSES").ok().as_deref(), + std::env::var("LAMP_EARS_CLASSES_MODEL").ok(), + &shared, + )?; let pump_shared = shared.clone(); let worker = thread::Builder::new() .name("ears-privacy-bus".into()) - .spawn(move || pump(bus, clock_id, send_input, receive_output, pump_shared))?; + .spawn(move || { + pump( + bus, + clock_id, + send_input, + receive_output, + receive_classes, + pump_shared, + ) + })?; log("privacy controls on; microphone waits for Voice's fresh permission"); if export { log( @@ -438,6 +565,7 @@ pub(super) fn run(arguments: &Arguments) -> io::Result<()> { let mut failures = 0u64; let mut retry_at = 0u64; let mut reported = 0u64; + let mut unpermitted = Unpermitted::default(); loop { let fence = shared.publication.generation(); if fence != seen_fence || fence == u64::MAX { @@ -480,6 +608,9 @@ pub(super) fn run(arguments: &Arguments) -> io::Result<()> { shared.pcm.invalidate(); } let permit = owner.maintain(lamp_ipc::monotonic_us())?; + if let Some(line) = unpermitted.note(permit.is_some(), lamp_ipc::monotonic_us()) { + log(line); + } shared.publication.update(permit, seen_fence); shared .pcm @@ -491,17 +622,26 @@ pub(super) fn run(arguments: &Arguments) -> io::Result<()> { shared.speaking.load(Ordering::Acquire), || seen_fence == shared.publication.generation(), |samples, permit, sequence, captured_at_us| { + let stamp = lamp_ears::pcm::CaptureStamp { + permit, + fence: seen_fence, + sequence, + captured_at_us, + }; shared.pcm.offer( samples, &shared.publication, - lamp_ears::pcm::CaptureStamp { - permit, - fence: seen_fence, - sequence, - captured_at_us, - }, + stamp, lamp_ipc::monotonic_us(), ); + if let Some(classes) = &classes { + classes.offer( + samples, + &shared.publication, + stamp, + lamp_ipc::monotonic_us(), + ); + } }, ) { Ok(Some(emission)) => { @@ -551,3 +691,50 @@ pub(super) fn run(arguments: &Arguments) -> io::Result<()> { } close.and(result) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The reviewer on #1923 (P3): with the switch anything but exactly + /// `on`, no classifier thread, no handoff for capture and no publisher + /// for the pump. + #[test] + fn classes_start_nothing_unless_the_switch_is_on() { + let shared = Arc::new(Shared::new(false)); + for switch in [None, Some(""), Some("off"), Some("ON"), Some("on ")] { + let (handoff, receiver) = start_classes(switch, None, &shared).unwrap(); + assert!(handoff.is_none() && receiver.is_none(), "{switch:?}"); + } + assert_eq!( + Arc::strong_count(&shared), + 1, + "no thread holds the shared state" + ); + } + + /// Third's yes (04:37): said once, after 10 s with no permission, and + /// never again in the process's life; a permission in between starts the + /// 10 s again. + #[test] + fn no_permission_is_said_once_after_ten_seconds() { + let mut unpermitted = Unpermitted::default(); + let s = |seconds: u64| seconds * 1_000_000; + assert_eq!(unpermitted.note(false, s(0)), None); + assert_eq!(unpermitted.note(true, s(9)), None); + assert_eq!(unpermitted.note(false, s(10)), None, "permitted at 9 s"); + assert_eq!(unpermitted.note(false, s(19)), None); + assert!( + unpermitted + .note(false, s(20)) + .is_some_and(|line| line.contains("no permission from Voice")) + ); + assert_eq!(unpermitted.note(false, s(60)), None, "once only"); + assert_eq!(unpermitted.note(true, s(61)), None); + assert_eq!( + unpermitted.note(false, s(200)), + None, + "once in the process's life" + ); + } +} diff --git a/lampOS/docs/ears.md b/lampOS/docs/ears.md index 63ba3743dd..5763385685 100644 --- a/lampOS/docs/ears.md +++ b/lampOS/docs/ears.md @@ -36,6 +36,7 @@ The voice keeps mic2 to itself; the two parts never share a device. | `ears/level` | `RoomSound` | latest, every 100 ms | The last 100 ms in dBFS; the room's quiet (the tenth percentile of the last 30 s heard while the lamp was not speaking; none for the first two seconds); whether the lamp's own voice is in it | | `ears/sudden` | `SuddenSound` | event | Its loudest 20 ms, the quiet it rose out of, how long it stayed 10 dB over the quiet in the 300 ms it was watched, whether that was under 200 ms ("short": a knock, a clap) or not (a voice, a chair, music), and how long ago it began | | `diagnostics` | `Diagnostic` | every second | `ears`: ok, or an error with why the microphone does not open; the quiet now, how many sudden sounds, overruns | +| `ears/classes` | `SoundClasses` | latest, every 500 ms | Only with `LAMP_EARS_CLASSES=on` and `LAMP_PRIVACY_CONTROLS=on`: speech, music and laughter scores of the last patch, or why there are none (below, and [sound-events.md](sound-events.md)) | **When a sound is sudden.** A 20 ms block at least 30 dB over the quiet (`--rise-db`, 10 to 60) and at least -40 dBFS, after 200 ms that averaged under 10 dB over the quiet. @@ -252,3 +253,23 @@ retention enforcement. `lamp-bus record` can persist raw PCM using the caller's umask and its serialized-file size cap (64 MiB by default). Operator permission, private storage and retention limits are required policy, not a property the publisher can enforce after handing a block to a subscriber. + + +## Optional room sound classes (#1669) + +`LAMP_EARS_CLASSES=on` (exactly `on`) scores speech, music and laughter with +a pinned YAMNet and publishes them on `ears/classes`. It is off unless set, +and runs only with `LAMP_PRIVACY_CONTROLS=on`; otherwise one journal line +says no classifier runs. `LAMP_EARS_CLASSES_MODEL` is the absolute path to +`yamnet.onnx`, with `yamnet.data` beside it; both must match their pinned +SHA256 or the topic says `invalid_model`. No unit sets either. + +Capture offers each admitted block, from the same post-read callback as the +PCM export, to a bounded queue of 500 ms; it never waits, and a full queue +drops the block and purges. A separate thread holds at most one patch of +one uninterrupted capture era (under 1.5 s, capped at 2 s) in memory and +writes nothing; any gap, mute, epoch or fence change, overflow or age past +2 s purges it. The Bus pump publishes a score only while that era still +holds authority and the result is under 2 s old. The retention, timing and +measurements are in [sound-events.md](sound-events.md), "The publisher +behind LAMP_EARS_CLASSES". Nothing consumes the topic yet. diff --git a/lampOS/docs/msgs.md b/lampOS/docs/msgs.md index 3f5f386f60..38f6af28da 100644 --- a/lampOS/docs/msgs.md +++ b/lampOS/docs/msgs.md @@ -409,7 +409,7 @@ topic catalogue. Neither carries any sound. |---|---|---|---|---| | `RoomSound` | `ears/level` | latest | 100 ms / none | The room microphone's last 100 ms in dBFS, the room's quiet (none for the first two seconds), whether the lamp's own voice is in it | | `SuddenSound` | `ears/sudden` | event | none / none | A sound that rose suddenly out of the quiet: its loudest 20 ms, the quiet, how long it stayed loud in 300 ms, whether that was short, how long ago it began | -| `SoundClasses` | `ears/classes` | latest | 500 ms / 1,500 ms | Unavailable with no scores, or multi-label speech/music/laughter scores with original window times, completion, capture epoch, model id and own-voice evidence; typed contract only, no live publisher yet (#1669, sound-events.md) | +| `SoundClasses` | `ears/classes` | latest | 500 ms / 1,500 ms | Unavailable with no scores, or multi-label speech/music/laughter scores with original window times, completion, capture epoch, model id and own-voice evidence; Ears publishes it only with `LAMP_EARS_CLASSES=on` and `LAMP_PRIVACY_CONTROLS=on`, off by default (#1669, sound-events.md) | Levels are dBFS from -120 (digital silence) to 0; `check()` refuses anything else, and a peak under its quiet. diff --git a/lampOS/docs/sound-events.md b/lampOS/docs/sound-events.md index 05ca00c3cc..891066dfa4 100644 --- a/lampOS/docs/sound-events.md +++ b/lampOS/docs/sound-events.md @@ -2,10 +2,14 @@ Status: the first builder accepted this topic and model direction on [19 October at 09:02](https://github.com/autonomous-ai/lampOS/issues/1669#issuecomment-6258073073). -Third reviewed the typed contract on 19 October; classifier retention still -needs validation before a live classifier uses it. Ears publishes levels and -sudden changes, not music or laughter. -This proposal reserves no production behaviour and changes no deployment unit. +Third reviewed the typed contract on 19 October. A publisher now exists in +Ears behind `LAMP_EARS_CLASSES=on`, off unless set and only with +`LAMP_PRIVACY_CONTROLS=on` (see "The publisher behind LAMP_EARS_CLASSES" +below). Third's privacy read of its two-second retention found no P1 or P2 +(19 October); its scores still need the empty-room and busy-room +measurement, and the owner's word, before any unit sets it. +No deployment unit changes; by default Ears publishes levels and sudden +changes, not music or laughter. ## Purpose and boundaries @@ -63,8 +67,8 @@ voice; it is not permission for a skill to act. `check()` validates shape/ranges and the raw-age bound. `check_at(now_us)` additionally rejects future or stale results. Neither checks privacy authority, clock identity or current capture epoch: those remain required at the publisher/consumer boundary. Unknown JSON -fields are refused, so an unavailable message cannot hide stale scores. There -is no runtime publisher in this checkpoint. +fields are refused, so an unavailable message cannot hide stale scores. The +only publisher is Ears behind its switch, below. There is no `safe_to_act` field. Consumer gates also require fresh voice state, no active conversation/input, no hand/pause fence, and the relevant skill's @@ -97,16 +101,18 @@ Proposed separate bounded retention for classification: enable classification. Its missing capture boundary is not silently repaired by trusting voice/speaking metadata. -This is a proposed additional retention contract, not a change already made to -`Emission` or `PublicationGate`. It needs a privacy-owner read before a live -collector is installed. Offline model/measurement work can proceed independently. +This additional retention contract is implemented behind the switch below, +without changing `Emission` or `PublicationGate`. It still needs a +privacy-owner read before any unit sets the switch. ## Model and measurement plan YAMNet is one candidate: TensorFlow's maintained implementation classifies 521 AudioSet classes with MobileNet-v1, accepting 16 kHz mono audio. The existing Rust inference stack and model/operator compatibility still need a concrete -check; no model is selected or advertised as running yet. +check. The switch-gated publisher pins Qualcomm AI Hub's YAMNet ONNX float +export, release v0.59.0, for evaluation only; distribution review of its +weights is outstanding, so they are not committed or deployed. Primary sources: - [TensorFlow YAMNet tutorial](https://www.tensorflow.org/hub/tutorials/yamnet) @@ -141,3 +147,101 @@ person addressing it. Departure is not inferred from sound classification. The offline voice-eval exporter recognizes `ears/classes` in its omitted-topic counts. It does not export score payloads or make them behavioral evidence. + +## The publisher behind LAMP_EARS_CLASSES + +`lamp-ears` with `LAMP_EARS_CLASSES=on` (exactly `on`) and +`LAMP_PRIVACY_CONTROLS=on` publishes `ears/classes` +(`crates/ears/src/classes.rs`). With the switch on and privacy controls off, +one journal line says no classifier runs, and none does. Off, nothing below +exists: no queue, thread or model. + +**Model.** `LAMP_EARS_CLASSES_MODEL` is the absolute path to `yamnet.onnx`, +with `yamnet.data` beside it. Both are pinned in the code by size and +SHA256 (graph 25,789 bytes `970ea63f…6471`, weights 14,915,108 bytes +`1d6dff48…7ed31`; `model_id` `yamnet-qai-v0.59.0`). They are read once, +checked, and parsed from the checked bytes in memory, so a file swapped +after the check is never loaded. A missing, relative or mismatched model +publishes `invalid_model` every 500 ms and retains no audio. Loading and +inference run on their own thread (`ears-classes`), never on capture or the +Bus pump. Scores are independent sigmoids of the logits for Speech (0), +Music (132) and Laughter (13). + +**Frontend and windows.** As the offline probe: 16 kHz, FFT 512, periodic +Hann 400 centered in it, hop 160, magnitude, 64 HTK mel bands 125–7500 Hz, +`log(mel + 0.001)`, patches of 96 frames every 50 frames (500 ms). A capture +era's first patch reflects at its start and needs 15,456 samples; later +patches take 15,712 real samples. A window's `start_us`/`end_us` are whole +20 ms blocks on the capture clock (read-completion time minus 20 ms, and the +last block's read completion), so they cover the patch's full FFT extent. +`own_voice` is always `unknown`: Ears' speaking flag turns a stale voice +state into "not speaking", which is not evidence of a quiet lamp. + +**Handoff and retention.** After the capture owner's post-read lease and +fence checks, the same callback that feeds the PCM export offers the block +to a bounded queue of 25 blocks (500 ms): a copy of 640 bytes, no wait, no +allocation. Only a block whose own permit and fence still hold is copied. +A full queue drops the block and marks the window overloaded. The worker +keeps whole blocks of one era from the next patch's start: at most about +16,000 samples, plus what waits in the queue, under 1.5 s and never over +the 2 s cap. It purges (zero-filled, best effort as for the PCM export) +on a sequence gap, a block from another epoch or fence, the era's +authority lapsing (checked on every 20 ms wake), 100 ms without a block, +an overflow, the oldest retained sample passing 2 s, or an inference error. +A later block starts a new era; it never completes the old one. What is +made from the samples goes the same way: each patch's 96x64 log-mel +features (about a second of the room, roughly invertible) and the FFT's +buffers are zero-filled once the patch is scored, best effort as the +samples are; the model's own copy of its input is the inference library's +and is not. + +On the privacy route, lamp-ears says once, after 10 s with no permission +from Voice (since its start or the last one), that the microphone stays +closed until one comes: muted, or Controls has no privacy source, as on a +lamp with no privacy slide in its pins (lamp-4ace on 20 October, 04:13). + +**Publication.** The worker sends results to the existing Bus pump through +a queue of four (dropped when full). Before each publish the pump checks +that the era's epoch and fence still hold a live lease now and that the +result passes `check_at(now)` (two-second raw age); otherwise the result is +discarded. The header time of a scored message is the window's end, never +the publish time. Results are published under the era's current lease, not +the lease of their last block: a lease lasts 50 ms and inference can take +longer, but any lapse or mute closes the era, so "same epoch, live now" +means capture was never interrupted since the samples were admitted. + +`unavailable` says `warming_up` until an era's first patch, `privacy` when +the era lost authority, `device_gap` on a gap or 100 ms of no blocks, +`overloaded` on overflow, age or a scored era a whole second late, +`invalid_model`, and `inference_failed`. It is said at once after scores and +otherwise every 500 ms (at most every 100 ms). With no blocks and no live +lease (a lamp muted from the start, or before the first grant) it says +`privacy` from 100 ms after it starts. + +A laughter and speech timeline is a new kind of personal signal. Nothing +reads or records `ears/classes` yet; a subscriber that records it (the +journal, records, the cloud) gets its own privacy read first (Third on +#1923). + +**Measured on the development Mac (Apple Silicon, host only, 19 October).** +Against the targets above: + +| What | Target | Measured | +|---|---|---| +| Capture handoff per 20 ms block | under 1 ms, independent of the classifier | release: p50 42 ns, p95 84 ns, max 13 µs over 1,000 blocks; with the worker stalled and the queue full, p50 42 ns, max 125 ns (debug: p50 167 ns) | +| Per patch, frontend and inference | not set; a patch every 500 ms | release: p50 2.8 ms, max 3.1 ms over 20 patches; model load 75 ms (debug: 102 ms per patch) | +| Music onset to scored result, p95 | ≤ 2 s | not measured; by construction a sound is first fully inside a patch 0.5–1 s after onset, plus inference | +| Laughter, p95 | ≤ 1.5 s | not measured | + +Silence scores equal the offline probe's (speech 8.63e-8, music 7.07e-15), +and streamed patches' features equal the offline frontend's over the whole +signal. Not measured: ARM64 inference time and CPU on a lamp, Linux ALSA +timing, onset latency on real sounds, and any accuracy or false rate. + +Host tests (`cargo test -p lamp-ears classes`) cover the switch, frontend +bounds and values, the handoff cost and a full queue, unauthorized blocks, +streamed windows against the offline features, gap, mute, epoch, fence and +age purges, the worker's scores then `privacy` with the pump's recheck, and +pins refused before parsing. The model-backed test runs only with +`LAMP_EARS_CLASSES_TEST_MODEL` set to a present `yamnet.onnx`, and is +skipped otherwise. diff --git a/lampOS/docs/switches.md b/lampOS/docs/switches.md index 3eb74fbe12..c978af54fc 100644 --- a/lampOS/docs/switches.md +++ b/lampOS/docs/switches.md @@ -1,7 +1,7 @@ # Switches Every setting that turns a behaviour on in `lamp-voice`, `lamp-body`, `lamp-eyes`, -`lamp-expression`, `lamp-name`, `lamp-skills`, `lamp-tasks`, `lamp-controls` and `lamp-cloud`: what it does, +`lamp-expression`, `lamp-name`, `lamp-skills`, `lamp-tasks`, `lamp-controls`, `lamp-ears` and `lamp-cloud`: what it does, what lamp-4ace and the product set, the pull request that added it with the lamp check it named, and the product default proposed for it. Made for the owner's question of 17 October, "why off?", and for switching lamp-4ace on group by @@ -290,6 +290,7 @@ its lamp-4ace run is Monday's, with a person there. | `lamp-skills --tools-dir` | absolute folder; unset = no skill tools. Skills' own tools offered to the voice, which reads them from `LAMP_SKILLS_DIR` (the product's `/run/lampos-skills`, lamp-skills' own folder since 19 Oct) | — | set | #984 | skills.md: "not yet run on a lamp" | on (already in product); lamp-4ace's unit lacks it | The product passes it; nothing recorded on a lamp | on | | `lamp-controls --enable` | flag; off (shadow). Gestures act, not only watched | set | set | commit 1c765da78 (direct) | Watching only on lamp-4ace from 14 Oct 11:03, then enabled; shown | on (already) | Without it the button and pads say `enabled: false` and nothing acts | on | | `lamp-controls --touch` | flag; off. The touch path: head and strip pads, and the reset button held 5 s, on controls/touch | — | set | #728 | Per-pad thresholds, faulty E4/E8, latency listed; not yet shown | on (already in product) | The product needs it for the 5 s reset hold (#1167); without it the reset did nothing | on | +| `LAMP_EARS_CLASSES` (`lamp-ears`) | `on` exactly; off unless set, and only with LAMP_PRIVACY_CONTROLS=on (otherwise one journal line and no classifier). Speech, music and laughter scores on `ears/classes` every 500 ms from the pinned YAMNet at `LAMP_EARS_CLASSES_MODEL` (`yamnet.onnx`, `yamnet.data` beside it, both pinned by SHA256); at most one patch of one capture era (under 2 s) held in memory, none written (sound-events.md) | — | — | #1669 (part a) | Host tests only (handoff p95 84 ns a block, 2.8 ms a patch on the Mac); on a lamp: an empty-room and a busy-room measurement, per-patch time and CPU on ARM64, false music/laughter per hour of office speech | off until that measurement and the owner's word (Third's privacy read: no P1 or P2) | New in-memory retention of room audio, no threshold and nothing consuming it yet | until | | `LAMP_CLOUD` (`lamp-cloud`) | `on`; off unless set. The lamp on the company broker with setup's account, V1's `info` and `data` replies, and the backend's 15 s ping, so the Autonomous app shows it online and its page reads the volume, the microphones and health, and reads, sets and previews the person's light (#1396, `cloud.md`) | — | — | #1418 (Ten's #1404, #1396) | The backend's record for the lamp goes `connected`, and the app's page reads and sets the volume, on one lamp with the backend; the Light row: get, a drag (each preview shown, the person's light back 10 s after the last), a save (kept, `light/state` agrees) (#1464) | off until its lamp check | Needs the backend's word on what marks a lamp connected, the broker's port and the `device` field | until | | `LAMP_MESSAGES` (`lamp-cloud`) | `on`; off unless set, and only with `LAMP_CLOUD`. A text from the Autonomous app (V1's `chat.send`) is acked with a run and put on `chat/in`, and what comes back on `chat/out` goes out as V1's `chat.event`; refused at once when nothing on the lamp takes texts, and a run nothing answers in 60 s ends as an error. The second route's worker answers in text on its model, with the lamp's character, notes, memory and functions; a task asked for in a text is started only by a "yes" typed in the same chat (#1673) | — | — | #1673 | Tests only: the ack and the events through the test broker, the bus through `Real`, the stream's gathering and its minute, and the worker's text turn with a scripted model through the bus | off until the app's chat shows a reply on one lamp with the second route's config | Needs the second route's config (`LAMP_VOICE_SECOND_CONFIG`), set only in lamp-4ace's drop-in today; without it every text is refused plainly, and the journal says why | until | | `LAMP_REPORTS` (`lamp-cloud`, `lamp-voice`) | `on`; off unless set, and only with `LAMP_CLOUD`. Health and crash reports to the company, only with the person's yes at setup (`share_reports`) or from the app (`reports.set`): numbers, enums and listed names only, on `/report` at QoS 1, a health every 15 min and crashes from systemd's exit variables (with the `exit-note` drop-in). In the voice it declares `reports_stop`, which can only turn them off (#1683, `cloud.md`, "Health and crash reports") | — | — | #1683 | Tests only (the canary, the gates, the stop, the bounds, the test broker). Named for lamp-06c1 under a hold, with a broker on the Mac: turn latency with and without, CPU under 0.5 % added, under 200 KB a day, no flash write without a crash, a `kill -SEGV` reported within 60 s, the stop by voice and from the app, the reset | off until its lamp check | The backend drops reports until its store is up, and the person's yes needs the app's setup screen | until |