diff --git a/templates/clips/changelog/2026-07-29-meeting-notes-no-longer-transcribe-the-other-side-twice-when.md b/templates/clips/changelog/2026-07-29-meeting-notes-no-longer-transcribe-the-other-side-twice-when.md new file mode 100644 index 0000000000..2bd7e3fe86 --- /dev/null +++ b/templates/clips/changelog/2026-07-29-meeting-notes-no-longer-transcribe-the-other-side-twice-when.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-29 +--- + +Meeting notes no longer transcribe the other side twice when you are on speakers instead of headphones diff --git a/templates/clips/desktop/src-tauri/src/echo_guard.rs b/templates/clips/desktop/src-tauri/src/echo_guard.rs new file mode 100644 index 0000000000..55a3230d40 --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/echo_guard.rs @@ -0,0 +1,443 @@ +//! Speaker-bleed detection for the meeting microphone stream. +//! +//! Without headphones the microphone re-records whatever the speakers play, so +//! the remote side reaches Whisper twice: once cleanly on the system stream and +//! once, mangled, on the mic. Downstream text de-duplication can only catch the +//! copies that happen to transcribe alike, and echo is exactly the audio +//! Whisper transcribes worst — so the leak is cut here instead. Mic audio whose +//! loudness envelope tracks the system-audio envelope at a constant delay is +//! playback bleed, not speech, and never reaches inference. +//! +//! Dropping real speech is far worse here than letting echo through, so the +//! gate is built to fail open: +//! - With headphones the reference is just as loud but uncorrelated with the +//! mic, so it stays open without any output-device detection. +//! - During double-talk the user's own voice is energy the reference cannot +//! explain, which breaks the correlation and keeps the utterance. +//! - The verdict is taken per one-second window and every window has to +//! agree, so a single sentence of the user's cannot be outvoted by the +//! minute of remote speech it interrupted. +//! +//! Envelopes, not waveforms: the speaker→mic path adds room reverb, clipping, +//! and device resampling that destroy sample-level correlation but leave the +//! loudness contour intact. + +use std::collections::VecDeque; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// Envelope resolution. Short enough to follow syllables, long enough that a +/// capture buffer lands in one or two frames. +const FRAME_MS: u64 = 20; +/// Longest speaker→microphone round trip we search for. Covers output device +/// buffering plus room propagation. +const MAX_ECHO_DELAY_MS: u64 = 400; +/// Playback older than this can never explain a mic utterance we are about to +/// finalize, so the reference ring never needs to grow past it. +const REFERENCE_RETENTION: Duration = Duration::from_secs(30); +/// An utterance is judged one window at a time rather than as a whole, and is +/// only suppressed when every window is echo. Whisper keeps buffering until it +/// hears a pause, so an utterance can run for tens of seconds — long enough +/// that a whole-buffer verdict would let a wall of remote speech outvote the +/// second in which the user cut in. One second is the shortest window whose +/// envelope still carries enough syllables to correlate. +const WINDOW_FRAMES: usize = 1000 / FRAME_MS as usize; +/// A window with less speech than this has nothing to explain, so it neither +/// confirms nor denies echo. +const MIN_VOICED_FRAMES: usize = 100 / FRAME_MS as usize; +/// Mic frames quieter than this are not speech and do not need explaining. +/// Matches the whisper worker's own voice-activity threshold. +const VOICED_RMS: f32 = 0.006; +/// Reference frames quieter than this count as "nothing was playing". +const PLAYBACK_RMS: f32 = 0.002; +/// Share of voiced mic frames that must coincide with playback. A single +/// stretch of the user talking into silence drops the utterance below this. +const MIN_COVERAGE: f32 = 0.9; +/// Pearson correlation of the two dB envelopes at the best delay. +const MIN_CORRELATION: f32 = 0.7; +/// Both envelopes must actually vary, otherwise correlation is measuring noise +/// between two near-constant lines. Steady background playback fails this and +/// the utterance is kept. +const MIN_DB_DEVIATION: f32 = 3.0; + +const MAX_LAG_FRAMES: usize = MAX_ECHO_DELAY_MS as usize / FRAME_MS as usize; + +/// Loudness of one capture buffer, kept with the wall-clock window it covers so +/// mic and system streams can be aligned without a shared sample clock. +#[derive(Clone, Copy)] +struct ReferenceSpan { + start: Instant, + end: Instant, + rms: f32, +} + +/// Rolling record of what the speakers have been playing. +pub(crate) struct EchoGuard { + spans: Mutex>, +} + +impl EchoGuard { + pub(crate) fn new() -> Self { + Self { + spans: Mutex::new(VecDeque::new()), + } + } + + /// Record one system-audio capture buffer. Called from the realtime audio + /// callback: one pass over the samples, one push, no allocation beyond the + /// ring's amortized growth. + pub(crate) fn note_playback(&self, samples: &[f32], src_rate: f64) { + if samples.is_empty() || src_rate <= 0.0 { + return; + } + let end = Instant::now(); + let duration = Duration::from_secs_f64(samples.len() as f64 / src_rate); + let rms = (samples.iter().map(|s| s * s).sum::() / samples.len() as f32).sqrt(); + let mut spans = self + .spans + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + spans.push_back(ReferenceSpan { + start: end.checked_sub(duration).unwrap_or(end), + end, + rms, + }); + while spans + .front() + .is_some_and(|span| end.saturating_duration_since(span.end) > REFERENCE_RETENTION) + { + spans.pop_front(); + } + } + + /// Whether `samples` (16 kHz mono, captured starting at `buffer_start`) is + /// the speakers bleeding back into the microphone rather than speech. + pub(crate) fn is_playback_echo(&self, samples: &[f32], buffer_start: Instant) -> bool { + let mic = envelope_16k(samples); + if mic.len() < WINDOW_FRAMES { + return false; + } + // The reference has to start MAX_LAG_FRAMES early so every candidate + // delay has real playback to line up against. + let lag = Duration::from_millis(MAX_LAG_FRAMES as u64 * FRAME_MS); + let reference_start = buffer_start.checked_sub(lag).unwrap_or(buffer_start); + let reference = self.reference_envelope(reference_start, mic.len() + MAX_LAG_FRAMES); + is_echo(&mic, &reference) + } + + /// Sample the playback envelope onto the same `FRAME_MS` grid the mic uses, + /// taking the loudest overlapping span for each frame. + fn reference_envelope(&self, from: Instant, frames: usize) -> Vec { + let mut envelope = vec![0.0f32; frames]; + let spans: Vec = { + let spans = self + .spans + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + spans.iter().copied().collect() + }; + let frame = Duration::from_millis(FRAME_MS); + let until = from + frame * frames as u32; + for span in spans { + if span.end <= from || span.start >= until { + continue; + } + let first = span.start.saturating_duration_since(from).as_nanos() / frame.as_nanos(); + let end = span.end.saturating_duration_since(from).as_nanos(); + let end_exclusive = end.div_ceil(frame.as_nanos()); + for slot in envelope + .iter_mut() + .take((end_exclusive as usize).min(frames)) + .skip((first as usize).min(frames)) + { + *slot = slot.max(span.rms); + } + } + envelope + } +} + +/// Per-frame RMS of 16 kHz mono samples. A trailing partial frame is dropped — +/// its loudness is not comparable to a full one. +fn envelope_16k(samples: &[f32]) -> Vec { + let frame = (16_000 * FRAME_MS as usize) / 1000; + samples + .chunks_exact(frame) + .map(|chunk| (chunk.iter().map(|s| s * s).sum::() / chunk.len() as f32).sqrt()) + .collect() +} + +/// Decide whether `mic` is playback bleed. `reference` is the playback envelope +/// on the same grid but starting `MAX_LAG_FRAMES` earlier, so a window of `mic` +/// at offset `o` is judged against `reference[o..o + WINDOW_FRAMES + lag]`. +/// +/// Every speech-carrying window has to look like echo. One window that does not +/// keeps the whole utterance, because that window is the user talking. +/// +/// Split out from `EchoGuard` so the decision is testable without audio devices +/// or wall-clock timing. +fn is_echo(mic: &[f32], reference: &[f32]) -> bool { + if mic.len() < WINDOW_FRAMES || reference.len() < mic.len() + MAX_LAG_FRAMES { + return false; + } + let mut judged = 0u32; + for offset in window_offsets(mic.len()) { + let window = &mic[offset..offset + WINDOW_FRAMES]; + if window.iter().filter(|&&level| level > VOICED_RMS).count() < MIN_VOICED_FRAMES { + continue; + } + judged += 1; + let reference = &reference[offset..offset + WINDOW_FRAMES + MAX_LAG_FRAMES]; + if !meets_echo_thresholds( + playback_coverage(window, reference), + best_delay_correlation(window, reference), + ) { + return false; + } + } + judged > 0 +} + +/// Window start frames covering all of `frames`. The last window is anchored to +/// the end rather than dropped, so speech in a trailing part-window — a "hang +/// on, actually" right before the pause that ended the utterance — is still +/// judged on its own instead of inheriting the verdict of the echo before it. +fn window_offsets(frames: usize) -> Vec { + let Some(last) = frames.checked_sub(WINDOW_FRAMES) else { + return Vec::new(); + }; + let mut offsets: Vec = (0..=last).step_by(WINDOW_FRAMES).collect(); + if offsets.last() != Some(&last) { + offsets.push(last); + } + offsets +} + +fn meets_echo_thresholds(coverage: f32, correlation: f32) -> bool { + coverage >= MIN_COVERAGE && correlation >= MIN_CORRELATION +} + +/// Share of the mic's voiced frames that had playback somewhere inside the echo +/// delay window. Voice arriving while the speakers were silent cannot be echo. +fn playback_coverage(mic: &[f32], reference: &[f32]) -> f32 { + let mut voiced = 0u32; + let mut covered = 0u32; + for (i, &level) in mic.iter().enumerate() { + if level <= VOICED_RMS { + continue; + } + voiced += 1; + if reference[i..=i + MAX_LAG_FRAMES] + .iter() + .any(|&r| r > PLAYBACK_RMS) + { + covered += 1; + } + } + if voiced == 0 { + return 0.0; + } + covered as f32 / voiced as f32 +} + +/// Best Pearson correlation between the mic and reference dB envelopes across +/// every candidate echo delay. +fn best_delay_correlation(mic: &[f32], reference: &[f32]) -> f32 { + let mic_db: Vec = mic.iter().copied().map(decibels).collect(); + let reference_db: Vec = reference.iter().copied().map(decibels).collect(); + (0..=MAX_LAG_FRAMES) + .map(|delay| { + let offset = MAX_LAG_FRAMES - delay; + correlation(&mic_db, &reference_db[offset..offset + mic_db.len()]) + }) + .fold(0.0f32, f32::max) +} + +fn decibels(rms: f32) -> f32 { + 20.0 * rms.max(1e-6).log10() +} + +/// Pearson correlation, or 0 when either series is too flat to correlate +/// meaningfully. +fn correlation(left: &[f32], right: &[f32]) -> f32 { + let n = left.len() as f32; + let left_mean = left.iter().sum::() / n; + let right_mean = right.iter().sum::() / n; + let mut covariance = 0.0f32; + let mut left_variance = 0.0f32; + let mut right_variance = 0.0f32; + for (&l, &r) in left.iter().zip(right) { + let l = l - left_mean; + let r = r - right_mean; + covariance += l * r; + left_variance += l * l; + right_variance += r * r; + } + let left_deviation = (left_variance / n).sqrt(); + let right_deviation = (right_variance / n).sqrt(); + if left_deviation < MIN_DB_DEVIATION || right_deviation < MIN_DB_DEVIATION { + return 0.0; + } + covariance / (n * left_deviation * right_deviation) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FRAMES: usize = 150; + const DELAY: usize = 5; + + /// Speech-like envelope: alternating loud and quiet stretches. + fn speech(frames: usize, seed: usize) -> Vec { + (0..frames) + .map(|i| { + let phase = (i + seed * 7) % 40; + if phase < 24 { + 0.02 + 0.02 * ((i * (seed + 3)) % 5) as f32 / 5.0 + } else { + 0.0005 + } + }) + .collect() + } + + /// Place `mic` inside a reference track delayed by `DELAY` frames and + /// attenuated the way a speaker→mic path attenuates. + fn reference_echoing(mic: &[f32], gain: f32) -> Vec { + let mut reference = vec![0.0f32; mic.len() + MAX_LAG_FRAMES]; + for (i, &level) in mic.iter().enumerate() { + reference[i + MAX_LAG_FRAMES - DELAY] = level / gain; + } + reference + } + + #[test] + fn speaker_bleed_is_detected_as_echo() { + let mic = speech(FRAMES, 1); + let reference = reference_echoing(&mic, 8.0); + assert!(is_echo(&mic, &reference)); + } + + #[test] + fn headphones_keep_the_utterance_even_though_playback_is_loud() { + // Reference is continuously loud (remote side talking into the user's + // headphones) but its contour is unrelated to the mic. + let mic = speech(FRAMES, 1); + let mut reference = vec![0.0f32; mic.len() + MAX_LAG_FRAMES]; + for (i, level) in speech(reference.len(), 9).into_iter().enumerate() { + reference[i] = level; + } + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn double_talk_keeps_the_utterance() { + // Echo plus the user speaking through the reference's quiet stretches. + let remote = speech(FRAMES, 1); + let reference = reference_echoing(&remote, 8.0); + let mic: Vec = remote + .iter() + .enumerate() + .map(|(i, &level)| if i % 40 >= 24 { 0.05 } else { level }) + .collect(); + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn silent_playback_keeps_the_utterance() { + let mic = speech(FRAMES, 1); + let reference = vec![0.0f32; mic.len() + MAX_LAG_FRAMES]; + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn steady_playback_is_never_mistaken_for_echo() { + // Constant tone under a constant mic level: coverage is total, but + // neither envelope varies so there is nothing to correlate. + let mic = vec![0.03f32; FRAMES]; + let reference = vec![0.004f32; FRAMES + MAX_LAG_FRAMES]; + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn short_utterances_are_always_kept() { + let mic = speech(WINDOW_FRAMES - 1, 1); + let reference = reference_echoing(&mic, 8.0); + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn a_late_interruption_saves_the_whole_utterance() { + // Eight seconds of the remote side echoing off the speakers, then the + // user cuts in for the last second. A whole-buffer verdict would let + // the echo outvote the interruption and discard both. + let remote = speech(400, 1); + let reference = reference_echoing(&remote, 8.0); + let mut mic = remote; + for frame in mic.iter_mut().skip(350) { + *frame = 0.05; + } + assert!(!is_echo(&mic, &reference)); + } + + #[test] + fn window_offsets_always_reach_the_end_of_the_utterance() { + let offsets = window_offsets(WINDOW_FRAMES * 2 + 7); + assert_eq!(offsets.first(), Some(&0)); + assert_eq!(offsets.last(), Some(&(WINDOW_FRAMES + 7))); + } + + #[test] + fn window_offsets_are_empty_for_short_input() { + assert!(window_offsets(WINDOW_FRAMES - 1).is_empty()); + } + + #[test] + fn poisoned_reference_lock_does_not_disable_the_guard() { + let guard = std::sync::Arc::new(EchoGuard::new()); + let poisoning_guard = guard.clone(); + assert!(std::thread::spawn(move || { + let _spans = poisoning_guard.spans.lock().unwrap(); + panic!("poison reference lock"); + }) + .join() + .is_err()); + + guard.note_playback(&[0.5; 480], 48_000.0); + assert_eq!( + guard + .spans + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .len(), + 1 + ); + } + + #[test] + fn reference_envelope_aligns_playback_onto_the_mic_frame_grid() { + let guard = EchoGuard::new(); + let from = Instant::now(); + guard.spans.lock().unwrap().push_back(ReferenceSpan { + start: from + Duration::from_millis(5), + end: from + Duration::from_millis(25), + rms: 0.5, + }); + + assert_eq!(guard.reference_envelope(from, 3), vec![0.5, 0.5, 0.0]); + } + + #[test] + fn echo_thresholds_reject_values_just_below_the_boundaries() { + assert!(!meets_echo_thresholds( + f32::from_bits(MIN_COVERAGE.to_bits() - 1), + MIN_CORRELATION, + )); + assert!(!meets_echo_thresholds( + MIN_COVERAGE, + f32::from_bits(MIN_CORRELATION.to_bits() - 1), + )); + assert!(meets_echo_thresholds(MIN_COVERAGE, MIN_CORRELATION)); + } +} diff --git a/templates/clips/desktop/src-tauri/src/lib.rs b/templates/clips/desktop/src-tauri/src/lib.rs index c51abf29d3..050285edca 100644 --- a/templates/clips/desktop/src-tauri/src/lib.rs +++ b/templates/clips/desktop/src-tauri/src/lib.rs @@ -11,6 +11,7 @@ mod capture_graph; mod clips; mod config; mod debug; +mod echo_guard; mod eventkit; mod logfile; mod meetings_watcher; diff --git a/templates/clips/desktop/src-tauri/src/whisper_speech.rs b/templates/clips/desktop/src-tauri/src/whisper_speech.rs index 525dddc181..3ccd1b8eca 100644 --- a/templates/clips/desktop/src-tauri/src/whisper_speech.rs +++ b/templates/clips/desktop/src-tauri/src/whisper_speech.rs @@ -146,6 +146,7 @@ mod macos { use crate::capture_audio_bus::{ try_subscribe, AudioSources, AudioSubscription, SubscriptionAttempt, }; + use crate::echo_guard::EchoGuard; use crate::native_speech::macos::{ start_raw_mic_capture, MicVoiceProcessingMode, RawMicCapture, }; @@ -441,6 +442,11 @@ mod macos { /// Recording capture only persists finals and disables this expensive /// repeated inference; meeting capture keeps it enabled. emit_partials: bool, + /// Shared speaker-bleed reference, present only while both streams are + /// capturing. The system stream records what the speakers played into + /// it; the mic stream reads it back to drop its own echo of that + /// playback before inference. See `echo_guard`. + echo_guard: Option>, } impl WhisperStream { @@ -452,6 +458,7 @@ mod macos { ctx: Arc, stream_start: Instant, emit_partials: bool, + echo_guard: Option>, ) -> Arc { let done = Arc::new(AtomicBool::new(false)); let stream = Arc::new(WhisperStream { @@ -468,6 +475,7 @@ mod macos { }), reset_generation: AtomicU32::new(0), emit_partials, + echo_guard, }); let worker_stream = stream.clone(); std::thread::spawn(move || { @@ -492,6 +500,11 @@ mod macos { if !self.running.load(Ordering::SeqCst) { return; } + if self.source == "system" { + if let Some(guard) = &self.echo_guard { + guard.note_playback(frames, self.src_rate.load(Ordering::SeqCst) as f64); + } + } if let Ok(mut buf) = self.buf.lock() { buf.extend_from_slice(frames); } @@ -514,15 +527,48 @@ mod macos { .unwrap_or(0) } - /// Mark the start of a fresh buffer (called when the buffer is cleared + /// Mark the start of a fresh buffer (called when the buffer is drained /// on finalize) so the next utterance's whisper timestamps offset - /// correctly onto the meeting timeline. - fn reset_buffer_start(&self) { + /// correctly onto the meeting timeline. `pending` is how much audio + /// survived the drain, which is how far before now the buffer begins. + fn reset_buffer_start(&self, pending: Duration) { + let now = Instant::now(); if let Ok(mut timeline) = self.timeline.lock() { - timeline.buffer_start = Instant::now(); + timeline.buffer_start = buffer_start_after_drain(now, pending); } } + /// Wall-clock start of the audio currently sitting in `buf`, used to + /// line that audio up against the playback reference. + fn buffer_start(&self) -> Instant { + self.timeline + .lock() + .map(|timeline| timeline.buffer_start) + .unwrap_or_else(|_| Instant::now()) + } + + /// Whether `samples` (16 kHz mono) is the speakers bleeding into the + /// microphone rather than someone talking. Only the mic can be + /// contaminated: a call app never plays the local user back. + fn is_playback_echo(&self, samples: &[f32]) -> bool { + if self.source != "mic" { + return false; + } + self.echo_guard + .as_ref() + .is_some_and(|guard| guard.is_playback_echo(samples, self.buffer_start())) + } + + /// Clear whichever live partial this stream last rendered. Suppressed + /// echo emits no final, so without this the overlay would keep showing + /// the partial that led up to it. + fn clear_partial(&self) { + let _ = self.app.emit( + "voice:partial-transcript", + serde_json::json!({ "text": "", "source": self.source }), + ); + } + /// Rebase timestamps to "now" and discard any audio captured while the /// recorder was warming up/counting down. fn reset_timeline(&self) { @@ -682,6 +728,10 @@ mod macos { sum / count as f32 } + fn buffer_start_after_drain(now: Instant, pending: Duration) -> Instant { + now.checked_sub(pending).unwrap_or(now) + } + fn partial_inference_due( emit_partials: bool, had_voice: bool, @@ -694,6 +744,18 @@ mod macos { && since_last_infer > Duration::from_millis(1200) } + fn partial_inference_timestamp( + previous: Instant, + inference_ran: bool, + now: Instant, + ) -> Instant { + if inference_ran { + now + } else { + previous + } + } + fn utterance_finalize_due(have_secs: f32, silence: Duration) -> bool { (have_secs > 0.4 && silence > Duration::from_millis(800)) || have_secs > 25.0 } @@ -790,20 +852,32 @@ mod macos { if stream.reset_generation.load(Ordering::SeqCst) != seen_reset_generation { continue; } - let segs = infer(&mut state, resample_state.samples(), lang); - stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms()); + if stream.is_playback_echo(resample_state.samples()) { + // A dropped utterance is indistinguishable from silence + // in the transcript, so say so here: this log is the + // only way a false positive is diagnosable afterwards. + eprintln!("[whisper-mic] suppressed {have_secs:.1}s of speaker bleed"); + stream.clear_partial(); + } else { + let segs = infer(&mut state, resample_state.samples(), lang); + stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms()); + } } + let mut pending = 0usize; if let Ok(mut b) = stream.buf.lock() { let to_drain = n_processed.min(b.len()); b.drain(..to_drain); + pending = b.len(); } // Raw indices shift after the drain above (front-truncated), // so the resample cache is invalid regardless of whether this // utterance ran inference — rebuild fresh from whatever's left. resample_state.drop_all(); - // New buffer begins now — advance the timeline offset so the - // next utterance's whisper timestamps map correctly. - stream.reset_buffer_start(); + // Advance the timeline offset so the next utterance's whisper + // timestamps map correctly. Inference can take seconds, and + // audio kept arriving throughout it, so the new buffer starts + // as far back as the audio it already holds — not at "now". + stream.reset_buffer_start(Duration::from_secs_f32(pending as f32 / src_rate)); last_raw_len = 0; had_voice = false; last_infer = Instant::now(); @@ -825,9 +899,15 @@ mod macos { if stream.reset_generation.load(Ordering::SeqCst) != seen_reset_generation { continue; } - let segs = infer(&mut state, resample_state.samples(), lang); - stream.emit_transcript("voice:partial-transcript", &segs, stream.offset_ms()); - last_infer = Instant::now(); + let inference_ran = if stream.is_playback_echo(resample_state.samples()) { + stream.clear_partial(); + false + } else { + let segs = infer(&mut state, resample_state.samples(), lang); + stream.emit_transcript("voice:partial-transcript", &segs, stream.offset_ms()); + true + }; + last_infer = partial_inference_timestamp(last_infer, inference_ran, Instant::now()); } } @@ -835,7 +915,10 @@ mod macos { let raw = stream.buf.lock().map(|b| b.clone()).unwrap_or_default(); let src_rate = stream.src_rate.load(Ordering::SeqCst) as f64; let samples = resample_to_16k(&raw, src_rate); - if had_voice && samples.len() as f32 / SAMPLE_RATE_16K > 0.3 { + if had_voice + && samples.len() as f32 / SAMPLE_RATE_16K > 0.3 + && !stream.is_playback_echo(&samples) + { let segs = infer(&mut state, &samples, lang); stream.emit_transcript("voice:final-transcript", &segs, stream.offset_ms()); } @@ -1028,6 +1111,9 @@ mod macos { // competing VoiceProcessingIO mic input. Older macOS versions (and a // failed SCK start) keep the existing split-capture fallback. let session_start = Instant::now(); + // Only a session that captures both streams can tell speaker bleed + // from speech, so a mic-only session leaves the guard unarmed. + let echo_guard = capture_system.then(|| Arc::new(EchoGuard::new())); let mic_stream = WhisperStream::new( app.clone(), "mic", @@ -1036,6 +1122,7 @@ mod macos { ctx.clone(), session_start, emit_partials, + echo_guard.clone(), ); let sys_stream = capture_system.then(|| { WhisperStream::new( @@ -1046,6 +1133,7 @@ mod macos { ctx.clone(), session_start, emit_partials, + echo_guard.clone(), ) }); let mic_for_cb = mic_stream.clone(); @@ -1311,11 +1399,12 @@ mod macos { #[cfg(test)] mod tests { - use std::time::Duration; + use std::time::{Duration, Instant}; use super::{ - partial_inference_due, resample_to_16k, should_use_combined_sck_capture, - split_mic_capture_options, utterance_finalize_due, IncrementalResample, SessionOwner, + buffer_start_after_drain, partial_inference_due, partial_inference_timestamp, + resample_to_16k, should_use_combined_sck_capture, split_mic_capture_options, + utterance_finalize_due, IncrementalResample, SessionOwner, }; use crate::native_speech::macos::MicVoiceProcessingMode; @@ -1364,6 +1453,17 @@ mod macos { ); } + #[test] + fn buffer_start_accounts_for_audio_captured_during_inference() { + let now = Instant::now(); + let pending = Duration::from_millis(750); + + assert_eq!( + buffer_start_after_drain(now, pending), + now.checked_sub(pending).unwrap() + ); + } + #[test] fn recording_mode_never_runs_live_partial_inference() { assert!(!partial_inference_due( @@ -1390,6 +1490,15 @@ mod macos { )); } + #[test] + fn echo_suppressed_partial_does_not_reset_retry_cadence() { + let previous = Instant::now(); + let now = previous + Duration::from_secs(2); + + assert_eq!(partial_inference_timestamp(previous, false, now), previous); + assert_eq!(partial_inference_timestamp(previous, true, now), now); + } + #[test] fn recording_mode_keeps_silence_and_long_utterance_finalization() { assert!(utterance_finalize_due(1.0, Duration::from_millis(801))); diff --git a/templates/clips/desktop/src/hooks/useMeetingTranscription.ts b/templates/clips/desktop/src/hooks/useMeetingTranscription.ts index 872c19406c..9cc32e81a1 100644 --- a/templates/clips/desktop/src/hooks/useMeetingTranscription.ts +++ b/templates/clips/desktop/src/hooks/useMeetingTranscription.ts @@ -8,11 +8,14 @@ import { appendFinalTranscript, onFinalTranscript, restartTranscriptionEngine, - speakerFor, startTranscriptionEngine, stopTranscriptionEngine, + transcriptFullText, + transcriptLineFromSegment, + transcriptSegments, type SourcedTranscriptSegment, type TranscriptionEngine, + type TranscriptLine, } from "../lib/transcription-engine"; import { normalizeServerUrl } from "../lib/url"; @@ -31,8 +34,7 @@ export interface MeetingTranscriptionPayload { interface MeetingTranscriptionSession { meetingId: string; recordingId: string; - lines: string[]; - segments: SourcedTranscriptSegment[]; + lines: TranscriptLine[]; unlisten: Array<() => void>; flushTimer: ReturnType | null; stopping: boolean; @@ -52,6 +54,22 @@ interface MeetingTranscriptionSession { dirtySeq: number; } +/** What the pill overlay needs to render a line: text, side, and timestamp. + * The verbatim segments stay behind in the session. */ +interface PillTranscriptLine { + text: string; + source: "mic" | "system"; + startMs?: number; +} + +function pillTranscriptLines(lines: TranscriptLine[]): PillTranscriptLine[] { + return lines.map((line) => ({ + text: line.text, + source: line.source, + startMs: line.startMs ?? undefined, + })); +} + type CallClipsAction = ( name: string, body: Record, @@ -79,11 +97,7 @@ export function useMeetingTranscription({ const pendingPillInitRef = useRef<{ meetingId: string; initialNotes: string; - preloadedLines?: Array<{ - text: string; - source: "mic" | "system"; - startMs?: number; - }>; + preloadedLines?: PillTranscriptLine[]; } | null>(null); const normalizedServerUrl = useMemo( @@ -119,8 +133,8 @@ export function useMeetingTranscription({ const run = (async () => { await callClipsAction("save-browser-transcript", { recordingId: session.recordingId, - fullText: session.lines.join("\n\n"), - segments: session.segments, + fullText: transcriptFullText(session.lines), + segments: transcriptSegments(session.lines), source: session.engine, overwriteReady: true, }); @@ -314,7 +328,6 @@ export function useMeetingTranscription({ meetingId: resolvedMeetingId, recordingId, lines: [], - segments: [], unlisten: [], flushTimer: null, stopping: false, @@ -363,13 +376,7 @@ export function useMeetingTranscription({ })), } : event; - if ( - appendFinalTranscript( - timelineEvent, - session.lines, - session.segments, - ) - ) { + if (appendFinalTranscript(timelineEvent, session.lines)) { scheduleFlush(); } }), @@ -539,11 +546,7 @@ export function useMeetingTranscription({ pendingPillInitRef.current = { meetingId: resolvedMeetingId, initialNotes, - preloadedLines: session.segments.map((segment) => ({ - text: segment.text, - source: segment.source, - startMs: segment.startMs, - })), + preloadedLines: pillTranscriptLines(session.lines), }; emit("clips:meeting-notes-init", { meetingId: resolvedMeetingId, @@ -561,25 +564,16 @@ export function useMeetingTranscription({ source?: "mic" | "system"; }>; if (segs.length > 0) { - const preloadedLineStrings = segs.map( - (s) => `${speakerFor(s.source)}: ${s.text}`, + const storedLines = segs.map((s) => + transcriptLineFromSegment({ + startMs: s.startMs ?? 0, + endMs: s.endMs ?? 0, + text: s.text, + source: s.source ?? "mic", + }), ); - const preloadedSegments = segs.map((s) => ({ - startMs: s.startMs ?? 0, - endMs: s.endMs ?? 0, - text: s.text, - source: s.source ?? ("mic" as const), - })); - session.lines = [...preloadedLineStrings, ...session.lines]; - session.segments = [ - ...preloadedSegments, - ...session.segments, - ]; - const preloadedLines = session.segments.map((s) => ({ - text: s.text, - source: s.source, - startMs: s.startMs, - })); + session.lines = [...storedLines, ...session.lines]; + const preloadedLines = pillTranscriptLines(session.lines); // Store in ref so clips:pill-ready can re-emit if the // pill window mounts after this fetch resolves. if ( @@ -648,15 +642,10 @@ export function useMeetingTranscription({ .then((history) => { if (sessionRef.current !== session) return; const historyLines = history.segments.map( - (segment) => `${speakerFor(segment.source)}: ${segment.text}`, + transcriptLineFromSegment, ); session.lines = [...historyLines, ...session.lines]; - session.segments = [...history.segments, ...session.segments]; - const preloadedLines = session.segments.map((segment) => ({ - text: segment.text, - source: segment.source, - startMs: segment.startMs, - })); + const preloadedLines = pillTranscriptLines(session.lines); if (pendingPillInitRef.current?.meetingId === resolvedMeetingId) { pendingPillInitRef.current = { ...pendingPillInitRef.current, diff --git a/templates/clips/desktop/src/lib/transcription-capture.ts b/templates/clips/desktop/src/lib/transcription-capture.ts index 1eb2d0bb93..8818667e95 100644 --- a/templates/clips/desktop/src/lib/transcription-capture.ts +++ b/templates/clips/desktop/src/lib/transcription-capture.ts @@ -19,7 +19,10 @@ import { startTranscriptionEngine, stopTranscriptionEngine, TranscriptionEngine, + transcriptFullText, + transcriptSegments, type SourcedTranscriptSegment, + type TranscriptLine, } from "./transcription-engine"; /** Grace period after stop for whisper to emit any flushed trailing finals. */ @@ -307,8 +310,7 @@ export async function startTranscriptionCapture( voiceProcessing?: boolean; }, ): Promise { - const lines: string[] = []; - const segments: SourcedTranscriptSegment[] = []; + const lines: TranscriptLine[] = []; let disposed = false; let paused = false; let desiredPaused = false; @@ -331,8 +333,8 @@ export async function startTranscriptionCapture( }; const captured = (): CapturedTranscript => ({ - text: lines.join("\n\n").trim(), - segments, + text: transcriptFullText(lines), + segments: transcriptSegments(lines), }); let engine: TranscriptionEngine; @@ -340,7 +342,7 @@ export async function startTranscriptionCapture( unlistens.push( await onFinalTranscript((event) => { if (disposed) return; - appendFinalTranscript(event, lines, segments); + appendFinalTranscript(event, lines); }), ); diff --git a/templates/clips/desktop/src/lib/transcription-engine.test.ts b/templates/clips/desktop/src/lib/transcription-engine.test.ts index 97b41a0d21..a18c38bfaa 100644 --- a/templates/clips/desktop/src/lib/transcription-engine.test.ts +++ b/templates/clips/desktop/src/lib/transcription-engine.test.ts @@ -8,11 +8,24 @@ vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn() })); import { appendFinalTranscript, recordingTranscriptionLanguage, + isMicEcho, restartTranscriptionEngine, - type SourcedTranscriptSegment, startTranscriptionEngine, + transcriptFullText, + transcriptSegments, + type TranscriptLine, } from "./transcription-engine"; +/** A final-transcript event carrying one segment of `text`. */ +function said( + source: "mic" | "system", + text: string, + startMs: number, + endMs = startMs + 2_000, +) { + return { text, source, segments: [{ startMs, endMs, text }] } as const; +} + beforeEach(() => { invokeMock.mockReset(); invokeMock.mockResolvedValue(undefined); @@ -22,88 +35,286 @@ describe("recording transcription language", () => { it("leaves local Whisper recordings on auto-detect instead of forcing the UI locale", () => { expect(recordingTranscriptionLanguage()).toBeNull(); }); +}); - it("drops overlapping duplicate speech from the other audio source", () => { - const lines: string[] = []; - const segments: SourcedTranscriptSegment[] = []; +describe("transcript echo suppression", () => { + it("drops mic speech that only echoes system audio already captured", () => { + const lines: TranscriptLine[] = []; + appendFinalTranscript( + said("system", "Send the pull request button", 1_000), + lines, + ); expect( appendFinalTranscript( - { - text: "Send the pull request button", - source: "mic", - segments: [ - { - startMs: 1_000, - endMs: 2_000, - text: "Send the pull request button", - }, - ], - }, + said("mic", "Send the pull request button", 1_100), lines, - segments, ), - ).toBe(true); + ).toBe(false); + expect(transcriptFullText(lines)).toBe( + "Them: Send the pull request button", + ); + expect(transcriptSegments(lines)).toHaveLength(1); + }); + + it("retracts a mic echo once the system copy of it arrives", () => { + const lines: TranscriptLine[] = []; + + // The mic finalizes first, so without retraction the remote speaker's + // words would stay attributed to the user. + appendFinalTranscript( + said("mic", "Send the pull request button", 1_100), + lines, + ); + appendFinalTranscript( + said("system", "Send the pull request button", 1_000), + lines, + ); + + expect(transcriptFullText(lines)).toBe( + "Them: Send the pull request button", + ); + expect(transcriptSegments(lines)).toHaveLength(1); + }); + + it("treats mangled echo as echo", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "So I think we should ship the redesign on Friday", 4_000), + lines, + ); + // Whisper transcribes speaker bleed badly: words drop out and change. expect( appendFinalTranscript( - { - text: "Send the pull request button", - source: "system", - segments: [ - { - startMs: 1_100, - endMs: 2_100, - text: "Send the pull request button", - }, - ], - }, + said("mic", "So I think we should ship a redesign Friday", 4_300), lines, - segments, ), ).toBe(false); + }); + + it("keeps the user talking over the remote side", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "So I think we should ship the redesign on Friday", 4_000), + lines, + ); + expect( + appendFinalTranscript( + said("mic", "Wait, can we talk about QA first", 4_500), + lines, + ), + ).toBe(true); + expect(lines).toHaveLength(2); + }); + + it("keeps matching speech once the conversation has moved on", () => { + const lines: TranscriptLine[] = []; - expect(lines).toEqual(["Me: Send the pull request button"]); - expect(segments).toHaveLength(1); + appendFinalTranscript( + said("system", "Please review the changes", 1_000), + lines, + ); + for (let index = 0; index < 6; index++) { + appendFinalTranscript( + said("system", `Unrelated remark number ${index}`, 5_000 + index), + lines, + ); + } + expect( + appendFinalTranscript( + said("mic", "Please review the changes", 30_000), + lines, + ), + ).toBe(true); + + expect(lines).toHaveLength(8); + expect(transcriptSegments(lines)).toHaveLength(8); }); - it("keeps matching speech when it happens at a different time", () => { - const lines: string[] = []; - const segments: SourcedTranscriptSegment[] = []; - const event = { - text: "Please review the changes", - segments: [ - { - startMs: 1_000, - endMs: 2_000, - text: "Please review the changes", - }, - ], - }; + it("keeps a deliberate repeat after the loose echo time window", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "So seventy five centimetres, got it", 1_000), + lines, + ); expect( - appendFinalTranscript({ ...event, source: "mic" }, lines, segments), + appendFinalTranscript( + said("mic", "So seventy five centimetres, got it", 30_000), + lines, + ), ).toBe(true); + expect(lines).toHaveLength(2); + }); + + it("does not use or retract preloaded history as live echo evidence", () => { + const lines: TranscriptLine[] = [ + { + source: "system", + text: "Please review the changes", + startMs: 1_000, + segments: [], + historical: true, + }, + { + source: "mic", + text: "Send the pull request button", + startMs: 2_000, + segments: [], + historical: true, + }, + ]; + expect( appendFinalTranscript( - { - ...event, - source: "system", - segments: [ - { - startMs: 3_000, - endMs: 4_000, - text: "Please review the changes", - }, - ], - }, + said("mic", "Please review the changes", 1_200), lines, - segments, ), ).toBe(true); + appendFinalTranscript( + said("system", "Send the pull request button", 2_200), + lines, + ); + + expect(lines.filter((line) => line.historical)).toHaveLength(2); + expect(lines).toHaveLength(4); + }); + + it("keeps short agreements that merely repeat a common word", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "Does that work for everyone", 2_000), + lines, + ); + expect( + appendFinalTranscript(said("mic", "Yeah that works", 2_400), lines), + ).toBe(true); + }); + // Echo repeats a whole utterance. A brief interjection whose words all + // happen to appear, in order, somewhere in a long remote passage is the user + // talking, and silently deleting that is far worse than keeping echo. + it.each([ + ["Sorry, go ahead", "Right, go ahead and start whenever you are ready"], + ["Yeah, I think so", "I don't think so, we should just ship it"], + [ + "I think we should do that", + "So I was thinking we should not do the second one, that is my take", + ], + ])("keeps %j spoken over the remote side", (mine, theirs) => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript(said("system", theirs, 5_000, 12_000), lines); + expect(appendFinalTranscript(said("mic", mine, 6_000, 8_000), lines)).toBe( + true, + ); expect(lines).toHaveLength(2); - expect(segments).toHaveLength(2); + }); + + // Captured off a real speaker-mode call. Whisper hears the bleed well enough + // to keep the sentence structure but mangles the nouns and the digits, which + // is why both exact and set-based matching let it through. + it("matches a real mangled echo of a long utterance", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said( + "system", + "I'm going to test it out on my garden hedge. This particular model is the 751 and the 75 just means it's got a 75 centimetre cut in blade.", + 0, + 10_000, + ), + lines, + ); + expect( + appendFinalTranscript( + said( + "mic", + "I'm going to test it out on my garden page. This particular model is the 751 and the 752 has a 75% to me to cut in blade.", + 200, + 10_200, + ), + lines, + ), + ).toBe(false); + }); + + it("matches echo that straddles two system lines", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "Let us start with the", 1_000), + lines, + ); + appendFinalTranscript( + said("system", "roadmap for next quarter", 3_000), + lines, + ); + expect( + appendFinalTranscript( + said( + "mic", + "Let us start with the roadmap for next quarter", + 1_200, + 5_000, + ), + lines, + ), + ).toBe(false); + }); + + it("does not let an unrelated neighbouring line bury the match", () => { + const lines: TranscriptLine[] = []; + + appendFinalTranscript( + said("system", "Anyway that is everything from my side today", 1_000), + lines, + ); + appendFinalTranscript( + said("system", "Any questions before we go", 3_000), + lines, + ); + expect( + appendFinalTranscript( + said("mic", "Any questions before we go", 3_300), + lines, + ), + ).toBe(false); + }); +}); + +describe("in-flight partials", () => { + it("suppresses a mic partial that mirrors the system partial", () => { + const inFlight: TranscriptLine[] = [ + { + source: "system", + text: "so the next thing on the list is the pricing page rewrite", + startMs: null, + segments: [], + }, + ]; + + expect( + isMicEcho("so the next thing on the list is the pricing page", inFlight), + ).toBe(true); + }); + + it("keeps a mic partial of the user answering", () => { + const inFlight: TranscriptLine[] = [ + { + source: "system", + text: "so the next thing on the list is the pricing page rewrite", + startMs: null, + segments: [], + }, + ]; + + expect(isMicEcho("right, who is picking that up", inFlight)).toBe(false); }); }); diff --git a/templates/clips/desktop/src/lib/transcription-engine.ts b/templates/clips/desktop/src/lib/transcription-engine.ts index c90866a722..4e98b472a3 100644 --- a/templates/clips/desktop/src/lib/transcription-engine.ts +++ b/templates/clips/desktop/src/lib/transcription-engine.ts @@ -30,9 +30,6 @@ export interface SourcedTranscriptSegment extends TranscriptSegment { source: TranscriptSource; } -const DUPLICATE_TOKEN_OVERLAP = 0.72; -const DUPLICATE_TIME_OVERLAP = 0.35; - export interface FinalTranscriptEvent { /** Raw text (not trimmed); callers decide whether to skip empties. */ text: string; @@ -86,133 +83,212 @@ function transcriptWords(text: string): string[] { return normalized ? normalized.split(/\s+/) : []; } -function tokenOverlap(left: string[], right: string[]): number { - if (left.length === 0 || right.length === 0) return 0; - - const rightWords = new Set(right); - const sharedWords = new Set(left.filter((word) => rightWords.has(word))); - return sharedWords.size / Math.min(new Set(left).size, new Set(right).size); -} - -function timeOverlapRatio( - left: SourcedTranscriptSegment, - right: SourcedTranscriptSegment, -): number { - const overlapStart = Math.max(left.startMs, right.startMs); - const overlapEnd = Math.min(left.endMs, right.endMs); - const overlapMs = Math.max(0, overlapEnd - overlapStart); - const shorterDuration = Math.max( - 1, - Math.min(left.endMs - left.startMs, right.endMs - right.startMs), - ); - - return overlapMs / shorterDuration; +// --------------------------------------------------------------------------- +// Echo de-duplication +// --------------------------------------------------------------------------- +// +// Without headphones the microphone re-records whatever the speakers play, so +// the remote side can reach the transcript twice: cleanly on the system stream +// and, whenever the acoustic guard in `echo_guard.rs` was not confident enough +// to drop it, again on the mic. The leak only ever runs one way, because a call +// app never plays the local user back. So when both streams carry the same +// words at the same time, the system copy is the real one and the mic copy is +// echo — no matter which stream happened to finalize first. + +/** Share of the *longer* of the two word lists that has to match. Scoring the + * longer side is what keeps a brief interjection ("yeah, I think so") alive + * next to a long remote passage that happens to contain those words in order: + * echo repeats a whole utterance, it does not sprinkle a few words into one. */ +const ECHO_MATCH_RATIO = 0.65; +/** Fewer matched words than this is coincidence, not evidence. */ +const ECHO_MIN_WORDS = 4; +/** Words that have to agree before a long in-order run counts as echo on its + * own. Two people do not independently say eight of the same words, in the + * same order, at the same moment. */ +const ECHO_LONG_MATCH_WORDS = 8; +/** How many recently appended lines count as "at the same time". + * + * Arrival order, not timestamps: each stream's Whisper timestamps are + * estimates against its own rolling buffer, and the two streams cut those + * buffers at their own silences, so the same words routinely carry stamps + * seconds apart. Both finals still *arrive* within a second or two of each + * other, because they are transcribed from the same sound. Finalized lines also + * get a loose timestamp bound below to reject much later deliberate repeats. */ +const ECHO_RECENT_LINES = 6; +/** Cross-stream timestamps are approximate, but echo finals should still begin + * within this loose bound of each other. */ +const ECHO_MAX_START_DELTA_MS = 15_000; + +/** Length of the longest common subsequence of two word lists. Subsequence + * rather than set intersection because echo repeats the words *in order*, + * while two people using the same vocabulary do not — and rather than exact + * equality because Whisper mangles echo with dropped and substituted words. */ +function commonWordRun(left: string[], right: string[]): number { + let previous = new Array(right.length + 1).fill(0); + let current = new Array(right.length + 1).fill(0); + for (const word of left) { + for (let index = 0; index < right.length; index++) { + current[index + 1] = + word === right[index] + ? previous[index] + 1 + : Math.max(current[index], previous[index + 1]); + } + [previous, current] = [current, previous]; + } + return previous[right.length]; } -function isDuplicateTranscriptSegment( - existing: SourcedTranscriptSegment, - incoming: SourcedTranscriptSegment, +/** Whether `text` heard on the mic is the speakers bleeding back into it. + * + * Exported because the live overlay has to make the same call on in-flight + * partials, which never reach `appendFinalTranscript`. + * + * Scored against every contiguous run of the recent system lines, not against + * all of them glued together: the two streams cut speech at different points, + * so one mic line can echo a single system line or straddle two, and gluing an + * unrelated third one in would bury the match. */ +export function isMicEcho( + text: string, + lines: TranscriptLine[], + startMs: number | null = null, ): boolean { - if (existing.source === incoming.source) return false; - - const existingText = normalizedTranscriptText(existing.text); - const incomingText = normalizedTranscriptText(incoming.text); - const existingWords = transcriptWords(existing.text); - const incomingWords = transcriptWords(incoming.text); - - if ( - !existingText || - !incomingText || - existingWords.length < 3 || - incomingWords.length < 3 - ) { - return false; + const words = transcriptWords(text); + if (words.length < ECHO_MIN_WORDS) return false; + + const nearby = lines + .slice(-ECHO_RECENT_LINES) + .filter( + (line) => + line.source === "system" && + !line.historical && + (startMs === null || + line.startMs === null || + Math.abs(startMs - line.startMs) <= ECHO_MAX_START_DELTA_MS), + ) + .map((line) => transcriptWords(line.text)); + + for (let start = 0; start < nearby.length; start++) { + const run: string[] = []; + for (let end = start; end < nearby.length; end++) { + run.push(...nearby[end]); + const matched = commonWordRun(words, run); + if (matched < ECHO_MIN_WORDS) continue; + // Either the two are the same length and mostly agree, or they agree on + // a stretch long enough that nothing but echo explains it. + const sameUtterance = + matched / Math.max(words.length, run.length) >= ECHO_MATCH_RATIO; + const longRun = + matched >= ECHO_LONG_MATCH_WORDS && + matched / words.length >= ECHO_MATCH_RATIO; + if (sameUtterance || longRun) return true; + } } + return false; +} - if (timeOverlapRatio(existing, incoming) < DUPLICATE_TIME_OVERLAP) { - return false; +/** Drop the recent mic lines that a just-appended system line exposes as echo. */ +function retractMicEcho(lines: TranscriptLine[]): void { + const snapshot = [...lines]; + const oldest = Math.max(0, snapshot.length - ECHO_RECENT_LINES); + const removals: number[] = []; + for (let index = snapshot.length - 1; index >= oldest; index--) { + const line = snapshot[index]; + if (line.source !== "mic" || line.historical) continue; + const evidence = snapshot.slice(index, index + ECHO_RECENT_LINES); + if (isMicEcho(line.text, evidence, line.startMs)) removals.push(index); } + for (const index of removals) lines.splice(index, 1); +} - return ( - existingText === incomingText || - tokenOverlap(existingWords, incomingWords) >= DUPLICATE_TOKEN_OVERLAP - ); +// --------------------------------------------------------------------------- +// Transcript lines +// --------------------------------------------------------------------------- + +/** One speaker-labelled transcript line. `segments` carries the verbatim + * Whisper timings behind the line and is empty for the mic-only fallback + * engines, which report no timestamps. */ +export interface TranscriptLine { + source: TranscriptSource; + /** Preloaded lines are display/persistence data, not live echo evidence. */ + historical?: boolean; + /** Meeting-timeline position, or null from an engine that reports none. + * Not 0 — the overlay renders a timestamp for every line that has one, and + * "start of the meeting" is a different claim from "unknown". */ + startMs: number | null; + text: string; + segments: SourcedTranscriptSegment[]; } -function isDuplicateTranscriptLine( - lines: string[], - source: TranscriptSource, - text: string, -): boolean { - const words = transcriptWords(text); - if (words.length < 4) return false; +function lineFromSegments( + segments: SourcedTranscriptSegment[], +): TranscriptLine { + return { + source: segments[0].source, + startMs: segments[0].startMs, + text: segments.map((segment) => segment.text).join(" "), + segments, + }; +} - const speaker = speakerFor(source); - return lines.some((line) => { - const separatorIndex = line.indexOf(":"); - if (separatorIndex === -1 || line.slice(0, separatorIndex) === speaker) { - return false; - } +/** Rebuild a line from a stored segment, for preloaded transcript history. */ +export function transcriptLineFromSegment( + segment: SourcedTranscriptSegment, +): TranscriptLine { + return { ...lineFromSegments([segment]), historical: true }; +} - const existingText = line.slice(separatorIndex + 1); - const existingWords = transcriptWords(existingText); - return ( - normalizedTranscriptText(existingText) === - normalizedTranscriptText(text) || - tokenOverlap(existingWords, words) >= DUPLICATE_TOKEN_OVERLAP - ); - }); +/** Speaker-labelled text, as persisted by `save-browser-transcript`. */ +export function transcriptFullText(lines: TranscriptLine[]): string { + return lines + .map((line) => `${speakerFor(line.source)}: ${line.text}`) + .join("\n\n") + .trim(); +} + +/** Flattened verbatim segments, as persisted alongside the text. */ +export function transcriptSegments( + lines: TranscriptLine[], +): SourcedTranscriptSegment[] { + return lines.flatMap((line) => line.segments); } /** - * Fold a final-transcript event into a running transcript: appends a - * speaker-labelled line and the event's (non-empty) segments tagged with the - * event source. Mutates `lines`/`segments` in place. Returns true if anything - * was appended (i.e. the event had non-empty text). + * Fold a final-transcript event into a running transcript, dropping mic speech + * that only echoes the system audio and retracting mic lines that a later + * system line exposes as echo. Mutates `lines` in place; returns whether the + * transcript changed. */ export function appendFinalTranscript( event: FinalTranscriptEvent, - lines: string[], - segments: SourcedTranscriptSegment[], + lines: TranscriptLine[], ): boolean { const text = event.text.trim(); if (!text) return false; - if (event.segments.length === 0) { - if (isDuplicateTranscriptLine(lines, event.source, text)) return false; - - lines.push(`${speakerFor(event.source)}: ${text}`); + const segments: SourcedTranscriptSegment[] = event.segments + .map((segment) => ({ + startMs: segment.startMs, + endMs: segment.endMs, + text: segment.text?.trim() ?? "", + source: event.source, + })) + .filter((segment) => segment.text.length > 0); + + // One event is one stream's take on one utterance, so the whole line is the + // unit that is or is not echo. Engines without timestamps still produce a + // line, just one carrying no segments behind it. + const line: TranscriptLine = segments.length + ? lineFromSegments(segments) + : { source: event.source, startMs: null, text, segments: [] }; + + if (line.source === "mic") { + if (isMicEcho(line.text, lines, line.startMs)) return false; + lines.push(line); return true; } - const uniqueSegments = event.segments.filter((segment) => { - const segText = segment.text?.trim(); - if (!segText) return false; - - return !segments.some((existing) => - isDuplicateTranscriptSegment(existing, { - ...segment, - text: segText, - source: event.source, - }), - ); - }); - - if (uniqueSegments.length === 0) return false; - - lines.push( - `${speakerFor(event.source)}: ${uniqueSegments - .map((segment) => segment.text.trim()) - .join(" ")}`, - ); - for (const seg of uniqueSegments) { - segments.push({ - startMs: seg.startMs, - endMs: seg.endMs, - text: seg.text.trim(), - source: event.source, - }); - } + lines.push(line); + retractMicEcho(lines); return true; } diff --git a/templates/clips/desktop/src/overlays/live-transcript.tsx b/templates/clips/desktop/src/overlays/live-transcript.tsx index 2bdcadf793..93a8195402 100644 --- a/templates/clips/desktop/src/overlays/live-transcript.tsx +++ b/templates/clips/desktop/src/overlays/live-transcript.tsx @@ -1,8 +1,11 @@ -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { + appendFinalTranscript, + isMicEcho, onFinalTranscript, onPartialTranscript, + type TranscriptLine, } from "../lib/transcription-engine"; type Source = "mic" | "system"; @@ -13,6 +16,17 @@ export interface FinalLine { startMs?: number; } +/** Preloaded history arrives without its verbatim segments. */ +function historyLine(line: FinalLine): TranscriptLine { + return { + text: line.text, + source: line.source, + startMs: line.startMs ?? null, + segments: [], + historical: true, + }; +} + /** Format ms since meeting start as m:ss. */ function formatTimestamp(ms: number): string { const total = Math.floor(ms / 1000); @@ -38,10 +52,12 @@ export function LiveTranscript({ onLinesChange, initialLines, }: { - onLinesChange?: (lines: FinalLine[]) => void; + onLinesChange?: (lines: TranscriptLine[]) => void; initialLines?: FinalLine[]; } = {}) { - const [finals, setFinals] = useState(initialLines ?? []); + const [finals, setFinals] = useState( + () => initialLines?.map(historyLine) ?? [], + ); const [micPartial, setMicPartial] = useState(""); const [sysPartial, setSysPartial] = useState(""); const scrollRef = useRef(null); @@ -69,7 +85,7 @@ export function LiveTranscript({ // older history), instead of being overwritten. if (preloadAppliedRef.current) return; preloadAppliedRef.current = true; - setFinals((prev) => [...lines, ...prev]); + setFinals((prev) => [...lines.map(historyLine), ...prev]); }, [initialLines]); useEffect(() => { @@ -97,12 +113,13 @@ export function LiveTranscript({ }), ); trackListen( - onFinalTranscript(({ text, source, segments }) => { - const txt = text.trim(); - if (!txt) return; - const startMs = segments[0]?.startMs; - setFinals((prev) => [...prev, { text: txt, source, startMs }]); - if (source === "system") setSysPartial(""); + onFinalTranscript((event) => { + if (!event.text.trim()) return; + setFinals((prev) => { + const next = [...prev]; + return appendFinalTranscript(event, next) ? next : prev; + }); + if (event.source === "system") setSysPartial(""); else setMicPartial(""); }), ); @@ -126,9 +143,21 @@ export function LiveTranscript({ el.scrollTop = el.scrollHeight; }, [finals, micPartial, sysPartial]); + // Partials never reach `appendFinalTranscript`, so speaker bleed shows up + // here as a live "You" bubble mirroring what the remote side is still + // saying. Judge the in-flight mic text against the in-flight system text too, + // since neither has been committed to a line yet. + const spokenMicPartial = useMemo(() => { + if (!micPartial) return ""; + const inFlight: TranscriptLine[] = sysPartial + ? [{ source: "system", text: sysPartial, startMs: null, segments: [] }] + : []; + return isMicEcho(micPartial, [...finals, ...inFlight]) ? "" : micPartial; + }, [micPartial, sysPartial, finals]); + return (
- {finals.length === 0 && !micPartial && !sysPartial ? ( + {finals.length === 0 && !spokenMicPartial && !sysPartial ? (
Listening…
) : null} {finals.map((line, i) => ( @@ -142,8 +171,8 @@ export function LiveTranscript({ {sysPartial ? ( ) : null} - {micPartial ? ( - + {spokenMicPartial ? ( + ) : null}
); @@ -164,7 +193,7 @@ function ChatBubble({ source: Source; text: string; pending?: boolean; - startMs?: number; + startMs?: number | null; }) { const isYou = source === "mic"; const label = isYou ? "You" : "Them"; diff --git a/templates/clips/desktop/src/overlays/recording-pill.tsx b/templates/clips/desktop/src/overlays/recording-pill.tsx index 33e53f12ce..b4189d17f3 100644 --- a/templates/clips/desktop/src/overlays/recording-pill.tsx +++ b/templates/clips/desktop/src/overlays/recording-pill.tsx @@ -17,7 +17,7 @@ import { getCurrentWindow } from "@tauri-apps/api/window"; import { useCallback, useEffect, useRef, useState } from "react"; import { isDirectPillClick, type ScreenPoint } from "../lib/pill-interaction"; -import { speakerFor } from "../lib/transcription-engine"; +import { speakerFor, type TranscriptLine } from "../lib/transcription-engine"; import { LiveAudioBars } from "./live-audio-bars"; import { LiveTranscript, type FinalLine } from "./live-transcript"; import { PillLogo } from "./pill-logo"; @@ -52,7 +52,7 @@ export function RecordingPill() { ); const finished = finishedMeetingId !== null; const [error, setError] = useState(null); - const transcriptLinesRef = useRef([]); + const transcriptLinesRef = useRef([]); const [hasTranscriptLines, setHasTranscriptLines] = useState(false); const [transcriptCopied, setTranscriptCopied] = useState(false); const [preloadedLines, setPreloadedLines] = useState([]); @@ -269,7 +269,7 @@ export function RecordingPill() { // Stable callback for LiveTranscript to push locked-in lines up. Stable // identity matters — it's a dep of an effect inside LiveTranscript. - const handleTranscriptLines = useCallback((lines: FinalLine[]) => { + const handleTranscriptLines = useCallback((lines: TranscriptLine[]) => { transcriptLinesRef.current = lines; setHasTranscriptLines(lines.length > 0); }, []);