diff --git a/v2/crates/wifi-densepose-train/build.rs b/v2/crates/wifi-densepose-train/build.rs new file mode 100644 index 0000000000..661d99f51c --- /dev/null +++ b/v2/crates/wifi-densepose-train/build.rs @@ -0,0 +1,38 @@ +//! Build script: when the `tch-backend` feature binds libtorch from the system +//! PyTorch install (`LIBTORCH_USE_PYTORCH=1`), embed an rpath to the PyTorch +//! `lib` directory so the produced binaries and test executables can locate +//! `libtorch_cpu` at runtime. torch-sys only adds the rpath to its own +//! cc-built static library, which does not propagate to downstream binaries on +//! macOS/Linux. A strict no-op otherwise (default / `--no-default-features` +//! builds, including the Python wheel, are unaffected). +fn main() { + println!("cargo:rerun-if-env-changed=LIBTORCH_USE_PYTORCH"); + println!("cargo:rerun-if-env-changed=PYTHON_SYS_EXECUTABLE"); + if std::env::var_os("CARGO_FEATURE_TCH_BACKEND").is_none() + || std::env::var_os("LIBTORCH_USE_PYTORCH").is_none() + { + return; + } + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + if target_os != "macos" && target_os != "linux" { + return; + } + // Same interpreter selection torch-sys uses, defaulting to python3. + let python = std::env::var("PYTHON_SYS_EXECUTABLE").unwrap_or_else(|_| "python3".into()); + let output = std::process::Command::new(python) + .args([ + "-c", + "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))", + ]) + .output(); + if let Ok(output) = output { + if output.status.success() { + let lib_dir = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !lib_dir.is_empty() { + // Plain rustc-link-arg (not -bins/-tests): it is the only + // variant that also reaches the lib unit-test binary. + println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}"); + } + } + } +} diff --git a/v2/crates/wifi-densepose-train/src/bin/train.rs b/v2/crates/wifi-densepose-train/src/bin/train.rs index 5fe7bab2f3..e61ce5b1be 100644 --- a/v2/crates/wifi-densepose-train/src/bin/train.rs +++ b/v2/crates/wifi-densepose-train/src/bin/train.rs @@ -55,6 +55,29 @@ struct Args { #[arg(long, value_name = "DIR")] data_dir: Option, + /// MM-Fi directory to use as the REAL validation set (e.g. a held-out + /// subject or session). Takes precedence over the automatic + /// subject-disjoint split of `--data-dir`. Must share no subject with + /// `--data-dir` (checked; overlap is refused as a train/val leak). + #[arg(long, value_name = "DIR")] + val_dir: Option, + + /// Evaluation-only mode: load `--checkpoint`, run the validation loop over + /// `--val-dir` (no training), print PCK@0.2 / OKS and exit. Requires the + /// `tch-backend` feature. + #[arg(long, default_value_t = false, requires_all = ["checkpoint", "val_dir"])] + eval_only: bool, + + /// Checkpoint (tch VarStore `.pt` or `.safetensors`) to load in + /// `--eval-only` mode. + #[arg(long, value_name = "FILE", requires = "eval_only")] + checkpoint: Option, + + /// In `--eval-only` mode, dump per-sample predicted + GT keypoints as JSONL + /// to this path (dataset order, one line per window). + #[arg(long, value_name = "FILE", requires = "eval_only")] + dump_preds: Option, + /// Override the checkpoint output directory from the config. #[arg(long, value_name = "DIR")] checkpoint_dir: Option, @@ -137,6 +160,22 @@ fn main() { log_config_summary(&config); + // ------------------------------------------------------------------ + // Eval-only mode: load checkpoint, run the validation loop, exit + // ------------------------------------------------------------------ + + if args.eval_only { + // clap enforces `--checkpoint` and `--val-dir` alongside `--eval-only`. + let (Some(ckpt), Some(val_dir)) = (args.checkpoint.as_deref(), args.val_dir.as_deref()) + else { + error!("--eval-only requires --checkpoint and --val-dir "); + std::process::exit(2); + }; + let eval_ds = load_mmfi_or_exit(val_dir, &config, "eval"); + run_eval(config, &eval_ds, ckpt, args.dump_preds.as_deref()); + return; + } + // ------------------------------------------------------------------ // Build datasets // ------------------------------------------------------------------ @@ -204,6 +243,35 @@ fn main() { info!("Dataset: {} samples", train_ds.len()); + // An explicit held-out validation directory takes precedence over the + // automatic split. Refuse any subject overlap with the training set: + // that would be exactly the leak ADR-155 §Tier-1.2 forbids. + if let Some(val_dir) = args.val_dir.as_deref() { + let val_ds = load_mmfi_or_exit(val_dir, &config, "validation"); + let train_subjects = train_ds.subjects(); + let shared: Vec = val_ds + .subjects() + .into_iter() + .filter(|s| train_subjects.contains(s)) + .collect(); + if !shared.is_empty() { + error!( + "--val-dir shares subject(s) {shared:?} with --data-dir; a \ + subject-overlapping validation set leaks (ADR-155 §Tier-1.2). \ + Use a held-out subject directory." + ); + std::process::exit(1); + } + info!( + "Using REAL held-out validation set: {} windows (subjects {:?}) from {}", + val_ds.len(), + val_ds.subjects(), + val_dir.display() + ); + run_training(config, &train_ds, &val_ds); + return; + } + // ADR-155 §Tier-1.2: prefer a REAL, leak-free, subject-disjoint split so // any reported PCK/OKS is honest. MM-Fi windows are stride-1 (≈99% // overlap), so an index-level split would leak; a synthetic val set @@ -295,6 +363,83 @@ fn run_training(_config: TrainingConfig, train_ds: &dyn CsiDataset, val_ds: &dyn info!("Config and dataset infrastructure: OK"); } +// --------------------------------------------------------------------------- +// --val-dir / --eval-only helpers +// --------------------------------------------------------------------------- + +/// Load an MM-Fi directory with the run's window/subcarrier/keypoint shape, or +/// exit with an error if it cannot be loaded or holds no windows. +fn load_mmfi_or_exit(dir: &std::path::Path, config: &TrainingConfig, what: &str) -> MmFiDataset { + let ds = match MmFiDataset::discover( + dir, + config.window_frames, + config.num_subcarriers, + config.num_keypoints, + ) { + Ok(ds) => ds, + Err(e) => { + error!("Failed to load {what} dataset from {}: {e}", dir.display()); + std::process::exit(1); + } + }; + if ds.is_empty() { + error!("{what} dataset is empty — {}", dir.display()); + std::process::exit(1); + } + ds +} + +/// `--eval-only`: load a checkpoint into a fresh model and run the same +/// validation loop training uses ([`Trainer::evaluate`]). +#[cfg(feature = "tch-backend")] +fn run_eval( + config: TrainingConfig, + ds: &dyn CsiDataset, + checkpoint: &std::path::Path, + dump: Option<&std::path::Path>, +) { + use wifi_densepose_train::trainer::Trainer; + + let mut trainer = Trainer::new(config); + match trainer.load_checkpoint(checkpoint) { + Ok(epoch) => info!("Loaded checkpoint {} (epoch {epoch})", checkpoint.display()), + Err(e) => { + error!("Failed to load checkpoint: {e}"); + std::process::exit(1); + } + } + info!("Evaluating on {} windows", ds.len()); + + let result = match dump { + Some(path) => { + info!("Dumping per-sample predictions to {}", path.display()); + trainer.evaluate_with_dump(ds, path) + } + None => trainer.evaluate(ds), + }; + match result { + Ok(m) => info!("EVAL RESULT: {}", m.summary()), + Err(e) => { + error!("Evaluation failed: {e}"); + std::process::exit(1); + } + } +} + +#[cfg(not(feature = "tch-backend"))] +fn run_eval( + _config: TrainingConfig, + _ds: &dyn CsiDataset, + _checkpoint: &std::path::Path, + _dump: Option<&std::path::Path>, +) { + error!( + "--eval-only requires the `tch-backend` feature: \ + cargo run --features tch-backend --bin train" + ); + std::process::exit(1); +} + // --------------------------------------------------------------------------- // run_smoke_test — synthetic-validation path (NOT a reportable metric) // --------------------------------------------------------------------------- diff --git a/v2/crates/wifi-densepose-train/src/losses.rs b/v2/crates/wifi-densepose-train/src/losses.rs index 1efdd96d9f..1053eb7a0b 100644 --- a/v2/crates/wifi-densepose-train/src/losses.rs +++ b/v2/crates/wifi-densepose-train/src/losses.rs @@ -692,8 +692,11 @@ mod tests { let cy = (kp_y * s).round() as usize; let peak = hm[[cy, cx]]; + // kp 0.5 on a 64-px map is the half-pixel point 31.5, so the max + // attainable at pixel 31 or 32 is exp(-(0.5^2+0.5^2)/(2*2^2)) ≈ 0.9394. + // The old 0.95 threshold was mathematically unreachable. assert!( - peak > 0.95, + peak > 0.93, "Peak value {peak} should be close to 1.0 at centre" ); @@ -931,7 +934,10 @@ mod tests { let vis = Tensor::ones(&[1i64, 1], (Kind::Float, dev)); let hm = generate_gaussian_heatmaps(&kpts, &vis, 8, 1.5); let max_val: f64 = hm.max().double_value(&[]); - assert!(max_val > 0.9, "Peak value {max_val} should be > 0.9"); + // kp 0.5 on an 8-px map is the half-pixel point 3.5, so the max + // attainable at pixel 3 or 4 is exp(-(0.5^2+0.5^2)/(2*1.5^2)) ≈ 0.8948. + // The old 0.9 threshold was mathematically unreachable. + assert!(max_val > 0.89, "Peak value {max_val} should be > 0.89"); } #[test] diff --git a/v2/crates/wifi-densepose-train/src/trainer.rs b/v2/crates/wifi-densepose-train/src/trainer.rs index a9f8d6702f..90fb70b67e 100644 --- a/v2/crates/wifi-densepose-train/src/trainer.rs +++ b/v2/crates/wifi-densepose-train/src/trainer.rs @@ -364,11 +364,43 @@ impl Trainer { /// Runs inference (no gradient) over the full dataset using the configured /// batch size. pub fn evaluate(&self, dataset: &dyn CsiDataset) -> Result { + self.evaluate_inner(dataset, None) + } + + /// Evaluate exactly like [`Trainer::evaluate`] (same loop, same metric), + /// additionally writing per-sample predicted + ground-truth keypoints as + /// JSONL to `dump_path`. + /// + /// Each line is `{"idx":N,"pred":[[x,y],..],"gt":[[x,y],..],"vis":[..]}` + /// in dataset order (evaluation never shuffles), so `idx` is the dataset + /// window index. Used by `train --eval-only --dump-preds`. + pub fn evaluate_with_dump( + &self, + dataset: &dyn CsiDataset, + dump_path: &Path, + ) -> Result { + let file = std::fs::File::create(dump_path) + .map_err(|e| TrainError::training_step(format!("create dump file: {e}")))?; + let mut sink = std::io::BufWriter::new(file); + let result = self.evaluate_inner(dataset, Some(&mut sink))?; + sink.flush() + .map_err(|e| TrainError::training_step(format!("flush dump file: {e}")))?; + Ok(result) + } + + /// Shared evaluation loop behind [`Trainer::evaluate`] and + /// [`Trainer::evaluate_with_dump`]. + fn evaluate_inner( + &self, + dataset: &dyn CsiDataset, + mut dump: Option<&mut dyn IoWrite>, + ) -> Result { if dataset.is_empty() { return Err(TrainError::EmptyDataset); } let mut acc = MetricsAccumulator::default_threshold(); + let mut global_idx: usize = 0; let batches = make_batches( dataset, @@ -393,6 +425,25 @@ impl Trainer { let vis_np = extract_vis_ndarray(vis_batch, b); acc.update(&pred_kp_np, >_kp_np, &vis_np); + + if let Some(sink) = dump.as_deref_mut() { + let pred_v: Vec<[f32; 2]> = pred_kp_np + .rows() + .into_iter() + .map(|r| [r[0], r[1]]) + .collect(); + let gt_v: Vec<[f32; 2]> = + gt_kp_np.rows().into_iter().map(|r| [r[0], r[1]]).collect(); + let line = serde_json::json!({ + "idx": global_idx, + "pred": pred_v, + "gt": gt_v, + "vis": vis_np.to_vec(), + }); + writeln!(sink, "{line}") + .map_err(|e| TrainError::training_step(format!("write dump line: {e}")))?; + } + global_idx += 1; } }