Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions v2/crates/wifi-densepose-train/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! Build script: when the `tch-backend` feature binds libtorch from the system
//! PyTorch install (`LIBTORCH_USE_PYTORCH=1`), embed an rpath to the PyTorch
//! `lib` directory so the produced binaries and test executables can locate
//! `libtorch_cpu` at runtime. torch-sys only adds the rpath to its own
//! cc-built static library, which does not propagate to downstream binaries on
//! macOS/Linux. A strict no-op otherwise (default / `--no-default-features`
//! builds, including the Python wheel, are unaffected).
fn main() {
println!("cargo:rerun-if-env-changed=LIBTORCH_USE_PYTORCH");
println!("cargo:rerun-if-env-changed=PYTHON_SYS_EXECUTABLE");
if std::env::var_os("CARGO_FEATURE_TCH_BACKEND").is_none()
|| std::env::var_os("LIBTORCH_USE_PYTORCH").is_none()
{
return;
}
let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
if target_os != "macos" && target_os != "linux" {
return;
}
// Same interpreter selection torch-sys uses, defaulting to python3.
let python = std::env::var("PYTHON_SYS_EXECUTABLE").unwrap_or_else(|_| "python3".into());
let output = std::process::Command::new(python)
.args([
"-c",
"import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))",
])
.output();
if let Ok(output) = output {
if output.status.success() {
let lib_dir = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !lib_dir.is_empty() {
// Plain rustc-link-arg (not -bins/-tests): it is the only
// variant that also reaches the lib unit-test binary.
println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}");
}
}
}
}
145 changes: 145 additions & 0 deletions v2/crates/wifi-densepose-train/src/bin/train.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,29 @@ struct Args {
#[arg(long, value_name = "DIR")]
data_dir: Option<PathBuf>,

/// MM-Fi directory to use as the REAL validation set (e.g. a held-out
/// subject or session). Takes precedence over the automatic
/// subject-disjoint split of `--data-dir`. Must share no subject with
/// `--data-dir` (checked; overlap is refused as a train/val leak).
#[arg(long, value_name = "DIR")]
val_dir: Option<PathBuf>,

/// Evaluation-only mode: load `--checkpoint`, run the validation loop over
/// `--val-dir` (no training), print PCK@0.2 / OKS and exit. Requires the
/// `tch-backend` feature.
#[arg(long, default_value_t = false, requires_all = ["checkpoint", "val_dir"])]
eval_only: bool,

/// Checkpoint (tch VarStore `.pt` or `.safetensors`) to load in
/// `--eval-only` mode.
#[arg(long, value_name = "FILE", requires = "eval_only")]
checkpoint: Option<PathBuf>,

/// In `--eval-only` mode, dump per-sample predicted + GT keypoints as JSONL
/// to this path (dataset order, one line per window).
#[arg(long, value_name = "FILE", requires = "eval_only")]
dump_preds: Option<PathBuf>,

/// Override the checkpoint output directory from the config.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#[arg(long, value_name = "DIR")]
checkpoint_dir: Option<PathBuf>,
Expand Down Expand Up @@ -137,6 +160,22 @@ fn main() {

log_config_summary(&config);

// ------------------------------------------------------------------
// Eval-only mode: load checkpoint, run the validation loop, exit

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// Eval-only mode: load checkpoint, run the validation loop, exit
// Eval-only mode: load checkpoint, run the validation loop, exit```suggestion
// Eval-only mode: load checkpoint, run the validation loop, exit**[factory reset]()**

// ------------------------------------------------------------------

if args.eval_only {
// clap enforces `--checkpoint` and `--val-dir` alongside `--eval-only`.
let (Some(ckpt), Some(val_dir)) = (args.checkpoint.as_deref(), args.val_dir.as_deref())
else {
error!("--eval-only requires --checkpoint <FILE> and --val-dir <DIR>");
std::process::exit(2);
};
let eval_ds = load_mmfi_or_exit(val_dir, &config, "eval");
run_eval(config, &eval_ds, ckpt, args.dump_preds.as_deref());
return;
}

// ------------------------------------------------------------------
// Build datasets
// ------------------------------------------------------------------
Expand Down Expand Up @@ -204,6 +243,35 @@ fn main() {

info!("Dataset: {} samples", train_ds.len());

// An explicit held-out validation directory takes precedence over the
// automatic split. Refuse any subject overlap with the training set:
// that would be exactly the leak ADR-155 §Tier-1.2 forbids.
if let Some(val_dir) = args.val_dir.as_deref() {
let val_ds = load_mmfi_or_exit(val_dir, &config, "validation");
let train_subjects = train_ds.subjects();
let shared: Vec<u32> = val_ds
.subjects()
.into_iter()
.filter(|s| train_subjects.contains(s))
.collect();
if !shared.is_empty() {
error!(
"--val-dir shares subject(s) {shared:?} with --data-dir; a \
subject-overlapping validation set leaks (ADR-155 §Tier-1.2). \
Use a held-out subject directory."
);
std::process::exit(1);
}
info!(
"Using REAL held-out validation set: {} windows (subjects {:?}) from {}",
val_ds.len(),
val_ds.subjects(),
val_dir.display()
);
run_training(config, &train_ds, &val_ds);
return;
}

// ADR-155 §Tier-1.2: prefer a REAL, leak-free, subject-disjoint split so
// any reported PCK/OKS is honest. MM-Fi windows are stride-1 (≈99%
// overlap), so an index-level split would leak; a synthetic val set
Expand Down Expand Up @@ -295,6 +363,83 @@ fn run_training(_config: TrainingConfig, train_ds: &dyn CsiDataset, val_ds: &dyn
info!("Config and dataset infrastructure: OK");
}

// ---------------------------------------------------------------------------
// --val-dir / --eval-only helpers
// ---------------------------------------------------------------------------

/// Load an MM-Fi directory with the run's window/subcarrier/keypoint shape, or
/// exit with an error if it cannot be loaded or holds no windows.
fn load_mmfi_or_exit(dir: &std::path::Path, config: &TrainingConfig, what: &str) -> MmFiDataset {
let ds = match MmFiDataset::discover(
dir,
config.window_frames,
config.num_subcarriers,
config.num_keypoints,
) {
Ok(ds) => ds,
Err(e) => {
error!("Failed to load {what} dataset from {}: {e}", dir.display());
std::process::exit(1);
}
};
if ds.is_empty() {
error!("{what} dataset is empty — {}", dir.display());
std::process::exit(1);
}
ds
}

/// `--eval-only`: load a checkpoint into a fresh model and run the same
/// validation loop training uses ([`Trainer::evaluate`]).
#[cfg(feature = "tch-backend")]
fn run_eval(
config: TrainingConfig,
ds: &dyn CsiDataset,
checkpoint: &std::path::Path,
dump: Option<&std::path::Path>,
) {
use wifi_densepose_train::trainer::Trainer;

let mut trainer = Trainer::new(config);
match trainer.load_checkpoint(checkpoint) {
Ok(epoch) => info!("Loaded checkpoint {} (epoch {epoch})", checkpoint.display()),
Err(e) => {
error!("Failed to load checkpoint: {e}");
std::process::exit(1);
}
}
info!("Evaluating on {} windows", ds.len());

let result = match dump {
Some(path) => {
info!("Dumping per-sample predictions to {}", path.display());
trainer.evaluate_with_dump(ds, path)
}
None => trainer.evaluate(ds),
};
match result {
Ok(m) => info!("EVAL RESULT: {}", m.summary()),
Err(e) => {
error!("Evaluation failed: {e}");
std::process::exit(1);
}
}
}

#[cfg(not(feature = "tch-backend"))]
fn run_eval(
_config: TrainingConfig,
_ds: &dyn CsiDataset,
_checkpoint: &std::path::Path,
_dump: Option<&std::path::Path>,
) {
error!(
"--eval-only requires the `tch-backend` feature: \
cargo run --features tch-backend --bin train"
);
std::process::exit(1);
}

// ---------------------------------------------------------------------------
// run_smoke_test — synthetic-validation path (NOT a reportable metric)
// ---------------------------------------------------------------------------
Expand Down
10 changes: 8 additions & 2 deletions v2/crates/wifi-densepose-train/src/losses.rs
Original file line number Diff line number Diff line change
Expand Up @@ -692,8 +692,11 @@ mod tests {
let cy = (kp_y * s).round() as usize;

let peak = hm[[cy, cx]];
// kp 0.5 on a 64-px map is the half-pixel point 31.5, so the max
// attainable at pixel 31 or 32 is exp(-(0.5^2+0.5^2)/(2*2^2)) ≈ 0.9394.
// The old 0.95 threshold was mathematically unreachable.
assert!(
peak > 0.95,
peak > 0.93,
"Peak value {peak} should be close to 1.0 at centre"
);

Expand Down Expand Up @@ -931,7 +934,10 @@ mod tests {
let vis = Tensor::ones(&[1i64, 1], (Kind::Float, dev));
let hm = generate_gaussian_heatmaps(&kpts, &vis, 8, 1.5);
let max_val: f64 = hm.max().double_value(&[]);
assert!(max_val > 0.9, "Peak value {max_val} should be > 0.9");
// kp 0.5 on an 8-px map is the half-pixel point 3.5, so the max
// attainable at pixel 3 or 4 is exp(-(0.5^2+0.5^2)/(2*1.5^2)) ≈ 0.8948.
// The old 0.9 threshold was mathematically unreachable.
assert!(max_val > 0.89, "Peak value {max_val} should be > 0.89");
}

#[test]
Expand Down
51 changes: 51 additions & 0 deletions v2/crates/wifi-densepose-train/src/trainer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -364,11 +364,43 @@ impl Trainer {
/// Runs inference (no gradient) over the full dataset using the configured
/// batch size.
pub fn evaluate(&self, dataset: &dyn CsiDataset) -> Result<MetricsResult, TrainError> {
self.evaluate_inner(dataset, None)
}

/// Evaluate exactly like [`Trainer::evaluate`] (same loop, same metric),
/// additionally writing per-sample predicted + ground-truth keypoints as
/// JSONL to `dump_path`.
///
/// Each line is `{"idx":N,"pred":[[x,y],..],"gt":[[x,y],..],"vis":[..]}`
/// in dataset order (evaluation never shuffles), so `idx` is the dataset
/// window index. Used by `train --eval-only --dump-preds`.
pub fn evaluate_with_dump(
&self,
dataset: &dyn CsiDataset,
dump_path: &Path,
) -> Result<MetricsResult, TrainError> {
let file = std::fs::File::create(dump_path)
.map_err(|e| TrainError::training_step(format!("create dump file: {e}")))?;
let mut sink = std::io::BufWriter::new(file);
let result = self.evaluate_inner(dataset, Some(&mut sink))?;
sink.flush()
.map_err(|e| TrainError::training_step(format!("flush dump file: {e}")))?;
Ok(result)
}

/// Shared evaluation loop behind [`Trainer::evaluate`] and
/// [`Trainer::evaluate_with_dump`].
fn evaluate_inner(
&self,
dataset: &dyn CsiDataset,
mut dump: Option<&mut dyn IoWrite>,
) -> Result<MetricsResult, TrainError> {
if dataset.is_empty() {
return Err(TrainError::EmptyDataset);
}

let mut acc = MetricsAccumulator::default_threshold();
let mut global_idx: usize = 0;

let batches = make_batches(
dataset,
Expand All @@ -393,6 +425,25 @@ impl Trainer {
let vis_np = extract_vis_ndarray(vis_batch, b);

acc.update(&pred_kp_np, &gt_kp_np, &vis_np);

if let Some(sink) = dump.as_deref_mut() {
let pred_v: Vec<[f32; 2]> = pred_kp_np
.rows()
.into_iter()
.map(|r| [r[0], r[1]])
.collect();
let gt_v: Vec<[f32; 2]> =
gt_kp_np.rows().into_iter().map(|r| [r[0], r[1]]).collect();
let line = serde_json::json!({
"idx": global_idx,
"pred": pred_v,
"gt": gt_v,
"vis": vis_np.to_vec(),
});
writeln!(sink, "{line}")
.map_err(|e| TrainError::training_step(format!("write dump line: {e}")))?;
}
global_idx += 1;
}
}

Expand Down
Loading