Repository navigation
fix(train): tch-backend rpath build.rs + unreachable test thresholds + --val-dir / --eval-only (rebased residual) #1014
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
ruvnet
merged 1 commit into
ruvnet:main
from
stuinfla:fix/trainer-tch024-and-real-validation
Nov 11, 2026
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| //! Build script: when the `tch-backend` feature binds libtorch from the system | ||
| //! PyTorch install (`LIBTORCH_USE_PYTORCH=1`), embed an rpath to the PyTorch | ||
| //! `lib` directory so the produced binaries and test executables can locate | ||
| //! `libtorch_cpu` at runtime. torch-sys only adds the rpath to its own | ||
| //! cc-built static library, which does not propagate to downstream binaries on | ||
| //! macOS/Linux. A strict no-op otherwise (default / `--no-default-features` | ||
| //! builds, including the Python wheel, are unaffected). | ||
| fn main() { | ||
| println!("cargo:rerun-if-env-changed=LIBTORCH_USE_PYTORCH"); | ||
| println!("cargo:rerun-if-env-changed=PYTHON_SYS_EXECUTABLE"); | ||
| if std::env::var_os("CARGO_FEATURE_TCH_BACKEND").is_none() | ||
| || std::env::var_os("LIBTORCH_USE_PYTORCH").is_none() | ||
| { | ||
| return; | ||
| } | ||
| let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); | ||
| if target_os != "macos" && target_os != "linux" { | ||
| return; | ||
| } | ||
| // Same interpreter selection torch-sys uses, defaulting to python3. | ||
| let python = std::env::var("PYTHON_SYS_EXECUTABLE").unwrap_or_else(|_| "python3".into()); | ||
| let output = std::process::Command::new(python) | ||
| .args([ | ||
| "-c", | ||
| "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))", | ||
| ]) | ||
| .output(); | ||
| if let Ok(output) = output { | ||
| if output.status.success() { | ||
| let lib_dir = String::from_utf8_lossy(&output.stdout).trim().to_string(); | ||
| if !lib_dir.is_empty() { | ||
| // Plain rustc-link-arg (not -bins/-tests): it is the only | ||
| // variant that also reaches the lib unit-test binary. | ||
| println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}"); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -55,6 +55,29 @@ struct Args { | |||||||
| #[arg(long, value_name = "DIR")] | ||||||||
| data_dir: Option<PathBuf>, | ||||||||
|
|
||||||||
| /// MM-Fi directory to use as the REAL validation set (e.g. a held-out | ||||||||
| /// subject or session). Takes precedence over the automatic | ||||||||
| /// subject-disjoint split of `--data-dir`. Must share no subject with | ||||||||
| /// `--data-dir` (checked; overlap is refused as a train/val leak). | ||||||||
| #[arg(long, value_name = "DIR")] | ||||||||
| val_dir: Option<PathBuf>, | ||||||||
|
|
||||||||
| /// Evaluation-only mode: load `--checkpoint`, run the validation loop over | ||||||||
| /// `--val-dir` (no training), print PCK@0.2 / OKS and exit. Requires the | ||||||||
| /// `tch-backend` feature. | ||||||||
| #[arg(long, default_value_t = false, requires_all = ["checkpoint", "val_dir"])] | ||||||||
| eval_only: bool, | ||||||||
|
|
||||||||
| /// Checkpoint (tch VarStore `.pt` or `.safetensors`) to load in | ||||||||
| /// `--eval-only` mode. | ||||||||
| #[arg(long, value_name = "FILE", requires = "eval_only")] | ||||||||
| checkpoint: Option<PathBuf>, | ||||||||
|
|
||||||||
| /// In `--eval-only` mode, dump per-sample predicted + GT keypoints as JSONL | ||||||||
| /// to this path (dataset order, one line per window). | ||||||||
| #[arg(long, value_name = "FILE", requires = "eval_only")] | ||||||||
| dump_preds: Option<PathBuf>, | ||||||||
|
|
||||||||
| /// Override the checkpoint output directory from the config. | ||||||||
| #[arg(long, value_name = "DIR")] | ||||||||
| checkpoint_dir: Option<PathBuf>, | ||||||||
|
|
@@ -137,6 +160,22 @@ fn main() { | |||||||
|
|
||||||||
| log_config_summary(&config); | ||||||||
|
|
||||||||
| // ------------------------------------------------------------------ | ||||||||
| // Eval-only mode: load checkpoint, run the validation loop, exit | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||
| // ------------------------------------------------------------------ | ||||||||
|
|
||||||||
| if args.eval_only { | ||||||||
| // clap enforces `--checkpoint` and `--val-dir` alongside `--eval-only`. | ||||||||
| let (Some(ckpt), Some(val_dir)) = (args.checkpoint.as_deref(), args.val_dir.as_deref()) | ||||||||
| else { | ||||||||
| error!("--eval-only requires --checkpoint <FILE> and --val-dir <DIR>"); | ||||||||
| std::process::exit(2); | ||||||||
| }; | ||||||||
| let eval_ds = load_mmfi_or_exit(val_dir, &config, "eval"); | ||||||||
| run_eval(config, &eval_ds, ckpt, args.dump_preds.as_deref()); | ||||||||
| return; | ||||||||
| } | ||||||||
|
|
||||||||
| // ------------------------------------------------------------------ | ||||||||
| // Build datasets | ||||||||
| // ------------------------------------------------------------------ | ||||||||
|
|
@@ -204,6 +243,35 @@ fn main() { | |||||||
|
|
||||||||
| info!("Dataset: {} samples", train_ds.len()); | ||||||||
|
|
||||||||
| // An explicit held-out validation directory takes precedence over the | ||||||||
| // automatic split. Refuse any subject overlap with the training set: | ||||||||
| // that would be exactly the leak ADR-155 §Tier-1.2 forbids. | ||||||||
| if let Some(val_dir) = args.val_dir.as_deref() { | ||||||||
| let val_ds = load_mmfi_or_exit(val_dir, &config, "validation"); | ||||||||
| let train_subjects = train_ds.subjects(); | ||||||||
| let shared: Vec<u32> = val_ds | ||||||||
| .subjects() | ||||||||
| .into_iter() | ||||||||
| .filter(|s| train_subjects.contains(s)) | ||||||||
| .collect(); | ||||||||
| if !shared.is_empty() { | ||||||||
| error!( | ||||||||
| "--val-dir shares subject(s) {shared:?} with --data-dir; a \ | ||||||||
| subject-overlapping validation set leaks (ADR-155 §Tier-1.2). \ | ||||||||
| Use a held-out subject directory." | ||||||||
| ); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
| info!( | ||||||||
| "Using REAL held-out validation set: {} windows (subjects {:?}) from {}", | ||||||||
| val_ds.len(), | ||||||||
| val_ds.subjects(), | ||||||||
| val_dir.display() | ||||||||
| ); | ||||||||
| run_training(config, &train_ds, &val_ds); | ||||||||
| return; | ||||||||
| } | ||||||||
|
|
||||||||
| // ADR-155 §Tier-1.2: prefer a REAL, leak-free, subject-disjoint split so | ||||||||
| // any reported PCK/OKS is honest. MM-Fi windows are stride-1 (≈99% | ||||||||
| // overlap), so an index-level split would leak; a synthetic val set | ||||||||
|
|
@@ -295,6 +363,83 @@ fn run_training(_config: TrainingConfig, train_ds: &dyn CsiDataset, val_ds: &dyn | |||||||
| info!("Config and dataset infrastructure: OK"); | ||||||||
| } | ||||||||
|
|
||||||||
| // --------------------------------------------------------------------------- | ||||||||
| // --val-dir / --eval-only helpers | ||||||||
| // --------------------------------------------------------------------------- | ||||||||
|
|
||||||||
| /// Load an MM-Fi directory with the run's window/subcarrier/keypoint shape, or | ||||||||
| /// exit with an error if it cannot be loaded or holds no windows. | ||||||||
| fn load_mmfi_or_exit(dir: &std::path::Path, config: &TrainingConfig, what: &str) -> MmFiDataset { | ||||||||
| let ds = match MmFiDataset::discover( | ||||||||
| dir, | ||||||||
| config.window_frames, | ||||||||
| config.num_subcarriers, | ||||||||
| config.num_keypoints, | ||||||||
| ) { | ||||||||
| Ok(ds) => ds, | ||||||||
| Err(e) => { | ||||||||
| error!("Failed to load {what} dataset from {}: {e}", dir.display()); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
| }; | ||||||||
| if ds.is_empty() { | ||||||||
| error!("{what} dataset is empty — {}", dir.display()); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
| ds | ||||||||
| } | ||||||||
|
|
||||||||
| /// `--eval-only`: load a checkpoint into a fresh model and run the same | ||||||||
| /// validation loop training uses ([`Trainer::evaluate`]). | ||||||||
| #[cfg(feature = "tch-backend")] | ||||||||
| fn run_eval( | ||||||||
| config: TrainingConfig, | ||||||||
| ds: &dyn CsiDataset, | ||||||||
| checkpoint: &std::path::Path, | ||||||||
| dump: Option<&std::path::Path>, | ||||||||
| ) { | ||||||||
| use wifi_densepose_train::trainer::Trainer; | ||||||||
|
|
||||||||
| let mut trainer = Trainer::new(config); | ||||||||
| match trainer.load_checkpoint(checkpoint) { | ||||||||
| Ok(epoch) => info!("Loaded checkpoint {} (epoch {epoch})", checkpoint.display()), | ||||||||
| Err(e) => { | ||||||||
| error!("Failed to load checkpoint: {e}"); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
| } | ||||||||
| info!("Evaluating on {} windows", ds.len()); | ||||||||
|
|
||||||||
| let result = match dump { | ||||||||
| Some(path) => { | ||||||||
| info!("Dumping per-sample predictions to {}", path.display()); | ||||||||
| trainer.evaluate_with_dump(ds, path) | ||||||||
| } | ||||||||
| None => trainer.evaluate(ds), | ||||||||
| }; | ||||||||
| match result { | ||||||||
| Ok(m) => info!("EVAL RESULT: {}", m.summary()), | ||||||||
| Err(e) => { | ||||||||
| error!("Evaluation failed: {e}"); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| #[cfg(not(feature = "tch-backend"))] | ||||||||
| fn run_eval( | ||||||||
| _config: TrainingConfig, | ||||||||
| _ds: &dyn CsiDataset, | ||||||||
| _checkpoint: &std::path::Path, | ||||||||
| _dump: Option<&std::path::Path>, | ||||||||
| ) { | ||||||||
| error!( | ||||||||
| "--eval-only requires the `tch-backend` feature: \ | ||||||||
| cargo run --features tch-backend --bin train" | ||||||||
| ); | ||||||||
| std::process::exit(1); | ||||||||
| } | ||||||||
|
|
||||||||
| // --------------------------------------------------------------------------- | ||||||||
| // run_smoke_test — synthetic-validation path (NOT a reportable metric) | ||||||||
| // --------------------------------------------------------------------------- | ||||||||
|
|
||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@claude