From 975f771ec2a3b457cbf6e1dd8909867c54f977be Mon Sep 17 00:00:00 2001 From: martin-velay Date: Wed, 25 Mar 2026 16:31:07 +0100 Subject: [PATCH 1/2] [src] Integrate DVPlan tool commands - Integrate DVPlan subcommands. These commands will be run only if: - DVPlan tool is available - Coverage is enabled - A "".vplan" file is found in the cuurent sim_cfg.hjson file - When the condition are met, at the end the regression/simulation, the DVPlan tool will: - Check the coverage of the specification - Generate a derived vPlan in the simulation directory - Perform back-annotation of the vPlan with the coverage data - Compute the coverage of the vPlan - Generate a report based on obtained annotated vPlan Signed-off-by: martin-velay --- src/dvsim/job/deploy.py | 79 ++++++++++++++++++++++++++++++++++++++++- src/dvsim/sim/data.py | 2 ++ src/dvsim/sim/flow.py | 17 +++++++++ src/dvsim/sim/report.py | 4 ++- 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/src/dvsim/job/deploy.py b/src/dvsim/job/deploy.py index 42aeb55a..41caee37 100644 --- a/src/dvsim/job/deploy.py +++ b/src/dvsim/job/deploy.py @@ -30,9 +30,9 @@ from dvsim.modes import BuildMode from dvsim.sim.flow import SimCfg - __all__ = ( "CompileSim", + "CovVPlan", "Deploy", ) @@ -1028,3 +1028,80 @@ def _set_attrs(self) -> None: self.qual_name = self.target self.full_name = f"{self.sim_cfg.name}{self._variant_suffix}:{self.qual_name}" self.input_dirs += [self.cov_merge_db_dir] + + +class CovVPlan(Deploy): + """Abstraction for generating a Verification Plan (vPlan) report using DVPlan.""" + + target = "cov_vplan" + weight = 10 + + def __init__(self, cov_report_job, sim_cfg) -> None: + self.report_job = cov_report_job + super().__init__(sim_cfg) + self.dependencies.append(cov_report_job) + + def _define_attrs(self) -> None: + super()._define_attrs() + self.mandatory_cmd_attrs.update( + { + "proj_root": False, + "vplan": False, + } + ) + self.mandatory_misc_attrs.update( + { + "dut_instance": False, + } + ) + + def _set_attrs(self) -> None: + self.cov_vplan_dir = f"{self.sim_cfg.scratch_path}/{self.target}" + + super()._set_attrs() + self.qual_name = self.target + self.full_name = f"{self.sim_cfg.name}{self._variant_suffix}:{self.qual_name}" + + self.prepare_opts = self.sim_cfg.cov_vplan_prepare_opts + self.process_opts = self.sim_cfg.cov_vplan_process_opts + + # Calculate IP root. + vplan_path = Path(self.vplan) + self.ip_root = str(vplan_path.parent.parent) + + # Use fixed output filenames so the report location is always predictable. + self.annotated_hjson = f"{self.odir}/vplan_annotated.hjson" + self.gen_html = f"{self.odir}/vplan_annotated.html" + self.output_dirs = [self.odir] + + def _construct_cmd(self) -> str: + """Construct the pure bash shell command, bypassing the base Makefile assumption.""" + import shlex + import shutil + + if shutil.which("dvplan") is None: + fallback = ( + "echo 'WARNING: dvplan tool not installed in PATH. Skipping vPlan generation.'" + ) + return f"/usr/bin/env bash -c {shlex.quote(fallback)}" + + def format_opts(opts): + return " ".join(opts) if isinstance(opts, list) else str(opts) + + prepare_opts_str = format_opts(self.prepare_opts) + process_opts_str = format_opts(self.process_opts) + + prepare_cmd = f"dvplan prepare_vplan {prepare_opts_str} {self.ip_root} {self.vplan} {self.annotated_hjson}" + prepare_cmd = " ".join(prepare_cmd.split()) + + vendor_tool = f"{self.sim_cfg.tool}_report" + report_path = self.report_job.cov_report_dir + + process_cmd = ( + f"dvplan process_results {process_opts_str} --coverage {vendor_tool} {report_path} " + f"-R {self.gen_html} -s {self.sim_cfg.name} {self.dut_instance} {self.annotated_hjson}" + ) + process_cmd = " ".join(process_cmd.split()) + + full_command = f"set -e; mkdir -p {self.odir}; {prepare_cmd} && {process_cmd}" + return f"/usr/bin/env bash -c {shlex.quote(full_command)}" diff --git a/src/dvsim/sim/data.py b/src/dvsim/sim/data.py index f9427da4..7b40dbff 100644 --- a/src/dvsim/sim/data.py +++ b/src/dvsim/sim/data.py @@ -192,6 +192,8 @@ class SimFlowResults(BaseModel): """Coverage metrics.""" cov_report_page: Path | None """Optional path linking to the generated coverage report dashboard page.""" + vplan_report_page: Path | None + """Optional path linking to the generated verification plan (vPlan) reports.""" failed_jobs: BucketedFailures """Bucketed failed job overview.""" diff --git a/src/dvsim/sim/flow.py b/src/dvsim/sim/flow.py index f02ed9da..c3bc6239 100644 --- a/src/dvsim/sim/flow.py +++ b/src/dvsim/sim/flow.py @@ -22,6 +22,7 @@ CovMerge, CovReport, CovUnr, + CovVPlan, RunTest, ) from dvsim.job.status import JobStatus @@ -157,6 +158,13 @@ def __init__(self, flow_cfg_file, hjson_data, args, mk_config) -> None: self.cov_report_dir = "" self.cov_report_page = "" + # Options for vPlan processing + # dut_instance is the hierarchical testbench path to the DUT (e.g. "tb.dut"), + # distinct from `name`/`qual_name` which identify the sim config itself. + self.dut_instance = "" + self.cov_vplan_prepare_opts = [] + self.cov_vplan_process_opts = [] + # Options from tools - for building and running tests self.build_cmd = "" self.flist_gen_cmd = "" @@ -550,6 +558,10 @@ def _create_deploy_objects(self) -> None: self.cov_report_deploy = CovReport(self.cov_merge_deploy, self) self.deploy += [self.cov_merge_deploy, self.cov_report_deploy] + if getattr(self, "vplan", False): + self.cov_vplan_deploy = CovVPlan(self.cov_report_deploy, self) + self.deploy.append(self.cov_vplan_deploy) + def _cov_analyze(self) -> None: """Open GUI tool for coverage analysis. @@ -821,6 +833,10 @@ def make_test_result(tr) -> TestResult | None: cov_report_dir = self.cov_report_dir or "cov_report" cov_report_page = Path(cov_report_dir, self.cov_report_page) + vplan_report_page = None + if getattr(self, "cov_vplan_deploy", None): + vplan_report_page = Path(self.scratch_path) / CovVPlan.target / "vplan_annotated.html" + failures = BucketedFailures.from_job_status(results=run_results) if failures.buckets: self.errors_seen = True @@ -835,6 +851,7 @@ def make_test_result(tr) -> TestResult | None: stages=stages, coverage=coverage_model, cov_report_page=cov_report_page, + vplan_report_page=vplan_report_page, failed_jobs=failures, passed=total_passed, total=total_runs, diff --git a/src/dvsim/sim/report.py b/src/dvsim/sim/report.py index 56824ac3..60b0a268 100644 --- a/src/dvsim/sim/report.py +++ b/src/dvsim/sim/report.py @@ -329,10 +329,12 @@ def render_coverage_table(self, results: SimFlowResults) -> str: for k, v in results.coverage.flattened().items() if v is not None } - if not cov_results and not results.cov_report_page: + if not cov_results and not results.cov_report_page and not results.vplan_report_page: return "" report_md = "## Coverage Results" + if results.vplan_report_page: + report_md += f"\n### [vPlan Dashboard]({results.vplan_report_page})" if results.cov_report_page: report_md += f"\n### [Coverage Dashboard]({results.cov_report_page})" if cov_results: From c48e69e639a75787aca36d0181f77e6104640d27 Mon Sep 17 00:00:00 2001 From: martin-velay Date: Wed, 22 Apr 2026 17:36:46 +0200 Subject: [PATCH 2/2] [src] Add vPlan coverage to final summary report Signed-off-by: martin-velay --- src/dvsim/job/deploy.py | 27 +++++++++++++++++++++++++++ src/dvsim/sim/data.py | 2 ++ src/dvsim/sim/flow.py | 3 +++ src/dvsim/sim/report.py | 2 ++ 4 files changed, 34 insertions(+) diff --git a/src/dvsim/job/deploy.py b/src/dvsim/job/deploy.py index 41caee37..b7a86345 100644 --- a/src/dvsim/job/deploy.py +++ b/src/dvsim/job/deploy.py @@ -1038,6 +1038,8 @@ class CovVPlan(Deploy): def __init__(self, cov_report_job, sim_cfg) -> None: self.report_job = cov_report_job + # Populated by post_finish() once the job completes successfully. + self.vplan_coverage: float | None = None super().__init__(sim_cfg) self.dependencies.append(cov_report_job) @@ -1074,6 +1076,31 @@ def _set_attrs(self) -> None: self.gen_html = f"{self.odir}/vplan_annotated.html" self.output_dirs = [self.odir] + def post_finish(self) -> Callable[[JobStatus], None]: + """Get post finish callback.""" + + def callback(status: JobStatus) -> None: + """Extract the overall vPlan normalised coverage from the annotated HJSON.""" + if self.dry_run or status != JobStatus.PASSED: + return + hjson_path = Path(self.annotated_hjson) + if not hjson_path.exists(): + return + try: + import hjson # noqa: PLC0415 + + with hjson_path.open() as f: + data = hjson.load(f) + # HJSON vPlans are keyed: {dut_name: {fields...}} + root_node = next(iter(data.values()), {}) + raw = root_node.get("Normalized_Coverage") + if raw is not None: + self.vplan_coverage = float(str(raw).rstrip(" %")) + except Exception: # noqa: BLE001 + log.debug("Could not extract vPlan coverage from '%s'.", hjson_path) + + return callback + def _construct_cmd(self) -> str: """Construct the pure bash shell command, bypassing the base Makefile assumption.""" import shlex diff --git a/src/dvsim/sim/data.py b/src/dvsim/sim/data.py index 7b40dbff..888dfad1 100644 --- a/src/dvsim/sim/data.py +++ b/src/dvsim/sim/data.py @@ -194,6 +194,8 @@ class SimFlowResults(BaseModel): """Optional path linking to the generated coverage report dashboard page.""" vplan_report_page: Path | None """Optional path linking to the generated verification plan (vPlan) reports.""" + vplan_coverage: float | None = None + """Overall normalised coverage (%) extracted from the back-annotated vPlan.""" failed_jobs: BucketedFailures """Bucketed failed job overview.""" diff --git a/src/dvsim/sim/flow.py b/src/dvsim/sim/flow.py index c3bc6239..bbd907a0 100644 --- a/src/dvsim/sim/flow.py +++ b/src/dvsim/sim/flow.py @@ -834,8 +834,10 @@ def make_test_result(tr) -> TestResult | None: cov_report_page = Path(cov_report_dir, self.cov_report_page) vplan_report_page = None + vplan_coverage = None if getattr(self, "cov_vplan_deploy", None): vplan_report_page = Path(self.scratch_path) / CovVPlan.target / "vplan_annotated.html" + vplan_coverage = self.cov_vplan_deploy.vplan_coverage failures = BucketedFailures.from_job_status(results=run_results) if failures.buckets: @@ -852,6 +854,7 @@ def make_test_result(tr) -> TestResult | None: coverage=coverage_model, cov_report_page=cov_report_page, vplan_report_page=vplan_report_page, + vplan_coverage=vplan_coverage, failed_jobs=failures, passed=total_passed, total=total_runs, diff --git a/src/dvsim/sim/report.py b/src/dvsim/sim/report.py index 60b0a268..8fc08c45 100644 --- a/src/dvsim/sim/report.py +++ b/src/dvsim/sim/report.py @@ -335,6 +335,8 @@ def render_coverage_table(self, results: SimFlowResults) -> str: report_md = "## Coverage Results" if results.vplan_report_page: report_md += f"\n### [vPlan Dashboard]({results.vplan_report_page})" + if results.vplan_coverage is not None: + report_md += f"\n\nVerification Plan Coverage: {results.vplan_coverage:.2f} %\n" if results.cov_report_page: report_md += f"\n### [Coverage Dashboard]({results.cov_report_page})" if cov_results: