From a617036df43d014034e46cab91eebcd19abb3c38 Mon Sep 17 00:00:00 2001 From: Lukas Burgholzer Date: Mon, 27 Jul 2026 15:59:33 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Add=20MLIR=20mapping=20ben?= =?UTF-8?q?chmark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add deterministic frontier- and routing-heavy workloads plus a paired A/B runner that records raw timings, build metadata, robust summaries, and bootstrap confidence intervals. Document correctness and route-quality validation limits separately from timing. Assisted-by: GPT-5.6 via Codex --- eval/CMakeLists.txt | 7 + eval/mapping_evaluation.md | 38 ++++++ eval/mapping_evaluation.py | 273 +++++++++++++++++++++++++++++++++++++ eval/mlir_mapping.cpp | 189 +++++++++++++++++++++++++ pyproject.toml | 1 + 5 files changed, 508 insertions(+) create mode 100644 eval/mapping_evaluation.md create mode 100644 eval/mapping_evaluation.py create mode 100644 eval/mlir_mapping.cpp diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 4ae857d0a9..9e1b17e38a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -10,3 +10,10 @@ add_executable(mqt-core-dd-eval eval_dd_package.cpp) target_link_libraries( mqt-core-dd-eval PRIVATE MQT::CoreDD MQT::CoreAlgorithms MQT::CoreCircuitOptimizer MQT::ProjectOptions MQT::ProjectWarnings) + +if(BUILD_MQT_CORE_MLIR) + add_executable(mqt-core-mlir-mapping-eval mlir_mapping.cpp) + target_link_libraries( + mqt-core-mlir-mapping-eval PRIVATE MLIRParser MLIRQCOProgramBuilder MLIRQTensorUtils + MLIRQCOTransforms MQT::ProjectOptions MQT::ProjectWarnings) +endif() diff --git a/eval/mapping_evaluation.md b/eval/mapping_evaluation.md new file mode 100644 index 0000000000..d2c8042420 --- /dev/null +++ b/eval/mapping_evaluation.md @@ -0,0 +1,38 @@ +# MLIR mapping performance evaluation + +The `mqt-core-mlir-mapping-eval` executable times a pass manager containing only +the QCO mapping pass. Program construction, MLIR context initialization, and +module verification happen outside the timed interval. The generated circuits +and mapper seed are deterministic. The default mapper configuration matches the +PR #1930 evaluation: 20 lookahead steps, lambda 0.5, one refinement iteration, +and 18 initial-layout trials. + +Configure and build an optimized executable with: + +```console +MLIR_DIR=/path/to/llvm/lib/cmake/mlir cmake --preset release \ + -DBUILD_MQT_CORE_BENCHMARKS=ON +cmake --build build/release --target mqt-core-mlir-mapping-eval +``` + +To compare executables from two worktrees, run: + +```console +python eval/mapping_evaluation.py \ + /path/to/baseline/build/release/eval/mqt-core-mlir-mapping-eval \ + /path/to/candidate/build/release/eval/mqt-core-mlir-mapping-eval \ + --output mapping-results.json +``` + +The runner randomizes the execution order within each pair. Its JSON output +contains the raw nanosecond samples, source-worktree revisions and dirty states, +executable SHA-256 digests, CMake and compiler metadata, medians, median +absolute deviations, 10% trimmed means, and paired median speedups with a +bootstrap 95% confidence interval. Use the same machine without other +substantial workloads for both executables. + +Do not use byte-for-byte mapped IR equality as a cross-process correctness +check. Ready operations are stored in a pointer-keyed `DenseMap`, so address +layout can change their iteration order and select an equally valid alternative +route. Validate mapping correctness with focused tests and evaluate route +quality separately from pass execution time. diff --git a/eval/mapping_evaluation.py b/eval/mapping_evaluation.py new file mode 100644 index 0000000000..af11ac657d --- /dev/null +++ b/eval/mapping_evaluation.py @@ -0,0 +1,273 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Run paired A/B measurements of the MLIR mapping benchmark.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import random +import statistics +import subprocess +import sys +from pathlib import Path +from typing import Any + +SCENARIOS = ("frontier", "routing") + + +def run(command: list[str], *, cwd: Path | None = None) -> str: + """Run a command. + + Returns: + The command's standard output without surrounding whitespace. + """ + return subprocess.run( # ruff:ignore[subprocess-without-shell-equals-true] + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def worktree_metadata(executable: Path) -> dict[str, str | bool]: + """Return source-worktree metadata associated with an executable path.""" + directory = str(executable.parent) + return { + "revision": run(["git", "-C", directory, "rev-parse", "HEAD"]), + "dirty": bool(run(["git", "-C", directory, "status", "--porcelain"])), + } + + +def executable_digest(executable: Path) -> str: + """Return the SHA-256 digest of an executable.""" + digest = hashlib.sha256() + with executable.open("rb") as file: + while chunk := file.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def cmake_metadata(executable: Path) -> dict[str, str]: + """Read compiler and MLIR metadata from the executable's CMake cache. + + Returns: + Available build type, compiler, and MLIR configuration values. + """ + cache = executable.parents[1] / "CMakeCache.txt" + if not cache.exists(): + return {} + + values: dict[str, str] = {} + for line in cache.read_text(encoding="utf-8").splitlines(): + key, separator, value = line.partition("=") + if separator and key.split(":", maxsplit=1)[0] in { + "CMAKE_BUILD_TYPE", + "CMAKE_CXX_COMPILER", + "MLIR_DIR", + }: + values[key.split(":", maxsplit=1)[0]] = value + + compiler = values.get("CMAKE_CXX_COMPILER") + if compiler: + values["CMAKE_CXX_COMPILER_VERSION"] = run([compiler, "--version"]).splitlines()[0] + return values + + +def measure( + executable: Path, + *, + scenario: str, + qubits: int, + layers: int, + seed: int, + lookahead: int, + lambda_: float, + iterations: int, + trials: int, +) -> int: + """Measure one mapping pass execution. + + Returns: + The elapsed time in nanoseconds. + """ + output = run([ + str(executable), + f"--scenario={scenario}", + f"--qubits={qubits}", + f"--layers={layers}", + f"--seed={seed}", + f"--lookahead={lookahead}", + f"--lambda={lambda_}", + f"--iterations={iterations}", + f"--trials={trials}", + ]) + return int(output) + + +def trimmed_mean(samples: list[float], proportion: float = 0.1) -> float: + """Return a symmetrically trimmed mean. + + Returns: + The mean after removing the requested proportion from both tails. + """ + ordered = sorted(samples) + count = int(len(ordered) * proportion) + selected = ordered[count:-count] if count else ordered + return statistics.fmean(selected) + + +def summarize(samples: list[int]) -> dict[str, float]: + """Summarize samples. + + Returns: + Robust summary statistics in milliseconds. + """ + milliseconds = [sample / 1_000_000 for sample in samples] + median = statistics.median(milliseconds) + return { + "median_ms": median, + "mad_ms": statistics.median(abs(sample - median) for sample in milliseconds), + "trimmed_mean_ms": trimmed_mean(milliseconds), + } + + +def paired_speedup(baseline: list[int], candidate: list[int], *, seed: int) -> dict[str, float]: + """Summarize paired speedups. + + Returns: + The median ratio and its bootstrap 95% confidence interval. + """ + ratios = [before / after for before, after in zip(baseline, candidate, strict=True)] + point = statistics.median(ratios) + rng = random.Random(seed) # ruff:ignore[suspicious-non-cryptographic-random-usage] + bootstraps = sorted(statistics.median(rng.choices(ratios, k=len(ratios))) for _ in range(10_000)) + return { + "median_ratio": point, + "percent": (point - 1) * 100, + "bootstrap_95_percent_low": (bootstraps[249] - 1) * 100, + "bootstrap_95_percent_high": (bootstraps[9749] - 1) * 100, + } + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments. + + Returns: + The parsed arguments. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidate", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--samples", type=int, default=25) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--qubits", type=int, default=36) + parser.add_argument("--layers", type=int, default=120) + parser.add_argument("--seed", type=int, default=1930) + parser.add_argument("--lookahead", type=int, default=20) + parser.add_argument("--lambda", dest="lambda_", type=float, default=0.5) + parser.add_argument("--iterations", type=int, default=1) + parser.add_argument("--trials", type=int, default=18) + parser.add_argument("--scenario", choices=SCENARIOS, action="append") + return parser.parse_args() + + +def main() -> None: + """Run the benchmark comparison and write raw and summarized results.""" + args = parse_args() + scenarios = args.scenario or list(SCENARIOS) + executables = { + "baseline": args.baseline.resolve(), + "candidate": args.candidate.resolve(), + } + results: dict[str, Any] = { + "configuration": { + "samples": args.samples, + "warmups": args.warmups, + "qubits": args.qubits, + "layers": args.layers, + "seed": args.seed, + "lookahead": args.lookahead, + "lambda": args.lambda_, + "iterations": args.iterations, + "trials": args.trials, + }, + "executables": { + name: { + "path": str(executable), + "worktree": worktree_metadata(executable), + "sha256": executable_digest(executable), + "cmake": cmake_metadata(executable), + } + for name, executable in executables.items() + }, + "scenarios": {}, + } + + order_rng = random.Random(args.seed) # ruff:ignore[suspicious-non-cryptographic-random-usage] + for scenario in scenarios: + for _ in range(args.warmups): + for executable in executables.values(): + measure( + executable, + scenario=scenario, + qubits=args.qubits, + layers=args.layers, + seed=args.seed, + lookahead=args.lookahead, + lambda_=args.lambda_, + iterations=args.iterations, + trials=args.trials, + ) + + raw = {"baseline": [], "candidate": []} + orders: list[list[str]] = [] + for _ in range(args.samples): + order = list(executables) + order_rng.shuffle(order) + orders.append(order) + for name in order: + raw[name].append( + measure( + executables[name], + scenario=scenario, + qubits=args.qubits, + layers=args.layers, + seed=args.seed, + lookahead=args.lookahead, + lambda_=args.lambda_, + iterations=args.iterations, + trials=args.trials, + ) + ) + + results["scenarios"][scenario] = { + "execution_order": orders, + "raw_nanoseconds": raw, + "baseline": summarize(raw["baseline"]), + "candidate": summarize(raw["candidate"]), + "speedup": paired_speedup(raw["baseline"], raw["candidate"], seed=args.seed), + } + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + for scenario, result in results["scenarios"].items(): + speedup = result["speedup"] + sys.stdout.write( + f"{scenario}: {speedup['percent']:+.2f}% " + f"[{speedup['bootstrap_95_percent_low']:+.2f}%, " + f"{speedup['bootstrap_95_percent_high']:+.2f}%]\n" + ) + + +if __name__ == "__main__": + main() diff --git a/eval/mlir_mapping.cpp b/eval/mlir_mapping.cpp new file mode 100644 index 0000000000..5ae2b1b69f --- /dev/null +++ b/eval/mlir_mapping.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2023 - 2026 Chair for Design Automation, TUM + * Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH + * All rights reserved. + * + * SPDX-License-Identifier: MIT + * + * Licensed under the MIT License + */ + +#include "mlir/Dialect/QCO/Builder/QCOProgramBuilder.h" +#include "mlir/Dialect/QCO/IR/QCODialect.h" +#include "mlir/Dialect/QCO/Transforms/Mapping/Mapping.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlir; +using namespace mlir::qco; + +namespace { + +llvm::cl::opt + scenario("scenario", llvm::cl::desc("Workload: frontier or routing"), + llvm::cl::value_desc("name"), llvm::cl::init("frontier")); +llvm::cl::opt nqubits("qubits", llvm::cl::desc("Number of qubits"), + llvm::cl::init(36)); +llvm::cl::opt nlayers("layers", llvm::cl::desc("Number of gate layers"), + llvm::cl::init(120)); +llvm::cl::opt seed("seed", llvm::cl::desc("Deterministic random seed"), + llvm::cl::init(1930)); +llvm::cl::opt + nlookahead("lookahead", llvm::cl::desc("Number of mapping lookahead steps"), + llvm::cl::init(20)); +llvm::cl::opt lambda("lambda", + llvm::cl::desc("Mapping cost decay factor"), + llvm::cl::init(0.5F)); +llvm::cl::opt + niterations("iterations", + llvm::cl::desc("Number of mapping refinement iterations"), + llvm::cl::init(1)); +llvm::cl::opt ntrials("trials", + llvm::cl::desc("Number of initial layout trials"), + llvm::cl::init(18)); + +using CouplingSet = llvm::DenseSet>; + +CouplingSet makeGrid(const size_t count) { + const auto width = + static_cast(std::ceil(std::sqrt(static_cast(count)))); + CouplingSet couplingSet; + for (size_t i = 0; i < count; ++i) { + if (i % width + 1 < width && i + 1 < count) { + couplingSet.insert({i, i + 1}); + couplingSet.insert({i + 1, i}); + } + if (i + width < count) { + couplingSet.insert({i, i + width}); + couplingSet.insert({i + width, i}); + } + } + return couplingSet; +} + +OwningOpRef makeProgram(MLIRContext& context) { + QCOProgramBuilder builder(&context); + builder.initialize(llvm::SmallVector(nqubits, builder.getI1Type())); + + Value tensor = builder.qtensorAlloc(static_cast(nqubits)); + llvm::SmallVector qubits(nqubits); + llvm::SmallVector bits(nqubits); + for (size_t i = 0; i < nqubits; ++i) { + std::tie(tensor, qubits[i]) = + builder.qtensorExtract(tensor, static_cast(i)); + } + + std::mt19937_64 rng(seed); + llvm::SmallVector order(nqubits); + std::iota(order.begin(), order.end(), 0); + + if (scenario == "frontier") { + for (size_t layer = 0; layer < nlayers; ++layer) { + std::ranges::shuffle(order, rng); + for (size_t i = 0; i + 1 < order.size(); i += 2) { + const auto first = order[i]; + const auto second = order[i + 1]; + if (layer % 2 == 0) { + std::tie(qubits[first], qubits[second]) = + builder.cx(qubits[first], qubits[second]); + } else { + std::tie(qubits[first], qubits[second]) = + builder.cz(qubits[first], qubits[second]); + } + } + } + } else if (scenario == "routing") { + size_t active = 0; + for (size_t layer = 0; layer < nlayers; ++layer) { + size_t target = active; + while (target == active) { + target = rng() % nqubits; + } + std::tie(qubits[active], qubits[target]) = + builder.cx(qubits[active], qubits[target]); + active = target; + qubits[active] = builder.h(qubits[active]); + } + } else { + llvm::errs() << "unknown scenario: " << scenario << '\n'; + return {}; + } + + qubits = builder.barrier(qubits); + for (size_t i = 0; i < nqubits; ++i) { + std::tie(qubits[i], bits[i]) = builder.measure(qubits[i]); + tensor = builder.qtensorInsert(qubits[i], tensor, static_cast(i)); + } + builder.qtensorDealloc(tensor); + return builder.finalize(bits); +} + +} // namespace + +int main(int argc, char** argv) { + llvm::cl::ParseCommandLineOptions(argc, argv); + if (nqubits < 2) { + llvm::errs() << "--qubits must be at least 2\n"; + return 2; + } + + DialectRegistry registry; + registry.insert(); + MLIRContext context(registry); + context.loadAllAvailableDialects(); + + auto module = makeProgram(context); + if (!module) { + return 2; + } + + PassManager pm(&context); + pm.enableVerifier(false); + pm.addPass( + createMappingPass(makeGrid(nqubits), MappingPassOptions{ + .nlookahead = nlookahead, + .lambda = lambda, + .niterations = niterations, + .ntrials = ntrials, + .seed = seed, + })); + + const auto start = std::chrono::steady_clock::now(); + const auto result = pm.run(module.get()); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + if (failed(result)) { + llvm::errs() << "mapping failed\n"; + return 1; + } + if (failed(verify(module.get()))) { + llvm::errs() << "mapped module verification failed\n"; + return 1; + } + std::cout << elapsed.count() << '\n'; + return 0; +} diff --git a/pyproject.toml b/pyproject.toml index 013cd3eb87..5ad055c68d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,6 +242,7 @@ known-first-party = ["mqt.core"] "python/mqt/core/_compat/**.py" = ["TID251", "A005"] "python/mqt/core/__main__.py" = ["T201"] "eval/dd_evaluation.py" = ["T201"] +"eval/mapping_evaluation.py" = ["INP001"] [tool.ruff.lint.pydocstyle] convention = "google"