added runtime evaluation

src/test/runtime_evaluation/evaluation.py

@@ -0,0 +1,201 @@
+import antimony
+import libsbml
+from pathlib import Path
+import re
+import time
+import csv
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+from scipy.stats import t
+
+
+COLORS = {
+    "sbml_to_antimony": "#0072B2",
+    "antimony_to_sbml": "#E69F00",
+}
+
+
+def load_results(results_file_name):
+    results_file = Path(results_file_name)
+
+    if results_file.exists():
+        with open(results_file) as f:
+            return list(csv.DictReader(f))
+
+    return None
+
+
+def get_sbml_models(models_dir):
+    return [
+        {
+            "path": p,
+            "biomodel_id": re.search(r"BIOMD\d+", str(p)).group(),
+            "file_size": p.stat().st_size,
+        }
+        for p in Path(models_dir).rglob("*.xml")
+    ]
+
+
+def save_results(results):
+    with open("evaluation_results.csv", "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=results[0].keys())
+        writer.writeheader()
+        writer.writerows(results)
+
+
+def benchmark():
+    results = load_results("evaluation_results.csv")
+    if results is not None:
+        return results
+
+    results = []
+    models = get_sbml_models("./large_biomodels")
+    for model in models:
+        sbml_document = libsbml.readSBMLFromFile(str(model["path"]))
+        sbml_string = libsbml.writeSBMLToString(sbml_document)
+
+        for repeat in range(10):
+            # SBML -> Antimony
+            antimony.clearPreviousLoads()
+            start = time.perf_counter()
+            antimony.loadSBMLString(sbml_string)
+            antimony_string = antimony.getAntimonyString(None)
+            t1 = time.perf_counter() - start
+
+            # Antimony -> SBML
+            antimony.clearPreviousLoads()
+            start = time.perf_counter()
+            antimony.loadAntimonyString(antimony_string)
+            module = antimony.getMainModuleName()
+            antimony.getSBMLString(module)
+            t2 = time.perf_counter() - start
+
+            results.append({
+                "biomodel_id": model["biomodel_id"],
+                "file_size": model["file_size"],
+                "repeat": repeat + 1,
+                "sbml_to_antimony_s": t1,
+                "antimony_to_sbml_s": t2,
+            })
+
+    save_results(results)
+    return results
+
+
+def process_results(results):
+    df = pd.DataFrame(results)
+
+    numeric_cols = [
+        "file_size",
+        "sbml_to_antimony_s",
+        "antimony_to_sbml_s",
+    ]
+    df[numeric_cols] = df[numeric_cols].apply(pd.to_numeric)
+
+    df["model"] = df["biomodel_id"].str.replace("BIOMD", "").astype(int)
+
+    return df.groupby("model").agg(
+        file_size=("file_size", "first"),
+        s2a_mean=("sbml_to_antimony_s", "mean"),
+        s2a_sem=("sbml_to_antimony_s", "sem"),
+        a2s_mean=("antimony_to_sbml_s", "mean"),
+        a2s_sem=("antimony_to_sbml_s", "sem"),
+        n=("sbml_to_antimony_s", "count"),
+    ).sort_values("file_size", ascending=False)
+
+
+def set_plot_style():
+    plt.rcParams.update({
+        "font.family": "sans-serif",
+        "font.sans-serif": ["Helvetica Neue", "Arial", "DejaVu Sans"],
+        "font.size": 11,
+        "axes.titlesize": 13,
+        "axes.labelsize": 11,
+        "xtick.labelsize": 9.5,
+        "ytick.labelsize": 9.5,
+        "legend.fontsize": 10,
+        "axes.edgecolor": "#888888",
+        "text.color": "#222222",
+        "axes.labelcolor": "#222222",
+        "xtick.color": "#444444",
+        "ytick.color": "#444444",
+    })
+
+
+def add_bars(ax, results, x, width, ci, error_style):
+    bars = [
+        ("s2a", -width / 2, "sbml_to_antimony", "SBML $\\rightarrow$ Antimony"),
+        ("a2s",  width / 2, "antimony_to_sbml", "Antimony $\\rightarrow$ SBML"),
+    ]
+
+    for prefix, offset, color, label in bars:
+        ax.bar(
+            x + offset,
+            results[f"{prefix}_mean"],
+            width,
+            yerr=ci * results[f"{prefix}_sem"],
+            error_kw=error_style,
+            color=COLORS[color],
+            alpha=0.9,
+            label=label,
+        )
+
+
+def format_axes(ax, results, x):
+    ax.set_xticks(x)
+    ax.set_xticklabels(results.index, fontsize=11)
+
+    for xi, size in zip(x, results["file_size"]):
+        ax.annotate(
+            f"({size / 1e6:.1f} MB)",
+            xy=(xi, 0),
+            xycoords=("data", "axes fraction"),
+            xytext=(0, -18),
+            textcoords="offset points",
+            ha="center",
+            va="top",
+            fontsize=9.5,
+            color="#444444",
+        )
+
+    ax.set_xlabel("BioModels ID (SBML file size)", labelpad=26)
+    ax.set_ylabel("Runtime (s)")
+    ax.spines[["top", "right"]].set_visible(False)
+    ax.spines[["left", "bottom"]].set_color("#888888")
+    ax.tick_params(axis="both", length=0)
+    ax.yaxis.grid(True, linewidth=0.6, alpha=0.15)
+    ax.set_axisbelow(True)
+    ax.legend(frameon=False, loc="upper right", fontsize=11)
+
+
+def plot_results(results):
+    set_plot_style()
+
+    ci = t.ppf(0.975, results["n"] - 1)
+    x = np.arange(len(results))
+    width = 0.38
+
+    fig, ax = plt.subplots(figsize=(10, 5.5))
+    error_style = {
+        "elinewidth": 1,
+        "ecolor": "#333333",
+        "capsize": 0,
+        "alpha": 0.8,
+    }
+    add_bars(ax, results, x, width, ci, error_style)
+    format_axes(ax, results, x)
+
+    fig.tight_layout()
+    fig.savefig("evaluation_results.png", bbox_inches="tight")
+    plt.show()
+
+
+def main():
+    results = benchmark()
+    processed_results = process_results(results)
+    plot_results(processed_results)
+
+
+if __name__ == "__main__":
+    main()

src/test/runtime_evaluation/evaluation_results.csv

@@ -0,0 +1,101 @@
+biomodel_id,file_size,repeat,sbml_to_antimony_s,antimony_to_sbml_s
+BIOMD0000000470,2692426,1,3.212773833423853,1.3017169162631035
+BIOMD0000000470,2692426,2,3.326765707693994,1.3592691663652658
+BIOMD0000000470,2692426,3,3.3125697914510965,1.3175972495228052
+BIOMD0000000470,2692426,4,3.39835958275944,1.3968776250258088
+BIOMD0000000470,2692426,5,3.394289250485599,1.4064087914302945
+BIOMD0000000470,2692426,6,3.4363871663808823,1.4702202919870615
+BIOMD0000000470,2692426,7,3.4166142493486404,1.3566110413521528
+BIOMD0000000470,2692426,8,3.424558916129172,1.3703443743288517
+BIOMD0000000470,2692426,9,3.4286542497575283,1.460510041564703
+BIOMD0000000470,2692426,10,3.453912124969065,1.3676570001989603
+BIOMD0000000497,2990743,1,2.3939167922362685,1.0680274162441492
+BIOMD0000000497,2990743,2,2.384712249971926,1.0147971250116825
+BIOMD0000000497,2990743,3,2.408152874559164,1.0301422085613012
+BIOMD0000000497,2990743,4,2.3828032091259956,1.0439454587176442
+BIOMD0000000497,2990743,5,2.4304674584418535,1.0426343334838748
+BIOMD0000000497,2990743,6,2.412156166508794,1.0183689994737506
+BIOMD0000000497,2990743,7,2.38263095729053,1.0226880414411426
+BIOMD0000000497,2990743,8,2.4051017072051764,1.042955375276506
+BIOMD0000000497,2990743,9,2.37899037450552,1.0605667913332582
+BIOMD0000000497,2990743,10,2.416272707283497,1.0276097087189555
+BIOMD0000000235,1920104,1,1.4192438749596477,1.3092668326571584
+BIOMD0000000235,1920104,2,1.444253332912922,1.2973423330113292
+BIOMD0000000235,1920104,3,1.4514105003327131,1.394904707558453
+BIOMD0000000235,1920104,4,1.3953018747270107,1.264883124269545
+BIOMD0000000235,1920104,5,1.4460438750684261,1.2940603327006102
+BIOMD0000000235,1920104,6,1.4569834172725677,1.2786425417289138
+BIOMD0000000235,1920104,7,1.4465779168531299,1.266563250683248
+BIOMD0000000235,1920104,8,1.46436308324337,1.245904041454196
+BIOMD0000000235,1920104,9,1.3906325418502092,1.1874207081273198
+BIOMD0000000235,1920104,10,1.458469333127141,1.2230471670627594
+BIOMD0000000496,2970772,1,2.378688333556056,1.000006791204214
+BIOMD0000000496,2970772,2,2.3712347503751516,1.0084619587287307
+BIOMD0000000496,2970772,3,2.36628704238683,1.0116182090714574
+BIOMD0000000496,2970772,4,2.42696304153651,1.0064173340797424
+BIOMD0000000496,2970772,5,2.3554868325591087,1.0012326668947935
+BIOMD0000000496,2970772,6,2.3571212496608496,1.0029447078704834
+BIOMD0000000496,2970772,7,2.4030913757160306,1.007418374530971
+BIOMD0000000496,2970772,8,2.3769887909293175,0.987738249823451
+BIOMD0000000496,2970772,9,2.3648392912000418,0.99305041693151
+BIOMD0000000496,2970772,10,2.356204957701266,0.9857649160549045
+BIOMD0000001063,12085389,1,10.997095917351544,10.210589250549674
+BIOMD0000001063,12085389,2,10.36745133344084,9.962318957783282
+BIOMD0000001063,12085389,3,10.624579540453851,11.167006624862552
+BIOMD0000001063,12085389,4,10.444218958728015,11.173593875020742
+BIOMD0000001063,12085389,5,10.645713042467833,12.198524791747332
+BIOMD0000001063,12085389,6,11.065570291131735,9.638518375344574
+BIOMD0000001063,12085389,7,10.712383375503123,9.874975667335093
+BIOMD0000001063,12085389,8,9.359557708725333,9.423894625157118
+BIOMD0000001063,12085389,9,11.977634583599865,9.516215625219047
+BIOMD0000001063,12085389,10,13.00681862514466,12.05973166693002
+BIOMD0000001062,11801832,1,7.793687708675861,6.884549374692142
+BIOMD0000001062,11801832,2,8.201154874637723,6.668806582689285
+BIOMD0000001062,11801832,3,7.577913083136082,6.811934916302562
+BIOMD0000001062,11801832,4,7.57874858379364,6.5120872082188725
+BIOMD0000001062,11801832,5,7.546625958755612,6.916838415898383
+BIOMD0000001062,11801832,6,7.336775708943605,6.614929624833167
+BIOMD0000001062,11801832,7,8.29043583292514,6.670048833824694
+BIOMD0000001062,11801832,8,8.20492083299905,7.060642667114735
+BIOMD0000001062,11801832,9,7.985298791900277,6.844037124887109
+BIOMD0000001062,11801832,10,7.976388582959771,6.991870624944568
+BIOMD0000000472,2075342,1,2.5015401244163513,0.9825301673263311
+BIOMD0000000472,2075342,2,2.5997344590723515,0.945056083612144
+BIOMD0000000472,2075342,3,2.46114429179579,0.9338538330048323
+BIOMD0000000472,2075342,4,2.48821850027889,0.9620546251535416
+BIOMD0000000472,2075342,5,2.4783693747594953,1.0010205823928118
+BIOMD0000000472,2075342,6,2.559977541677654,0.9290708750486374
+BIOMD0000000472,2075342,7,2.425911749713123,0.9284449173137546
+BIOMD0000000472,2075342,8,2.4779784167185426,0.9553226670250297
+BIOMD0000000472,2075342,9,2.4568369165062904,0.9566129995509982
+BIOMD0000000472,2075342,10,2.455169625580311,0.95647266600281
+BIOMD0000000473,2256315,1,2.5742618339136243,0.9974809996783733
+BIOMD0000000473,2256315,2,2.539823042228818,0.9979968750849366
+BIOMD0000000473,2256315,3,2.5622578747570515,0.9690846242010593
+BIOMD0000000473,2256315,4,2.590264375321567,0.981288124807179
+BIOMD0000000473,2256315,5,2.5305427089333534,1.0029274998232722
+BIOMD0000000473,2256315,6,2.5600485000759363,0.9880852922797203
+BIOMD0000000473,2256315,7,2.5849630841985345,0.9902595421299338
+BIOMD0000000473,2256315,8,2.6174014573916793,1.0689068334177136
+BIOMD0000000473,2256315,9,2.6495518339797854,1.0134632075205445
+BIOMD0000000473,2256315,10,2.5666908342391253,1.0185406254604459
+BIOMD0000000469,2335963,1,3.4232753748074174,1.4929964151233435
+BIOMD0000000469,2335963,2,3.3989814585074782,1.5027222083881497
+BIOMD0000000469,2335963,3,3.4534294176846743,1.5024404590949416
+BIOMD0000000469,2335963,4,3.437159833498299,1.4965718751773238
+BIOMD0000000469,2335963,5,3.4930639173835516,1.4460595836862922
+BIOMD0000000469,2335963,6,3.429328833706677,1.548036958090961
+BIOMD0000000469,2335963,7,3.417153916321695,1.5130366254597902
+BIOMD0000000469,2335963,8,3.573352249339223,1.5267899166792631
+BIOMD0000000469,2335963,9,3.4748254995793104,1.5553724588826299
+BIOMD0000000469,2335963,10,3.4255086667835712,1.461430124938488
+BIOMD0000001061,4454287,1,2.4210686665028334,2.182412374764681
+BIOMD0000001061,4454287,2,2.4336356669664383,2.1524445405229926
+BIOMD0000001061,4454287,3,2.451794500462711,2.156989832408726
+BIOMD0000001061,4454287,4,2.507272000424564,2.135230374522507
+BIOMD0000001061,4454287,5,2.4566615829244256,2.152782291173935
+BIOMD0000001061,4454287,6,2.4916114574298263,2.170709041878581
+BIOMD0000001061,4454287,7,2.470538209192455,2.10925433319062
+BIOMD0000001061,4454287,8,2.4755009170621634,2.091311374679208
+BIOMD0000001061,4454287,9,2.454078665934503,2.1980863753706217
+BIOMD0000001061,4454287,10,2.3942296663299203,2.194705666974187