Run STF tests using only one VM

README.md

@@ -168,6 +168,25 @@ all relevant ones can be easily checked out from [our test vectors repository](h
 Obviously it's also possible to run just single test case or part of the test
 cases by altering the glob pattern in the path.
 
+#### Selecting PVM Backend
+
+By default, test vectors are run with both PVM backends (built-in and Ananas).
+You can select a specific PVM backend using the `--pvm` option:
+
+```bash
+# Run tests with built-in PVM only
+$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm builtin
+
+# Run tests with Ananas PVM only
+$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm ananas
+
+# Run tests with both PVMs (default)
+$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner
+```
+
+This option is useful for debugging PVM-specific issues or running faster tests
+by testing only one implementation at a time.
+
 ### Running JSON RPC E2E tests
 
 To run JSON RPC E2E test-vectors [test-vectors](https://github.com/fluffylabs/test-vectors) 

bin/test-runner/common.test.ts

@@ -0,0 +1,41 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { deepEqual } from "@typeberry/utils";
+import { parseArgs, SelectedPvm } from "./common.js";
+
+describe("test runner common", () => {
+  it("should parse pvm argument", () => {
+    const args = ["--pvm", "ananas", "file1.json", "file2.json"];
+
+    const result = parseArgs(args);
+
+    deepEqual(result, {
+      initialFiles: ["file1.json", "file2.json"],
+      pvms: [SelectedPvm.Ananas],
+    });
+  });
+
+  it("should have both pvms by default", () => {
+    const args = ["file1.json", "file2.json"];
+
+    const result = parseArgs(args);
+
+    deepEqual(result, {
+      initialFiles: ["file1.json", "file2.json"],
+      pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin],
+    });
+  });
+
+  it("should throw on invalid pvm", () => {
+    const args = ["--pvm=invalid", "file1.json", "file2.json"];
+
+    assert.throws(
+      () => {
+        const _result = parseArgs(args);
+      },
+      {
+        message: "Unknown pvm value: invalid. Use one of ananas, builtin.",
+      },
+    );
+  });
+});

bin/test-runner/common.ts

@@ -6,14 +6,36 @@ import path from "node:path";
 import test, { type TestContext } from "node:test";
 import util from "node:util";
 import { type Decode, Decoder } from "@typeberry/codec";
-import { type ChainSpec, tinyChainSpec } from "@typeberry/config";
+import { type ChainSpec, PvmBackend, tinyChainSpec } from "@typeberry/config";
 import { initWasm } from "@typeberry/crypto";
 import { type FromJson, parseFromJson } from "@typeberry/json-parser";
 import { Level, Logger } from "@typeberry/logger";
+import { assertNever } from "@typeberry/utils";
+import minimist from "minimist";
 
 Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
 export const logger = Logger.new(import.meta.filename, "test-runner");
 
+export enum SelectedPvm {
+  Ananas = "ananas",
+  Builtin = "builtin",
+}
+export const ALL_PVMS = [SelectedPvm.Ananas, SelectedPvm.Builtin];
+export function selectedPvmToBackend(pvm: SelectedPvm): PvmBackend {
+  switch (pvm) {
+    case SelectedPvm.Ananas:
+      return PvmBackend.Ananas;
+    case SelectedPvm.Builtin:
+      return PvmBackend.BuiltIn;
+    default:
+      assertNever(pvm);
+  }
+}
+
+export type GlobalsOptions = {
+  pvms: SelectedPvm[];
+};
+
 export class RunnerBuilder<T, V> implements Runner<T, V> {
   public readonly parsers: testFile.Kind<T>[] = [];
   public readonly variants: V[] = [];
@@ -112,22 +134,56 @@ export namespace testFile {
       };
 }
 
+export function parseArgs(argv: string[]) {
+  const PVM_OPTION = "pvm";
+  const parsed = minimist(argv);
+  const pvms = getPvms(parsed[PVM_OPTION]);
+
+  return {
+    initialFiles: parsed._,
+    pvms,
+  };
+
+  function getPvms(parsed: string | undefined): SelectedPvm[] {
+    const allPvms = ALL_PVMS.slice();
+
+    if (parsed === undefined) {
+      return allPvms;
+    }
+
+    const opts = parsed.split(",").map((x) => x.trim());
+    const result: SelectedPvm[] = [];
+    for (const o of opts) {
+      const idx = allPvms.indexOf(o as SelectedPvm);
+      if (idx !== -1) {
+        result.push(allPvms[idx]);
+      } else {
+        throw new Error(`Unknown pvm value: ${o}. Use one of ${allPvms.join(", ")}.`);
+      }
+    }
+    return result;
+  }
+}
+
 export async function main(
   runners: Runner<unknown, unknown>[],
-  initialFiles: string[],
   directoryToScan: string,
   {
+    initialFiles,
+    pvms,
     patterns = [testFile.bin, testFile.json],
     accepted,
     ignored,
   }: {
+    initialFiles: string[];
+    pvms: SelectedPvm[];
     patterns?: (testFile.bin | testFile.json)[];
     accepted?: {
       [testFile.bin]?: string[];
       [testFile.json]?: string[];
     };
     ignored?: string[];
-  } = {},
+  },
 ) {
   await initWasm();
   const relPath = `${import.meta.dirname}/../..`;
@@ -177,7 +233,7 @@ export async function main(
       // 3. If the list is defined, we make sure that the path is on that list.
       (accepted[testFileContent.kind] ?? []).some((x) => absolutePath.includes(x));
 
-    const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath);
+    const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath, { pvms });
     for (const test of testVariants) {
       test.shouldSkip = !isAccepted;
       tests.push(test);
@@ -258,6 +314,7 @@ function prepareTest<T, V>(
   testContent: testFile.Content,
   fileName: string,
   fullPath: string,
+  globalOptions: GlobalsOptions,
 ): TestAndRunner<V>[] {
   const errors: [string, unknown][] = [];
   const handleError = (name: string, e: unknown) => errors.push([name, e]);
@@ -286,7 +343,7 @@ function prepareTest<T, V>(
         if (parser.kind === testFile.bin && testContent.kind === testFile.bin) {
           try {
             const parsedTest = Decoder.decodeObject(parser.codec, testContent.content, chainSpec);
-            matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec));
+            matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec, globalOptions));
           } catch (e) {
             handleError(path, e);
           }
@@ -295,7 +352,7 @@ function prepareTest<T, V>(
         if (parser.kind === testFile.json && testContent.kind === testFile.json) {
           try {
             const parsedTest = parseFromJson(testContent.content, parser.fromJson);
-            matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec));
+            matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec, globalOptions));
           } catch (e) {
             handleError(path, e);
           }
@@ -330,9 +387,15 @@ function prepareTest<T, V>(
     variants: V[],
     parsedTest: T,
     chainSpec: ChainSpec,
+    globalOptions: GlobalsOptions,
   ) {
     const results: TestAndRunner<V>[] = [];
-    const possibleVariants: V[] = variants.length === 0 ? [noneVariant] : variants;
+    let possibleVariants: V[] = variants.length === 0 ? [noneVariant] : variants;
+    // a bit hacky way to detect pvm-variants and filtering.
+    const idx = ALL_PVMS.indexOf(possibleVariants[0] as SelectedPvm);
+    if (idx !== -1) {
+      possibleVariants = possibleVariants.filter((x) => globalOptions.pvms.includes(x as SelectedPvm));
+    }
 
     for (const variant of possibleVariants) {
       results.push({

bin/test-runner/jam-conformance-070.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/jam-conformance/fuzz-reports/0.7.0/traces", {
+main(runners, "test-vectors/jam-conformance/fuzz-reports/0.7.0/traces", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   ignored: [
     // CORRECT: note [seko] test rejected at block parsing stage, which is considered valid behavior

bin/test-runner/jam-conformance-071.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/jam-conformance/fuzz-reports/0.7.1/traces", {
+main(runners, "test-vectors/jam-conformance/fuzz-reports/0.7.1/traces", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   ignored: [
     // genesis file is unparsable

bin/test-runner/jamduna-067.ts

@@ -1,15 +1,16 @@
 import { StateTransition } from "@typeberry/state-vectors";
-import { logger, main, runner } from "./common.js";
+import { logger, main, parseArgs, runner, SelectedPvm } from "./common.js";
 import { runStateTransition } from "./state-transition/state-transition.js";
 
 const runners = [
   runner("state_transition", runStateTransition)
     .fromJson(StateTransition.fromJson)
     .fromBin(StateTransition.Codec)
-    .withVariants(["ananas", "builtin"]),
+    .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin]),
 ].map((x) => x.build());
 
-main(runners, process.argv.slice(2), "test-vectors/jamduna_067", {
+main(runners, "test-vectors/jamduna_067", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   accepted: {
     ".json": ["traces/"],

bin/test-runner/javajam.ts

@@ -1,15 +1,16 @@
 import { StateTransition } from "@typeberry/state-vectors";
-import { logger, main, runner } from "./common.js";
+import { logger, main, parseArgs, runner, SelectedPvm } from "./common.js";
 import { runStateTransition } from "./state-transition/state-transition.js";
 
 const runners = [
   runner("state_transition", runStateTransition)
     .fromJson(StateTransition.fromJson)
     .fromBin(StateTransition.Codec)
-    .withVariants(["ananas", "builtin"]),
+    .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin]),
 ].map((x) => x.build());
 
-main(runners, process.argv.slice(2), "test-vectors/javajam", {
+main(runners, "test-vectors/javajam", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   accepted: {
     ".json": ["stf/state_transitions/"],

bin/test-runner/package.json

@@ -30,7 +30,8 @@
     "@typeberry/transition": "*",
     "@typeberry/trie": "*",
     "@typeberry/utils": "*",
-    "json-bigint-patch": "0.0.8"
+    "json-bigint-patch": "0.0.8",
+    "minimist": "^1.2.8"
   },
   "scripts": {
     "start": "tsx --test-timeout=900000 ./index.ts",

bin/test-runner/state-transition/state-transition.ts

@@ -3,7 +3,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { Block, emptyBlock } from "@typeberry/block";
 import { Decoder, Encoder } from "@typeberry/codec";
-import { ChainSpec, PvmBackend, tinyChainSpec } from "@typeberry/config";
+import { ChainSpec, tinyChainSpec } from "@typeberry/config";
 import { InMemoryBlocks } from "@typeberry/database";
 import { Blake2b, keccak, WithHash } from "@typeberry/hash";
 import { tryAsU32 } from "@typeberry/numbers";
@@ -13,7 +13,7 @@ import { TransitionHasher } from "@typeberry/transition";
 import { BlockVerifier } from "@typeberry/transition/block-verifier.js";
 import { OnChain } from "@typeberry/transition/chain-stf.js";
 import { deepEqual, resultToString } from "@typeberry/utils";
-import type { RunOptions } from "../common.js";
+import { type RunOptions, type SelectedPvm, selectedPvmToBackend } from "../common.js";
 import { loadState } from "./state-loader.js";
 
 const keccakHasher = keccak.KeccakHasher.create();
@@ -65,12 +65,8 @@ const jamConformance070V0Spec = new ChainSpec({
   maxLookupAnchorAge: tryAsU32(14_400),
 });
 
-export async function runStateTransition(
-  testContent: StateTransition,
-  options: RunOptions,
-  variant: "ananas" | "builtin",
-) {
-  const pvm = variant === "ananas" ? PvmBackend.Ananas : PvmBackend.BuiltIn;
+export async function runStateTransition(testContent: StateTransition, options: RunOptions, variant: SelectedPvm) {
+  const pvm = selectedPvmToBackend(variant);
   const blake2b = await Blake2b.createHasher();
   // a bit of a hack, but the new value for `maxLookupAnchorAge` was proposed with V1
   // version of the fuzzer, yet these tests were still depending on the older value.

bin/test-runner/w3f-davxy-067.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_067", {
+main(runners, "test-vectors/w3f-davxy_067", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   accepted: {
     ".json": ["traces"],

bin/test-runner/w3f-davxy-070.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_070", {
+main(runners, "test-vectors/w3f-davxy_070", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   accepted: {
     ".json": ["traces"],

bin/test-runner/w3f-davxy-071.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_071", {
+main(runners, "test-vectors/w3f-davxy_071", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   accepted: {
     ".json": ["traces", "codec", "stf"],

bin/test-runner/w3f.ts

@@ -1,7 +1,8 @@
-import { logger, main } from "./common.js";
+import { logger, main, parseArgs } from "./common.js";
 import { runners } from "./w3f/runners.js";
 
-main(runners, process.argv.slice(2), "test-vectors/w3f-fluffy", {
+main(runners, "test-vectors/w3f-fluffy", {
+  ...parseArgs(process.argv.slice(2)),
   patterns: [".json"],
   ignored: [
     "genesis.json",

bin/test-runner/w3f/runners.ts

@@ -13,7 +13,7 @@ import {
 } from "@typeberry/block-json";
 import { fullChainSpec, tinyChainSpec } from "@typeberry/config";
 import { StateTransition } from "@typeberry/state-vectors";
-import { runner } from "../common.js";
+import { runner, SelectedPvm } from "../common.js";
 import { runStateTransition } from "../state-transition/state-transition.js";
 import { AccumulateTest, runAccumulateTest } from "./accumulate.js";
 import { AssurancesTestFull, AssurancesTestTiny, runAssurancesTestFull, runAssurancesTestTiny } from "./assurances.js";
@@ -46,7 +46,7 @@ import { runShufflingTests, shufflingTestsFromJson } from "./shuffling.js";
 import { runStatisticsTestFull, runStatisticsTestTiny, StatisticsTestFull, StatisticsTestTiny } from "./statistics.js";
 import { runTrieTest, trieTestSuiteFromJson } from "./trie.js";
 
-const pvms: ("ananas" | "builtin")[] = ["ananas", "builtin"];
+const pvms: SelectedPvm[] = [SelectedPvm.Ananas, SelectedPvm.Builtin];
 const tiny = [tinyChainSpec];
 const full = [fullChainSpec];
 const tinyFull = [...tiny, ...full];