diff --git a/README.md b/README.md index c8d695bef..425d9a7d1 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,25 @@ all relevant ones can be easily checked out from [our test vectors repository](h Obviously it's also possible to run just single test case or part of the test cases by altering the glob pattern in the path. +#### Selecting PVM Backend + +By default, test vectors are run with both PVM backends (built-in and Ananas). +You can select a specific PVM backend using the `--pvm` option: + +```bash +# Run tests with built-in PVM only +$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm builtin + +# Run tests with Ananas PVM only +$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm ananas + +# Run tests with both PVMs (default) +$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner +``` + +This option is useful for debugging PVM-specific issues or running faster tests +by testing only one implementation at a time. + ### Running JSON RPC E2E tests To run JSON RPC E2E test-vectors [test-vectors](https://github.com/fluffylabs/test-vectors) diff --git a/bin/test-runner/common.test.ts b/bin/test-runner/common.test.ts new file mode 100644 index 000000000..7d3859368 --- /dev/null +++ b/bin/test-runner/common.test.ts @@ -0,0 +1,41 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { deepEqual } from "@typeberry/utils"; +import { parseArgs, SelectedPvm } from "./common.js"; + +describe("test runner common", () => { + it("should parse pvm argument", () => { + const args = ["--pvm", "ananas", "file1.json", "file2.json"]; + + const result = parseArgs(args); + + deepEqual(result, { + initialFiles: ["file1.json", "file2.json"], + pvms: [SelectedPvm.Ananas], + }); + }); + + it("should have both pvms by default", () => { + const args = ["file1.json", "file2.json"]; + + const result = parseArgs(args); + + deepEqual(result, { + initialFiles: ["file1.json", "file2.json"], + pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin], + }); + }); + + it("should throw on invalid pvm", () => { + const args = ["--pvm=invalid", "file1.json", "file2.json"]; + + assert.throws( + () => { + const _result = parseArgs(args); + }, + { + message: "Unknown pvm value: invalid. Use one of ananas, builtin.", + }, + ); + }); +}); diff --git a/bin/test-runner/common.ts b/bin/test-runner/common.ts index 731d93e9e..9c3844bfd 100644 --- a/bin/test-runner/common.ts +++ b/bin/test-runner/common.ts @@ -6,14 +6,36 @@ import path from "node:path"; import test, { type TestContext } from "node:test"; import util from "node:util"; import { type Decode, Decoder } from "@typeberry/codec"; -import { type ChainSpec, tinyChainSpec } from "@typeberry/config"; +import { type ChainSpec, PvmBackend, tinyChainSpec } from "@typeberry/config"; import { initWasm } from "@typeberry/crypto"; import { type FromJson, parseFromJson } from "@typeberry/json-parser"; import { Level, Logger } from "@typeberry/logger"; +import { assertNever } from "@typeberry/utils"; +import minimist from "minimist"; Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG); export const logger = Logger.new(import.meta.filename, "test-runner"); +export enum SelectedPvm { + Ananas = "ananas", + Builtin = "builtin", +} +export const ALL_PVMS = [SelectedPvm.Ananas, SelectedPvm.Builtin]; +export function selectedPvmToBackend(pvm: SelectedPvm): PvmBackend { + switch (pvm) { + case SelectedPvm.Ananas: + return PvmBackend.Ananas; + case SelectedPvm.Builtin: + return PvmBackend.BuiltIn; + default: + assertNever(pvm); + } +} + +export type GlobalsOptions = { + pvms: SelectedPvm[]; +}; + export class RunnerBuilder implements Runner { public readonly parsers: testFile.Kind[] = []; public readonly variants: V[] = []; @@ -112,22 +134,56 @@ export namespace testFile { }; } +export function parseArgs(argv: string[]) { + const PVM_OPTION = "pvm"; + const parsed = minimist(argv); + const pvms = getPvms(parsed[PVM_OPTION]); + + return { + initialFiles: parsed._, + pvms, + }; + + function getPvms(parsed: string | undefined): SelectedPvm[] { + const allPvms = ALL_PVMS.slice(); + + if (parsed === undefined) { + return allPvms; + } + + const opts = parsed.split(",").map((x) => x.trim()); + const result: SelectedPvm[] = []; + for (const o of opts) { + const pvm = allPvms.find((p) => p === o); + if (pvm !== undefined) { + result.push(pvm); + } else { + throw new Error(`Unknown pvm value: ${o}. Use one of ${allPvms.join(", ")}.`); + } + } + return result; + } +} + export async function main( runners: Runner[], - initialFiles: string[], directoryToScan: string, { + initialFiles, + pvms, patterns = [testFile.bin, testFile.json], accepted, ignored, }: { + initialFiles: string[]; + pvms: SelectedPvm[]; patterns?: (testFile.bin | testFile.json)[]; accepted?: { [testFile.bin]?: string[]; [testFile.json]?: string[]; }; ignored?: string[]; - } = {}, + }, ) { await initWasm(); const relPath = `${import.meta.dirname}/../..`; @@ -177,7 +233,7 @@ export async function main( // 3. If the list is defined, we make sure that the path is on that list. (accepted[testFileContent.kind] ?? []).some((x) => absolutePath.includes(x)); - const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath); + const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath, { pvms }); for (const test of testVariants) { test.shouldSkip = !isAccepted; tests.push(test); @@ -258,6 +314,7 @@ function prepareTest( testContent: testFile.Content, fileName: string, fullPath: string, + globalOptions: GlobalsOptions, ): TestAndRunner[] { const errors: [string, unknown][] = []; const handleError = (name: string, e: unknown) => errors.push([name, e]); @@ -286,7 +343,7 @@ function prepareTest( if (parser.kind === testFile.bin && testContent.kind === testFile.bin) { try { const parsedTest = Decoder.decodeObject(parser.codec, testContent.content, chainSpec); - matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec)); + matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec, globalOptions)); } catch (e) { handleError(path, e); } @@ -295,7 +352,7 @@ function prepareTest( if (parser.kind === testFile.json && testContent.kind === testFile.json) { try { const parsedTest = parseFromJson(testContent.content, parser.fromJson); - matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec)); + matchingRunners.push(...createTestDefinitions(path, run, variants, parsedTest, chainSpec, globalOptions)); } catch (e) { handleError(path, e); } @@ -330,9 +387,15 @@ function prepareTest( variants: V[], parsedTest: T, chainSpec: ChainSpec, + globalOptions: GlobalsOptions, ) { const results: TestAndRunner[] = []; - const possibleVariants: V[] = variants.length === 0 ? [noneVariant] : variants; + let possibleVariants: V[] = variants.length === 0 ? [noneVariant] : variants; + // a bit hacky way to detect pvm-variants and filtering. + const idx = ALL_PVMS.indexOf(possibleVariants[0] as SelectedPvm); + if (idx !== -1) { + possibleVariants = possibleVariants.filter((x) => globalOptions.pvms.includes(x as SelectedPvm)); + } for (const variant of possibleVariants) { results.push({ diff --git a/bin/test-runner/jam-conformance-070.ts b/bin/test-runner/jam-conformance-070.ts index 8fb07b075..87799a54d 100644 --- a/bin/test-runner/jam-conformance-070.ts +++ b/bin/test-runner/jam-conformance-070.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/jam-conformance/fuzz-reports/0.7.0/traces", { +main(runners, "test-vectors/jam-conformance/fuzz-reports/0.7.0/traces", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], ignored: [ // CORRECT: note [seko] test rejected at block parsing stage, which is considered valid behavior diff --git a/bin/test-runner/jam-conformance-071.ts b/bin/test-runner/jam-conformance-071.ts index f8c5d810f..7b78dfdfb 100644 --- a/bin/test-runner/jam-conformance-071.ts +++ b/bin/test-runner/jam-conformance-071.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/jam-conformance/fuzz-reports/0.7.1/traces", { +main(runners, "test-vectors/jam-conformance/fuzz-reports/0.7.1/traces", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], ignored: [ // genesis file is unparsable diff --git a/bin/test-runner/jamduna-067.ts b/bin/test-runner/jamduna-067.ts index 7b8a40735..6f13183f0 100644 --- a/bin/test-runner/jamduna-067.ts +++ b/bin/test-runner/jamduna-067.ts @@ -1,15 +1,16 @@ import { StateTransition } from "@typeberry/state-vectors"; -import { logger, main, runner } from "./common.js"; +import { logger, main, parseArgs, runner, SelectedPvm } from "./common.js"; import { runStateTransition } from "./state-transition/state-transition.js"; const runners = [ runner("state_transition", runStateTransition) .fromJson(StateTransition.fromJson) .fromBin(StateTransition.Codec) - .withVariants(["ananas", "builtin"]), + .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin]), ].map((x) => x.build()); -main(runners, process.argv.slice(2), "test-vectors/jamduna_067", { +main(runners, "test-vectors/jamduna_067", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], accepted: { ".json": ["traces/"], diff --git a/bin/test-runner/javajam.ts b/bin/test-runner/javajam.ts index 1e67c1d58..da311acb8 100644 --- a/bin/test-runner/javajam.ts +++ b/bin/test-runner/javajam.ts @@ -1,15 +1,16 @@ import { StateTransition } from "@typeberry/state-vectors"; -import { logger, main, runner } from "./common.js"; +import { logger, main, parseArgs, runner, SelectedPvm } from "./common.js"; import { runStateTransition } from "./state-transition/state-transition.js"; const runners = [ runner("state_transition", runStateTransition) .fromJson(StateTransition.fromJson) .fromBin(StateTransition.Codec) - .withVariants(["ananas", "builtin"]), + .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin]), ].map((x) => x.build()); -main(runners, process.argv.slice(2), "test-vectors/javajam", { +main(runners, "test-vectors/javajam", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], accepted: { ".json": ["stf/state_transitions/"], diff --git a/bin/test-runner/package.json b/bin/test-runner/package.json index 733ffae64..15a79023a 100644 --- a/bin/test-runner/package.json +++ b/bin/test-runner/package.json @@ -30,7 +30,8 @@ "@typeberry/transition": "*", "@typeberry/trie": "*", "@typeberry/utils": "*", - "json-bigint-patch": "0.0.8" + "json-bigint-patch": "0.0.8", + "minimist": "^1.2.8" }, "scripts": { "start": "tsx --test-timeout=900000 ./index.ts", diff --git a/bin/test-runner/state-transition/state-transition.ts b/bin/test-runner/state-transition/state-transition.ts index 78c1a1217..fec266711 100644 --- a/bin/test-runner/state-transition/state-transition.ts +++ b/bin/test-runner/state-transition/state-transition.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import { Block, emptyBlock } from "@typeberry/block"; import { Decoder, Encoder } from "@typeberry/codec"; -import { ChainSpec, PvmBackend, tinyChainSpec } from "@typeberry/config"; +import { ChainSpec, tinyChainSpec } from "@typeberry/config"; import { InMemoryBlocks } from "@typeberry/database"; import { Blake2b, keccak, WithHash } from "@typeberry/hash"; import { tryAsU32 } from "@typeberry/numbers"; @@ -13,7 +13,7 @@ import { TransitionHasher } from "@typeberry/transition"; import { BlockVerifier } from "@typeberry/transition/block-verifier.js"; import { OnChain } from "@typeberry/transition/chain-stf.js"; import { deepEqual, resultToString } from "@typeberry/utils"; -import type { RunOptions } from "../common.js"; +import { type RunOptions, type SelectedPvm, selectedPvmToBackend } from "../common.js"; import { loadState } from "./state-loader.js"; const keccakHasher = keccak.KeccakHasher.create(); @@ -65,12 +65,8 @@ const jamConformance070V0Spec = new ChainSpec({ maxLookupAnchorAge: tryAsU32(14_400), }); -export async function runStateTransition( - testContent: StateTransition, - options: RunOptions, - variant: "ananas" | "builtin", -) { - const pvm = variant === "ananas" ? PvmBackend.Ananas : PvmBackend.BuiltIn; +export async function runStateTransition(testContent: StateTransition, options: RunOptions, variant: SelectedPvm) { + const pvm = selectedPvmToBackend(variant); const blake2b = await Blake2b.createHasher(); // a bit of a hack, but the new value for `maxLookupAnchorAge` was proposed with V1 // version of the fuzzer, yet these tests were still depending on the older value. diff --git a/bin/test-runner/w3f-davxy-067.ts b/bin/test-runner/w3f-davxy-067.ts index 5efd9e704..36101c139 100644 --- a/bin/test-runner/w3f-davxy-067.ts +++ b/bin/test-runner/w3f-davxy-067.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_067", { +main(runners, "test-vectors/w3f-davxy_067", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], accepted: { ".json": ["traces"], diff --git a/bin/test-runner/w3f-davxy-070.ts b/bin/test-runner/w3f-davxy-070.ts index 1e5a19e4a..0bf7eaeaa 100644 --- a/bin/test-runner/w3f-davxy-070.ts +++ b/bin/test-runner/w3f-davxy-070.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_070", { +main(runners, "test-vectors/w3f-davxy_070", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], accepted: { ".json": ["traces"], diff --git a/bin/test-runner/w3f-davxy-071.ts b/bin/test-runner/w3f-davxy-071.ts index 0e4ae6fd4..8954ba458 100644 --- a/bin/test-runner/w3f-davxy-071.ts +++ b/bin/test-runner/w3f-davxy-071.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/w3f-davxy_071", { +main(runners, "test-vectors/w3f-davxy_071", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], accepted: { ".json": ["traces", "codec", "stf"], diff --git a/bin/test-runner/w3f.ts b/bin/test-runner/w3f.ts index 4db263236..4be2d629d 100644 --- a/bin/test-runner/w3f.ts +++ b/bin/test-runner/w3f.ts @@ -1,7 +1,8 @@ -import { logger, main } from "./common.js"; +import { logger, main, parseArgs } from "./common.js"; import { runners } from "./w3f/runners.js"; -main(runners, process.argv.slice(2), "test-vectors/w3f-fluffy", { +main(runners, "test-vectors/w3f-fluffy", { + ...parseArgs(process.argv.slice(2)), patterns: [".json"], ignored: [ "genesis.json", diff --git a/bin/test-runner/w3f/runners.ts b/bin/test-runner/w3f/runners.ts index bd6dce29f..9296abd98 100644 --- a/bin/test-runner/w3f/runners.ts +++ b/bin/test-runner/w3f/runners.ts @@ -13,7 +13,7 @@ import { } from "@typeberry/block-json"; import { fullChainSpec, tinyChainSpec } from "@typeberry/config"; import { StateTransition } from "@typeberry/state-vectors"; -import { runner } from "../common.js"; +import { runner, SelectedPvm } from "../common.js"; import { runStateTransition } from "../state-transition/state-transition.js"; import { AccumulateTest, runAccumulateTest } from "./accumulate.js"; import { AssurancesTestFull, AssurancesTestTiny, runAssurancesTestFull, runAssurancesTestTiny } from "./assurances.js"; @@ -46,7 +46,7 @@ import { runShufflingTests, shufflingTestsFromJson } from "./shuffling.js"; import { runStatisticsTestFull, runStatisticsTestTiny, StatisticsTestFull, StatisticsTestTiny } from "./statistics.js"; import { runTrieTest, trieTestSuiteFromJson } from "./trie.js"; -const pvms: ("ananas" | "builtin")[] = ["ananas", "builtin"]; +const pvms: SelectedPvm[] = [SelectedPvm.Ananas, SelectedPvm.Builtin]; const tiny = [tinyChainSpec]; const full = [fullChainSpec]; const tinyFull = [...tiny, ...full]; diff --git a/package-lock.json b/package-lock.json index 8445946c6..c3bba6d50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -326,7 +326,8 @@ "@typeberry/transition": "*", "@typeberry/trie": "*", "@typeberry/utils": "*", - "json-bigint-patch": "0.0.8" + "json-bigint-patch": "0.0.8", + "minimist": "^1.2.8" } }, "bin/typeberry": {