Refactor test runner

bin/test-runner/common.ts

@@ -14,56 +14,121 @@ import { Level, Logger } from "@typeberry/logger";
 Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
 export const logger = Logger.new(import.meta.filename, "test-runner");
 
-export function runner<T>(
-  name: string,
-  fromJson: FromJson<T>,
-  run: (test: T, path: string, t: TestContext, chainSpec: ChainSpec) => Promise<void>,
-  chainSpec: ChainSpec = tinyChainSpec,
-  fromCodec?: Decode<T>,
-): Runner<unknown> {
-  return { name, fromJson, fromCodec, run, chainSpec } as Runner<unknown>;
+export class RunnerBuilder<T, V> implements Runner<T, V> {
+  public readonly parsers: testFile.Kind<T>[] = [];
+  public readonly variants: V[] = [];
+  public readonly chainSpecs: ChainSpec[] = [];
+
+  constructor(
+    public readonly path: string,
+    public readonly run: RunFunction<T, V>,
+  ) {}
+
+  fromJson(fromJson: FromJson<T>) {
+    this.parsers.push({ kind: testFile.json, fromJson });
+    return this;
+  }
+
+  fromBin(codec: Decode<T>) {
+    this.parsers.push({ kind: testFile.bin, codec });
+    return this;
+  }
+
+  withChainSpecDetection(chainSpec: ChainSpec[]) {
+    this.chainSpecs.push(...chainSpec);
+    return this;
+  }
+
+  withVariants(variants: V[]) {
+    this.variants.push(...variants);
+    return this;
+  }
+
+  build(): Runner<unknown, unknown> {
+    const { path, run, parsers, variants, chainSpecs } = this;
+    if (parsers.length === 0) {
+      throw new Error(`No parsers for ${path}!`);
+    }
+
+    return {
+      path,
+      run,
+      parsers,
+      variants,
+      chainSpecs,
+    } as Runner<unknown, unknown>;
+  }
 }
 
-export type Runner<T> = {
-  name: string;
-  fromJson: FromJson<T>;
-  run: (test: T, path: string, t: TestContext, chainSpec: ChainSpec) => Promise<void>;
-  fromCodec?: Decode<T>;
+/** Test runner builder function. */
+export function runner<T, V = never>(path: string, run: RunFunction<T, V>, chainSpecs?: ChainSpec[]) {
+  const builder = new RunnerBuilder(path, run);
+  if (chainSpecs !== undefined) {
+    return builder.withChainSpecDetection(chainSpecs);
+  }
+  return builder;
+}
+
+export type RunOptions = {
+  test: TestContext;
   chainSpec: ChainSpec;
+  path: string;
 };
 
-enum TestFileKind {
-  Binary = 0,
-  JSON = 1,
-}
+export type RunFunction<T, V> = (test: T, options: RunOptions, variant: V) => Promise<void>;
 
-type TestFile =
-  | {
-      kind: TestFileKind.JSON;
-      content: unknown;
-    }
-  | {
-      kind: TestFileKind.Binary;
-      content: Uint8Array;
-    };
+export type Runner<T, V> = {
+  path: string;
+  parsers: testFile.Kind<T>[];
+  run: RunFunction<T, V>;
+  variants: V[];
+  chainSpecs: ChainSpec[];
+};
+
+export namespace testFile {
+  export const json = ".json";
+  export type json = typeof json;
+  export const bin = ".bin";
+  export type bin = typeof bin;
+
+  export type Kind<T> =
+    | {
+        kind: json;
+        fromJson: FromJson<T>;
+      }
+    | {
+        kind: bin;
+        codec: Decode<T>;
+      };
+
+  export type Content =
+    | {
+        kind: json;
+        content: unknown;
+      }
+    | {
+        kind: bin;
+        content: Uint8Array;
+      };
+}
 
 export async function main(
-  runners: Runner<unknown>[],
+  runners: Runner<unknown, unknown>[],
   initialFiles: string[],
   directoryToScan: string,
   {
-    pattern = ".json",
+    pattern = testFile.json,
     accepted,
     ignored,
   }: {
-    pattern?: string;
+    pattern?: testFile.json | testFile.bin;
     accepted?: string[];
     ignored?: string[];
   } = {},
 ) {
   await initWasm();
   const relPath = `${import.meta.dirname}/../..`;
-  const tests: TestAndRunner[] = [];
+  const tests: TestAndRunner<unknown>[] = [];
   const ignoredPatterns = ignored ?? [];
 
   let testFiles = initialFiles;
@@ -73,38 +138,38 @@ export async function main(
   }
 
   logger.info`Preparing tests for ${testFiles.length} files.`;
-  for (const testFile of testFiles) {
-    const absolutePath = path.resolve(`${relPath}/${testFile}`);
+  for (const testFilePath of testFiles) {
+    const absolutePath = path.resolve(`${relPath}/${testFilePath}`);
 
     if (ignoredPatterns.some((x) => absolutePath.includes(x))) {
       logger.log`Ignoring: ${absolutePath}`;
       continue;
     }
 
-    let testFileContent: TestFile;
-    if (absolutePath.endsWith(".bin")) {
+    let testFileContent: testFile.Content;
+    if (absolutePath.endsWith(testFile.bin)) {
       const content: Buffer = await fs.readFile(absolutePath);
       testFileContent = {
-        kind: TestFileKind.Binary,
+        kind: ".bin",
         content: new Uint8Array(content),
       };
     } else {
       const content = await fs.readFile(absolutePath, "utf8");
       testFileContent = {
-        kind: TestFileKind.JSON,
+        kind: ".json",
         content: JSON.parse(content),
       };
     }
 
-    const test = prepareTest(runners, testFileContent, testFile, absolutePath);
-
-    test.shouldSkip = accepted !== undefined && !accepted.some((x) => absolutePath.includes(x));
-
-    tests.push(test);
+    const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath);
+    for (const test of testVariants) {
+      test.shouldSkip = accepted !== undefined && !accepted.some((x) => absolutePath.includes(x));
+      tests.push(test);
+    }
   }
 
   // aggregate the tests by their runner.
-  const aggregated = new Map<string, TestAndRunner[]>();
+  const aggregated = new Map<string, TestAndRunner<unknown>[]>();
   for (const test of tests) {
     const sameRunner = aggregated.get(test.runner) ?? [];
     sameRunner.push(test);
@@ -136,10 +201,11 @@ export async function main(
             const runnersBatch = testRunners.slice(i * batchSize, (i + 1) * batchSize);
             for (const runner of runnersBatch) {
               const fileName = runner.file.replace(pathToReplace, "");
+              const testCase = `${runner.variant}` !== "" ? `[${runner.variant}] ${fileName}` : fileName;
               if (runner.shouldSkip) {
-                test.it.skip(fileName, runner.test);
+                test.it.skip(testCase, runner.test);
               } else {
-                test.it(fileName, { timeout }, runner.test);
+                test.it(testCase, { timeout }, runner.test);
               }
             }
           },
@@ -163,67 +229,115 @@ async function scanDir(relPath: string, dir: string, filePattern: string): Promi
   }
 }
 
-type TestAndRunner = {
+type TestAndRunner<V> = {
   shouldSkip: boolean;
   runner: string;
   file: string;
+  variant: V;
   test: (ctx: TestContext) => Promise<void>;
 };
 
-function prepareTest(runners: Runner<unknown>[], testContent: TestFile, file: string, path: string): TestAndRunner {
+function prepareTest<T, V>(
+  runners: Runner<T, V>[],
+  testContent: testFile.Content,
+  fileName: string,
+  fullPath: string,
+): TestAndRunner<V>[] {
   const errors: [string, unknown][] = [];
   const handleError = (name: string, e: unknown) => errors.push([name, e]);
+  // NOTE This is not safe, but if the test does not specify
+  // variants it means it doesn't care about them.
+  const noneVariant = "" as V;
 
   // Find the first runner that is able to parse the input data.
-  for (const { name, fromJson, fromCodec, run, chainSpec } of runners) {
-    // NOTE: this `if` statement is needed to distinguish between tiny and full chain spec
-    // without this `if` some tests (for example statistics) will be run twice
-    if (!name.split("/").every((pathPart) => path.includes(pathPart))) {
-      continue;
-    }
+  for (const { path, parsers, run, variants, chainSpecs } of runners) {
+    const specs = chainSpecs.length > 0 ? chainSpecs : [tinyChainSpec];
+    const matchPath = chainSpecs.length > 0;
+    for (const chainSpec of specs) {
+      // NOTE: this `if` statement is intended to speed up parsing of the test files
+      // instead of trying each and every runner, we make sure that the absolute
+      // path to the file includes each part of our "test path" definition.
+      if (!path.split("/").every((pathPart) => fullPath.includes(pathPart))) {
+        continue;
+      }
+      // if we care about the chain spec, we also need to match the path
+      if (matchPath && !fullPath.includes(chainSpec.name)) {
+        continue;
+      }
 
-    const createTestDefinition = (parsedTest: unknown): TestAndRunner => ({
-      shouldSkip: false,
-      runner: name,
-      file,
-      test: (ctx) => {
-        logger.log`[${name}] running test from ${file}`;
-        logger.trace` ${util.inspect(parsedTest)}`;
-        return run(parsedTest, path, ctx, chainSpec);
-      },
-    });
+      for (const parser of parsers) {
+        if (parser.kind === testFile.bin && testContent.kind === testFile.bin) {
+          try {
+            const parsedTest = Decoder.decodeObject(parser.codec, testContent.content, chainSpec);
+            return createTestDefinitions(path, run, variants, parsedTest, chainSpec);
+          } catch (e) {
+            handleError(path, e);
+          }
+        }
 
-    if (testContent.kind === TestFileKind.Binary) {
-      if (fromCodec !== undefined) {
-        try {
-          const parsedTest = Decoder.decodeObject(fromCodec, testContent.content, chainSpec);
-          return createTestDefinition(parsedTest);
-        } catch (e) {
-          handleError(name, e);
+        if (parser.kind === testFile.json && testContent.kind === testFile.json) {
+          try {
+            const parsedTest = parseFromJson(testContent.content, parser.fromJson);
+            return createTestDefinitions(path, run, variants, parsedTest, chainSpec);
+          } catch (e) {
+            handleError(path, e);
+          }
+        }
+
+        if (testContent.kind !== parser.kind) {
+          handleError(path, new Error(`Mismatching parser: got ${parser.kind}, need: ${testContent.kind}`));
         }
-      } else {
-        handleError(name, new Error(`Missing codec definition for: ${name}`));
-      }
-    } else {
-      try {
-        const parsedTest = parseFromJson(testContent.content, fromJson);
-        return createTestDefinition(parsedTest);
-      } catch (e) {
-        handleError(name, e);
       }
     }
   }
 
-  return {
-    shouldSkip: false,
-    runner: "Invalid",
-    file,
-    test: () => {
-      for (const [runner, error] of errors) {
-        logger.error`[${runner}] Parsing error: ${error}`;
-      }
+  return [
+    {
+      shouldSkip: false,
+      runner: "Invalid",
+      file: fileName,
+      variant: noneVariant,
+      test: () => {
+        for (const [runner, error] of errors) {
+          logger.error`[${runner}] Parsing error: ${error}`;
+        }
 
-      fail(`Unrecognized test case in ${file}`);
+        fail(`Unrecognized test case in ${fileName}`);
+      },
     },
-  };
+  ];
+
+  function createTestDefinitions(
+    path: string,
+    run: RunFunction<T, V>,
+    variants: V[],
+    parsedTest: T,
+    chainSpec: ChainSpec,
+  ) {
+    const results: TestAndRunner<V>[] = [];
+    const possibleVariants: V[] = variants.length === 0 ? [noneVariant] : variants;
+
+    for (const variant of possibleVariants) {
+      results.push({
+        shouldSkip: false,
+        runner: path,
+        file: fileName,
+        variant,
+        test: (ctx) => {
+          logger.log`[${path}:${variant}] running test from ${fileName} (spec: ${chainSpec.name})`;
+          logger.trace` ${util.inspect(parsedTest, true, 2)}`;
+          return run(
+            parsedTest,
+            {
+              test: ctx,
+              path: fullPath,
+              chainSpec,
+            },
+            variant,
+          );
+        },
+      });
+    }
+    return results;
+  }
 }

bin/test-runner/jamduna-067.ts

@@ -1,7 +1,12 @@
 import { logger, main, runner } from "./common.js";
 import { runStateTransition, StateTransition } from "./state-transition/state-transition.js";
 
-const runners = [runner("traces", StateTransition.fromJson, runStateTransition)];
+const runners = [
+  runner("state_transition", runStateTransition)
+    .fromJson(StateTransition.fromJson)
+    .fromBin(StateTransition.Codec)
+    .withVariants(["ananas", "builtin"]),
+].map((x) => x.build());
 
 main(runners, process.argv.slice(2), "test-vectors/jamduna_067", {
   accepted: ["traces/"],