Refactor Authorship

bin/jam/args.ts

@@ -1,5 +1,5 @@
 import { type PvmBackend, PvmBackendNames } from "@typeberry/config";
-import { DEFAULT_CONFIG, DEV_CONFIG, NODE_DEFAULTS } from "@typeberry/config-node";
+import { DEFAULT_CONFIG, DEV_TINY_CONFIG, NODE_DEFAULTS } from "@typeberry/config-node";
 import { logger } from "@typeberry/node";
 import { isU16, type U16 } from "@typeberry/numbers";
 import { version } from "@typeberry/utils";
@@ -28,7 +28,7 @@ Options:
   --${ARGS.NAME}                Override node name. Affects networking key and db location.
                         [default: ${NODE_DEFAULTS.name}]
   --${ARGS.CONFIG}              Configuration directives. If specified more than once, they are evaluated and merged from left to right.
-                        A configuration directive can be a path to a config file, an inline JSON object, a pseudo-jq query or one of predefined configs ['${DEV_CONFIG}', '${DEFAULT_CONFIG}'].
+                        A configuration directive can be a path to a config file, an inline JSON object, a pseudo-jq query or one of predefined configs ['${DEV_TINY_CONFIG}', '${DEFAULT_CONFIG}'].
                         Pseudo-jq queries are a way to modify the config using a subset of jq syntax.
                         Example: --${ARGS.CONFIG}=dev --${ARGS.CONFIG}=.chain_spec+={"bootnodes": []}      -- will modify only the bootnodes property of the chain spec (merge).
                         Example: --${ARGS.CONFIG}=dev --${ARGS.CONFIG}=.chain_spec={"bootnodes": []}       -- will replace the entire chain spec property with the provided JSON object.
@@ -150,7 +150,7 @@ export function parseArgs(input: string[], withRelPath: (v: string) => string):
       return { command: Command.Run, args: data };
     }
     case Command.Dev: {
-      const data = parseSharedOptions(args, [DEV_CONFIG]);
+      const data = parseSharedOptions(args, [DEV_TINY_CONFIG]);
       const indexOrAll = args._.shift();
       if (indexOrAll === undefined) {
         throw new Error("Missing dev-validator index.");

bin/jam/helpers/generate-full-genesis.ts

@@ -0,0 +1,161 @@
+#!/usr/bin/env tsx
+// biome-ignore-all lint/suspicious/noConsole: bin file
+
+// Generate a "full"-flavor dev config JSON with 1023 trivial-seed validators.
+// Output goes to stdout; redirect to e.g. packages/configs/typeberry-dev-full.json
+// then run with:
+//   npm start -- --config packages/configs/typeberry-dev-full.json dev all --fast-forward
+
+import {
+  Block,
+  DisputesExtrinsic,
+  Extrinsic,
+  Header,
+  reencodeAsView,
+  tryAsPerEpochBlock,
+  tryAsPerValidator,
+  tryAsTimeSlot,
+  tryAsValidatorIndex,
+} from "@typeberry/block";
+import { Bytes, BytesBlob } from "@typeberry/bytes";
+import { Encoder } from "@typeberry/codec";
+import { asKnownSize } from "@typeberry/collections";
+import { fullChainSpec } from "@typeberry/config";
+import { BLS_KEY_BYTES, initWasm } from "@typeberry/crypto";
+import {
+  deriveBandersnatchPublicKey,
+  deriveBandersnatchSecretKey,
+  deriveEd25519PublicKey,
+  deriveEd25519SecretKey,
+  trivialSeed,
+} from "@typeberry/crypto/key-derivation.js";
+import { Blake2b } from "@typeberry/hash";
+import { tryAsU32 } from "@typeberry/numbers";
+import bandersnatchVrf from "@typeberry/safrole/bandersnatch-vrf.js";
+import { BandernsatchWasm } from "@typeberry/safrole/bandersnatch-wasm.js";
+import { InMemoryState, SafroleSealingKeysData, VALIDATOR_META_BYTES, ValidatorData } from "@typeberry/state";
+import { StateEntries } from "@typeberry/state-merkleization";
+import { TransitionHasher } from "@typeberry/transition";
+import { asOpaqueType } from "@typeberry/utils";
+
+async function main() {
+  await initWasm();
+  const blake2b = await Blake2b.createHasher();
+  const bandersnatch = await BandernsatchWasm.new();
+  const spec = fullChainSpec;
+  const n = spec.validatorsCount;
+
+  console.error(`Deriving ${n} validator keys...`);
+  const validators: ValidatorData[] = [];
+  for (let i = 0; i < n; i++) {
+    const seed = trivialSeed(tryAsU32(i));
+    const bandersnatchSecret = deriveBandersnatchSecretKey(seed, blake2b);
+    const ed25519Secret = deriveEd25519SecretKey(seed, blake2b);
+    const bandersnatchPub = deriveBandersnatchPublicKey(bandersnatchSecret);
+    const ed25519Pub = await deriveEd25519PublicKey(ed25519Secret);
+    validators.push(
+      ValidatorData.create({
+        bandersnatch: bandersnatchPub,
+        ed25519: ed25519Pub,
+        bls: Bytes.zero(BLS_KEY_BYTES).asOpaque(),
+        metadata: Bytes.zero(VALIDATOR_META_BYTES).asOpaque(),
+      }),
+    );
+    if ((i + 1) % 100 === 0) {
+      console.error(`  ${i + 1}/${n}`);
+    }
+  }
+
+  console.error("Computing ring commitment...");
+  const ringRootResult = await bandersnatchVrf.getRingCommitment(
+    bandersnatch,
+    validators.map((v) => v.bandersnatch),
+  );
+  if (ringRootResult.isError) {
+    throw new Error(`Failed to compute ring commitment: ${ringRootResult.error}`);
+  }
+  const epochRoot = ringRootResult.ok;
+
+  console.error("Building genesis state...");
+  const state = InMemoryState.empty(spec);
+  const perValidator = tryAsPerValidator(validators, spec);
+  state.designatedValidatorData = perValidator;
+  state.nextValidatorData = perValidator;
+  state.currentValidatorData = perValidator;
+  state.previousValidatorData = perValidator;
+  state.epochRoot = epochRoot;
+  // Fallback sealing keys: cycle bandersnatch keys across epoch slots. The first
+  // epoch transition will recompute this via safrole; we only need a valid
+  // SafroleSealingKeysData here so the genesis state encodes successfully.
+  state.sealingKeySeries = SafroleSealingKeysData.keys(
+    tryAsPerEpochBlock(
+      Array.from({ length: spec.epochLength }, (_, i) => validators[i % n].bandersnatch),
+      spec,
+    ),
+  );
+
+  console.error("Serialising state entries...");
+  const stateEntries = StateEntries.serializeInMemory(spec, blake2b, state);
+  const stateRoot = stateEntries.getRootHash(blake2b);
+  console.error(`Genesis state root: ${stateRoot}`);
+
+  console.error("Building genesis header...");
+  const hasher = await TransitionHasher.create();
+  const extrinsic = Extrinsic.create({
+    tickets: asOpaqueType(asKnownSize([])),
+    preimages: [],
+    guarantees: asOpaqueType(asKnownSize([])),
+    assurances: asOpaqueType(asKnownSize([])),
+    disputes: DisputesExtrinsic.create({ verdicts: [], culprits: [], faults: [] }),
+  });
+  const extrinsicView = reencodeAsView(Extrinsic.Codec, extrinsic, spec);
+  const extrinsicHash = hasher.extrinsic(extrinsicView).hash;
+
+  const header = Header.create({
+    parentHeaderHash: Bytes.zero(32).asOpaque(),
+    priorStateRoot: Bytes.zero(32).asOpaque(),
+    extrinsicHash,
+    timeSlotIndex: tryAsTimeSlot(0),
+    epochMarker: null,
+    ticketsMarker: null,
+    bandersnatchBlockAuthorIndex: tryAsValidatorIndex(0xffff),
+    entropySource: Bytes.zero(96).asOpaque(),
+    offendersMarker: [],
+    seal: Bytes.zero(96).asOpaque(),
+  });
+  const encodedHeader = Encoder.encodeObject(Header.Codec, header, spec);
+
+  // Sanity check: roundtrip the genesis block through codec under fullChainSpec
+  // so we catch any size/shape mismatches now instead of at node startup.
+  reencodeAsView(Block.Codec, Block.create({ header, extrinsic }), spec);
+
+  console.error("Encoding JSON...");
+  const stateMap: Record<string, string> = {};
+  for (const [key, value] of stateEntries) {
+    stateMap[hexNoPrefix(key.raw)] = hexNoPrefix(value.raw);
+  }
+
+  const out = {
+    $schema: "https://fluffylabs.dev/typeberry/schemas/config-v1.schema.json",
+    version: 1,
+    flavor: "full",
+    authorship: {},
+    chain_spec: {
+      id: "typeberry-dev-full",
+      bootnodes: [],
+      genesis_header: hexNoPrefix(encodedHeader.raw),
+      genesis_state: stateMap,
+    },
+  };
+  process.stdout.write(JSON.stringify(out, null, 2));
+  process.stdout.write("\n");
+}
+
+function hexNoPrefix(bytes: Uint8Array): string {
+  return BytesBlob.blobFrom(bytes).toString().slice(2);
+}
+
+main().catch((err) => {
+  console.error("Error:", err);
+  process.exit(1);
+});

bin/jam/package.json

@@ -9,7 +9,10 @@
   "main": "index.ts",
   "bin": "./index.ts",
   "dependencies": {
+    "@typeberry/block": "*",
     "@typeberry/bytes": "*",
+    "@typeberry/codec": "*",
+    "@typeberry/collections": "*",
     "@typeberry/config": "*",
     "@typeberry/config-node": "*",
     "@typeberry/crypto": "*",
@@ -18,7 +21,11 @@
     "@typeberry/networking": "*",
     "@typeberry/node": "*",
     "@typeberry/numbers": "*",
+    "@typeberry/safrole": "*",
+    "@typeberry/state": "*",
+    "@typeberry/state-merkleization": "*",
     "@typeberry/telemetry": "*",
+    "@typeberry/transition": "*",
     "@typeberry/utils": "*",
     "@typeberry/workers-api": "*",
     "minimist": "1.2.8"
@@ -29,7 +36,7 @@
     "build": "./build-for-npm.sh",
     "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')",
     "test:e2e": "JAM_LOG=trace tsx --test test/e2e.ts",
-    "tiny-network": "tsx ./tiny-network.ts"
+    "tiny-network": "tsx ./helpers/tiny-network.ts"
   },
   "author": "Fluffy Labs",
   "license": "MPL-2.0",

biome.jsonc

@@ -64,7 +64,8 @@
       "!**/benchmarks/**/output/**/*",
       "!**/test-vectors/**/*",
       "!**/packages/misc/builder/pkg.ts",
-      "!**/web/docs"
+      "!**/web/docs",
+      "!**/packages/configs/typeberry-dev-full.json"
     ]
   }
 }

packages/configs/index.ts

@@ -1,7 +1,9 @@
 import defaultConfig from "./typeberry-default.json" with { type: "json" };
-import devConfig from "./typeberry-dev.json" with { type: "json" };
+import devFullConfig from "./typeberry-dev-full.json" with { type: "json" };
+import devTinyConfig from "./typeberry-dev-tiny.json" with { type: "json" };
 
 export const configs = {
   default: defaultConfig,
-  dev: devConfig,
+  devTiny: devTinyConfig,
+  devFull: devFullConfig,
 };

packages/core/bytes/bytes.ts

@@ -240,6 +240,15 @@ export class Bytes<T extends number> extends BytesBlob {
   asOpaque<R>(): Opaque<Bytes<T>, TokenOf<R, Bytes<T>>> {
     return asOpaqueType<Bytes<T>, TokenOf<R, Bytes<T>>>(this);
   }
+
+  toStringTruncated() {
+    if (this.raw.length > 8) {
+      const start = bytesToHexString(this.raw.subarray(0, 2));
+      const end = bytesToHexString(this.raw.subarray(this.raw.length - 2));
+      return `${start}...${end.substring(2)}`;
+    }
+    return `${this.toString()}`;
+  }
 }
 
 function byteFromString(s: string): number {

packages/core/utils/debug.test.ts

@@ -31,23 +31,29 @@ describe("utils::lazyInspect", () => {
 
 describe("utils::memoryUsage", () => {
   it("should report all memory fields", () => {
-    const usage = memoryUsage();
+    const usage = memoryUsage(true);
     for (const field of ["rss=", "heap=", "external=", "arrayBuffers="]) {
       assert.ok(usage.includes(field), `expected "${field}" in "${usage}"`);
     }
   });
+  it("should report all memory fields without details", () => {
+    const usage = memoryUsage(false);
+    for (const field of ["rss=", "heap="]) {
+      assert.ok(usage.includes(field), `expected "${field}" in "${usage}"`);
+    }
+  });
 });
 
 describe("utils::memoryTracker", () => {
   it("should not include a delta on the first call", () => {
-    const tracker = memoryTracker();
-    assert.ok(!tracker().includes("Δrss"));
+    const tracker = memoryTracker(true);
+    assert.ok(!tracker.toString().includes("Δrss"));
   });
 
   it("should include a delta on subsequent calls", () => {
-    const tracker = memoryTracker();
-    tracker();
-    const second = tracker();
+    const tracker = memoryTracker(true);
+    tracker.toString();
+    const second = tracker.toString();
     assert.ok(second.includes("Δrss="), `expected delta in "${second}"`);
     assert.ok(second.includes("ΔarrayBuffers="), `expected delta in "${second}"`);
   });

packages/core/utils/debug.ts

@@ -175,28 +175,34 @@ function rawMemoryUsage(): NodeJS.MemoryUsage | null {
  * `arrayBuffers` should allow tracking WASM memory, since every instance backs its
  * memory with `ArrayBuffer`.
  */
-export function memoryUsage(): string {
+export function memoryUsage(withDetails: boolean): string {
   const m = rawMemoryUsage();
   if (m === null) {
     return "";
   }
-  return `rss=${toMb(m.rss)}MB heap=${toMb(m.heapUsed)}/${toMb(m.heapTotal)}MB external=${toMb(m.external)}MB arrayBuffers=${toMb(m.arrayBuffers)}MB`;
+  if (withDetails) {
+    return `rss=${toMb(m.rss)}MB heap=${toMb(m.heapUsed)}/${toMb(m.heapTotal)}MB external=${toMb(m.external)}MB arrayBuffers=${toMb(m.arrayBuffers)}MB`;
+  }
+
+  return `rss=${toMb(m.rss)}MB heap=${toMb(m.heapUsed)}/${toMb(m.heapTotal)}MB`;
 }
 
 /** Create a stateful memory usage reporter. */
-export function memoryTracker(): () => string {
+export function memoryTracker(withDetails: boolean): { toString(): string } {
   let prev: NodeJS.MemoryUsage | null = null;
-  return () => {
-    const m = rawMemoryUsage();
-    if (m === null) {
-      return "";
-    }
-    const delta =
-      prev === null
-        ? ""
-        : ` (Δrss=${signedMb(m.rss - prev.rss)}MB ΔarrayBuffers=${signedMb(m.arrayBuffers - prev.arrayBuffers)}MB)`;
-    prev = m;
-    return `${memoryUsage()}${delta}`;
+  return {
+    toString() {
+      const m = rawMemoryUsage();
+      if (m === null) {
+        return "";
+      }
+      const delta =
+        prev === null || withDetails === false
+          ? ""
+          : ` (Δrss=${signedMb(m.rss - prev.rss)}MB ΔarrayBuffers=${signedMb(m.arrayBuffers - prev.arrayBuffers)}MB)`;
+      prev = m;
+      return `${memoryUsage(withDetails)}${delta}`;
+    },
   };
 }
 

packages/jam/config-node/node-config.test.ts

@@ -86,7 +86,7 @@ describe("loadConfig", () => {
 
   it("should load dev config", () => {
     const config = loadConfig(["dev"], withRelPath);
-    assert.deepStrictEqual(config, parseFromJson(configs.dev, NodeConfiguration.fromJson));
+    assert.deepStrictEqual(config, parseFromJson(configs.devTiny, NodeConfiguration.fromJson));
   });
 
   it("should parse inline json config and deep merge onto previous entries", () => {
@@ -200,9 +200,9 @@ describe("loadConfig", () => {
       config,
       parseFromJson(
         {
-          ...configs.dev,
+          ...configs.devTiny,
           database_base_path: "/test/path-1",
-          chain_spec: { ...configs.dev.chain_spec, bootnodes: [] },
+          chain_spec: { ...configs.devTiny.chain_spec, bootnodes: [] },
         },
         NodeConfiguration.fromJson,
       ),