From 30a4c24e1e7bacdb7f4c6bf690055c8ca9b1a975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 14 Jul 2026 10:34:58 +0200 Subject: [PATCH] Offline networking worker. --- bin/lib/README.md | 72 ++++++++++ bin/lib/examples/networking-offline.test.ts | 42 ++++++ bin/lib/exports/crypto-browser.ts | 8 ++ bin/lib/exports/networking-offline.ts | 6 + bin/lib/index.ts | 2 + bin/lib/package.json | 3 + package-lock.json | 28 +++- package.json | 1 + packages/core/crypto/README.md | 14 ++ packages/core/crypto/browser.test.ts | 18 +++ packages/core/crypto/browser.ts | 49 +++++++ packages/core/listener/index.test.ts | 61 ++++++++ packages/core/listener/index.ts | 11 +- packages/core/listener/package.json | 3 + packages/core/utils/test.ts | 16 ++- packages/jam/block/assurances.ts | 2 +- packages/jam/block/disputes.ts | 7 +- packages/jam/block/guarantees.ts | 2 +- packages/jam/block/header.ts | 10 +- packages/jam/block/tickets.ts | 2 +- packages/jam/networking-offline/README.md | 44 ++++++ packages/jam/networking-offline/index.test.ts | 74 ++++++++++ packages/jam/networking-offline/index.ts | 132 ++++++++++++++++++ packages/jam/networking-offline/package.json | 24 ++++ packages/workers/api/channel.ts | 2 +- .../comms-authorship-network/protocol.ts | 3 +- .../tickets-message.ts | 2 +- packages/workers/jam-network/messages.ts | 28 ++++ packages/workers/jam-network/protocol.ts | 28 +--- 29 files changed, 652 insertions(+), 42 deletions(-) create mode 100644 bin/lib/examples/networking-offline.test.ts create mode 100644 bin/lib/exports/crypto-browser.ts create mode 100644 bin/lib/exports/networking-offline.ts create mode 100644 packages/core/crypto/README.md create mode 100644 packages/core/crypto/browser.test.ts create mode 100644 packages/core/crypto/browser.ts create mode 100644 packages/core/listener/index.test.ts create mode 100644 packages/jam/networking-offline/README.md create mode 100644 packages/jam/networking-offline/index.test.ts create mode 100644 packages/jam/networking-offline/index.ts create mode 100644 packages/jam/networking-offline/package.json create mode 100644 packages/workers/jam-network/messages.ts diff --git a/bin/lib/README.md b/bin/lib/README.md index 25fec293b..bb3024368 100644 --- a/bin/lib/README.md +++ b/bin/lib/README.md @@ -52,6 +52,7 @@ The following modules are available as subpath imports (e.g., `@typeberry/lib/bl - `config` - Configuration types - `config-node` - Node configuration utilities - `crypto` - Cryptographic primitives (Ed25519, Sr25519, BLS) +- `crypto-browser` - Browser-safe cryptographic sizes and opaque data types - `database` - Database abstractions - `erasure-coding` - Erasure coding implementation - `fuzz-proto` - Fuzzing protocol support @@ -61,6 +62,7 @@ The following modules are available as subpath imports (e.g., `@typeberry/lib/bl - `json-parser` - JSON parsing utilities - `logger` - Logging framework - `mmr` - Merkle Mountain Range implementation +- `networking-offline` - Programmatically controlled offline JAM networking implementation - `numbers` - Fixed-size numeric types - `ordering` - Ordering and comparison utilities - `pvm-host-calls` - PVM host call implementations @@ -76,6 +78,76 @@ The following modules are available as subpath imports (e.g., `@typeberry/lib/bl - `utils` - General utilities - `workers-api` - Workers API utilities +### Browser-safe crypto metadata + +Use `@typeberry/lib/crypto-browser` when browser code only needs encoded +cryptographic sizes or opaque key and signature types: + +```typescript +import { ED25519_SIGNATURE_BYTES } from "@typeberry/lib/crypto-browser"; +import type { Ed25519Signature } from "@typeberry/lib/crypto-browser"; +``` + +This entry point does not load signing, verification, native bindings, or WASM +initialization. Use `@typeberry/lib/crypto` when those operations are required. + +### Offline networking controller + +The offline networking module is a browser-safe implementation of the standard +networking-worker protocol. A node host can wire its `network` endpoint to an +importer and best-header source, while callers control peer traffic through +`offline`: + + +```typescript +import { startOfflineNetworkingWorker } from "@typeberry/lib/networking-offline"; +import { Block, emptyBlock, reencodeAsView } from "@typeberry/lib/block"; +import { Bytes } from "@typeberry/lib/bytes"; +import { tinyChainSpec } from "@typeberry/lib/config"; +import { HASH_SIZE, WithHash } from "@typeberry/lib/hash"; +import { DirectPort } from "@typeberry/lib/workers-api"; + +type BlockView = import("@typeberry/lib/block").BlockView; +type HeaderHash = import("@typeberry/lib/block").HeaderHash; + +const authorshipPorts = DirectPort.pair(); +const worker = startOfflineNetworkingWorker(authorshipPorts[0]); +// Connect authorshipPorts[1] to block authorship. + +// Connect blocks received through offline networking to the importer. +const importedBlocks: BlockView[] = []; +worker.network.setOnBlocks(async (blocks) => { + importedBlocks.push(...blocks); +}); + +const block = reencodeAsView(Block.Codec, emptyBlock(), tinyChainSpec); +await worker.offline.submitBlock(block); +assert.strictEqual(importedBlocks[0], block); + +// Outgoing online-network announcements are observable programmatically. +let announcedHeader: unknown = null; +worker.offline.announcedHeaders.once((header) => { + announcedHeader = header; +}); +const header = WithHash.new(Bytes.zero(HASH_SIZE).asOpaque(), block.header.view()); +await worker.network.sendNewHeader(header); +assert.strictEqual(announcedHeader, header); + +await worker.finish(); +``` + + +`submitBlock` and `submitBlocks` resolve once the standard networking protocol +has delivered the blocks; importer acceptance remains intentionally separate. +`submitTickets` returns the connected authorship module's validation decision. +The announcement events expose headers and ticket messages that online +networking would distribute to peers. Calling `finish()` is idempotent, and +submission methods reject after shutdown. + +The concrete `@typeberry/node` package deliberately does not compose this +controller yet. A future node integration should expose an RPC or equivalent +control surface so external callers can submit offline network traffic. + ## Examples All examples below are extracted from actual test files in `examples/` directory, ensuring they compile and work correctly. diff --git a/bin/lib/examples/networking-offline.test.ts b/bin/lib/examples/networking-offline.test.ts new file mode 100644 index 000000000..f432b94b0 --- /dev/null +++ b/bin/lib/examples/networking-offline.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; + +describe("Offline Networking Example", () => { + it("should submit blocks and observe outgoing announcements", async () => { + // + const { startOfflineNetworkingWorker } = await import("@typeberry/lib/networking-offline"); + const { Block, emptyBlock, reencodeAsView } = await import("@typeberry/lib/block"); + type BlockView = import("@typeberry/lib/block").BlockView; + type HeaderHash = import("@typeberry/lib/block").HeaderHash; + const { Bytes } = await import("@typeberry/lib/bytes"); + const { tinyChainSpec } = await import("@typeberry/lib/config"); + const { HASH_SIZE, WithHash } = await import("@typeberry/lib/hash"); + const { DirectPort } = await import("@typeberry/lib/workers-api"); + + const authorshipPorts = DirectPort.pair(); + const worker = startOfflineNetworkingWorker(authorshipPorts[0]); + // Connect authorshipPorts[1] to block authorship. + + // Connect blocks received through offline networking to the importer. + const importedBlocks: BlockView[] = []; + worker.network.setOnBlocks(async (blocks) => { + importedBlocks.push(...blocks); + }); + + const block = reencodeAsView(Block.Codec, emptyBlock(), tinyChainSpec); + await worker.offline.submitBlock(block); + assert.strictEqual(importedBlocks[0], block); + + // Outgoing online-network announcements are observable programmatically. + let announcedHeader: unknown = null; + worker.offline.announcedHeaders.once((header) => { + announcedHeader = header; + }); + const header = WithHash.new(Bytes.zero(HASH_SIZE).asOpaque(), block.header.view()); + await worker.network.sendNewHeader(header); + assert.strictEqual(announcedHeader, header); + + await worker.finish(); + // + }); +}); diff --git a/bin/lib/exports/crypto-browser.ts b/bin/lib/exports/crypto-browser.ts new file mode 100644 index 000000000..990835117 --- /dev/null +++ b/bin/lib/exports/crypto-browser.ts @@ -0,0 +1,8 @@ +/** + * Browser-safe cryptographic sizes and opaque key/signature types. + * + * This module does not expose signing, verification, or native/WASM setup. + * + * @module crypto-browser + */ +export * from "@typeberry/crypto/browser.js"; diff --git a/bin/lib/exports/networking-offline.ts b/bin/lib/exports/networking-offline.ts new file mode 100644 index 000000000..fc8e73b70 --- /dev/null +++ b/bin/lib/exports/networking-offline.ts @@ -0,0 +1,6 @@ +/** + * Programmatically controlled offline implementation of the JAM networking protocol. + * + * @module networking-offline + */ +export * from "@typeberry/networking-offline"; diff --git a/bin/lib/index.ts b/bin/lib/index.ts index 529a9d465..1a316321f 100644 --- a/bin/lib/index.ts +++ b/bin/lib/index.ts @@ -9,6 +9,7 @@ export * as collections from "./exports/collections.js"; export * as config from "./exports/config.js"; export * as config_node from "./exports/config-node.js"; export * as crypto from "./exports/crypto.js"; +export * as crypto_browser from "./exports/crypto-browser.js"; export * as database from "./exports/database.js"; export * as erasure_coding from "./exports/erasure-coding.js"; export * as fuzz_proto from "./exports/fuzz-proto.js"; @@ -18,6 +19,7 @@ export * as jam_host_calls from "./exports/jam-host-calls.js"; export * as json_parser from "./exports/json-parser.js"; export * as logger from "./exports/logger.js"; export * as mmr from "./exports/mmr.js"; +export * as networking_offline from "./exports/networking-offline.js"; export * as numbers from "./exports/numbers.js"; export * as ordering from "./exports/ordering.js"; export * as pvm_host_calls from "./exports/pvm-host-calls.js"; diff --git a/bin/lib/package.json b/bin/lib/package.json index ecaceaf78..7a483265c 100644 --- a/bin/lib/package.json +++ b/bin/lib/package.json @@ -17,6 +17,7 @@ "./config": "./exports/config.js", "./config-node": "./exports/config-node.js", "./crypto": "./exports/crypto.js", + "./crypto-browser": "./exports/crypto-browser.js", "./database": "./exports/database.js", "./erasure-coding": "./exports/erasure-coding.js", "./fuzz-proto": "./exports/fuzz-proto.js", @@ -26,6 +27,7 @@ "./json-parser": "./exports/json-parser.js", "./logger": "./exports/logger.js", "./mmr": "./exports/mmr.js", + "./networking-offline": "./exports/networking-offline.js", "./numbers": "./exports/numbers.js", "./ordering": "./exports/ordering.js", "./package.json": "./package.json", @@ -66,6 +68,7 @@ "@typeberry/json-parser": "*", "@typeberry/logger": "*", "@typeberry/mmr": "*", + "@typeberry/networking-offline": "*", "@typeberry/numbers": "*", "@typeberry/ordering": "*", "@typeberry/pvm-host-calls": "*", diff --git a/package-lock.json b/package-lock.json index f3f678877..344fbbd5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,6 +51,7 @@ "packages/jam/in-core", "packages/jam/jam-host-calls", "packages/jam/jamnp-s", + "packages/jam/networking-offline", "packages/jam/node", "packages/jam/rpc", "packages/jam/rpc-client", @@ -220,6 +221,7 @@ "@typeberry/logger": "*", "@typeberry/mmr": "*", "@typeberry/native": "0.5.1", + "@typeberry/networking-offline": "*", "@typeberry/numbers": "*", "@typeberry/ordering": "*", "@typeberry/pvm-host-calls": "*", @@ -3695,6 +3697,10 @@ "resolved": "packages/core/networking", "link": true }, + "node_modules/@typeberry/networking-offline": { + "resolved": "packages/jam/networking-offline", + "link": true + }, "node_modules/@typeberry/node": { "resolved": "packages/jam/node", "link": true @@ -8769,7 +8775,10 @@ "packages/core/listener": { "name": "@typeberry/listener", "version": "0.11.0", - "license": "MPL-2.0" + "license": "MPL-2.0", + "dependencies": { + "eventemitter3": "^5.0.1" + } }, "packages/core/logger": { "name": "@typeberry/logger", @@ -9115,6 +9124,23 @@ "@typeberry/utils": "*" } }, + "packages/jam/networking-offline": { + "name": "@typeberry/networking-offline", + "version": "0.11.0", + "license": "MPL-2.0", + "dependencies": { + "@typeberry/block": "*", + "@typeberry/comms-authorship-network": "*", + "@typeberry/hash": "*", + "@typeberry/jam-network": "*", + "@typeberry/listener": "*", + "@typeberry/workers-api": "*" + }, + "devDependencies": { + "@typeberry/bytes": "*", + "@typeberry/config": "*" + } + }, "packages/jam/node": { "name": "@typeberry/node", "version": "0.11.0", diff --git a/package.json b/package.json index b7134a562..b166240b0 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "packages/jam/in-core", "packages/jam/jam-host-calls", "packages/jam/jamnp-s", + "packages/jam/networking-offline", "packages/jam/node", "packages/jam/rpc", "packages/jam/rpc-client", diff --git a/packages/core/crypto/README.md b/packages/core/crypto/README.md new file mode 100644 index 000000000..54dc96e12 --- /dev/null +++ b/packages/core/crypto/README.md @@ -0,0 +1,14 @@ +# Crypto entry points + +`@typeberry/crypto` is the complete cryptographic implementation. It includes +native/WASM initialization and signing or verification operations. + +`@typeberry/crypto/browser.js` is the dependency-light entry point for code +which only needs encoded key/signature sizes and their opaque TypeScript types. +It has no runtime imports and does not expose cryptographic operations. Its +constants intentionally mirror the canonical values in `ed25519.ts` and +`bandersnatch.ts`; their literal type annotations and unit test catch drift. + +Browser-facing codecs should import from `@typeberry/crypto/browser.js` rather +than the package root, so merely describing encoded data does not load native +implementations. diff --git a/packages/core/crypto/browser.test.ts b/packages/core/crypto/browser.test.ts new file mode 100644 index 000000000..0b2572918 --- /dev/null +++ b/packages/core/crypto/browser.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import * as bandersnatch from "./bandersnatch.js"; +import * as browser from "./browser.js"; +import * as ed25519 from "./ed25519.js"; + +describe("browser-safe crypto constants", () => { + it("match their canonical implementation values", () => { + assert.strictEqual(browser.ED25519_PRIV_KEY_BYTES, ed25519.ED25519_PRIV_KEY_BYTES); + assert.strictEqual(browser.ED25519_KEY_BYTES, ed25519.ED25519_KEY_BYTES); + assert.strictEqual(browser.ED25519_SIGNATURE_BYTES, ed25519.ED25519_SIGNATURE_BYTES); + assert.strictEqual(browser.BANDERSNATCH_KEY_BYTES, bandersnatch.BANDERSNATCH_KEY_BYTES); + assert.strictEqual(browser.BANDERSNATCH_VRF_SIGNATURE_BYTES, bandersnatch.BANDERSNATCH_VRF_SIGNATURE_BYTES); + assert.strictEqual(browser.BANDERSNATCH_RING_ROOT_BYTES, bandersnatch.BANDERSNATCH_RING_ROOT_BYTES); + assert.strictEqual(browser.BANDERSNATCH_PROOF_BYTES, bandersnatch.BANDERSNATCH_PROOF_BYTES); + assert.strictEqual(browser.BLS_KEY_BYTES, bandersnatch.BLS_KEY_BYTES); + }); +}); diff --git a/packages/core/crypto/browser.ts b/packages/core/crypto/browser.ts new file mode 100644 index 000000000..569a7a7f5 --- /dev/null +++ b/packages/core/crypto/browser.ts @@ -0,0 +1,49 @@ +/** + * Browser-safe cryptographic sizes and opaque data types. + * + * This module intentionally has no runtime imports. The annotations on the + * mirrored constants make TypeScript fail if their canonical values in the + * implementation modules change. + */ + +type CanonicalEd25519PrivateKeyBytes = typeof import("./ed25519.js").ED25519_PRIV_KEY_BYTES; +type CanonicalEd25519KeyBytes = typeof import("./ed25519.js").ED25519_KEY_BYTES; +type CanonicalEd25519SignatureBytes = typeof import("./ed25519.js").ED25519_SIGNATURE_BYTES; +type CanonicalBandersnatchKeyBytes = typeof import("./bandersnatch.js").BANDERSNATCH_KEY_BYTES; +type CanonicalBandersnatchVrfSignatureBytes = typeof import("./bandersnatch.js").BANDERSNATCH_VRF_SIGNATURE_BYTES; +type CanonicalBandersnatchRingRootBytes = typeof import("./bandersnatch.js").BANDERSNATCH_RING_ROOT_BYTES; +type CanonicalBandersnatchProofBytes = typeof import("./bandersnatch.js").BANDERSNATCH_PROOF_BYTES; +type CanonicalBlsKeyBytes = typeof import("./bandersnatch.js").BLS_KEY_BYTES; + +export const ED25519_PRIV_KEY_BYTES: CanonicalEd25519PrivateKeyBytes = 32; +export type ED25519_PRIV_KEY_BYTES = typeof ED25519_PRIV_KEY_BYTES; + +export const ED25519_KEY_BYTES: CanonicalEd25519KeyBytes = 32; +export type ED25519_KEY_BYTES = typeof ED25519_KEY_BYTES; + +export const ED25519_SIGNATURE_BYTES: CanonicalEd25519SignatureBytes = 64; +export type ED25519_SIGNATURE_BYTES = typeof ED25519_SIGNATURE_BYTES; + +export const BANDERSNATCH_KEY_BYTES: CanonicalBandersnatchKeyBytes = 32; +export type BANDERSNATCH_KEY_BYTES = typeof BANDERSNATCH_KEY_BYTES; + +export const BANDERSNATCH_VRF_SIGNATURE_BYTES: CanonicalBandersnatchVrfSignatureBytes = 96; +export type BANDERSNATCH_VRF_SIGNATURE_BYTES = typeof BANDERSNATCH_VRF_SIGNATURE_BYTES; + +export const BANDERSNATCH_RING_ROOT_BYTES: CanonicalBandersnatchRingRootBytes = 144; +export type BANDERSNATCH_RING_ROOT_BYTES = typeof BANDERSNATCH_RING_ROOT_BYTES; + +export const BANDERSNATCH_PROOF_BYTES: CanonicalBandersnatchProofBytes = 784; +export type BANDERSNATCH_PROOF_BYTES = typeof BANDERSNATCH_PROOF_BYTES; + +export const BLS_KEY_BYTES: CanonicalBlsKeyBytes = 144; +export type BLS_KEY_BYTES = typeof BLS_KEY_BYTES; + +export type { + BandersnatchKey, + BandersnatchProof, + BandersnatchRingRoot, + BandersnatchVrfSignature, + BlsKey, +} from "./bandersnatch.js"; +export type { Ed25519Key, Ed25519Signature } from "./ed25519.js"; diff --git a/packages/core/listener/index.test.ts b/packages/core/listener/index.test.ts new file mode 100644 index 000000000..64c29e846 --- /dev/null +++ b/packages/core/listener/index.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Listener } from "./index.js"; + +describe("Listener", () => { + it("supports persistent, one-shot, done, and removed listeners", () => { + const event = new Listener(); + const values: number[] = []; + const persistent = (value: number) => values.push(value); + let done = false; + + event.on(persistent); + event.once((value) => values.push(value * 10)); + event.onceDone(() => { + done = true; + }); + + event.emit(1); + event.off(persistent); + event.emit(2); + event.on((value) => values.push(value * 100)); + event.markDone(); + event.emit(3); + + assert.deepStrictEqual(values, [1, 10]); + assert.strictEqual(done, true); + }); + + it("applies subscription changes on the next emission", () => { + const event = new Listener(); + const values: string[] = []; + const lateListener = (value: number) => values.push(`late:${value}`); + const secondListener = (value: number) => values.push(`second:${value}`); + + event.on((value) => { + values.push(`first:${value}`); + event.off(secondListener); + event.on(lateListener); + }); + event.on(secondListener); + + event.emit(1); + event.emit(2); + + assert.deepStrictEqual(values, ["first:1", "second:1", "first:2", "late:2"]); + }); + + it("defers callback handlers until after their response", async () => { + const event = new Listener(); + let received: number | null = null; + event.on((value) => { + received = value; + }); + + await event.callbackHandler()(1); + assert.strictEqual(received, null); + + await new Promise((resolve) => globalThis.setTimeout(resolve, 0)); + assert.strictEqual(received, 1); + }); +}); diff --git a/packages/core/listener/index.ts b/packages/core/listener/index.ts index 3d61d2694..deb31335b 100644 --- a/packages/core/listener/index.ts +++ b/packages/core/listener/index.ts @@ -1,11 +1,16 @@ -import { EventEmitter } from "node:events"; +import { EventEmitter } from "eventemitter3"; const EVENT = Symbol(); const EVENT_DONE = Symbol(); +type ListenerEvents = { + [EVENT]: [data: T]; + [EVENT_DONE]: []; +}; + /** A typed version of event emitter. */ export class Listener { - private readonly emitter = new EventEmitter(); + private readonly emitter = new EventEmitter>(); emit(data: T) { this.emitter.emit(EVENT, data); @@ -43,7 +48,7 @@ export class Listener { // to avoid deadlocks and since we don't care about that signal, // we simply return the response immediately and process the emitting // later - setImmediate(() => this.emit(req)); + globalThis.setTimeout(() => this.emit(req), 0); }; } } diff --git a/packages/core/listener/package.json b/packages/core/listener/package.json index 96b02801b..7749c6672 100644 --- a/packages/core/listener/package.json +++ b/packages/core/listener/package.json @@ -3,6 +3,9 @@ "version": "0.11.0", "description": "Typed event-emitter and listener.", "main": "index.ts", + "dependencies": { + "eventemitter3": "^5.0.1" + }, "scripts": { "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')" }, diff --git a/packages/core/utils/test.ts b/packages/core/utils/test.ts index 7b74d39e1..c18bb9021 100644 --- a/packages/core/utils/test.ts +++ b/packages/core/utils/test.ts @@ -1,7 +1,6 @@ /** * Utilities for tests. */ -import assert from "node:assert"; import { inspect } from "./debug.js"; import type { Result } from "./result.js"; @@ -62,14 +61,15 @@ export function deepEqual( * * This workaround can be removed when this issue is resolved: https://github.com/nodejs/node/issues/57242 */ - const [major, minor] = process.versions.node.split(".").map(Number); - const isOoMWorkaroundNeeded = major > 22 || (major === 22 && minor >= 12); + const nodeVersion = typeof process === "undefined" ? null : process.versions?.node; + const [major = 0, minor = 0] = nodeVersion?.split(".").map(Number) ?? []; + const isOoMWorkaroundNeeded = nodeVersion !== null && (major > 22 || (major === 22 && minor >= 12)); const message = isOoMWorkaroundNeeded ? new Error(`${actual} != ${expected}`) : undefined; const actualDisp = actual === null || actual === undefined ? actual : `${inspect(actual)}`; const expectedDisp = expected === null || expected === undefined ? expected : `${inspect(expected)}`; try { - assert.strictEqual(actualDisp, expectedDisp, message); + strictEqual(actualDisp, expectedDisp, message); } catch (e) { if (isOoMWorkaroundNeeded && !oomWarningPrinted) { // biome-ignore lint/suspicious/noConsole: warning @@ -188,12 +188,18 @@ export function deepEqual( errors.tryAndCatch(() => { // fallback - assert.strictEqual(actual, expected); + strictEqual(actual, expected); }, ctx); return errors.exitOrThrow(); } +function strictEqual(actual: unknown, expected: unknown, message?: Error): void { + if (!Object.is(actual, expected)) { + throw message ?? new Error(`Expected ${inspect(actual)} to strictly equal ${inspect(expected)}`); + } +} + function getAllKeysSorted(a: (keyof T)[], b: (keyof T)[]): (keyof T)[] { const all = new Set(a.concat(b)); return Array.from(all).sort(); diff --git a/packages/jam/block/assurances.ts b/packages/jam/block/assurances.ts index d00fc123d..3f7d822cb 100644 --- a/packages/jam/block/assurances.ts +++ b/packages/jam/block/assurances.ts @@ -1,7 +1,7 @@ import type { BitVec } from "@typeberry/bytes"; import { type CodecRecord, codec, type DescribedBy } from "@typeberry/codec"; import type { KnownSizeArray } from "@typeberry/collections"; -import { ED25519_SIGNATURE_BYTES, type Ed25519Signature } from "@typeberry/crypto"; +import { ED25519_SIGNATURE_BYTES, type Ed25519Signature } from "@typeberry/crypto/browser.js"; import { HASH_SIZE } from "@typeberry/hash"; import { WithDebug } from "@typeberry/utils"; import { codecKnownSizeArray, codecWithContext } from "./codec-utils.js"; diff --git a/packages/jam/block/disputes.ts b/packages/jam/block/disputes.ts index 6d0fcc8d1..edb6a383d 100644 --- a/packages/jam/block/disputes.ts +++ b/packages/jam/block/disputes.ts @@ -1,6 +1,11 @@ import { type CodecRecord, codec } from "@typeberry/codec"; import { asKnownSize, type KnownSizeArray } from "@typeberry/collections"; -import { ED25519_KEY_BYTES, ED25519_SIGNATURE_BYTES, type Ed25519Key, type Ed25519Signature } from "@typeberry/crypto"; +import { + ED25519_KEY_BYTES, + ED25519_SIGNATURE_BYTES, + type Ed25519Key, + type Ed25519Signature, +} from "@typeberry/crypto/browser.js"; import { HASH_SIZE } from "@typeberry/hash"; import { seeThrough, WithDebug } from "@typeberry/utils"; import { codecWithContext } from "./codec-utils.js"; diff --git a/packages/jam/block/guarantees.ts b/packages/jam/block/guarantees.ts index 56be74cc3..59c42a832 100644 --- a/packages/jam/block/guarantees.ts +++ b/packages/jam/block/guarantees.ts @@ -1,6 +1,6 @@ import { type CodecRecord, codec, type DescribedBy } from "@typeberry/codec"; import type { KnownSizeArray } from "@typeberry/collections"; -import { ED25519_SIGNATURE_BYTES, type Ed25519Signature } from "@typeberry/crypto"; +import { ED25519_SIGNATURE_BYTES, type Ed25519Signature } from "@typeberry/crypto/browser.js"; import { WithDebug } from "@typeberry/utils"; import { codecKnownSizeArray, codecWithContext } from "./codec-utils.js"; import type { TimeSlot, ValidatorIndex } from "./common.js"; diff --git a/packages/jam/block/header.ts b/packages/jam/block/header.ts index b8e4e2cf2..8e7ff6638 100644 --- a/packages/jam/block/header.ts +++ b/packages/jam/block/header.ts @@ -1,7 +1,13 @@ import { Bytes, BytesBlob } from "@typeberry/bytes"; import { type CodecRecord, codec, type DescribedBy } from "@typeberry/codec"; -import { BANDERSNATCH_KEY_BYTES, type BandersnatchKey, ED25519_KEY_BYTES, type Ed25519Key } from "@typeberry/crypto"; -import { BANDERSNATCH_VRF_SIGNATURE_BYTES, type BandersnatchVrfSignature } from "@typeberry/crypto/bandersnatch.js"; +import { + BANDERSNATCH_KEY_BYTES, + BANDERSNATCH_VRF_SIGNATURE_BYTES, + type BandersnatchKey, + type BandersnatchVrfSignature, + ED25519_KEY_BYTES, + type Ed25519Key, +} from "@typeberry/crypto/browser.js"; import { HASH_SIZE, WithHash } from "@typeberry/hash"; import { WithDebug } from "@typeberry/utils"; import { diff --git a/packages/jam/block/tickets.ts b/packages/jam/block/tickets.ts index dba83be97..31a8107dd 100644 --- a/packages/jam/block/tickets.ts +++ b/packages/jam/block/tickets.ts @@ -1,7 +1,7 @@ import type { Bytes } from "@typeberry/bytes"; import { type CodecRecord, codec } from "@typeberry/codec"; import type { KnownSizeArray } from "@typeberry/collections"; -import { BANDERSNATCH_PROOF_BYTES, type BandersnatchProof } from "@typeberry/crypto/bandersnatch.js"; +import { BANDERSNATCH_PROOF_BYTES, type BandersnatchProof } from "@typeberry/crypto/browser.js"; import { HASH_SIZE } from "@typeberry/hash"; import { tryAsU8, tryAsU32, type U8 } from "@typeberry/numbers"; import { asOpaqueType, type Opaque, WithDebug } from "@typeberry/utils"; diff --git a/packages/jam/networking-offline/README.md b/packages/jam/networking-offline/README.md new file mode 100644 index 000000000..144cbd788 --- /dev/null +++ b/packages/jam/networking-offline/README.md @@ -0,0 +1,44 @@ +# Offline networking + +`@typeberry/networking-offline` implements the same message boundaries as the +online JAM networking worker, but replaces peer traffic with a programmatic +controller. It does not import `@typeberry/node`, the online networking +implementation, Node worker utilities, or native cryptographic bindings. + +`startOfflineNetworkingWorker(authorshipPort)` does not spawn an operating +system or Web Worker. The name reflects that it implements the standard +networking-worker protocol in the same thread. + +## API surfaces + +The returned object deliberately has two sides: + +- `network` is the node-facing `NetworkingApi`. A host connects its importer to + `setOnBlocks` and forwards newly imported headers with `sendNewHeader`. +- `offline` is the caller-facing `OfflineNetworking` controller. It accepts + blocks and tickets programmatically and exposes the announcements that online + networking would send to peers. + +The supplied port is the offline-networking end of the authorship/networking +protocol. Its peer must be connected to block authorship for ticket submission +and outgoing ticket announcements. A host without authorship should attach a +`receivedTickets` handler which returns `false`. + +`submitBlock` and `submitBlocks` confirm protocol delivery, not importer +acceptance. `submitTickets` returns authorship's validation decision. `finish()` +is idempotent; submissions reject after it completes. + +## Dependency boundary + +Direct subpath imports in this package are intentional. In particular, do not +replace them with the `@typeberry/jam-network`, `@typeberry/workers-api`, or +`@typeberry/comms-authorship-network` barrels without checking the resulting +dependency graph. Those barrels also expose environment-specific +implementations. + +Run the focused checks with: + +```sh +npm test -w @typeberry/networking-offline +npm run build -w @typeberry/lib +``` diff --git a/packages/jam/networking-offline/index.test.ts b/packages/jam/networking-offline/index.test.ts new file mode 100644 index 000000000..d267cd7c3 --- /dev/null +++ b/packages/jam/networking-offline/index.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Block, emptyBlock, type HeaderHash, reencodeAsView, tryAsEpoch } from "@typeberry/block"; +import { Bytes } from "@typeberry/bytes"; +import { + protocol as authorshipProtocol, + type NetworkingComms, + TicketsMessage, +} from "@typeberry/comms-authorship-network"; +import { tinyChainSpec } from "@typeberry/config"; +import { HASH_SIZE, WithHash } from "@typeberry/hash"; +import { Channel, DirectPort } from "@typeberry/workers-api"; +import { startOfflineNetworkingWorker } from "./index.js"; + +describe("OfflineNetworking", () => { + it("routes all inbound and outbound networking messages", async () => { + const [networkingPort, authorshipPort] = DirectPort.pair(); + const worker = startOfflineNetworkingWorker(networkingPort); + const authorship: NetworkingComms = Channel.tx(authorshipProtocol, authorshipPort); + + const block = reencodeAsView(Block.Codec, emptyBlock(), tinyChainSpec); + let submittedBlock: typeof block | null = null; + worker.network.setOnBlocks(async (blocks) => { + submittedBlock = blocks[0]; + }); + await worker.offline.submitBlock(block); + assert.strictEqual(submittedBlock, block); + + const header = WithHash.new(Bytes.zero(HASH_SIZE).asOpaque(), block.header.view()); + let announcedHeader: typeof header | null = null; + worker.offline.announcedHeaders.once((value) => { + announcedHeader = value; + }); + await worker.network.sendNewHeader(header); + assert.strictEqual(announcedHeader, header); + + const tickets = TicketsMessage.create({ epochIndex: tryAsEpoch(1), tickets: [] }); + let announcedTickets: TicketsMessage | null = null; + worker.offline.announcedTickets.once((value) => { + announcedTickets = value; + }); + await authorship.sendTickets(tickets); + assert.strictEqual(announcedTickets, tickets); + + let replacement: TicketsMessage | null = null; + worker.offline.ticketPoolReplacements.once((value) => { + replacement = value; + }); + await authorship.sendReplaceTicketPool(tickets); + assert.strictEqual(replacement, tickets); + + const receivedTickets: TicketsMessage[] = []; + let acceptTickets = true; + authorship.setOnReceivedTickets(async (value) => { + receivedTickets.push(value); + return acceptTickets; + }); + assert.strictEqual(await worker.offline.submitTickets(tickets.epochIndex, tickets.tickets), true); + acceptTickets = false; + assert.strictEqual(await worker.offline.submitTickets(tickets.epochIndex, tickets.tickets), false); + assert.strictEqual(receivedTickets[0].epochIndex, tickets.epochIndex); + assert.strictEqual(receivedTickets[0].tickets, tickets.tickets); + assert.strictEqual(receivedTickets.length, 2); + + const firstFinish = worker.finish(); + assert.strictEqual(worker.finish(), firstFinish); + await firstFinish; + await assert.rejects(worker.offline.submitBlock(block), /Offline networking has finished/); + await assert.rejects( + worker.offline.submitTickets(tickets.epochIndex, tickets.tickets), + /Offline networking has finished/, + ); + }); +}); diff --git a/packages/jam/networking-offline/index.ts b/packages/jam/networking-offline/index.ts new file mode 100644 index 000000000..5898e8c82 --- /dev/null +++ b/packages/jam/networking-offline/index.ts @@ -0,0 +1,132 @@ +import type { BlockView, HeaderHash, HeaderView } from "@typeberry/block"; +import type { SignedTicket } from "@typeberry/block/tickets.js"; +import { type AuthorshipComms, protocol as authorshipProtocol } from "@typeberry/comms-authorship-network/protocol.js"; +import { TicketsMessage } from "@typeberry/comms-authorship-network/tickets-message.js"; +import type { WithHash } from "@typeberry/hash"; +import { + type NetworkingApi, + type NetworkingInternal, + protocol as networkingProtocol, +} from "@typeberry/jam-network/messages.js"; +import { Listener } from "@typeberry/listener"; +import { Channel } from "@typeberry/workers-api/channel.js"; +import type { Port } from "@typeberry/workers-api/port.js"; +import { startSameThread } from "@typeberry/workers-api/protocol.js"; + +/** + * Programmatic handle for an offline networking worker. + * + * Inputs model messages received from peers. Events model messages which the + * online networking worker would broadcast to peers. + */ +export class OfflineNetworking { + /** Best-header announcements produced after successful imports. */ + readonly announcedHeaders = new Listener>(); + /** Locally authored tickets which should be distributed to peers. */ + readonly announcedTickets = new Listener(); + /** Authoritative ticket-pool snapshots produced on epoch boundaries. */ + readonly ticketPoolReplacements = new Listener(); + private isFinished = false; + + private constructor( + private readonly networkingComms: NetworkingInternal, + private readonly authorshipComms: AuthorshipComms, + /** Resolves after the offline networking protocol has finished. */ + readonly finished: Promise, + ) {} + + static start(networkingComms: NetworkingInternal, authorshipComms: AuthorshipComms): OfflineNetworking { + let finish: () => void = () => {}; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const offline = new OfflineNetworking(networkingComms, authorshipComms, finished); + + networkingComms.setOnNewHeader(async (header) => { + offline.announcedHeaders.emit(header); + }); + networkingComms.setOnFinish(async () => { + offline.isFinished = true; + offline.announcedHeaders.markDone(); + offline.announcedTickets.markDone(); + offline.ticketPoolReplacements.markDone(); + finish(); + }); + + authorshipComms.setOnTickets(async (tickets) => { + offline.announcedTickets.emit(tickets); + }); + authorshipComms.setOnReplaceTicketPool(async (tickets) => { + offline.ticketPoolReplacements.emit(tickets); + }); + + return offline; + } + + /** Submit one block as though it had been received from a peer. */ + async submitBlock(block: BlockView): Promise { + await this.submitBlocks([block]); + } + + /** Submit a batch of blocks as though it had been received from peers. */ + async submitBlocks(blocks: BlockView[]): Promise { + this.assertActive(); + await this.networkingComms.sendBlocks(blocks); + } + + /** + * Submit tickets as though they had been received from a peer. + * + * The result is the real authorship worker's validation decision for the + * complete batch. + */ + async submitTickets(epochIndex: TicketsMessage["epochIndex"], tickets: SignedTicket[]): Promise { + this.assertActive(); + return await this.authorshipComms.sendReceivedTickets(TicketsMessage.create({ epochIndex, tickets })); + } + + private assertActive(): void { + if (this.isFinished) { + throw new Error("Offline networking has finished"); + } + } +} + +export type OfflineNetworkingWorker = { + /** Node-facing side of the standard networking-worker protocol. */ + network: NetworkingApi; + /** Programmatic replacement for network peer traffic. */ + offline: OfflineNetworking; + /** Stop the offline worker and close its channels. */ + finish(): Promise; +}; + +/** + * Start a same-thread networking worker controlled through {@link OfflineNetworking}. + * + * The authorship port may lead to either a same-thread or a worker-thread + * authorship module, so persistent and in-memory nodes exercise the same APIs. + */ +export function startOfflineNetworkingWorker(authorshipPort: Port): OfflineNetworkingWorker { + const { api: network, internal } = startSameThread(networkingProtocol); + const authorshipComms = Channel.rx(authorshipProtocol, authorshipPort); + const offline = OfflineNetworking.start(internal, authorshipComms); + let finishPromise: Promise | null = null; + + return { + network, + offline, + finish: () => { + finishPromise ??= (async () => { + try { + await network.sendFinish(); + await offline.finished; + } finally { + network.destroy(); + authorshipComms.destroy(); + } + })(); + return finishPromise; + }, + }; +} diff --git a/packages/jam/networking-offline/package.json b/packages/jam/networking-offline/package.json new file mode 100644 index 000000000..c93080597 --- /dev/null +++ b/packages/jam/networking-offline/package.json @@ -0,0 +1,24 @@ +{ + "name": "@typeberry/networking-offline", + "version": "0.11.0", + "description": "Programmatically controlled offline JAM networking implementation.", + "main": "index.ts", + "dependencies": { + "@typeberry/block": "*", + "@typeberry/comms-authorship-network": "*", + "@typeberry/hash": "*", + "@typeberry/jam-network": "*", + "@typeberry/listener": "*", + "@typeberry/workers-api": "*" + }, + "devDependencies": { + "@typeberry/bytes": "*", + "@typeberry/config": "*" + }, + "scripts": { + "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')" + }, + "author": "Fluffy Labs", + "license": "MPL-2.0", + "type": "module" +} diff --git a/packages/workers/api/channel.ts b/packages/workers/api/channel.ts index ed4224a92..8606b1cb2 100644 --- a/packages/workers/api/channel.ts +++ b/packages/workers/api/channel.ts @@ -1,5 +1,5 @@ import { Logger } from "@typeberry/logger"; -import { check } from "@typeberry/utils"; +import { check } from "@typeberry/utils/debug.js"; import type { Port } from "./port.js"; import type { Destroy, diff --git a/packages/workers/comms-authorship-network/protocol.ts b/packages/workers/comms-authorship-network/protocol.ts index a9c467692..9dc7a7e65 100644 --- a/packages/workers/comms-authorship-network/protocol.ts +++ b/packages/workers/comms-authorship-network/protocol.ts @@ -1,5 +1,6 @@ import { codec } from "@typeberry/codec"; -import { type Api, createProtocol, type Internal } from "@typeberry/workers-api"; +import { createProtocol } from "@typeberry/workers-api/protocol.js"; +import type { Api, Internal } from "@typeberry/workers-api/types.js"; import { TicketsMessage } from "./tickets-message.js"; /** diff --git a/packages/workers/comms-authorship-network/tickets-message.ts b/packages/workers/comms-authorship-network/tickets-message.ts index c77468d30..409eedeb0 100644 --- a/packages/workers/comms-authorship-network/tickets-message.ts +++ b/packages/workers/comms-authorship-network/tickets-message.ts @@ -1,7 +1,7 @@ import type { Epoch } from "@typeberry/block"; import { SignedTicket } from "@typeberry/block/tickets.js"; import { type CodecRecord, codec } from "@typeberry/codec"; -import { WithDebug } from "@typeberry/utils"; +import { WithDebug } from "@typeberry/utils/debug.js"; export class TicketsMessage extends WithDebug { static Codec = codec.Class(TicketsMessage, { diff --git a/packages/workers/jam-network/messages.ts b/packages/workers/jam-network/messages.ts new file mode 100644 index 000000000..704c3678a --- /dev/null +++ b/packages/workers/jam-network/messages.ts @@ -0,0 +1,28 @@ +import { Block } from "@typeberry/block/block.js"; +import { headerViewWithHashCodec } from "@typeberry/block/header.js"; +import { codec } from "@typeberry/codec"; +import { createProtocol } from "@typeberry/workers-api/protocol.js"; +import type { Api, Internal } from "@typeberry/workers-api/types.js"; + +/** Browser-safe protocol shared by real and manually controlled networking workers. */ +export const protocol = createProtocol("net", { + toWorker: { + newHeader: { + request: headerViewWithHashCodec, + response: codec.nothing, + }, + finish: { + request: codec.nothing, + response: codec.nothing, + }, + }, + fromWorker: { + blocks: { + request: codec.sequenceVarLen(Block.Codec.View), + response: codec.nothing, + }, + }, +}); + +export type NetworkingInternal = Internal; +export type NetworkingApi = Api; diff --git a/packages/workers/jam-network/protocol.ts b/packages/workers/jam-network/protocol.ts index 824fc977d..764c2a5c0 100644 --- a/packages/workers/jam-network/protocol.ts +++ b/packages/workers/jam-network/protocol.ts @@ -1,10 +1,12 @@ -import { Block, type HeaderHash, headerViewWithHashCodec } from "@typeberry/block"; +import type { HeaderHash } from "@typeberry/block"; import { type CodecRecord, codec } from "@typeberry/codec"; import { ED25519_PRIV_KEY_BYTES, type Ed25519SecretSeed } from "@typeberry/crypto"; import { HASH_SIZE } from "@typeberry/hash"; import type { U16 } from "@typeberry/numbers"; import { WithDebug } from "@typeberry/utils"; -import { type Api, createProtocol, type Internal } from "@typeberry/workers-api"; + +export type { NetworkingApi, NetworkingInternal } from "./messages.js"; +export { protocol } from "./messages.js"; /** Network-specific worker initialisatation. */ export class NetworkingConfig extends WithDebug { @@ -35,25 +37,3 @@ export class NetworkingConfig extends WithDebug { super(); } } - -export const protocol = createProtocol("net", { - toWorker: { - newHeader: { - request: headerViewWithHashCodec, - response: codec.nothing, - }, - finish: { - request: codec.nothing, - response: codec.nothing, - }, - }, - fromWorker: { - blocks: { - request: codec.sequenceVarLen(Block.Codec.View), - response: codec.nothing, - }, - }, -}); - -export type NetworkingInternal = Internal; -export type NetworkingApi = Api;