Skip to content
8 changes: 2 additions & 6 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,8 @@ jobs:
cd ./baseline
npm ci
npm run build
if npm run bench:baseline 2>/dev/null; then
cp ./bench/baseline.json ../baseline.json
else
echo "bench:baseline script not found on base branch, skipping comparison"
echo '{}' > ../baseline.json
fi
npm run bench:baseline
cp ./bench/baseline.json ../baseline.json
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
# Run benchmarks on current PR
- name: Run PR benchmark
run: |
Expand Down
21 changes: 19 additions & 2 deletions assembly/api-debugger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,21 @@ import { decodeSpi } from "./spi";

let interpreter: Interpreter | null = null;

export function resetJAM(program: u8[], pc: u32, initialGas: Gas, args: u8[], hasMetadata: boolean = false): void {
export function resetJAM(
program: u8[],
pc: u32,
initialGas: Gas,
args: u8[],
hasMetadata: boolean = false,
useBlockGas: boolean = false,
): void {
const code = hasMetadata ? extractCodeAndMetadata(liftBytes(program)).code : liftBytes(program);

const p = decodeSpi(code, liftBytes(args), 128);
const int = new Interpreter(p.program, p.registers, p.memory);
int.nextPc = <u32>pc;
int.gas.set(initialGas);
int.useBlockGas = useBlockGas;

if (interpreter !== null) {
(<Interpreter>interpreter).memory.free();
Expand All @@ -26,14 +34,21 @@ export function resetJAM(program: u8[], pc: u32, initialGas: Gas, args: u8[], ha
interpreter = int;
}

export function resetGeneric(program: u8[], flatRegisters: u8[], initialGas: Gas, hasMetadata: boolean = false): void {
export function resetGeneric(
program: u8[],
flatRegisters: u8[],
initialGas: Gas,
hasMetadata: boolean = false,
useBlockGas: boolean = false,
): void {
const code = hasMetadata ? extractCodeAndMetadata(liftBytes(program)).code : liftBytes(program);

const p = deblob(code);
const registers: Registers = newRegisters();
fillRegisters(registers, flatRegisters);
const int = new Interpreter(p, registers);
int.gas.set(initialGas);
int.useBlockGas = useBlockGas;

if (interpreter !== null) {
(<Interpreter>interpreter).memory.free();
Expand All @@ -49,6 +64,7 @@ export function resetGenericWithMemory(
chunks: Uint8Array,
initialGas: Gas,
hasMetadata: boolean = false,
useBlockGas: boolean = false,
): void {
const code = hasMetadata ? extractCodeAndMetadata(liftBytes(program)).code : liftBytes(program);

Expand All @@ -61,6 +77,7 @@ export function resetGenericWithMemory(

const int = new Interpreter(p, registers, memory);
int.gas.set(initialGas);
int.useBlockGas = useBlockGas;

interpreter = int;
}
Expand Down
5 changes: 3 additions & 2 deletions assembly/api-internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,18 @@ export function buildMemory(builder: MemoryBuilder, pages: InitialPage[], chunks
}

/** Initialize new VM for execution. */
export function vmInit(input: VmInput, useSbrkGas: boolean = false): Interpreter {
export function vmInit(input: VmInput, useSbrkGas: boolean = false, useBlockGas: boolean = false): Interpreter {
const int = new Interpreter(input.program, input.registers, input.memory);
int.useSbrkGas = useSbrkGas;
int.useBlockGas = useBlockGas;
int.nextPc = input.pc;
int.gas.set(input.gas);
return int;
}

/** Initialize & run & destroy a VM in a single go. */
export function vmRunOnce(input: VmInput, options: VmRunOptions): VmOutput {
const int = vmInit(input, options.useSbrkGas);
const int = vmInit(input, options.useSbrkGas, options.useBlockGas);
vmExecute(int, options.logs);
return vmDestroy(int, options.dumpMemory);
}
Expand Down
1 change: 1 addition & 0 deletions assembly/api-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export class InitialChunk {

export class VmRunOptions {
useSbrkGas: boolean = false;
useBlockGas: boolean = false;
logs: boolean = false;
dumpMemory: boolean = false;
}
Expand Down
6 changes: 4 additions & 2 deletions assembly/api-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ export function runProgram(
logs: boolean = false,
useSbrkGas: boolean = false,
dumpMemory: boolean = false,
useBlockGas: boolean = false,
): VmOutput {
const vmInput = new VmInput(program.program, program.memory, program.registers);
vmInput.gas = i64(initialGas);
Expand All @@ -111,6 +112,7 @@ export function runProgram(
const vmOptions = new VmRunOptions();
vmOptions.logs = logs;
vmOptions.useSbrkGas = useSbrkGas;
vmOptions.useBlockGas = useBlockGas;
vmOptions.dumpMemory = dumpMemory;

return vmRunOnce(vmInput, vmOptions);
Expand All @@ -126,11 +128,11 @@ const pvms = new Map<u32, Interpreter>();
*
* NOTE: the PVM MUST be de-allocated using `pvmDestroy`.
*/
export function pvmStart(program: StandardProgram, useSbrkGas: boolean = false): u32 {
export function pvmStart(program: StandardProgram, useSbrkGas: boolean = false, useBlockGas: boolean = false): u32 {
const vmInput = new VmInput(program.program, program.memory, program.registers);

nextPvmId += 1;
pvms.set(nextPvmId, vmInit(vmInput, useSbrkGas));
pvms.set(nextPvmId, vmInit(vmInput, useSbrkGas, useBlockGas));
return nextPvmId;
}

Expand Down
22 changes: 18 additions & 4 deletions assembly/interpreter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export class Interpreter {
public exitCode: u32;
public nextPc: u32;
public useSbrkGas: boolean;
public useBlockGas: boolean;

private djumpRes: DjumpResult = new DjumpResult();
private argsRes: Args = new Args();
Expand All @@ -64,6 +65,7 @@ export class Interpreter {
this.exitCode = 0;
this.nextPc = 0;
this.useSbrkGas = false;
this.useBlockGas = false;
}

nextSteps(nSteps: u32 = 1): boolean {
Expand Down Expand Up @@ -93,6 +95,8 @@ export class Interpreter {
const mask = program.mask;
const argsRes = this.argsRes;
const outcomeRes = this.outcomeRes;
const useBlockGas = this.useBlockGas;
const blockGasCosts = program.blockGasCosts;

for (let i: u32 = 0; i < nSteps; i++) {
// reset some stuff at start
Expand All @@ -115,10 +119,20 @@ export class Interpreter {
const instruction = unchecked(code[pc]);
const iData = <i32>instruction < INSTRUCTIONS.length ? unchecked(INSTRUCTIONS[instruction]) : MISSING_INSTRUCTION;

// check gas (might be done for each block instead).
if (this.gas.sub(iData.gas)) {
this.status = Status.OOG;
return false;
// check gas: per-block or per-instruction
if (useBlockGas) {
const blockGas = portable.staticArrayAt(blockGasCosts, pc);
if (blockGas > 0) {
if (this.gas.sub(blockGas)) {
this.status = Status.OOG;
return false;
}
}
} else {
if (this.gas.sub(iData.gas)) {
this.status = Status.OOG;
return false;
}
}

if (iData === MISSING_INSTRUCTION) {
Expand Down
45 changes: 44 additions & 1 deletion assembly/program.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,12 +229,55 @@ export class JumpTable {
}

export class Program {
/**
* Pre-computed gas cost per basic block, indexed by PC.
* At block-start PCs, stores the total gas cost for the entire block; 0 elsewhere.
*/
public readonly blockGasCosts: StaticArray<u64>;

constructor(
public readonly code: u8[],
public readonly mask: Mask,
public readonly jumpTable: JumpTable,
public readonly basicBlocks: BasicBlocks,
) {}
) {
const len = code.length;
const blockGasCosts = new StaticArray<u64>(len);

// Two-pass approach: first accumulate gas per block, then store at block-start PCs
let blockStartPc: i32 = -1;
let blockGas: u64 = u64(0);

for (let i = 0; i < len; i++) {
if (!mask.isInstruction(i)) {
continue;
}

const instruction = code[i];
const iData = <i32>instruction < INSTRUCTIONS.length ? INSTRUCTIONS[instruction] : MISSING_INSTRUCTION;

if (basicBlocks.isStart(i)) {
// Store previous block's gas
if (blockStartPc >= 0) {
blockGasCosts[blockStartPc] = blockGas;
}
blockStartPc = i;
blockGas = iData.gas;
} else {
blockGas = portable.u64_add(blockGas, iData.gas);
}

// Skip argument bytes
i += mask.skipBytesToNextInstruction(i);
}

// Store the final block's gas
if (blockStartPc >= 0) {
blockGasCosts[blockStartPc] = blockGas;
}

this.blockGasCosts = blockGasCosts;
}

toString(): string {
return `Program { code: ${this.code}, mask: ${this.mask}, jumpTable: ${this.jumpTable}, basicBlocks: ${this.basicBlocks} }`;
Expand Down
59 changes: 23 additions & 36 deletions bench/compare.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
#!/usr/bin/env node
/**
* Benchmark Comparison Tool
*
*
* Compares two benchmark result JSON files and reports regressions/improvements.
*
*
* Usage:
* tsx bench/compare.ts <baseline.json> <results.json> [--threshold <percent>]
*
*
* Options:
* --threshold <n> Regression threshold as percentage (default: 5%)
* --verbose Show detailed per-trace comparison
*/

import { readFileSync, existsSync } from "node:fs";
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { parseArgs } from "node:util";

Expand All @@ -36,7 +36,7 @@ Options:
}

const [baselinePath, resultsPath] = positionals;
const threshold = parseFloat(values.threshold!);
const threshold = parseFloat(values.threshold ?? "5");

// Validate threshold
if (!Number.isFinite(threshold) || threshold < 0) {
Expand Down Expand Up @@ -81,12 +81,8 @@ const baselineSuite = baseline as SuiteResult;
const resultsSuite = results as SuiteResult;

// Build maps for easy lookup
const baselineTraces = new Map(
baselineSuite.traces.map((t) => [t.name, t])
);
const resultsTraces = new Map(
resultsSuite.traces.map((t) => [t.name, t])
);
const baselineTraces = new Map(baselineSuite.traces.map((t) => [t.name, t]));
const resultsTraces = new Map(resultsSuite.traces.map((t) => [t.name, t]));

interface Comparison {
name: string;
Expand All @@ -98,8 +94,8 @@ interface Comparison {
}

const comparisons: Comparison[] = [];
let regressions: Comparison[] = [];
let improvements: Comparison[] = [];
const regressions: Comparison[] = [];
const improvements: Comparison[] = [];

// Compare traces
for (const [name, baselineTrace] of baselineTraces) {
Expand All @@ -110,11 +106,11 @@ for (const [name, baselineTrace] of baselineTraces) {
}

const diffMs = currentTrace.medianMs - baselineTrace.medianMs;

// Guard against division by zero
let diffPercent: number;
if (baselineTrace.medianMs === 0) {
diffPercent = diffMs === 0 ? 0 : (diffMs > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY);
diffPercent = diffMs === 0 ? 0 : diffMs > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
} else {
diffPercent = (diffMs / baselineTrace.medianMs) * 100;
}
Expand Down Expand Up @@ -163,38 +159,33 @@ console.log(`Threshold: ${threshold}%\n`);

console.log("--- Summary ---\n");
console.log(
`Total trace time: ${baselineSuite.summary.totalTraceMedianMs.toFixed(1)}ms -> ${resultsSuite.summary.totalTraceMedianMs.toFixed(1)}ms`
`Total trace time: ${baselineSuite.summary.totalTraceMedianMs.toFixed(1)}ms -> ${resultsSuite.summary.totalTraceMedianMs.toFixed(1)}ms`,
);
const totalDiff =
resultsSuite.summary.totalTraceMedianMs -
baselineSuite.summary.totalTraceMedianMs;
const totalDiff = resultsSuite.summary.totalTraceMedianMs - baselineSuite.summary.totalTraceMedianMs;

// Guard against division by zero
let totalDiffPercent: number;
if (baselineSuite.summary.totalTraceMedianMs === 0) {
totalDiffPercent = totalDiff === 0 ? 0 : (totalDiff > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY);
totalDiffPercent = totalDiff === 0 ? 0 : totalDiff > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
} else {
totalDiffPercent = (totalDiff / baselineSuite.summary.totalTraceMedianMs) * 100;
}
`Difference: ${totalDiff >= 0 ? "+" : ""}${totalDiff.toFixed(1)}ms (${totalDiffPercent.toFixed(2)}%)`
);
console.log(`Difference: ${totalDiff >= 0 ? "+" : ""}${totalDiff.toFixed(1)}ms (${totalDiffPercent.toFixed(2)}%)`);

if (baselineSuite.w3f && resultsSuite.w3f) {
const w3fDiff =
resultsSuite.w3f.medianMs - baselineSuite.w3f.medianMs;

const w3fDiff = resultsSuite.w3f.medianMs - baselineSuite.w3f.medianMs;

// Guard against division by zero
let w3fDiffPercent: number;
if (baselineSuite.w3f.medianMs === 0) {
w3fDiffPercent = w3fDiff === 0 ? 0 : (w3fDiff > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY);
w3fDiffPercent = w3fDiff === 0 ? 0 : w3fDiff > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
} else {
w3fDiffPercent = (w3fDiff / baselineSuite.w3f.medianMs) * 100;
}
`\nW3F suite: ${baselineSuite.w3f.medianMs.toFixed(1)}ms -> ${resultsSuite.w3f.medianMs.toFixed(1)}ms`
);
console.log(
`Difference: ${w3fDiff >= 0 ? "+" : ""}${w3fDiff.toFixed(1)}ms (${w3fDiffPercent.toFixed(2)}%)`
`\nW3F suite: ${baselineSuite.w3f.medianMs.toFixed(1)}ms -> ${resultsSuite.w3f.medianMs.toFixed(1)}ms`,
);
console.log(`Difference: ${w3fDiff >= 0 ? "+" : ""}${w3fDiff.toFixed(1)}ms (${w3fDiffPercent.toFixed(2)}%)`);
}

console.log(`\nRegressions: ${regressions.length}`);
Expand All @@ -203,18 +194,14 @@ console.log(`Improvements: ${improvements.length}`);
if (regressions.length > 0) {
console.log("\n--- Regressions (worst first) ---\n");
for (const r of regressions) {
console.log(
` ${r.name.padEnd(40)} ${r.currentMs.toFixed(1).padStart(8)}ms (+${r.diffPercent.toFixed(1)}%)`
);
console.log(` ${r.name.padEnd(40)} ${r.currentMs.toFixed(1).padStart(8)}ms (+${r.diffPercent.toFixed(1)}%)`);
}
}

if (improvements.length > 0) {
console.log("\n--- Improvements ---\n");
for (const i of improvements) {
console.log(
` ${i.name.padEnd(40)} ${i.currentMs.toFixed(1).padStart(8)}ms (${i.diffPercent.toFixed(1)}%)`
);
console.log(` ${i.name.padEnd(40)} ${i.currentMs.toFixed(1).padStart(8)}ms (${i.diffPercent.toFixed(1)}%)`);
}
}

Expand All @@ -225,7 +212,7 @@ if (values.verbose && comparisons.length > 0) {
const sign = c.diffMs >= 0 ? "+" : "";
const marker = c.status === "regression" ? "⚠️" : c.status === "improvement" ? "✓" : " ";
console.log(
` ${marker} ${c.name.padEnd(40)} ${c.baselineMs.toFixed(1).padStart(7)}ms -> ${c.currentMs.toFixed(1).padStart(7)}ms (${sign}${c.diffPercent.toFixed(1)}%)`
` ${marker} ${c.name.padEnd(40)} ${c.baselineMs.toFixed(1).padStart(7)}ms -> ${c.currentMs.toFixed(1).padStart(7)}ms (${sign}${c.diffPercent.toFixed(1)}%)`,
);
}
}
Expand Down
Loading
Loading