feat(ci): Brillig execution reports

.github/workflows/benchmark.yml

@@ -130,3 +130,45 @@ jobs:
           # delete the comment in case changes no longer impact circuit sizes
           delete: ${{ !steps.brillig_diff.outputs.markdown }}
           message: ${{ steps.brillig_diff.outputs.markdown }}
+
+      # Brillig execution profiling (runtime opcode counts)
+      - name: Profile Brillig execution
+        run: ./scripts/build-brillig-execution-report.sh
+
+      - name: Compare Brillig execution reports
+        id: brillig_execution_diff
+        # Will fail on first run (no baseline on main yet) — remove continue-on-error
+        # once the first main build has uploaded a baseline artifact.
+        continue-on-error: true
+        uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1
+        with:
+          report: brillig_execution_report.json
+          header: |
+            # Changes to number of Brillig opcodes executed
+          brillig_report: true
+          summaryQuantile: 0.9
+
+      - name: Store Brillig execution benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: "Brillig Execution (opcodes executed)"
+          tool: "customSmallerIsBetter"
+          output-file-path: "benchmark-brillig-execution.json"
+          gh-pages-branch: "gh-pages"
+          benchmark-data-dir-path: "dev/bench"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: ${{ github.ref == 'refs/heads/main' }}
+          comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
+          comment-on-alert: true
+          alert-threshold: "105%"
+          fail-on-alert: false
+          max-items-in-chart: 50
+          skip-fetch-gh-pages: true
+
+      - name: Add Brillig execution diff to sticky comment
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: brillig_execution
+          delete: ${{ !steps.brillig_execution_diff.outputs.markdown }}
+          message: ${{ steps.brillig_execution_diff.outputs.markdown }}

.gitignore

@@ -1,5 +1,9 @@
 target
 .vscode/launch.json
-export/* 
+export/*
+benchmark-inputs/
 gates_report.json
 opcode_report.json
+brillig_report.json
+brillig_execution_report.json
+benchmark-*.json

scripts/build-brillig-execution-report.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -e
+
+INSPECTOR=${INSPECTOR:-noir-inspector}
+cd $(dirname "$0")/../
+
+artifacts_path="./export"
+inputs_path="./benchmark-inputs"
+
+# Generate inputs (each library provides its own generate-benchmark-inputs.sh)
+./scripts/generate-benchmark-inputs.sh "$artifacts_path" "$inputs_path"
+
+# Run execution profiling on each artifact
+REPORTS=$(jq --null-input '[]')
+
+# Known failures (skipped):
+#   batch_invert_10_elements_{U256,U2048}: U256/U2048 have has_multiplicative_inverse=false
+#   sqrt_{U256,U2048}: sqrt requires a prime field (asserts has_multiplicative_inverse)
+#   to_field_BLS12_377Fr: validate_gt in limbs_to_field uses MOD_BITS=253 for range
+#     checking the subtraction result, but GRUMPKIN_MODULUS[2]=0x3064 needs 14 bits
+#     (see https://github.com/noir-lang/noir-bignum/issues/266)
+
+for artifact in $(ls "$artifacts_path"/*.json 2>/dev/null); do
+    name=$(basename "$artifact" .json)
+    input_file="$inputs_path/${name}_input.json"
+
+    if [ ! -f "$input_file" ]; then
+        echo "Skipping $name (no input file)"
+        continue
+    fi
+
+    echo "Profiling execution: $name"
+    OP_CODE_INFO=$($INSPECTOR info --json --profile-execution \
+        --input-file "$input_file" \
+        "$artifact") || { echo "FAILED: $name"; continue; }
+
+    REPORTS=$(echo "$OP_CODE_INFO" | jq -c '.programs[0] | del(.functions)' \
+        | jq -c --argjson old_reports "$REPORTS" '$old_reports + [.]')
+done
+
+echo "$REPORTS" | jq '{ programs: . }' > brillig_execution_report.json
+
+# Convert to benchmark-action format
+jq -r '[.programs[] | .package_name as $pkg | .unconstrained_functions[] | {
+    "name": (if ($pkg // "") == "" then .name else "\($pkg | sub("^null/"; ""))/\(.name)" end),
+    "unit": "brillig_opcodes_executed",
+    "value": (.opcodes // 0)
+}]' brillig_execution_report.json > benchmark-brillig-execution.json
+
+echo "Reports written: brillig_execution_report.json, benchmark-brillig-execution.json"

scripts/generate-benchmark-inputs.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -e
+
+# Generate JSON input files for noir-inspector --profile-execution from
+# compiled artifact ABIs.  Each library can replace this script with its
+# own input-generation logic.
+#
+# Usage: ./scripts/generate-benchmark-inputs.sh [artifacts_dir] [output_dir]
+
+cd $(dirname "$0")/../
+
+artifacts_path="${1:-./export}"
+inputs_path="${2:-./benchmark-inputs}"
+mkdir -p "$inputs_path"
+
+# jq function that recursively generates a valid input value from an ABI type.
+# `seed` controls the base value: different seeds yield distinct values so that
+# e.g. a != b for assert_is_not_equal and b != 0 for division.
+#
+# For integer arrays (BigNum limbs, byte arrays) only the first element carries
+# the seed value; the rest are zero.  This keeps BigNum values small and safely
+# within any field modulus.
+JQ_GEN='
+def gen(seed):
+  if .kind == "field" then (seed | tostring)
+  elif .kind == "boolean" then false
+  elif .kind == "integer" then
+    if .width <= 8 then (seed % 256) else (seed | tostring) end
+  elif .kind == "array" then
+    .type as $elem | .length as $len |
+    if $elem.kind == "integer" then
+      [$elem | gen(seed)] + [range($len - 1) | $elem | gen(0)]
+    else
+      [range($len) | $elem | gen(seed)]
+    end
+  elif .kind == "struct" then
+    [.fields[] | {key: .name, value: (.type | gen(seed))}] | from_entries
+  else (seed | tostring)
+  end;
+'
+
+count=0
+for artifact in $(ls "$artifacts_path"/*.json 2>/dev/null); do
+    name=$(basename "$artifact" .json)
+    input_file="$inputs_path/${name}_input.json"
+
+    # evaluate_quadratic_expression requires sum(lhs*rhs) + add = 0 mod p.
+    # All-zero BigNums satisfy this trivially.
+    if echo "$name" | grep -q "evaluate_quadratic_expression"; then
+        jq "$JQ_GEN"'
+          [.abi.parameters | to_entries[] |
+            {key: .value.name, value: (.value.type | gen(0))}
+          ] | from_entries
+        ' "$artifact" > "$input_file"
+    else
+        jq "$JQ_GEN"'
+          [.abi.parameters | to_entries[] |
+            (.key + 1) as $seed |
+            {key: .value.name, value: (.value.type | gen($seed))}
+          ] | from_entries
+        ' "$artifact" > "$input_file"
+    fi
+
+    count=$((count + 1))
+done
+
+echo "Generated $count input files in $inputs_path"