diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 93bccf62..e9d57824 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -130,3 +130,45 @@ jobs: # delete the comment in case changes no longer impact circuit sizes delete: ${{ !steps.brillig_diff.outputs.markdown }} message: ${{ steps.brillig_diff.outputs.markdown }} + + # Brillig execution profiling (runtime opcode counts) + - name: Profile Brillig execution + run: ./scripts/build-brillig-execution-report.sh + + - name: Compare Brillig execution reports + id: brillig_execution_diff + # Will fail on first run (no baseline on main yet) — remove continue-on-error + # once the first main build has uploaded a baseline artifact. + continue-on-error: true + uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 + with: + report: brillig_execution_report.json + header: | + # Changes to number of Brillig opcodes executed + brillig_report: true + summaryQuantile: 0.9 + + - name: Store Brillig execution benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: "Brillig Execution (opcodes executed)" + tool: "customSmallerIsBetter" + output-file-path: "benchmark-brillig-execution.json" + gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "dev/bench" + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: ${{ github.ref == 'refs/heads/main' }} + comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} + comment-on-alert: true + alert-threshold: "105%" + fail-on-alert: false + max-items-in-chart: 50 + skip-fetch-gh-pages: true + + - name: Add Brillig execution diff to sticky comment + if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: brillig_execution + delete: ${{ !steps.brillig_execution_diff.outputs.markdown }} + message: ${{ steps.brillig_execution_diff.outputs.markdown }} diff --git a/.gitignore b/.gitignore index 1e06e6d4..09848661 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ target .vscode/launch.json -export/* +export/* +benchmark-inputs/ gates_report.json opcode_report.json +brillig_report.json +brillig_execution_report.json +benchmark-*.json diff --git a/scripts/build-brillig-execution-report.sh b/scripts/build-brillig-execution-report.sh new file mode 100755 index 00000000..27dc7f86 --- /dev/null +++ b/scripts/build-brillig-execution-report.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -e + +INSPECTOR=${INSPECTOR:-noir-inspector} +cd $(dirname "$0")/../ + +artifacts_path="./export" +inputs_path="./benchmark-inputs" + +# Generate inputs (each library provides its own generate-benchmark-inputs.sh) +./scripts/generate-benchmark-inputs.sh "$artifacts_path" "$inputs_path" + +# Run execution profiling on each artifact +REPORTS=$(jq --null-input '[]') + +# Known failures (skipped): +# batch_invert_10_elements_{U256,U2048}: U256/U2048 have has_multiplicative_inverse=false +# sqrt_{U256,U2048}: sqrt requires a prime field (asserts has_multiplicative_inverse) +# to_field_BLS12_377Fr: validate_gt in limbs_to_field uses MOD_BITS=253 for range +# checking the subtraction result, but GRUMPKIN_MODULUS[2]=0x3064 needs 14 bits +# (see https://github.com/noir-lang/noir-bignum/issues/266) + +for artifact in $(ls "$artifacts_path"/*.json 2>/dev/null); do + name=$(basename "$artifact" .json) + input_file="$inputs_path/${name}_input.json" + + if [ ! -f "$input_file" ]; then + echo "Skipping $name (no input file)" + continue + fi + + echo "Profiling execution: $name" + OP_CODE_INFO=$($INSPECTOR info --json --profile-execution \ + --input-file "$input_file" \ + "$artifact") || { echo "FAILED: $name"; continue; } + + REPORTS=$(echo "$OP_CODE_INFO" | jq -c '.programs[0] | del(.functions)' \ + | jq -c --argjson old_reports "$REPORTS" '$old_reports + [.]') +done + +echo "$REPORTS" | jq '{ programs: . }' > brillig_execution_report.json + +# Convert to benchmark-action format +jq -r '[.programs[] | .package_name as $pkg | .unconstrained_functions[] | { + "name": (if ($pkg // "") == "" then .name else "\($pkg | sub("^null/"; ""))/\(.name)" end), + "unit": "brillig_opcodes_executed", + "value": (.opcodes // 0) +}]' brillig_execution_report.json > benchmark-brillig-execution.json + +echo "Reports written: brillig_execution_report.json, benchmark-brillig-execution.json" diff --git a/scripts/generate-benchmark-inputs.sh b/scripts/generate-benchmark-inputs.sh new file mode 100755 index 00000000..cedfa986 --- /dev/null +++ b/scripts/generate-benchmark-inputs.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -e + +# Generate JSON input files for noir-inspector --profile-execution from +# compiled artifact ABIs. Each library can replace this script with its +# own input-generation logic. +# +# Usage: ./scripts/generate-benchmark-inputs.sh [artifacts_dir] [output_dir] + +cd $(dirname "$0")/../ + +artifacts_path="${1:-./export}" +inputs_path="${2:-./benchmark-inputs}" +mkdir -p "$inputs_path" + +# jq function that recursively generates a valid input value from an ABI type. +# `seed` controls the base value: different seeds yield distinct values so that +# e.g. a != b for assert_is_not_equal and b != 0 for division. +# +# For integer arrays (BigNum limbs, byte arrays) only the first element carries +# the seed value; the rest are zero. This keeps BigNum values small and safely +# within any field modulus. +JQ_GEN=' +def gen(seed): + if .kind == "field" then (seed | tostring) + elif .kind == "boolean" then false + elif .kind == "integer" then + if .width <= 8 then (seed % 256) else (seed | tostring) end + elif .kind == "array" then + .type as $elem | .length as $len | + if $elem.kind == "integer" then + [$elem | gen(seed)] + [range($len - 1) | $elem | gen(0)] + else + [range($len) | $elem | gen(seed)] + end + elif .kind == "struct" then + [.fields[] | {key: .name, value: (.type | gen(seed))}] | from_entries + else (seed | tostring) + end; +' + +count=0 +for artifact in $(ls "$artifacts_path"/*.json 2>/dev/null); do + name=$(basename "$artifact" .json) + input_file="$inputs_path/${name}_input.json" + + # evaluate_quadratic_expression requires sum(lhs*rhs) + add = 0 mod p. + # All-zero BigNums satisfy this trivially. + if echo "$name" | grep -q "evaluate_quadratic_expression"; then + jq "$JQ_GEN"' + [.abi.parameters | to_entries[] | + {key: .value.name, value: (.value.type | gen(0))} + ] | from_entries + ' "$artifact" > "$input_file" + else + jq "$JQ_GEN"' + [.abi.parameters | to_entries[] | + (.key + 1) as $seed | + {key: .value.name, value: (.value.type | gen($seed))} + ] | from_entries + ' "$artifact" > "$input_file" + fi + + count=$((count + 1)) +done + +echo "Generated $count input files in $inputs_path"