Add pyperformance benchmark run to buildbot

README

@@ -1,19 +1,25 @@
 .. -*- mode: rst -*-
 
-Moved from heptapod
-===================
+Current buidlbot status
+=======================
 
 This is the code for PyPy's buildbot, powering buildbot.pypy.org. It was
-imported in April 2026 from https://foss.heptapod.net/pypy/buildbot.
+imported in April 2026 from https://foss.heptapod.net/pypy/buildbot. The
+service is managed via ssh into the machine, and running the
+``restart_buildmaster_when_not_running`` script. It uses a python2.7 venv.
 
 Everything has been tested with builbot 0.8.8 on CPython2.7
 Note you must pin "automat==20.2" "incremental==21.3.0"
 Testing must use "pytest<4"
 
+
 Patch the source
 ================
 
-buildbot 0.8.8 is really old. There are some patches needed, see patches.patch
+buildbot 0.8.8 is really old. There are some patches needed, see patches.patch.
+If you are starting from a new db, you will need to also patch the installed
+migrate and buildbot packages for newer sqlite3: patch-migrate.patch,
+patch-buildbot.patch.
 
 How to hack the PyPy buildbot
 ==============================
@@ -36,7 +42,7 @@ How to run the PyPy buildbot
 If you want to run buildbot in production, you need to make sure that the
 function ``pypybuildbot.util.we_are_debugging`` returns ``False`` in your
 environment.  At the moment of writing, debugging is enabled everywhere but on
-cobra.
+cobra, which is not really where the buildbot runs.
 
 You still need to fill ``master/slaveinfo.py`` with the passwords of the
 various slaves you want to use.

bot2/pypybuildbot/bulk_upload.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""
+Bulk upload benchmark results to a codespeed instance in a single POST.
+
+Supports two input formats:
+  legacy  - result.json produced by runner.py (unladen_swallow/perf format)
+  pyperf  - result.json produced by pyperformance (pyperf format)
+
+Revision and branch are read from the JSON file when possible (legacy format);
+pass --revision / --branch on the command line to override or when using pyperf.
+
+Examples:
+  # legacy, jit-off results
+  ./bulk_upload.py result.json -e pypy-c-64 -H benchmarker
+
+  # legacy, jit-on results (avg_base values)
+  ./bulk_upload.py result.json -e pypy-c-jit-64 -H benchmarker --baseline
+
+  # pyperformance
+  ./bulk_upload.py pyperf.json -e pypy-c-64 -H benchmarker -r abc123 -B py3.11
+"""
+import argparse
+import json
+import os
+import statistics
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+def detect_format(data):
+    if 'benchmarks' in data and isinstance(data.get('benchmarks'), list):
+        benchmarks = data['benchmarks']
+        if benchmarks and 'runs' in benchmarks[0]:
+            return 'pyperf'
+    if 'results' in data:
+        return 'legacy'
+    raise ValueError("Unrecognised result file format")
+
+
+def parse_legacy(data, changed):
+    """Return list of (name, value, std_dev) from a runner.py result.json.
+
+    changed=True  -> upload avg_changed values (jit-off run, executable pypy-c-NN)
+    changed=False -> upload avg_base values   (jit-on  run, executable pypy-c-jit-NN)
+    """
+    records = []
+    for name, result_type, result_data in data['results']:
+        value = std_dev = None
+
+        if result_type == 'ComparisonResult':
+            if changed:
+                value = result_data.get('avg_changed')
+                std_dev = result_data.get('std_changed')
+            else:
+                value = result_data.get('avg_base')
+                std_dev = result_data.get('std_base')
+
+        elif result_type == 'SimpleComparisonResult':
+            value = result_data['changed_time'] if changed else result_data['base_time']
+
+        elif result_type == 'RawResult':
+            times = result_data['changed_times'] if changed else result_data['base_times']
+            if times:
+                value = times[0] if len(times) == 1 else statistics.mean(times)
+                std_dev = statistics.stdev(times) if len(times) > 1 else None
+
+        if not value:
+            continue
+        records.append((name, value, std_dev))
+    return records
+
+
+def parse_pyperf(data):
+    """Return (records, suite_version) from a pyperformance/pyperf result.json.
+
+    records is a list of (name, mean_seconds, std_dev_or_None).
+    suite_version is taken from the first benchmark's metadata 'version' field.
+    """
+    records = []
+    suite_version = ''
+
+    for bench in data.get('benchmarks', []):
+        meta = bench.get('metadata', {})
+        name = meta.get('name', '')
+        if not name:
+            continue
+        if not suite_version:
+            suite_version = meta.get('version', '')
+
+        # pyperf stores values as floats (seconds) inside each run.
+        # Each value entry is either a plain float or a (loops, time_per_loop) pair.
+        values = []
+        for run in bench.get('runs', []):
+            for v in run.get('values', []):
+                values.append(v[1] if isinstance(v, (list, tuple)) else v)
+
+        if not values:
+            continue
+
+        mean = statistics.mean(values)
+        std_dev = statistics.stdev(values) if len(values) > 1 else None
+        records.append((name, mean, std_dev))
+
+    return records, suite_version
+
+
+def build_codespeed_record(name, value, std_dev, args,
+                           source='legacy', suite_version=''):
+    record = {
+        'commitid':     args.revision,
+        'branch':       args.branch,
+        'project':      args.project,
+        'executable':   args.executable,
+        'environment':  args.host,
+        'benchmark':    name,
+        'result_value': value,
+        'source':       source,
+    }
+    if std_dev is not None:
+        record['std_dev'] = std_dev
+    if suite_version:
+        record['suite_version'] = suite_version
+    return record
+
+
+def send_bulk(records, url, username=None, password=None):
+    params = urllib.parse.urlencode({'json': json.dumps(records)}).encode('utf-8')
+
+    if username and password:
+        mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        mgr.add_password(None, url, username, password)
+        opener = urllib.request.build_opener(urllib.request.HTTPBasicAuthHandler(mgr))
+    else:
+        opener = urllib.request.build_opener()
+
+    retries = [1, 3, 6, 20]
+    while True:
+        try:
+            req = urllib.request.Request(url + 'result/add/json/', params)
+            resp = opener.open(req)
+            print(resp.read().decode())
+            return
+        except urllib.error.HTTPError as e:
+            body = e.read().decode(errors='replace')
+            print(f"Upload failed with HTTP {e.code}: {body}")
+            if e.code < 500:
+                sys.exit(1)
+            if not retries:
+                sys.exit(1)
+            delay = retries.pop(0)
+            print(f"Retrying in {delay}s...")
+            time.sleep(delay)
+        except urllib.error.URLError:
+            if not retries:
+                raise
+            delay = retries.pop(0)
+            print(f"Upload failed, retrying in {delay}s...")
+            time.sleep(delay)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument('jsonfile', help='Path to result JSON file')
+    parser.add_argument('-e', '--executable', required=True,
+                        help='Executable name in codespeed')
+    parser.add_argument('-H', '--host', required=True,
+                        help='Environment/host name in codespeed')
+    parser.add_argument('-r', '--revision', default=None,
+                        help='Commit id (read from file if omitted)')
+    parser.add_argument('-B', '--branch', default=None,
+                        help='Branch name (read from file if omitted)')
+    parser.add_argument('-P', '--project', default='PyPy',
+                        help='Project name in codespeed (default: PyPy)')
+    parser.add_argument('-u', '--url', default='https://speed.pypy.org/',
+                        help='Base URL of codespeed instance')
+    parser.add_argument('-b', '--baseline', action='store_true',
+                        help='Upload avg_base values (legacy format only; '
+                             'use for the jit-on executable)')
+    args = parser.parse_args()
+
+    with open(args.jsonfile) as f:
+        data = json.load(f)
+
+    fmt = detect_format(data)
+
+    if args.revision is None:
+        args.revision = data.get('revision') or ''
+    if args.branch is None:
+        args.branch = data.get('branch') or 'default'
+
+    if not args.revision:
+        parser.error('--revision is required (could not read from file)')
+
+    if not args.url.endswith('/'):
+        args.url += '/'
+
+    if fmt == 'legacy':
+        raw = parse_legacy(data, changed=not args.baseline)
+        records = [
+            build_codespeed_record(name, value, std_dev, args, source='legacy')
+            for name, value, std_dev in raw
+        ]
+    else:
+        raw, suite_version = parse_pyperf(data)
+        records = [
+            build_codespeed_record(name, value, std_dev, args,
+                                   source='pyperformance',
+                                   suite_version=suite_version)
+            for name, value, std_dev in raw
+        ]
+
+    if not records:
+        print("No results to upload.", file=sys.stderr)
+        sys.exit(1)
+
+    username = os.environ.get('SPEED_UPLOAD_USER')
+    password = os.environ.get('SPEED_UPLOAD_PASSWORD')
+    auth_note = f" as {username!r}" if username and password else " (no credentials)"
+
+    print(f"Uploading {len(records)} results ({fmt} format) "
+          f"for {args.executable!r} env {args.host!r} to {args.url}{auth_note}")
+    send_bulk(records, args.url, username=username, password=password)
+    print("Done.")
+
+
+if __name__ == '__main__':
+    main()

bot2/pypybuildbot/master.py

@@ -27,14 +27,25 @@ def force(self, owner, builder_name, **kwargs):
         return ForceScheduler.force(self, owner, builder_name, **kwargs)
 
 
+import re as _re
+_PYPERFORMANCE_SPEC_RE = _re.compile(
+    r'^('
+    r'pyperformance\s*([><=!~^][^;]*)?'          # PyPI: pyperformance, pyperformance==1.2
+    r'|git\+https://github\.com/python/pyperformance(@\S+)?'  # GitHub: python/pyperformance only
+    r')$'
+)
+
 class BenchmarkForceScheduler(CustomForceScheduler):
     '''
-    A ForceScheduler with an extra field: benchmark_branch
+    A ForceScheduler with extra fields: benchmark_branch and pyperformance_spec
     '''
     def __init__(self, name, builderNames,
             benchmark_branch=StringParameter(name="benchmark_branch",
-                                             label="Benchmark repo branch:",
+                                             label="Legacy benchmark repo branch:",
                                              default="default", length=20),
+            pyperformance_spec=StringParameter(name="pyperformance_spec",
+                                               label="pyperformance pip specifier:",
+                                               default="pyperformance", length=60),
             properties=[ CodebaseParameter('PyPy repo', label='PyPy Repo')],
             **kwargs):
         CustomForceScheduler.__init__(self, name, builderNames, **kwargs)
@@ -45,8 +56,19 @@ def __init__(self, name, builderNames,
                          pypy_branch)
         self.all_fields.append(benchmark_branch)
         self.forcedProperties.append(benchmark_branch)
+        self.all_fields.append(pyperformance_spec)
+        self.forcedProperties.append(pyperformance_spec)
 
     def force(self, owner, builderNames=None, **kwargs):
+        spec = kwargs.get('pyperformance_spec', 'pyperformance')
+        if isinstance(spec, list):
+            spec = spec[0] if spec else 'pyperformance'
+        spec = spec.strip()
+        if not _PYPERFORMANCE_SPEC_RE.match(spec):
+            raise ValidationError(
+                "pyperformance pip specifier must be 'pyperformance', "
+                "'pyperformance==<version>', or "
+                "'git+https://github.com/python/pyperformance@<ref>'")
         CustomForceScheduler.force(self, owner, builderNames, **kwargs)
 
 # Forbid "stop build" without a reason that starts with "!"
@@ -239,10 +261,12 @@ def _checkStopBuild(self, reason=""):
     pypyjit=True,
     app_tests=True)
 
-pypyJITBenchmarkFactory = pypybuilds.JITBenchmark(host='benchmarker')
+pypyJITBenchmarkFactory = pypybuilds.JITBenchmark(host='benchmarker',
+                                                         upload_credentials=upload_credentials)
 pypyJITBenchmarkFactory64 = pypybuilds.JITBenchmark(platform='linux64',
                                                            host='benchmarker',
-                                                           postfix='-64')
+                                                           postfix='-64',
+                                                           upload_credentials=upload_credentials)
 pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmarkSingleRun(
     platform='linux64',
     host='speed_python',

bot2/slaveinfo.py

@@ -1,3 +1,6 @@
 
 # Mapping from slave name to slave password
-passwords = {}
+passwords = {'localhost': 'localhost'}
+
+# Credentials for uploading benchmark results to speed.pypy.org
+upload_credentials = {'username': '', 'password': ''}

master/master.cfg

@@ -19,6 +19,8 @@ httpPortNumber = 8099
 slaveinfo = load('slaveinfo')
 passwords = slaveinfo.passwords
 
+upload_credentials = getattr(slaveinfo, 'upload_credentials', None)
+
 execfile(os.path.join(botdir, 'pypybuildbot', 'master.py'))
 
 if we_are_debugging():

patch-buildbot.patch

@@ -0,0 +1,43 @@
+25,26c25,50
+<     sourcestamps_table = sa.Table('sourcestamps', metadata, autoload=True)
+<     buildsets_table = sa.Table('buildsets', metadata, autoload=True)
+---
+>     # Define tables explicitly to avoid SQLite reflection issues caused by
+>     # migration_tmp references left in sqlite_master by migration 004's ALTER TABLE.
+>     sa.Table('patches', metadata,
+>         sa.Column('id', sa.Integer, primary_key=True),
+>         sa.Column('patchlevel', sa.Integer, nullable=False),
+>         sa.Column('patch_base64', sa.Text, nullable=False),
+>         sa.Column('subdir', sa.Text),
+>     )
+>     sourcestamps_table = sa.Table('sourcestamps', metadata,
+>         sa.Column('id', sa.Integer, primary_key=True),
+>         sa.Column('branch', sa.String(256)),
+>         sa.Column('revision', sa.String(256)),
+>         sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')),
+>         sa.Column('repository', sa.Text, nullable=False, server_default=''),
+>         sa.Column('project', sa.Text, nullable=False, server_default=''),
+>     )
+>     buildsets_table = sa.Table('buildsets', metadata,
+>         sa.Column('id', sa.Integer, primary_key=True),
+>         sa.Column('external_idstring', sa.String(256)),
+>         sa.Column('reason', sa.String(256)),
+>         sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
+>         sa.Column('submitted_at', sa.Integer, nullable=False),
+>         sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")),
+>         sa.Column('complete_at', sa.Integer),
+>         sa.Column('results', sa.SmallInteger),
+>     )
+45c69,78
+<     buildsets_table = sa.Table('buildsets', metadata, autoload=True)
+---
+>     buildsets_table = sa.Table('buildsets', metadata,
+>         sa.Column('id', sa.Integer, primary_key=True),
+>         sa.Column('external_idstring', sa.String(256)),
+>         sa.Column('reason', sa.String(256)),
+>         sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id'), nullable=False),
+>         sa.Column('submitted_at', sa.Integer, nullable=False),
+>         sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")),
+>         sa.Column('complete_at', sa.Integer),
+>         sa.Column('results', sa.SmallInteger),
+>     )