From e63d611e1b59d6ca9c6dd55ed9c0341f9ab89255 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 2 May 2026 20:02:07 +0300 Subject: [PATCH 01/21] use pyperformance on pypy3*, do bulk benchmark upload Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 124 ++++++++++++++--- bot2/pypybuildbot/bulk_upload.py | 220 +++++++++++++++++++++++++++++++ bot2/pypybuildbot/master.py | 6 +- bot2/slaveinfo.py | 3 + 4 files changed, 336 insertions(+), 17 deletions(-) create mode 100644 bot2/pypybuildbot/bulk_upload.py diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 85bca87..dd5ca66 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -893,7 +893,8 @@ def __init__(self, platform='linux', host='speed_python', postfix=''): workdir=".")) class JITBenchmark(factory.BuildFactory): - def __init__(self, platform='linux', host='benchmarker', postfix=''): + def __init__(self, platform='linux', host='benchmarker', postfix='', + upload_credentials=None): factory.BuildFactory.__init__(self) # @@ -981,21 +982,10 @@ def get_cmd(props): '--changed', target, '--baseline', target, '--args', ',--jit off', - '--upload', - '--upload-executable', exe + postfix, - '--upload-project', project, - # use only the hash in the revision '--revision', rev, '--branch', branch, - '--upload-urls', 'https://speed.pypy.org/', - '--upload-baseline', - '--upload-baseline-executable', exe + '-jit' + postfix, - '--upload-baseline-project', project, - '--upload-baseline-revision', rev, - '--upload-baseline-branch', branch, - '--upload-baseline-urls', 'https://speed.pypy.org/', - ] - return command + ] + return command self.addStep(ShellCmd( # this step needs exclusive access to the CPU @@ -1004,7 +994,111 @@ def get_cmd(props): command=get_cmd, workdir='./benchmarks', timeout=3600)) - # a bit obscure hack to get both os.path.expand and a property + + upload_env = {} + if upload_credentials: + upload_env = { + 'SPEED_UPLOAD_USER': upload_credentials.get('username', ''), + 'SPEED_UPLOAD_PASSWORD': upload_credentials.get('password', ''), + } + + # Push bulk_upload.py to the slave so upload steps can use it + self.addStep(transfer.FileDownload( + mastersrc=os.path.join(os.path.dirname(__file__), 'bulk_upload.py'), + slavedest='bulk_upload.py', + workdir='.')) + + def _props_for_upload(props): + target = props.getProperty('target_path') + exe = os.path.split(target)[-1][:-2] + project = props.getProperty('project', default='PyPy') + rev = props.getProperty('got_revision').split(':')[-1] + branch = props.getProperty('branch') or 'main' + if branch == 'None': + branch = 'main' + return exe, project, rev, branch + + @renderer + def get_upload_changed_cmd(props): + exe, project, rev, branch = _props_for_upload(props) + return ['python3', 'bulk_upload.py', 'benchmarks/result.json', + '-e', exe + postfix, '-H', host, + '-P', project, '-r', rev, '-B', branch, + '-u', 'https://speed.pypy.org/'] + + @renderer + def get_upload_baseline_cmd(props): + exe, project, rev, branch = _props_for_upload(props) + return ['python3', 'bulk_upload.py', 'benchmarks/result.json', + '-e', exe + '-jit' + postfix, '-H', host, + '-P', project, '-r', rev, '-B', branch, + '-u', 'https://speed.pypy.org/', + '--baseline'] + + self.addStep(ShellCmd( + description='upload legacy results (jit-off)', + command=get_upload_changed_cmd, + env=upload_env, + workdir='.')) + self.addStep(ShellCmd( + description='upload legacy results (jit-on)', + command=get_upload_baseline_cmd, + env=upload_env, + workdir='.')) + + # Pyperformance: only when the target binary is pypy3* + def is_py3_target(step): + target = step.build.getProperty('target_path') or '' + return os.path.basename(target).startswith('pypy3') + + @renderer + def get_pyperformance_venv_cmd(props): + target = props.getProperty('target_path') + return [target, '-m', 'venv', 'pyperformance_venv'] + + @renderer + def get_pyperformance_install_cmd(props): + return ['./pyperformance_venv/bin/pip', 'install', '--upgrade', + 'pyperformance'] + + @renderer + def get_pyperformance_run_cmd(props): + return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', + 'run', '--output', 'pyperformance_result.json'] + + @renderer + def get_pyperformance_upload_cmd(props): + exe, project, rev, branch = _props_for_upload(props) + return ['python3', 'bulk_upload.py', 'pyperformance_result.json', + '-e', exe + postfix, '-H', host, + '-P', project, '-r', rev, '-B', branch, + '-u', 'https://speed.pypy.org/'] + + self.addStep(ShellCmd( + description='create pyperformance venv', + command=get_pyperformance_venv_cmd, + doStepIf=is_py3_target, + workdir='.')) + self.addStep(ShellCmd( + description='install pyperformance', + command=get_pyperformance_install_cmd, + doStepIf=is_py3_target, + workdir='.')) + self.addStep(ShellCmd( + description='run pyperformance', + command=get_pyperformance_run_cmd, + locks=[lock.access('exclusive')], + doStepIf=is_py3_target, + workdir='.', + timeout=7200)) + self.addStep(ShellCmd( + description='upload pyperformance results', + command=get_pyperformance_upload_cmd, + env=upload_env, + doStepIf=is_py3_target, + workdir='.')) + + # Archive the legacy result file on the master filename = '%(got_revision)s' + (postfix or '') resfile = os.path.expanduser("~/bench_results/%s.json" % filename) self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", diff --git a/bot2/pypybuildbot/bulk_upload.py b/bot2/pypybuildbot/bulk_upload.py new file mode 100644 index 0000000..fac2925 --- /dev/null +++ b/bot2/pypybuildbot/bulk_upload.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Bulk upload benchmark results to a codespeed instance in a single POST. + +Supports two input formats: + legacy - result.json produced by runner.py (unladen_swallow/perf format) + pyperf - result.json produced by pyperformance (pyperf format) + +Revision and branch are read from the JSON file when possible (legacy format); +pass --revision / --branch on the command line to override or when using pyperf. + +Examples: + # legacy, jit-off results + ./bulk_upload.py result.json -e pypy-c-64 -H benchmarker + + # legacy, jit-on results (avg_base values) + ./bulk_upload.py result.json -e pypy-c-jit-64 -H benchmarker --baseline + + # pyperformance + ./bulk_upload.py pyperf.json -e pypy-c-64 -H benchmarker -r abc123 -B py3.11 +""" +import argparse +import json +import os +import statistics +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +def detect_format(data): + if 'benchmarks' in data and isinstance(data.get('benchmarks'), list): + benchmarks = data['benchmarks'] + if benchmarks and 'runs' in benchmarks[0]: + return 'pyperf' + if 'results' in data: + return 'legacy' + raise ValueError("Unrecognised result file format") + + +def parse_legacy(data, changed): + """Return list of (name, value, std_dev) from a runner.py result.json. + + changed=True -> upload avg_changed values (jit-off run, executable pypy-c-NN) + changed=False -> upload avg_base values (jit-on run, executable pypy-c-jit-NN) + """ + records = [] + for name, result_type, result_data in data['results']: + value = std_dev = None + + if result_type == 'ComparisonResult': + if changed: + value = result_data.get('avg_changed') + std_dev = result_data.get('std_changed') + else: + value = result_data.get('avg_base') + std_dev = result_data.get('std_base') + + elif result_type == 'SimpleComparisonResult': + value = result_data['changed_time'] if changed else result_data['base_time'] + + elif result_type == 'RawResult': + times = result_data['changed_times'] if changed else result_data['base_times'] + if times: + value = times[0] if len(times) == 1 else statistics.mean(times) + std_dev = statistics.stdev(times) if len(times) > 1 else None + + if not value: + continue + records.append((name, value, std_dev)) + return records + + +def parse_pyperf(data): + """Return (records, suite_version) from a pyperformance/pyperf result.json. + + records is a list of (name, mean_seconds, std_dev_or_None). + suite_version is taken from the first benchmark's metadata 'version' field. + """ + records = [] + suite_version = '' + + for bench in data.get('benchmarks', []): + meta = bench.get('metadata', {}) + name = meta.get('name', '') + if not name: + continue + if not suite_version: + suite_version = meta.get('version', '') + + # pyperf stores values as floats (seconds) inside each run. + # Each value entry is either a plain float or a (loops, time_per_loop) pair. + values = [] + for run in bench.get('runs', []): + for v in run.get('values', []): + values.append(v[1] if isinstance(v, (list, tuple)) else v) + + if not values: + continue + + mean = statistics.mean(values) + std_dev = statistics.stdev(values) if len(values) > 1 else None + records.append((name, mean, std_dev)) + + return records, suite_version + + +def build_codespeed_record(name, value, std_dev, args, + source='legacy', suite_version=''): + record = { + 'commitid': args.revision, + 'branch': args.branch, + 'project': args.project, + 'executable': args.executable, + 'environment': args.host, + 'benchmark': name, + 'result_value': value, + 'source': source, + } + if std_dev is not None: + record['std_dev'] = std_dev + if suite_version: + record['suite_version'] = suite_version + return record + + +def send_bulk(records, url, username=None, password=None): + params = urllib.parse.urlencode({'json': json.dumps(records)}).encode('utf-8') + + if username and password: + mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + mgr.add_password(None, url, username, password) + opener = urllib.request.build_opener(urllib.request.HTTPBasicAuthHandler(mgr)) + else: + opener = urllib.request.build_opener() + + retries = [1, 3, 6, 20] + while True: + try: + req = urllib.request.Request(url + 'result/add/json/', params) + resp = opener.open(req) + print(resp.read().decode()) + return + except urllib.error.URLError: + if not retries: + raise + delay = retries.pop(0) + print(f"Upload failed, retrying in {delay}s...") + time.sleep(delay) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('jsonfile', help='Path to result JSON file') + parser.add_argument('-e', '--executable', required=True, + help='Executable name in codespeed') + parser.add_argument('-H', '--host', required=True, + help='Environment/host name in codespeed') + parser.add_argument('-r', '--revision', default=None, + help='Commit id (read from file if omitted)') + parser.add_argument('-B', '--branch', default=None, + help='Branch name (read from file if omitted)') + parser.add_argument('-P', '--project', default='PyPy', + help='Project name in codespeed (default: PyPy)') + parser.add_argument('-u', '--url', default='https://speed.pypy.org/', + help='Base URL of codespeed instance') + parser.add_argument('-b', '--baseline', action='store_true', + help='Upload avg_base values (legacy format only; ' + 'use for the jit-on executable)') + args = parser.parse_args() + + with open(args.jsonfile) as f: + data = json.load(f) + + fmt = detect_format(data) + + if args.revision is None: + args.revision = data.get('revision') or '' + if args.branch is None: + args.branch = data.get('branch') or 'default' + + if not args.revision: + parser.error('--revision is required (could not read from file)') + + if fmt == 'legacy': + raw = parse_legacy(data, changed=not args.baseline) + records = [ + build_codespeed_record(name, value, std_dev, args, source='legacy') + for name, value, std_dev in raw + ] + else: + raw, suite_version = parse_pyperf(data) + records = [ + build_codespeed_record(name, value, std_dev, args, + source='pyperformance', + suite_version=suite_version) + for name, value, std_dev in raw + ] + + if not records: + print("No results to upload.", file=sys.stderr) + sys.exit(1) + + username = os.environ.get('SPEED_UPLOAD_USER') + password = os.environ.get('SPEED_UPLOAD_PASSWORD') + auth_note = f" as {username!r}" if username and password else " (no credentials)" + + print(f"Uploading {len(records)} results ({fmt} format) " + f"for {args.executable!r} to {args.url}{auth_note}") + send_bulk(records, args.url, username=username, password=password) + print("Done.") + + +if __name__ == '__main__': + main() diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py index bf01870..5f2e729 100644 --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -239,10 +239,12 @@ def _checkStopBuild(self, reason=""): pypyjit=True, app_tests=True) -pypyJITBenchmarkFactory = pypybuilds.JITBenchmark(host='benchmarker') +pypyJITBenchmarkFactory = pypybuilds.JITBenchmark(host='benchmarker', + upload_credentials=upload_credentials) pypyJITBenchmarkFactory64 = pypybuilds.JITBenchmark(platform='linux64', host='benchmarker', - postfix='-64') + postfix='-64', + upload_credentials=upload_credentials) pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmarkSingleRun( platform='linux64', host='speed_python', diff --git a/bot2/slaveinfo.py b/bot2/slaveinfo.py index aeccc4a..3027458 100644 --- a/bot2/slaveinfo.py +++ b/bot2/slaveinfo.py @@ -1,3 +1,6 @@ # Mapping from slave name to slave password passwords = {} + +# Credentials for uploading benchmark results to speed.pypy.org +upload_credentials = {'username': '', 'password': ''} From e7cf51f7e99a14d2f13861e9bbeb6f50d3c69fee Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 2 May 2026 21:33:51 +0300 Subject: [PATCH 02/21] add defaults --- bot2/slaveinfo.py | 2 +- master/master.cfg | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bot2/slaveinfo.py b/bot2/slaveinfo.py index 3027458..b11daf5 100644 --- a/bot2/slaveinfo.py +++ b/bot2/slaveinfo.py @@ -1,6 +1,6 @@ # Mapping from slave name to slave password -passwords = {} +passwords = {'localhost': 'localhost'} # Credentials for uploading benchmark results to speed.pypy.org upload_credentials = {'username': '', 'password': ''} diff --git a/master/master.cfg b/master/master.cfg index 54b0ffb..99fa4e4 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -19,6 +19,8 @@ httpPortNumber = 8099 slaveinfo = load('slaveinfo') passwords = slaveinfo.passwords +upload_credentials = getattr(slaveinfo, 'upload_credentials', None) + execfile(os.path.join(botdir, 'pypybuildbot', 'master.py')) if we_are_debugging(): From 5eaf5abfe3f60afb3b0ce0a5e9eb6979b1a34067 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 2 May 2026 22:40:48 +0300 Subject: [PATCH 03/21] add patches for old buildbot with new sqlite3 --- README | 16 +++++++++++----- patch-buildbot.patch | 43 +++++++++++++++++++++++++++++++++++++++++++ patch-migrate.patch | 18 ++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 patch-buildbot.patch create mode 100644 patch-migrate.patch diff --git a/README b/README index 36bb489..bb4f77f 100644 --- a/README +++ b/README @@ -1,19 +1,25 @@ .. -*- mode: rst -*- -Moved from heptapod -=================== +Current buidlbot status +======================= This is the code for PyPy's buildbot, powering buildbot.pypy.org. It was -imported in April 2026 from https://foss.heptapod.net/pypy/buildbot. +imported in April 2026 from https://foss.heptapod.net/pypy/buildbot. The +service is managed via ssh into the machine, and running the +``restart_buildmaster_when_not_running`` script. It uses a python2.7 venv. Everything has been tested with builbot 0.8.8 on CPython2.7 Note you must pin "automat==20.2" "incremental==21.3.0" Testing must use "pytest<4" + Patch the source ================ -buildbot 0.8.8 is really old. There are some patches needed, see patches.patch +buildbot 0.8.8 is really old. There are some patches needed, see patches.patch. +If you are starting from a new db, you will need to also patch the installed +migrate and buildbot packages for newer sqlite3: patch-migrate.patch, +patch-buildbot.patch. How to hack the PyPy buildbot ============================== @@ -36,7 +42,7 @@ How to run the PyPy buildbot If you want to run buildbot in production, you need to make sure that the function ``pypybuildbot.util.we_are_debugging`` returns ``False`` in your environment. At the moment of writing, debugging is enabled everywhere but on -cobra. +cobra, which is not really where the buildbot runs. You still need to fill ``master/slaveinfo.py`` with the passwords of the various slaves you want to use. diff --git a/patch-buildbot.patch b/patch-buildbot.patch new file mode 100644 index 0000000..230616e --- /dev/null +++ b/patch-buildbot.patch @@ -0,0 +1,43 @@ +25,26c25,50 +< sourcestamps_table = sa.Table('sourcestamps', metadata, autoload=True) +< buildsets_table = sa.Table('buildsets', metadata, autoload=True) +--- +> # Define tables explicitly to avoid SQLite reflection issues caused by +> # migration_tmp references left in sqlite_master by migration 004's ALTER TABLE. +> sa.Table('patches', metadata, +> sa.Column('id', sa.Integer, primary_key=True), +> sa.Column('patchlevel', sa.Integer, nullable=False), +> sa.Column('patch_base64', sa.Text, nullable=False), +> sa.Column('subdir', sa.Text), +> ) +> sourcestamps_table = sa.Table('sourcestamps', metadata, +> sa.Column('id', sa.Integer, primary_key=True), +> sa.Column('branch', sa.String(256)), +> sa.Column('revision', sa.String(256)), +> sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), +> sa.Column('repository', sa.Text, nullable=False, server_default=''), +> sa.Column('project', sa.Text, nullable=False, server_default=''), +> ) +> buildsets_table = sa.Table('buildsets', metadata, +> sa.Column('id', sa.Integer, primary_key=True), +> sa.Column('external_idstring', sa.String(256)), +> sa.Column('reason', sa.String(256)), +> sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), +> sa.Column('submitted_at', sa.Integer, nullable=False), +> sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), +> sa.Column('complete_at', sa.Integer), +> sa.Column('results', sa.SmallInteger), +> ) +45c69,78 +< buildsets_table = sa.Table('buildsets', metadata, autoload=True) +--- +> buildsets_table = sa.Table('buildsets', metadata, +> sa.Column('id', sa.Integer, primary_key=True), +> sa.Column('external_idstring', sa.String(256)), +> sa.Column('reason', sa.String(256)), +> sa.Column('sourcestampsetid', sa.Integer, sa.ForeignKey('sourcestampsets.id'), nullable=False), +> sa.Column('submitted_at', sa.Integer, nullable=False), +> sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), +> sa.Column('complete_at', sa.Integer), +> sa.Column('results', sa.SmallInteger), +> ) diff --git a/patch-migrate.patch b/patch-migrate.patch new file mode 100644 index 0000000..29bbee2 --- /dev/null +++ b/patch-migrate.patch @@ -0,0 +1,18 @@ +29a30,37 +> # SQLite 3.26.0+ propagates RENAME TABLE into FK references in other +> # tables, which breaks the migration_tmp pattern. legacy_alter_table +> # restores the pre-3.26.0 behaviour for the duration of this rename. +> try: +> self.connection.execute('PRAGMA legacy_alter_table = ON') +> except Exception: +> pass +> +45c53,58 +< +--- +> +> try: +> self.connection.execute('PRAGMA legacy_alter_table = OFF') +> except Exception: +> pass +> From 4721e66a76640b253f6b06cc2ce3955442714758 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 2 May 2026 23:16:41 +0300 Subject: [PATCH 04/21] allow overriding speed.pypy.org and benchmark runner name Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 35 ++++++++++++++++++++------------ bot2/pypybuildbot/bulk_upload.py | 2 +- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index dd5ca66..0351744 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -403,8 +403,7 @@ def setup_steps(platform, factory, workdir=None, repourl='https://github.com/pypy/pypy/', force_branch=None): factory.addStep(shell.SetPropertyFromCommand( - command=['python', '-c', "import tempfile, os ;print" - " tempfile.gettempdir() + os.path.sep"], + command=['python', '-c', "import tempfile, os; print(tempfile.gettempdir() + os.path.sep)"], property="target_tmpdir", env={'TMPDIR': "${TMPDIR}"}, )) @@ -995,12 +994,22 @@ def get_cmd(props): workdir='./benchmarks', timeout=3600)) - upload_env = {} + upload_env = { + 'SPEED_UPLOAD_URL': os.environ.get('SPEED_UPLOAD_URL', 'https://speed.pypy.org/'), + 'SPEED_UPLOAD_HOST': os.environ.get('SPEED_UPLOAD_HOST', host), + } if upload_credentials: - upload_env = { - 'SPEED_UPLOAD_USER': upload_credentials.get('username', ''), - 'SPEED_UPLOAD_PASSWORD': upload_credentials.get('password', ''), - } + upload_env['SPEED_UPLOAD_USER'] = upload_credentials.get('username', '') + upload_env['SPEED_UPLOAD_PASSWORD'] = upload_credentials.get('password', '') + + def extract_upload_config(rc, stdout, stderr): + return {'upload_url': upload_env['SPEED_UPLOAD_URL'], + 'upload_host': upload_env['SPEED_UPLOAD_HOST']} + self.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', 'print("ok")'], + extract_fn=extract_upload_config, + description='set upload config', + )) # Push bulk_upload.py to the slave so upload steps can use it self.addStep(transfer.FileDownload( @@ -1022,17 +1031,17 @@ def _props_for_upload(props): def get_upload_changed_cmd(props): exe, project, rev, branch = _props_for_upload(props) return ['python3', 'bulk_upload.py', 'benchmarks/result.json', - '-e', exe + postfix, '-H', host, + '-e', exe + postfix, '-H', upload_env['SPEED_UPLOAD_HOST'], '-P', project, '-r', rev, '-B', branch, - '-u', 'https://speed.pypy.org/'] + '-u', upload_env['SPEED_UPLOAD_URL']] @renderer def get_upload_baseline_cmd(props): exe, project, rev, branch = _props_for_upload(props) return ['python3', 'bulk_upload.py', 'benchmarks/result.json', - '-e', exe + '-jit' + postfix, '-H', host, + '-e', exe + '-jit' + postfix, '-H', upload_env['SPEED_UPLOAD_HOST'], '-P', project, '-r', rev, '-B', branch, - '-u', 'https://speed.pypy.org/', + '-u', upload_env['SPEED_UPLOAD_URL'], '--baseline'] self.addStep(ShellCmd( @@ -1070,9 +1079,9 @@ def get_pyperformance_run_cmd(props): def get_pyperformance_upload_cmd(props): exe, project, rev, branch = _props_for_upload(props) return ['python3', 'bulk_upload.py', 'pyperformance_result.json', - '-e', exe + postfix, '-H', host, + '-e', exe + postfix, '-H', upload_env['SPEED_UPLOAD_HOST'], '-P', project, '-r', rev, '-B', branch, - '-u', 'https://speed.pypy.org/'] + '-u', upload_env['SPEED_UPLOAD_URL']] self.addStep(ShellCmd( description='create pyperformance venv', diff --git a/bot2/pypybuildbot/bulk_upload.py b/bot2/pypybuildbot/bulk_upload.py index fac2925..d8ec582 100644 --- a/bot2/pypybuildbot/bulk_upload.py +++ b/bot2/pypybuildbot/bulk_upload.py @@ -211,7 +211,7 @@ def main(): auth_note = f" as {username!r}" if username and password else " (no credentials)" print(f"Uploading {len(records)} results ({fmt} format) " - f"for {args.executable!r} to {args.url}{auth_note}") + f"for {args.executable!r} env {args.host!r} to {args.url}{auth_note}") send_bulk(records, args.url, username=username, password=password) print("Done.") From f69bdf88fa1ea7e936d4b1ae22ca774ec3da6091 Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 3 May 2026 07:39:19 +0300 Subject: [PATCH 05/21] print 400 error body Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/bulk_upload.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bot2/pypybuildbot/bulk_upload.py b/bot2/pypybuildbot/bulk_upload.py index d8ec582..39ab8b5 100644 --- a/bot2/pypybuildbot/bulk_upload.py +++ b/bot2/pypybuildbot/bulk_upload.py @@ -143,6 +143,16 @@ def send_bulk(records, url, username=None, password=None): resp = opener.open(req) print(resp.read().decode()) return + except urllib.error.HTTPError as e: + body = e.read().decode(errors='replace') + print(f"Upload failed with HTTP {e.code}: {body}") + if e.code < 500: + sys.exit(1) + if not retries: + sys.exit(1) + delay = retries.pop(0) + print(f"Retrying in {delay}s...") + time.sleep(delay) except urllib.error.URLError: if not retries: raise From 10203f9e3fe0a2bdf13fc8a71db8ef6c1bdf7e8e Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 3 May 2026 10:03:14 +0300 Subject: [PATCH 06/21] refactor --- bot2/pypybuildbot/builds.py | 40 ++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 0351744..5044b4d 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -951,12 +951,23 @@ def __init__(self, platform='linux', host='benchmarker', postfix='', else: assert False, 'unknown host %s' % host - def extract_info(rc, stdout, stderr): - if rc == 0: - return json.loads(stdout) - else: - return {} - + upload_env = { + 'SPEED_UPLOAD_URL': os.environ.get('SPEED_UPLOAD_URL', 'https://speed.pypy.org/'), + 'SPEED_UPLOAD_HOST': os.environ.get('SPEED_UPLOAD_HOST', host), + } + if upload_credentials: + upload_env['SPEED_UPLOAD_USER'] = upload_credentials.get('username', '') + upload_env['SPEED_UPLOAD_PASSWORD'] = upload_credentials.get('password', '') + + def extract_upload_config(rc, stdout, stderr): + return {'upload_url': upload_env['SPEED_UPLOAD_URL'], + 'upload_host': upload_env['SPEED_UPLOAD_HOST']} + self.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', 'print("ok")'], + extract_fn=extract_upload_config, + description='set upload config', + )) + self.addStep( Translate( translationArgs=['-Ojit'], @@ -994,23 +1005,6 @@ def get_cmd(props): workdir='./benchmarks', timeout=3600)) - upload_env = { - 'SPEED_UPLOAD_URL': os.environ.get('SPEED_UPLOAD_URL', 'https://speed.pypy.org/'), - 'SPEED_UPLOAD_HOST': os.environ.get('SPEED_UPLOAD_HOST', host), - } - if upload_credentials: - upload_env['SPEED_UPLOAD_USER'] = upload_credentials.get('username', '') - upload_env['SPEED_UPLOAD_PASSWORD'] = upload_credentials.get('password', '') - - def extract_upload_config(rc, stdout, stderr): - return {'upload_url': upload_env['SPEED_UPLOAD_URL'], - 'upload_host': upload_env['SPEED_UPLOAD_HOST']} - self.addStep(shell.SetPropertyFromCommand( - command=['python', '-c', 'print("ok")'], - extract_fn=extract_upload_config, - description='set upload config', - )) - # Push bulk_upload.py to the slave so upload steps can use it self.addStep(transfer.FileDownload( mastersrc=os.path.join(os.path.dirname(__file__), 'bulk_upload.py'), From dd67517769bb2db681e0feb19d79b76e1e89a6ba Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 3 May 2026 12:46:45 +0300 Subject: [PATCH 07/21] tweak upload url for missing slash --- bot2/pypybuildbot/bulk_upload.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot2/pypybuildbot/bulk_upload.py b/bot2/pypybuildbot/bulk_upload.py index 39ab8b5..8c75d09 100644 --- a/bot2/pypybuildbot/bulk_upload.py +++ b/bot2/pypybuildbot/bulk_upload.py @@ -197,6 +197,9 @@ def main(): if not args.revision: parser.error('--revision is required (could not read from file)') + if not args.url.endswith('/'): + args.url += '/' + if fmt == 'legacy': raw = parse_legacy(data, changed=not args.baseline) records = [ From 1b4742f22c8041a10830389ba013e033fcff9afc Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 3 May 2026 22:27:29 +0300 Subject: [PATCH 08/21] patch failures in pyperformance Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 67 ++++++++++++++++++++++- patches/patch_bm_pickle_pypy.py | 38 +++++++++++++ patches/patch_sqlalchemy_identity_pypy.py | 50 +++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 patches/patch_bm_pickle_pypy.py create mode 100644 patches/patch_sqlalchemy_identity_pypy.py diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 5044b4d..36065dd 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -8,6 +8,7 @@ from buildbot import locks from pypybuildbot.util import symlink_force from buildbot.status.results import SKIPPED, SUCCESS +import glob import os import json @@ -1073,6 +1074,27 @@ def get_pyperformance_run_cmd(props): def get_pyperformance_upload_cmd(props): exe, project, rev, branch = _props_for_upload(props) return ['python3', 'bulk_upload.py', 'pyperformance_result.json', + '-e', exe + '-jit' + postfix, '-H', upload_env['SPEED_UPLOAD_HOST'], + '-P', project, '-r', rev, '-B', branch, + '-u', upload_env['SPEED_UPLOAD_URL']] + + @renderer + def get_pyperformance_nojit_wrapper_cmd(props): + target = props.getProperty('target_path') + script = '#!/bin/sh\nexec %s --jit off "$@"\n' % target + return ['python3', '-c', + 'open("pypy_nojit","w").write(%r);import os;os.chmod("pypy_nojit",0o755)' % script] + + @renderer + def get_pyperformance_nojit_run_cmd(props): + return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', + 'run', '--output', 'pyperformance_nojit_result.json', + '--python', './pypy_nojit'] + + @renderer + def get_pyperformance_nojit_upload_cmd(props): + exe, project, rev, branch = _props_for_upload(props) + return ['python3', 'bulk_upload.py', 'pyperformance_nojit_result.json', '-e', exe + postfix, '-H', upload_env['SPEED_UPLOAD_HOST'], '-P', project, '-r', rev, '-B', branch, '-u', upload_env['SPEED_UPLOAD_URL']] @@ -1087,19 +1109,60 @@ def get_pyperformance_upload_cmd(props): command=get_pyperformance_install_cmd, doStepIf=is_py3_target, workdir='.')) + + # Transfer all PyPy-compatibility patch scripts from master to worker, + # then apply them in a single step. + _patches_dir = os.path.normpath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', 'patches')) + for _patch_file in sorted(glob.glob( + os.path.join(_patches_dir, 'patch_*_pypy.py'))): + self.addStep(transfer.FileDownload( + mastersrc=_patch_file, + slavedest=os.path.basename(_patch_file), + doStepIf=is_py3_target, + workdir='.')) self.addStep(ShellCmd( - description='run pyperformance', + description='apply PyPy compatibility patches', + command=['python3', '-c', + 'import glob, subprocess, sys\n' + 'scripts = sorted(glob.glob("patch_*_pypy.py"))\n' + 'print("Applying patches:", scripts)\n' + 'for f in scripts:\n' + ' subprocess.check_call([sys.executable, f])\n'], + doStepIf=is_py3_target, + workdir='.')) + + self.addStep(ShellCmd( + description='run pyperformance (jit)', command=get_pyperformance_run_cmd, locks=[lock.access('exclusive')], doStepIf=is_py3_target, workdir='.', timeout=7200)) self.addStep(ShellCmd( - description='upload pyperformance results', + description='upload pyperformance results (jit)', command=get_pyperformance_upload_cmd, env=upload_env, doStepIf=is_py3_target, workdir='.')) + self.addStep(ShellCmd( + description='create pyperformance nojit wrapper', + command=get_pyperformance_nojit_wrapper_cmd, + doStepIf=is_py3_target, + workdir='.')) + self.addStep(ShellCmd( + description='run pyperformance (nojit)', + command=get_pyperformance_nojit_run_cmd, + locks=[lock.access('exclusive')], + doStepIf=is_py3_target, + workdir='.', + timeout=7200)) + self.addStep(ShellCmd( + description='upload pyperformance results (nojit)', + command=get_pyperformance_nojit_upload_cmd, + env=upload_env, + doStepIf=is_py3_target, + workdir='.')) # Archive the legacy result file on the master filename = '%(got_revision)s' + (postfix or '') diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py new file mode 100644 index 0000000..df1db8d --- /dev/null +++ b/patches/patch_bm_pickle_pypy.py @@ -0,0 +1,38 @@ +""" +Patch pyperformance bm_pickle for PyPy compatibility. + +PyPy provides a _pickle accelerator module, so the existing IS_PYPY check +in the pure-python guard incorrectly raises RuntimeError. Move IS_PYPY to +the accelerated-module check instead so PyPy can use its C extension. + +Source: https://github.com/python/pyperformance/pull/461 +""" +import glob +import sys + +GLOB = ("./pyperformance_venv/lib/python*/site-packages" + "/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py") + +files = glob.glob(GLOB) +if not files: + print("WARNING: bm_pickle run_benchmark.py not found – skipping patch") + sys.exit(0) + +for path in files: + txt = open(path).read() + original = txt + + txt = txt.replace( + "if not (options.pure_python or IS_PYPY):", + "if not (options.pure_python):", + ) + txt = txt.replace( + "if not is_accelerated_module(pickle):", + "if not is_accelerated_module(pickle) and not IS_PYPY:", + ) + + if txt == original: + print(f"NOTE: {path} already patched or pattern not found") + else: + open(path, "w").write(txt) + print(f"Patched {path}") diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py new file mode 100644 index 0000000..939b891 --- /dev/null +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -0,0 +1,50 @@ +""" +Patch SQLAlchemy identity map for PyPy GC compatibility. + +PyPy's garbage collector may collect an object referenced by a weakref before +the weakref is checked, causing 'NoneType has no attribute __dict__' errors in +the ORM identity map. Guard the call to _manage_removed_state with an +explicit liveness check on the weakref. + +Source: https://github.com/sqlalchemy/sqlalchemy/discussions/13274 +""" +import glob +import re +import sys + +GLOB = ("./pyperformance_venv/lib/python*/site-packages" + "/sqlalchemy/orm/identity.py") + +files = glob.glob(GLOB) +if not files: + print("WARNING: sqlalchemy identity.py not found – skipping patch") + sys.exit(0) + +# Replace every bare call to _manage_removed_state(existing_non_none) with a +# weakref liveness check. Capture indentation so the replacement is correctly +# indented regardless of nesting level. +PATTERN = re.compile( + r"^( +)self\._manage_removed_state\(existing_non_none\)\s*$", + re.MULTILINE, +) + + +def _replacement(m): + i = m.group(1) + return ( + f"{i}if existing_non_none.obj() is not None:\n" + f"{i} self._manage_removed_state(existing_non_none)\n" + f"{i}else:\n" + f"{i} existing = None" + ) + + +for path in files: + txt = open(path).read() + new_txt, count = PATTERN.subn(_replacement, txt) + + if count == 0: + print(f"NOTE: {path} already patched or pattern not found") + else: + open(path, "w").write(new_txt) + print(f"Patched {path} ({count} site(s))") From 30474eb6b394f5af4fdfeed634be551728ad075d Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 08:49:39 +0300 Subject: [PATCH 09/21] split pyperformance into 'create venv' and 'run' so we can apply patches also run the benchmarks fast to save some time fix patch downloading and application Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 43 +++++++++++++++++++---- patches/patch_bm_pickle_pypy.py | 2 +- patches/patch_sqlalchemy_identity_pypy.py | 2 +- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 36065dd..8389baf 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -989,7 +989,7 @@ def get_cmd(props): branch = props.getProperty('branch') if branch == 'None' or branch is None: branch = 'default' - command=["python", "-u", "runner.py", '--output-filename', 'result.json', + command=["python", "-u", "runner.py", '--fast', '--output-filename', 'result.json', '--changed', target, '--baseline', target, '--args', ',--jit off', @@ -1065,10 +1065,21 @@ def get_pyperformance_install_cmd(props): return ['./pyperformance_venv/bin/pip', 'install', '--upgrade', 'pyperformance'] + _bench_venv = 'pyperformance_bench_venv' + @renderer - def get_pyperformance_run_cmd(props): + def get_pyperformance_bench_venv_cmd(props): + target = props.getProperty('target_path') return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', - 'run', '--output', 'pyperformance_result.json'] + 'venv', 'create', '--venv', _bench_venv, '-p', target] + + @renderer + def get_pyperformance_run_cmd(props): + return ['bash', '-c', + 'rm -f pyperformance_result.json && ' + './pyperformance_venv/bin/python -m pyperformance run ' + '--venv ' + _bench_venv + ' -f ' + '--output pyperformance_result.json'] @renderer def get_pyperformance_upload_cmd(props): @@ -1087,9 +1098,12 @@ def get_pyperformance_nojit_wrapper_cmd(props): @renderer def get_pyperformance_nojit_run_cmd(props): - return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', - 'run', '--output', 'pyperformance_nojit_result.json', - '--python', './pypy_nojit'] + return ['bash', '-c', + 'rm -f pyperformance_nojit_result.json && ' + './pyperformance_venv/bin/python -m pyperformance run ' + '--venv ' + _bench_venv + ' -f ' + '--output pyperformance_nojit_result.json ' + '--python ./pypy_nojit'] @renderer def get_pyperformance_nojit_upload_cmd(props): @@ -1109,11 +1123,16 @@ def get_pyperformance_nojit_upload_cmd(props): command=get_pyperformance_install_cmd, doStepIf=is_py3_target, workdir='.')) + self.addStep(ShellCmd( + description='create pyperformance benchmark venv', + command=get_pyperformance_bench_venv_cmd, + doStepIf=is_py3_target, + workdir='.')) # Transfer all PyPy-compatibility patch scripts from master to worker, # then apply them in a single step. _patches_dir = os.path.normpath( - os.path.join(os.path.dirname(__file__), '..', '..', '..', 'patches')) + os.path.join(os.path.dirname(__file__), '..', '..', 'patches')) for _patch_file in sorted(glob.glob( os.path.join(_patches_dir, 'patch_*_pypy.py'))): self.addStep(transfer.FileDownload( @@ -1170,6 +1189,16 @@ def get_pyperformance_nojit_upload_cmd(props): self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", masterdest=WithProperties(resfile), workdir=".")) + pyresfile = os.path.expanduser("~/bench_results/%s-pyperformance.json" % filename) + self.addStep(transfer.FileUpload(slavesrc="pyperformance_result.json", + masterdest=WithProperties(pyresfile), + doStepIf=is_py3_target, + workdir=".")) + pynoresfile = os.path.expanduser("~/bench_results/%s-pyperformance-nojit.json" % filename) + self.addStep(transfer.FileUpload(slavesrc="pyperformance_nojit_result.json", + masterdest=WithProperties(pynoresfile), + doStepIf=is_py3_target, + workdir=".")) class CPythonBenchmark(factory.BuildFactory): diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py index df1db8d..9577b37 100644 --- a/patches/patch_bm_pickle_pypy.py +++ b/patches/patch_bm_pickle_pypy.py @@ -10,7 +10,7 @@ import glob import sys -GLOB = ("./pyperformance_venv/lib/python*/site-packages" +GLOB = ("./pyperformance_bench_venv/lib/python*/site-packages" "/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py") files = glob.glob(GLOB) diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index 939b891..7309697 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -12,7 +12,7 @@ import re import sys -GLOB = ("./pyperformance_venv/lib/python*/site-packages" +GLOB = ("./pyperformance_bench_venv/lib/python*/site-packages" "/sqlalchemy/orm/identity.py") files = glob.glob(GLOB) From 853e491f04764d1f15d7c6ae4253aadcb655e711 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 10:22:24 +0300 Subject: [PATCH 10/21] no --venv in pyperformance run --- bot2/pypybuildbot/builds.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 8389baf..b7b2dd2 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -969,15 +969,16 @@ def extract_upload_config(rc, stdout, stderr): description='set upload config', )) - self.addStep( - Translate( - translationArgs=['-Ojit'], - targetArgs=[], - haltOnFailure=True, - # this step can be executed in parallel with other builds - locks=[lock.access('counting')], - ) - ) + # self.addStep( + # Translate( + # translationArgs=['-Ojit'], + # targetArgs=[], + # haltOnFailure=True, + # # this step can be executed in parallel with other builds + # locks=[lock.access('counting')], + # ) + # ) + @renderer def get_cmd(props): # set from testrunner/get_info.py @@ -1077,8 +1078,7 @@ def get_pyperformance_bench_venv_cmd(props): def get_pyperformance_run_cmd(props): return ['bash', '-c', 'rm -f pyperformance_result.json && ' - './pyperformance_venv/bin/python -m pyperformance run ' - '--venv ' + _bench_venv + ' -f ' + './pyperformance_venv/bin/python -m pyperformance run -f ' '--output pyperformance_result.json'] @renderer @@ -1100,8 +1100,7 @@ def get_pyperformance_nojit_wrapper_cmd(props): def get_pyperformance_nojit_run_cmd(props): return ['bash', '-c', 'rm -f pyperformance_nojit_result.json && ' - './pyperformance_venv/bin/python -m pyperformance run ' - '--venv ' + _bench_venv + ' -f ' + './pyperformance_venv/bin/python -m pyperformance run -f ' '--output pyperformance_nojit_result.json ' '--python ./pypy_nojit'] From 8a429be917b916d7001116097afa988f0d81cd30 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 10:28:18 +0300 Subject: [PATCH 11/21] re-enable translation --- bot2/pypybuildbot/builds.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index b7b2dd2..4d8166d 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -969,15 +969,15 @@ def extract_upload_config(rc, stdout, stderr): description='set upload config', )) - # self.addStep( - # Translate( - # translationArgs=['-Ojit'], - # targetArgs=[], - # haltOnFailure=True, - # # this step can be executed in parallel with other builds - # locks=[lock.access('counting')], - # ) - # ) + self.addStep( + Translate( + translationArgs=['-Ojit'], + targetArgs=[], + haltOnFailure=True, + # this step can be executed in parallel with other builds + locks=[lock.access('counting')], + ) + ) @renderer def get_cmd(props): From 74cf7f5251253c520f5f9c3c4cd5a5ec61128a7e Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 13:18:40 +0300 Subject: [PATCH 12/21] debug patch application, for now reuse the build env --- bot2/pypybuildbot/builds.py | 148 ++++++++++++---------- patches/patch_bm_pickle_pypy.py | 6 +- patches/patch_sqlalchemy_identity_pypy.py | 6 +- 3 files changed, 86 insertions(+), 74 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 4d8166d..17da27b 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -386,18 +386,6 @@ def update_hg(platform, factory, repourl, workdir, revision, use_branch, workdir=workdir, logEnviron=False)) -def update_git(platform, factory, repourl, workdir, branch='main', - alwaysUseLatest=False): - factory.addStep( - Git( - repourl=repourl, - mode='full', - method='fresh', - workdir=workdir, - branch=branch, - alwaysUseLatest=alwaysUseLatest, - timeout=40*60, - logEnviron=False)) def setup_steps(platform, factory, workdir=None, @@ -425,7 +413,16 @@ def setup_steps(platform, factory, workdir=None, revision=WithProperties("%(revision)s") # update_hg(platform, factory, repourl, workdir, revision, use_branch=True, # force_branch=force_branch, wipe_bookmarks=True) - update_git(platform, factory, repourl, workdir, branch=force_branch) + # TODO: re-enable git checkout once pyperformance pipeline is stable + if False: + factory.addStep(Git( + repourl=repourl, + mode='full', + method='fresh', + workdir=workdir, + branch=force_branch, + timeout=40*60, + logEnviron=False)) # factory.addStep(CheckGotRevision(workdir=workdir)) @@ -902,46 +899,48 @@ def __init__(self, platform='linux', host='benchmarker', postfix='', # benchmark_branch is the branch in the benchmark repo, # the rest refer to the pypy version to benchmark - # Since we want to use the benchmark_branch, copy the hg update steps - if platform in ("win32", "win64"): - command = "if NOT EXIST .hg rmdir /q /s ." - else: - command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" - self.addStep(ShellCmd(description="rmdir?", - command=command, - workdir='./benchmarks', - haltOnFailure=False)) - # - if platform in ("win32", "win64"): - command = "if NOT EXIST .hg %s" - else: - command = "if [ ! -d .hg ]; then %s; fi" - command = command % ("hg clone -U " + repourl + " .") - self.addStep(ShellCmd(description="hg clone", - command=command, - workdir='./benchmarks', - timeout=3600, - haltOnFailure=True)) - # - self.addStep( - ShellCmd(description="benchmrk: hg purge", - command="hg --config extensions.purge= purge --all", - workdir='./benchmarks', - haltOnFailure=True)) - # - self.addStep(ShellCmd(description="benchmrk: hg pull", - command="hg pull %s" % repourl, - workdir='./benchmarks')) - # - # update with the branch - self.addStep(ShellCmd(description="benchmrk: hg update", - command=Interpolate("hg update --clean %(prop:benchmark_branch)s"), - workdir='./benchmarks')) - - self.addStep(ShellCmd(description="benchmrk: hg report revision", - command=Interpolate("hg parents --template='got_revision:{rev}:{node}'"), - workdir='./benchmarks')) + # TODO: re-enable hg benchmarks checkout and git+translate when + # iterating on pyperformance pipeline is complete + if False: + # Since we want to use the benchmark_branch, copy the hg update steps + if platform in ("win32", "win64"): + command = "if NOT EXIST .hg rmdir /q /s ." + else: + command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" + self.addStep(ShellCmd(description="rmdir?", + command=command, + workdir='./benchmarks', + haltOnFailure=False)) + # + if platform in ("win32", "win64"): + command = "if NOT EXIST .hg %s" + else: + command = "if [ ! -d .hg ]; then %s; fi" + command = command % ("hg clone -U " + repourl + " .") + self.addStep(ShellCmd(description="hg clone", + command=command, + workdir='./benchmarks', + timeout=3600, + haltOnFailure=True)) + # + self.addStep( + ShellCmd(description="benchmrk: hg purge", + command="hg --config extensions.purge= purge --all", + workdir='./benchmarks', + haltOnFailure=True)) + # + self.addStep(ShellCmd(description="benchmrk: hg pull", + command="hg pull %s" % repourl, + workdir='./benchmarks')) + # + # update with the branch + self.addStep(ShellCmd(description="benchmrk: hg update", + command=Interpolate("hg update --clean %(prop:benchmark_branch)s"), + workdir='./benchmarks')) + self.addStep(ShellCmd(description="benchmrk: hg report revision", + command=Interpolate("hg parents --template='got_revision:{rev}:{node}'"), + workdir='./benchmarks')) # setup_steps(platform, self) @@ -969,15 +968,17 @@ def extract_upload_config(rc, stdout, stderr): description='set upload config', )) - self.addStep( - Translate( - translationArgs=['-Ojit'], - targetArgs=[], - haltOnFailure=True, - # this step can be executed in parallel with other builds - locks=[lock.access('counting')], + # TODO: re-enable translation once pyperformance pipeline is stable + if False: + self.addStep( + Translate( + translationArgs=['-Ojit'], + targetArgs=[], + haltOnFailure=True, + # this step can be executed in parallel with other builds + locks=[lock.access('counting')], + ) ) - ) @renderer def get_cmd(props): @@ -1066,13 +1067,11 @@ def get_pyperformance_install_cmd(props): return ['./pyperformance_venv/bin/pip', 'install', '--upgrade', 'pyperformance'] - _bench_venv = 'pyperformance_bench_venv' - @renderer def get_pyperformance_bench_venv_cmd(props): target = props.getProperty('target_path') return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', - 'venv', 'create', '--venv', _bench_venv, '-p', target] + 'venv', 'create', '-p', target] @renderer def get_pyperformance_run_cmd(props): @@ -1142,12 +1141,20 @@ def get_pyperformance_nojit_upload_cmd(props): self.addStep(ShellCmd( description='apply PyPy compatibility patches', command=['python3', '-c', - 'import glob, subprocess, sys\n' + 'import glob, os, subprocess, sys\n' + 'print("cwd:", os.getcwd())\n' + 'print("cwd contents:", sorted(os.listdir(".")))\n' + 'venv_dir = "venv"\n' + 'if os.path.isdir(venv_dir):\n' + ' print("venv/ contents:", sorted(os.listdir(venv_dir)))\n' + 'else:\n' + ' print("venv/ directory does not exist")\n' 'scripts = sorted(glob.glob("patch_*_pypy.py"))\n' 'print("Applying patches:", scripts)\n' 'for f in scripts:\n' ' subprocess.check_call([sys.executable, f])\n'], doStepIf=is_py3_target, + haltOnFailure=True, workdir='.')) self.addStep(ShellCmd( @@ -1383,10 +1390,15 @@ def __init__(self, platform='linux', # obtain a pypy-compatible branch of numpy numpy_url = 'https://foss.heptapod.net/pypy/numpy' - update_git(platform, self, numpy_url, 'numpy_src', branch='master', - alwaysUseLatest=True, # ignore pypy rev number when - # triggered by a pypy build - ) + self.addStep(Git( + repourl=numpy_url, + mode='full', + method='fresh', + workdir='numpy_src', + branch='master', + alwaysUseLatest=True, + timeout=40*60, + logEnviron=False)) self.addStep(ShellCmd( description="install numpy", diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py index 9577b37..805e67d 100644 --- a/patches/patch_bm_pickle_pypy.py +++ b/patches/patch_bm_pickle_pypy.py @@ -10,13 +10,13 @@ import glob import sys -GLOB = ("./pyperformance_bench_venv/lib/python*/site-packages" +GLOB = ("./venv/*/lib/python*/site-packages" "/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py") files = glob.glob(GLOB) if not files: - print("WARNING: bm_pickle run_benchmark.py not found – skipping patch") - sys.exit(0) + print("ERROR: bm_pickle run_benchmark.py not found – patch failed") + sys.exit(1) for path in files: txt = open(path).read() diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index 7309697..f0a07de 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -12,13 +12,13 @@ import re import sys -GLOB = ("./pyperformance_bench_venv/lib/python*/site-packages" +GLOB = ("./venv/*/lib/python*/site-packages" "/sqlalchemy/orm/identity.py") files = glob.glob(GLOB) if not files: - print("WARNING: sqlalchemy identity.py not found – skipping patch") - sys.exit(0) + print("ERROR: sqlalchemy identity.py not found – patch failed") + sys.exit(1) # Replace every bare call to _manage_removed_state(existing_non_none) with a # weakref liveness check. Capture indentation so the replacement is correctly From ba951dd8ff275f8a9c624519feb1cdafefb72a37 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 14:04:00 +0300 Subject: [PATCH 13/21] reorder and fix patching Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 39 +++++++++++------------ patches/patch_bm_pickle_pypy.py | 2 +- patches/patch_sqlalchemy_identity_pypy.py | 2 +- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 17da27b..5a0213a 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1000,14 +1000,6 @@ def get_cmd(props): ] return command - self.addStep(ShellCmd( - # this step needs exclusive access to the CPU - locks=[lock.access('exclusive')], - description="run benchmarks on top of pypy-c", - command=get_cmd, - workdir='./benchmarks', - timeout=3600)) - # Push bulk_upload.py to the slave so upload steps can use it self.addStep(transfer.FileDownload( mastersrc=os.path.join(os.path.dirname(__file__), 'bulk_upload.py'), @@ -1041,17 +1033,6 @@ def get_upload_baseline_cmd(props): '-u', upload_env['SPEED_UPLOAD_URL'], '--baseline'] - self.addStep(ShellCmd( - description='upload legacy results (jit-off)', - command=get_upload_changed_cmd, - env=upload_env, - workdir='.')) - self.addStep(ShellCmd( - description='upload legacy results (jit-on)', - command=get_upload_baseline_cmd, - env=upload_env, - workdir='.')) - # Pyperformance: only when the target binary is pypy3* def is_py3_target(step): target = step.build.getProperty('target_path') or '' @@ -1071,7 +1052,7 @@ def get_pyperformance_install_cmd(props): def get_pyperformance_bench_venv_cmd(props): target = props.getProperty('target_path') return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', - 'venv', 'create', '-p', target] + 'venv', 'recreate', '-p', target] @renderer def get_pyperformance_run_cmd(props): @@ -1189,6 +1170,24 @@ def get_pyperformance_nojit_upload_cmd(props): doStepIf=is_py3_target, workdir='.')) + self.addStep(ShellCmd( + # this step needs exclusive access to the CPU + locks=[lock.access('exclusive')], + description="run benchmarks on top of pypy-c", + command=get_cmd, + workdir='./benchmarks', + timeout=3600)) + self.addStep(ShellCmd( + description='upload legacy results (jit-off)', + command=get_upload_changed_cmd, + env=upload_env, + workdir='.')) + self.addStep(ShellCmd( + description='upload legacy results (jit-on)', + command=get_upload_baseline_cmd, + env=upload_env, + workdir='.')) + # Archive the legacy result file on the master filename = '%(got_revision)s' + (postfix or '') resfile = os.path.expanduser("~/bench_results/%s.json" % filename) diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py index 805e67d..b919cbc 100644 --- a/patches/patch_bm_pickle_pypy.py +++ b/patches/patch_bm_pickle_pypy.py @@ -10,7 +10,7 @@ import glob import sys -GLOB = ("./venv/*/lib/python*/site-packages" +GLOB = ("./venv/*/lib/*/site-packages" "/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py") files = glob.glob(GLOB) diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index f0a07de..aaa4694 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -12,7 +12,7 @@ import re import sys -GLOB = ("./venv/*/lib/python*/site-packages" +GLOB = ("./venv/*/lib/*/site-packages" "/sqlalchemy/orm/identity.py") files = glob.glob(GLOB) From 40749525cf5046afb378f6779f15fe06059964c1 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 14:50:58 +0300 Subject: [PATCH 14/21] tweak target name so only one venv is created Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 5a0213a..48e9014 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1056,9 +1056,11 @@ def get_pyperformance_bench_venv_cmd(props): @renderer def get_pyperformance_run_cmd(props): + target = props.getProperty('target_path') return ['bash', '-c', 'rm -f pyperformance_result.json && ' './pyperformance_venv/bin/python -m pyperformance run -f ' + '--python ' + target + ' ' '--output pyperformance_result.json'] @renderer From 7e03921d8b27289f41603db1e147492a730102d1 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 17:22:36 +0300 Subject: [PATCH 15/21] tweak file locations for patches, make sure jit-off works Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 42 ++++-------- patches/patch_bm_pickle_pypy.py | 51 ++++++++++---- patches/patch_sqlalchemy_identity_pypy.py | 84 ++++++++++++++++------- 3 files changed, 107 insertions(+), 70 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 48e9014..b4de54b 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1054,14 +1054,15 @@ def get_pyperformance_bench_venv_cmd(props): return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', 'venv', 'recreate', '-p', target] - @renderer - def get_pyperformance_run_cmd(props): - target = props.getProperty('target_path') - return ['bash', '-c', - 'rm -f pyperformance_result.json && ' - './pyperformance_venv/bin/python -m pyperformance run -f ' - '--python ' + target + ' ' - '--output pyperformance_result.json'] + def get_pyperformance_run_cmd(outfile): + @renderer + def _cmd(props): + target = props.getProperty('target_path') + return ['bash', '-c', + 'rm -f %s && ' + './pyperformance_venv/bin/python -m pyperformance run -f ' + '--python %s --output %s' % (outfile, target, outfile)] + return _cmd @renderer def get_pyperformance_upload_cmd(props): @@ -1071,21 +1072,6 @@ def get_pyperformance_upload_cmd(props): '-P', project, '-r', rev, '-B', branch, '-u', upload_env['SPEED_UPLOAD_URL']] - @renderer - def get_pyperformance_nojit_wrapper_cmd(props): - target = props.getProperty('target_path') - script = '#!/bin/sh\nexec %s --jit off "$@"\n' % target - return ['python3', '-c', - 'open("pypy_nojit","w").write(%r);import os;os.chmod("pypy_nojit",0o755)' % script] - - @renderer - def get_pyperformance_nojit_run_cmd(props): - return ['bash', '-c', - 'rm -f pyperformance_nojit_result.json && ' - './pyperformance_venv/bin/python -m pyperformance run -f ' - '--output pyperformance_nojit_result.json ' - '--python ./pypy_nojit'] - @renderer def get_pyperformance_nojit_upload_cmd(props): exe, project, rev, branch = _props_for_upload(props) @@ -1142,7 +1128,7 @@ def get_pyperformance_nojit_upload_cmd(props): self.addStep(ShellCmd( description='run pyperformance (jit)', - command=get_pyperformance_run_cmd, + command=get_pyperformance_run_cmd('pyperformance_result.json'), locks=[lock.access('exclusive')], doStepIf=is_py3_target, workdir='.', @@ -1153,14 +1139,10 @@ def get_pyperformance_nojit_upload_cmd(props): env=upload_env, doStepIf=is_py3_target, workdir='.')) - self.addStep(ShellCmd( - description='create pyperformance nojit wrapper', - command=get_pyperformance_nojit_wrapper_cmd, - doStepIf=is_py3_target, - workdir='.')) self.addStep(ShellCmd( description='run pyperformance (nojit)', - command=get_pyperformance_nojit_run_cmd, + command=get_pyperformance_run_cmd('pyperformance_nojit_result.json'), + env={'PYPY_DISABLE_JIT': '1'}, locks=[lock.access('exclusive')], doStepIf=is_py3_target, workdir='.', diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py index b919cbc..f51683e 100644 --- a/patches/patch_bm_pickle_pypy.py +++ b/patches/patch_bm_pickle_pypy.py @@ -8,31 +8,54 @@ Source: https://github.com/python/pyperformance/pull/461 """ import glob +import os import sys -GLOB = ("./venv/*/lib/*/site-packages" - "/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py") +GLOBS = [ + "./*/lib/*/site-packages/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py", + "./*/lib/*/site-packages/benchmarks/bm_pickle/run_benchmark.py", +] + +# Deduplicate by real path in case any entries are symlinks to the same file +seen = {} +for pattern in GLOBS: + for path in glob.glob(pattern): + seen[os.path.realpath(path)] = path +files = list(seen.values()) -files = glob.glob(GLOB) if not files: print("ERROR: bm_pickle run_benchmark.py not found – patch failed") sys.exit(1) +OLD_PURE = "if not (options.pure_python or IS_PYPY):" +NEW_PURE = "if not (options.pure_python):" +OLD_ACCEL = "if not is_accelerated_module(pickle):" +NEW_ACCEL = "if not is_accelerated_module(pickle) and not IS_PYPY:" + +failed = False for path in files: txt = open(path).read() original = txt - txt = txt.replace( - "if not (options.pure_python or IS_PYPY):", - "if not (options.pure_python):", - ) - txt = txt.replace( - "if not is_accelerated_module(pickle):", - "if not is_accelerated_module(pickle) and not IS_PYPY:", - ) - - if txt == original: - print(f"NOTE: {path} already patched or pattern not found") + if OLD_PURE in txt: + txt = txt.replace(OLD_PURE, NEW_PURE) + elif NEW_PURE in txt: + print(f"NOTE: {path} pure_python guard already patched") + else: + print(f"ERROR: {path} pure_python guard pattern not found – version mismatch?") + failed = True + + if OLD_ACCEL in txt: + txt = txt.replace(OLD_ACCEL, NEW_ACCEL) + elif NEW_ACCEL in txt: + print(f"NOTE: {path} accelerated-module guard already patched") else: + print(f"ERROR: {path} accelerated-module guard pattern not found – version mismatch?") + failed = True + + if txt != original: open(path, "w").write(txt) print(f"Patched {path}") + +if failed: + sys.exit(1) diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index aaa4694..5717014 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -3,48 +3,80 @@ PyPy's garbage collector may collect an object referenced by a weakref before the weakref is checked, causing 'NoneType has no attribute __dict__' errors in -the ORM identity map. Guard the call to _manage_removed_state with an +the ORM identity map. Guard each call to _manage_removed_state with an explicit liveness check on the weakref. Source: https://github.com/sqlalchemy/sqlalchemy/discussions/13274 """ import glob -import re +import os import sys -GLOB = ("./venv/*/lib/*/site-packages" - "/sqlalchemy/orm/identity.py") +GLOBS = [ + "./*/lib/*/site-packages/sqlalchemy/orm/identity.py", +] + +seen = {} +for pattern in GLOBS: + for path in glob.glob(pattern): + seen[os.path.realpath(path)] = path +files = list(seen.values()) -files = glob.glob(GLOB) if not files: print("ERROR: sqlalchemy identity.py not found – patch failed") sys.exit(1) -# Replace every bare call to _manage_removed_state(existing_non_none) with a -# weakref liveness check. Capture indentation so the replacement is correctly -# indented regardless of nesting level. -PATTERN = re.compile( - r"^( +)self\._manage_removed_state\(existing_non_none\)\s*$", - re.MULTILINE, +# In replace(): if the GC collected existing, set existing = None so the +# caller gets None back rather than a dead weakref. +OLD_REPLACE = ( + " if existing is not state:\n" + " self._manage_removed_state(existing)\n" + " else:\n" + " return None" +) +NEW_REPLACE = ( + " if existing is not state:\n" + " if existing.obj() is not None:\n" + " self._manage_removed_state(existing)\n" + " else:\n" + " existing = None\n" + " else:\n" + " return None" ) +# In safe_discard(): no return value, so no else clause needed. +OLD_DISCARD = ( + " self._manage_removed_state(state)\n" +) +NEW_DISCARD = ( + " if state.obj() is not None:\n" + " self._manage_removed_state(state)\n" +) -def _replacement(m): - i = m.group(1) - return ( - f"{i}if existing_non_none.obj() is not None:\n" - f"{i} self._manage_removed_state(existing_non_none)\n" - f"{i}else:\n" - f"{i} existing = None" - ) - - +failed = False for path in files: txt = open(path).read() - new_txt, count = PATTERN.subn(_replacement, txt) + original = txt - if count == 0: - print(f"NOTE: {path} already patched or pattern not found") + if OLD_REPLACE in txt: + txt = txt.replace(OLD_REPLACE, NEW_REPLACE) + elif NEW_REPLACE in txt: + print(f"NOTE: {path} replace() already patched") else: - open(path, "w").write(new_txt) - print(f"Patched {path} ({count} site(s))") + print(f"ERROR: {path} replace() pattern not found – version mismatch?") + failed = True + + if OLD_DISCARD in txt: + txt = txt.replace(OLD_DISCARD, NEW_DISCARD) + elif NEW_DISCARD in txt: + print(f"NOTE: {path} safe_discard() already patched") + else: + print(f"ERROR: {path} safe_discard() pattern not found – version mismatch?") + failed = True + + if txt != original: + open(path, "w").write(txt) + print(f"Patched {path}") + +if failed: + sys.exit(1) From 90b64172a4f9329abcd71cbf702f952f63093e0f Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 17:31:51 +0300 Subject: [PATCH 16/21] tweak patching Co-Authored-By: Claude Sonnet 4.6 --- patches/patch_bm_pickle_pypy.py | 2 ++ patches/patch_sqlalchemy_identity_pypy.py | 1 + 2 files changed, 3 insertions(+) diff --git a/patches/patch_bm_pickle_pypy.py b/patches/patch_bm_pickle_pypy.py index f51683e..410c171 100644 --- a/patches/patch_bm_pickle_pypy.py +++ b/patches/patch_bm_pickle_pypy.py @@ -14,6 +14,8 @@ GLOBS = [ "./*/lib/*/site-packages/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py", "./*/lib/*/site-packages/benchmarks/bm_pickle/run_benchmark.py", + "./*/*/lib/*/site-packages/pyperformance/data-files/benchmarks/bm_pickle/run_benchmark.py", + "./*/*/lib/*/site-packages/benchmarks/bm_pickle/run_benchmark.py", ] # Deduplicate by real path in case any entries are symlinks to the same file diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index 5717014..47bf81f 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -14,6 +14,7 @@ GLOBS = [ "./*/lib/*/site-packages/sqlalchemy/orm/identity.py", + "./*/*/lib/*/site-packages/sqlalchemy/orm/identity.py", ] seen = {} From ed4896e744a1941f49bc160cd8abb95c66745e5f Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 21:32:54 +0300 Subject: [PATCH 17/21] revert debug stuff, add pyperformance pip specifier Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/builds.py | 115 ++++++++++++++++-------------------- bot2/pypybuildbot/master.py | 23 +++++++- 2 files changed, 73 insertions(+), 65 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index b4de54b..448c4c5 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -413,9 +413,7 @@ def setup_steps(platform, factory, workdir=None, revision=WithProperties("%(revision)s") # update_hg(platform, factory, repourl, workdir, revision, use_branch=True, # force_branch=force_branch, wipe_bookmarks=True) - # TODO: re-enable git checkout once pyperformance pipeline is stable - if False: - factory.addStep(Git( + factory.addStep(Git( repourl=repourl, mode='full', method='fresh', @@ -899,50 +897,40 @@ def __init__(self, platform='linux', host='benchmarker', postfix='', # benchmark_branch is the branch in the benchmark repo, # the rest refer to the pypy version to benchmark - # TODO: re-enable hg benchmarks checkout and git+translate when - # iterating on pyperformance pipeline is complete - if False: - # Since we want to use the benchmark_branch, copy the hg update steps - if platform in ("win32", "win64"): - command = "if NOT EXIST .hg rmdir /q /s ." - else: - command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" - self.addStep(ShellCmd(description="rmdir?", - command=command, - workdir='./benchmarks', - haltOnFailure=False)) - # - if platform in ("win32", "win64"): - command = "if NOT EXIST .hg %s" - else: - command = "if [ ! -d .hg ]; then %s; fi" - command = command % ("hg clone -U " + repourl + " .") - self.addStep(ShellCmd(description="hg clone", - command=command, - workdir='./benchmarks', - timeout=3600, - haltOnFailure=True)) - # - self.addStep( - ShellCmd(description="benchmrk: hg purge", - command="hg --config extensions.purge= purge --all", - workdir='./benchmarks', - haltOnFailure=True)) - # - self.addStep(ShellCmd(description="benchmrk: hg pull", - command="hg pull %s" % repourl, - workdir='./benchmarks')) - # - # update with the branch - self.addStep(ShellCmd(description="benchmrk: hg update", - command=Interpolate("hg update --clean %(prop:benchmark_branch)s"), - workdir='./benchmarks')) - - self.addStep(ShellCmd(description="benchmrk: hg report revision", - command=Interpolate("hg parents --template='got_revision:{rev}:{node}'"), - workdir='./benchmarks')) + # Since we want to use the benchmark_branch, copy the hg update steps + if platform in ("win32", "win64"): + command = "if NOT EXIST .hg rmdir /q /s ." + else: + command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" + self.addStep(ShellCmd(description="rmdir?", + command=command, + workdir='./benchmarks', + haltOnFailure=False)) + if platform in ("win32", "win64"): + command = "if NOT EXIST .hg %s" + else: + command = "if [ ! -d .hg ]; then %s; fi" + command = command % ("hg clone -U " + repourl + " .") + self.addStep(ShellCmd(description="hg clone", + command=command, + workdir='./benchmarks', + timeout=3600, + haltOnFailure=True)) + self.addStep( + ShellCmd(description="benchmrk: hg purge", + command="hg --config extensions.purge= purge --all", + workdir='./benchmarks', + haltOnFailure=True)) + self.addStep(ShellCmd(description="benchmrk: hg pull", + command="hg pull %s" % repourl, + workdir='./benchmarks')) + self.addStep(ShellCmd(description="benchmrk: hg update", + command=Interpolate("hg update --clean %(prop:benchmark_branch)s"), + workdir='./benchmarks')) + self.addStep(ShellCmd(description="benchmrk: hg report revision", + command=Interpolate("hg parents --template='got_revision:{rev}:{node}'"), + workdir='./benchmarks')) - # setup_steps(platform, self) if host == 'benchmarker': lock = BenchmarkerLock @@ -968,17 +956,15 @@ def extract_upload_config(rc, stdout, stderr): description='set upload config', )) - # TODO: re-enable translation once pyperformance pipeline is stable - if False: - self.addStep( - Translate( - translationArgs=['-Ojit'], - targetArgs=[], - haltOnFailure=True, - # this step can be executed in parallel with other builds - locks=[lock.access('counting')], - ) + self.addStep( + Translate( + translationArgs=['-Ojit'], + targetArgs=[], + haltOnFailure=True, + # this step can be executed in parallel with other builds + locks=[lock.access('counting')], ) + ) @renderer def get_cmd(props): @@ -991,7 +977,7 @@ def get_cmd(props): branch = props.getProperty('branch') if branch == 'None' or branch is None: branch = 'default' - command=["python", "-u", "runner.py", '--fast', '--output-filename', 'result.json', + command=["python", "-u", "runner.py", '--output-filename', 'result.json', '--changed', target, '--baseline', target, '--args', ',--jit off', @@ -1045,8 +1031,8 @@ def get_pyperformance_venv_cmd(props): @renderer def get_pyperformance_install_cmd(props): - return ['./pyperformance_venv/bin/pip', 'install', '--upgrade', - 'pyperformance'] + spec = props.getProperty('pyperformance_spec', default='pyperformance').strip() + return ['./pyperformance_venv/bin/pip', 'install', '--upgrade', spec] @renderer def get_pyperformance_bench_venv_cmd(props): @@ -1054,14 +1040,16 @@ def get_pyperformance_bench_venv_cmd(props): return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', 'venv', 'recreate', '-p', target] - def get_pyperformance_run_cmd(outfile): + def get_pyperformance_run_cmd(outfile, inherit_environ=None): @renderer def _cmd(props): target = props.getProperty('target_path') + inherit = ('--inherit-environ %s ' % inherit_environ + if inherit_environ else '') return ['bash', '-c', 'rm -f %s && ' - './pyperformance_venv/bin/python -m pyperformance run -f ' - '--python %s --output %s' % (outfile, target, outfile)] + './pyperformance_venv/bin/python -m pyperformance run ' + '%s--python %s --output %s' % (outfile, inherit, target, outfile)] return _cmd @renderer @@ -1141,7 +1129,8 @@ def get_pyperformance_nojit_upload_cmd(props): workdir='.')) self.addStep(ShellCmd( description='run pyperformance (nojit)', - command=get_pyperformance_run_cmd('pyperformance_nojit_result.json'), + command=get_pyperformance_run_cmd('pyperformance_nojit_result.json', + inherit_environ='PYPY_DISABLE_JIT'), env={'PYPY_DISABLE_JIT': '1'}, locks=[lock.access('exclusive')], doStepIf=is_py3_target, diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py index 5f2e729..71dc605 100644 --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -27,14 +27,25 @@ def force(self, owner, builder_name, **kwargs): return ForceScheduler.force(self, owner, builder_name, **kwargs) +import re as _re +_PYPERFORMANCE_SPEC_RE = _re.compile( + r'^(' + r'pyperformance\s*([><=!~^][^;]*)?' # PyPI: pyperformance, pyperformance==1.2 + r'|git\+https://github\.com/python/pyperformance(@\S+)?' # GitHub: python/pyperformance only + r')$' +) + class BenchmarkForceScheduler(CustomForceScheduler): ''' - A ForceScheduler with an extra field: benchmark_branch + A ForceScheduler with extra fields: benchmark_branch and pyperformance_spec ''' def __init__(self, name, builderNames, benchmark_branch=StringParameter(name="benchmark_branch", - label="Benchmark repo branch:", + label="Legacy benchmark repo branch:", default="default", length=20), + pyperformance_spec=StringParameter(name="pyperformance_spec", + label="pyperformance pip specifier:", + default="pyperformance", length=60), properties=[ CodebaseParameter('PyPy repo', label='PyPy Repo')], **kwargs): CustomForceScheduler.__init__(self, name, builderNames, **kwargs) @@ -45,8 +56,16 @@ def __init__(self, name, builderNames, pypy_branch) self.all_fields.append(benchmark_branch) self.forcedProperties.append(benchmark_branch) + self.all_fields.append(pyperformance_spec) + self.forcedProperties.append(pyperformance_spec) def force(self, owner, builderNames=None, **kwargs): + spec = kwargs.get('pyperformance_spec', 'pyperformance').strip() + if not _PYPERFORMANCE_SPEC_RE.match(spec): + raise ValidationError( + "pyperformance pip specifier must be 'pyperformance', " + "'pyperformance==', or " + "'git+https://github.com/python/pyperformance@'") CustomForceScheduler.force(self, owner, builderNames, **kwargs) # Forbid "stop build" without a reason that starts with "!" From 9f4047e97090af5a76e3d3837a4abbfeb2f54eff Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 4 May 2026 21:40:33 +0300 Subject: [PATCH 18/21] typo Co-Authored-By: Claude Sonnet 4.6 --- bot2/pypybuildbot/master.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py index 71dc605..72711d6 100644 --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -60,7 +60,10 @@ def __init__(self, name, builderNames, self.forcedProperties.append(pyperformance_spec) def force(self, owner, builderNames=None, **kwargs): - spec = kwargs.get('pyperformance_spec', 'pyperformance').strip() + spec = kwargs.get('pyperformance_spec', 'pyperformance') + if isinstance(spec, list): + spec = spec[0] if spec else 'pyperformance' + spec = spec.strip() if not _PYPERFORMANCE_SPEC_RE.match(spec): raise ValidationError( "pyperformance pip specifier must be 'pyperformance', " From c9d5660d331a7dbebf2f186f6c5fe3d81ab2fc84 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 5 May 2026 08:23:51 +0300 Subject: [PATCH 19/21] use 'fast' -f for non-jit pyperformance benchmarks --- bot2/pypybuildbot/builds.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 448c4c5..518fb89 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1040,16 +1040,17 @@ def get_pyperformance_bench_venv_cmd(props): return ['./pyperformance_venv/bin/python', '-m', 'pyperformance', 'venv', 'recreate', '-p', target] - def get_pyperformance_run_cmd(outfile, inherit_environ=None): + def get_pyperformance_run_cmd(outfile, inherit_environ=None, fast=False): @renderer def _cmd(props): target = props.getProperty('target_path') inherit = ('--inherit-environ %s ' % inherit_environ if inherit_environ else '') + fast_flag = '-f ' if fast else '' return ['bash', '-c', 'rm -f %s && ' './pyperformance_venv/bin/python -m pyperformance run ' - '%s--python %s --output %s' % (outfile, inherit, target, outfile)] + '%s%s--python %s --output %s' % (outfile, fast_flag, inherit, target, outfile)] return _cmd @renderer @@ -1130,7 +1131,8 @@ def get_pyperformance_nojit_upload_cmd(props): self.addStep(ShellCmd( description='run pyperformance (nojit)', command=get_pyperformance_run_cmd('pyperformance_nojit_result.json', - inherit_environ='PYPY_DISABLE_JIT'), + inherit_environ='PYPY_DISABLE_JIT', + fast=True), env={'PYPY_DISABLE_JIT': '1'}, locks=[lock.access('exclusive')], doStepIf=is_py3_target, From 4d3d1d5118ba05bfd735b4e35af1d191e991a66b Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 10 May 2026 07:18:52 +0300 Subject: [PATCH 20/21] fix sqlalchemy patch to patch runner instead --- patches/patch_sqlalchemy_identity_pypy.py | 74 +++++++++-------------- 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/patches/patch_sqlalchemy_identity_pypy.py b/patches/patch_sqlalchemy_identity_pypy.py index 47bf81f..a4e819e 100644 --- a/patches/patch_sqlalchemy_identity_pypy.py +++ b/patches/patch_sqlalchemy_identity_pypy.py @@ -1,20 +1,27 @@ """ -Patch SQLAlchemy identity map for PyPy GC compatibility. +Patch bm_sqlalchemy_declarative for PyPy GC compatibility. -PyPy's garbage collector may collect an object referenced by a weakref before -the weakref is checked, causing 'NoneType has no attribute __dict__' errors in -the ORM identity map. Guard each call to _manage_removed_state with an -explicit liveness check on the weakref. +SQLite reuses primary key IDs after a full table DELETE; the next loop +iteration inserts new objects with the same IDs as the previous run. On +CPython, refcounting immediately collects the old objects, clearing the +identity map's weakrefs. On PyPy, GC is deferred, so the old objects +remain alive and SQLAlchemy emits a SAWarning for every inserted row on +every loop, flooding stderr and causing the benchmark to fail. -Source: https://github.com/sqlalchemy/sqlalchemy/discussions/13274 +Fix: call session.expunge_all() after the bulk deletes to explicitly clear +stale session state before timing begins. + +Source: https://github.com/python/pyperformance/pull/472 """ import glob import os import sys GLOBS = [ - "./*/lib/*/site-packages/sqlalchemy/orm/identity.py", - "./*/*/lib/*/site-packages/sqlalchemy/orm/identity.py", + "./*/lib/*/site-packages/pyperformance/data-files/benchmarks/bm_sqlalchemy_declarative/run_benchmark.py", + "./*/lib/*/site-packages/benchmarks/bm_sqlalchemy_declarative/run_benchmark.py", + "./*/*/lib/*/site-packages/pyperformance/data-files/benchmarks/bm_sqlalchemy_declarative/run_benchmark.py", + "./*/*/lib/*/site-packages/benchmarks/bm_sqlalchemy_declarative/run_benchmark.py", ] seen = {} @@ -24,34 +31,17 @@ files = list(seen.values()) if not files: - print("ERROR: sqlalchemy identity.py not found – patch failed") + print("ERROR: bm_sqlalchemy_declarative/run_benchmark.py not found – patch failed") sys.exit(1) -# In replace(): if the GC collected existing, set existing = None so the -# caller gets None back rather than a dead weakref. -OLD_REPLACE = ( - " if existing is not state:\n" - " self._manage_removed_state(existing)\n" - " else:\n" - " return None" -) -NEW_REPLACE = ( - " if existing is not state:\n" - " if existing.obj() is not None:\n" - " self._manage_removed_state(existing)\n" - " else:\n" - " existing = None\n" - " else:\n" - " return None" -) - -# In safe_discard(): no return value, so no else clause needed. -OLD_DISCARD = ( - " self._manage_removed_state(state)\n" +OLD = ( + " session.query(Person).delete(synchronize_session=False)\n" + " session.query(Address).delete(synchronize_session=False)\n" ) -NEW_DISCARD = ( - " if state.obj() is not None:\n" - " self._manage_removed_state(state)\n" +NEW = ( + " session.query(Person).delete(synchronize_session=False)\n" + " session.query(Address).delete(synchronize_session=False)\n" + " session.expunge_all()\n" ) failed = False @@ -59,20 +49,12 @@ txt = open(path).read() original = txt - if OLD_REPLACE in txt: - txt = txt.replace(OLD_REPLACE, NEW_REPLACE) - elif NEW_REPLACE in txt: - print(f"NOTE: {path} replace() already patched") - else: - print(f"ERROR: {path} replace() pattern not found – version mismatch?") - failed = True - - if OLD_DISCARD in txt: - txt = txt.replace(OLD_DISCARD, NEW_DISCARD) - elif NEW_DISCARD in txt: - print(f"NOTE: {path} safe_discard() already patched") + if OLD in txt: + txt = txt.replace(OLD, NEW) + elif NEW in txt: + print(f"NOTE: {path} already patched") else: - print(f"ERROR: {path} safe_discard() pattern not found – version mismatch?") + print(f"ERROR: {path} pattern not found – version mismatch?") failed = True if txt != original: From 6c24188b7dffaa6c7ba7b5bc1a5180a972b7de8c Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 10 May 2026 11:25:06 +0300 Subject: [PATCH 21/21] add step to clean out venvs --- bot2/pypybuildbot/builds.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py index 518fb89..823acea 100644 --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1069,6 +1069,11 @@ def get_pyperformance_nojit_upload_cmd(props): '-P', project, '-r', rev, '-B', branch, '-u', upload_env['SPEED_UPLOAD_URL']] + self.addStep(ShellCmd( + description='clean up old pypy venvs', + command=['sh', '-c', 'rm -rf venv pyperformance_venv'], + doStepIf=is_py3_target, + workdir='.')) self.addStep(ShellCmd( description='create pyperformance venv', command=get_pyperformance_venv_cmd,