ci: add codspeed benchmarks

.github/workflows/ci.yaml

@@ -6,11 +6,11 @@ env:
 
 on:
   push:
-    branches: [main, ci/*, dependabot/*, renovate/*]
+    branches: [ main, ci/*, dependabot/*, renovate/* ]
     tags:
       - "v*.*.*"
   pull_request:
-    branches: [main]
+    branches: [ main ]
 
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
@@ -57,7 +57,7 @@ jobs:
 
   tests:
     name: 🔬 ${{ matrix.session.job_name }}
-    needs: [pre, generate-jobs-tests]
+    needs: [ pre, generate-jobs-tests ]
     if: github.ref_type == 'tag' || needs.pre.outputs.should_skip != 'true'
     runs-on: ubuntu-latest
     strategy:
@@ -119,9 +119,46 @@ jobs:
           name: test-results-${{ matrix.session.session }}
           path: junit-${{ matrix.session.session }}.xml
 
+  benchmarks:
+    name: 📈 Run benchmarks
+    needs: [ tests, generate-jobs-tests ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Set up Docker Buildx
+        id: docker-buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Pip and nox cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache
+          key:
+            ${{ runner.os }}-benchmarks-${{ env.pythonLocation }}-${{ hashFiles('**/uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-benchmarks-${{ env.pythonLocation }}
+
+      - name: Setup mise
+        uses: jdx/mise-action@v3
+        with:
+          cache: true
+          log_level: debug
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: instrumentation
+          run: mise run test:benchmarks
+          token: ${{ secrets.CODSPEED_TOKEN }} # optional for public repos
+
+
   upload-coverage:
     name: 🆙 Upload Coverage
-    needs: [tests, generate-jobs-tests]
+    needs: [ tests, generate-jobs-tests ]
     runs-on: ubuntu-latest
 
     steps:
@@ -155,7 +192,7 @@ jobs:
 
   upload-test-results:
     name: 📊 Upload test results
-    needs: [tests, generate-jobs-tests]
+    needs: [ tests, generate-jobs-tests ]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -208,6 +245,7 @@ jobs:
       - pre
       - tests
       - lint
+      - benchmarks
     steps:
       - name: Mark workflow as failed if lint or test did not pass or were cancelled
         if: contains(fromJson('["failure", "cancelled"]'), needs.tests.result) || contains(fromJson('["failure", "cancelled"]'), needs.lint.result)

.gitignore

@@ -192,3 +192,5 @@ mise.local.toml
 .windsurf
 
 CLAUDE.md
+
+.codspeed

mise.toml

@@ -47,6 +47,11 @@ description = "Install dependencies"
 hide = true
 run = "uv sync --all-extras --dev"
 
+[tasks."_install:test"]
+description = "Install test dependencies"
+hide = true
+run = "uv sync --all-extras --only-group test"
+
 [tasks."_install:geo"]
 description = "Install dependencies with geo extras"
 hide = true
@@ -149,6 +154,12 @@ description = "Run snapshot-based tests and update snapshots"
 depends = "_install"
 run = "uv run pytest {{vars.local_pytest_options}} -m snapshot --snapshot-update"
 
+[tasks."test:benchmarks"]
+description = "Run benchmarks"
+depends = "_install:test"
+run = "uv run pytest --codspeed tests/benchmarks"
+
+
 # ###############
 # CI
 # ###############

pyproject.toml

@@ -80,6 +80,7 @@ test = [
     "pytest-xdist",
     "pytest-databases[postgres,mysql]",
     "pytest-lazy-fixtures",
+    "pytest-codspeed",
     "syrupy",
     "sqlparse",
     "covdefaults",
@@ -296,12 +297,13 @@ markers = [
     "snapshot: snapshot-based tests using syrupy",
     "integration: SQLAlchemy integration tests",
     "geo: Geoalchemy2 integration tests",
-    "aiosqlite: SQLite (aiosqlite) Tests",
-    "sqlite: SQLite Tests",
-    "asyncmy: MySQL (asyncmy) Tests",
-    "psycopg_async: SQLAlchemy Postgres (psycopg async) Tests",
-    "psycopg_sync: SQLAlchemy Postgres (psycopg sync) Tests",
-    "asyncpg: SQLAlchemy Postgres (asyncpg) Tests",
+    "aiosqlite: SQLite (aiosqlite) tests",
+    "sqlite: SQLite tests",
+    "asyncmy: MySQL (asyncmy) tests",
+    "psycopg_async: SQLAlchemy Postgres (psycopg async) tests",
+    "psycopg_sync: SQLAlchemy Postgres (psycopg sync) tests",
+    "asyncpg: SQLAlchemy Postgres (asyncpg) tests",
+    "benchmark: Benchmark tests"
 ]
 testpaths = ["tests"]
 xfail_strict = true

tasks.md

@@ -146,6 +146,14 @@ Run tests
 
 #### `[test]`
 
+## `test:benchmarks`
+
+- Depends: _install:test
+
+- **Usage**: `test:benchmarks`
+
+Run benchmarks
+
 ## `test:coverage`
 
 - Depends: _install

tests/benchmarks/conftest.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from tests.integration.fixtures import (
+    aiosqlite_engine,
+    any_query,
+    any_session,
+    async_engine,
+    async_mutation,
+    async_query,
+    async_session,
+    asyncmy_engine,
+    asyncpg_engine,
+    before_create_all_statements,
+    config,
+    db_features,
+    dialect,
+    engine,
+    fx_metadata,
+    mapper,
+    no_session_query,
+    postgis_service,
+    postgres_database_service,
+    psycopg_async_engine,
+    psycopg_engine,
+    query_tracker,
+    raw_arrays,
+    raw_colors,
+    raw_date_times,
+    raw_farms,
+    raw_fruits,
+    raw_geo,
+    raw_geo_flipped,
+    raw_groups,
+    raw_intervals,
+    raw_json,
+    raw_topics,
+    raw_users,
+    seed_db_async,
+    seed_db_sync,
+    seed_insert_statements,
+    session,
+    sqlite_engine,
+    sync_mutation,
+    sync_query,
+)
+
+from .fixtures import fx_strawchemy_mysql, fx_strawchemy_postgresql, fx_strawchemy_sqlite
+
+__all__ = (
+    "aiosqlite_engine",
+    "any_query",
+    "any_session",
+    "async_engine",
+    "async_mutation",
+    "async_query",
+    "async_session",
+    "asyncmy_engine",
+    "asyncpg_engine",
+    "before_create_all_statements",
+    "config",
+    "db_features",
+    "dialect",
+    "engine",
+    "fx_metadata",
+    "fx_strawchemy_mysql",
+    "fx_strawchemy_postgresql",
+    "fx_strawchemy_sqlite",
+    "mapper",
+    "no_session_query",
+    "postgis_service",
+    "postgres_database_service",
+    "psycopg_async_engine",
+    "psycopg_engine",
+    "query_tracker",
+    "raw_arrays",
+    "raw_colors",
+    "raw_date_times",
+    "raw_farms",
+    "raw_fruits",
+    "raw_geo",
+    "raw_geo_flipped",
+    "raw_groups",
+    "raw_intervals",
+    "raw_json",
+    "raw_topics",
+    "raw_users",
+    "seed_db_async",
+    "seed_db_sync",
+    "seed_insert_statements",
+    "session",
+    "sqlite_engine",
+    "sync_mutation",
+    "sync_query",
+)

tests/benchmarks/fixtures.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import pytest
+from strawchemy import Strawchemy
+
+
+@pytest.fixture(name="strawchemy_postgresql")
+def fx_strawchemy_postgresql() -> Strawchemy:
+    return Strawchemy("postgresql")
+
+
+@pytest.fixture(name="strawchemy_sqlite")
+def fx_strawchemy_sqlite() -> Strawchemy:
+    return Strawchemy("sqlite")
+
+
+@pytest.fixture(name="strawchemy_mysql")
+def fx_strawchemy_mysql() -> Strawchemy:
+    return Strawchemy("mysql")

tests/benchmarks/test_build_schema.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from pytest_lazy_fixtures import lf
+from strawchemy import StrawchemySyncRepository
+
+from tests.integration.models import User
+
+if TYPE_CHECKING:
+    from pytest_codspeed import BenchmarkFixture
+    from strawchemy import Strawchemy
+
+
+@pytest.mark.benchmark(group="schema")
+@pytest.mark.parametrize("strawchemy", [lf("strawchemy_postgresql"), lf("strawchemy_sqlite"), lf("strawchemy_mysql")])
+def test_build_user_type(strawchemy: Strawchemy) -> None:
+    strawchemy.type(User, include="all", override=True)(type("UserType", (), {}))
+
+
+@pytest.mark.benchmark(group="schema")
+@pytest.mark.parametrize("strawchemy", [lf("strawchemy_postgresql"), lf("strawchemy_sqlite"), lf("strawchemy_mysql")])
+def test_build_user_filter(strawchemy: Strawchemy) -> None:
+    strawchemy.filter(User, include="all", override=True)(type("UserFilter", (), {}))
+
+
+@pytest.mark.benchmark(group="schema")
+@pytest.mark.parametrize("strawchemy", [lf("strawchemy_postgresql"), lf("strawchemy_sqlite"), lf("strawchemy_mysql")])
+def test_build_user_create_input(strawchemy: Strawchemy) -> None:
+    strawchemy.create_input(User, include="all", override=True)(type("UserFilter", (), {}))
+
+
+@pytest.mark.benchmark(group="schema")
+@pytest.mark.parametrize("strawchemy", [lf("strawchemy_postgresql"), lf("strawchemy_sqlite"), lf("strawchemy_mysql")])
+def test_build_user_pk_update_input(strawchemy: Strawchemy) -> None:
+    strawchemy.create_input(User, include="all", override=True)(type("UserFilter", (), {}))
+
+
+@pytest.mark.benchmark(group="schema")
+@pytest.mark.parametrize("strawchemy", [lf("strawchemy_postgresql"), lf("strawchemy_sqlite"), lf("strawchemy_mysql")])
+def test_build_user_query_field(strawchemy: Strawchemy, benchmark: BenchmarkFixture) -> None:
+    @strawchemy.filter(User, include="all", override=True)
+    class UserFilter: ...
+
+    @strawchemy.order(User, include="all", override=True)
+    class UserOrderBy: ...
+
+    @strawchemy.distinct_on(User, include="all", override=True)
+    class UserDistinctOn: ...
+
+    benchmark(
+        lambda: strawchemy.field(
+            filter_input=UserFilter,
+            order_by=UserOrderBy,
+            distinct_on=UserDistinctOn,
+            repository_type=StrawchemySyncRepository,
+        )
+    )

tests/benchmarks/test_queries.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import pytest
+
+from tests.integration.typing import RawRecordData
+from tests.typing import AnyQueryExecutor
+from tests.utils import maybe_async
+
+
+@pytest.mark.benchmark(group="queries")
+async def test_single(any_query: AnyQueryExecutor, raw_users: RawRecordData) -> None:
+    await maybe_async(
+        any_query(
+            """
+            query GetUser($id: Int!) {
+                user(id: $id) {
+                    name
+            }
+            }
+            """,
+            {"id": raw_users[0]["id"]},
+        )
+    )