C++ optimization and abi3 wheel support for portable binaries

.github/workflows/python-package.yml

@@ -5,9 +5,9 @@ name: tests
 
 on:
   push:
-    branches: [ "*" ]
+    branches: [ "main" ]
   pull_request:
-    branches: [ "*" ]
+    branches: [ "main" ]
 
 jobs:
   prepare-cache:
@@ -32,7 +32,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.13', '3.14.0-beta.4', 'pypy-3.9', 'pypy-3.11']
+        # Test across Python versions to verify abi3 wheel compatibility
+        # CPython: minimum (3.9), intermediate versions, latest stable
+        # PyPy: versions we build wheels for
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy-3.9', 'pypy-3.10']
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/wheels.yml

@@ -49,14 +49,17 @@ jobs:
         run: python -m cibuildwheel --output-dir wheelhouse
         # Options (https://cibuildwheel.readthedocs.io/en/stable/options/)
         env:
-          CIBW_SKIP: cp36-* cp37-* cp38-* pp37-* pp38-* *musllinux* *win_pp* 
+          # Build abi3 wheel for CPython 3.9+ (one wheel for all versions)
+          # Build version-specific wheels for PyPy (which doesn't support abi3)
+          CIBW_BUILD: cp39-* pp39-* pp310-*
+          CIBW_SKIP: "*musllinux* *win_pp*"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9"
           CIBW_BEFORE_BUILD_MACOS: python3 -m pip install --upgrade setuptools wheel cffi
           CIBW_ARCHS_MACOS: "x86_64 arm64"
           CIBW_ARCHS_WINDOWS: "AMD64"
           CIBW_BEFORE_BUILD_WINDOWS: python -m pip install "setuptools<72"
           CIBW_ARCHS_LINUX: "x86_64"
           CIBW_BUILD_FRONTEND: "build"
-          # CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9"
 
       - uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -74,7 +74,7 @@ log/
 console/
 
 # Virtual environments
-venv
+venv*/
 p3*/
 pypy*/
 

AGENTS.md

@@ -22,6 +22,16 @@ python tools/binpack.py
 python tools/dawgbuilder.py
 ```
 
+### Rebuilding C++ Extensions
+
+**Always use `pip install -e .` to rebuild C++ extensions** (not `bin_build.py` directly):
+
+```bash
+{path-to-venv}/bin/python -m pip install -e . --no-build-isolation
+```
+
+Running `bin_build.py` directly creates `.so` files in the wrong location (`islenska/` instead of `src/islenska/`) because it doesn't respect the `src/` layout from `pyproject.toml`.
+
 ### Testing
 ```bash
 # Run all tests

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=61.0", "cffi>=1.15.1"]
+requires = ["setuptools>=61.0", "cffi>=1.15.1", "wheel>=0.38.0"]
 build-backend = "setuptools.build_meta"
 
 [project]

setup.py

@@ -4,11 +4,20 @@
 All package metadata is defined in pyproject.toml.
 """
 
+import platform
 from setuptools import setup
 
 # The cffi_modules and zip_safe settings are not yet supported in pyproject.toml
 # and must be defined here.
+
+# Use stable ABI for CPython to create portable wheels across Python versions
+# PyPy doesn't support the stable ABI, so we skip this for PyPy builds
+options = {}
+if platform.python_implementation() == "CPython":
+    options["py_limited_api"] = "cp39"  # Requires Python 3.9+
+
 setup(
     zip_safe=True,
     cffi_modules=["src/islenska/bin_build.py:ffibuilder"],
+    options={"bdist_wheel": options},
 )

src/islenska/bin_build.py

@@ -54,11 +54,30 @@
 
 declarations = """
 
+    // From bin.h
     typedef unsigned int UINT;
     typedef uint8_t BYTE;
-
     UINT mapping(const BYTE* pbMap, const BYTE* pszWordLatin);
 
+    // From dawgdictionary.h
+    typedef void* DawgHandle;
+    DawgHandle dawg_load(const BYTE* pbMap);
+    void dawg_unload(DawgHandle handle);
+    bool dawg_contains(DawgHandle handle, const char* word);
+    char* dawg_find_combinations(DawgHandle handle, const char* word);
+    void dawg_free_string(char* str);
+
+    // From bincompress.h
+    typedef void* BcHandle;
+    BcHandle bin_compressed_init(const BYTE* pbMap);
+    void bin_compressed_close(BcHandle handle);
+    bool bin_compressed_contains(BcHandle handle, const char* word);
+    char* bin_compressed_lookup(BcHandle handle, const char* word, const char* cat, const char* lemma, int utg);
+    char* bin_compressed_lookup_ksnid(BcHandle handle, const char* word, const char* cat, const char* lemma, int utg);
+    char* bin_compressed_lemma_forms(BcHandle handle, int bin_id);
+    char* bin_compressed_lookup_id(BcHandle handle, int bin_id);
+    void bin_compressed_free_string(char* str);
+
 """
 
 # Do the magic CFFI incantations necessary to get CFFI and setuptools
@@ -83,15 +102,20 @@
 
 ffibuilder.cdef(declarations)  # type: ignore
 
+# Use stable ABI for CPython to create portable wheels across Python versions.
+# PyPy doesn't support the stable ABI, so we create version-specific wheels for it.
+py_limited_api = "cp39" if IMPLEMENTATION == "CPython" else False
+
 ffibuilder.set_source(  # type: ignore
     "islenska._bin",
     # bin.cpp is written in C++ but must export a pure C interface.
     # This is the reason for the "extern 'C' { ... }" wrapper.
     'extern "C" {\n' + declarations + "\n}\n",
     source_extension=".cpp",
-    sources=["src/islenska/bin.cpp"],
+    sources=["src/islenska/bin.cpp", "src/islenska/dawgdictionary.cpp", "src/islenska/bincompress.cpp"],
     extra_compile_args=extra_compile_args,
     extra_link_args=extra_link_args,
+    py_limited_api=py_limited_api,
 )
 
 if __name__ == "__main__":