diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3960c0d6..bf68cb46c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,11 +174,6 @@ jobs: matrix: python-version: ['3.12', '3.13'] - # needed to allow julia-actions/cache to delete old caches that it has created - permissions: - actions: write - contents: read - steps: - name: Check out repository uses: actions/checkout@v7 @@ -196,19 +191,9 @@ jobs: .tox/ key: "${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-ci-${{ github.job }}" - - name: Install julia - uses: julia-actions/setup-julia@v3 - with: - version: 1.11 - - name: Install dependencies run: .github/workflows/install_deps.sh - - name: Install PEtabJL dependencies - run: > - julia -e 'using Pkg; Pkg.add("PEtab"); - Pkg.add("OrdinaryDiffEq"); Pkg.add("Sundials")' - - name: Run tests timeout-minutes: 25 run: tox -e julia diff --git a/pypesto/objective/julia/base.py b/pypesto/objective/julia/base.py index 6e6fd75d9..ac461cd16 100644 --- a/pypesto/objective/julia/base.py +++ b/pypesto/objective/julia/base.py @@ -1,7 +1,14 @@ -"""Interface to Julia via pyjulia.""" +"""Interface to Julia via juliacall.""" from collections.abc import Callable +# Import juliacall early to avoid conflicts with other libraries (especially numpy) +# See: https://juliapy.github.io/PythonCall.jl/dev/faq/ +try: + from juliacall import Main as jl # noqa: F401 +except ImportError: + jl = None + import numpy as np from ..function import Objective @@ -27,10 +34,10 @@ def _read_source(module_name: str, source_file: str) -> None: module_name: Julia module name. source_file: Qualified Julia source file. """ - from julia import Main + from juliacall import Main as jl - if not hasattr(Main, module_name): - Main.include(source_file) + if not hasattr(jl, module_name): + jl.include(source_file) class JuliaObjective(Objective): @@ -40,45 +47,18 @@ class JuliaObjective(Objective): It expects the corresponding Julia objects to be defined in a `source_file` within a `module`. - We use the PyJulia package to access Julia from inside Python. - It can be installed via `pip install pypesto[julia]`, however requires - additional Julia dependencies to be installed via: + We use the juliacall package (part of PythonCall.jl) to access Julia + from inside Python. It can be installed via `pip install pypesto[julia]`. - >>> python -c "import julia; julia.install()" + juliacall automatically manages the Julia installation and configuration, + so no additional setup steps are required beyond pip installation. For further information, see - https://pyjulia.readthedocs.io/en/latest/installation.html. - - There are some known problems, e.g. with statically linked Python - interpreters, see - https://pyjulia.readthedocs.io/en/latest/troubleshooting.html - for details. - Possible solutions are to pass ``compiled_modules=False`` to the Julia - constructor early in your code: - - >>> from julia.api import Julia - >>> jl = Julia(compiled_modules=False) - - This however slows down loading and using Julia packages, especially for - large ones. - An alternative is to use the ``python-jl`` command shipped with PyJulia: - - >>> python-jl MY_SCRIPT.py - - This basically launches a Python interpreter inside Julia. - When using Jupyter notebooks, this wrapper can be installed as an - additional kernel via: - - >>> python -m ipykernel install --name python-jl [--prefix=/path/to/python/env] - - And changing the first argument in - ``/path/to/python/env/share/jupyter/kernels/python-jl/kernel.json`` - to ``python-jl``. + https://juliapy.github.io/PythonCall.jl/stable/juliacall/ - Model simulations are eagerly converted to Python objects - (specifically, `numpy.ndarray` and `pandas.DataFrame`). - This can introduce overhead and could be avoided by an alternative - lazy implementation. + Model simulations are efficiently handled with minimal overhead. + By default, juliacall wraps mutable objects instead of copying them, + providing better performance than PyJulia. Parameters ---------- @@ -103,10 +83,10 @@ def __init__( ): # lazy imports try: - from julia import Main # noqa: F401 + from juliacall import Main as jl # noqa: F401 except ImportError: raise ImportError( - "Install PyJulia, e.g. via `pip install pypesto[julia]`, " + "Install juliacall, e.g. via `pip install pypesto[julia]`, " "and see the class documentation", ) from None @@ -133,10 +113,10 @@ def get(self, name: str, as_array: bool = False) -> Callable | None: Use this function to access any variable from the Julia module. """ - from julia import Main + from juliacall import Main as jl if name is not None: - ret = getattr(getattr(Main, self.module), name, None) + ret = getattr(getattr(jl, self.module), name, None) if as_array: ret = _as_array(ret) return ret diff --git a/pypesto/objective/julia/petabJl.py b/pypesto/objective/julia/petabJl.py index f9ddefb22..ebb24908c 100644 --- a/pypesto/objective/julia/petabJl.py +++ b/pypesto/objective/julia/petabJl.py @@ -3,6 +3,13 @@ import logging import os +# Import juliacall early to avoid conflicts with other libraries (especially numpy) +# See: https://juliapy.github.io/PythonCall.jl/dev/faq/ +try: + from juliacall import Main as jl # noqa: F401 +except ImportError: + jl = None + import numpy as np from .base import JuliaObjective, _read_source @@ -37,12 +44,24 @@ def __init__( """Initialize objective.""" # lazy imports try: - from julia import Main, Pkg # noqa: F401 + from juliacall import Main as jl # noqa: F401 + + # Load Pkg into Julia session + jl.seval("using Pkg") + jl.Pkg.activate(".") - Pkg.activate(".") + # Install required packages if not already available + # This ensures packages are available even when precompile=False + try: + jl.seval("using OrdinaryDiffEq") + except Exception: + logger.info("Installing required Julia packages...") + jl.Pkg.add("OrdinaryDiffEq") + jl.Pkg.add("PEtab") + jl.Pkg.add("Sundials") except ImportError: raise ImportError( - "Install PyJulia, e.g. via `pip install pypesto[julia]`, " + "Install juliacall, e.g. via `pip install pypesto[julia]`, " "and see the class documentation", ) from None @@ -61,6 +80,12 @@ def __init__( petab_jl_problem = self.get(petab_problem_name) self.petab_jl_problem = petab_jl_problem + if petab_jl_problem is None: + raise ValueError( + f"Could not find petab problem '{petab_problem_name}' in module '{module}'. " + f"Make sure the Julia module defines this variable." + ) + # get functions fun = self.petab_jl_problem.nllh grad = self.petab_jl_problem.grad @@ -87,15 +112,24 @@ def __setstate__(self, state): setattr(self, key, value) # lazy imports try: - from julia import ( - Main, # noqa: F401 - Pkg, - ) + from juliacall import Main as jl # noqa: F401 + + # Load Pkg into Julia session + jl.seval("using Pkg") + jl.Pkg.activate(".") - Pkg.activate(".") + # Install required packages if not already available + # This ensures packages are available even when precompile=False + try: + jl.seval("using OrdinaryDiffEq") + except Exception: + logger.info("Installing required Julia packages...") + jl.Pkg.add("OrdinaryDiffEq") + jl.Pkg.add("PEtab") + jl.Pkg.add("Sundials") except ImportError: raise ImportError( - "Install PyJulia, e.g. via `pip install pypesto[julia]`, " + "Install juliacall, e.g. via `pip install pypesto[julia]`, " "and see the class documentation", ) from None # Include module if not already included @@ -104,6 +138,12 @@ def __setstate__(self, state): petab_jl_problem = self.get(self._petab_problem_name) self.petab_jl_problem = petab_jl_problem + if petab_jl_problem is None: + raise ValueError( + f"Could not find petab problem '{self._petab_problem_name}' in module '{self.module}'. " + f"Make sure the Julia module defines this variable." + ) + # get functions fun = self.petab_jl_problem.nllh grad = self.petab_jl_problem.grad @@ -138,19 +178,20 @@ def precompile_model(self, force_compile: bool = False): return None # lazy imports try: - from julia import Main # noqa: F401 + from juliacall import Main as jl # noqa: F401 except ImportError: raise ImportError( - "Install PyJulia, e.g. via `pip install pypesto[julia]`, " + "Install juliacall, e.g. via `pip install pypesto[julia]`, " "and see the class documentation", ) from None # setting up a local project, where the precompilation will be done in - from julia import Pkg - Pkg.activate(".") + # Load Pkg into Julia session + jl.seval("using Pkg") + jl.Pkg.activate(".") # create a Project f"{self.module}_pre". try: - Pkg.generate(f"{directory}/{self.module}_pre") + jl.Pkg.generate(f"{directory}/{self.module}_pre") except Exception: logger.info("Module is already generated. Skipping generate...") # Adjust the precompilation file @@ -169,16 +210,16 @@ def precompile_model(self, force_compile: bool = False): os.rename("dummy_temp_file.jl", self.source_file) try: - Pkg.develop(path=f"{directory}/{self.module}_pre") + jl.Pkg.develop(path=f"{directory}/{self.module}_pre") except Exception: logger.info("Module is already developed. Skipping develop...") - Pkg.activate(f"{directory}/{self.module}_pre/") + jl.Pkg.activate(f"{directory}/{self.module}_pre/") # add dependencies - Pkg.add("PrecompileTools") - Pkg.add("OrdinaryDiffEq") - Pkg.add("PEtab") - Pkg.add("Sundials") - Pkg.precompile() + jl.Pkg.add("PrecompileTools") + jl.Pkg.add("OrdinaryDiffEq") + jl.Pkg.add("PEtab") + jl.Pkg.add("Sundials") + jl.Pkg.precompile() def write_precompilation_module(module, source_file_orig): diff --git a/pypesto/objective/julia/petab_jl_importer.py b/pypesto/objective/julia/petab_jl_importer.py index 1a81ef450..7f17c5582 100644 --- a/pypesto/objective/julia/petab_jl_importer.py +++ b/pypesto/objective/julia/petab_jl_importer.py @@ -6,6 +6,13 @@ import os.path from collections.abc import Iterable +# Import juliacall early to avoid conflicts with other libraries (especially numpy) +# See: https://juliapy.github.io/PythonCall.jl/dev/faq/ +try: + from juliacall import Main as jl # noqa: F401 +except ImportError: + jl = None + import numpy as np from pypesto.objective.julia import PEtabJlObjective @@ -116,10 +123,10 @@ def create_objective( """ # lazy imports try: - from julia import Main # noqa: F401 + from juliacall import Main as jl # noqa: F401 except ImportError: raise ImportError( - "Install PyJulia, e.g. via `pip install pypesto[julia]`, " + "Install juliacall, e.g. via `pip install pypesto[julia]`, " "and see the class documentation", ) from None if self.source_file is None: diff --git a/pyproject.toml b/pyproject.toml index 86182e3de..7b84f5576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,7 +131,7 @@ mltools = [ ] julia = [ - "julia>=0.5.7", + "juliacall>=0.9.31", "ipython>=8.4.0", "pygments>=2.12.0", ] diff --git a/test/julia/test_pyjulia.py b/test/julia/test_pyjulia.py index 71b0ded48..20ea9d251 100644 --- a/test/julia/test_pyjulia.py +++ b/test/julia/test_pyjulia.py @@ -1,5 +1,4 @@ import os -import unittest import numpy as np @@ -8,8 +7,6 @@ from pypesto.objective.julia import JuliaObjective, display_source_ipython from pypesto.objective.julia.petab_jl_importer import PetabJlImporter -# The pyjulia wrapper appears to ignore global noqas, thus per line here - def test_pyjulia_pipeline(): """Test that a pipeline with julia objective works.""" @@ -67,9 +64,6 @@ def test_pyjulia_pipeline(): ) -# TODO: REACTIVATE JULIA TESTS - These tests have been temporarily paused -# and need to be reactivated. -@unittest.skip("Julia tests are temporarily disabled.") def test_petabJL_interface(): """Test the interface to PEtab.jl with provided solutions from julia.""" model_name = "boehm_JProteomeRes2014" @@ -117,7 +111,6 @@ def test_petabJL_interface(): assert np.allclose(hess, hess_ref) # noqa: S101 -@unittest.skip("Julia tests are temporarily disabled.") def test_petabJL_from_module(): """Test that PEtab.jl is integrated properly.""" # create objective @@ -139,7 +132,6 @@ def test_petabJL_from_module(): ) -@unittest.skip("Julia tests are temporarily disabled.") def test_petabJL_from_yaml(): """Test that PEtab.jl from yaml file is running smoothly.""" yaml_file = ( diff --git a/tox.ini b/tox.ini index 8942f8cf4..2a4ff983d 100644 --- a/tox.ini +++ b/tox.ini @@ -106,8 +106,7 @@ description = [testenv:julia] extras = test,julia commands = - python -c "import julia; julia.install()" - python-jl -m pytest --cov=pypesto --cov-report=xml --cov-append \ + pytest --cov=pypesto --cov-report=xml --cov-append \ test/julia description = Test Julia interface