Migration to juliacall

.github/workflows/ci.yml

@@ -174,11 +174,6 @@ jobs:
       matrix:
         python-version: ['3.12', '3.13']
 
-    # needed to allow julia-actions/cache to delete old caches that it has created
-    permissions:
-      actions: write
-      contents: read
-
     steps:
     - name: Check out repository
       uses: actions/checkout@v7
@@ -196,19 +191,9 @@ jobs:
           .tox/
         key: "${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-ci-${{ github.job }}"
 
-    - name: Install julia
-      uses: julia-actions/setup-julia@v3
-      with:
-        version: 1.11
-
     - name: Install dependencies
       run: .github/workflows/install_deps.sh
 
-    - name: Install PEtabJL dependencies
-      run: >
-        julia -e 'using Pkg; Pkg.add("PEtab");
-        Pkg.add("OrdinaryDiffEq"); Pkg.add("Sundials")'
-
     - name: Run tests
       timeout-minutes: 25
       run: tox -e julia

pypesto/objective/julia/base.py

@@ -1,7 +1,14 @@
-"""Interface to Julia via pyjulia."""
+"""Interface to Julia via juliacall."""
 
 from collections.abc import Callable
 
+# Import juliacall early to avoid conflicts with other libraries (especially numpy)
+# See: https://juliapy.github.io/PythonCall.jl/dev/faq/
+try:
+    from juliacall import Main as jl  # noqa: F401
+except ImportError:
+    jl = None
+
 import numpy as np
 
 from ..function import Objective
@@ -27,10 +34,10 @@ def _read_source(module_name: str, source_file: str) -> None:
     module_name: Julia module name.
     source_file: Qualified Julia source file.
     """
-    from julia import Main
+    from juliacall import Main as jl
 
-    if not hasattr(Main, module_name):
-        Main.include(source_file)
+    if not hasattr(jl, module_name):
+        jl.include(source_file)
 
 
 class JuliaObjective(Objective):
@@ -40,45 +47,18 @@ class JuliaObjective(Objective):
     It expects the corresponding Julia objects to be defined in a
     `source_file` within a `module`.
 
-    We use the PyJulia package to access Julia from inside Python.
-    It can be installed via `pip install pypesto[julia]`, however requires
-    additional Julia dependencies to be installed via:
+    We use the juliacall package (part of PythonCall.jl) to access Julia
+    from inside Python. It can be installed via `pip install pypesto[julia]`.
 
-    >>> python -c "import julia; julia.install()"
+    juliacall automatically manages the Julia installation and configuration,
+    so no additional setup steps are required beyond pip installation.
 
     For further information, see
-    https://pyjulia.readthedocs.io/en/latest/installation.html.
-
-    There are some known problems, e.g. with statically linked Python
-    interpreters, see
-    https://pyjulia.readthedocs.io/en/latest/troubleshooting.html
-    for details.
-    Possible solutions are to pass ``compiled_modules=False`` to the Julia
-    constructor early in your code:
-
-    >>> from julia.api import Julia
-    >>> jl = Julia(compiled_modules=False)
-
-    This however slows down loading and using Julia packages, especially for
-    large ones.
-    An alternative is to use the ``python-jl`` command shipped with PyJulia:
-
-    >>> python-jl MY_SCRIPT.py
-
-    This basically launches a Python interpreter inside Julia.
-    When using Jupyter notebooks, this wrapper can be installed as an
-    additional kernel via:
-
-    >>> python -m ipykernel install --name python-jl [--prefix=/path/to/python/env]
-
-    And changing the first argument in
-    ``/path/to/python/env/share/jupyter/kernels/python-jl/kernel.json``
-    to ``python-jl``.
+    https://juliapy.github.io/PythonCall.jl/stable/juliacall/
 
-    Model simulations are eagerly converted to Python objects
-    (specifically, `numpy.ndarray` and `pandas.DataFrame`).
-    This can introduce overhead and could be avoided by an alternative
-    lazy implementation.
+    Model simulations are efficiently handled with minimal overhead.
+    By default, juliacall wraps mutable objects instead of copying them,
+    providing better performance than PyJulia.
 
     Parameters
     ----------
@@ -103,10 +83,10 @@ def __init__(
     ):
         # lazy imports
         try:
-            from julia import Main  # noqa: F401
+            from juliacall import Main as jl  # noqa: F401
         except ImportError:
             raise ImportError(
-                "Install PyJulia, e.g. via `pip install pypesto[julia]`, "
+                "Install juliacall, e.g. via `pip install pypesto[julia]`, "
                 "and see the class documentation",
             ) from None
 
@@ -133,10 +113,10 @@ def get(self, name: str, as_array: bool = False) -> Callable | None:
 
         Use this function to access any variable from the Julia module.
         """
-        from julia import Main
+        from juliacall import Main as jl
 
         if name is not None:
-            ret = getattr(getattr(Main, self.module), name, None)
+            ret = getattr(getattr(jl, self.module), name, None)
             if as_array:
                 ret = _as_array(ret)
             return ret

pypesto/objective/julia/petabJl.py

@@ -3,6 +3,13 @@
 import logging
 import os
 
+# Import juliacall early to avoid conflicts with other libraries (especially numpy)
+# See: https://juliapy.github.io/PythonCall.jl/dev/faq/
+try:
+    from juliacall import Main as jl  # noqa: F401
+except ImportError:
+    jl = None
+
 import numpy as np
 
 from .base import JuliaObjective, _read_source
@@ -37,12 +44,24 @@ def __init__(
         """Initialize objective."""
         # lazy imports
         try:
-            from julia import Main, Pkg  # noqa: F401
+            from juliacall import Main as jl  # noqa: F401
+
+            # Load Pkg into Julia session
+            jl.seval("using Pkg")
+            jl.Pkg.activate(".")
 
-            Pkg.activate(".")
+            # Install required packages if not already available
+            # This ensures packages are available even when precompile=False
+            try:
+                jl.seval("using OrdinaryDiffEq")
+            except Exception:
+                logger.info("Installing required Julia packages...")
+                jl.Pkg.add("OrdinaryDiffEq")
+                jl.Pkg.add("PEtab")
+                jl.Pkg.add("Sundials")
         except ImportError:
             raise ImportError(
-                "Install PyJulia, e.g. via `pip install pypesto[julia]`, "
+                "Install juliacall, e.g. via `pip install pypesto[julia]`, "
                 "and see the class documentation",
             ) from None
 
@@ -61,6 +80,12 @@ def __init__(
         petab_jl_problem = self.get(petab_problem_name)
         self.petab_jl_problem = petab_jl_problem
 
+        if petab_jl_problem is None:
+            raise ValueError(
+                f"Could not find petab problem '{petab_problem_name}' in module '{module}'. "
+                f"Make sure the Julia module defines this variable."
+            )
+
         # get functions
         fun = self.petab_jl_problem.nllh
         grad = self.petab_jl_problem.grad
@@ -87,15 +112,24 @@ def __setstate__(self, state):
             setattr(self, key, value)
         # lazy imports
         try:
-            from julia import (
-                Main,  # noqa: F401
-                Pkg,
-            )
+            from juliacall import Main as jl  # noqa: F401
+
+            # Load Pkg into Julia session
+            jl.seval("using Pkg")
+            jl.Pkg.activate(".")
 
-            Pkg.activate(".")
+            # Install required packages if not already available
+            # This ensures packages are available even when precompile=False
+            try:
+                jl.seval("using OrdinaryDiffEq")
+            except Exception:
+                logger.info("Installing required Julia packages...")
+                jl.Pkg.add("OrdinaryDiffEq")
+                jl.Pkg.add("PEtab")
+                jl.Pkg.add("Sundials")
         except ImportError:
             raise ImportError(
-                "Install PyJulia, e.g. via `pip install pypesto[julia]`, "
+                "Install juliacall, e.g. via `pip install pypesto[julia]`, "
                 "and see the class documentation",
             ) from None
         # Include module if not already included
@@ -104,6 +138,12 @@ def __setstate__(self, state):
         petab_jl_problem = self.get(self._petab_problem_name)
         self.petab_jl_problem = petab_jl_problem
 
+        if petab_jl_problem is None:
+            raise ValueError(
+                f"Could not find petab problem '{self._petab_problem_name}' in module '{self.module}'. "
+                f"Make sure the Julia module defines this variable."
+            )
+
         # get functions
         fun = self.petab_jl_problem.nllh
         grad = self.petab_jl_problem.grad
@@ -138,19 +178,20 @@ def precompile_model(self, force_compile: bool = False):
             return None
         # lazy imports
         try:
-            from julia import Main  # noqa: F401
+            from juliacall import Main as jl  # noqa: F401
         except ImportError:
             raise ImportError(
-                "Install PyJulia, e.g. via `pip install pypesto[julia]`, "
+                "Install juliacall, e.g. via `pip install pypesto[julia]`, "
                 "and see the class documentation",
             ) from None
         # setting up a local project, where the precompilation will be done in
-        from julia import Pkg
 
-        Pkg.activate(".")
+        # Load Pkg into Julia session
+        jl.seval("using Pkg")
+        jl.Pkg.activate(".")
         # create a Project f"{self.module}_pre".
         try:
-            Pkg.generate(f"{directory}/{self.module}_pre")
+            jl.Pkg.generate(f"{directory}/{self.module}_pre")
         except Exception:
             logger.info("Module is already generated. Skipping generate...")
         # Adjust the precompilation file
@@ -169,16 +210,16 @@ def precompile_model(self, force_compile: bool = False):
                 os.rename("dummy_temp_file.jl", self.source_file)
 
         try:
-            Pkg.develop(path=f"{directory}/{self.module}_pre")
+            jl.Pkg.develop(path=f"{directory}/{self.module}_pre")
         except Exception:
             logger.info("Module is already developed. Skipping develop...")
-        Pkg.activate(f"{directory}/{self.module}_pre/")
+        jl.Pkg.activate(f"{directory}/{self.module}_pre/")
         # add dependencies
-        Pkg.add("PrecompileTools")
-        Pkg.add("OrdinaryDiffEq")
-        Pkg.add("PEtab")
-        Pkg.add("Sundials")
-        Pkg.precompile()
+        jl.Pkg.add("PrecompileTools")
+        jl.Pkg.add("OrdinaryDiffEq")
+        jl.Pkg.add("PEtab")
+        jl.Pkg.add("Sundials")
+        jl.Pkg.precompile()
 
 
 def write_precompilation_module(module, source_file_orig):

pypesto/objective/julia/petab_jl_importer.py

@@ -6,6 +6,13 @@
 import os.path
 from collections.abc import Iterable
 
+# Import juliacall early to avoid conflicts with other libraries (especially numpy)
+# See: https://juliapy.github.io/PythonCall.jl/dev/faq/
+try:
+    from juliacall import Main as jl  # noqa: F401
+except ImportError:
+    jl = None
+
 import numpy as np
 
 from pypesto.objective.julia import PEtabJlObjective
@@ -116,10 +123,10 @@ def create_objective(
         """
         # lazy imports
         try:
-            from julia import Main  # noqa: F401
+            from juliacall import Main as jl  # noqa: F401
         except ImportError:
             raise ImportError(
-                "Install PyJulia, e.g. via `pip install pypesto[julia]`, "
+                "Install juliacall, e.g. via `pip install pypesto[julia]`, "
                 "and see the class documentation",
             ) from None
         if self.source_file is None:

pyproject.toml

@@ -131,7 +131,7 @@ mltools = [
 ]
 
 julia = [
-  "julia>=0.5.7",
+  "juliacall>=0.9.31",
   "ipython>=8.4.0",
   "pygments>=2.12.0",
 ]

test/julia/test_pyjulia.py

@@ -1,5 +1,4 @@
 import os
-import unittest
 
 import numpy as np
 
@@ -8,8 +7,6 @@
 from pypesto.objective.julia import JuliaObjective, display_source_ipython
 from pypesto.objective.julia.petab_jl_importer import PetabJlImporter
 
-# The pyjulia wrapper appears to ignore global noqas, thus per line here
-
 
 def test_pyjulia_pipeline():
     """Test that a pipeline with julia objective works."""
@@ -67,9 +64,6 @@ def test_pyjulia_pipeline():
     )
 
 
-# TODO: REACTIVATE JULIA TESTS - These tests have been temporarily paused
-#  and need to be reactivated.
-@unittest.skip("Julia tests are temporarily disabled.")
 def test_petabJL_interface():
     """Test the interface to PEtab.jl with provided solutions from julia."""
     model_name = "boehm_JProteomeRes2014"
@@ -117,7 +111,6 @@ def test_petabJL_interface():
     assert np.allclose(hess, hess_ref)  # noqa: S101
 
 
-@unittest.skip("Julia tests are temporarily disabled.")
 def test_petabJL_from_module():
     """Test that PEtab.jl is integrated properly."""
     # create objective
@@ -139,7 +132,6 @@ def test_petabJL_from_module():
     )
 
 
-@unittest.skip("Julia tests are temporarily disabled.")
 def test_petabJL_from_yaml():
     """Test that PEtab.jl from yaml file is running smoothly."""
     yaml_file = (

tox.ini

@@ -106,8 +106,7 @@ description =
 [testenv:julia]
 extras = test,julia
 commands =
-    python -c "import julia; julia.install()"
-    python-jl -m pytest --cov=pypesto --cov-report=xml --cov-append \
+    pytest --cov=pypesto --cov-report=xml --cov-append \
         test/julia
 description =
     Test Julia interface