feat(compiler): incremental compile cache & warm hot-reload daemon (REFLEX_COMPILE_CACHE)

packages/reflex-base/src/reflex_base/environment.py

@@ -611,6 +611,18 @@ class EnvironmentVariables:
     # If this env var is set to "yes", App.compile will be a no-op
     REFLEX_SKIP_COMPILE: EnvVar[bool] = env_var(False, internal=True)
 
+    # Experimental: incremental compile cache. A fresh compile process (e.g. a
+    # reflex-run hot-reload worker) reuses each page's compiled output from an
+    # on-disk manifest and recompiles only the pages whose source changed,
+    # backed by an in-process per-page cache for repeat compiles in one process.
+    # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py.
+    REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False)
+
+    # When the compile cache reuses pages, also run a full compile and assert
+    # byte-identical output, falling back to the full result on any mismatch.
+    # Doubles compile time; for validating the cache on an app.
+    REFLEX_COMPILE_CACHE_VERIFY: EnvVar[bool] = env_var(False)
+
     # Inherited by uvicorn/granian reload workers so the backend can distinguish
     # dev reload-capable worker boots from other backend starts. Never set in prod.
     REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True)

packages/reflex-base/src/reflex_base/plugins/compiler.py

@@ -6,6 +6,7 @@
 import dataclasses
 import inspect
 from collections.abc import Callable, Sequence
+from contextlib import AbstractContextManager
 from contextvars import ContextVar, Token
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, TypeVar, cast
@@ -35,6 +36,15 @@
 _BaseComponentT = TypeVar("_BaseComponentT", bound=BaseComponent)
 
 
+#: Optional per-page source-read recorder, installed by an incremental compile
+#: cache. When set, :meth:`CompileContext.compile` wraps each page's evaluation
+#: in ``page_source_recorder()``; the context manager yields a set that collects
+#: the source files (e.g. markdown/data) read during that page's eval, which is
+#: then stored on the page's ``source_files``. ``None`` (default) disables
+#: recording, so the compile path is unchanged when no cache is active.
+page_source_recorder: Callable[[], AbstractContextManager[set[str]]] | None = None
+
+
 class PageDefinition(Protocol):
     """Protocol for page-like objects compiled by :class:`CompileContext`."""
 
@@ -690,6 +700,18 @@ class PageContext(BaseContext):
     output_path: str | None = None
     output_code: str | None = None
     source_module: str | None = None
+    # Source files (e.g. markdown/data) read while evaluating this page, when a
+    # per-page read recorder is installed (see ``page_source_recorder``). Lets an
+    # incremental cache depend on the exact non-import inputs a page consumed, so
+    # editing one page's data invalidates only that page. Empty when no recorder
+    # is active.
+    source_files: set[str] = dataclasses.field(default_factory=set)
+    # Auto-memo components first registered while compiling THIS page, keyed by
+    # ``(tag, source_module)``. Lets an incremental cache attribute memo
+    # contributions per page so a skipped page can re-register them.
+    memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field(
+        default_factory=dict
+    )
     # Stack of ``id(component)`` for components whose subtree is
     # memoize-suppressed. Populated by ``MemoizeStatefulPlugin`` when it
     # encounters a ``MemoizationLeaf``-style snapshot boundary and popped on
@@ -794,6 +816,7 @@ def compile(
         """
         from reflex.compiler import compiler
         from reflex.state import all_base_state_classes
+        from reflex_base.vars.base import reset_unique_variable_names
 
         self.ensure_context_attached()
         self.compiled_pages.clear()
@@ -803,15 +826,32 @@ def compile(
         self.memoize_wrappers.clear()
         self.auto_memo_components.clear()
 
+        # Reset the deterministic ref-name generator so a second in-process
+        # compile reproduces the same auto-generated names as the first (these
+        # names feed auto-memo content hashes, so drift breaks reproducibility).
+        reset_unique_variable_names()
+
+        recorder = page_source_recorder
         for page in self.pages:
             page_fn = page.component
             n_states_before = len(all_base_state_classes)
-            page_ctx = self.hooks.eval_page(
-                page_fn,
-                page=page,
-                compile_context=self,
-                **kwargs,
-            )
+            if recorder is not None:
+                with recorder() as read_set:
+                    page_ctx = self.hooks.eval_page(
+                        page_fn,
+                        page=page,
+                        compile_context=self,
+                        **kwargs,
+                    )
+                if page_ctx is not None:
+                    page_ctx.source_files = read_set
+            else:
+                page_ctx = self.hooks.eval_page(
+                    page_fn,
+                    page=page,
+                    compile_context=self,
+                    **kwargs,
+                )
             if page_ctx is None:
                 page_name = getattr(page_fn, "__name__", repr(page_fn))
                 msg = (
@@ -836,6 +876,7 @@ def compile(
             self.compiled_pages.values(),
             strict=True,
         ):
+            memo_before = set(self.auto_memo_components)
             with page_ctx:
                 page_ctx.root_component = self.hooks.compile_component(
                     page_ctx.root_component,
@@ -848,6 +889,12 @@ def compile(
                     compile_context=self,
                     **kwargs,
                 )
+            # Attribute newly-registered auto-memo components to this page.
+            page_ctx.memo_contributions = {
+                key: value
+                for key, value in self.auto_memo_components.items()
+                if key not in memo_before
+            }
 
             page_ctx.frontend_imports = page_ctx.merged_imports(collapse=True)
             self.all_imports = merge_imports(

packages/reflex-base/src/reflex_base/vars/base.py

@@ -9,6 +9,7 @@
 import functools
 import inspect
 import json
+import random
 import re
 import string
 import uuid
@@ -44,7 +45,6 @@
 from reflex_base.constants.state import FIELD_MARKER
 from reflex_base.utils import console, exceptions, imports, serializers, types
 from reflex_base.utils.compat import annotations_from_namespace
-from reflex_base.utils.decorator import once
 from reflex_base.utils.exceptions import (
     ComputedVarSignatureError,
     UntypedComputedVarError,
@@ -3211,12 +3211,20 @@ def get_uuid_string_var() -> Var:
 # Set of unique variable names.
 USED_VARIABLES = set()
 
+_UNIQUE_NAME_RNG = random.Random(42)
 
-@once
-def _rng():
-    import random
 
-    return random.Random(42)
+def reset_unique_variable_names() -> None:
+    """Reset the deterministic unique-name generator to its initial state.
+
+    ``get_unique_variable_name`` draws from a seeded RNG and dedups against
+    ``USED_VARIABLES``; both persist process-wide, so a second in-process compile
+    would draw *different* ref names than the first (the RNG state and the used
+    set carry over). Resetting them before each compile makes the generated names
+    reproducible across compiles — names only need to be unique within a compile.
+    """
+    USED_VARIABLES.clear()
+    _UNIQUE_NAME_RNG.seed(42)
 
 
 def get_unique_variable_name() -> str:
@@ -3225,7 +3233,7 @@ def get_unique_variable_name() -> str:
     Returns:
         The unique variable name.
     """
-    name = "".join([_rng().choice(string.ascii_lowercase) for _ in range(8)])
+    name = "".join([_UNIQUE_NAME_RNG.choice(string.ascii_lowercase) for _ in range(8)])
     if name not in USED_VARIABLES:
         USED_VARIABLES.add(name)
         return name