Skip to content

Commit aaec2e0

Browse files
authored
Merge pull request #490 from posit-dev/refactor/build-modules
refactor: move repository builds, the site server and build timings out of GreatDocs
2 parents af17c76 + 4bf84ad commit aaec2e0

33 files changed

Lines changed: 2454 additions & 2509 deletions

‎great_docs/_build/freeze.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from pathlib import Path
1414
from typing import TYPE_CHECKING
1515

16-
from .._content_naming import strip_numeric_prefix
16+
from .._content.naming import strip_numeric_prefix
1717
from .._subprocess import TEXT_MODE_KWARGS
1818
from .._utils import recognised_build_dirs, validate_tree_symlinks
1919
from . import quarto

‎great_docs/_build/from_repo.py‎

Lines changed: 360 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,360 @@
1+
"""
2+
Documentation builds from a Git repository in an isolated environment
3+
4+
Each build clones the repository into a temporary directory, installs great-docs and the
5+
repository's package into a temporary virtual environment, builds the site with that
6+
environment's great-docs, and copies the site out.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import os
12+
import shutil
13+
import subprocess
14+
import sys
15+
import tempfile
16+
import tomllib
17+
import venv
18+
from pathlib import Path
19+
from typing import TYPE_CHECKING, Literal
20+
21+
from yaml12 import read_yaml
22+
23+
from .._layout import Layout, LayoutError
24+
from .._subprocess import TEXT_MODE_KWARGS
25+
from .._yaml import YamlMapping, YamlValue
26+
27+
if TYPE_CHECKING:
28+
from collections.abc import Callable
29+
30+
__all__ = ["build", "history_needed", "install_extras"]
31+
32+
# Optional-dependency groups likely to hold what a documentation build needs
33+
_DOCS_EXTRAS = ("dev", "docs", "doc", "all")
34+
35+
36+
def build(
37+
repo_url: str,
38+
*,
39+
branch: str | None = None,
40+
output_dir: str | None = None,
41+
config_path: str | None = None,
42+
refresh: bool = True,
43+
version_tags: list[str] | None = None,
44+
latest_only: bool = False,
45+
shallow: bool = False,
46+
report: Callable[[str], None],
47+
) -> Path:
48+
"""
49+
Build a Git repository's documentation site
50+
51+
The build clones the repository, installs great-docs and the repository's package into a
52+
temporary virtual environment, runs that environment's `great-docs build`, and copies
53+
the site out. The temporary clone and environment are removed afterwards.
54+
55+
Unless `shallow` is true, the clone is deepened as far as the selected configuration
56+
needs: the full history for versioned docs or page dates, tags for source links.
57+
58+
Parameters
59+
----------
60+
repo_url
61+
Git URL of the repository to clone (HTTPS or SSH).
62+
branch
63+
Branch or tag to check out. If `None`, use the repository's default branch.
64+
output_dir
65+
Directory to copy the built site into, replacing any directory already there. If
66+
`None`, use the checkout's deployment path relative to the current working directory.
67+
config_path
68+
Configuration path relative to the checkout.
69+
refresh
70+
Whether to re-discover package exports before building.
71+
version_tags
72+
Version tags to build, instead of all of them.
73+
latest_only
74+
Whether to build only the latest version.
75+
shallow
76+
Whether to keep the shallow clone without fetching history. Versioned docs, page dates
77+
and tag-based source links then do not work.
78+
report
79+
A callback that receives each progress line for the user.
80+
81+
Returns
82+
-------
83+
:
84+
The absolute path of the copied site.
85+
86+
Raises
87+
------
88+
LayoutError
89+
If `config_path` is absolute or resolves outside the checkout.
90+
RuntimeError
91+
If cloning, an installation or the build fails, or the build leaves no site.
92+
"""
93+
if config_path is not None and Path(config_path).is_absolute():
94+
raise LayoutError("Remote configuration must be a relative path inside the checkout")
95+
96+
with tempfile.TemporaryDirectory(
97+
prefix="great-docs-remote-", ignore_cleanup_errors=True
98+
) as temporary:
99+
checkout = Path(temporary) / "repo"
100+
layout = _clone(
101+
repo_url,
102+
checkout,
103+
branch=branch,
104+
config_path=config_path,
105+
shallow=shallow,
106+
report=report,
107+
)
108+
output_path = (
109+
Path(output_dir).resolve()
110+
if output_dir
111+
else Path.cwd() / layout.site_dir.relative_to(checkout.resolve())
112+
)
113+
scripts = _install(checkout, Path(temporary) / "venv", report=report)
114+
115+
report("🔨 Building documentation...")
116+
command = [str(scripts / _executable("great-docs")), "build"]
117+
command += ["--project-path", str(checkout)]
118+
if config_path is not None:
119+
command += ["--config", str(layout.config_path)]
120+
if not refresh:
121+
command.append("--no-refresh")
122+
if version_tags:
123+
command += ["--versions", ",".join(version_tags)]
124+
if latest_only:
125+
command.append("--latest-only")
126+
# The build inherits this environment, including `GITHUB_TOKEN` and `GH_TOKEN`
127+
environment = {
128+
**os.environ,
129+
"VIRTUAL_ENV": str(scripts.parent),
130+
"PATH": str(scripts) + os.pathsep + os.environ.get("PATH", ""),
131+
}
132+
result = subprocess.run(command, env=environment, cwd=str(checkout))
133+
if result.returncode != 0:
134+
raise RuntimeError(f"great-docs build failed (exit {result.returncode})")
135+
136+
site_dir = layout.site_dir
137+
if not site_dir.exists():
138+
raise RuntimeError(f"Build completed but _site/ directory not found at {site_dir}")
139+
report(f"📂 Copying built site to {output_path}...")
140+
output_path.parent.mkdir(parents=True, exist_ok=True)
141+
if output_path.exists():
142+
shutil.rmtree(output_path)
143+
_ = shutil.copytree(site_dir, output_path)
144+
report(f"✅ Site built successfully: {output_path}")
145+
return output_path
146+
147+
148+
def install_extras(project_dir: Path) -> str:
149+
"""
150+
Choose the optional-dependency groups to install for a documentation build
151+
152+
Parameters
153+
----------
154+
project_dir
155+
The project root, which holds `pyproject.toml`.
156+
157+
Returns
158+
-------
159+
:
160+
The groups among `dev`, `docs`, `doc` and `all` that `pyproject.toml` defines, joined
161+
by commas, such as `"dev,docs"`. An empty string when none are defined or the file is
162+
unreadable or malformed.
163+
"""
164+
try:
165+
with (project_dir / "pyproject.toml").open("rb") as file:
166+
manifest = YamlMapping(tomllib.load(file))
167+
except (tomllib.TOMLDecodeError, UnicodeDecodeError, OSError):
168+
return ""
169+
groups = manifest.mapping("project").mapping("optional-dependencies")
170+
return ",".join(group for group in _DOCS_EXTRAS if group in groups)
171+
172+
173+
def history_needed(
174+
checkout: Path, *, config_path: Path | None = None
175+
) -> Literal["full", "tags", "none"]:
176+
"""
177+
Determine the Git history a checkout's documentation build needs
178+
179+
Parameters
180+
----------
181+
checkout
182+
The repository root.
183+
config_path
184+
The selected configuration file in the checkout. If `None`, use the configuration the
185+
checkout's layout finds.
186+
187+
Returns
188+
-------
189+
:
190+
`"full"` when the configuration lists `versions` or turns on `show_dates`: versioned
191+
builds check out tags, and page dates read `git log`. `"none"` when it sets
192+
`source.branch`, or when the configuration file is missing, unreadable or malformed.
193+
`"tags"` otherwise, so that source links can point at the tag of the checked-out
194+
commit.
195+
"""
196+
path = Layout.make(checkout, config_path).config_path
197+
try:
198+
value: YamlValue = read_yaml(path)
199+
except (OSError, ValueError):
200+
return "none"
201+
settings = YamlMapping(value) if isinstance(value, dict) else YamlMapping()
202+
203+
# This read bypasses `Config`, which moves the legacy `site.show_dates` to the top level
204+
show_dates = settings.get("show_dates")
205+
if show_dates is None:
206+
show_dates = settings.mapping("site").get("show_dates")
207+
if settings.get("versions") or show_dates:
208+
return "full"
209+
if settings.mapping("source").get("branch"):
210+
return "none"
211+
return "tags"
212+
213+
214+
def _clone(
215+
repo_url: str,
216+
checkout: Path,
217+
*,
218+
branch: str | None,
219+
config_path: str | None,
220+
shallow: bool,
221+
report: Callable[[str], None],
222+
) -> Layout:
223+
"""
224+
Clone a repository with the Git history its documentation build needs
225+
226+
Parameters
227+
----------
228+
repo_url
229+
Git URL of the repository to clone.
230+
checkout
231+
The directory to clone into.
232+
branch
233+
Branch or tag to check out, or `None` for the default branch.
234+
config_path
235+
Configuration path relative to the checkout, or `None` for the one the layout finds.
236+
shallow
237+
Whether to keep the shallow clone without fetching history.
238+
report
239+
A callback that receives each progress line for the user.
240+
241+
Returns
242+
-------
243+
:
244+
The checkout's layout under the selected configuration.
245+
246+
Raises
247+
------
248+
LayoutError
249+
If `config_path` resolves outside the checkout.
250+
RuntimeError
251+
If cloning fails.
252+
"""
253+
report(f"📦 Cloning {repo_url}...")
254+
branch_option = ["--branch", branch] if branch else []
255+
result = _run(["git", "clone", "--depth=1", *branch_option, repo_url, str(checkout)])
256+
if result.returncode != 0:
257+
raise RuntimeError(f"git clone failed (exit {result.returncode}):\n{result.stderr.strip()}")
258+
259+
selected = (checkout / config_path).resolve() if config_path else None
260+
if selected is not None and not selected.is_relative_to(checkout.resolve()):
261+
raise LayoutError("Remote configuration must remain inside the checkout")
262+
layout = Layout.make(checkout, selected)
263+
264+
# Fetch failures leave the shallow history, which builds with fewer features
265+
if not shallow:
266+
needed = history_needed(checkout, config_path=layout.config_path)
267+
if needed == "full":
268+
report(" Fetching full history (versioned docs / page dates)...")
269+
_ = _run(["git", "fetch", "--unshallow"], cwd=checkout)
270+
_ = _run(["git", "fetch", "--tags"], cwd=checkout)
271+
elif needed == "tags":
272+
report(" Fetching tags for source link detection...")
273+
_ = _run(["git", "fetch", "--tags"], cwd=checkout)
274+
report(" Cloned to temporary directory")
275+
return layout
276+
277+
278+
def _install(checkout: Path, environment: Path, *, report: Callable[[str], None]) -> Path:
279+
"""
280+
Create a virtual environment with great-docs and the checkout's package
281+
282+
If installing great-docs from PyPI fails, the build installs it from the running source
283+
tree, as during development. The checkout's package is installed in editable mode with
284+
its documentation extras, or without them if installing with extras fails.
285+
286+
Parameters
287+
----------
288+
checkout
289+
The repository root.
290+
environment
291+
The directory for the virtual environment.
292+
report
293+
A callback that receives each progress line for the user.
294+
295+
Returns
296+
-------
297+
:
298+
The environment's scripts directory.
299+
300+
Raises
301+
------
302+
RuntimeError
303+
If great-docs or the package fails to install.
304+
"""
305+
report("🐍 Creating temporary virtual environment...")
306+
venv.create(str(environment), with_pip=True, clear=True)
307+
scripts = environment / ("Scripts" if sys.platform == "win32" else "bin")
308+
pip = str(scripts / _executable("pip"))
309+
310+
report("📥 Installing great-docs into temporary environment...")
311+
result = _run([pip, "install", "great-docs"])
312+
if result.returncode != 0:
313+
result = _run([pip, "install", str(Path(__file__).resolve().parents[2])])
314+
if result.returncode != 0:
315+
raise RuntimeError(f"Failed to install great-docs:\n{result.stderr.strip()}")
316+
317+
report("📥 Installing target package...")
318+
extras = install_extras(checkout)
319+
result = _run([pip, "install", "-e", f"{checkout}[{extras}]" if extras else str(checkout)])
320+
if result.returncode != 0:
321+
result = _run([pip, "install", "-e", str(checkout)])
322+
if result.returncode != 0:
323+
raise RuntimeError(f"Failed to install target package:\n{result.stderr.strip()}")
324+
return scripts
325+
326+
327+
def _executable(name: str) -> str:
328+
"""
329+
Return a virtual environment's script name for the platform
330+
331+
Parameters
332+
----------
333+
name
334+
The script's name without an extension.
335+
336+
Returns
337+
-------
338+
:
339+
The file name, with `.exe` on Windows.
340+
"""
341+
return f"{name}.exe" if sys.platform == "win32" else name
342+
343+
344+
def _run(command: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
345+
"""
346+
Run a command and capture its output as text
347+
348+
Parameters
349+
----------
350+
command
351+
The command and its arguments.
352+
cwd
353+
The working directory, or `None` for the current one.
354+
355+
Returns
356+
-------
357+
:
358+
The finished process.
359+
"""
360+
return subprocess.run(command, cwd=cwd, capture_output=True, **TEXT_MODE_KWARGS)

0 commit comments

Comments
 (0)