From b7818d79a825301e1c06f4cee6c7e9aac5a47851 Mon Sep 17 00:00:00 2001 From: Isaiah Akorita Date: Fri, 19 Jun 2026 16:02:53 +0100 Subject: [PATCH 1/4] add build script for llms.txt and markdown docs --- .github/workflows/docs.yaml | 2 +- .gitignore | 3 + pixi.toml | 6 +- scripts/build_llms_txt.py | 161 ++++++++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 scripts/build_llms_txt.py diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 72a30fc36..3c2fbdf8a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -104,7 +104,7 @@ jobs: echo "Deploying from ref ${GITHUB_REF#refs/*/}" echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - name: build docs - run: sphinx-build -j auto -b html doc builtdocs + run: sphinx-build -j 1 -d .sphinx-doctrees -b html doc builtdocs - name: report failure if: failure() run: cat /tmp/sphinx-*.log | tail -n 100 diff --git a/.gitignore b/.gitignore index 32283cea1..678bfc1ca 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ hvplot/_version.py # pixi .pixi pixi.lock + +# Sphinx doctree cache +.sphinx-doctrees/ diff --git a/pixi.toml b/pixi.toml index 2b98151e2..8367919cf 100644 --- a/pixi.toml +++ b/pixi.toml @@ -234,6 +234,7 @@ nbsite = ">=0.9.0a12" sphinxext-rediraffe = "*" numpydoc = "*" sphinxcontrib-mermaid = "*" +nbconvert = ">=7" [feature.doc.activation.env] MOZ_HEADLESS = "1" @@ -242,11 +243,12 @@ DISPLAY = ":99.0" HVPLOT_PATCH_PLOT_DOCSTRING_SIGNATURE = "false" [feature.doc.tasks] -docs-build-sphinx = 'sphinx-build -j auto -b html doc builtdocs' +docs-build-sphinx = 'sphinx-build -j 1 -d .sphinx-doctrees -b html doc builtdocs' _docs-install = 'python -m pip install --no-deps --disable-pip-version-check -e .' +_docs-markdown = 'python scripts/build_llms_txt.py' # Depends on _docs-install instead of install as install # in the default environment -docs-build = { depends-on = ["_docs-install", "docs-build-sphinx"] } +docs-build = { depends-on = ["_docs-install", "docs-build-sphinx", "_docs-markdown"] } docs-server = 'python -m http.server 5500 --directory ./builtdocs' # ================== BUILD ==================== diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py new file mode 100644 index 000000000..fc9bec9f3 --- /dev/null +++ b/scripts/build_llms_txt.py @@ -0,0 +1,161 @@ +""" +Build markdown docs and llms.txt for hvPlot's LLM-facing documentation set. + +Run as part of the docs build pipeline: + pixi run docs-build +""" + +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).parent.parent +DOC_DIR = ROOT / 'doc' +BUILTDOCS_DIR = ROOT / 'builtdocs' +OUTPUT_DIR = BUILTDOCS_DIR / 'markdown' +MARKDOWN_BASE_URL = '/markdown' + +INCLUDED_REFERENCE_FILES = { + Path('ref/index.md'), + Path('ref/installation.md'), + Path('ref/data_libraries.ipynb'), + Path('ref/plotting_extensions.ipynb'), + Path('ref/deprecations.md'), + Path('ref/api/index.md'), +} + +INCLUDED_DIR_PREFIXES = ( + Path('tutorials'), + Path('ref/plotting_options'), + Path('ref/api/manual'), + Path('ref/api_compatibility'), +) + +EXCLUDED_DIR_NAMES = {'gallery', 'user_guide', '.ipynb_checkpoints'} + + +def _is_included(rel_path: Path) -> bool: + if any(part in EXCLUDED_DIR_NAMES for part in rel_path.parts): + return False + + if rel_path.suffix not in {'.md', '.ipynb'}: + return False + + if rel_path in INCLUDED_REFERENCE_FILES: + return True + + return any(rel_path.is_relative_to(prefix) for prefix in INCLUDED_DIR_PREFIXES) + + +def _iter_source_docs() -> list[Path]: + docs = [ + path + for path in DOC_DIR.rglob('*') + if path.is_file() and _is_included(path.relative_to(DOC_DIR)) + ] + return sorted(docs) + + +def _convert_notebook(notebook_path: Path, output_dir: Path) -> bool: + output_dir.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + [ + sys.executable, + '-m', + 'jupyter', + 'nbconvert', + '--to', + 'markdown', + '--output-dir', + str(output_dir), + str(notebook_path), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + rel = notebook_path.relative_to(DOC_DIR) + print(f' Warning: failed to convert {rel}: {result.stderr.strip()}') + return False + return True + + +def build_markdown_docs() -> list[Path]: + generated: list[Path] = [] + for source in _iter_source_docs(): + rel = source.relative_to(DOC_DIR) + destination_dir = OUTPUT_DIR / rel.parent + + if source.suffix == '.md': + destination = OUTPUT_DIR / rel + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + generated.append(rel) + print(f' Copied {rel}') + continue + + if _convert_notebook(source, destination_dir): + generated.append(rel.with_suffix('.md')) + print(f' Converted {rel}') + + return generated + + +def _build_links(paths: list[Path]) -> list[str]: + return [ + f'- [{path.stem.replace("_", " ")}]({MARKDOWN_BASE_URL}/{path.as_posix()})' + for path in sorted(paths) + ] + + +def generate_llms_txt(generated_paths: list[Path]) -> None: + tutorial_paths = [p for p in generated_paths if p.is_relative_to(Path('tutorials'))] + ref_paths = [ + p + for p in generated_paths + if p.is_relative_to(Path('ref')) and not p.is_relative_to(Path('ref/api/manual')) + ] + api_manual_paths = [p for p in generated_paths if p.is_relative_to(Path('ref/api/manual'))] + + lines = [ + '# hvPlot', + '', + 'hvPlot is a high-level plotting API for the HoloViz ecosystem built on HoloViews.', + 'This file points to markdown documentation selected for code-writing utility.', + '', + f'All links resolve under {MARKDOWN_BASE_URL}/.', + '', + ] + + if tutorial_paths: + lines.extend(['## Tutorials', '']) + lines.extend(_build_links(tutorial_paths)) + lines.append('') + + if ref_paths: + lines.extend(['## Reference', '']) + lines.extend(_build_links(ref_paths)) + lines.append('') + + if api_manual_paths: + lines.extend(['## API Manual', '']) + lines.extend(_build_links(api_manual_paths)) + lines.append('') + + llms_txt = BUILTDOCS_DIR / 'llms.txt' + llms_txt.write_text('\n'.join(lines), encoding='utf-8') + print(f'Generated {llms_txt}') + + +def main() -> None: + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + print('Building markdown docs for hvPlot scope...') + generated = build_markdown_docs() + print('Generating llms.txt...') + generate_llms_txt(generated) + print('Done!') + + +if __name__ == '__main__': + main() From d930bef9ddbb7502f73fec75562d2abdac384d86 Mon Sep 17 00:00:00 2001 From: Isaiah Akorita Date: Fri, 19 Jun 2026 16:05:06 +0100 Subject: [PATCH 2/4] remove streaming ref --- doc/user_guide/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/user_guide/index.md b/doc/user_guide/index.md index 6d84b26f9..7b8577b07 100644 --- a/doc/user_guide/index.md +++ b/doc/user_guide/index.md @@ -11,7 +11,7 @@ commandline, and from a script. Another section will introduce you to generating [subplots](Subplots.ipynb) from your data. Once the basics are covered you can learn how to use the plotting API -for specific types of data including [streaming data](Streaming.ipynb), [gridded data](Gridded_Data.ipynb) +for specific types of data including [gridded data](Gridded_Data.ipynb), [network graphs](NetworkX.ipynb), [geographic data](Geographic_Data.ipynb), and [timeseries data](Timeseries_Data.ipynb). These sections are not meant to be read in a particular order; you should take a look at any that seem @@ -78,7 +78,6 @@ Plotting Extensions Exploring data Viewing Subplots -Streaming Gridded Data Network Graphs Geographic Data From 7080de3ac7dadcb3d668fd9ac921919d0bf081e7 Mon Sep 17 00:00:00 2001 From: Isaiah Akorita Date: Tue, 23 Jun 2026 15:45:58 +0100 Subject: [PATCH 3/4] update script --- scripts/build_llms_txt.py | 46 ++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py index fc9bec9f3..6e4e5fb9b 100644 --- a/scripts/build_llms_txt.py +++ b/scripts/build_llms_txt.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +from collections.abc import Callable from pathlib import Path ROOT = Path(__file__).parent.parent @@ -102,9 +103,34 @@ def build_markdown_docs() -> list[Path]: return generated -def _build_links(paths: list[Path]) -> list[str]: +def _default_label(path: Path) -> str: + return path.stem.replace('_', ' ') + + +def _api_manual_label(path: Path) -> str: + name = path.stem + for prefix in ('hvplot.hvPlot.', 'hvplot.plotting.'): + if name.startswith(prefix): + name = name.removeprefix(prefix) + break + return name.replace('_', ' ') + + +def _reference_label(path: Path) -> str: + if path.stem != 'index': + return _default_label(path) + + if path.parent == Path('ref'): + return 'reference' + + return path.parent.name.replace('_', ' ') + + +def _build_links( + paths: list[Path], label_builder: Callable[[Path], str] = _default_label +) -> list[str]: return [ - f'- [{path.stem.replace("_", " ")}]({MARKDOWN_BASE_URL}/{path.as_posix()})' + f'- [{label_builder(path)}]({MARKDOWN_BASE_URL}/{path.as_posix()})' for path in sorted(paths) ] @@ -130,17 +156,27 @@ def generate_llms_txt(generated_paths: list[Path]) -> None: if tutorial_paths: lines.extend(['## Tutorials', '']) + lines.extend( + [ + 'Step-by-step guides to help you master hvPlot and explore the full HoloViz ecosystem.', + '', + ] + ) lines.extend(_build_links(tutorial_paths)) lines.append('') if ref_paths: lines.extend(['## Reference', '']) - lines.extend(_build_links(ref_paths)) + lines.extend( + ['API reference and pages that provide detailed information about hvPlot’s usage.', ''] + ) + lines.extend(_build_links(ref_paths, _reference_label)) lines.append('') if api_manual_paths: - lines.extend(['## API Manual', '']) - lines.extend(_build_links(api_manual_paths)) + lines.extend(['## API', '']) + lines.extend(['hvPlot plotting APIs', '']) + lines.extend(_build_links(api_manual_paths, _api_manual_label)) lines.append('') llms_txt = BUILTDOCS_DIR / 'llms.txt' From 217c6e087f322acad7eac6ff7e52397d30ded15b Mon Sep 17 00:00:00 2001 From: Isaiah Akorita Date: Thu, 2 Jul 2026 13:07:55 +0100 Subject: [PATCH 4/4] include all sections except user guide --- scripts/build_llms_txt.py | 64 +++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py index 6e4e5fb9b..a39637d16 100644 --- a/scripts/build_llms_txt.py +++ b/scripts/build_llms_txt.py @@ -17,23 +17,7 @@ OUTPUT_DIR = BUILTDOCS_DIR / 'markdown' MARKDOWN_BASE_URL = '/markdown' -INCLUDED_REFERENCE_FILES = { - Path('ref/index.md'), - Path('ref/installation.md'), - Path('ref/data_libraries.ipynb'), - Path('ref/plotting_extensions.ipynb'), - Path('ref/deprecations.md'), - Path('ref/api/index.md'), -} - -INCLUDED_DIR_PREFIXES = ( - Path('tutorials'), - Path('ref/plotting_options'), - Path('ref/api/manual'), - Path('ref/api_compatibility'), -) - -EXCLUDED_DIR_NAMES = {'gallery', 'user_guide', '.ipynb_checkpoints'} +EXCLUDED_DIR_NAMES = {'user_guide', '.ipynb_checkpoints'} def _is_included(rel_path: Path) -> bool: @@ -43,10 +27,7 @@ def _is_included(rel_path: Path) -> bool: if rel_path.suffix not in {'.md', '.ipynb'}: return False - if rel_path in INCLUDED_REFERENCE_FILES: - return True - - return any(rel_path.is_relative_to(prefix) for prefix in INCLUDED_DIR_PREFIXES) + return True def _iter_source_docs() -> list[Path]: @@ -107,6 +88,14 @@ def _default_label(path: Path) -> str: return path.stem.replace('_', ' ') +def _section_label(path: Path) -> str: + if path == Path('index.md'): + return 'home' + if path.parent == Path('.'): + return path.stem.replace('_', ' ') + return path.parent.as_posix().replace('-', ' ') + + def _api_manual_label(path: Path) -> str: name = path.stem for prefix in ('hvplot.hvPlot.', 'hvplot.plotting.'): @@ -135,25 +124,40 @@ def _build_links( ] +def _top_level_paths(generated_paths: list[Path]) -> list[Path]: + return [p for p in generated_paths if len(p.parts) == 1] + + +def _section_paths(generated_paths: list[Path], section: str) -> list[Path]: + return [p for p in generated_paths if p.is_relative_to(Path(section))] + + def generate_llms_txt(generated_paths: list[Path]) -> None: - tutorial_paths = [p for p in generated_paths if p.is_relative_to(Path('tutorials'))] + top_level_paths = _top_level_paths(generated_paths) + tutorial_paths = _section_paths(generated_paths, 'tutorials') + gallery_paths = _section_paths(generated_paths, 'gallery') ref_paths = [ p - for p in generated_paths - if p.is_relative_to(Path('ref')) and not p.is_relative_to(Path('ref/api/manual')) + for p in _section_paths(generated_paths, 'ref') + if not p.is_relative_to(Path('ref/api/manual')) ] - api_manual_paths = [p for p in generated_paths if p.is_relative_to(Path('ref/api/manual'))] + api_manual_paths = _section_paths(generated_paths, 'ref/api/manual') lines = [ '# hvPlot', '', 'hvPlot is a high-level plotting API for the HoloViz ecosystem built on HoloViews.', - 'This file points to markdown documentation selected for code-writing utility.', + 'This file points to the generated markdown documentation selected for code-writing utility.', '', f'All links resolve under {MARKDOWN_BASE_URL}/.', '', ] + if top_level_paths: + lines.extend(['## Home', '']) + lines.extend(_build_links(top_level_paths, _section_label)) + lines.append('') + if tutorial_paths: lines.extend(['## Tutorials', '']) lines.extend( @@ -165,6 +169,14 @@ def generate_llms_txt(generated_paths: list[Path]) -> None: lines.extend(_build_links(tutorial_paths)) lines.append('') + if gallery_paths: + lines.extend(['## Gallery', '']) + lines.extend( + ['Example visualizations using hvPlot with different backends and datasets.', ''] + ) + lines.extend(_build_links(gallery_paths)) + lines.append('') + if ref_paths: lines.extend(['## Reference', '']) lines.extend(