Add build script for llms.txt and markdown docs

.github/workflows/docs.yaml

@@ -104,7 +104,7 @@ jobs:
           echo "Deploying from ref ${GITHUB_REF#refs/*/}"
           echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
       - name: build docs
-        run: sphinx-build -j auto -b html doc builtdocs
+        run: sphinx-build -j 1 -d .sphinx-doctrees -b html doc builtdocs
       - name: report failure
         if: failure()
         run: cat /tmp/sphinx-*.log | tail -n 100

.gitignore

@@ -132,3 +132,6 @@ hvplot/_version.py
 # pixi
 .pixi
 pixi.lock
+
+# Sphinx doctree cache
+.sphinx-doctrees/

doc/user_guide/index.md

@@ -11,7 +11,7 @@ commandline, and from a script. Another section will introduce you to
 generating [subplots](Subplots.ipynb) from your data.
 
 Once the basics are covered you can learn how to use the plotting API
-for specific types of data including [streaming data](Streaming.ipynb), [gridded data](Gridded_Data.ipynb)
+for specific types of data including [gridded data](Gridded_Data.ipynb),
 [network graphs](NetworkX.ipynb), [geographic data](Geographic_Data.ipynb),
 and [timeseries data](Timeseries_Data.ipynb). These sections are not meant
 to be read in a particular order; you should take a look at any that seem
@@ -78,7 +78,6 @@ Plotting Extensions <Plotting_Extensions>
 Exploring data <Explorer>
 Viewing <Viewing>
 Subplots <Subplots>
-Streaming <Streaming>
 Gridded Data <Gridded_Data>
 Network Graphs <NetworkX>
 Geographic Data <Geographic_Data>

pixi.toml

@@ -234,6 +234,7 @@ nbsite = ">=0.9.0a12"
 sphinxext-rediraffe = "*"
 numpydoc = "*"
 sphinxcontrib-mermaid = "*"
+nbconvert = ">=7"
 
 [feature.doc.activation.env]
 MOZ_HEADLESS = "1"
@@ -242,11 +243,12 @@ DISPLAY = ":99.0"
 HVPLOT_PATCH_PLOT_DOCSTRING_SIGNATURE = "false"
 
 [feature.doc.tasks]
-docs-build-sphinx = 'sphinx-build -j auto -b html doc builtdocs'
+docs-build-sphinx = 'sphinx-build -j 1 -d .sphinx-doctrees -b html doc builtdocs'
 _docs-install = 'python -m pip install --no-deps --disable-pip-version-check -e .'
+_docs-markdown = 'python scripts/build_llms_txt.py'
 # Depends on _docs-install instead of install as install
 # in the default environment
-docs-build = { depends-on = ["_docs-install", "docs-build-sphinx"] }
+docs-build = { depends-on = ["_docs-install", "docs-build-sphinx", "_docs-markdown"] }
 docs-server = 'python -m http.server 5500 --directory ./builtdocs'
 
 # ================== BUILD ====================

scripts/build_llms_txt.py

@@ -0,0 +1,209 @@
+"""
+Build markdown docs and llms.txt for hvPlot's LLM-facing documentation set.
+
+Run as part of the docs build pipeline:
+    pixi run docs-build
+"""
+
+import shutil
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent
+DOC_DIR = ROOT / 'doc'
+BUILTDOCS_DIR = ROOT / 'builtdocs'
+OUTPUT_DIR = BUILTDOCS_DIR / 'markdown'
+MARKDOWN_BASE_URL = '/markdown'
+
+EXCLUDED_DIR_NAMES = {'user_guide', '.ipynb_checkpoints'}
+
+
+def _is_included(rel_path: Path) -> bool:
+    if any(part in EXCLUDED_DIR_NAMES for part in rel_path.parts):
+        return False
+
+    if rel_path.suffix not in {'.md', '.ipynb'}:
+        return False
+
+    return True
+
+
+def _iter_source_docs() -> list[Path]:
+    docs = [
+        path
+        for path in DOC_DIR.rglob('*')
+        if path.is_file() and _is_included(path.relative_to(DOC_DIR))
+    ]
+    return sorted(docs)
+
+
+def _convert_notebook(notebook_path: Path, output_dir: Path) -> bool:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    result = subprocess.run(
+        [
+            sys.executable,
+            '-m',
+            'jupyter',
+            'nbconvert',
+            '--to',
+            'markdown',
+            '--output-dir',
+            str(output_dir),
+            str(notebook_path),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        rel = notebook_path.relative_to(DOC_DIR)
+        print(f'  Warning: failed to convert {rel}: {result.stderr.strip()}')
+        return False
+    return True
+
+
+def build_markdown_docs() -> list[Path]:
+    generated: list[Path] = []
+    for source in _iter_source_docs():
+        rel = source.relative_to(DOC_DIR)
+        destination_dir = OUTPUT_DIR / rel.parent
+
+        if source.suffix == '.md':
+            destination = OUTPUT_DIR / rel
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source, destination)
+            generated.append(rel)
+            print(f'  Copied {rel}')
+            continue
+
+        if _convert_notebook(source, destination_dir):
+            generated.append(rel.with_suffix('.md'))
+            print(f'  Converted {rel}')
+
+    return generated
+
+
+def _default_label(path: Path) -> str:
+    return path.stem.replace('_', ' ')
+
+
+def _section_label(path: Path) -> str:
+    if path == Path('index.md'):
+        return 'home'
+    if path.parent == Path('.'):
+        return path.stem.replace('_', ' ')
+    return path.parent.as_posix().replace('-', ' ')
+
+
+def _api_manual_label(path: Path) -> str:
+    name = path.stem
+    for prefix in ('hvplot.hvPlot.', 'hvplot.plotting.'):
+        if name.startswith(prefix):
+            name = name.removeprefix(prefix)
+            break
+    return name.replace('_', ' ')
+
+
+def _reference_label(path: Path) -> str:
+    if path.stem != 'index':
+        return _default_label(path)
+
+    if path.parent == Path('ref'):
+        return 'reference'
+
+    return path.parent.name.replace('_', ' ')
+
+
+def _build_links(
+    paths: list[Path], label_builder: Callable[[Path], str] = _default_label
+) -> list[str]:
+    return [
+        f'- [{label_builder(path)}]({MARKDOWN_BASE_URL}/{path.as_posix()})'
+        for path in sorted(paths)
+    ]
+
+
+def _top_level_paths(generated_paths: list[Path]) -> list[Path]:
+    return [p for p in generated_paths if len(p.parts) == 1]
+
+
+def _section_paths(generated_paths: list[Path], section: str) -> list[Path]:
+    return [p for p in generated_paths if p.is_relative_to(Path(section))]
+
+
+def generate_llms_txt(generated_paths: list[Path]) -> None:
+    top_level_paths = _top_level_paths(generated_paths)
+    tutorial_paths = _section_paths(generated_paths, 'tutorials')
+    gallery_paths = _section_paths(generated_paths, 'gallery')
+    ref_paths = [
+        p
+        for p in _section_paths(generated_paths, 'ref')
+        if not p.is_relative_to(Path('ref/api/manual'))
+    ]
+    api_manual_paths = _section_paths(generated_paths, 'ref/api/manual')
+
+    lines = [
+        '# hvPlot',
+        '',
+        'hvPlot is a high-level plotting API for the HoloViz ecosystem built on HoloViews.',
+        'This file points to the generated markdown documentation selected for code-writing utility.',
+        '',
+        f'All links resolve under {MARKDOWN_BASE_URL}/.',
+        '',
+    ]
+
+    if top_level_paths:
+        lines.extend(['## Home', ''])
+        lines.extend(_build_links(top_level_paths, _section_label))
+        lines.append('')
+
+    if tutorial_paths:
+        lines.extend(['## Tutorials', ''])
+        lines.extend(
+            [
+                'Step-by-step guides to help you master hvPlot and explore the full HoloViz ecosystem.',
+                '',
+            ]
+        )
+        lines.extend(_build_links(tutorial_paths))
+        lines.append('')
+
+    if gallery_paths:
+        lines.extend(['## Gallery', ''])
+        lines.extend(
+            ['Example visualizations using hvPlot with different backends and datasets.', '']
+        )
+        lines.extend(_build_links(gallery_paths))
+        lines.append('')
+
+    if ref_paths:
+        lines.extend(['## Reference', ''])
+        lines.extend(
+            ['API reference and pages that provide detailed information about hvPlot’s usage.', '']
+        )
+        lines.extend(_build_links(ref_paths, _reference_label))
+        lines.append('')
+
+    if api_manual_paths:
+        lines.extend(['## API', ''])
+        lines.extend(['hvPlot plotting APIs', ''])
+        lines.extend(_build_links(api_manual_paths, _api_manual_label))
+        lines.append('')
+
+    llms_txt = BUILTDOCS_DIR / 'llms.txt'
+    llms_txt.write_text('\n'.join(lines), encoding='utf-8')
+    print(f'Generated {llms_txt}')
+
+
+def main() -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    print('Building markdown docs for hvPlot scope...')
+    generated = build_markdown_docs()
+    print('Generating llms.txt...')
+    generate_llms_txt(generated)
+    print('Done!')
+
+
+if __name__ == '__main__':
+    main()