harden: use yaml.safe_load in generate_index_of_sub_section.py...

.github/workflows/build-tests.yml

@@ -3,10 +3,10 @@ name: Build tests and GitHub release
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
-      - master
+    - main
 
 jobs:
   run-py-tests:
@@ -16,42 +16,87 @@ jobs:
         os: ["ubuntu-24.04"]
         python-version:
           - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
+          # No need to run with many Py versions - as long as one is passing, we should be OK
+          # - "3.11"
+          # - "3.12"
+          # - "3.13"
 
     steps:
       - name: Checkout repository and submodules
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           lfs: true
           submodules: true
 
       - name: Setup python 3
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Build documentation
-        uses: Exabyte-io/action-mkdocs-build@main
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Build documentation (legacy)
+        run: |
+          python -m mkdocs build -f mkdocs.yml
+
+      - name: Build split sites
+        run: |
+          python -m mkdocs build -f mkdocs-guide.yml      -d site/guide
+          python -m mkdocs build -f mkdocs-interface.yml  -d site/interface
+          python -m mkdocs build -f mkdocs-concepts.yml   -d site/reference
+          python -m mkdocs build -f mkdocs-resources.yml  -d site/resources
+          python -m mkdocs build -f mkdocs-developers.yml -d site/developers
+          python -m mkdocs build -f mkdocs-cli.yml        -d site/command-line
+          python -m mkdocs build -f mkdocs-standards.yml  -d site/standards
+
+          # Fix and copy subsite homepages to root of each subsite
+          fix_homepage() {
+            local src="$1"
+            local dst="$2"
+            if [ -f "$src" ]; then
+              sed \
+                -e 's|"base": ".."|"base": "."|' \
+                -e 's|"\.\./assets/|"./assets/|g' \
+                -e 's|"\.\./search/|"./search/|g' \
+                -e 's|"\.\./extra/|"./extra/|g' \
+                -e 's|"\.\./images/|"./images/|g' \
+                -e 's|href="\.\./|href="./|g' \
+                -e 's|src="\.\./|src="./|g' \
+                "$src" > "$dst"
+            fi
+          }
+          fix_homepage site/guide/index-guide/index.html              site/guide/index.html
+          fix_homepage site/interface/index-interface/index.html       site/interface/index.html
+          fix_homepage site/reference/index-concepts/index.html       site/reference/index.html
+          fix_homepage site/resources/index-resources/index.html      site/resources/index.html
+          fix_homepage site/developers/index-developers/index.html    site/developers/index.html
+          fix_homepage site/command-line/index-cli/index.html                  site/command-line/index.html
+          fix_homepage site/standards/index-standards/index.html      site/standards/index.html
+
+
+
+      - name: Check for broken internal links
+        run: python scripts/links/check-links.py site
 
   publish-py-package:
     needs:
       - run-py-tests
     runs-on: ubuntu-24.04
-    if: (github.repository != 'Exabyte-io/template-definitions-js-py') && (github.ref_name == 'master')
+    if: (github.ref_name == 'main')
 
     steps:
       - name: Checkout this repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           lfs: true
 
       - name: Checkout actions repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          repository: Exabyte-io/actions
+          repository: mat3ra/actions
           token: ${{ secrets.BOT_GITHUB_TOKEN }}
           path: actions
 

.github/workflows/docs-agent-index.yml

@@ -0,0 +1,39 @@
+name: Refresh the documentation agent index
+
+# The agent answers from a snapshot of this repository's content, baked into
+# its container image. When the content changes, the snapshot is stale until
+# the service is rebuilt.
+#
+# This workflow does no ingestion of its own: all agent logic lives in the
+# documentation-agent repository, which checks this repository out at the
+# commit below and builds its own index. The two pipelines stay decoupled —
+# this one only says "the documentation moved, here is where".
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "lang/en/docs/**"
+  workflow_dispatch:
+
+jobs:
+  notify:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Tell the agent repository to rebuild
+        # A token with `repo` scope on mat3ra/documentation-agent. The default
+        # GITHUB_TOKEN cannot dispatch across repositories.
+        env:
+          TOKEN: ${{ secrets.DOCS_AGENT_DISPATCH_TOKEN }}
+        run: |
+          if [ -z "$TOKEN" ]; then
+            echo "DOCS_AGENT_DISPATCH_TOKEN is not set; skipping the rebuild trigger." >&2
+            exit 0
+          fi
+          curl -fsS -X POST \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            https://api.github.com/repos/mat3ra/documentation-agent/dispatches \
+            -d "{\"event_type\":\"documentation-updated\",\"client_payload\":{\"sha\":\"${GITHUB_SHA}\"}}"
+          echo "Requested a rebuild from documentation@${GITHUB_SHA}"

.github/workflows/s3-deploy.yml

@@ -18,23 +18,58 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           lfs: true
           submodules: true
 
       - name: Set python 3 version
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.10"
 
-      - name: Build pages
-        uses: Exabyte-io/action-mkdocs-build@main
+      - name: Build pages (legacy)
+        uses: mat3ra/action-mkdocs-build@main
+
+      - name: Build split sites
+        run: |
+          python -m mkdocs build -f mkdocs-guide.yml      -d site/guide
+          python -m mkdocs build -f mkdocs-interface.yml  -d site/interface
+          python -m mkdocs build -f mkdocs-concepts.yml   -d site/reference
+          python -m mkdocs build -f mkdocs-resources.yml  -d site/resources
+          python -m mkdocs build -f mkdocs-developers.yml -d site/developers
+          python -m mkdocs build -f mkdocs-cli.yml        -d site/command-line
+          python -m mkdocs build -f mkdocs-standards.yml  -d site/standards
+
+          # Fix and copy subsite homepages to root of each subsite
+          fix_homepage() {
+            local src="$1"
+            local dst="$2"
+            if [ -f "$src" ]; then
+              sed \
+                -e 's|"base": ".."|"base": "."|' \
+                -e 's|"\.\./assets/|"./assets/|g' \
+                -e 's|"\.\./search/|"./search/|g' \
+                -e 's|"\.\./extra/|"./extra/|g' \
+                -e 's|"\.\./images/|"./images/|g' \
+                -e 's|href="\.\./|href="./|g' \
+                -e 's|src="\.\./|src="./|g' \
+                "$src" > "$dst"
+            fi
+          }
+          fix_homepage site/guide/index-guide/index.html              site/guide/index.html
+          fix_homepage site/interface/index-interface/index.html       site/interface/index.html
+          fix_homepage site/reference/index-concepts/index.html       site/reference/index.html
+          fix_homepage site/resources/index-resources/index.html      site/resources/index.html
+          fix_homepage site/developers/index-developers/index.html    site/developers/index.html
+          fix_homepage site/command-line/index-cli/index.html                  site/command-line/index.html
+          fix_homepage site/standards/index-standards/index.html      site/standards/index.html
+
 
       - name: Deploy to Production (dev branch, old docs)
         if: github.ref == 'refs/heads/dev'
-        uses: Reggionick/s3-deploy@v4
+        uses: Reggionick/s3-deploy@04c48f45adfd7a34c66348cf9cd1b6cd117cc467 # v4.0.0
         with:
           folder: site
           bucket: docs.mat3ra.com
@@ -46,7 +81,7 @@ jobs:
 
       - name: Deploy to Development (main branch, new docs)
         if: github.ref == 'refs/heads/main'
-        uses: Reggionick/s3-deploy@v4
+        uses: Reggionick/s3-deploy@04c48f45adfd7a34c66348cf9cd1b6cd117cc467 # v4.0.0
         with:
           folder: site
           bucket: docs-new.mat3ra.com
@@ -55,3 +90,4 @@ jobs:
           invalidation: /*
           no-cache: true
           private: true
+

.github/workflows/widget-tests.yml

@@ -0,0 +1,56 @@
+name: Ask AI widget tests
+
+# The widget ships to every documentation page, so a regression in it is a
+# regression on the whole site. These tests need no cloud access and no model
+# calls: the agent service is faked at the network boundary.
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "extra/js/docs-agent.js"
+      - "extra/css/docs-agent.css"
+      - "tests/widget/**"
+      - ".github/workflows/widget-tests.yml"
+  pull_request:
+    paths:
+      - "extra/js/docs-agent.js"
+      - "extra/css/docs-agent.css"
+      - "tests/widget/**"
+      - ".github/workflows/widget-tests.yml"
+
+jobs:
+  playwright:
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: tests/widget
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install
+        run: npm ci || npm install
+
+      - name: Install the browser
+        run: npx playwright install --with-deps chromium
+
+      - name: Run the tests
+        run: npx playwright test
+
+      - name: Upload the report on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: tests/widget/playwright-report
+          retention-days: 7

.gitignore

@@ -16,5 +16,19 @@ service-account-key.json
 scripts/*.mp3
 scripts/*.mp4
 
+# RAG demo generated index
+scripts/rag/chunks.jsonl
+
 # Pyenv
 .python-version
+# Agents workdir
+tmp
+.local-mkdocs*.yml
+
+# Local clone of the documentation-agent service repo (see plans/)
+reference/
+
+# Widget browser tests
+tests/widget/node_modules/
+tests/widget/test-results/
+tests/widget/playwright-report/

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "esse"]
 	path = data/esse
-	url = https://github.com/Exabyte-io/esse.git
+	url = https://github.com/mat3ra/esse.git