Migrating Hugo modules and Discourse data updates worflows

.github/workflows/update-discourse-data.yml

@@ -5,26 +5,25 @@ on:
   schedule:
     - cron: "0 6 * * *"
 
+concurrency:
+  group: update-discourse-data
+  cancel-in-progress: false
+
 jobs:
   update-discourse-data:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     permissions:
       contents: write
 
     steps:
-      - uses: actions/create-github-app-token@v3
-        id: app-token
-        with:
-          client-id: ${{ vars.WEBSITE_UPDATER_CLIENT_ID }}
-          private-key: ${{ secrets.WEBSITE_UPDATER_KEY }}
-
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
-          token: ${{ steps.app-token.outputs.token }}
+          fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
         with:
           python-version: "3.14"
 
@@ -33,19 +32,15 @@ jobs:
           python tools/fetch-news.py
           python tools/fetch-faq.py
 
-      - name: Configure Git author
+      - name: Commit and push changes
         run: |
+          git add static/assets/data/news.json static/assets/data/faq.json
+          if git diff --cached --quiet; then
+            echo "Discourse data is already up to date."
+            exit 0
+          fi
+
           git config --local user.name "precice-bot"
           git config --local user.email "info@precice.org"
-
-      - name: Commit changes (if any)
-        run: |
-          git add assets/data/news.json
-          git add assets/data/faq.json
-          git commit -m "Update Discourse data data [skip ci]" || echo "No changes to commit"
-
-      - name: Push commit
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ steps.app-token.outputs.token }}
-          branch: ${{ github.ref }}
+          git commit --message "Update Discourse data [skip ci]"
+          git push origin "HEAD:${GITHUB_REF_NAME}"

.github/workflows/update-submodules.yml

@@ -1,39 +1,67 @@
-name: Update submodules
+name: Update Hugo modules
+
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "5 0 * * *"
+
+concurrency:
+  group: update-hugo-modules
+  cancel-in-progress: false
+
+env:
+  GO_VERSION: "1.26.3"
+  HUGO_VERSION: "0.163.3"
+  PYTHON_VERSION: "3.14"
 
 jobs:
   update:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
-      contents: 'write'
-    outputs:
-      success: ${{ steps.commit.outputs.outcome  == 'success' }}
+      contents: write
+
     steps:
-      - uses: actions/create-github-app-token@v3
-        id: app-token
-        with:
-          client-id: ${{ vars.WEBSITE_UPDATER_CLIENT_ID }}
-          private-key: ${{ secrets.WEBSITE_UPDATER_KEY }}
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
+          fetch-depth: 0
           submodules: false
-          lfs: true
-          token: ${{ steps.app-token.outputs.token }}
-      - name: Update submodules
-        run: git submodule update --init --remote --force
-      - name: Commit update
-        id: commit
-        continue-on-error: true
+          lfs: false
+          token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }}
+
+      - name: Set up Go
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: go.sum
+
+      - name: Set up Hugo
+        uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3
+        with:
+          hugo-version: ${{ env.HUGO_VERSION }}
+          extended: true
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Synchronize Hugo module revisions
+        run: python3 tools/sync_hugo_modules.py
+
+      - name: Verify updated modules
+        run: hugo mod verify
+
+      - name: Commit and push update
         run: |
+          if git diff --quiet -- go.mod go.sum; then
+            echo "Hugo modules are already up to date."
+            exit 0
+          fi
+
           git config --local user.name "precice-bot"
           git config --local user.email "info@precice.org"
-          git commit -m "Update submodules" -a
-      - name: Push commit
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ steps.app-token.outputs.token }}
-          branch: ${{ github.ref }}
+          git add go.mod go.sum
+          git commit --message "Update Hugo modules"
+          git push origin "HEAD:${GITHUB_REF_NAME}"

static/js/forum-fetch.js

@@ -1,5 +1,8 @@
 console.log("forum-fetch.js loaded!");
 
+// Resolve data from the site root even when the site is served from a subpath.
+const siteRoot = new URL("../", document.currentScript.src);
+
 document.addEventListener("DOMContentLoaded", async function () {
   console.log("FAQ loader running...");
 
@@ -22,7 +25,7 @@ document.addEventListener("DOMContentLoaded", async function () {
   }
 
   try {
-    const res = await fetch("/assets/data/faq.json");
+    const res = await fetch(new URL("assets/data/faq.json", siteRoot));
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
 
     const data = await res.json();
@@ -108,8 +111,7 @@ document.addEventListener("DOMContentLoaded", async function () {
           new Date(t.last_posted_at).toLocaleDateString("en-GB") +
           " | Replies: " +
           t.posts_count +
-          " | Views: " +
-          t.views;
+          (t.views !== undefined && t.views !== null ? " | Views: " + t.views : "");
 
         card.appendChild(h4);
         card.appendChild(excerptP);

tools/fetch-faq.py

@@ -2,25 +2,40 @@
 import os
 import json
 import re
+import time
 import urllib.request
-from datetime import datetime
 
 DISCOURSE_BASE = "https://precice.discourse.group"
-OUTPUT_FILE = "./assets/data/faq.json"
+OUTPUT_FILE = "./static/assets/data/faq.json"
+VIEW_THRESHOLD = 50
+USER_AGENT = "preCICE-Website-Updater/1.0 (https://precice.org)"
 
 
 def http_get_json(url: str):
     """GET URL and return parsed JSON using only stdlib."""
-    with urllib.request.urlopen(url) as r:
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req) as r:
         return json.load(r)
 
 
 def strip_html(text: str) -> str:
     return re.sub(r"<[^>]+>", "", text)
 
 
+def load_existing_topics() -> dict:
+    if not os.path.exists(OUTPUT_FILE):
+        return {}
+    try:
+        with open(OUTPUT_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            return {t["id"]: t for t in data.get("topics", []) if "id" in t}
+    except Exception:
+        return {}
+
+
 def fetch_excerpt(topic_id: int) -> str:
     try:
+        time.sleep(0.1)
         topic_data = http_get_json(f"{DISCOURSE_BASE}/t/{topic_id}.json")
         raw = topic_data.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "")
         cleaned = strip_html(raw)
@@ -38,25 +53,48 @@ def fetch_faq():
         topic_list = data.get("topic_list", {}).get("topics", [])
         print(f"Found {len(topic_list)} FAQ topics. Fetching excerpts...")
 
+        existing_topics = load_existing_topics()
         topics = []
         for t in topic_list:
-            excerpt = fetch_excerpt(t["id"])
+            topic_id = t["id"]
+            new_views = t.get("views", 0) or 0
+            new_last_posted = t.get("last_posted_at")
+
+            old_topic = existing_topics.get(topic_id, {})
+            old_views = old_topic.get("views", 0) or 0
+            old_excerpt = old_topic.get("excerpt", "")
+
+            # Preserve view count if difference is below threshold
+            if topic_id in existing_topics:
+                if abs(new_views - old_views) < VIEW_THRESHOLD:
+                    views = old_views
+                else:
+                    views = new_views
+            else:
+                views = new_views
+
+            # Only fetch excerpt if topic is new, updated, or missing excerpt
+            if old_excerpt and old_topic.get("last_posted_at") == new_last_posted:
+                excerpt = old_excerpt
+            else:
+                fetched_excerpt = fetch_excerpt(topic_id)
+                excerpt = fetched_excerpt if fetched_excerpt else old_excerpt
+
             topics.append({
-                "id": t["id"],
+                "id": topic_id,
                 "title": t["title"],
                 "slug": t["slug"],
-                "url": f"{DISCOURSE_BASE}/t/{t['slug']}/{t['id']}",
+                "url": f"{DISCOURSE_BASE}/t/{t['slug']}/{topic_id}",
                 "created_at": t.get("created_at"),
-                "last_posted_at": t.get("last_posted_at"),
-                "views": t.get("views"),
+                "last_posted_at": new_last_posted,
+                "views": views,
                 "posts_count": t.get("posts_count"),
                 "like_count": t.get("like_count"),
                 "excerpt": excerpt,
             })
 
         payload = {
             "source": "preCICE Discourse (FAQ)",
-            "generated_at": datetime.utcnow().isoformat(),
             "topics": topics,
         }
 

tools/fetch-news.py

@@ -1,50 +1,91 @@
 import json
 import os
 import re
-from urllib.request import urlopen
+import time
+import urllib.request
 
 DISCOURSE_URL = "https://precice.discourse.group/c/news/5.json"
-OUTPUT_FILE = "./assets/data/news.json"
+OUTPUT_FILE = "./static/assets/data/news.json"
+VIEW_THRESHOLD = 50
+USER_AGENT = "preCICE-Website-Updater/1.0 (https://precice.org)"
 
 
 def fetch_json(url: str):
-    with urlopen(url) as res:
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req) as res:
         return json.loads(res.read().decode("utf-8"))
 
 
 def strip_html(html: str) -> str:
     return re.sub(r"<[^>]*>", "", html)
 
 
+def load_existing_topics() -> dict:
+    if not os.path.exists(OUTPUT_FILE):
+        return {}
+    try:
+        with open(OUTPUT_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            return {t["id"]: t for t in data.get("topics", []) if "id" in t}
+    except Exception:
+        return {}
+
+
 def main():
     try:
         data = fetch_json(DISCOURSE_URL)
         topics = data.get("topic_list", {}).get("topics", [])
 
+        existing_topics = load_existing_topics()
         news = []
         for topic in topics:
-            detail = fetch_json(f"https://precice.discourse.group/t/{topic['id']}.json")
-            cooked = detail.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "")
-            text = strip_html(cooked).strip()
+            topic_id = topic["id"]
+            new_views = topic.get("views", 0) or 0
+            new_last_posted = topic.get("last_posted_at")
+
+            old_topic = existing_topics.get(topic_id, {})
+            old_views = old_topic.get("views", 0) or 0
+            old_description = old_topic.get("description", "")
+
+            # Preserve view count if difference is below threshold
+            if topic_id in existing_topics:
+                if abs(new_views - old_views) < VIEW_THRESHOLD:
+                    views = old_views
+                else:
+                    views = new_views
+            else:
+                views = new_views
 
-            excerpt = " ".join(text.split()[:30]) + "..."
+            # Only fetch detail if topic is new, updated, or missing description
+            if old_description and old_topic.get("last_posted_at") == new_last_posted:
+                excerpt = old_description
+            else:
+                try:
+                    time.sleep(0.1)
+                    detail = fetch_json(f"https://precice.discourse.group/t/{topic_id}.json")
+                    cooked = detail.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "")
+                    text = strip_html(cooked).strip()
+                    excerpt = " ".join(text.split()[:30]) + "..." if text else old_description
+                except Exception as e:
+                    print(f"Could not fetch detail for news topic {topic_id}: {e}")
+                    excerpt = old_description
 
             news.append({
-                "id": topic["id"],
+                "id": topic_id,
                 "title": topic["title"],
                 "slug": topic["slug"],
-                "url": f"https://precice.discourse.group/t/{topic['slug']}/{topic['id']}",
+                "url": f"https://precice.discourse.group/t/{topic['slug']}/{topic_id}",
                 "created_at": topic.get("created_at"),
-                "last_posted_at": topic.get("last_posted_at"),
+                "last_posted_at": new_last_posted,
                 "like_count": topic.get("like_count"),
                 "posts_count": topic.get("posts_count"),
-                "views": topic.get("views"),
+                "views": views,
                 "description": excerpt,
             })
 
         os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
         with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
-            json.dump({"generated_at": __import__("datetime").datetime.utcnow().isoformat(), "topics": news}, f, indent=2)
+            json.dump({"topics": news}, f, indent=2)
 
         print(f"News data saved to {OUTPUT_FILE}")