diff --git a/.github/workflows/update-discourse-data.yml b/.github/workflows/update-discourse-data.yml index 45a500139c6..3d437c817d5 100644 --- a/.github/workflows/update-discourse-data.yml +++ b/.github/workflows/update-discourse-data.yml @@ -5,26 +5,25 @@ on: schedule: - cron: "0 6 * * *" +concurrency: + group: update-discourse-data + cancel-in-progress: false + jobs: update-discourse-data: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write steps: - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ vars.WEBSITE_UPDATER_CLIENT_ID }} - private-key: ${{ secrets.WEBSITE_UPDATER_KEY }} - - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.14" @@ -33,19 +32,15 @@ jobs: python tools/fetch-news.py python tools/fetch-faq.py - - name: Configure Git author + - name: Commit and push changes run: | + git add static/assets/data/news.json static/assets/data/faq.json + if git diff --cached --quiet; then + echo "Discourse data is already up to date." + exit 0 + fi + git config --local user.name "precice-bot" git config --local user.email "info@precice.org" - - - name: Commit changes (if any) - run: | - git add assets/data/news.json - git add assets/data/faq.json - git commit -m "Update Discourse data data [skip ci]" || echo "No changes to commit" - - - name: Push commit - uses: ad-m/github-push-action@master - with: - github_token: ${{ steps.app-token.outputs.token }} - branch: ${{ github.ref }} + git commit --message "Update Discourse data [skip ci]" + git push origin "HEAD:${GITHUB_REF_NAME}" diff --git a/.github/workflows/update-submodules.yml b/.github/workflows/update-submodules.yml index cdb39006d77..fdbc46d5161 100644 --- a/.github/workflows/update-submodules.yml +++ b/.github/workflows/update-submodules.yml @@ -1,39 +1,67 @@ -name: Update submodules +name: Update Hugo modules + on: workflow_dispatch: schedule: - - cron: "0 0 * * *" + - cron: "5 0 * * *" + +concurrency: + group: update-hugo-modules + cancel-in-progress: false + +env: + GO_VERSION: "1.26.3" + HUGO_VERSION: "0.163.3" + PYTHON_VERSION: "3.14" jobs: update: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: - contents: 'write' - outputs: - success: ${{ steps.commit.outputs.outcome == 'success' }} + contents: write + steps: - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ vars.WEBSITE_UPDATER_CLIENT_ID }} - private-key: ${{ secrets.WEBSITE_UPDATER_KEY }} - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: + fetch-depth: 0 submodules: false - lfs: true - token: ${{ steps.app-token.outputs.token }} - - name: Update submodules - run: git submodule update --init --remote --force - - name: Commit update - id: commit - continue-on-error: true + lfs: false + token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: go.sum + + - name: Set up Hugo + uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3 + with: + hugo-version: ${{ env.HUGO_VERSION }} + extended: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Synchronize Hugo module revisions + run: python3 tools/sync_hugo_modules.py + + - name: Verify updated modules + run: hugo mod verify + + - name: Commit and push update run: | + if git diff --quiet -- go.mod go.sum; then + echo "Hugo modules are already up to date." + exit 0 + fi + git config --local user.name "precice-bot" git config --local user.email "info@precice.org" - git commit -m "Update submodules" -a - - name: Push commit - uses: ad-m/github-push-action@master - with: - github_token: ${{ steps.app-token.outputs.token }} - branch: ${{ github.ref }} + git add go.mod go.sum + git commit --message "Update Hugo modules" + git push origin "HEAD:${GITHUB_REF_NAME}" diff --git a/static/js/forum-fetch.js b/static/js/forum-fetch.js index 403823cc92d..bf7e0a63893 100644 --- a/static/js/forum-fetch.js +++ b/static/js/forum-fetch.js @@ -1,5 +1,8 @@ console.log("forum-fetch.js loaded!"); +// Resolve data from the site root even when the site is served from a subpath. +const siteRoot = new URL("../", document.currentScript.src); + document.addEventListener("DOMContentLoaded", async function () { console.log("FAQ loader running..."); @@ -22,7 +25,7 @@ document.addEventListener("DOMContentLoaded", async function () { } try { - const res = await fetch("/assets/data/faq.json"); + const res = await fetch(new URL("assets/data/faq.json", siteRoot)); if (!res.ok) throw new Error(`HTTP ${res.status}`); const data = await res.json(); @@ -108,8 +111,7 @@ document.addEventListener("DOMContentLoaded", async function () { new Date(t.last_posted_at).toLocaleDateString("en-GB") + " | Replies: " + t.posts_count + - " | Views: " + - t.views; + (t.views !== undefined && t.views !== null ? " | Views: " + t.views : ""); card.appendChild(h4); card.appendChild(excerptP); diff --git a/tools/fetch-faq.py b/tools/fetch-faq.py index ffa668a90e5..211a97a81a3 100644 --- a/tools/fetch-faq.py +++ b/tools/fetch-faq.py @@ -2,16 +2,19 @@ import os import json import re +import time import urllib.request -from datetime import datetime DISCOURSE_BASE = "https://precice.discourse.group" -OUTPUT_FILE = "./assets/data/faq.json" +OUTPUT_FILE = "./static/assets/data/faq.json" +VIEW_THRESHOLD = 50 +USER_AGENT = "preCICE-Website-Updater/1.0 (https://precice.org)" def http_get_json(url: str): """GET URL and return parsed JSON using only stdlib.""" - with urllib.request.urlopen(url) as r: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req) as r: return json.load(r) @@ -19,8 +22,20 @@ def strip_html(text: str) -> str: return re.sub(r"<[^>]+>", "", text) +def load_existing_topics() -> dict: + if not os.path.exists(OUTPUT_FILE): + return {} + try: + with open(OUTPUT_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + return {t["id"]: t for t in data.get("topics", []) if "id" in t} + except Exception: + return {} + + def fetch_excerpt(topic_id: int) -> str: try: + time.sleep(0.1) topic_data = http_get_json(f"{DISCOURSE_BASE}/t/{topic_id}.json") raw = topic_data.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "") cleaned = strip_html(raw) @@ -38,17 +53,41 @@ def fetch_faq(): topic_list = data.get("topic_list", {}).get("topics", []) print(f"Found {len(topic_list)} FAQ topics. Fetching excerpts...") + existing_topics = load_existing_topics() topics = [] for t in topic_list: - excerpt = fetch_excerpt(t["id"]) + topic_id = t["id"] + new_views = t.get("views", 0) or 0 + new_last_posted = t.get("last_posted_at") + + old_topic = existing_topics.get(topic_id, {}) + old_views = old_topic.get("views", 0) or 0 + old_excerpt = old_topic.get("excerpt", "") + + # Preserve view count if difference is below threshold + if topic_id in existing_topics: + if abs(new_views - old_views) < VIEW_THRESHOLD: + views = old_views + else: + views = new_views + else: + views = new_views + + # Only fetch excerpt if topic is new, updated, or missing excerpt + if old_excerpt and old_topic.get("last_posted_at") == new_last_posted: + excerpt = old_excerpt + else: + fetched_excerpt = fetch_excerpt(topic_id) + excerpt = fetched_excerpt if fetched_excerpt else old_excerpt + topics.append({ - "id": t["id"], + "id": topic_id, "title": t["title"], "slug": t["slug"], - "url": f"{DISCOURSE_BASE}/t/{t['slug']}/{t['id']}", + "url": f"{DISCOURSE_BASE}/t/{t['slug']}/{topic_id}", "created_at": t.get("created_at"), - "last_posted_at": t.get("last_posted_at"), - "views": t.get("views"), + "last_posted_at": new_last_posted, + "views": views, "posts_count": t.get("posts_count"), "like_count": t.get("like_count"), "excerpt": excerpt, @@ -56,7 +95,6 @@ def fetch_faq(): payload = { "source": "preCICE Discourse (FAQ)", - "generated_at": datetime.utcnow().isoformat(), "topics": topics, } diff --git a/tools/fetch-news.py b/tools/fetch-news.py index c0d7aca0399..259f13cec08 100644 --- a/tools/fetch-news.py +++ b/tools/fetch-news.py @@ -1,14 +1,18 @@ import json import os import re -from urllib.request import urlopen +import time +import urllib.request DISCOURSE_URL = "https://precice.discourse.group/c/news/5.json" -OUTPUT_FILE = "./assets/data/news.json" +OUTPUT_FILE = "./static/assets/data/news.json" +VIEW_THRESHOLD = 50 +USER_AGENT = "preCICE-Website-Updater/1.0 (https://precice.org)" def fetch_json(url: str): - with urlopen(url) as res: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req) as res: return json.loads(res.read().decode("utf-8")) @@ -16,35 +20,72 @@ def strip_html(html: str) -> str: return re.sub(r"<[^>]*>", "", html) +def load_existing_topics() -> dict: + if not os.path.exists(OUTPUT_FILE): + return {} + try: + with open(OUTPUT_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + return {t["id"]: t for t in data.get("topics", []) if "id" in t} + except Exception: + return {} + + def main(): try: data = fetch_json(DISCOURSE_URL) topics = data.get("topic_list", {}).get("topics", []) + existing_topics = load_existing_topics() news = [] for topic in topics: - detail = fetch_json(f"https://precice.discourse.group/t/{topic['id']}.json") - cooked = detail.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "") - text = strip_html(cooked).strip() + topic_id = topic["id"] + new_views = topic.get("views", 0) or 0 + new_last_posted = topic.get("last_posted_at") + + old_topic = existing_topics.get(topic_id, {}) + old_views = old_topic.get("views", 0) or 0 + old_description = old_topic.get("description", "") + + # Preserve view count if difference is below threshold + if topic_id in existing_topics: + if abs(new_views - old_views) < VIEW_THRESHOLD: + views = old_views + else: + views = new_views + else: + views = new_views - excerpt = " ".join(text.split()[:30]) + "..." + # Only fetch detail if topic is new, updated, or missing description + if old_description and old_topic.get("last_posted_at") == new_last_posted: + excerpt = old_description + else: + try: + time.sleep(0.1) + detail = fetch_json(f"https://precice.discourse.group/t/{topic_id}.json") + cooked = detail.get("post_stream", {}).get("posts", [{}])[0].get("cooked", "") + text = strip_html(cooked).strip() + excerpt = " ".join(text.split()[:30]) + "..." if text else old_description + except Exception as e: + print(f"Could not fetch detail for news topic {topic_id}: {e}") + excerpt = old_description news.append({ - "id": topic["id"], + "id": topic_id, "title": topic["title"], "slug": topic["slug"], - "url": f"https://precice.discourse.group/t/{topic['slug']}/{topic['id']}", + "url": f"https://precice.discourse.group/t/{topic['slug']}/{topic_id}", "created_at": topic.get("created_at"), - "last_posted_at": topic.get("last_posted_at"), + "last_posted_at": new_last_posted, "like_count": topic.get("like_count"), "posts_count": topic.get("posts_count"), - "views": topic.get("views"), + "views": views, "description": excerpt, }) os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True) with open(OUTPUT_FILE, "w", encoding="utf-8") as f: - json.dump({"generated_at": __import__("datetime").datetime.utcnow().isoformat(), "topics": news}, f, indent=2) + json.dump({"topics": news}, f, indent=2) print(f"News data saved to {OUTPUT_FILE}") diff --git a/tools/sync_hugo_modules.py b/tools/sync_hugo_modules.py new file mode 100644 index 00000000000..5acf5f915f1 --- /dev/null +++ b/tools/sync_hugo_modules.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Synchronize Hugo module revisions with their upstream default branches.""" + +import argparse +from pathlib import Path +import re +import subprocess +import sys + + +PRECICE_MODULE_PREFIX = "github.com/precice/" +IMPORT_HEADER = "[[imports]]" +IMPORT_PATH = re.compile(r'^path\s*=\s*"([^"]+)"\s*$') + + +def run(command: list[str], *, cwd: Path, capture_output: bool = False) -> str: + completed = subprocess.run( + command, + cwd=cwd, + check=False, + text=True, + capture_output=capture_output, + ) + if completed.returncode: + if capture_output: + sys.stderr.write(completed.stderr) + raise RuntimeError(f"Command failed: {' '.join(command)}") + return completed.stdout if capture_output else "" + + +def hugo_imports(module_toml: Path) -> list[str]: + imports: list[str] = [] + in_import = False + + for raw_line in module_toml.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if line.startswith("[["): + in_import = line == IMPORT_HEADER + continue + if not in_import: + continue + match = IMPORT_PATH.match(line) + if match: + imports.append(match.group(1)) + in_import = False + + return imports + + +def module_revisions(repository: Path, module_toml: Path) -> list[tuple[str, str]]: + revisions: list[tuple[str, str]] = [] + + for module in hugo_imports(module_toml): + if not module.startswith(PRECICE_MODULE_PREFIX): + continue + output = run( + ["git", "ls-remote", f"https://{module}.git", "HEAD"], + cwd=repository, + capture_output=True, + ) + for line in output.splitlines(): + fields = line.split() + if len(fields) == 2 and fields[1] == "HEAD": + revisions.append((module, fields[0])) + break + else: + raise RuntimeError(f"Could not resolve the default branch for {module}") + + return revisions + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Synchronize Hugo module revisions with upstream default branches." + ) + parser.add_argument("--module-toml", default="config/_default/module.toml") + parser.add_argument("--hugo-bin", default="hugo") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + repository = Path.cwd() + revisions = module_revisions(repository, repository / args.module_toml) + dependencies = [f"{module}@{revision}" for module, revision in revisions] + + for dependency in dependencies: + print(f"Synchronizing {dependency}") + + if args.dry_run: + return + + run([args.hugo_bin, "mod", "get", *dependencies], cwd=repository) + run([args.hugo_bin, "mod", "tidy"], cwd=repository) + + +if __name__ == "__main__": + try: + main() + except (OSError, RuntimeError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + sys.exit(1)