Self-hosted: static SEO files, canonical policy and responsive media

apps/self-hosted/hosting/api/package.json

@@ -11,17 +11,18 @@
     "test": "vitest run"
   },
   "dependencies": {
+    "@ecency/render-helper": "^2.5.26",
     "@ecency/sdk": "^2.2.0",
     "@hiveio/x402": "^0.1.2",
     "@hono/node-server": "^2.0.11",
     "@hono/zod-validator": "^0.4.0",
+    "date-fns": "^3.6.0",
     "hono": "^4.4.0",
     "jsonwebtoken": "^9.0.0",
+    "nanoid": "^5.0.0",
     "pg": "^8.12.0",
     "redis": "^4.6.0",
-    "zod": "^3.23.0",
-    "date-fns": "^3.6.0",
-    "nanoid": "^5.0.0"
+    "zod": "^3.23.0"
   },
   "devDependencies": {
     "@types/jsonwebtoken": "^9.0.0",

apps/self-hosted/hosting/api/src/services/config-publish-lock.test.ts

@@ -16,7 +16,10 @@ vi.mock('../db/client', () => ({
 }));
 
 vi.mock('./tenant-service', () => ({
-  TenantService: { getByUsername: mocks.getByUsername },
+  TenantService: {
+    getByUsername: mocks.getByUsername,
+    getBlogUrl: (t: any) => `https://${t.username}.blogs.ecency.com`,
+  },
 }));
 
 const { ConfigService } = await import('./config-service');

apps/self-hosted/hosting/api/src/services/config-service.ts

@@ -7,19 +7,23 @@
 import { promises as fs } from 'fs';
 import path from 'path';
 import { withAdvisoryLock } from '../db/client';
+import { escapeHtml } from '../utils/escape-html';
 import { TenantService, type Tenant } from './tenant-service';
+import {
+  buildRobotsTxt,
+  buildRssXml,
+  buildSitemapXml,
+  canonicalHomeUrl,
+  fetchTenantPosts,
+  SEO_FRESH_MS,
+} from './seo-files';
 
 const CONFIG_DIR = process.env.CONFIG_DIR || '/app/configs';
 
-/** Escape a string for safe interpolation into HTML text and attribute values. */
-export function escapeHtml(value: string): string {
-  return value
-    .replace(/&/g, '&')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#39;');
-}
+// Re-exported for existing consumers; the implementation lives in its own
+// util so seo-files can use it without importing this module back
+// (config-service consumes seo-files below).
+export { escapeHtml } from '../utils/escape-html';
 
 // Per-tenant write chains: every config-file write for a tenant runs strictly after the
 // previous one, so the periodic sync and a concurrent tenant update can never interleave.
@@ -178,6 +182,11 @@ export const ConfigService = {
           ]
         : []),
       `<meta name="twitter:card" content="${ogImage ? 'summary_large_image' : 'summary'}" />`,
+      // Custom-domain tenants canonicalize to themselves, subdomain tenants
+      // to the ecency.com SSR page (see canonicalHomeUrl); the feed link is
+      // in the SSI snippet so crawlers see it, not only JS runtimes.
+      `<link rel="canonical" href="${escapeHtml(canonicalHomeUrl(tenant))}" />`,
+      `<link rel="alternate" type="application/rss+xml" title="${title}" href="${escapeHtml(TenantService.getBlogUrl(tenant) + '/rss.xml')}" />`,
       `<link rel="icon" href="${favicon}" />`,
       '',
     ].join('\n');
@@ -188,6 +197,56 @@ export const ConfigService = {
     return path.join(CONFIG_DIR, username.toLowerCase() + '.meta.html');
   },
 
+  /** Paths of a tenant's static SEO files (robots, sitemap, rss). */
+  getSeoPaths(username: string): { robots: string; sitemap: string; rss: string } {
+    const base = path.join(CONFIG_DIR, username.toLowerCase());
+    return {
+      robots: `${base}.robots.txt`,
+      sitemap: `${base}.sitemap.xml`,
+      rss: `${base}.rss.xml`,
+    };
+  },
+
+  /**
+   * Regenerate a tenant's static SEO files when they are stale. Called from
+   * the sync pass inside the per-tenant lock. Freshness is mtime-based, so
+   * after an unchanged regeneration the files are touched: writeIfChanged
+   * deliberately skips identical writes, and without the touch every later
+   * pass would refetch the feed for a blog that has not posted.
+   */
+  async writeSeoFilesIfStale(tenant: Tenant): Promise<void> {
+    const paths = this.getSeoPaths(tenant.username);
+    const all = [paths.robots, paths.sitemap, paths.rss];
+
+    let fresh = true;
+    for (const p of all) {
+      try {
+        const stat = await fs.stat(p);
+        if (Date.now() - stat.mtimeMs > SEO_FRESH_MS) {
+          fresh = false;
+          break;
+        }
+      } catch {
+        fresh = false;
+        break;
+      }
+    }
+    if (fresh) return;
+
+    // One bounded chain page; a failure here is caught by the sync pass's
+    // per-tenant isolation and yesterday's files keep serving.
+    const posts = await fetchTenantPosts(tenant);
+    await fs.mkdir(CONFIG_DIR, { recursive: true });
+    await this.writeIfChanged(paths.robots, buildRobotsTxt(tenant));
+    await this.writeIfChanged(paths.sitemap, buildSitemapXml(tenant, posts));
+    await this.writeIfChanged(paths.rss, buildRssXml(tenant, posts));
+
+    const now = new Date();
+    for (const p of all) {
+      await fs.utimes(p, now, now).catch(() => {});
+    }
+  },
+
   /**
    * Delete config file for a tenant. Serialized through the same per-tenant lock as
    * writes, so an in-flight sync write cannot resurrect the file after deletion.
@@ -198,7 +257,14 @@ export const ConfigService = {
 
   /** The actual unlink; only ever invoked under the per-tenant lock. */
   async removeConfigFileUnlocked(username: string): Promise<void> {
-    for (const filePath of [this.getConfigPath(username), this.getMetaPath(username)]) {
+    const seo = this.getSeoPaths(username);
+    for (const filePath of [
+      this.getConfigPath(username),
+      this.getMetaPath(username),
+      seo.robots,
+      seo.sitemap,
+      seo.rss,
+    ]) {
       try {
         await fs.unlink(filePath);
         console.log('[ConfigService] Deleted config:', filePath);
@@ -253,6 +319,9 @@ export const ConfigService = {
       for (const f of files) {
         if (f === 'default.json') continue;
         if (f.endsWith('.meta.html')) names.add(f.slice(0, -'.meta.html'.length));
+        else if (f.endsWith('.robots.txt')) names.add(f.slice(0, -'.robots.txt'.length));
+        else if (f.endsWith('.sitemap.xml')) names.add(f.slice(0, -'.sitemap.xml'.length));
+        else if (f.endsWith('.rss.xml')) names.add(f.slice(0, -'.rss.xml'.length));
         else if (f.endsWith('.json')) names.add(f.slice(0, -'.json'.length));
       }
       return [...names];
@@ -298,6 +367,7 @@ export const ConfigService = {
           const fresh = await TenantService.getByUsername(tenant.username);
           if (!fresh || !isPublishableTenant(fresh)) return;
           await this.writeConfigFile(fresh);
+          await this.writeSeoFilesIfStale(fresh);
         });
       } catch (err) {
         failed++;

apps/self-hosted/hosting/api/src/services/hivesigner-registry.test.ts

@@ -36,6 +36,7 @@ vi.mock('../db/client', () => ({
 
 vi.mock('./tenant-service', () => ({
   TenantService: {
+    getBlogUrl: (t: any) => `https://${t.username}.blogs.ecency.com`,
     getActiveTenants: mocks.getActiveTenants,
     applyConfigDocument: mocks.applyConfigDocument,
     getByUsername: mocks.getByUsername,

apps/self-hosted/hosting/api/src/services/post-meta.test.ts

@@ -74,8 +74,10 @@ describe('buildMetaForUri', () => {
     });
     expect(html).not.toContain('<script>');
     expect(html).toContain('og:type" content="article"');
-    expect(html).toContain(
-      'og:image" content="https://i.ecency.com/1200x630/https://img.example/meta.png"',
+    // render-helper's modern proxy path: a hashed /p/ URL with explicit
+    // dimensions, not the legacy redirecting /WxH/ route.
+    expect(html).toMatch(
+      /og:image" content="https:\/\/i\.ecency\.com\/p\/[A-Za-z0-9]+\?format=match&amp;mode=fit&amp;width=1200&amp;height=630"/,
     );
     expect(html).toContain('Some words to read here');
     expect(html).toContain(

apps/self-hosted/hosting/api/src/services/post-meta.ts

@@ -1,7 +1,24 @@
+import { createRequire } from 'node:module';
 import { callRPC } from '@ecency/sdk/hive';
+
+// render-helper through its CJS build: the package's node ESM entry carries
+// a directory import ('remarkable/linkify') that Node refuses, so the ESM
+// path crashes at module load. Tracked as a render-helper packaging fix;
+// until it ships, CJS resolution handles the directory import fine.
+const requireCjs = createRequire(import.meta.url);
+const { catchPostImage } = requireCjs('@ecency/render-helper') as {
+  catchPostImage: (
+    entry: unknown,
+    width?: number,
+    height?: number,
+    format?: string,
+  ) => string | null;
+};
 import type { Tenant } from '../types';
 import { ConfigService, escapeHtml } from './config-service';
 import { TenantService } from './tenant-service';
+import { canonicalPostUrl } from './seo-files';
+import { excerptOf } from '../utils/excerpt';
 
 /**
  * Per-post head metadata for link unfurls. The SPA sets OG tags client-side,
@@ -85,43 +102,6 @@ async function getPostCached(author: string, permlink: string): Promise<any | nu
   return post;
 }
 
-/** Strip markdown/html noise the way an excerpt should read. */
-function excerptOf(body: unknown, max = 200): string {
-  if (typeof body !== 'string') return '';
-  const text = body
-    .replace(/```[\s\S]*?```/g, ' ')
-    .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
-    .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
-    .replace(/<[^>]*>/g, ' ')
-    .replace(/^#{1,6}\s+/gm, '')
-    .replace(/[`*_>~|]/g, ' ')
-    .replace(/https?:\/\/\S+/g, ' ')
-    .replace(/\s+/g, ' ')
-    .trim();
-  return text.length > max ? `${text.slice(0, max - 1).trimEnd()}…` : text;
-}
-
-/** The post's cover: json_metadata image first, then the first body image. */
-function coverOf(post: any): string | null {
-  const metaImage = post?.json_metadata?.image?.[0];
-  if (typeof metaImage === 'string' && /^https?:\/\//i.test(metaImage)) {
-    return metaImage;
-  }
-  const body = typeof post?.body === 'string' ? post.body : '';
-  const md = /!\[[^\]]*\]\((https?:\/\/[^\s)]+)\)/.exec(body);
-  if (md) return md[1];
-  const html = /<img[^>]+src=["'](https?:\/\/[^\s"']+)["']/.exec(body);
-  if (html) return html[1];
-  return null;
-}
-
-function proxyBaseOf(tenant: Tenant): string {
-  const configured = (tenant.config as any)?.configuration?.general?.imageProxy;
-  return typeof configured === 'string' && /^https?:\/\//i.test(configured)
-    ? configured.replace(/\/+$/, '')
-    : 'https://i.ecency.com';
-}
-
 /**
  * The head snippet for one post. Falls back to the tenant snippet whenever
  * the URI is not a post or the post cannot be resolved, so this endpoint
@@ -149,12 +129,12 @@ export async function buildMetaForUri(tenant: Tenant, uri: unknown): Promise<str
     excerptOf(post.body) || `A post by @${parsed.author}`,
   );
 
-  const coverRaw = coverOf(post);
-  // Unfurl targets get a crawler-friendly size through the image proxy; the
-  // raw URL is chain-authored and untrusted, so it is escaped like the rest.
-  const ogImage = coverRaw
-    ? escapeHtml(`${proxyBaseOf(tenant)}/1200x630/${coverRaw}`)
-    : null;
+  // render-helper's own cover extraction and proxying, the same pipeline the
+  // apps render with: it understands string-form metadata, entities and code
+  // fences, and emits the modern proxy path instead of the legacy sized
+  // route that answers with a redirect. Chain-authored, so escaped.
+  const coverProxied = catchPostImage(post, 1200, 630, 'match');
+  const ogImage = coverProxied ? escapeHtml(coverProxied) : null;
 
   const canonical = escapeHtml(
     `${TenantService.getBlogUrl(tenant)}/@${parsed.author}/${parsed.permlink}`,
@@ -175,6 +155,9 @@ export async function buildMetaForUri(tenant: Tenant, uri: unknown): Promise<str
         ]
       : []),
     `<meta name="twitter:card" content="${ogImage ? 'summary_large_image' : 'summary'}" />`,
+    // Same canonical policy as the tenant snippet: the owner's own domain
+    // when one is verified, the ecency.com SSR post otherwise.
+    `<link rel="canonical" href="${escapeHtml(canonicalPostUrl(tenant, parsed.author, parsed.permlink))}" />`,
     '',
   ].join('\n');
 }