Add Hlido Agent Trust guardrail (closes #1689)

plugins/hlido/agentTrust.ts

@@ -0,0 +1,73 @@
+import {
+  HookEventType,
+  PluginContext,
+  PluginHandler,
+  PluginParameters,
+} from '../types';
+import { getScorecard } from './globals';
+
+export const handler: PluginHandler = async (
+  context: PluginContext,
+  parameters: PluginParameters,
+  _eventType: HookEventType
+) => {
+  let error = null;
+  let verdict = true;
+  let data: any = null;
+
+  try {
+    const slug: string | undefined =
+      parameters.slug || (context.provider as string | undefined);
+    if (!slug) {
+      return {
+        error: new Error('hlido: no `slug` parameter and no request provider to check'),
+        verdict: true,
+        data: null,
+      };
+    }
+
+    const minScore =
+      typeof parameters.minScore === 'number' ? parameters.minScore : 70;
+    const failOnUnreviewed = parameters.failOnUnreviewed === true;
+
+    const scorecard = await getScorecard(slug, parameters.timeout);
+
+    if (!scorecard) {
+      verdict = !failOnUnreviewed;
+      data = {
+        slug,
+        reviewed: false,
+        explanation: `No independent Hlido review for "${slug}". ${
+          failOnUnreviewed ? 'Blocked (failOnUnreviewed).' : 'Passed (no review to gate on).'
+        }`,
+        check_url: `https://hlido.eu/check/?agent=${encodeURIComponent(slug)}`,
+      };
+      return { error, verdict, data };
+    }
+
+    const score = typeof scorecard.score === 'number' ? scorecard.score : null;
+    verdict = score !== null ? score >= minScore : !failOnUnreviewed;
+    data = {
+      slug,
+      reviewed: true,
+      score,
+      tier: scorecard.tier ?? null,
+      minScore,
+      passed: verdict,
+      explanation:
+        score !== null
+          ? `Hlido trust score ${score} (${scorecard.tier}) vs minimum ${minScore} → ${
+              verdict ? 'passed' : 'blocked'
+            }.`
+          : `Hlido review exists but has no numeric score; ${
+              verdict ? 'passed' : 'blocked'
+            }.`,
+      review_url: `https://hlido.eu/reviews/${encodeURIComponent(slug)}/`,
+    };
+  } catch (e: any) {
+    delete e.stack;
+    error = e;
+  }
+
+  return { error, verdict, data };
+};

plugins/hlido/globals.ts

@@ -0,0 +1,32 @@
+export const HLIDO_BASE_URL = 'https://hlido.eu';
+
+export interface HlidoScorecard {
+  slug: string;
+  name?: string;
+  score?: number | null;
+  tier?: string | null;
+  staleness_after?: string | null;
+}
+
+// Fetch a public Hlido scorecard. Public + CORS-open + no auth; returns null on
+// 404 (unreviewed) so the caller can distinguish "unreviewed" from "error".
+export const getScorecard = async (
+  slug: string,
+  timeout?: number
+): Promise<HlidoScorecard | null> => {
+  const controller = new AbortController();
+  const id = timeout
+    ? setTimeout(() => controller.abort(), timeout)
+    : undefined;
+  try {
+    const res = await fetch(
+      `${HLIDO_BASE_URL}/data/scorecards/${encodeURIComponent(slug)}.json`,
+      { signal: controller.signal }
+    );
+    if (res.status === 404) return null;
+    if (!res.ok) throw new Error(`Hlido API returned ${res.status}`);
+    return (await res.json()) as HlidoScorecard;
+  } finally {
+    if (id) clearTimeout(id);
+  }
+};

plugins/hlido/hlido.test.ts

@@ -0,0 +1,45 @@
+import { handler as agentTrust } from './agentTrust';
+import { PluginContext, PluginParameters } from '../types';
+
+// These tests hit the public Hlido API (no auth). `aider` is a long-standing
+// high-scoring reviewed agent; a random slug is used for the unreviewed path.
+describe('hlido agentTrust guardrail', () => {
+  const context: PluginContext = { requestType: 'chatComplete' };
+
+  it('passes when the agent scores at or above the threshold', async () => {
+    const parameters: PluginParameters = { slug: 'aider', minScore: 50 };
+    const result = await agentTrust(context, parameters, 'beforeRequestHook');
+    expect(result.error).toBeNull();
+    expect(result.verdict).toBe(true);
+    expect(result.data.reviewed).toBe(true);
+    expect(typeof result.data.score).toBe('number');
+  });
+
+  it('blocks when the agent scores below the threshold', async () => {
+    const parameters: PluginParameters = { slug: 'aider', minScore: 101 };
+    const result = await agentTrust(context, parameters, 'beforeRequestHook');
+    expect(result.error).toBeNull();
+    expect(result.verdict).toBe(false);
+  });
+
+  it('passes an unreviewed agent by default', async () => {
+    const parameters: PluginParameters = {
+      slug: 'this-agent-does-not-exist-xyz',
+    };
+    const result = await agentTrust(context, parameters, 'beforeRequestHook');
+    expect(result.error).toBeNull();
+    expect(result.verdict).toBe(true);
+    expect(result.data.reviewed).toBe(false);
+  });
+
+  it('blocks an unreviewed agent when failOnUnreviewed is true', async () => {
+    const parameters: PluginParameters = {
+      slug: 'this-agent-does-not-exist-xyz',
+      failOnUnreviewed: true,
+    };
+    const result = await agentTrust(context, parameters, 'beforeRequestHook');
+    expect(result.error).toBeNull();
+    expect(result.verdict).toBe(false);
+    expect(result.data.reviewed).toBe(false);
+  });
+});

plugins/hlido/manifest.json

@@ -0,0 +1,62 @@
+{
+  "id": "hlido",
+  "name": "Hlido Agent Trust",
+  "description": "Gate LLM/agent requests on Hlido's independent, evidence-backed trust scores for shipped AI agent products. No API key required — Hlido's review data is public. Blocks (or flags) requests routed to an agent that scores below your threshold or has no independent review.",
+  "credentials": {
+    "type": "object",
+    "properties": {},
+    "required": []
+  },
+  "functions": [
+    {
+      "name": "Agent Trust Check",
+      "id": "agentTrust",
+      "supportedHooks": ["beforeRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Verify the target agent/model against its independent Hlido trust score before the request runs. Uses the public Hlido review corpus (no auth)."
+        }
+      ],
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "slug": {
+            "type": "string",
+            "label": "Hlido agent slug",
+            "description": [
+              {
+                "type": "subHeading",
+                "text": "The Hlido slug of the agent/tool to verify (e.g. 'aider'). Look it up at https://hlido.eu/check/. If omitted, the request provider name is used as a best-effort match."
+              }
+            ]
+          },
+          "minScore": {
+            "type": "number",
+            "label": "Minimum trust score",
+            "description": [
+              {
+                "type": "subHeading",
+                "text": "Fail the guardrail when the agent's Hlido score (0–100) is below this. Default 70 (STEADY tier)."
+              }
+            ],
+            "default": 70
+          },
+          "failOnUnreviewed": {
+            "type": "boolean",
+            "label": "Fail if unreviewed",
+            "description": [
+              {
+                "type": "subHeading",
+                "text": "When true, fail the guardrail if the agent has no independent Hlido review. When false (default), unreviewed agents pass with a note."
+              }
+            ],
+            "default": false
+          }
+        },
+        "required": ["slug"]
+      }
+    }
+  ]
+}

plugins/index.ts

@@ -47,6 +47,7 @@ import { handler as pangeatextGuard } from './pangea/textGuard';
 import { handler as promptfooPii } from './promptfoo/pii';
 import { handler as promptfooHarm } from './promptfoo/harm';
 import { handler as promptfooGuard } from './promptfoo/guard';
+import { handler as hlidoAgentTrust } from './hlido/agentTrust';
 import { handler as pangeapii } from './pangea/pii';
 import { pluginHandler as bedrockHandler } from './bedrock/index';
 import { handler as acuvityScan } from './acuvity/scan';
@@ -138,6 +139,9 @@ export const plugins = {
     textGuard: pangeatextGuard,
     pii: pangeapii,
   },
+  hlido: {
+    agentTrust: hlidoAgentTrust,
+  },
   promptfoo: {
     pii: promptfooPii,
     harm: promptfooHarm,