Add SQL filters to metadata backfill and correct diagnostic URLs

src/api/video.test.ts

@@ -130,8 +130,10 @@ test("proxy login-required detail response hands off to the authenticated direct
   assert.deepEqual(warnings, []);
 });
 
-test("a direct login-required detail response after proxy handoff still fails", async () => {
+test("direct detail failure diagnostics omit an absent video identifier", async () => {
   const originalAdapter = webInterfaceClient.defaults.adapter;
+  const originalConsoleError = console.error;
+  const errors: string[] = [];
   const restoreWbiState = setTestWbiState();
   const proxyAdapter: AxiosAdapter = async (request) => ({
     config: request,
@@ -141,22 +143,45 @@ test("a direct login-required detail response after proxy handoff still fails",
     statusText: "OK",
   });
   const directClient = axios.create();
-  directClient.defaults.adapter = async (request) => ({
-    config: request,
-    data: response(-403),
-    headers: {},
-    status: 200,
-    statusText: "OK",
-  });
+  const requestUris: string[] = [];
+  directClient.defaults.adapter = async (request) => {
+    requestUris.push(directClient.getUri(request));
+    return {
+      config: request,
+      data: response(-403),
+      headers: {},
+      status: 200,
+      statusText: "OK",
+    };
+  };
 
   webInterfaceClient.defaults.adapter = proxyAdapter;
+  console.error = (...args: unknown[]) => errors.push(args.join(" "));
   try {
-    await assert.rejects(
-      fetchVideoFullDetail({ bvid: "BVdirect-failure" }, directClient),
-      /Fetch video full detail failed/,
-    );
+    for (const params of [
+      { bvid: "BVdirect-failure" },
+      { aid: 123 },
+    ] as const) {
+      await assert.rejects(
+        fetchVideoFullDetail(params, directClient),
+        /Fetch video full detail failed/,
+      );
+    }
   } finally {
     webInterfaceClient.defaults.adapter = originalAdapter;
+    console.error = originalConsoleError;
     restoreWbiState();
   }
+
+  assert.deepEqual(requestUris, [
+    "/view/detail?bvid=BVdirect-failure",
+    "/view/detail?aid=123",
+  ]);
+  const diagnosticUrls = errors
+    .filter((message) => message.includes("API Error for URL:"))
+    .map(
+      (message) =>
+        message.replace(/^.*API Error for URL: /, "").split(" Error:", 1)[0],
+    );
+  assert.deepEqual(diagnosticUrls, requestUris);
 });

src/api/video.ts

@@ -140,10 +140,7 @@ export const fetchVideoFullDetail = async (
       return null;
     }
 
-    const baseUrl = directClient.defaults.baseURL || "";
-    const fullUrl = `${baseUrl}${endpoint}?bvid=${params.bvid || ""}&aid=${
-      params.aid || ""
-    }`;
+    const fullUrl = directClient.getUri({ url: endpoint, params });
     logger.error(`API Error for URL: ${fullUrl}`, error);
     if (error instanceof Error) {
       logger.error(error.stack);

src/database/videos.test.ts

@@ -7,6 +7,7 @@ import { backfillMissionIds } from "./schema/videos";
 import {
   getProcessedVideoAids,
   getProcessedVideoAidsMissingPidV2,
+  getProcessedVideoMetadataCandidates,
   markVideoDeleted,
   markVideoProcessedWithCollectionState,
   markVideosProcessedWithCollectionState,
@@ -211,6 +212,40 @@ test("missing pid lookup queries only a bounded AID set", async () => {
   assert.deepEqual(calls[0]?.values, [["1", "2"]]);
 });
 
+test("metadata candidates combine a raw predicate with existing bounds", async () => {
+  const calls: QueryCall[] = [];
+  const pool = {
+    async query(sql: string, values?: unknown[]) {
+      calls.push({ sql, values });
+      return { rows: [], rowCount: 0 };
+    },
+  } as unknown as Pool;
+  const createdBefore = new Date("2026-09-22T00:00:00Z");
+
+  await getProcessedVideoMetadataCandidates(pool, {
+    afterAid: 2_746_490n,
+    throughAid: 9_999_999n,
+    createdBefore,
+    onlyMissingPidV2: false,
+    where: "aid >= 2746491 OR pid_v2 IS NULL",
+    limit: 100,
+  });
+
+  assert.match(calls[0]?.sql ?? "", /AND \(aid >= 2746491 OR pid_v2 IS NULL\)/);
+  assert.match(calls[0]?.sql ?? "", /aid > \$1::bigint/);
+  assert.match(
+    calls[0]?.sql ?? "",
+    /\(\$4::boolean = false OR pid_v2 IS NULL\)/,
+  );
+  assert.deepEqual(calls[0]?.values, [
+    "2746490",
+    "9999999",
+    createdBefore,
+    false,
+    100,
+  ]);
+});
+
 test("recommendation refresh deduplicates inputs and preserves absent card fields", async () => {
   const calls: QueryCall[] = [];
   const query = {

src/database/videos.ts

@@ -556,6 +556,7 @@ export interface ProcessedVideoMetadataSweep {
   createdBefore: Date;
   onlyMissingPidV2?: boolean;
   throughAid: bigint;
+  where?: string;
   limit: number;
 }
 
@@ -579,13 +580,14 @@ export async function getProcessedVideoMetadataCandidates(
   pool: Pool,
   options: ProcessedVideoMetadataSweep,
 ): Promise<ProcessedVideoMetadataCandidate[]> {
+  const whereClause = options.where ? `\n       AND (${options.where})` : "";
   const result = await pool.query(
     `SELECT aid, bvid, pid_v2
      FROM processed_videos
      WHERE aid > $1::bigint
        AND aid <= $2::bigint
        AND created_at <= $3::timestamptz
-       AND ($4::boolean = false OR pid_v2 IS NULL)
+       AND ($4::boolean = false OR pid_v2 IS NULL)${whereClause}
      ORDER BY aid ASC
      LIMIT $5`,
     [

src/index.ts

@@ -97,14 +97,18 @@ async function main() {
   const args = process.argv.slice(2);
 
   if (args.includes("--update-info")) {
-    const { parsePidV2Whitelist, runUpdateInfo } = await import(
-      "./scripts/update-info"
-    );
+    const {
+      parsePidV2Whitelist,
+      parseUpdateInfoPredicateArgument,
+      runUpdateInfo,
+    } = await import("./scripts/update-info");
     const whitelistValue =
       updateInfoWhitelistArgument(args) ??
       config.processing.filtering.pidV2Whitelist.join(",");
+    const where = parseUpdateInfoPredicateArgument(args);
     await runUpdateInfo({
       pidV2Whitelist: parsePidV2Whitelist(whitelistValue),
+      ...(where === undefined ? {} : { where }),
     });
     return;
   }

src/scripts/update-info.test.ts

@@ -11,7 +11,11 @@ import type {
   RecommendedVideo,
   VideoData,
 } from "../types/index.js";
-import { parsePidV2Whitelist, runUpdateInfo } from "./update-info.js";
+import {
+  parsePidV2Whitelist,
+  parseUpdateInfoPredicateArgument,
+  runUpdateInfo,
+} from "./update-info.js";
 
 function related(
   aid: number,
@@ -86,6 +90,11 @@ function video(aid: number): VideoData {
 class FakeDatabase {
   closed = false;
   calls = { dictionary: 0, membership: 0, refresh: 0, persist: 0 };
+  readonly candidateRequests: Array<{
+    afterAid: bigint;
+    onlyMissingPidV2?: boolean;
+    where?: string;
+  }> = [];
   readonly pidV2Names = new Map<number, string>();
   readonly pidV2ByAid = new Map<number, number>();
   constructor(
@@ -102,7 +111,9 @@ class FakeDatabase {
   async getProcessedVideoMetadataCandidates(options: {
     afterAid: bigint;
     onlyMissingPidV2?: boolean;
+    where?: string;
   }) {
+    this.candidateRequests.push(options);
     return this.sources
       .filter((aid) => BigInt(aid) > options.afterAid)
       .filter((aid) => !options.onlyMissingPidV2 || !this.pidV2ByAid.has(aid))
@@ -212,6 +223,30 @@ test("concurrent collectors share the configured capacity for source and admissi
   assert.equal(admissionRequests, 50);
 });
 
+test("manual updater passes its source predicate to both sweeps", async () => {
+  const database = new FakeDatabase([4, 5]);
+  const where = "aid >= 4 AND pid_v2 IS NULL";
+  const fetched: number[] = [];
+  const result = await runUpdateInfo({
+    where,
+    database,
+    detailsService: parser,
+    pidV2Whitelist: new Set(),
+    onProgress: () => {},
+    fetchDetail: async (id) => {
+      const aid = Number(String(id).replace("BV", ""));
+      fetched.push(aid);
+      return detail(aid, 10);
+    },
+  });
+  assert.equal(result.scanned, 2);
+  assert.deepEqual(fetched, [4, 5, 4, 5]);
+  assert.deepEqual(
+    database.candidateRequests.map((request) => request.where),
+    [where, where],
+  );
+});
+
 test("manual updater preserves the original AID cutoff and closes the database", async () => {
   const database = new FakeDatabase([1, 2]);
   const result = await runUpdateInfo({
@@ -222,6 +257,10 @@ test("manual updater preserves the original AID cutoff and closes the database",
     fetchDetail: async (id) => detail(Number(String(id).replace("BV", "")), 10),
   });
   assert.equal(result.scanned, 2);
+  assert.deepEqual(
+    database.candidateRequests.map((request) => request.where),
+    [undefined, undefined],
+  );
   assert.equal(database.closed, true);
 });
 
@@ -291,6 +330,41 @@ test("partial related cards refresh only their supplied existing fields", async
   assert.deepEqual(refreshed, { aid: 2n, description: "", pidV2: 7 });
 });
 
+test("update-info predicate parser accepts pnpm forwarding and ignores whitelist values", () => {
+  const predicate = "aid >= 2746491 AND pid_v2 IS NULL";
+  assert.equal(
+    parseUpdateInfoPredicateArgument([
+      "--update-info",
+      "--",
+      predicate,
+      "--pid-v2-whitelist",
+      "1003,1005,1007",
+    ]),
+    predicate,
+  );
+  assert.equal(
+    parseUpdateInfoPredicateArgument(["--update-info", predicate]),
+    predicate,
+  );
+  assert.equal(
+    parseUpdateInfoPredicateArgument([
+      "--update-info",
+      "--pid-v2-whitelist",
+      "1003,1005,1007",
+    ]),
+    undefined,
+  );
+  assert.equal(
+    parseUpdateInfoPredicateArgument([
+      "--update-info",
+      "--",
+      "--pid-v2-whitelist",
+      "1003,1005,1007",
+    ]),
+    undefined,
+  );
+});
+
 test("whitelist parser rejects malformed explicit values", () => {
   assert.deepEqual(parsePidV2Whitelist(undefined), new Set());
   assert.deepEqual(parsePidV2Whitelist("7, 11"), new Set([7, 11]));

src/scripts/update-info.ts

@@ -29,6 +29,7 @@ interface UpdateInfoDatabase extends RecommendationRefreshDatabase {
 
 export interface UpdateInfoOptions {
   afterAid?: bigint;
+  where?: string;
   database?: UpdateInfoDatabase;
   detailsService?: RecommendationDetailParser;
   fetchDetail?: (
@@ -55,6 +56,16 @@ function validPidV2(value: unknown): value is number {
   );
 }
 
+/** Read the optional positional source predicate used by the update-info CLI. */
+export function parseUpdateInfoPredicateArgument(
+  args: string[],
+): string | undefined {
+  const updateInfoIndex = args.indexOf("--update-info");
+  const following = args[updateInfoIndex + 1];
+  const candidate = following === "--" ? args[updateInfoIndex + 2] : following;
+  return candidate && !candidate.startsWith("--") ? candidate : undefined;
+}
+
 /** Parse a comma-separated list of explicit pid_v2 values. */
 export function parsePidV2Whitelist(value: string | undefined): Set<number> {
   if (value === undefined || value.trim() === "") return new Set();
@@ -154,6 +165,7 @@ export async function runUpdateInfo(
       createdBefore,
       throughAid,
       ...(options.afterAid === undefined ? {} : { afterAid: options.afterAid }),
+      ...(options.where === undefined ? {} : { where: options.where }),
     };
     const collector = new RecommendationRefreshService({
       database,