Sync email-only breaches to Remote Settings (mntor-5363)

src/app/functions/universal/breach.ts

@@ -6,6 +6,16 @@ import { DataClassEffected } from "../../../utils/subscriberBreaches";
 
 // TODO: Move pure functions that operate on breaches to this file
 
+/**
+ * The data types we name, in our own kebab-case spelling. This is what we
+ * store and what our APIs speak. See HibpLabelByDataType for HIBP's spelling.
+ *
+ * Not the full list. HIBP adds a data type whenever it sees a new kind of
+ * data, and locales/en/data-classes.ftl lists around 170.
+ *
+ * These are roughly the data types the public breach detail page gives advice
+ * for, see getAllPriorityDataClasses in src/utils/recommendations.ts.
+ */
 export const BreachDataTypes = {
   Passwords: "passwords",
   Email: "email-addresses",
@@ -22,6 +32,39 @@ export const BreachDataTypes = {
   General: "general",
 } as const;
 
+/**
+ * The same data types, spelled the way HIBP spells them. Use these only against
+ * a raw HibpGetBreachesResponse. Anything that came from our database already
+ * speaks BreachDataTypes, because formatDataClass converts it at ingestion.
+ *
+ * Keys match BreachDataTypes, so one data type's two spellings line up on one
+ * key. This cannot be a transform, the difference is not mechanical: `pins` is
+ * `PINs` and `ip-addresses` is `IP addresses`. `General` is ours alone with no
+ * HIBP counterpart, hence the Omit.
+ */
+export const HibpLabelByDataType = {
+  Passwords: "Passwords",
+  Email: "Email addresses",
+  SSN: "Social security numbers",
+  CreditCard: "Partial credit card data",
+  BankAccount: "Bank account numbers",
+  PIN: "PINs",
+  IP: "IP addresses",
+  Address: "Physical addresses",
+  DoB: "Dates of birth",
+  Phone: "Phone numbers",
+  SecurityQuestions: "Security questions and answers",
+  HistoricalPasswords: "Historical passwords",
+} as const satisfies Omit<
+  Record<keyof typeof BreachDataTypes, string>,
+  "General"
+>;
+
+/**
+ * The subset of BreachDataTypes that Monitor asks a user to act on, and so the
+ * only ones that appear in the guided resolution flow and in breach alert
+ * emails. The rest are shown as context only.
+ */
 export const ResolutionRelevantBreachDataTypes = {
   Passwords: BreachDataTypes.Passwords,
   Email: BreachDataTypes.Email,

src/scripts/cronjobs/updateBreachesInRemoteSettings/updateBreachesInRemoteSettings.test.ts

@@ -124,6 +124,39 @@ describe("updateBreachesInRemoteSettings job", () => {
       await main(mockLog);
       expect(reviewSpy).not.toHaveBeenCalled();
     });
+    it("syncs a breach that leaked emails but no passwords", async () => {
+      const emailOnly = HibpData[0] as HIBP.HibpGetBreachesResponse[number];
+      expect(emailOnly.DataClasses).not.toContain("Passwords");
+      vi.mocked(HIBP.fetchHibpBreaches).mockResolvedValue([emailOnly]);
+      fetchBreachesSpy.mockResolvedValueOnce(new Set([]));
+      addBreachSpy.mockResolvedValue(undefined);
+      reviewSpy.mockResolvedValue(undefined);
+
+      await main(mockLog);
+
+      // Pins the whole record
+      expect(addBreachSpy).toHaveBeenCalledWith({
+        Name: emailOnly.Name,
+        Domain: emailOnly.Domain,
+        BreachDate: emailOnly.BreachDate,
+        PwnCount: emailOnly.PwnCount,
+        AddedDate: emailOnly.AddedDate,
+        DataClasses: emailOnly.DataClasses,
+      });
+    });
+    it("skips a breach that leaked neither passwords nor emails", async () => {
+      const neither = {
+        ...(HibpData[0] as HIBP.HibpGetBreachesResponse[number]),
+        Name: "NoUsableDataClasses",
+        DataClasses: ["Names", "Phone numbers"],
+      };
+      vi.mocked(HIBP.fetchHibpBreaches).mockResolvedValue([neither]);
+      fetchBreachesSpy.mockResolvedValueOnce(new Set([]));
+
+      await main(mockLog);
+
+      expect(addBreachSpy).not.toHaveBeenCalled();
+    });
     it("happy path: logs counts, posts filtered breaches, requests review, and exits", async () => {
       const breaches = HibpData.slice(0, 3) as HIBP.HibpGetBreachesResponse;
       vi.mocked(HIBP.fetchHibpBreaches).mockResolvedValue(breaches);
@@ -133,8 +166,7 @@ describe("updateBreachesInRemoteSettings job", () => {
 
       await main(mockLog);
       expect(vi.mocked(HIBP.fetchHibpBreaches)).toHaveBeenCalledTimes(1);
-      // First mock breach is ignored because of no passwords
-      expect(addBreachSpy).toHaveBeenCalledTimes(2);
+      expect(addBreachSpy).toHaveBeenCalledTimes(3);
       expect(reviewSpy).toHaveBeenCalledTimes(1);
     });
     it("happy path: exits with code 1 if review fails", async () => {
@@ -149,7 +181,7 @@ describe("updateBreachesInRemoteSettings job", () => {
 
       await main(mockLog);
       expect(vi.mocked(HIBP.fetchHibpBreaches)).toHaveBeenCalledTimes(1);
-      expect(addBreachSpy).toHaveBeenCalledTimes(2);
+      expect(addBreachSpy).toHaveBeenCalledTimes(3);
       expect(reviewSpy).toHaveBeenCalledTimes(1);
       expect(process.exitCode).toEqual(1);
 

src/scripts/cronjobs/updateBreachesInRemoteSettings/updateBreachesInRemoteSettings.ts

@@ -9,14 +9,13 @@
 
 import * as Sentry from "@sentry/node";
 import * as HIBP from "../../../utils/hibp";
-import { RemoteSettingsClient } from "../../../utils/remoteSettingsClient";
+import {
+  RemoteSettingsClient,
+  type RemoteSettingsBreach,
+} from "../../../utils/remoteSettingsClient";
 import { type Logger } from "winston";
 import { config } from "../../../config";
-
-type RemoteSettingsBreach = Pick<
-  HIBP.HibpGetBreachesResponse[number],
-  "Name" | "Domain" | "BreachDate" | "PwnCount" | "AddedDate" | "DataClasses"
->;
+import { HibpLabelByDataType } from "../../../app/functions/universal/breach";
 
 export type UpdateBreachesJobConfig = {
   user: string;
@@ -75,15 +74,16 @@ export async function main(parentLogger: Logger) {
   });
   const allHibpBreaches = await HIBP.fetchHibpBreaches();
   const remoteBreaches = await remoteSettings.fetchRemoteBreachNames();
-  // Get all valid password breaches that aren't currently in remote settings
+  // Get all valid password or email breaches that aren't currently in remote settings
   const newBreaches = allHibpBreaches.filter((breach) => {
     return (
       !breach.IsRetired &&
       !breach.IsSpamList &&
       !breach.IsFabricated &&
       breach.IsVerified &&
       breach.Domain !== "" &&
-      breach.DataClasses.includes("Passwords") &&
+      (breach.DataClasses.includes(HibpLabelByDataType.Passwords) ||
+        breach.DataClasses.includes(HibpLabelByDataType.Email)) &&
       !remoteBreaches.has(breach.Name)
     );
   });

src/utils/hibp.ts

@@ -127,7 +127,11 @@ export type HibpGetBreachesResponse = Array<{
   Domain: string;
   AddedDate: ISO8601DateString;
   BreachDate: ISO8601DateString;
-  DataClasses: Array<keyof HibpBreachDataTypes>;
+  // Raw from HIBP, so Title Case: "Email addresses".
+  // Stays string[] because HIBP sends far more data types
+  // than HibpLabelByDataType names, and HIBP invents new ones.
+  // See HibpLikeDbBreach below for the kebab-case version.
+  DataClasses: string[];
   Description: string;
   LogoPath: string;
   IsFabricated: boolean;
@@ -203,7 +207,7 @@ export function formatDataClass(dataClass: string): string {
  */
 // TODO: Add unit test when changing this code:
 /* c8 ignore start */
-function formatDataClassesArray(dataClasses: Array<keyof HibpBreachDataTypes>) {
+function formatDataClassesArray(dataClasses: string[]) {
   return dataClasses.map((dataClass) => formatDataClass(dataClass)) as Array<
     HibpBreachDataTypes[keyof HibpBreachDataTypes]
   >;
@@ -232,6 +236,7 @@ export type HibpLikeDbBreach = {
   PwnCount: BreachRow["pwn_count"];
   Description: BreachRow["description"];
   LogoPath: BreachRow["logo_path"];
+  // Converted at ingestion, so kebab-case: "email-addresses".
   DataClasses: BreachRow["data_classes"];
   IsVerified: BreachRow["is_verified"];
   IsFabricated: BreachRow["is_fabricated"];

src/utils/remoteSettingsClient.ts

@@ -5,7 +5,8 @@
 import type { Logger } from "winston";
 import type { HibpGetBreachesResponse } from "./hibp";
 
-type RemoteSettingsBreach = Pick<
+/** The fields we publish to Remote Settings. */
+export type RemoteSettingsBreach = Pick<
   HibpGetBreachesResponse[number],
   "Name" | "Domain" | "BreachDate" | "PwnCount" | "AddedDate" | "DataClasses"
 >;