diff --git a/src/renderer/src/extensions/analytics/mixpanel/MixpanelAnalytics.ts b/src/renderer/src/extensions/analytics/mixpanel/MixpanelAnalytics.ts index 861bc1fd4..9b4e45183 100644 --- a/src/renderer/src/extensions/analytics/mixpanel/MixpanelAnalytics.ts +++ b/src/renderer/src/extensions/analytics/mixpanel/MixpanelAnalytics.ts @@ -8,6 +8,7 @@ import type { } from "../../nexus_integration/types/IValidateKeyData"; import { MIXPANEL_PROD_TOKEN, MIXPANEL_DEV_TOKEN } from "../constants"; import { analyticsServiceLog } from "../utils/analyticsLog"; +import { validateEventProperties } from "./eventSchemas"; import type { MixpanelEvent } from "./MixpanelEvents"; class MixpanelAnalytics { @@ -188,15 +189,29 @@ class MixpanelAnalytics { return; } + // Event classes forward caller-supplied properties verbatim and are public API, so the + // payload is validated here at the one send boundary. Dropping a malformed event is + // deliberate: an unjoinable partial row skews the funnels more than a missing row does. + const validation = validateEventProperties(event.eventName, event.properties); + if (validation.status === "invalid") { + analyticsServiceLog("mixpanel", "warn", "Event not tracked (properties failed validation)", { + eventName: event.eventName, + issues: validation.error.issues, + properties: event.properties, + }); + return; + } + // Track event with mixpanel-browser // Super properties are automatically included // IP address and geolocation are automatically tracked - mixpanel.track(event.eventName, event.properties); + const properties = validation.status === "valid" ? validation.properties : event.properties; + mixpanel.track(event.eventName, properties); analyticsServiceLog("mixpanel", "debug", "Event tracked", { eventName: event.eventName, game_id: this.registeredGameId(), - properties: event.properties, + properties, }); } } diff --git a/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.test.ts b/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.test.ts new file mode 100644 index 000000000..fd6498f4d --- /dev/null +++ b/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it } from "vitest"; + +import { EVENT_SCHEMAS, hasEventSchema, validateEventProperties } from "./eventSchemas"; +import type { CollectionInstallOutcomeProps, ModAnalyticsIdentity } from "./MixpanelEvents"; +import { + CollectionsInstallationCancelledEvent, + CollectionsInstallationCompletedEvent, + CollectionsInstallationFailedEvent, + CollectionsInstallationPausedEvent, + CollectionsInstallationResumedEvent, + CollectionsInstallationStartedEvent, + ModsInstallationCancelledEvent, + ModsInstallationCompletedEvent, + ModsInstallationFailedEvent, + ModsInstallationStartedEvent, +} from "./MixpanelEvents"; + +const outcomeProps: CollectionInstallOutcomeProps = { + collection_id: "479609", + revision_id: "12345", + game_id: 1704, + required_total: 10, + installed: 8, + failed: 2, + ignored: 0, + optional: 3, + duration_ms: 1000, + total_duration_ms: 2000, + pause_count: 1, + resume_count: 1, + was_resumed: true, +}; + +const modIdentity: ModAnalyticsIdentity = { + mod_id: "123", + file_id: "456", + game_id: 1704, + mod_uid: "mod-uid", + file_uid: "file-uid", + collection_id: "479609", + revision_id: "12345", +}; + +const modInstallProps = { ...modIdentity, install_kind: "fresh" as const }; + +describe("validateEventProperties", () => { + // Regression: a collections_installation_failed row reached Mixpanel in 2.4.0 as + // {"0":"4","1":"7",...}, a bare id string spread into the properties object. + it("rejects a bare string passed as the properties", () => { + const result = validateEventProperties("collections_installation_failed", "479609"); + expect(result.status).toBe("invalid"); + }); + + it("rejects a string that was spread into an object", () => { + const spread = { ...("479609" as unknown as object) }; + const result = validateEventProperties("collections_installation_failed", spread); + expect(result.status).toBe("invalid"); + }); + + it.each([null, undefined, 42, [], true])("rejects non-object properties: %s", (properties) => { + expect(validateEventProperties("collections_installation_failed", properties).status).toBe( + "invalid", + ); + }); + + it("rejects an empty object, so a fully stripped payload can't pass as valid", () => { + expect(validateEventProperties("collections_installation_cancelled", {}).status).toBe( + "invalid", + ); + }); + + it("rejects a payload whose every key is unrecognized", () => { + expect( + validateEventProperties("collections_installation_cancelled", { 0: "4", 1: "7" }).status, + ).toBe("invalid"); + }); + + it("accepts a collection event carrying only part of the snapshot", () => { + // no id is mandatory, so one recognized field is enough + expect( + validateEventProperties("collections_installation_cancelled", { duration_ms: 1000 }).status, + ).toBe("valid"); + }); + + it("rejects a payload with a mistyped field", () => { + const result = validateEventProperties("collections_installation_cancelled", { + ...outcomeProps, + collection_id: 479609, // number where the contract says string + }); + expect(result.status).toBe("invalid"); + }); + + it("tolerates a missing file_id, absent for bundled and direct downloads", () => { + const { file_id: _omitted, ...withoutFileId } = modInstallProps; + expect(validateEventProperties("mods_installation_started", { ...withoutFileId }).status).toBe( + "valid", + ); + }); + + it("rejects a value outside a closed vocabulary", () => { + const result = validateEventProperties("collections_installation_failed", { + ...outcomeProps, + failure_stage: "something_else", + }); + expect(result.status).toBe("invalid"); + }); + + it("strips unknown keys rather than rejecting the event", () => { + const result = validateEventProperties("collections_installation_cancelled", { + ...outcomeProps, + junk: "should not reach mixpanel", + 0: "4", + }); + expect(result.status).toBe("valid"); + if (result.status !== "valid") return; + expect(result.properties).toEqual({ ...outcomeProps }); + expect(result.properties).not.toHaveProperty("junk"); + expect(result.properties).not.toHaveProperty("0"); + }); + + it("keeps a registered optional field when present", () => { + const result = validateEventProperties("collections_installation_failed", { + ...outcomeProps, + failure_stage: "postprocessing", + error_code: "EPERM", + }); + expect(result.status).toBe("valid"); + if (result.status !== "valid") return; + expect(result.properties.error_code).toBe("EPERM"); + }); + + it("reports an unregistered event name as unchecked so it is still sent", () => { + expect(validateEventProperties("app_launched", { anything: 1 }).status).toBe("unchecked"); + }); + + it("allows a null collection_id on a mod installed outside a collection", () => { + const result = validateEventProperties("mods_installation_started", { + ...modInstallProps, + collection_id: null, + revision_id: null, + }); + expect(result.status).toBe("valid"); + }); + + // Read through optional chaining / parseInt, so undefined and NaN are reachable at + // runtime despite the declared types. Rejecting on those would drop good events. + it("tolerates a missing revision_id, which buildOutcomeProps does not guard", () => { + const { revision_id: _omitted, ...withoutRevision } = outcomeProps; + expect( + validateEventProperties("collections_installation_cancelled", { ...withoutRevision }).status, + ).toBe("valid"); + }); + + it("tolerates a NaN game_id, which parseInt can produce", () => { + const result = validateEventProperties("collections_installation_cancelled", { + ...outcomeProps, + game_id: NaN, + }); + expect(result.status).toBe("valid"); + }); + + it("tolerates a missing mod_id, which resolveModIdentity does not guard", () => { + const { mod_id: _omitted, ...withoutModId } = modInstallProps; + expect(validateEventProperties("mods_installation_started", { ...withoutModId }).status).toBe( + "valid", + ); + }); + + it("still rejects a mistyped install_kind", () => { + expect( + validateEventProperties("mods_installation_started", { + ...modInstallProps, + install_kind: "made_up", + }).status, + ).toBe("invalid"); + }); +}); + +describe("event schema coverage", () => { + // Built through the real event classes so a renamed event, or a field a schema gets + // wrong, fails here rather than dropping real events in production. + const covered = [ + new CollectionsInstallationStartedEvent(outcomeProps), + new CollectionsInstallationResumedEvent(outcomeProps), + new CollectionsInstallationCompletedEvent(outcomeProps), + new CollectionsInstallationFailedEvent({ + ...outcomeProps, + failure_stage: "member_install", + }), + new CollectionsInstallationCancelledEvent(outcomeProps), + new CollectionsInstallationPausedEvent({ ...outcomeProps, trigger: "user" }), + new ModsInstallationStartedEvent(modInstallProps), + new ModsInstallationCompletedEvent({ ...modInstallProps, duration_ms: 500 }), + new ModsInstallationCancelledEvent(modInstallProps), + new ModsInstallationFailedEvent({ + ...modInstallProps, + error_code: "EPERM", + error_message: "denied", + }), + ]; + + it.each(covered.map((event) => [event.eventName, event] as const))( + "%s has a registered schema", + (_name, event) => { + expect(hasEventSchema(event.eventName)).toBe(true); + }, + ); + + it.each(covered.map((event) => [event.eventName, event] as const))( + "%s validates as its class constructs it", + (_name, event) => { + const result = validateEventProperties(event.eventName, event.properties); + expect(result.status).toBe("valid"); + }, + ); + + // Unknown keys are stripped, so a property a producer emits but its schema omits would + // disappear from the dataset with no other signal. + it.each(covered.map((event) => [event.eventName, event] as const))( + "%s keeps every property its class emits", + (_name, event) => { + const result = validateEventProperties(event.eventName, event.properties); + expect(result.status).toBe("valid"); + if (result.status !== "valid") return; + expect(Object.keys(result.properties).sort()).toEqual(Object.keys(event.properties).sort()); + }, + ); + + it("registers a schema for every covered event and nothing else", () => { + expect(Object.keys(EVENT_SCHEMAS).sort()).toEqual(covered.map((e) => e.eventName).sort()); + }); +}); diff --git a/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.ts b/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.ts new file mode 100644 index 000000000..fef85d0dd --- /dev/null +++ b/src/renderer/src/extensions/analytics/mixpanel/eventSchemas.ts @@ -0,0 +1,131 @@ +import { z } from "zod"; + +/** + * Runtime contracts for analytics events, enforced at the single send boundary in + * MixpanelAnalytics.trackEvent. + * + * `MixpanelEvent.properties` is `Record` and the event classes forward caller + * input verbatim, so the compiler can't police a payload. The classes are also public API + * (re-exported through util/api and vortex-api), so callers include untyped JS extensions. + * + * - `z.object` strips unknown keys rather than rejecting, so a stray field costs the field + * rather than the whole event. + * - Nothing read from download metadata is required. Those values come from + * nexusIdsFromDownloadId via optional chaining and `parseInt`, and file/mod ids are + * legitimately absent for bundled and direct downloads, so they can be undefined or NaN + * at runtime despite their declared types. Only the discriminators the emit site sets + * from its own arguments are required: `install_kind`, `failure_stage`, `trigger`. + * - `atLeastOneKnownField` is what keeps an empty or all-unknown payload out, so no single + * field has to be mandatory to reject garbage. + * + * The contracts are deliberately looser than the matching interfaces + * (CollectionInstallOutcomeProps, ModAnalyticsIdentity) and so aren't tied to them with + * `satisfies`; eventSchemas.test.ts covers drift by validating events built through the + * real classes. + * + * Covers the collection-install and mod-install families. Unlisted events pass unchecked. + */ + +/** Producers derive these by arithmetic or `parseInt`, so NaN is reachable. */ +const analyticsNumber = z.number().or(z.nan()); + +/** Rejects a payload that retained none of its recognized properties. */ +function atLeastOneKnownField(schema: T) { + return schema.refine((value) => Object.keys(value).length > 0, { + message: "no recognized analytics properties present", + }); +} + +/** Count/duration snapshot shared by every collection-install event. */ +const collectionInstallOutcome = z.object({ + collection_id: z.string().optional(), + revision_id: z.string().optional(), + game_id: analyticsNumber.optional(), + required_total: analyticsNumber.optional(), + installed: analyticsNumber.optional(), + failed: analyticsNumber.optional(), + ignored: analyticsNumber.optional(), + optional: analyticsNumber.optional(), + duration_ms: analyticsNumber.optional(), + total_duration_ms: analyticsNumber.optional(), + pause_count: analyticsNumber.optional(), + resume_count: analyticsNumber.optional(), + was_resumed: z.boolean().optional(), +}); + +/** Identity shared by every per-mod event. Null collection/revision means "not part of a collection". */ +const modAnalyticsIdentity = z.object({ + mod_id: z.string().optional(), + file_id: z.string().optional(), + game_id: analyticsNumber.optional(), + mod_uid: z.string().optional(), + file_uid: z.string().optional(), + collection_id: z.string().nullable().optional(), + revision_id: z.string().nullable().optional(), +}); + +/** Identity plus how the install came about, shared by the mods_installation_* events. */ +const modInstall = modAnalyticsIdentity.extend({ + install_kind: z.enum(["fresh", "version_update", "reinstall", "variant", "profile_replace"]), +}); + +/** + * eventName -> contract. An absent event is sent unvalidated, which is how the families + * not yet migrated keep working; `hasEventSchema` lets tests track coverage. + */ +export const EVENT_SCHEMAS: Record = { + collections_installation_started: atLeastOneKnownField( + collectionInstallOutcome.extend({ mod_count: analyticsNumber.optional() }), + ), + collections_installation_resumed: atLeastOneKnownField(collectionInstallOutcome), + collections_installation_completed: atLeastOneKnownField( + collectionInstallOutcome.extend({ mod_count: analyticsNumber.optional() }), + ), + collections_installation_failed: atLeastOneKnownField( + collectionInstallOutcome.extend({ + failure_stage: z.enum(["member_install", "postprocessing"]), + error_code: z.string().optional(), + }), + ), + collections_installation_cancelled: atLeastOneKnownField(collectionInstallOutcome), + collections_installation_paused: atLeastOneKnownField( + collectionInstallOutcome.extend({ trigger: z.string() }), + ), + + mods_installation_started: atLeastOneKnownField(modInstall), + mods_installation_completed: atLeastOneKnownField( + modInstall.extend({ duration_ms: analyticsNumber.optional() }), + ), + mods_installation_cancelled: atLeastOneKnownField(modInstall), + mods_installation_failed: atLeastOneKnownField( + modInstall.extend({ error_code: z.string(), error_message: z.string() }), + ), +}; + +export type EventValidation = + | { status: "valid"; properties: Record } + | { status: "invalid"; error: z.ZodError } + /** No contract registered for this event name; send the payload as-is. */ + | { status: "unchecked" }; + +/** + * Validates an event's properties against its contract, returning the payload with unknown + * keys stripped. Rejects a payload that isn't an object, retained no recognized property, + * or mistypes a discriminator: such a row can't be joined to its related events and skews + * the funnels, which is worse than no row. + */ +export function validateEventProperties(eventName: string, properties: unknown): EventValidation { + const schema = EVENT_SCHEMAS[eventName]; + if (schema === undefined) { + return { status: "unchecked" }; + } + const result = schema.safeParse(properties); + return result.success + ? { status: "valid", properties: result.data as Record } + : { status: "invalid", error: result.error }; +} + +/** Whether a contract is registered for this event name. */ +export function hasEventSchema(eventName: string): boolean { + return EVENT_SCHEMAS[eventName] !== undefined; +}