From b7d9f8e44760ce41c0f8435d7b051c3e7512328e Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 28 Aug 2026 21:59:36 +0000 Subject: [PATCH] fix(github): stabilize activity ingestion --- src/lib/github-activity-feed.ts | 2 +- src/lib/github-activity-worker-store.ts | 115 ++++- src/lib/github-commits-core.ts | 120 ++++- src/lib/github-commits-store.ts | 75 ++- tests/github-activity-feed-postgres.test.mjs | 511 ++++++++++++++++++- tests/github-commits.test.mjs | 133 +++++ 6 files changed, 924 insertions(+), 32 deletions(-) diff --git a/src/lib/github-activity-feed.ts b/src/lib/github-activity-feed.ts index 8bac4625..92607d9f 100644 --- a/src/lib/github-activity-feed.ts +++ b/src/lib/github-activity-feed.ts @@ -21,7 +21,7 @@ const readCachedActivity = unstable_cache( cursor, PUBLIC_GITHUB_ACTIVITY_DAY_PAGE_SIZE ), - ["public-github-activity-v5"], + ["public-github-activity-v6"], { revalidate: 900, tags: ["github-activity"] } ); diff --git a/src/lib/github-activity-worker-store.ts b/src/lib/github-activity-worker-store.ts index 689f84dd..4f2be727 100644 --- a/src/lib/github-activity-worker-store.ts +++ b/src/lib/github-activity-worker-store.ts @@ -1571,10 +1571,13 @@ const publishCompletedSummaryForCommit = async ( if (ready !== undefined) { await transaction .update(githubPublicActivities) - .set({ - publishedAt: sql`coalesce(${githubPublicActivities.publishedAt}, ${now})`, - }) - .where(eq(githubPublicActivities.publicId, ready.publicId)); + .set({ publishedAt: now }) + .where( + and( + eq(githubPublicActivities.publicId, ready.publicId), + isNull(githubPublicActivities.publishedAt) + ) + ); } }; @@ -1596,6 +1599,14 @@ const markGitHubCommitCanonicalized = async ( await publishCompletedSummaryForCommit(transaction, repositoryId, sha, now); }; +const comparableCommitHeadline = (message: string | null) => { + const headline = message?.split("\n", 1)[0]?.trim(); + if (headline === undefined || headline.length === 0) { + return null; + } + return headline.replace(/\s+\(#[1-9]\d*\)$/u, ""); +}; + export const canonicalizeGitHubCommitActivity = async ( repositoryId: string, sha: string, @@ -1605,7 +1616,10 @@ export const canonicalizeGitHubCommitActivity = async ( await getDatabase().transaction(async (transaction) => { const [candidate] = await transaction .select({ + authoredAt: githubCommits.authoredAt, + authorUserId: githubCommits.authorUserId, changeFingerprint: githubCommits.changeFingerprint, + committerAt: githubCommits.committerAt, fingerprintComplete: githubCommits.fingerprintComplete, firstObservedAt: githubCommits.firstObservedAt, fullMessage: githubCommits.fullMessage, @@ -1700,7 +1714,13 @@ export const canonicalizeGitHubCommitActivity = async ( } const copies = await transaction .select({ + authoredAt: githubCommits.authoredAt, + authorUserId: githubCommits.authorUserId, + canonicalPublicId: githubPublicActivities.canonicalPublicId, + committerAt: githubCommits.committerAt, firstObservedAt: githubCommits.firstObservedAt, + fullMessage: githubCommits.fullMessage, + parentShas: githubCommits.parentShas, publicId: githubPublicActivities.publicId, sha: githubCommits.sha, }) @@ -1715,8 +1735,10 @@ export const canonicalizeGitHubCommitActivity = async ( eq(githubCommits.changeFingerprint, candidate.changeFingerprint), eq(githubCommits.fingerprintComplete, true), eq(githubCommits.enrichmentState, "complete"), - isNull(githubPublicActivities.canonicalPublicId), - isNull(githubPublicActivities.hiddenAt) + or( + isNull(githubPublicActivities.hiddenAt), + isNotNull(githubPublicActivities.canonicalPublicId) + ) ) ) .orderBy(asc(githubCommits.firstObservedAt), asc(githubCommits.sha)); @@ -1732,13 +1754,45 @@ export const canonicalizeGitHubCommitActivity = async ( if (copy.sha === sha) { continue; } + const canonicalPublicId = copy.canonicalPublicId ?? copy.publicId; + if (canonicalPublicId === candidate.publicId) { + continue; + } const explicitCherryPick = copy.sha === cherryPickSource; + const directMergeParent = + (candidate.parentShas?.length ?? 0) > 1 && + candidate.parentShas?.includes(copy.sha) === true; + const sameAuthorSingleParent = + candidate.parentShas?.length === 1 && + copy.parentShas?.length === 1 && + candidate.authorUserId !== null && + candidate.authorUserId === copy.authorUserId; + const sameAuthoredCommit = + sameAuthorSingleParent && + candidate.authoredAt !== null && + copy.authoredAt !== null && + candidate.authoredAt.getTime() === copy.authoredAt.getTime() && + candidate.fullMessage !== null && + candidate.fullMessage === copy.fullMessage; + const candidateHeadline = comparableCommitHeadline(candidate.fullMessage); + const copyHeadline = comparableCommitHeadline(copy.fullMessage); + const sameHeadlineCommit = + sameAuthorSingleParent && + candidateHeadline !== null && + candidateHeadline === copyHeadline; + const candidateOrder = + candidate.committerAt?.getTime() ?? candidate.firstObservedAt.getTime(); + const copyOrder = + copy.committerAt?.getTime() ?? copy.firstObservedAt.getTime(); if ( !explicitCherryPick && - (copy.firstObservedAt > candidate.firstObservedAt || - (copy.firstObservedAt.getTime() === - candidate.firstObservedAt.getTime() && - copy.sha > sha)) + !directMergeParent && + (copyOrder > candidateOrder || + (copyOrder === candidateOrder && + (copy.firstObservedAt > candidate.firstObservedAt || + (copy.firstObservedAt.getTime() === + candidate.firstObservedAt.getTime() && + copy.sha > sha)))) ) { continue; } @@ -1758,20 +1812,40 @@ export const canonicalizeGitHubCommitActivity = async ( ); } )?.[0]; - if (!explicitCherryPick && sharedPullRequest === undefined) { + if ( + !explicitCherryPick && + !directMergeParent && + !sameAuthoredCommit && + !sameHeadlineCommit && + sharedPullRequest === undefined + ) { continue; } - const reason = explicitCherryPick - ? "cherry_pick" - : "pr_history_exact_copy"; + const reason = directMergeParent + ? "direct_parent_merge" + : explicitCherryPick + ? "cherry_pick" + : sameAuthoredCommit + ? "same_authored_exact_copy" + : sameHeadlineCommit + ? "same_author_headline_exact_copy" + : "pr_history_exact_copy"; const aliased = await setCanonicalAlias( transaction, candidate.publicId, - copy.publicId, + canonicalPublicId, reason, { fingerprint: candidate.changeFingerprint, fingerprintComplete: true, + authoredAt: sameAuthoredCommit + ? candidate.authoredAt?.toISOString() + : null, + directMergeParent, + headline: + !sameAuthoredCommit && sameHeadlineCommit + ? candidateHeadline + : null, pullRequestNodeId: sharedPullRequest ?? null, sourceSha: copy.sha, }, @@ -2105,10 +2179,13 @@ export const completeGitHubSummaryAttempt = async ( ) { await transaction .update(githubPublicActivities) - .set({ - publishedAt: sql`coalesce(${githubPublicActivities.publishedAt}, ${now})`, - }) - .where(eq(githubPublicActivities.publicId, attempt.activityPublicId)); + .set({ publishedAt: now }) + .where( + and( + eq(githubPublicActivities.publicId, attempt.activityPublicId), + isNull(githubPublicActivities.publishedAt) + ) + ); } return true; }); diff --git a/src/lib/github-commits-core.ts b/src/lib/github-commits-core.ts index 4823e98b..55aa6ab4 100644 --- a/src/lib/github-commits-core.ts +++ b/src/lib/github-commits-core.ts @@ -98,12 +98,19 @@ export interface GitHubPullRequestWebhookObservation { pullRequest: GitHubPullRequest; } +export interface GitHubPullRequestEventSignal { + action: string; + number: number; + repositoryId: string; +} + export interface GitHubEvent { issue: GitHubIssue | null; occurredAt: string; id: string; push: GitHubPush | null; pullRequest: GitHubPullRequest | null; + pullRequestSignal?: GitHubPullRequestEventSignal; } const isObject = (value: unknown): value is JsonObject => @@ -725,6 +732,82 @@ const issueEventFrom = ( }; }; +const sparsePullRequestRepositoryFrom = (value: unknown) => { + if (!isObject(value)) { + return null; + } + const url = normalizedText(value.url, 2000); + if (url === null || !url.startsWith("https://api.github.com/repos/")) { + return null; + } + const fullName = url.slice("https://api.github.com/repos/".length); + const repository = repositoryFrom({ id: value.id, name: fullName }); + const repositoryName = normalizedText(value.name, 100); + if ( + repository === null || + repositoryName === null || + repositoryName !== repository.fullName.split("/")[1] || + url !== `https://api.github.com/repos/${repository.fullName}` + ) { + return null; + } + return repository; +}; + +const sparsePullRequestBranchIsValid = ( + value: unknown, + expectedRepository: GitHubRepository | null +) => { + if (!isObject(value)) { + return false; + } + const repository = + value.repo === null ? null : sparsePullRequestRepositoryFrom(value.repo); + if ( + normalizedText(value.ref, 300) === null || + commitShaFrom(value.sha) === null || + (value.repo !== null && repository === null) + ) { + return false; + } + return ( + expectedRepository === null || + (repository !== null && repository.id === expectedRepository.id) + ); +}; + +const sparsePullRequestEventSignalFrom = ( + payload: JsonObject, + repository: GitHubRepository +): GitHubPullRequestEventSignal | null => { + if (!isObject(payload.pull_request)) { + return null; + } + const value = payload.pull_request; + const action = normalizedText(payload.action, 40)?.toLowerCase() ?? ""; + const payloadNumber = positiveInteger(payload.number); + const number = positiveInteger(value.number); + const id = pullRequestIdFrom(value.id); + const expectedUrl = + number === null + ? null + : `https://api.github.com/repos/${repository.fullName}/pulls/${number}`; + if ( + !PULL_REQUEST_ACTION.test(action) || + payloadNumber === null || + number === null || + payloadNumber !== number || + id === null || + expectedUrl === null || + value.url !== expectedUrl || + !sparsePullRequestBranchIsValid(value.base, repository) || + !sparsePullRequestBranchIsValid(value.head, null) + ) { + return null; + } + return { action, number, repositoryId: repository.id }; +}; + const pullRequestEventFrom = ( value: JsonObject, id: string, @@ -737,22 +820,41 @@ const pullRequestEventFrom = ( if (repository === null) { return null; } - const pullRequest = pullRequestFromGitHub( - value.payload.pull_request, - repository, - value.payload.action + const rawPullRequest = value.payload.pull_request; + if (isObject(rawPullRequest) && "user" in rawPullRequest) { + const pullRequest = pullRequestFromGitHub( + rawPullRequest, + repository, + value.payload.action + ); + if (pullRequest === null) { + return null; + } + return { + id, + issue: null, + occurredAt, + // Persistence admits a foreign-authored PR only when its stable node ID is + // already known through a tracked commit. Retaining it here lets a tracked + // maintainer's terminal event schedule the final reconciliation. + pullRequest, + push: null, + }; + } + + const pullRequestSignal = sparsePullRequestEventSignalFrom( + value.payload, + repository ); - if (pullRequest === null) { + if (pullRequestSignal === null) { return null; } return { id, issue: null, occurredAt, - // Persistence admits a foreign-authored PR only when its stable node ID is - // already known through a tracked commit. Retaining it here lets a tracked - // maintainer's terminal event schedule the final reconciliation. - pullRequest, + pullRequest: null, + pullRequestSignal, push: null, }; }; diff --git a/src/lib/github-commits-store.ts b/src/lib/github-commits-store.ts index 2ac0468b..1b59968a 100644 --- a/src/lib/github-commits-store.ts +++ b/src/lib/github-commits-store.ts @@ -17,6 +17,7 @@ import type { GitHubEvent, GitHubIssue, GitHubPullRequest, + GitHubPullRequestEventSignal, GitHubPush, GitHubRepository, GitHubRepositoryFacts, @@ -530,6 +531,61 @@ const upsertPullRequest = async ( return true; }; +const TERMINAL_PULL_REQUEST_EVENT_ACTIONS = new Set(["closed", "merged"]); + +const signalKnownPullRequestReconciliation = async ( + transaction: DatabaseTransaction, + account: TrackedGitHubAccount, + signal: GitHubPullRequestEventSignal, + occurredAt: Date, + observedAt: Date +) => { + const [known] = await transaction + .select({ + nodeId: githubPullRequests.nodeId, + providerUpdatedAt: githubPullRequests.providerUpdatedAt, + state: githubPullRequests.state, + }) + .from(githubPullRequests) + .where( + and( + eq(githubPullRequests.account, account), + eq(githubPullRequests.repositoryId, signal.repositoryId), + eq(githubPullRequests.number, signal.number) + ) + ) + .for("update"); + if (known === undefined) { + return false; + } + if (occurredAt < known.providerUpdatedAt) { + return false; + } + + const promoteToProvisionalClosed = + TERMINAL_PULL_REQUEST_EVENT_ACTIONS.has(signal.action) && + known.state === "open"; + await transaction + .update(githubPullRequests) + .set({ + nextReconcileAt: observedAt, + ...(promoteToProvisionalClosed + ? { + closedAt: occurredAt, + state: "closed" as const, + terminalAt: occurredAt, + } + : {}), + }) + .where( + and( + eq(githubPullRequests.account, account), + eq(githubPullRequests.nodeId, known.nodeId) + ) + ); + return true; +}; + const lockWebhookAccount = async ( transaction: DatabaseTransaction, account: TrackedGitHubAccount @@ -845,7 +901,12 @@ export const persistAccountIntake = async (input: { const [knownPullRequest] = await transaction .select({ account: githubPullRequests.account }) .from(githubPullRequests) - .where(eq(githubPullRequests.nodeId, event.pullRequest.nodeId)) + .where( + and( + eq(githubPullRequests.account, input.account), + eq(githubPullRequests.nodeId, event.pullRequest.nodeId) + ) + ) .limit(1); pullRequestAccount = trackedGitHubAccountFrom( knownPullRequest?.account @@ -863,6 +924,18 @@ export const persistAccountIntake = async (input: { result.pullRequests += 1; } } + if ( + event.pullRequestSignal !== undefined && + (await signalKnownPullRequestReconciliation( + transaction, + input.account, + event.pullRequestSignal, + new Date(event.occurredAt), + now + )) + ) { + result.pullRequests += 1; + } } await updateCheckpoint(transaction, input, now); diff --git a/tests/github-activity-feed-postgres.test.mjs b/tests/github-activity-feed-postgres.test.mjs index 8eaae506..ea0b6eae 100644 --- a/tests/github-activity-feed-postgres.test.mjs +++ b/tests/github-activity-feed-postgres.test.mjs @@ -61,9 +61,11 @@ describe.skipIf(!dockerAvailable)( "GitHub activity PostgreSQL projection", () => { let admin; + let canonicalizeGitHubCommitActivity; let claimDueGitHubPullRequests; let closeDatabase; let completeGitHubPullRequestReconciliation; + let completeGitHubSummaryAttempt; let containerId; let databaseUrl; let originalCronSecret; @@ -138,8 +140,12 @@ describe.skipIf(!dockerAvailable)( ({ closeDatabase } = await import("../src/db/client.ts")); ({ readPublicGitHubActivityPage } = await import("../src/lib/github-activity-store.ts")); - ({ claimDueGitHubPullRequests, completeGitHubPullRequestReconciliation } = - await import("../src/lib/github-activity-worker-store.ts")); + ({ + canonicalizeGitHubCommitActivity, + claimDueGitHubPullRequests, + completeGitHubPullRequestReconciliation, + completeGitHubSummaryAttempt, + } = await import("../src/lib/github-activity-worker-store.ts")); ({ persistAccountIntake } = await import("../src/lib/github-commits-store.ts")); }); @@ -635,6 +641,507 @@ describe.skipIf(!dockerAvailable)( next_reconcile_at: null, state: "merged", }); + + await admin` + insert into github_pull_requests ( + account, author_login, author_user_id, created_at, node_id, number, + provider_updated_at, repository_id, state, title, title_snapshot, url + ) values + ( + 'f0rr0', 'somebody-else', '999', '2025-01-01T00:00:00.000Z', + 'PR_sparse_terminal', 80, '2026-08-28T20:00:00.000Z', '202', + 'open', 'Immutable sparse-event title', 'Immutable sparse-event title', + 'https://github.com/example-org/upstream/pull/80' + ), + ( + 'f0rr0', 'somebody-else', '999', '2025-01-01T00:00:00.000Z', + 'PR_sparse_stale', 82, '2026-08-28T22:00:00.000Z', '202', + 'open', 'Newer open title', 'Newer open title', + 'https://github.com/example-org/upstream/pull/82' + ) + `; + const sparseTerminalSignal = { + action: "merged", + number: 80, + repositoryId: "202", + }; + const crossAccount = await persistAccountIntake({ + account: "yuppiestechdev", + events: [ + { + id: "900000004", + issue: null, + occurredAt: "2026-08-28T21:00:00.000Z", + pullRequest: null, + pullRequestSignal: sparseTerminalSignal, + push: null, + }, + ], + expectedCheckpoint: null, + gap: null, + latestEventId: "900000004", + }); + expect(crossAccount.pullRequests).toBe(0); + const [unchangedAcrossAccounts] = await admin` + select next_reconcile_at, state + from github_pull_requests + where node_id = 'PR_sparse_terminal' + `; + expect(unchangedAcrossAccounts).toEqual({ + next_reconcile_at: null, + state: "open", + }); + + const sparseIntake = await persistAccountIntake({ + account: "f0rr0", + events: [ + { + id: "900000005", + issue: null, + occurredAt: "2026-08-28T21:00:00.000Z", + pullRequest: null, + pullRequestSignal: sparseTerminalSignal, + push: null, + }, + { + id: "900000006", + issue: null, + occurredAt: "2026-08-28T21:00:01.000Z", + pullRequest: null, + pullRequestSignal: { + ...sparseTerminalSignal, + number: 81, + }, + push: null, + }, + { + id: "900000007", + issue: null, + occurredAt: "2026-08-28T21:00:00.000Z", + pullRequest: null, + pullRequestSignal: { + ...sparseTerminalSignal, + number: 82, + }, + push: null, + }, + ], + expectedCheckpoint: { + latestEventId: "900000003", + paused: false, + }, + gap: null, + latestEventId: "900000007", + }); + expect(sparseIntake.pullRequests).toBe(1); + const [provisionalTerminal] = await admin` + select closed_at, provider_updated_at, state, terminal_at, title + from github_pull_requests + where node_id = 'PR_sparse_terminal' + `; + expect({ + ...provisionalTerminal, + closed_at: new Date(provisionalTerminal.closed_at).toISOString(), + provider_updated_at: new Date( + provisionalTerminal.provider_updated_at + ).toISOString(), + terminal_at: new Date(provisionalTerminal.terminal_at).toISOString(), + }).toEqual({ + closed_at: "2026-08-28T21:00:00.000Z", + provider_updated_at: "2026-08-28T20:00:00.000Z", + state: "closed", + terminal_at: "2026-08-28T21:00:00.000Z", + title: "Immutable sparse-event title", + }); + const [staleTerminal] = await admin` + select next_reconcile_at, state, terminal_at + from github_pull_requests + where node_id = 'PR_sparse_stale' + `; + expect(staleTerminal).toEqual({ + next_reconcile_at: null, + state: "open", + terminal_at: null, + }); + expect( + await admin` + select node_id + from github_pull_requests + where repository_id = '202' and number = 81 + ` + ).toHaveLength(0); + expect( + await admin` + select public_id + from github_public_activities + where kind = 'pull_request' and source_node_id = 'PR_sparse_terminal' + ` + ).toHaveLength(0); + + const sparseClaims = await claimDueGitHubPullRequests( + "f0rr0", + 1, + 25, + new Date(Date.now() + 60_000) + ); + expect( + sparseClaims.some(({ nodeId }) => nodeId === "PR_sparse_terminal") + ).toBe(true); + expect( + sparseClaims.some(({ nodeId }) => nodeId === "PR_sparse_stale") + ).toBe(false); + + const summaryActivityId = "00000000-0000-4000-8000-000000000008"; + const summaryLeaseToken = "20000000-0000-4000-8000-000000000008"; + const summarySha = "9".repeat(40); + await admin` + insert into github_commits ( + activity_public_id, author_login, canonicalized_at, committed_at, + committer_at, enrichment_state, languages, message, repository, + repository_id, sha + ) values ( + ${summaryActivityId}, 'f0rr0', '2026-08-28T21:30:00.000Z', + '2026-08-28T21:00:00.000Z', '2026-08-28T21:00:00.000Z', + 'complete', '[]'::jsonb, 'Publish typed timestamp', 'f0rr0/source', + '101', ${summarySha} + ) + `; + await admin` + insert into github_public_activities ( + public_id, kind, occurred_at, repository_id, revision, source_node_id + ) values ( + ${summaryActivityId}, 'commit', '2026-08-28T21:00:00.000Z', '101', + 1, ${summarySha} + ) + `; + await admin` + insert into github_summary_attempts ( + activity_public_id, attempted_at, lease_token, lease_until, revision, + state + ) values ( + ${summaryActivityId}, '2026-08-28T21:31:00.000Z', + ${summaryLeaseToken}, '2026-08-28T21:36:00.000Z', 1, 'processing' + ) + `; + const summaryPublishedAt = new Date("2026-08-28T21:32:00.000Z"); + expect( + await completeGitHubSummaryAttempt( + { + activityPublicId: summaryActivityId, + author: "f0rr0", + committedAt: "2026-08-28T21:00:00.000Z", + leaseToken: summaryLeaseToken, + message: "Publish typed timestamp", + repository: "f0rr0/source", + repositoryId: "101", + revision: 1, + sha: summarySha, + }, + { + headline: "Typed publication succeeds", + inputHash: "a".repeat(64), + model: "test-model", + recipe: "test-recipe", + short: "The completed summary is published exactly once.", + }, + summaryPublishedAt + ) + ).toBe(true); + const [publishedSummary] = await admin` + select published_at, state + from github_public_activities + join github_summary_attempts + on github_summary_attempts.activity_public_id = github_public_activities.public_id + where github_public_activities.public_id = ${summaryActivityId} + `; + expect({ + published_at: new Date(publishedSummary.published_at).toISOString(), + state: publishedSummary.state, + }).toEqual({ + published_at: summaryPublishedAt.toISOString(), + state: "complete", + }); + + const exactCopyIds = { + controlCandidate: "00000000-0000-4000-8000-000000000014", + controlSource: "00000000-0000-4000-8000-000000000013", + headlineCandidate: "00000000-0000-4000-8000-000000000017", + headlineSource: "00000000-0000-4000-8000-000000000016", + merge: "00000000-0000-4000-8000-000000000012", + mergeParent: "00000000-0000-4000-8000-000000000011", + mergeRoot: "00000000-0000-4000-8000-000000000015", + rebased: "00000000-0000-4000-8000-000000000010", + rebasedSource: "00000000-0000-4000-8000-000000000009", + }; + const exactCopyShas = { + controlCandidate: `${"6".repeat(39)}a`, + controlSource: `${"5".repeat(39)}a`, + headlineCandidate: `${"1".repeat(39)}b`, + headlineSource: `${"f".repeat(39)}b`, + merge: `${"4".repeat(39)}a`, + mergeParent: `${"3".repeat(39)}a`, + mergeRoot: `${"7".repeat(39)}a`, + rebased: `${"2".repeat(39)}a`, + // Intentionally sorts after the rebased SHA: committer time, not SHA, + // must select the original when both were observed at the same time. + rebasedSource: `${"e".repeat(39)}a`, + }; + await admin` + insert into github_repositories ( + id, full_name, html_url, owner_avatar_url, owner_id, owner_login, + owner_type, visibility + ) values ( + '303', 'f0rr0/exact-copies', 'https://github.com/f0rr0/exact-copies', + 'https://avatars.githubusercontent.com/u/101', '101', 'f0rr0', + 'User', 'public' + ) + `; + await admin` + insert into github_public_activities ( + public_id, kind, occurred_at, repository_id, revision, source_node_id + ) values + ( + ${exactCopyIds.rebasedSource}, 'commit', + '2026-08-28T10:00:00.000Z', '303', 1, + ${exactCopyShas.rebasedSource} + ), + ( + ${exactCopyIds.rebased}, 'commit', '2026-08-28T12:00:00.000Z', + '303', 1, ${exactCopyShas.rebased} + ), + ( + ${exactCopyIds.mergeRoot}, 'commit', + '2026-08-28T10:15:00.000Z', '303', 1, + ${exactCopyShas.mergeRoot} + ), + ( + ${exactCopyIds.mergeParent}, 'commit', + '2026-08-28T10:30:00.000Z', '303', 1, + ${exactCopyShas.mergeParent} + ), + ( + ${exactCopyIds.merge}, 'commit', '2026-08-28T13:00:00.000Z', + '303', 1, ${exactCopyShas.merge} + ), + ( + ${exactCopyIds.controlSource}, 'commit', + '2026-08-28T09:00:00.000Z', '303', 1, + ${exactCopyShas.controlSource} + ), + ( + ${exactCopyIds.controlCandidate}, 'commit', + '2026-08-28T15:00:00.000Z', '303', 1, + ${exactCopyShas.controlCandidate} + ), + ( + ${exactCopyIds.headlineSource}, 'commit', + '2026-08-28T09:30:00.000Z', '303', 1, + ${exactCopyShas.headlineSource} + ), + ( + ${exactCopyIds.headlineCandidate}, 'commit', + '2026-08-28T10:30:00.000Z', '303', 1, + ${exactCopyShas.headlineCandidate} + ) + `; + await admin` + insert into github_commits ( + activity_public_id, author_login, authored_at, author_user_id, + change_fingerprint, committed_at, committer_at, enrichment_state, + fingerprint_complete, first_observed_at, full_message, message, + parent_shas, pr_discovery_state, repository, repository_id, + sha + ) values + ( + ${exactCopyIds.rebasedSource}, 'f0rr0', + '2026-08-28T08:00:00.000Z', '101', ${"a".repeat(64)}, + '2026-08-28T10:00:00.000Z', '2026-08-28T10:00:00.000Z', + 'complete', true, '2026-08-28T14:00:00.000Z', + 'Preserve exact authored metadata', 'Preserve exact authored metadata', + ${JSON.stringify(["7".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.rebasedSource} + ), + ( + ${exactCopyIds.rebased}, 'f0rr0', '2026-08-28T08:00:00.000Z', + '101', ${"a".repeat(64)}, '2026-08-28T12:00:00.000Z', + '2026-08-28T12:00:00.000Z', 'complete', true, + '2026-08-28T14:00:00.000Z', 'Preserve exact authored metadata', + 'Preserve exact authored metadata', + ${JSON.stringify(["8".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.rebased} + ), + ( + ${exactCopyIds.mergeRoot}, 'f0rr0', + '2026-08-28T08:30:00.000Z', '101', ${"b".repeat(64)}, + '2026-08-28T10:15:00.000Z', '2026-08-28T10:15:00.000Z', + 'complete', true, '2026-08-28T14:00:00.000Z', + 'Original canonical feature', 'Original canonical feature', + ${JSON.stringify(["d".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.mergeRoot} + ), + ( + ${exactCopyIds.mergeParent}, 'f0rr0', + '2026-08-28T09:00:00.000Z', '101', ${"b".repeat(64)}, + '2026-08-28T10:30:00.000Z', '2026-08-28T10:30:00.000Z', + 'complete', true, '2026-08-28T14:30:00.000Z', + 'Implement the feature', 'Implement the feature', + ${JSON.stringify(["9".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.mergeParent} + ), + ( + ${exactCopyIds.merge}, 'f0rr0', '2026-08-28T13:00:00.000Z', + '101', ${"b".repeat(64)}, '2026-08-28T13:00:00.000Z', + '2026-08-28T13:00:00.000Z', 'complete', true, + '2026-08-28T14:30:00.000Z', 'Merge the feature', + 'Merge the feature', + ${JSON.stringify([ + "a".repeat(40), + exactCopyShas.mergeParent, + ])}::jsonb, + 'complete', 'f0rr0/exact-copies', '303', ${exactCopyShas.merge} + ), + ( + ${exactCopyIds.controlSource}, 'f0rr0', + '2026-08-28T07:00:00.000Z', '101', ${"c".repeat(64)}, + '2026-08-28T09:00:00.000Z', '2026-08-28T09:00:00.000Z', + 'complete', true, '2026-08-28T15:30:00.000Z', + 'First intentional edit', 'First intentional edit', + ${JSON.stringify(["b".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.controlSource} + ), + ( + ${exactCopyIds.controlCandidate}, 'f0rr0', + '2026-08-28T07:00:00.000Z', '101', ${"c".repeat(64)}, + '2026-08-28T15:00:00.000Z', '2026-08-28T15:00:00.000Z', + 'complete', true, '2026-08-28T15:30:00.000Z', + 'Second intentional edit', 'Second intentional edit', + ${JSON.stringify(["c".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.controlCandidate} + ), + ( + ${exactCopyIds.headlineSource}, 'f0rr0', + '2026-08-28T07:30:00.000Z', '101', ${"d".repeat(64)}, + '2026-08-28T09:30:00.000Z', '2026-08-28T09:30:00.000Z', + 'complete', true, '2026-08-28T15:30:00.000Z', + 'Ship the squash-safe feature', 'Ship the squash-safe feature', + ${JSON.stringify(["4".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.headlineSource} + ), + ( + ${exactCopyIds.headlineCandidate}, 'f0rr0', + '2026-08-28T10:30:00.000Z', '101', ${"d".repeat(64)}, + '2026-08-28T10:30:00.000Z', '2026-08-28T10:30:00.000Z', + 'complete', true, '2026-08-28T15:30:00.000Z', + 'Ship the squash-safe feature (#123)\n\nReviewed squash body', + 'Ship the squash-safe feature (#123)', + ${JSON.stringify(["2".repeat(40)])}::jsonb, 'complete', + 'f0rr0/exact-copies', '303', ${exactCopyShas.headlineCandidate} + ) + `; + await admin` + update github_public_activities + set + alias_evidence = '{"preexistingAlias":true}'::jsonb, + alias_reason = 'same_authored_exact_copy', + canonical_public_id = ${exactCopyIds.mergeRoot}, + hidden_at = '2026-08-28T15:45:00.000Z' + where public_id = ${exactCopyIds.mergeParent} + `; + + expect( + await canonicalizeGitHubCommitActivity( + "303", + exactCopyShas.rebased, + new Date("2026-08-28T16:00:00.000Z") + ) + ).toEqual({ aliased: true, publicId: exactCopyIds.rebased }); + expect( + await canonicalizeGitHubCommitActivity( + "303", + exactCopyShas.merge, + new Date("2026-08-28T16:01:00.000Z") + ) + ).toEqual({ aliased: true, publicId: exactCopyIds.merge }); + expect( + await canonicalizeGitHubCommitActivity( + "303", + exactCopyShas.headlineCandidate, + new Date("2026-08-28T16:01:30.000Z") + ) + ).toEqual({ + aliased: true, + publicId: exactCopyIds.headlineCandidate, + }); + expect( + await canonicalizeGitHubCommitActivity( + "303", + exactCopyShas.controlCandidate, + new Date("2026-08-28T16:02:00.000Z") + ) + ).toEqual({ + aliased: false, + publicId: exactCopyIds.controlCandidate, + }); + + const exactCopyAliases = await admin` + select alias_evidence, alias_reason, canonical_public_id, public_id + from github_public_activities + where public_id in ( + ${exactCopyIds.rebased}, ${exactCopyIds.merge}, + ${exactCopyIds.controlCandidate}, ${exactCopyIds.headlineCandidate} + ) + order by public_id + `; + expect(exactCopyAliases).toEqual([ + { + alias_evidence: { + authoredAt: "2026-08-28T08:00:00.000Z", + directMergeParent: false, + fingerprint: "a".repeat(64), + fingerprintComplete: true, + headline: null, + pullRequestNodeId: null, + sourceSha: exactCopyShas.rebasedSource, + }, + alias_reason: "same_authored_exact_copy", + canonical_public_id: exactCopyIds.rebasedSource, + public_id: exactCopyIds.rebased, + }, + { + alias_evidence: { + authoredAt: null, + directMergeParent: true, + fingerprint: "b".repeat(64), + fingerprintComplete: true, + headline: null, + pullRequestNodeId: null, + sourceSha: exactCopyShas.mergeParent, + }, + alias_reason: "direct_parent_merge", + canonical_public_id: exactCopyIds.mergeRoot, + public_id: exactCopyIds.merge, + }, + { + alias_evidence: null, + alias_reason: null, + canonical_public_id: null, + public_id: exactCopyIds.controlCandidate, + }, + { + alias_evidence: { + authoredAt: null, + directMergeParent: false, + fingerprint: "d".repeat(64), + fingerprintComplete: true, + headline: "Ship the squash-safe feature", + pullRequestNodeId: null, + sourceSha: exactCopyShas.headlineSource, + }, + alias_reason: "same_author_headline_exact_copy", + canonical_public_id: exactCopyIds.headlineSource, + public_id: exactCopyIds.headlineCandidate, + }, + ]); }); } ); diff --git a/tests/github-commits.test.mjs b/tests/github-commits.test.mjs index ca98478e..66aa15fa 100644 --- a/tests/github-commits.test.mjs +++ b/tests/github-commits.test.mjs @@ -75,6 +75,46 @@ const pullRequest = { updated_at: "2026-08-26T12:02:00Z", user: { id: 456, login: "f0rr0" }, }; +const sparsePullRequest = { + base: { + ref: "main", + repo: { + id: 123, + name: "private-repo", + url: "https://api.github.com/repos/another-org/private-repo", + }, + sha: before, + }, + head: { + ref: "durable-intake", + repo: { + id: 123, + name: "private-repo", + url: "https://api.github.com/repos/another-org/private-repo", + }, + sha, + }, + id: 987, + number: 42, + url: "https://api.github.com/repos/another-org/private-repo/pulls/42", +}; +const sparsePullRequestEventFrom = (pullRequestValue, overrides = {}) => + githubEventFrom( + { + actor: { login: "f0rr0" }, + created_at: "2026-08-26T12:03:00Z", + id: "123456798", + payload: { + action: "merged", + number: 42, + pull_request: pullRequestValue, + ...overrides, + }, + repo: { id: 123, name: "another-org/private-repo" }, + type: "PullRequestEvent", + }, + "f0rr0" + ); const issue = { created_at: "2026-08-26T10:00:00Z", html_url: "https://github.com/another-org/private-repo/issues/91", @@ -665,6 +705,71 @@ describe("pull request observation normalization", () => { }); }); + test("checkpoints the sparse PullRequestEvent shape returned by the user Events API", () => { + expect( + githubEventFrom( + { + actor: { login: "f0rr0" }, + created_at: "2026-08-26T12:03:00Z", + id: "123456797", + payload: { + action: "merged", + number: 42, + pull_request: sparsePullRequest, + }, + repo: { id: 123, name: "another-org/private-repo" }, + type: "PullRequestEvent", + }, + "f0rr0" + ) + ).toEqual({ + id: "123456797", + issue: null, + occurredAt: "2026-08-26T12:03:00.000Z", + pullRequest: null, + pullRequestSignal: { + action: "merged", + number: 42, + repositoryId: "123", + }, + push: null, + }); + }); + + test("rejects malformed events that resemble sparse pull request signals", () => { + expect( + sparsePullRequestEventFrom({ ...sparsePullRequest, number: 41 }) + ).toBeNull(); + expect( + sparsePullRequestEventFrom({ + ...sparsePullRequest, + url: "https://example.com/repos/another-org/private-repo/pulls/42", + }) + ).toBeNull(); + expect( + sparsePullRequestEventFrom({ + ...sparsePullRequest, + base: { + ...sparsePullRequest.base, + repo: { ...sparsePullRequest.base.repo, id: 999 }, + }, + }) + ).toBeNull(); + expect( + sparsePullRequestEventFrom({ ...sparsePullRequest, head: null }) + ).toBeNull(); + expect( + sparsePullRequestEventFrom(sparsePullRequest, { action: "not valid" }) + ).toBeNull(); + expect( + sparsePullRequestEventFrom({ + ...sparsePullRequest, + updated_at: "not-a-date", + user: null, + }) + ).toBeNull(); + }); + test("retains foreign authors but rejects malformed provider timestamps", () => { const normalizedRepository = repositoryFrom(repository); expect(normalizedRepository).not.toBeNull(); @@ -785,6 +890,34 @@ describe("account pause state", () => { }); describe("bounded event collection", () => { + test("collects a real sparse pull request event without poisoning its page", async () => { + globalThis.fetch = async () => + Response.json([ + { + actor: { login: "f0rr0" }, + created_at: "2026-08-26T12:03:00Z", + id: "13", + payload: { + action: "merged", + number: 42, + pull_request: sparsePullRequest, + }, + repo: { id: 123, name: "another-org/private-repo" }, + type: "PullRequestEvent", + }, + accountEvent("12"), + ]); + + const collected = await collectGitHubEvents("f0rr0", "token", "12"); + expect(collected.latestEventId).toBe("13"); + expect(collected.events).toHaveLength(1); + expect(collected.events[0]?.pullRequestSignal).toEqual({ + action: "merged", + number: 42, + repositoryId: "123", + }); + }); + test("stops at the saved checkpoint without replaying it", async () => { globalThis.fetch = async () => Response.json([