Skip to content

Commit 88d76ed

Browse files
authored
Merge pull request #2332 from trycompai/main
[comp] Production Deploy
2 parents cda82f7 + e3a9867 commit 88d76ed

14 files changed

Lines changed: 517 additions & 64 deletions

File tree

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { filterUsersByOrgUnits } from './sync-ou-filter';
2+
3+
interface TestUser {
4+
primaryEmail: string;
5+
orgUnitPath: string;
6+
suspended?: boolean;
7+
}
8+
9+
describe('filterUsersByOrgUnits', () => {
10+
const users: TestUser[] = [
11+
{ primaryEmail: 'alice@example.com', orgUnitPath: '/' },
12+
{ primaryEmail: 'bob@example.com', orgUnitPath: '/Engineering' },
13+
{ primaryEmail: 'carol@example.com', orgUnitPath: '/Engineering/Frontend' },
14+
{ primaryEmail: 'dave@example.com', orgUnitPath: '/Marketing' },
15+
{ primaryEmail: 'eve@example.com', orgUnitPath: '/HR' },
16+
{
17+
primaryEmail: 'frank@example.com',
18+
orgUnitPath: '/Unlisted',
19+
suspended: true,
20+
},
21+
];
22+
23+
it('returns all users when no target OUs specified', () => {
24+
const result = filterUsersByOrgUnits(users, undefined);
25+
expect(result).toEqual(users);
26+
});
27+
28+
it('returns all users when target OUs is an empty array', () => {
29+
const result = filterUsersByOrgUnits(users, []);
30+
expect(result).toEqual(users);
31+
});
32+
33+
it('filters users to only those in selected OUs', () => {
34+
const result = filterUsersByOrgUnits(users, ['/Engineering']);
35+
expect(result.map((u) => u.primaryEmail)).toEqual([
36+
'bob@example.com',
37+
'carol@example.com',
38+
]);
39+
});
40+
41+
it('includes users in child OUs of selected OUs', () => {
42+
const result = filterUsersByOrgUnits(users, ['/Engineering']);
43+
expect(result.map((u) => u.primaryEmail)).toContain('carol@example.com');
44+
});
45+
46+
it('exact match on OU path works', () => {
47+
const result = filterUsersByOrgUnits(users, ['/Engineering/Frontend']);
48+
expect(result.map((u) => u.primaryEmail)).toEqual(['carol@example.com']);
49+
});
50+
51+
it('supports multiple target OUs', () => {
52+
const result = filterUsersByOrgUnits(users, ['/Engineering', '/Marketing']);
53+
expect(result.map((u) => u.primaryEmail)).toEqual([
54+
'bob@example.com',
55+
'carol@example.com',
56+
'dave@example.com',
57+
]);
58+
});
59+
60+
it('root OU includes all users', () => {
61+
const result = filterUsersByOrgUnits(users, ['/']);
62+
expect(result).toEqual(users);
63+
});
64+
65+
it('excludes users not in any selected OU', () => {
66+
const result = filterUsersByOrgUnits(users, ['/Engineering']);
67+
const emails = result.map((u) => u.primaryEmail);
68+
expect(emails).not.toContain('alice@example.com');
69+
expect(emails).not.toContain('dave@example.com');
70+
expect(emails).not.toContain('eve@example.com');
71+
expect(emails).not.toContain('frank@example.com');
72+
});
73+
74+
it('does not match partial OU path names', () => {
75+
// /Eng should NOT match /Engineering
76+
const result = filterUsersByOrgUnits(users, ['/Eng']);
77+
expect(result).toEqual([]);
78+
});
79+
80+
it('preserves suspended user status through filtering', () => {
81+
const result = filterUsersByOrgUnits(users, ['/Unlisted']);
82+
expect(result).toEqual([
83+
{
84+
primaryEmail: 'frank@example.com',
85+
orgUnitPath: '/Unlisted',
86+
suspended: true,
87+
},
88+
]);
89+
});
90+
});
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/**
2+
* Filters users by organizational unit paths.
3+
* Matches users whose orgUnitPath equals or is a child of any target OU.
4+
*
5+
* @param users - Array of objects with an orgUnitPath property
6+
* @param targetOrgUnits - Array of OU paths to include (undefined/empty = all users)
7+
* @returns Filtered array of users
8+
*/
9+
export function filterUsersByOrgUnits<
10+
T extends { orgUnitPath?: string },
11+
>(users: T[], targetOrgUnits: string[] | undefined): T[] {
12+
if (!targetOrgUnits || targetOrgUnits.length === 0) {
13+
return users;
14+
}
15+
16+
return users.filter((user) => {
17+
const userOu = user.orgUnitPath ?? '/';
18+
return targetOrgUnits.some(
19+
(ou) =>
20+
ou === '/' || userOu === ou || userOu.startsWith(`${ou}/`),
21+
);
22+
});
23+
}

‎apps/api/src/integration-platform/controllers/sync.controller.ts‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
import { RampRoleMappingService } from '../services/ramp-role-mapping.service';
3434
import { IntegrationSyncLoggerService } from '../services/integration-sync-logger.service';
3535
import { RampApiService } from '../services/ramp-api.service';
36+
import { filterUsersByOrgUnits } from './sync-ou-filter';
3637

3738
interface GoogleWorkspaceUser {
3839
id: string;
@@ -284,6 +285,19 @@ export class SyncController {
284285
string,
285286
unknown
286287
>;
288+
289+
// Filter by organizational unit if configured
290+
const targetOrgUnits = Array.isArray(syncVariables.target_org_units)
291+
? (syncVariables.target_org_units as string[])
292+
: undefined;
293+
const ouFilteredUsers = filterUsersByOrgUnits(users, targetOrgUnits);
294+
295+
if (targetOrgUnits && targetOrgUnits.length > 0) {
296+
this.logger.log(
297+
`Google Workspace OU filter kept ${ouFilteredUsers.length}/${users.length} users (OUs: ${targetOrgUnits.join(', ')})`,
298+
);
299+
}
300+
287301
const rawSyncFilterMode = syncVariables.sync_user_filter_mode;
288302
const syncFilterMode: GoogleWorkspaceSyncFilterMode =
289303
typeof rawSyncFilterMode === 'string' &&
@@ -307,7 +321,7 @@ export class SyncController {
307321
effectiveSyncFilterMode = 'all';
308322
}
309323

310-
const filteredUsers = users.filter((user) => {
324+
const filteredUsers = ouFilteredUsers.filter((user) => {
311325
const email = user.primaryEmail.toLowerCase();
312326

313327
if (effectiveSyncFilterMode === 'exclude' && excludedTerms.length > 0) {
@@ -322,7 +336,7 @@ export class SyncController {
322336
});
323337

324338
this.logger.log(
325-
`Google Workspace sync filter mode "${effectiveSyncFilterMode}" kept ${filteredUsers.length}/${users.length} users`,
339+
`Google Workspace sync filter mode "${effectiveSyncFilterMode}" kept ${filteredUsers.length}/${ouFilteredUsers.length} users`,
326340
);
327341

328342
// Active users to import/reactivate are based on the selected filter mode
@@ -336,10 +350,10 @@ export class SyncController {
336350
activeUsers.map((u) => u.primaryEmail.toLowerCase()),
337351
);
338352
const allSuspendedEmails = new Set(
339-
users.filter((u) => u.suspended).map((u) => u.primaryEmail.toLowerCase()),
353+
ouFilteredUsers.filter((u) => u.suspended).map((u) => u.primaryEmail.toLowerCase()),
340354
);
341355
const allActiveEmails = new Set(
342-
users
356+
ouFilteredUsers
343357
.filter((u) => !u.suspended)
344358
.map((u) => u.primaryEmail.toLowerCase()),
345359
);
@@ -467,7 +481,7 @@ export class SyncController {
467481

468482
const deactivationGwDomains =
469483
effectiveSyncFilterMode === 'include'
470-
? new Set(users.map((u) => u.primaryEmail.split('@')[1]?.toLowerCase()))
484+
? new Set(ouFilteredUsers.map((u) => u.primaryEmail.split('@')[1]?.toLowerCase()))
471485
: new Set(
472486
filteredUsers.map((u) =>
473487
u.primaryEmail.split('@')[1]?.toLowerCase(),

‎apps/api/src/trigger/policies/update-policy.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { logger, metadata, queue, schemaTask } from '@trigger.dev/sdk';
22
import { z } from 'zod';
33
import { processPolicyUpdate } from './update-policy-helpers';
44

5-
export const updatePolicyQueue = queue({ name: 'update-policy', concurrencyLimit: 50 });
5+
const updatePolicyQueue = queue({ name: 'update-policy', concurrencyLimit: 50 });
66

77
export const updatePolicy = schemaTask({
88
id: 'update-policy',

‎apps/api/src/trigger/vendor/vendor-risk-assessment/firecrawl-agent.ts‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import Firecrawl from '@mendable/firecrawl-js';
22
import { logger } from '@trigger.dev/sdk';
33
import { vendorRiskAssessmentAgentSchema } from './agent-schema';
44
import type { VendorRiskAssessmentDataV1 } from './agent-types';
5+
import { extractVendorDomain, validateVendorUrl } from './url-validation';
56

67
function normalizeUrl(url: string | null | undefined): string | null {
78
if (!url) return null;
@@ -58,19 +59,30 @@ export async function firecrawlAgentVendorRiskAssessment(params: {
5859

5960
const firecrawlClient = new Firecrawl({ apiKey });
6061

62+
// Extract vendor domain for URL validation
63+
const vendorDomain = extractVendorDomain(vendorWebsite);
64+
if (!vendorDomain) {
65+
logger.warn('Could not extract vendor domain for URL validation', {
66+
vendorWebsite,
67+
});
68+
return null;
69+
}
70+
6171
const prompt = `Complete cyber security research on the vendor "${vendorName}" with website ${vendorWebsite}.
6272
73+
CRITICAL: Only return URLs that belong to the domain "${vendorDomain}" or its subdomains (e.g., trust.${vendorDomain}, security.${vendorDomain}). Do NOT return URLs from any other domain. If you cannot find a page on ${vendorDomain}, return an empty string for that field rather than a URL from another website.
74+
6375
Extract the following information:
6476
1. **Certifications**: Find any security certifications they have (SOC 2 Type I, SOC 2 Type II, ISO 27001 etc). For each certification found, determine:
6577
- The type of certification
6678
- Whether it's verified/current, expired, or not certified
6779
- Any issue or expiry dates mentioned
68-
- Link to the compliance/trust page or report if available
80+
- Link to the compliance/trust page or report if available (must be on ${vendorDomain})
6981
70-
2. **Legal & Security Documents**: Find the direct URLs to:
82+
2. **Legal & Security Documents**: Find the direct URLs on ${vendorDomain} to:
7183
- Privacy Policy page (usually at /privacy, /privacy-policy, or linked in the footer)
7284
- Terms of Service page (usually at /terms, /tos, /terms-of-service, or linked in the footer)
73-
- Trust Center or Security page (typically could be at /trust, /security or trust.website.com or security.website.com)
85+
- Trust Center or Security page (typically could be at /trust, /security or trust.${vendorDomain} or security.${vendorDomain})
7486
7587
3. **Recent News**: Find recent news articles (last 12 months) about the company, especially:
7688
- Security incidents or data breaches
@@ -81,13 +93,26 @@ Extract the following information:
8193
8294
4. **Summary**: Provide an overall assessment of the vendor's security posture.
8395
84-
Focus on their official website (especially trust/security/compliance pages), press releases, and reputable news sources.`;
96+
Focus on their official website ${vendorWebsite} (especially trust/security/compliance pages), press releases, and reputable news sources.`;
97+
98+
// Provide seed URLs covering common legal/security paths so the agent
99+
// stays on the vendor's domain instead of wandering to unrelated sites.
100+
const seedUrls = [
101+
origin,
102+
`${origin}/privacy`,
103+
`${origin}/privacy-policy`,
104+
`${origin}/terms`,
105+
`${origin}/terms-of-service`,
106+
`${origin}/security`,
107+
`${origin}/trust`,
108+
`${origin}/legal`,
109+
`${origin}/compliance`,
110+
];
85111

86-
// Using SDK (no maxCredits override, no explicit polling here)
87-
// Important: avoid crawling huge sites with a wildcard (e.g. workspace.google.com).
88112
const agentResponse = await firecrawlClient.agent({
89113
prompt,
90-
urls: [origin],
114+
urls: seedUrls,
115+
strictConstrainToURLs: false, // allow following links from seed URLs, but seeds anchor it to the right domain
91116
schema: {
92117
type: 'object',
93118
properties: {
@@ -173,7 +198,10 @@ Focus on their official website (especially trust/security/compliance pages), pr
173198
});
174199

175200
const normalizedLinks = linkPairs
176-
.map((l) => ({ ...l, url: normalizeUrl(l.url) }))
201+
.map((l) => ({
202+
...l,
203+
url: validateVendorUrl(l.url, vendorDomain, l.label),
204+
}))
177205
.filter((l): l is { label: string; url: string } => Boolean(l.url));
178206

179207
const certifications =
@@ -182,7 +210,7 @@ Focus on their official website (especially trust/security/compliance pages), pr
182210
status: c.status ?? 'unknown',
183211
issuedAt: normalizeIso(c.issued_at ?? null),
184212
expiresAt: normalizeIso(c.expires_at ?? null),
185-
url: normalizeUrl(c.url ?? null),
213+
url: validateVendorUrl(c.url ?? null, vendorDomain, `cert:${c.type}`),
186214
})) ?? [];
187215

188216
const news =

‎apps/api/src/trigger/vendor/vendor-risk-assessment/firecrawl.ts‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { logger } from '@trigger.dev/sdk';
22
import { firecrawlVendorDataSchema, type FirecrawlVendorData } from './schema';
3+
import { extractVendorDomain, isUrlFromVendorDomain } from './url-validation';
34

45
type FirecrawlStartResponse = {
56
success: boolean;
@@ -48,6 +49,14 @@ function normalizeUrl(url: string | null | undefined): string | null {
4849
export async function firecrawlExtractVendorData(
4950
website: string,
5051
): Promise<FirecrawlVendorData | null> {
52+
// Extract vendor domain for URL validation
53+
const vendorDomain = extractVendorDomain(website);
54+
if (!vendorDomain) {
55+
logger.warn('Could not extract vendor domain for URL validation', {
56+
website,
57+
});
58+
return null;
59+
}
5160
const apiKey = process.env.FIRECRAWL_API_KEY;
5261
if (!apiKey) {
5362
logger.warn(
@@ -77,9 +86,11 @@ export async function firecrawlExtractVendorData(
7786
7887
Goal: return the MOST SPECIFIC, DIRECT URL for each document type below. Do not return general category pages.
7988
80-
You may crawl the site (including subdomains) and follow internal links. Trust portals are often linked in the header/footer under: "Trust", "Trust Center", "Security", "Compliance", "Legal", "Governance", "Privacy", "Data Processing", "DPA".
89+
CRITICAL: Only return URLs that belong to the domain "${vendorDomain}" or its subdomains (e.g., trust.${vendorDomain}, security.${vendorDomain}). Do NOT return URLs from any other domain. If you cannot find a page on ${vendorDomain}, return an empty string for that field.
8190
82-
Return ONLY absolute https URLs. If you cannot find a dedicated page that matches the definition, return an empty string.
91+
You may crawl the site (including subdomains of ${vendorDomain}) and follow internal links. Trust portals are often linked in the header/footer under: "Trust", "Trust Center", "Security", "Compliance", "Legal", "Governance", "Privacy", "Data Processing", "DPA".
92+
93+
Return ONLY absolute https URLs on ${vendorDomain}. If you cannot find a dedicated page that matches the definition, return an empty string.
8394
8495
DEFINITIONS (be strict):
8596
1) trust_portal_url:
@@ -193,13 +204,46 @@ When multiple candidates exist, choose the most direct URL that best matches the
193204
return null;
194205
}
195206

207+
// Normalize URLs and filter out any that don't belong to the vendor's domain
208+
const validateVendorUrl = (
209+
url: string | null | undefined,
210+
label: string,
211+
): string | null => {
212+
const normalized = normalizeUrl(url);
213+
if (!normalized) return null;
214+
if (!isUrlFromVendorDomain(normalized, vendorDomain)) {
215+
logger.warn('Filtered out URL from wrong domain', {
216+
vendorDomain,
217+
label,
218+
url: normalized,
219+
});
220+
return null;
221+
}
222+
return normalized;
223+
};
224+
196225
const normalized = {
197226
...parsed.data,
198-
privacy_policy_url: normalizeUrl(parsed.data.privacy_policy_url),
199-
terms_of_service_url: normalizeUrl(parsed.data.terms_of_service_url),
200-
security_overview_url: normalizeUrl(parsed.data.security_overview_url),
201-
trust_portal_url: normalizeUrl(parsed.data.trust_portal_url),
202-
soc2_report_url: normalizeUrl(parsed.data.soc2_report_url),
227+
privacy_policy_url: validateVendorUrl(
228+
parsed.data.privacy_policy_url,
229+
'privacy_policy',
230+
),
231+
terms_of_service_url: validateVendorUrl(
232+
parsed.data.terms_of_service_url,
233+
'terms_of_service',
234+
),
235+
security_overview_url: validateVendorUrl(
236+
parsed.data.security_overview_url,
237+
'security_overview',
238+
),
239+
trust_portal_url: validateVendorUrl(
240+
parsed.data.trust_portal_url,
241+
'trust_portal',
242+
),
243+
soc2_report_url: validateVendorUrl(
244+
parsed.data.soc2_report_url,
245+
'soc2_report',
246+
),
203247
};
204248

205249
logger.info('Firecrawl extraction completed', {

0 commit comments

Comments
 (0)