diff --git a/.changeset/replay-json-ld-events.md b/.changeset/replay-json-ld-events.md new file mode 100644 index 0000000000..8ee130cd13 --- /dev/null +++ b/.changeset/replay-json-ld-events.md @@ -0,0 +1,6 @@ +--- +'posthog-js': minor +'@posthog/types': patch +--- + +Add opt-in Schema.org JSON-LD capture to session replay through `session_recording.captureJsonLd`. When enabled, the recorder emits sanitized JSON-LD as custom replay events and excludes all script elements from replay snapshots. diff --git a/packages/browser/playwright/mocked/session-recording/session-recording-masking.spec.ts b/packages/browser/playwright/mocked/session-recording/session-recording-masking.spec.ts index a6f4701abf..259338725e 100644 --- a/packages/browser/playwright/mocked/session-recording/session-recording-masking.spec.ts +++ b/packages/browser/playwright/mocked/session-recording/session-recording-masking.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '../utils/posthog-playwright-test-base' -import { start } from '../utils/setup' +import { start, waitForSessionRecordingToStart } from '../utils/setup' import { Page } from '@playwright/test' import { CaptureResult } from '@/types' @@ -8,11 +8,12 @@ import { CaptureResult } from '@/types' const remoteMaskingTextSelector = '*' -const startOptions = (masking: Record) => ({ +const startOptions = (masking: Record, captureJsonLd = false) => ({ options: { session_recording: { // not the default but makes for easier test assertions compress_events: false, + captureJsonLd, }, }, flagsResponseOverrides: { @@ -55,6 +56,140 @@ function assertTheConfigIsAsExpected(snapshotEvents: CaptureResult[], expectedMa } test.describe('Session recording - masking', () => { + test('emits only sanitized JSON-LD in recording bytes', async ({ page, context }) => { + await page.addInitScript(() => { + const appendJsonLd = (value: Record, className = '') => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.className = className + script.setAttribute('data-private', 'PRIVATE_ATTRIBUTE') + script.textContent = JSON.stringify(value) + script.append(document.createComment('PRIVATE_COMMENT')) + document.head.append(script) + } + const appendInitialJsonLd = () => { + appendJsonLd({ + '@context': 'https://schema.org', + '@type': 'Product', + '@id': 'ALLOWED_PRODUCT_ID', + name: 'ALLOWED_INITIAL_PRODUCT', + email: 'PRIVATE_UNAPPROVED_EMAIL', + description: 'PRIVATE_DESCRIPTION', + url: 'https://example.com/?token=PRIVATE_URL_TOKEN', + brand: { + '@type': 'Person', + name: 'PRIVATE_NESTED_PERSON', + }, + manufacturer: { + '@type': 'Organization', + name: 'ALLOWED_MANUFACTURER', + legalName: 'ALLOWED_MANUFACTURER_LEGAL_NAME', + email: 'PRIVATE_MANUFACTURER_EMAIL', + }, + }) + appendJsonLd({ + '@context': 'https://schema.org', + '@graph': [ + { + '@type': 'WebSite', + inLanguage: 'ALLOWED_GRAPH_LANGUAGE', + email: 'PRIVATE_GRAPH_EMAIL', + }, + { + '@type': 'PrivateType', + name: 'PRIVATE_GRAPH_ENTITY', + }, + ], + }) + appendJsonLd( + { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'PRIVATE_MASKED_PRODUCT', + }, + 'json-ld-mask' + ) + } + if (document.head) { + appendInitialJsonLd() + } else { + document.onreadystatechange = () => { + if (document.head) { + document.onreadystatechange = null + appendInitialJsonLd() + } + } + } + }) + await start( + startOptions( + { + maskAllInputs: true, + maskTextSelector: '.json-ld-mask', + }, + true + ), + page, + context + ) + await waitForSessionRecordingToStart(page) + + await page.evaluate(() => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'ALLOWED_DYNAMIC_PRODUCT', + email: 'PRIVATE_DYNAMIC_EMAIL', + }) + document.head.append(script) + + const maskedScript = document.createElement('script') + maskedScript.type = 'application/ld+json' + maskedScript.className = 'json-ld-mask' + maskedScript.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'PRIVATE_DYNAMIC_MASKED_PRODUCT', + }) + document.head.append(maskedScript) + }) + await page.locator('[data-cy-input]').type('flush recording') + + const getEventBytes = async () => + JSON.stringify( + (await page.capturedEvents()) + .filter((event) => event.event === '$snapshot') + .flatMap((event) => event.properties['$snapshot_data']) + ) + await expect.poll(getEventBytes).toContain('ALLOWED_DYNAMIC_PRODUCT') + const eventBytes = await getEventBytes() + expect(eventBytes).toContain('ALLOWED_PRODUCT_ID') + expect(eventBytes).toContain('ALLOWED_INITIAL_PRODUCT') + expect(eventBytes).toContain('ALLOWED_DYNAMIC_PRODUCT') + expect(eventBytes).toContain('ALLOWED_MANUFACTURER') + expect(eventBytes).toContain('ALLOWED_MANUFACTURER_LEGAL_NAME') + expect(eventBytes).toContain('ALLOWED_GRAPH_LANGUAGE') + expect(eventBytes).not.toContain('"tagName":"script"') + for (const privateMarker of [ + 'PRIVATE_ATTRIBUTE', + 'PRIVATE_COMMENT', + 'PRIVATE_UNAPPROVED_EMAIL', + 'PRIVATE_DESCRIPTION', + 'PRIVATE_URL_TOKEN', + 'PRIVATE_NESTED_PERSON', + 'PRIVATE_MANUFACTURER_EMAIL', + 'PRIVATE_GRAPH_EMAIL', + 'PRIVATE_GRAPH_ENTITY', + 'PRIVATE_MASKED_PRODUCT', + 'PRIVATE_DYNAMIC_EMAIL', + 'PRIVATE_DYNAMIC_MASKED_PRODUCT', + ]) { + expect(eventBytes).not.toContain(privateMarker) + } + }) + test('masks text', async ({ page, context }) => { await start( startOptions({ diff --git a/packages/browser/src/__tests__/extensions/replay/json-ld.test.ts b/packages/browser/src/__tests__/extensions/replay/json-ld.test.ts new file mode 100644 index 0000000000..85a1b76805 --- /dev/null +++ b/packages/browser/src/__tests__/extensions/replay/json-ld.test.ts @@ -0,0 +1,658 @@ +import { sanitizeJsonLd, startJsonLdCapture } from '../../../extensions/replay/external/json-ld' + +const GOOGLE_SEARCH_TYPES = + '3DModel Accommodation Action AdministrativeArea AggregateOffer AggregateRating AlignmentObject Answer Article BedDetails Blog BlogPosting Book BorrowAction Brand BreadcrumbList BroadcastEvent Car Certification Clip Comment ContactPoint Country Course CreativeWork CreativeWorkSeason CreativeWorkSeries CreditCard DataCatalog DataDownload DataFeed Dataset DaySpa DefinedRegion DiscussionForumPosting EducationalOccupationalCredential Electrician EmployerAggregateRating EntryPoint Episode Event Game GeoCoordinates GeoShape HealthClub Hotel HowTo HowToDirection HowToSection HowToStep HowToTip ImageObject InteractionCounter ItemList JobPosting LearningResource Library LibrarySystem ListItem LocalBusiness LocationFeatureSpecification Locksmith LodgingBusiness MathSolver MediaObject MemberProgram MemberProgramTier MerchantReturnPolicy MerchantReturnPolicySeasonalOverride Message MobileApplication MonetaryAmount Movie MusicPlaylist MusicRecording NewsArticle NutritionInformation OccupationalExperienceRequirements Offer OfferShippingDetails OnlineStore OpeningHoursSpecification Organization PeopleAudience PerformingGroup Person Pharmacy Place Plumber PostalAddress PriceSpecification Product ProductGroup ProfilePage PropertyValue QAPage QuantitativeValue Question Quiz Rating ReadAction Recipe Restaurant Review SeekToAction ServicePeriod ShippingConditions ShippingDeliveryTime ShippingRateSettings ShippingService SocialMediaPosting SoftwareApplication SolveMathAction SpeakableSpecification State Store Thing UnitPriceSpecification VacationRental VideoGame VideoObject WatchAction WebApplication WebPage WebPageElement'.split( + ' ' + ) +const COMMON_SCHEMA_TYPES = + 'AboutPage AudioObject AutoDealer Bakery BarOrPub BusinessEvent CafeOrCoffeeShop CollegeOrUniversity CollectionPage ContactPage Corporation Dentist EducationEvent EducationalOrganization FAQPage Festival FoodEstablishment GovernmentOrganization IndividualProduct LegalService MedicalBusiness MusicEvent NGO OfferCatalog Photograph Physician PodcastEpisode PodcastSeries ProductModel RealEstateAgent ScholarlyArticle School SearchAction SearchResultsPage Service SiteNavigationElement SportsEvent SportsOrganization TVEpisode TVSeries TechArticle TheaterEvent WebSite'.split( + ' ' + ) + +function jsonLdScript(value: unknown): HTMLScriptElement { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify(value) + return script +} + +async function deliverMutations(): Promise { + await Promise.resolve() + await Promise.resolve() +} + +describe('JSON-LD replay capture', () => { + afterEach(() => { + document.body.replaceChildren() + }) + + it('accepts every Google-listed type', () => { + for (const type of GOOGLE_SEARCH_TYPES) { + expect(sanitizeJsonLd(JSON.stringify({ '@context': 'https://schema.org', '@type': type }))?.[0]).toEqual({ + '@context': 'https://schema.org', + '@type': type, + }) + } + }) + + it('accepts common Schema.org types outside the Google list', () => { + for (const type of COMMON_SCHEMA_TYPES) { + expect(sanitizeJsonLd(JSON.stringify({ '@context': 'https://schema.org', '@type': type }))?.[0]).toEqual({ + '@context': 'https://schema.org', + '@type': type, + }) + } + }) + + it('sanitizes root graphs and drops unsupported graph entities', () => { + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@graph': [ + { + '@type': 'WebSite', + datePublished: '2026-08-25', + email: 'private@example.com', + potentialAction: { + '@type': 'SearchAction', + actionStatus: 'https://schema.org/PotentialActionStatus', + target: 'https://example.com/search?q={private}', + }, + }, + { + '@type': 'FAQPage', + inLanguage: 'en', + text: 'Private question and answer', + }, + { + '@type': 'Person', + '@id': 'person-id', + name: 'Private name', + }, + { + '@type': 'PrivateType', + email: 'private@example.com', + }, + 'private@example.com', + ], + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@graph': [ + { + '@type': 'WebSite', + datePublished: '2026-08-25', + potentialAction: { + '@type': 'SearchAction', + actionStatus: 'https://schema.org/PotentialActionStatus', + }, + }, + { + '@type': 'FAQPage', + inLanguage: 'en', + }, + { + '@type': 'Person', + '@id': 'person-id', + }, + ], + }) + }) + + it.each(['BreadcrumbList', 'ItemList'])('sanitizes nested items in %s', (type) => { + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': type, + itemListElement: [ + { + '@type': 'ListItem', + position: 1, + name: 'Private label', + item: { + '@type': 'Product', + name: 'Camera', + email: 'private@example.com', + }, + }, + { + '@type': 'Person', + '@id': 'private-person', + }, + ], + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@type': type, + itemListElement: [ + { + '@type': 'ListItem', + position: 1, + item: { + '@type': 'Product', + name: 'Camera', + }, + }, + ], + }) + }) + + it('sanitizes offer catalogs and services', () => { + expect( + sanitizeJsonLd( + JSON.stringify([ + { + '@context': 'https://schema.org', + '@type': 'OfferCatalog', + name: 'Services', + itemListElement: { + '@type': 'Offer', + price: 100, + email: 'private@example.com', + }, + }, + { + '@context': 'https://schema.org', + '@type': 'Service', + name: 'Installation', + serviceType: 'Installation', + email: 'private@example.com', + provider: { + '@type': 'EducationalOrganization', + name: 'Acme', + telephone: '+44 0000 000000', + }, + }, + ]) + )?.[0] + ).toEqual([ + { + '@context': 'https://schema.org', + '@type': 'OfferCatalog', + name: 'Services', + itemListElement: { + '@type': 'Offer', + price: 100, + }, + }, + { + '@context': 'https://schema.org', + '@type': 'Service', + name: 'Installation', + serviceType: 'Installation', + provider: { + '@type': 'EducationalOrganization', + name: 'Acme', + }, + }, + ]) + }) + + it('sanitizes type arrays and full Schema.org type URLs', () => { + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': ['https://schema.org/Product', 'Car', 'PrivateType', 42], + name: 'Camera', + email: 'private@example.com', + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@type': ['Product', 'Car'], + name: 'Camera', + }) + + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'https://schema.org/Organization', + name: 'Acme', + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@type': 'Organization', + name: 'Acme', + }) + }) + + it('keeps only path-allowed properties and @id values', () => { + const sanitized = sanitizeJsonLd( + JSON.stringify({ + '@context': 'http://schema.org/', + '@type': 'Product', + '@id': 'https://example.com/products/123', + name: 'Camera', + email: 'private@example.com', + manufacturer: { + '@type': 'Organization', + '@id': 'https://example.com/organizations/acme', + name: 'Acme', + email: 'private@example.com', + }, + offers: { + '@type': 'Offer', + price: 100, + seller: { + '@type': 'Person', + name: 'Private name', + }, + }, + }) + ) + + expect(sanitized?.[0]).toEqual({ + '@context': 'https://schema.org', + '@type': 'Product', + '@id': 'https://example.com/products/123', + name: 'Camera', + manufacturer: { + '@type': 'Organization', + '@id': 'https://example.com/organizations/acme', + name: 'Acme', + }, + offers: { + '@type': 'Offer', + price: 100, + }, + }) + }) + + it('keeps scalar leaf values and drops non-scalar leaf values', () => { + const sanitized = sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: ['Camera', 2, true, null], + sku: { value: 'private' }, + color: ['black', { value: 'private' }], + category: false, + }) + ) + + expect(sanitized?.[0]).toEqual({ + '@context': 'https://schema.org', + '@type': 'Product', + name: ['Camera', 2, true, null], + category: false, + }) + }) + + it('keeps non-PII leaf properties without a type-specific path', () => { + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Thing', + availability: 'https://schema.org/InStock', + isAccessibleForFree: true, + name: 'Private name', + numberOfItems: 2, + priceCurrency: 'GBP', + ratingValue: 4.5, + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@type': 'Thing', + availability: 'https://schema.org/InStock', + isAccessibleForFree: true, + numberOfItems: 2, + priceCurrency: 'GBP', + ratingValue: 4.5, + }) + }) + + it('sanitizes root and nested entity arrays', () => { + expect( + sanitizeJsonLd( + JSON.stringify([ + { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + offers: [ + { '@type': 'Offer', price: 100, email: 'private@example.com' }, + { '@type': 'Person', '@id': 'private-person' }, + ], + }, + { '@context': 'https://schema.org', '@type': 'Organization', name: 'Acme' }, + ]) + )?.[0] + ).toEqual([ + { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + offers: [{ '@type': 'Offer', price: 100 }], + }, + { '@context': 'https://schema.org', '@type': 'Organization', name: 'Acme' }, + ]) + }) + + it.each([ + 'not json', + JSON.stringify({ '@context': 'https://example.com', '@type': 'Product' }), + JSON.stringify({ '@context': 'https://schema.org', '@type': 'PrivateType' }), + JSON.stringify({ '@context': 'https://schema.org', '@type': ['PrivateType', 'OtherPrivateType'] }), + JSON.stringify({ '@context': 'https://schema.org', '@graph': [{ '@type': 'PrivateType' }] }), + JSON.stringify({ '@context': 'https://schema.org', '@type': 'constructor', '@id': 'private@example.com' }), + JSON.stringify({ '@context': 'https://schema.org', '@type': 'toString', '@id': 'private@example.com' }), + JSON.stringify({ '@context': 'https://schema.org', '@type': '__proto__', '@id': 'private@example.com' }), + JSON.stringify([ + { '@context': 'https://schema.org', '@type': 'Product' }, + { '@context': 'https://schema.org', '@type': 'PrivateType' }, + ]), + ])('drops an invalid JSON-LD document', (value) => { + expect(sanitizeJsonLd(value)).toBeNull() + }) + + it.each(['ContactPoint', 'Person', 'PostalAddress'])('drops PII-bearing %s properties', (type) => { + expect( + sanitizeJsonLd( + JSON.stringify({ + '@context': 'https://schema.org', + '@type': type, + '@id': 'entity-id', + name: 'Private name', + email: 'private@example.com', + telephone: '+44 0000 000000', + streetAddress: 'Private address', + }) + )?.[0] + ).toEqual({ + '@context': 'https://schema.org', + '@type': type, + '@id': 'entity-id', + }) + }) + + it('ignores inherited JSON-LD properties', () => { + const properties = ['@context', '@type', '@id', 'name', 'ratingValue'] + const values = ['https://schema.org', 'Product', 'private-id', 'private-name', 5] + const descriptors = properties.map((property) => Object.getOwnPropertyDescriptor(Object.prototype, property)) + let inheritedContext: ReturnType + let inheritedType: ReturnType + let inheritedLeaves: ReturnType + + try { + properties.forEach((property, index) => { + Object.defineProperty(Object.prototype, property, { + configurable: true, + value: values[index], + }) + }) + inheritedContext = sanitizeJsonLd(JSON.stringify({ '@type': 'Product' })) + inheritedType = sanitizeJsonLd(JSON.stringify({ '@context': 'https://schema.org' })) + inheritedLeaves = sanitizeJsonLd(JSON.stringify({ '@context': 'https://schema.org', '@type': 'Product' })) + } finally { + properties.forEach((property, index) => { + const descriptor = descriptors[index] + if (descriptor) { + Object.defineProperty(Object.prototype, property, descriptor) + } else { + Reflect.deleteProperty(Object.prototype, property) + } + }) + } + + expect(inheritedContext).toBeNull() + expect(inheritedType).toBeNull() + expect(inheritedLeaves?.[0]).toEqual({ + '@context': 'https://schema.org', + '@type': 'Product', + }) + }) + + it('emits initial, added, and changed JSON-LD without duplicates', async () => { + const emit = jest.fn(() => true) + const initial = jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'One' }) + document.body.appendChild(initial) + + const capture = startJsonLdCapture(document, MutationObserver, { + blockClass: 'ph-no-capture', + maskTextClass: 'ph-mask', + emit, + }) + capture.scan() + + expect(emit).toHaveBeenCalledWith({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'One', + }) + + const added = jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'Two' }) + document.body.appendChild(added) + await deliverMutations() + + added.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Two', + email: 'private@example.com', + }) + await deliverMutations() + + added.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Three', + }) + await deliverMutations() + + expect(emit.mock.calls).toEqual([ + [{ '@context': 'https://schema.org', '@type': 'Product', name: 'One' }], + [{ '@context': 'https://schema.org', '@type': 'Product', name: 'Two' }], + [{ '@context': 'https://schema.org', '@type': 'Product', name: 'Three' }], + ]) + + capture.stop() + document.body.appendChild(jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'Four' })) + await deliverMutations() + expect(emit).toHaveBeenCalledTimes(3) + }) + + it('does not scan subtrees for ordinary text changes', async () => { + const text = document.createTextNode('before') + document.body.append(text) + const querySelectorAll = jest.spyOn(Element.prototype, 'querySelectorAll') + const capture = startJsonLdCapture(document, MutationObserver, { emit: jest.fn(() => true) }) + querySelectorAll.mockClear() + + text.data = 'after' + await deliverMutations() + + expect(querySelectorAll).not.toHaveBeenCalled() + querySelectorAll.mockRestore() + capture.stop() + }) + + it('limits the total JSON-LD emitted by one recorder', async () => { + for (let index = 0; index < 6; index++) { + document.body.appendChild( + jsonLdScript({ + '@context': 'https://schema.org', + '@type': 'Product', + name: `${index}${'x'.repeat(19_000)}`, + }) + ) + } + const emit = jest.fn(() => true) + + const capture = startJsonLdCapture(document, MutationObserver, { emit }) + capture.scan() + const querySelectorAll = jest.spyOn(Element.prototype, 'querySelectorAll') + const container = document.createElement('div') + container.appendChild(jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product' })) + document.body.append(container) + await deliverMutations() + + expect(emit).toHaveBeenCalledTimes(5) + expect(querySelectorAll).not.toHaveBeenCalled() + querySelectorAll.mockRestore() + capture.stop() + }) + + it('rescans after capture becomes enabled', () => { + let enabled = false + const emit = jest.fn(() => true) + document.body.appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'Camera' }) + ) + const capture = startJsonLdCapture(document, MutationObserver, { + emit, + getCaptureState: () => enabled, + }) + + expect(emit).not.toHaveBeenCalled() + enabled = true + capture.scan() + expect(emit).toHaveBeenCalledWith({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + }) + capture.stop() + }) + + it('drops scripts moved before capture into a masked shadow root or another document', async () => { + const emit = jest.fn(() => true) + const capture = startJsonLdCapture(document, MutationObserver, { + maskTextClass: 'ph-mask', + emit, + }) + const shadowHost = document.createElement('div') + shadowHost.className = 'ph-mask' + const shadowRoot = shadowHost.attachShadow({ mode: 'open' }) + document.body.append(shadowHost) + const shadowScript = jsonLdScript({ + '@context': 'https://schema.org', + '@type': 'Person', + '@id': 'shadow-private', + }) + document.body.append(shadowScript) + shadowRoot.append(shadowScript) + + const iframe = document.createElement('iframe') + document.body.append(iframe) + const frameScript = jsonLdScript({ + '@context': 'https://schema.org', + '@type': 'Person', + '@id': 'frame-private', + }) + document.body.append(frameScript) + iframe.contentDocument!.body.append(frameScript) + await deliverMutations() + + expect(emit).not.toHaveBeenCalled() + capture.stop() + }) + + it('drops JSON-LD inside text masks and blocked elements', async () => { + const emit = jest.fn(() => true) + document.body.innerHTML = '
' + document.body.children[0].appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Person', '@id': 'masked' }) + ) + document.body.children[1].appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Person', '@id': 'blocked' }) + ) + + const capture = startJsonLdCapture(document, MutationObserver, { + blockClass: 'ph-no-capture', + blockSelector: '.private', + maskTextClass: 'ph-mask', + emit, + }) + capture.scan() + + expect(emit).not.toHaveBeenCalled() + + document.body.children[0].appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Person', '@id': 'dynamic-masked' }) + ) + document.body.children[1].appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Person', '@id': 'dynamic-blocked' }) + ) + await deliverMutations() + expect(emit).not.toHaveBeenCalled() + + const transient = jsonLdScript({ '@context': 'https://schema.org', '@type': 'Person', '@id': 'transient' }) + document.body.children[0].appendChild(transient) + transient.remove() + await deliverMutations() + expect(emit).not.toHaveBeenCalled() + capture.stop() + }) + + it('ignores non-JSON-LD scripts until their type changes', async () => { + const emit = jest.fn(() => true) + const value = JSON.stringify({ '@context': 'https://schema.org', '@type': 'Product', name: 'Camera' }) + const scripts = ['', 'text/javascript', 'application/json'].map((type) => { + const script = document.createElement('script') + script.type = type + if (type === 'application/json') { + script.textContent = value + } + document.body.appendChild(script) + script.textContent = value + return script + }) + const capture = startJsonLdCapture(document, MutationObserver, { emit }) + + capture.scan() + expect(emit).not.toHaveBeenCalled() + + scripts[2].type = 'application/ld+json' + await deliverMutations() + expect(emit).toHaveBeenCalledWith({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + }) + capture.stop() + }) + + it('does not deduplicate an event that the recorder rejects', () => { + let acceptsEvents = false + const emit = jest.fn(() => acceptsEvents) + document.body.appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'Camera' }) + ) + const capture = startJsonLdCapture(document, MutationObserver, { emit }) + + capture.scan() + acceptsEvents = true + capture.scan() + capture.scan() + + expect(emit).toHaveBeenCalledTimes(2) + capture.stop() + }) + + it('retries an event that a forced scan cannot emit', () => { + let acceptsEvents = true + const emit = jest.fn(() => acceptsEvents) + document.body.appendChild( + jsonLdScript({ '@context': 'https://schema.org', '@type': 'Product', name: 'Camera' }) + ) + const capture = startJsonLdCapture(document, MutationObserver, { emit }) + + capture.scan() + acceptsEvents = false + capture.scan(true) + acceptsEvents = true + capture.scan() + + expect(emit).toHaveBeenCalledTimes(3) + capture.stop() + }) +}) diff --git a/packages/browser/src/__tests__/extensions/replay/lazy-sessionrecording.test.ts b/packages/browser/src/__tests__/extensions/replay/lazy-sessionrecording.test.ts index fd0fc32c10..1dbfc828e3 100644 --- a/packages/browser/src/__tests__/extensions/replay/lazy-sessionrecording.test.ts +++ b/packages/browser/src/__tests__/extensions/replay/lazy-sessionrecording.test.ts @@ -3461,6 +3461,230 @@ describe('Lazy SessionRecording', () => { }) }) + it('removes scripts when JSON-LD capture is enabled', () => { + posthog.config.session_recording.slimDOMOptions = { script: false, comment: true } + posthog.config.session_recording.captureJsonLd = true + + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + + expect(assignableWindow.__PosthogExtensions__.rrweb.record).toHaveBeenCalledWith( + expect.objectContaining({ + slimDOMOptions: { script: true, comment: true }, + }) + ) + }) + + it('emits sanitized JSON-LD only while capture is enabled', async () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + '@id': 'https://example.com/products/123', + name: 'Camera', + email: 'private@example.com', + }) + document.body.appendChild(script) + posthog.config.session_recording.captureJsonLd = true + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + + expect(_addCustomEvent).not.toHaveBeenCalledWith('$json_ld', expect.anything()) + _emit(createMetaSnapshot()) + await Promise.resolve() + + expect(_addCustomEvent).toHaveBeenCalledWith('$json_ld', { + '@context': 'https://schema.org', + '@type': 'Product', + '@id': 'https://example.com/products/123', + name: 'Camera', + }) + + posthog.config.session_recording.captureJsonLd = false + document.body.appendChild( + Object.assign(document.createElement('script'), { + type: 'application/ld+json', + textContent: JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'After disable', + }), + }) + ) + await Promise.resolve() + expect(_addCustomEvent).not.toHaveBeenCalledWith( + '$json_ld', + expect.objectContaining({ name: 'After disable' }) + ) + } finally { + document.querySelectorAll('script[type="application/ld+json"]').forEach((element) => element.remove()) + } + }) + + it('emits the latest JSON-LD after returning from idle', async () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Before idle', + }) + document.body.appendChild(script) + posthog.config.session_recording.captureJsonLd = true + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + _emit(createMetaSnapshot()) + await Promise.resolve() + _addCustomEvent.mockClear() + + const lazyRecorder = sessionRecording['_lazyLoadedSessionRecording'] + lazyRecorder['_isIdle'] = true + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'After idle', + }) + await Promise.resolve() + expect(_addCustomEvent).not.toHaveBeenCalledWith( + '$json_ld', + expect.objectContaining({ name: 'After idle' }) + ) + + _emit(createIncrementalSnapshot({ timestamp: Date.now() + 1 })) + await Promise.resolve() + expect(_addCustomEvent).toHaveBeenCalledWith('$json_ld', { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'After idle', + }) + } finally { + script.remove() + } + }) + + it('does not queue JSON-LD before rrweb is ready', async () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Before disable', + }) + document.body.appendChild(script) + posthog.config.session_recording.captureJsonLd = true + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + expect(_addCustomEvent).not.toHaveBeenCalledWith('$json_ld', expect.anything()) + + posthog.config.session_recording.captureJsonLd = false + _emit(createMetaSnapshot()) + await Promise.resolve() + + expect(_addCustomEvent).not.toHaveBeenCalledWith('$json_ld', expect.anything()) + expect(sessionRecording['_lazyLoadedSessionRecording']['_queuedRRWebEvents']).toEqual([]) + } finally { + script.remove() + } + }) + + it('restores JSON-LD after a pending-trigger snapshot truncates the buffer', async () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + }) + document.body.appendChild(script) + posthog.config.session_recording.captureJsonLd = true + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + _addCustomEvent.mockImplementation((tag: string, payload: unknown) => { + _emit(createCustomSnapshot({}, payload as Record, tag)) + }) + _emit(createMetaSnapshot({ data: { href: 'https://test.com/first' } })) + await Promise.resolve() + + const lazyRecorder = sessionRecording['_lazyLoadedSessionRecording'] + const pendingTrigger = jest + .spyOn(lazyRecorder['_strategy']!, 'hasPendingTriggers') + .mockReturnValue(true) + try { + _emit(createMetaSnapshot({ data: { href: 'https://test.com/second' } })) + _emit(createFullSnapshot()) + await Promise.resolve() + + const bufferedEvents = lazyRecorder['_buffer'].data + const fullSnapshotIndex = bufferedEvents.findIndex((event: eventWithTime) => event.type === 2) + const jsonLdIndexes = bufferedEvents + .map((event: eventWithTime, index: number) => (event.data?.tag === '$json_ld' ? index : -1)) + .filter((index: number) => index >= 0) + expect(bufferedEvents[0]).toEqual(createMetaSnapshot({ data: { href: 'https://test.com/second' } })) + expect(jsonLdIndexes).toHaveLength(1) + expect(jsonLdIndexes[0]).toBeGreaterThan(fullSnapshotIndex) + expect(bufferedEvents[jsonLdIndexes[0]]).toEqual( + createCustomSnapshot( + {}, + { '@context': 'https://schema.org', '@type': 'Product', name: 'Camera' }, + '$json_ld' + ) + ) + } finally { + pendingTrigger.mockRestore() + } + } finally { + _addCustomEvent.mockReset() + script.remove() + } + }) + + it('does not emit JSON-LD by default', () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + }) + document.body.appendChild(script) + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + + expect(_addCustomEvent.mock.calls.some(([tag]) => tag === '$json_ld')).toBe(false) + } finally { + script.remove() + } + }) + + it('does not enable JSON-LD capture for a truthy non-boolean value', () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Camera', + }) + document.body.appendChild(script) + posthog.config.session_recording.slimDOMOptions = { script: false } + posthog.config.session_recording.captureJsonLd = 'false' as unknown as boolean + + try { + sessionRecording.onRemoteConfig(makeFlagsResponse({ sessionRecording: { endpoint: '/s/' } })) + + expect(_addCustomEvent.mock.calls.some(([tag]) => tag === '$json_ld')).toBe(false) + expect(assignableWindow.__PosthogExtensions__.rrweb.record).toHaveBeenCalledWith( + expect.objectContaining({ slimDOMOptions: { script: false } }) + ) + } finally { + script.remove() + } + }) + it('contains and logs recorder-owned callback failures once without swallowing host failures', () => { sessionRecording.onRemoteConfig( makeFlagsResponse({ @@ -4266,7 +4490,54 @@ describe('Lazy SessionRecording', () => { }) describe('URL blocking', () => { + it('does not capture JSON-LD read on the initial blocked URL after navigation', async () => { + const script = document.createElement('script') + script.type = 'application/ld+json' + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Blocked page product', + }) + document.body.appendChild(script) + posthog.config.session_recording.captureJsonLd = true + fakeNavigateTo('https://test.com/blocked') + + try { + sessionRecording.onRemoteConfig( + makeFlagsResponse({ + sessionRecording: { + endpoint: '/s/', + urlBlocklist: [{ matching: 'regex', url: '/blocked' }], + }, + }) + ) + + fakeNavigateTo('https://test.com/allowed') + _emit(createMetaSnapshot()) + await Promise.resolve() + expect(_addCustomEvent).not.toHaveBeenCalledWith( + '$json_ld', + expect.objectContaining({ name: 'Blocked page product' }) + ) + + script.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Allowed page product', + }) + await Promise.resolve() + expect(_addCustomEvent).toHaveBeenCalledWith('$json_ld', { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Allowed page product', + }) + } finally { + script.remove() + } + }) + it('does not flush buffer and includes pause event when hitting blocked URL', async () => { + posthog.config.session_recording.captureJsonLd = true sessionRecording.onRemoteConfig( makeFlagsResponse({ sessionRecording: { @@ -4280,7 +4551,6 @@ describe('Lazy SessionRecording', () => { }, }) ) - // Emit some events before hitting blocked URL _emit(createIncrementalSnapshot({ data: { source: 1 } })) _emit(createIncrementalSnapshot({ data: { source: 2 } })) @@ -4294,6 +4564,20 @@ describe('Lazy SessionRecording', () => { _emit(createIncrementalSnapshot({ data: { source: 3 } })) _emit(createIncrementalSnapshot({ data: { source: 4 } })) + const blockedJsonLd = document.createElement('script') + blockedJsonLd.type = 'application/ld+json' + blockedJsonLd.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Blocked page product', + }) + document.body.appendChild(blockedJsonLd) + await Promise.resolve() + expect(_addCustomEvent).not.toHaveBeenCalledWith( + '$json_ld', + expect.objectContaining({ name: 'Blocked page product' }) + ) + expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data).toEqual([ { data: { @@ -4311,9 +4595,32 @@ describe('Lazy SessionRecording', () => { // Simulate URL change to allowed URL fakeNavigateTo('https://test.com/allowed') + blockedJsonLd.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Product changed before resume', + }) // Verify recording resumes with resume event _emit(createIncrementalSnapshot({ data: { source: 5 } })) + await Promise.resolve() + expect(_addCustomEvent).not.toHaveBeenCalledWith( + '$json_ld', + expect.objectContaining({ name: 'Product changed before resume' }) + ) + + blockedJsonLd.textContent = JSON.stringify({ + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Allowed page product', + }) + await Promise.resolve() + expect(_addCustomEvent).toHaveBeenCalledWith('$json_ld', { + '@context': 'https://schema.org', + '@type': 'Product', + name: 'Allowed page product', + }) + blockedJsonLd.remove() expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data).toStrictEqual([ { diff --git a/packages/browser/src/extensions/replay/external/json-ld.ts b/packages/browser/src/extensions/replay/external/json-ld.ts new file mode 100644 index 0000000000..c9aace61a6 --- /dev/null +++ b/packages/browser/src/extensions/replay/external/json-ld.ts @@ -0,0 +1,432 @@ +import { hasOwnProperty, isArray, isNull, isObject, isUndefined } from '@posthog/core' + +type JsonLdScalar = string | number | boolean | null +type JsonLdPropertyRule = true | readonly string[] +type JsonLdEntityRules = Record +type JsonLdRuleGroup = readonly [readonly string[], JsonLdEntityRules] + +const MAX_JSON_LD_LENGTH = 100_000 +const MAX_JSON_LD_OUTPUT_LENGTH = 20_000 +const SCHEMA_CONTEXT = 'https://schema.org' +const ANY_ENTITY_TYPES: readonly string[] = [] +const TYPE_INDEPENDENT_LEAF_PROPERTIES = + 'actionStatus availability bestRating contentRating encodingFormat eventAttendanceMode eventStatus highPrice inLanguage isAccessibleForFree isFamilyFriendly itemCondition itemListOrder lowPrice maximumAttendeeCapacity nonprofitStatus numberOfItems offerCount position price priceCurrency priceValidUntil publicAccess ratingCount ratingValue reviewCount smokingAllowed worstRating'.split( + ' ' + ) +const ACTION_TYPES = 'Action BorrowAction ReadAction SearchAction SeekToAction SolveMathAction WatchAction'.split(' ') +const ORGANIZATION_TYPES = + 'AutoDealer Bakery BarOrPub CafeOrCoffeeShop CollegeOrUniversity Corporation DaySpa Dentist EducationalOrganization Electrician FoodEstablishment GovernmentOrganization HealthClub Hotel LegalService Library LibrarySystem LocalBusiness Locksmith LodgingBusiness MedicalBusiness NGO OnlineStore Organization PerformingGroup Pharmacy Physician Plumber RealEstateAgent Restaurant School SportsOrganization Store'.split( + ' ' + ) +const PLACE_TYPES = 'Accommodation AdministrativeArea Country Place State'.split(' ') + +const ENTITY_RULES: Record = { + AggregateOffer: { + offers: ['Offer'], + }, + Brand: { + name: true, + }, + BreadcrumbList: { + itemListElement: ['ListItem'], + }, + CreativeWork: { + genre: true, + dateCreated: true, + dateModified: true, + datePublished: true, + expires: true, + learningResourceType: true, + educationalLevel: true, + educationalUse: true, + interactivityType: true, + aggregateRating: ['AggregateRating'], + potentialAction: ACTION_TYPES, + publisher: ORGANIZATION_TYPES, + }, + Event: { + startDate: true, + endDate: true, + previousStartDate: true, + aggregateRating: ['AggregateRating'], + offers: ['AggregateOffer', 'Offer'], + }, + ItemList: { + itemListElement: ['ListItem'], + }, + ListItem: { + item: ANY_ENTITY_TYPES, + }, + Offer: { + seller: ORGANIZATION_TYPES, + }, + Organization: { + name: true, + legalName: true, + foundingDate: true, + dissolutionDate: true, + aggregateRating: ['AggregateRating'], + brand: ['Brand'], + }, + Person: {}, + Place: { + aggregateRating: ['AggregateRating'], + }, + Product: { + name: true, + sku: true, + mpn: true, + gtin: true, + gtin8: true, + gtin12: true, + gtin13: true, + gtin14: true, + productID: true, + productGroupID: true, + asin: true, + model: true, + category: true, + color: true, + material: true, + pattern: true, + size: true, + productionDate: true, + releaseDate: true, + brand: ['Brand', 'Organization'], + manufacturer: ORGANIZATION_TYPES, + offers: ['Offer', 'AggregateOffer'], + aggregateRating: ['AggregateRating'], + }, + Service: { + name: true, + serviceType: true, + category: true, + provider: ORGANIZATION_TYPES, + areaServed: PLACE_TYPES, + offers: ['AggregateOffer', 'Offer'], + aggregateRating: ['AggregateRating'], + }, + OfferCatalog: { + name: true, + itemListElement: ANY_ENTITY_TYPES, + }, +} + +const EMPTY_ENTITY_RULES: JsonLdEntityRules = {} +const INHERITED_RULE_GROUPS: readonly JsonLdRuleGroup[] = [ + [ACTION_TYPES, EMPTY_ENTITY_RULES], + [ + '3DModel AboutPage Answer Article AudioObject Blog BlogPosting Book Clip CollectionPage Comment ContactPage Course CreativeWorkSeason CreativeWorkSeries DataCatalog DataDownload DataFeed Dataset DiscussionForumPosting Episode FAQPage Game HowTo HowToDirection HowToSection HowToStep HowToTip ImageObject LearningResource MediaObject Message MobileApplication Movie MusicPlaylist MusicRecording NewsArticle Photograph PodcastEpisode PodcastSeries ProfilePage QAPage Question Quiz Recipe Review ScholarlyArticle SearchResultsPage SiteNavigationElement SocialMediaPosting SoftwareApplication TVEpisode TVSeries TechArticle VacationRental VideoGame VideoObject WebApplication WebPage WebPageElement WebSite'.split( + ' ' + ), + ENTITY_RULES.CreativeWork, + ], + [ + 'BroadcastEvent BusinessEvent EducationEvent Festival MusicEvent SportsEvent TheaterEvent'.split(' '), + ENTITY_RULES.Event, + ], + [ORGANIZATION_TYPES, ENTITY_RULES.Organization], + [PLACE_TYPES, ENTITY_RULES.Place], + ['Car IndividualProduct ProductGroup ProductModel'.split(' '), ENTITY_RULES.Product], + ['AggregateRating EmployerAggregateRating Rating'.split(' '), EMPTY_ENTITY_RULES], +] +const TYPES_WITHOUT_PROPERTIES = + 'AlignmentObject BedDetails Certification ContactPoint CreditCard DefinedRegion EducationalOccupationalCredential EntryPoint GeoCoordinates GeoShape InteractionCounter JobPosting LocationFeatureSpecification MathSolver MemberProgram MemberProgramTier MerchantReturnPolicy MerchantReturnPolicySeasonalOverride MonetaryAmount NutritionInformation OccupationalExperienceRequirements OfferShippingDetails OpeningHoursSpecification PeopleAudience PostalAddress PriceSpecification PropertyValue QuantitativeValue ServicePeriod ShippingConditions ShippingDeliveryTime ShippingRateSettings ShippingService SpeakableSpecification Thing UnitPriceSpecification'.split( + ' ' + ) + +export const JSON_LD_EVENT_TAG = '$json_ld' + +function getOwnProperty(value: Record, property: string): unknown { + return hasOwnProperty.call(value, property) ? value[property] : undefined +} + +function isScalar(value: unknown): value is JsonLdScalar { + const type = typeof value + return isNull(value) || type === 'string' || type === 'number' || type === 'boolean' +} + +function sanitizeScalar(value: unknown): JsonLdScalar | JsonLdScalar[] | undefined { + return isScalar(value) || (isArray(value) && value.every(isScalar)) ? value : undefined +} + +function getEntityRules(type: string): JsonLdEntityRules | undefined { + if (hasOwnProperty.call(ENTITY_RULES, type)) { + return ENTITY_RULES[type] + } + for (const [types, rules] of INHERITED_RULE_GROUPS) { + if (types.includes(type)) { + return rules + } + } + return TYPES_WITHOUT_PROPERTIES.includes(type) ? EMPTY_ENTITY_RULES : undefined +} + +function getEntityTypes(value: unknown): string[] { + const values = typeof value === 'string' ? [value] : isArray(value) ? value : [] + return values + .filter((type): type is string => typeof type === 'string') + .map((type) => type.replace(/^https?:\/\/schema\.org\//, '')) + .filter((type) => !!getEntityRules(type)) +} + +function sanitizeEntity(value: unknown, allowedTypes?: readonly string[]): Record | null { + if (!isObject(value)) { + return null + } + const typeValue = getOwnProperty(value, '@type') + const types = getEntityTypes(typeValue).filter( + (type) => !allowedTypes || !allowedTypes.length || allowedTypes.includes(type) + ) + if (!types.length) { + return null + } + + const result: Record = { '@type': typeof typeValue === 'string' ? types[0] : types } + const id = sanitizeScalar(getOwnProperty(value, '@id')) + if (!isUndefined(id)) { + result['@id'] = id + } + + for (const property of TYPE_INDEPENDENT_LEAF_PROPERTIES) { + const scalar = sanitizeScalar(getOwnProperty(value, property)) + if (!isUndefined(scalar)) { + result[property] = scalar + } + } + + for (const type of types) { + const rules = getEntityRules(type)! + for (const property of Object.keys(rules)) { + const propertyValue = getOwnProperty(value, property) + const rule = rules[property] + if (rule === true) { + const scalar = sanitizeScalar(propertyValue) + if (!isUndefined(scalar)) { + result[property] = scalar + } + } else if (isArray(propertyValue)) { + const items = propertyValue.map((item) => sanitizeEntity(item, rule)).filter(isObject) + if (items.length) { + result[property] = items + } + } else { + const nestedEntity = sanitizeEntity(propertyValue, rule) + if (nestedEntity) { + result[property] = nestedEntity + } + } + } + } + + return result +} + +function sanitizeRoot(value: unknown): Record | null { + if (!isObject(value)) { + return null + } + const context = getOwnProperty(value, '@context') + if (typeof context !== 'string' || !/^https?:\/\/schema\.org\/?$/.test(context)) { + return null + } + + const entity = sanitizeEntity(value) + if (entity) { + return { '@context': SCHEMA_CONTEXT, ...entity } + } + + const graph = getOwnProperty(value, '@graph') + if (!isArray(graph)) { + return null + } + const entities = graph.map((entity) => sanitizeEntity(entity)).filter(isObject) + return entities.length ? { '@context': SCHEMA_CONTEXT, '@graph': entities } : null +} + +export function sanitizeJsonLd(text: string): [unknown, string] | null { + if (!text || text.length > MAX_JSON_LD_LENGTH) { + return null + } + + try { + const value: unknown = JSON.parse(text) + const sanitized = isArray(value) ? value.map(sanitizeRoot) : sanitizeRoot(value) + if (isNull(sanitized) || (isArray(sanitized) && (!sanitized.length || sanitized.some(isNull)))) { + return null + } + + const output = JSON.stringify(sanitized) + return output.length <= MAX_JSON_LD_OUTPUT_LENGTH ? [sanitized, output] : null + } catch { + return null + } +} + +function isJsonLdScript(node: Node): node is HTMLScriptElement { + return ( + node.nodeName === 'SCRIPT' && + (node as Element).getAttribute('type')?.trim().toLowerCase() === 'application/ld+json' + ) +} + +type JsonLdPrivacyOptions = { + blockClass?: string | RegExp + blockSelector?: string | null + maskTextClass?: string | RegExp + maskTextSelector?: string | null +} + +function matchesPrivacyRule(element: Element, classRule?: string | RegExp, selector?: string | null): boolean { + if ( + typeof classRule === 'string' + ? element.classList.contains(classRule) + : classRule && + Array.from(element.classList).some((className) => { + classRule.lastIndex = 0 + return classRule.test(className) + }) + ) { + return true + } + try { + return !!selector && element.matches(selector) + } catch { + return false + } +} + +function isWithinPrivacyBoundary(element: Element, options: JsonLdPrivacyOptions): boolean { + for (let current: Element | null = element; current; ) { + if ( + matchesPrivacyRule(current, options.blockClass, options.blockSelector) || + matchesPrivacyRule(current, options.maskTextClass, options.maskTextSelector) + ) { + return true + } + const parentNode: Node | null = current.parentNode + current = + current.parentElement || + (parentNode && parentNode.nodeType === parentNode.DOCUMENT_FRAGMENT_NODE + ? (parentNode as ShadowRoot).host + : null) + } + return false +} + +function getJsonLdScripts(node: Node): HTMLScriptElement[] { + if (isJsonLdScript(node)) { + return [node] + } + if (node.nodeType !== node.ELEMENT_NODE) { + return [] + } + return Array.from((node as Element).querySelectorAll('script')).filter(isJsonLdScript) +} + +export function startJsonLdCapture( + doc: Document, + MutationObserverClass: typeof MutationObserver, + options: JsonLdPrivacyOptions & { + emit: (jsonLd: unknown) => boolean + // Null updates the deduplication baseline without an event. + getCaptureState?: () => boolean | null + } +): { scan: (force?: boolean) => void; stop: () => void } { + const lastJsonByScript = new WeakMap() + const getCaptureState = options.getCaptureState || (() => true) + let remainingLength = MAX_JSON_LD_LENGTH + + const captureScript = (script: HTMLScriptElement): void => { + try { + const captureState = getCaptureState() + if ( + !remainingLength || + captureState === false || + !script.isConnected || + script.ownerDocument !== doc || + isWithinPrivacyBoundary(script, options) + ) { + return + } + const sanitized = sanitizeJsonLd(script.text) + if (!sanitized) { + lastJsonByScript.delete(script) + return + } + const [jsonLd, json] = sanitized + if (isNull(captureState)) { + lastJsonByScript.set(script, json) + return + } + if (lastJsonByScript.get(script) !== json) { + if (json.length > remainingLength) { + remainingLength = 0 + return + } + if (options.emit(jsonLd)) { + lastJsonByScript.set(script, json) + remainingLength -= json.length + } + } + } catch { + return + } + } + + try { + const observer = new MutationObserverClass((mutations) => { + try { + if (!remainingLength || getCaptureState() === false) { + return + } + const captureScripts = (node: Node): void => { + for (const script of getJsonLdScripts(node)) { + captureScript(script) + } + } + + for (const mutation of mutations) { + if (mutation.type === 'childList') { + if (isJsonLdScript(mutation.target)) { + captureScript(mutation.target) + } + mutation.addedNodes.forEach(captureScripts) + } else if (mutation.type === 'characterData') { + const parent = mutation.target.parentNode + if (parent && isJsonLdScript(parent)) { + captureScript(parent) + } + } else if (mutation.type === 'attributes' && isJsonLdScript(mutation.target)) { + captureScript(mutation.target) + } + } + } catch { + return + } + }) + + observer.observe(doc, { + attributes: true, + attributeFilter: ['type'], + characterData: true, + childList: true, + subtree: true, + }) + const scan = (force = false): void => { + if (!remainingLength || getCaptureState() === false) { + return + } + getJsonLdScripts(doc.documentElement).forEach((script) => { + if (force) { + lastJsonByScript.delete(script) + } + captureScript(script) + }) + } + + return { scan, stop: () => observer.disconnect() } + } catch { + return { scan: () => {}, stop: () => {} } + } +} diff --git a/packages/browser/src/extensions/replay/external/lazy-loaded-session-recorder.ts b/packages/browser/src/extensions/replay/external/lazy-loaded-session-recorder.ts index 09b8e34919..6344982799 100644 --- a/packages/browser/src/extensions/replay/external/lazy-loaded-session-recorder.ts +++ b/packages/browser/src/extensions/replay/external/lazy-loaded-session-recorder.ts @@ -90,6 +90,7 @@ import { decodeSamplingDecision, } from './recording-strategies' import { MASKED, PERSONAL_DATA_CAMPAIGN_PARAMS } from '@posthog/browser-common/utils/event-utils' +import { JSON_LD_EVENT_TAG, startJsonLdCapture } from './json-ld' const BASE_ENDPOINT = '/s/' const DEFAULT_CANVAS_QUALITY = 0.4 @@ -492,6 +493,8 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt */ private _forceAllowLocalhostNetworkCapture = false private _stopRrweb: listenerHandler | undefined = undefined + private _jsonLdCapture: ReturnType | undefined + private _jsonLdCaptureReady = false private _lastActivityTimestamp: number = Date.now() private _isActivatingTrigger: boolean = false /** @@ -895,6 +898,37 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt return this._tryRRWebMethod(newQueuedEvent(() => getRRWebRecord()!.addCustomEvent(tag, payload))) } + private _canCaptureJsonLd(): boolean { + return ( + this._jsonLdCaptureReady && + this._instance.config.session_recording?.captureJsonLd === true && + !this._urlTriggerMatching.urlBlocked && + this._isIdle !== true + ) + } + + private _tryAddJsonLdEvent(jsonLd: unknown): boolean { + if (!this._canCaptureJsonLd()) { + return false + } + try { + const rrwebRecord = getRRWebRecord() + if (!rrwebRecord) { + return false + } + rrwebRecord.addCustomEvent(JSON_LD_EVENT_TAG, jsonLd) + return this._canCaptureJsonLd() + } catch { + return false + } + } + + private _scheduleJsonLdScan(force = false): void { + // Run the scan after the current rrweb event updates the JSON-LD capture state. + // eslint-disable-next-line compat/compat + Promise.resolve().then(() => this._jsonLdCapture?.scan(force)) + } + private _pageViewFallBack() { try { if (this._instance.config.capture_pageview || !window) { @@ -997,6 +1031,7 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt // so we might not get the below custom event, but events will report the paused status. // which will allow debugging of sessions that start on blocked pages this._urlTriggerMatching.urlBlocked = true + this._jsonLdCapture?.scan() // Clear the snapshot timer since we don't want new snapshots while paused clearInterval(this._fullSnapshotTimer) @@ -1011,9 +1046,12 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt return } + // Baseline JSON-LD while capture is suppressed so blocked-page mutations cannot emit after resume. + this._jsonLdCapture?.scan() this._urlTriggerMatching.urlBlocked = false this._tryTakeFullSnapshot() + this._scheduleJsonLdScan() this._scheduleFullSnapshot() this._tryAddCustomEvent('recording resumed', { reason: 'left blocked url' }) @@ -1427,6 +1465,9 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt // Clear any queued rrweb events to prevent memory leaks from closures this._queuedRRWebEvents = [] + this._jsonLdCapture?.stop() + this._jsonLdCapture = undefined + this._jsonLdCaptureReady = false this._stopRrweb?.() this._stopRrweb = undefined } @@ -1781,7 +1822,9 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt } // Clear the buffer if waiting for a trigger and only keep data from after the current full snapshot - if (rawEvent.type === EventType.FullSnapshot && this._strategy?.hasPendingTriggers(this.sessionId)) { + const jsonLdRemovedFromPendingBuffer = + rawEvent.type === EventType.FullSnapshot && this._strategy?.hasPendingTriggers(this.sessionId) + if (jsonLdRemovedFromPendingBuffer) { this._clearBufferBeforeMostRecentMeta() } @@ -1869,6 +1912,14 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt event.timestamp = sessionIdlePayload.lastActivityTimestamp + sessionIdlePayload.threshold } + const jsonLdCaptureWasReady = this._jsonLdCaptureReady + this._jsonLdCaptureReady = true + if (jsonLdRemovedFromPendingBuffer) { + this._scheduleJsonLdScan(true) + } else if (!jsonLdCaptureWasReady) { + this._scheduleJsonLdScan() + } + const compressionEnabled = this._instance.config.session_recording.compress_events ?? true if ( @@ -2450,6 +2501,7 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt if (this._eventsDroppedWhileIdle > 0 && bufferCanShip) { this._tryTakeFullSnapshot() } + this._scheduleJsonLdScan() this._scheduleFullSnapshot() } } @@ -2573,6 +2625,17 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt } } + if ( + userSessionRecordingOptions?.captureJsonLd === true && + sessionRecordingOptions.slimDOMOptions !== true && + sessionRecordingOptions.slimDOMOptions !== 'all' + ) { + sessionRecordingOptions.slimDOMOptions = { + ...sessionRecordingOptions.slimDOMOptions, + script: true, + } + } + if (this._canvasRecording && this._canvasRecording.enabled) { sessionRecordingOptions.recordCanvas = true // canvas fps is owned by the canvas recording config; merge so that @@ -2689,6 +2752,30 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt this._rrwebError = false + if ( + userSessionRecordingOptions?.captureJsonLd === true && + document && + window?.MutationObserver && + !this._jsonLdCapture + ) { + this._jsonLdCapture = startJsonLdCapture(document, window.MutationObserver, { + blockClass: sessionRecordingOptions.blockClass, + blockSelector: sessionRecordingOptions.blockSelector, + maskTextClass: sessionRecordingOptions.maskTextClass, + maskTextSelector: sessionRecordingOptions.maskTextSelector, + getCaptureState: () => + this._instance.config.session_recording?.captureJsonLd !== true || + this._urlTriggerMatching.urlBlocked || + this._urlTriggerMatching.isCurrentUrlBlocked() + ? null + : this._canCaptureJsonLd(), + emit: (jsonLd) => this._tryAddJsonLdEvent(jsonLd), + }) + if (this._urlTriggerMatching.isCurrentUrlBlocked()) { + this._jsonLdCapture.scan() + } + } + // We reset the last activity timestamp, resetting the idle timer this._lastActivityTimestamp = Date.now() // stay unknown if we're not sure if we're idle or not diff --git a/packages/browser/src/extensions/replay/external/triggerMatching.ts b/packages/browser/src/extensions/replay/external/triggerMatching.ts index 4bb3301619..6904758ae9 100644 --- a/packages/browser/src/extensions/replay/external/triggerMatching.ts +++ b/packages/browser/src/extensions/replay/external/triggerMatching.ts @@ -317,6 +317,18 @@ export class URLTriggerMatching implements TriggerStatusMatching { return this._urlTriggerStatus(sessionId) } + isCurrentUrlBlocked(): boolean { + const url = getTargetingUrl(this._instance) + if (!url) { + return false + } + try { + return sessionRecordingUrlTriggerMatches(url, this._urlBlocklist, this._compiledBlocklistRegexes) + } catch { + return true + } + } + /** * Check URL blocklist and pause/resume recording accordingly * This is separate from trigger checking and is used by both V1 and V2 diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index 6c38bdeaf3..7a3963a1b9 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -50,6 +50,7 @@ "_callLoadToolbar", "_callbackFlavor", "_campaign_params_url", + "_canCaptureJsonLd", "_canUseSessionStorage", "_cancelAllPendingTours", "_cancelEventToItems", @@ -405,6 +406,8 @@ "_isWidgetSurveyOpen", "_is_persistence_disabled", "_is_supported", + "_jsonLdCapture", + "_jsonLdCaptureReady", "_keepaliveDisabled", "_lastActivityTimestamp", "_lastCheckedUrl", @@ -675,6 +678,7 @@ "_scheduleCheck", "_scheduleFlushBuffer", "_scheduleFullSnapshot", + "_scheduleJsonLdScan", "_scheduleNextRefresh", "_scopeName", "_scriptName", @@ -801,6 +805,7 @@ "_triggeringPage", "_trimToMaxBytes", "_tryAddCustomEvent", + "_tryAddJsonLdEvent", "_tryRRWebMethod", "_tryTakeFullSnapshot", "_typeByName", diff --git a/packages/types/src/__tests__/__snapshots__/config-snapshot.spec.ts.snap b/packages/types/src/__tests__/__snapshots__/config-snapshot.spec.ts.snap index 85cb70afb0..c7923fe7a8 100644 --- a/packages/types/src/__tests__/__snapshots__/config-snapshot.spec.ts.snap +++ b/packages/types/src/__tests__/__snapshots__/config-snapshot.spec.ts.snap @@ -500,6 +500,11 @@ exports[`config snapshot for PostHogConfig 1`] = ` "Partial", "\\"all\\"" ], + "captureJsonLd": [ + "undefined", + "false", + "true" + ], "collectFonts": [ "undefined", "false", diff --git a/packages/types/src/posthog-config.ts b/packages/types/src/posthog-config.ts index d3bf74aab7..03179ec6dc 100644 --- a/packages/types/src/posthog-config.ts +++ b/packages/types/src/posthog-config.ts @@ -686,6 +686,18 @@ export interface SessionRecordingOptions { */ slimDOMOptions?: true | Partial | 'all' + /** + * Captures sanitized Schema.org JSON-LD as session replay custom events. + * JSON-LD inside a text mask or blocked element is never captured. + * The recorder keeps `@id` values without changes. + * The event tag is `$json_ld`. The payload is a JSON-LD object or array. + * The recorder removes all script nodes from snapshots when this option is enabled. + * The JSON-LD observer starts only when this option is true at recording start. + * @see https://github.com/PostHog/posthog-js/blob/main/packages/browser/src/extensions/replay/external/json-ld.ts + * @default false + */ + captureJsonLd?: boolean + /** * Derived from `rrweb.record` options * @see https://github.com/rrweb-io/rrweb/blob/master/guide.md