Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/node-before-span-send.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'posthog-node': minor
'@posthog/core': minor
'@posthog/types': minor
---

Add a `traces.beforeSpanSend` hook that runs on every finished span before it is queued, so you can scrub sensitive attributes or drop spans entirely — return `null` to drop one, or pass an array of hooks to run left to right. The hook sees plain values rather than the OTLP wire encoding, span identity fields are read-only so edits cannot orphan child spans, and a hook that throws drops the span rather than exporting an unscrubbed one.
3 changes: 3 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ export type {
TraceSdkContext,
TracesHost,
} from './traces/types'
// The `beforeSpanSend` shapes come straight from @posthog/types: hooks see the
// public record, not core's internal one, which also carries `traceState`.
export type { SpanRecord, BeforeSpanSendFn } from '@posthog/types'
// Same barrel convention as logs and metrics for the user-facing tracing types.
export type {
Span,
Expand Down
128 changes: 128 additions & 0 deletions packages/core/src/traces/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type {
OtlpTracesPayload,
ResolvedTracesConfig,
SendTracesBatchOutcome,
SpanRecord,
TraceSdkContext,
} from './types'
import type { Logger } from '../types'
Expand All @@ -18,6 +19,7 @@ const resolveForTest = (partial?: Partial<ResolvedTracesConfig>): ResolvedTraces
flushIntervalMs: 5000,
maxExportBatchSize: 512,
maxQueueSize: 2048,
beforeSpanSend: [],
...partial,
})

Expand Down Expand Up @@ -469,6 +471,132 @@ describe('PostHogTraces', () => {
})
})

describe('beforeSpanSend', () => {
const endOneSpan = (beforeSpanSend: any): PostHogTraces => {
const traces = createTraces({ beforeSpanSend: [beforeSpanSend].flat() })
traces.startSpan('checkout', { attributes: { userId: 42 } }).end()
return traces
}

it('drops a span when the hook returns null', async () => {
await endOneSpan(() => null).flush()
expect(sentSpans()).toHaveLength(0)
})

it('drops the span when the hook throws', async () => {
await endOneSpan(() => {
throw new Error('scrubber broke')
}).flush()

expect(sentSpans()).toHaveLength(0)
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('beforeSpanSend failed'), expect.anything())
})

it('hands the hook plain values, not the OTLP encoding', () => {
const seen: unknown[] = []
endOneSpan((span: SpanRecord) => {
seen.push(span.attributes.userId)
return span
})

expect(seen).toEqual([42])
})

it('keeps the original ids when a hook rewrites them', async () => {
const traces = createTraces({
beforeSpanSend: [
(span: any) => {
span.traceId = '0'.repeat(32)
span.spanId = '1'.repeat(16)
return span
},
],
})
const started = traces.startSpan('checkout')
const originalTraceId = started.traceparent()!.split('-')[1]
started.end()
await traces.flush()

const [span] = sentSpans()
expect(span.traceId).toBe(originalTraceId)
expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('identity field'))
})

it('survives a hook that returns a frozen record', async () => {
const traces = createTraces({
beforeSpanSend: [(span: SpanRecord) => Object.freeze({ ...span, attributes: {} })],
})
const span = traces.startSpan('checkout')

expect(() => span.end()).not.toThrow()
await traces.flush()
expect(sentSpans()).toHaveLength(0)
})

it('rejects a timestamp the server could not decode', async () => {
const instance = createMockInstance()
const traces = createTraces(
{ beforeSpanSend: [(span: SpanRecord) => ({ ...span, startTime: span.startTime * 1e6 })] },
instance
)
traces.startSpan('poison').end()
await traces.flush()

const [span] = sentSpans(instance)
expect(span.startTimeUnixNano.length).toBeLessThanOrEqual(19)
})

it('logs when a hook drops a span', async () => {
await endOneSpan(() => null).flush()
expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('dropped a span'))
})

it('keeps tracestate a rebuilding hook would have dropped', async () => {
const instance = createMockInstance()
const traces = createTraces(
{ beforeSpanSend: [(span: SpanRecord) => ({ ...span, traceState: undefined }) as SpanRecord] },
instance
)
traces.startSpan('child', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`, tracestate: 'vendor=abc' }).end()
await traces.flush()

expect(sentSpans(instance)[0].traceState).toBe('vendor=abc')
})

it('runs hooks left to right and stops at the first null', async () => {
const order: string[] = []
await endOneSpan([
(span: SpanRecord) => {
order.push('first')
return span
},
() => {
order.push('second')
return null
},
(span: SpanRecord) => {
order.push('third')
return span
},
]).flush()

expect(order).toEqual(['first', 'second'])
expect(sentSpans()).toHaveLength(0)
})

it('exports the edits a hook made', async () => {
await endOneSpan((span: SpanRecord) => {
delete span.attributes.userId
span.name = 'redacted'
return span
}).flush()

const [span] = sentSpans()
expect(span.name).toBe('redacted')
expect(span.attributes?.find((attribute) => attribute.key === 'userId')).toBeUndefined()
})
})

describe('export', () => {
it('flushes when the queue reaches the batch size', async () => {
const traces = createTraces({ maxExportBatchSize: 2 })
Expand Down
87 changes: 85 additions & 2 deletions packages/core/src/traces/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,19 @@ import type {
import { NOOP_SPAN, PostHogSpan, describeError } from './span'
import { newSpanId, newTraceId } from './ids'
import { parseTraceparent, sanitizeTracestate } from './traceparent'
import { resolveStartTime, sanitizeName } from './sanitize'
import { clampEndTime, resolveStartTime, sanitizeName, toEpochMs } from './sanitize'
import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp'
import { isPromise, safeSetTimeout } from '../utils'

type SpanCallback<T> = (span: Span) => T

interface SpanIdentity {
traceId: string
spanId: string
parentSpanId?: string
traceState?: string
}

interface ParentContext {
traceId: string
parentSpanId?: string
Expand Down Expand Up @@ -296,13 +303,18 @@ export class PostHogTraces {
// Queue and export
// ==========================================================================

private _onSpanEnd(record: SpanRecord): void {
private _onSpanEnd(incoming: SpanRecord): void {
// Re-checked at end, not just at start: opting out mid-trace must stop the
// span exporting, without throwing into code holding a live handle.
if (this._instance.isDisabled || this._instance.optedOut) {
return
}

const record = this._runBeforeSpanSend(incoming)
if (!record) {
return
}

if (this._queue.length >= this._config.maxQueueSize) {
// Drop the incoming span rather than evicting queued ones: the oldest
// queued spans are completed parents whose children may already have been
Expand All @@ -323,6 +335,77 @@ export class PostHogTraces {
}
}

/**
* Runs the `beforeSpanSend` chain, returning the span to enqueue or `null` to
* drop it.
*
* A throwing hook drops the span. The hook is the documented scrubbing point,
* so a scrubber that breaks must not let the unscrubbed record through.
*
* Identity fields are restored afterwards: rewriting them would orphan
* children that already shipped with the original parent id.
*/
private _runBeforeSpanSend(record: SpanRecord): SpanRecord | null {
if (!this._config.beforeSpanSend.length) {
return record
}

// Snapshotted before any hook runs: a hook that mutates in place would
// otherwise leave nothing to restore from.
const identity = {
traceId: record.traceId,
spanId: record.spanId,
parentSpanId: record.parentSpanId,
traceState: record.traceState,
}
const originalTimes = { startTime: record.startTime, endTime: record.endTime }
let current = record
try {
for (const hook of this._config.beforeSpanSend) {
const result = hook(current)
if (!result) {
this._logger.debug('beforeSpanSend dropped a span')
return null
}
current = this._keepSpanIdentity(result, identity)
}

// Re-applied to whatever the hook returned: a hook can write a timestamp
// the server cannot decode, and one such span 400s the whole request it
// travels in, taking unrelated spans with it.
current.name = sanitizeName(current.name, 'Span name', this._logger)
current.startTime = toEpochMs(current.startTime) ?? originalTimes.startTime
current.endTime = clampEndTime(toEpochMs(current.endTime) ?? originalTimes.endTime, current.startTime)
return current
} catch (error) {
// Covers the hook and everything done to its return value: a frozen or
// hostile record must not throw out of `end()` into application code.
this._logger.warn('beforeSpanSend failed; dropping the span rather than exporting it unscrubbed', error)
return null
}
}

/**
* Restores the fields a hook must not change. Runs per hook so a later hook in
* the chain cannot sample on an id an earlier one forged.
*/
private _keepSpanIdentity(hooked: SpanRecord, original: SpanIdentity): SpanRecord {
if (
hooked.traceId !== original.traceId ||
hooked.spanId !== original.spanId ||
hooked.parentSpanId !== original.parentSpanId
) {
this._logger.debug('beforeSpanSend changed a span identity field; keeping the original ids')
}
hooked.traceId = original.traceId
hooked.spanId = original.spanId
hooked.parentSpanId = original.parentSpanId
// A hook that rebuilds the record instead of spreading it would otherwise
// drop tracestate, which is not part of the record the hook is handed.
hooked.traceState = original.traceState
return hooked
}

private _recordDrop(count: number, reason: string): void {
if (!this._droppedWarned) {
this._droppedWarned = true
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/traces/otlp.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ describe('OTLP span encoding', () => {
flushIntervalMs: 5000,
maxExportBatchSize: 512,
maxQueueSize: 2048,
beforeSpanSend: [],
...partial,
})

Expand Down
12 changes: 11 additions & 1 deletion packages/core/src/traces/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export type {
SpanTimeInput,
StartSpanOptions,
TracesConfig,
BeforeSpanSendFn,
OtlpSpan,
OtlpSpanAnyValue,
OtlpSpanEvent,
Expand All @@ -17,7 +18,15 @@ export type {
OtlpTracesPayload,
} from '@posthog/types'

import type { OtlpTracesPayload, Span, SpanAttributes, SpanKind, SpanStatusCode, TracesConfig } from '@posthog/types'
import type {
BeforeSpanSendFn,
OtlpTracesPayload,
Span,
SpanAttributes,
SpanKind,
SpanStatusCode,
TracesConfig,
} from '@posthog/types'

/** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for all three signals. */
export type SendTracesBatchOutcome =
Expand Down Expand Up @@ -112,4 +121,5 @@ export interface ResolvedTracesConfig extends TracesConfig {
* may already have been exported.
*/
maxQueueSize: number
beforeSpanSend: BeforeSpanSendFn[]
}
57 changes: 57 additions & 0 deletions packages/node/src/__tests__/traces.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,63 @@ describe('PostHog traces', () => {
})
})

describe('beforeSpanSend', () => {
it('scrubs attributes before they leave the process', async () => {
const client = createClient({
traces: {
serviceName: 'svc',
beforeSpanSend: (span: any) => {
delete span.attributes.password
return span
},
},
})
client.startSpan('login', { attributes: { password: 'hunter2', ok: true } }).end()
await client.shutdown()

const [span] = sentSpans()
expect(span.attributes?.find((a) => a.key === 'password')).toBeUndefined()
expect(span.attributes?.find((a) => a.key === 'ok')).toBeDefined()
})

it('runs an array of hooks through the client option', async () => {
const client = createClient({
traces: {
serviceName: 'svc',
beforeSpanSend: [
(span: any) => {
span.attributes.first = true
return span
},
(span: any) => {
span.attributes.second = true
return span
},
],
},
})
client.startSpan('checkout').end()
await client.shutdown()

const keys = sentSpans()[0].attributes?.map((a) => a.key)
expect(keys).toEqual(expect.arrayContaining(['first', 'second']))
})

it('drops a span the hook rejects', async () => {
const client = createClient({
traces: {
serviceName: 'svc',
beforeSpanSend: (span: any) => (span.attributes['http.route'] === '/health' ? null : span),
},
})
client.startSpan('GET /health', { attributes: { 'http.route': '/health' } }).end()
client.startSpan('GET /orders', { attributes: { 'http.route': '/orders' } }).end()
await client.shutdown()

expect(sentSpans().map((s) => s.name)).toEqual(['GET /orders'])
})
})

describe('shutdown', () => {
it('drains queued spans', async () => {
posthog.startSpan('a').end()
Expand Down
2 changes: 2 additions & 0 deletions packages/node/src/exports.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ export type {
SpanStatusCode,
SpanTimeInput,
StartSpanOptions,
SpanRecord,
BeforeSpanSendFn,
TracesConfig,
} from '@posthog/core'

Expand Down
Loading