feat(node): POC runtime metrics autocapture behind a feature flag

.changeset/wild-otters-collect-metrics.md

@@ -0,0 +1,5 @@
+---
+'posthog-node': minor
+---
+
+Proof of concept: autocapture low-level Node runtime metrics (CPU, memory, event loop delay and utilization, GC pauses, uptime, active handles) through `posthog.metrics`, with no instrumentation. Off unless the `metrics-sdk-autocapture` feature flag opens the gate (evaluated locally only, so it costs no request and no event) or `enableMetricsAutocapture: true` is set.

packages/node/references/posthog-node-references-latest.json

@@ -3456,6 +3456,16 @@
           "type": "MetricsConfig",
           "name": "metrics"
         },
+        {
+          "description": "PROOF OF CONCEPT - MAY CHANGE WITHOUT WARNING\n\nAutocapture low-level Node runtime metrics (CPU time and utilization, memory\nand heap limit, event loop delay and utilization, GC pauses, uptime, active\nhandles) through `posthog.metrics`, with no instrumentation of your own.\n\nLeave this unset to let the `metrics-sdk-autocapture` feature flag decide,\nre-evaluated every 30 seconds so it works as a remote kill switch. The flag\nis only ever evaluated locally, against cached flag definitions — so it\nrequires `secretKey`, must itself be locally evaluable, and never costs a\nrequest or a `$feature_flag_called` event. Set this option explicitly to opt\nin or out and skip flag evaluation entirely.",
+          "type": "boolean",
+          "name": "enableMetricsAutocapture"
+        },
+        {
+          "description": "PROOF OF CONCEPT - MAY CHANGE WITHOUT WARNING\n\nHow often runtime metrics are sampled when autocapture is on. Values below\n1000ms are clamped, and the default matches the metrics flush interval so\neach flush window carries roughly one sample per series.",
+          "type": "number",
+          "name": "metricsAutocaptureIntervalMs"
+        },
         {
           "description": "Credential that enables local feature flag evaluation and remote config.\n\nAccepts either a Personal API Key (`phx_...`) or a Project Secret API Key (`phs_...`).\nWhen provided, the client can evaluate feature flags locally and decrypt remote\nconfig payloads via `getRemoteConfigPayload`. Prefer this over the deprecated\n`personalApiKey` option; when both are set, `secretKey` takes precedence.",
           "type": "string",

packages/node/src/__tests__/metrics-autocapture.spec.ts

@@ -0,0 +1,290 @@
+import { PostHog } from '@/entrypoints/index.node'
+import type { PostHogOptions } from '@/types'
+import {
+  DEFAULT_SAMPLE_INTERVAL_MS,
+  GATE_POLL_INTERVAL_MS,
+  METRICS_AUTOCAPTURE_FLAG,
+} from '@/extensions/metrics-autocapture'
+import { parseCgroupCpuQuota } from '@/extensions/metrics-autocapture/runtime.node'
+import { waitForPromises } from './utils'
+
+jest.mock('../version', () => ({ version: '1.2.3' }))
+jest.spyOn(console, 'debug').mockImplementation()
+
+const mockedFetch = jest.spyOn(globalThis, 'fetch').mockImplementation()
+
+const options: PostHogOptions = {
+  host: 'http://example.com',
+  disableCompression: true,
+  fetchRetryCount: 0,
+  featureFlagsRequestMaxRetries: 0,
+  metrics: { serviceName: 'test-service' },
+}
+
+/**
+ * Serves flag definitions for local evaluation with the gate flag rolled out to
+ * either everyone or nobody, plus 200s for every write endpoint.
+ */
+const mockApi = (gateEnabled: boolean): void => {
+  mockedFetch.mockImplementation((url: any): Promise<any> => {
+    if (String(url).includes('flags/definitions')) {
+      return Promise.resolve({
+        status: 200,
+        text: () => Promise.resolve('ok'),
+        headers: { get: () => null },
+        json: () =>
+          Promise.resolve({
+            flags: [
+              {
+                id: 1,
+                name: 'Metrics SDK autocapture',
+                key: METRICS_AUTOCAPTURE_FLAG,
+                active: true,
+                filters: { groups: [{ rollout_percentage: gateEnabled ? 100 : 0 }] },
+              },
+            ],
+            group_type_mapping: {},
+            cohorts: {},
+          }),
+      })
+    }
+    return Promise.resolve({ status: 200, text: () => Promise.resolve('ok') })
+  })
+}
+
+const localEvaluationOptions: PostHogOptions = { ...options, secretKey: 'phx_test' }
+
+const flagsCalls = (): any[] => mockedFetch.mock.calls.filter((call) => String(call[0]).includes('/flags/?'))
+
+const metricNames = (): string[] => {
+  const names = new Set<string>()
+  for (const call of mockedFetch.mock.calls) {
+    if (!String(call[0]).includes('/i/v1/metrics')) {
+      continue
+    }
+    const body = JSON.parse((call[1] as any).body)
+    for (const metric of body.resourceMetrics[0].scopeMetrics[0].metrics) {
+      names.add(metric.name)
+    }
+  }
+  return [...names]
+}
+
+describe('PostHog Node.js metrics autocapture', () => {
+  let posthog: PostHog
+
+  jest.useFakeTimers()
+
+  beforeEach(() => {
+    mockedFetch.mockReset()
+    mockApi(true)
+  })
+
+  afterEach(async () => {
+    await posthog?.shutdown()
+  })
+
+  describe('when explicitly enabled', () => {
+    beforeEach(() => {
+      posthog = new PostHog('TEST_API_KEY', { ...options, enableMetricsAutocapture: true })
+    })
+
+    it('collects runtime metrics on the sample interval without any instrumentation', async () => {
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+
+      const names = metricNames()
+      expect(names).toEqual(
+        expect.arrayContaining([
+          'process.cpu.time',
+          'process.cpu.utilization',
+          'process.memory.usage',
+          'process.memory.heap_limit',
+          'process.event_loop.delay',
+          'process.event_loop.utilization',
+          'process.uptime',
+          'process.active_resources',
+        ])
+      )
+    })
+
+    it('breaks memory down by type and event loop delay by stat, and nothing else', async () => {
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+
+      const metricsCall = mockedFetch.mock.calls.find((call) => String(call[0]).includes('/i/v1/metrics'))!
+      const metrics = JSON.parse((metricsCall[1] as any).body).resourceMetrics[0].scopeMetrics[0].metrics
+      const byName = Object.fromEntries(metrics.map((m: any) => [m.name, m]))
+
+      const attributeValues = (metric: any, key: string): string[] => {
+        const dataPoints: any[] = metric.gauge ? metric.gauge.dataPoints : metric.sum.dataPoints
+        return dataPoints.map((point) => point.attributes.find((attr: any) => attr.key === key)?.value.stringValue)
+      }
+
+      expect(attributeValues(byName['process.memory.usage'], 'type').sort()).toEqual([
+        'array_buffers',
+        'external',
+        'heap_total',
+        'heap_used',
+        'rss',
+      ])
+      // `mean` is NaN until the event loop monitor has seen a tick, and NaN
+      // samples are dropped rather than shipped as a bogus data point.
+      const delayStats = attributeValues(byName['process.event_loop.delay'], 'stat').sort()
+      expect(delayStats).toEqual(expect.arrayContaining(['max', 'p50', 'p90', 'p99']))
+      expect(delayStats.every((stat) => ['mean', 'p50', 'p90', 'p99', 'max'].includes(stat))).toBe(true)
+      expect(attributeValues(byName['process.cpu.time'], 'state').sort()).toEqual(['system', 'user'])
+      // No per-host or per-instance attributes — those would be a series per box.
+      expect(byName['process.uptime'].gauge.dataPoints[0].attributes).toEqual([])
+    })
+
+    it('does not evaluate the gate flag', async () => {
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+
+      expect(flagsCalls()).toHaveLength(0)
+    })
+
+    it('respects a custom sample interval', async () => {
+      await posthog.shutdown()
+      posthog = new PostHog('TEST_API_KEY', {
+        ...options,
+        enableMetricsAutocapture: true,
+        metricsAutocaptureIntervalMs: 60000,
+      })
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+      expect(metricNames()).toHaveLength(0)
+
+      await jest.advanceTimersByTimeAsync(60000)
+      await posthog.metrics.flush()
+      expect(metricNames()).toContain('process.memory.usage')
+    })
+
+    it('stops sampling on shutdown', async () => {
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.shutdown()
+      mockedFetch.mockClear()
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS * 5)
+
+      expect(metricNames()).toHaveLength(0)
+    })
+  })
+
+  it('collects nothing when explicitly disabled', async () => {
+    posthog = new PostHog('TEST_API_KEY', { ...localEvaluationOptions, enableMetricsAutocapture: false })
+
+    await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS * 2)
+    await posthog.metrics.flush()
+
+    expect(metricNames()).toHaveLength(0)
+  })
+
+  it('stays off when left to the flag but local evaluation is unavailable', async () => {
+    // Without the poller the gate would cost a `/flags` request per poll, so it
+    // stays closed rather than adding an unasked-for request to every client.
+    posthog = new PostHog('TEST_API_KEY', options)
+    await waitForPromises()
+
+    await jest.advanceTimersByTimeAsync(GATE_POLL_INTERVAL_MS * 2)
+    await posthog.metrics.flush()
+
+    expect(metricNames()).toHaveLength(0)
+    expect(flagsCalls()).toHaveLength(0)
+  })
+
+  describe('when left to the feature flag', () => {
+    it('collects runtime metrics once the flag evaluates to true', async () => {
+      posthog = new PostHog('TEST_API_KEY', localEvaluationOptions)
+      await waitForPromises()
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+
+      expect(metricNames()).toContain('process.memory.usage')
+      // Local evaluation only: the gate never costs a `/flags` request.
+      expect(flagsCalls()).toHaveLength(0)
+    })
+
+    it('does not capture a $feature_flag_called event for its own gate', async () => {
+      // The SDK evaluating a flag about itself must not bill the user for an
+      // event, nor attach one to the synthetic gate distinct ID.
+      posthog = new PostHog('TEST_API_KEY', localEvaluationOptions)
+      await waitForPromises()
+      await jest.advanceTimersByTimeAsync(GATE_POLL_INTERVAL_MS)
+      await posthog.flush()
+
+      expect(mockedFetch.mock.calls.filter((call) => String(call[0]).includes('/batch/'))).toHaveLength(0)
+    })
+
+    it('collects nothing while the flag evaluates to false', async () => {
+      mockApi(false)
+      posthog = new PostHog('TEST_API_KEY', localEvaluationOptions)
+      await waitForPromises()
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS * 2)
+      await posthog.metrics.flush()
+
+      expect(metricNames()).toHaveLength(0)
+    })
+
+    it('acts as a kill switch: stops collecting when the flag is turned off', async () => {
+      posthog = new PostHog('TEST_API_KEY', localEvaluationOptions)
+      await waitForPromises()
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+      expect(metricNames()).toContain('process.memory.usage')
+
+      mockApi(false)
+      // One poll interval refreshes the cached definitions, the next re-evaluates
+      // the gate against them and closes it.
+      for (let i = 0; i < 3; i++) {
+        await jest.advanceTimersByTimeAsync(GATE_POLL_INTERVAL_MS)
+        await waitForPromises()
+      }
+      mockedFetch.mockClear()
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS * 3)
+      await posthog.metrics.flush()
+
+      expect(metricNames()).toHaveLength(0)
+    })
+
+    it('stays off and keeps polling when the definitions load fails', async () => {
+      mockedFetch.mockRejectedValue(new Error('connection refused'))
+      posthog = new PostHog('TEST_API_KEY', localEvaluationOptions)
+      await waitForPromises()
+
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS * 2)
+      expect(metricNames()).toHaveLength(0)
+
+      mockApi(true)
+      for (let i = 0; i < 3; i++) {
+        await jest.advanceTimersByTimeAsync(GATE_POLL_INTERVAL_MS)
+        await waitForPromises()
+      }
+      await jest.advanceTimersByTimeAsync(DEFAULT_SAMPLE_INTERVAL_MS)
+      await posthog.metrics.flush()
+
+      expect(metricNames()).toContain('process.memory.usage')
+    })
+  })
+})
+
+describe('cgroup CPU quota parsing', () => {
+  it.each([
+    // A pod limited to 500m: without this the utilization denominator would be
+    // the host's core count and the ratio would read 100x too low.
+    [{ cpuMax: '50000 100000' }, 0.5],
+    [{ cpuMax: '200000 100000\n' }, 2],
+    // Unlimited, in both cgroup versions — fall back to available parallelism.
+    [{ cpuMax: 'max 100000' }, undefined],
+    [{ cfsQuotaUs: '-1', cfsPeriodUs: '100000' }, undefined],
+    [{ cfsQuotaUs: '150000', cfsPeriodUs: '100000' }, 1.5],
+    // No cgroup filesystem at all.
+    [{}, undefined],
+  ])('parses %j as %s cores', (files, expected) => {
+    expect(parseCgroupCpuQuota(files)).toBe(expected)
+  })
+})

packages/node/src/client.ts

@@ -55,6 +55,8 @@ import {
   RequiresServerEvaluation,
 } from './extensions/feature-flags/feature-flags'
 import ErrorTracking from './extensions/error-tracking'
+import MetricsAutocapture from './extensions/metrics-autocapture'
+import type { RuntimeMetricsSampler } from './extensions/metrics-autocapture/types'
 import { PostHogMemoryStorage } from './storage-memory'
 import { ContextData, ContextOptions, IPostHogContext } from './extensions/context/types'
 import { type CaptureMode, resolveCaptureMode } from './capture-v1/config'
@@ -140,6 +142,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen
 
   private featureFlagsPoller?: FeatureFlagsPoller
   protected errorTracking: ErrorTracking
+  protected metricsAutocapture: MetricsAutocapture
   private maxCacheSize: number
   public readonly options: PostHogOptions
   protected readonly context?: IPostHogContext
@@ -266,10 +269,28 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen
     }
 
     this.errorTracking = new ErrorTracking(this, normalizedOptions, this._logger)
+    this.metricsAutocapture = new MetricsAutocapture(
+      this,
+      normalizedOptions,
+      this._logger,
+      // Its gate flag is evaluated locally only, so it can't resolve without the poller.
+      this.featureFlagsPoller !== undefined,
+      () => this.createRuntimeMetricsSampler()
+    )
+    this.metricsAutocapture.start()
     this.distinctIdHasSentFlagCalls = {}
     this.maxCacheSize = normalizedOptions.maxCacheSize || MAX_CACHE_SIZE
   }
 
+  /**
+   * The runtime metrics sampler used by metrics autocapture, or `undefined` when
+   * the runtime has no low-level metrics to offer. Overridden by the Node
+   * entrypoint; the base (and the edge build) opts out.
+   */
+  protected createRuntimeMetricsSampler(): RuntimeMetricsSampler | undefined {
+    return undefined
+  }
+
   protected override enqueue(
     type: string,
     message: any,
@@ -2573,6 +2594,10 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen
 
     await this.featureFlagsPoller?.stopPoller(shutdownTimeoutMs)
     this.errorTracking.shutdown()
+    // Stopped before the metrics flush below so no sample can land in a window
+    // that has already been drained, and so the event loop monitor and GC
+    // observer are torn down even if the flush times out.
+    this.metricsAutocapture.shutdown()
     if (this._metrics) {
       // Send whatever is aggregated in the current window, then clear the flush
       // timer so it can't fire after teardown. Raced against the shutdown budget:

packages/node/src/entrypoints/index.node.ts

@@ -7,6 +7,7 @@ import { createRelativePathModifier } from '../extensions/error-tracking/modifie
 import { PostHogBackendClient } from '../client'
 import { ErrorTracking as CoreErrorTracking } from '@posthog/core'
 import { PostHogContext } from '../extensions/context/context'
+import { RuntimeMetricsCollector } from '../extensions/metrics-autocapture/runtime.node'
 import { gzipCompress } from '../gzip.node'
 
 export class PostHog extends PostHogBackendClient {
@@ -22,6 +23,10 @@ export class PostHog extends PostHogBackendClient {
     return new PostHogContext()
   }
 
+  protected override createRuntimeMetricsSampler(): RuntimeMetricsCollector {
+    return new RuntimeMetricsCollector()
+  }
+
   protected override createErrorPropertiesBuilder(): CoreErrorTracking.ErrorPropertiesBuilder {
     return new CoreErrorTracking.ErrorPropertiesBuilder(
       [