Add correlated end-to-end wallet telemetry

packages/middleware/auth-express-middleware/src/__tests/ExpressTransportHardening.test.ts

@@ -15,7 +15,8 @@ function responseMock(): any {
     json: jest.fn(),
     text: jest.fn(),
     end: jest.fn(),
-    sendFile: jest.fn()
+    sendFile: jest.fn(),
+    once: jest.fn()
   }
   response.status.mockReturnValue(response)
   response.set.mockReturnValue(response)
@@ -740,4 +741,148 @@ describe('ExpressTransport hardening', () => {
     )
     expect(JSON.stringify(debug.mock.calls)).not.toContain('/public')
   })
+
+  it('emits a traceparent-linked auth span without request secrets', async () => {
+    const events: any[] = []
+    const middleware = createAuthMiddleware({
+      wallet: new MockWallet(new PrivateKey(1)),
+      allowUnauthenticated: true,
+      telemetry: {
+        sink: {
+          capture: event => events.push(event)
+        },
+        traceIdFactory: () => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+        spanIdFactory: () => 'bbbbbbbbbbbbbbbb'
+      }
+    })
+    const req = {
+      path: '/private/customer-record',
+      method: 'GET',
+      headers: {
+        authorization: 'Bearer never-report-this',
+        traceparent: '00-0123456789abcdef0123456789abcdef-fedcba9876543210-01'
+      }
+    } as any
+    const res = responseMock()
+    const next = jest.fn()
+
+    middleware(req, res, next)
+    await new Promise(resolve => setImmediate(resolve))
+
+    expect(next).toHaveBeenCalled()
+    expect(events).toHaveLength(1)
+    expect(events[0]).toMatchObject({
+      type: 'span',
+      name: 'wallet.auth.middleware',
+      traceId: '0123456789abcdef0123456789abcdef',
+      parentSpanId: 'fedcba9876543210',
+      spanStatus: 'ok',
+      attributes: {
+        'http.request.method': 'GET',
+        'auth.handshake': false,
+        'auth.signed_request': false,
+        'auth.disposition': 'continued'
+      }
+    })
+    expect(JSON.stringify(events)).not.toContain('customer-record')
+    expect(JSON.stringify(events)).not.toContain('never-report-this')
+  })
+
+  it('reports middleware errors and response completion without changing next semantics', async () => {
+    const events: any[] = []
+    const telemetry = {
+      sink: { capture: (event: unknown) => events.push(event) },
+      traceIdFactory: () => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+      spanIdFactory: () => 'bbbbbbbbbbbbbbbb'
+    }
+    const handleIncomingRequest = jest.spyOn(ExpressTransport.prototype, 'handleIncomingRequest')
+
+    handleIncomingRequest.mockImplementationOnce(async (_req, _res, next) => {
+      next(new Error('continued with error'))
+    })
+    const continued = createAuthMiddleware({
+      wallet: new MockWallet(new PrivateKey(1)),
+      allowUnauthenticated: true,
+      telemetry
+    })
+    const continuedNext = jest.fn()
+    continued(
+      {
+        path: '/public',
+        method: 'GET',
+        headers: {
+          traceparent: `00-${'0'.repeat(32)}-${'0'.repeat(16)}-01`
+        }
+      } as any,
+      responseMock(),
+      continuedNext
+    )
+    await flushPromises()
+    expect(continuedNext).toHaveBeenCalledWith(expect.any(Error))
+    expect(events.at(-1)).toMatchObject({
+      spanStatus: 'error',
+      attributes: { 'auth.disposition': 'continued' }
+    })
+
+    handleIncomingRequest.mockRejectedValueOnce(new Error('transport failed'))
+    const rejected = createAuthMiddleware({
+      wallet: new MockWallet(new PrivateKey(1)),
+      allowUnauthenticated: true,
+      telemetry
+    })
+    const rejectedNext = jest.fn()
+    rejected(
+      {
+        path: '/public',
+        method: 'GET',
+        headers: { traceparent: 'not-a-traceparent' }
+      } as any,
+      responseMock(),
+      rejectedNext
+    )
+    await flushPromises()
+    expect(rejectedNext).toHaveBeenCalledWith(expect.any(Error))
+    expect(events.at(-1)).toMatchObject({
+      spanStatus: 'error',
+      attributes: { 'auth.disposition': 'middleware_error' }
+    })
+
+    handleIncomingRequest.mockReturnValue(new Promise(() => {}))
+    const pending = createAuthMiddleware({
+      wallet: new MockWallet(new PrivateKey(1)),
+      allowUnauthenticated: true,
+      telemetry
+    })
+    for (const [statusCode, writableEnded, eventName, status, traceparent] of [
+      [503, true, 'finish', 'error', 'x'.repeat(129)],
+      [200, true, 'finish', 'ok', undefined],
+      [200, true, 'close', 'ok', `00-0123456789abcdef0123456789abcdef-${'0'.repeat(16)}-01`],
+      [200, false, 'close', 'cancelled', undefined]
+    ] as const) {
+      const res = responseMock()
+      res.statusCode = statusCode
+      res.writableEnded = writableEnded
+      pending(
+        {
+          path: '/public',
+          method: 'GET',
+          headers: traceparent === undefined ? {} : { traceparent }
+        } as any,
+        res,
+        jest.fn()
+      )
+      const callback = res.once.mock.calls.find(([name]: [string]) => name === eventName)?.[1]
+      callback()
+      callback()
+      expect(events.at(-1)).toMatchObject({
+        spanStatus: status,
+        attributes: {
+          'auth.disposition': eventName === 'finish' ? 'responded' : 'connection_closed',
+          'http.response.status_code': statusCode
+        }
+      })
+    }
+
+    handleIncomingRequest.mockRestore()
+  })
 })

packages/middleware/auth-express-middleware/src/index.ts

@@ -5,10 +5,12 @@ import {
   Peer,
   SessionManager,
   PublicKey,
+  Telemetry,
   type AsyncSessionManager,
   type AuthMessage,
   type PubKeyHex,
   type RequestedCertificateSet,
+  type TelemetryConfig,
   type Transport,
   type VerifiableCertificate,
   type WalletInterface
@@ -34,6 +36,22 @@ const WELL_KNOWN_AUTH_PATH = '/.well-known/auth'
 const DEFAULT_REQUEST_TIMEOUT_MS = 30_000
 const DEFAULT_MAX_PENDING_REQUESTS = 1_000
 const MAX_AUTH_HEADER_LENGTH = 4_096
+const TRACEPARENT_PATTERN = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i
+
+function parseTraceparent(
+  value: unknown
+): { traceId: string; spanId: string; traceFlags: number } | undefined {
+  if (typeof value !== 'string' || value.length > 128) return undefined
+  const match = TRACEPARENT_PATTERN.exec(value.trim())
+  if (match == null || /^0{32}$/.test(match[1]) || /^0{16}$/.test(match[2])) {
+    return undefined
+  }
+  return {
+    traceId: match[1].toLowerCase(),
+    spanId: match[2].toLowerCase(),
+    traceFlags: Number.parseInt(match[3], 16)
+  }
+}
 
 interface PendingHandle {
   res: Response
@@ -100,6 +118,12 @@ export interface AuthMiddlewareOptions {
    * 30-second timeout and 1,000 concurrent request records per process.
    */
   transportLimits?: Partial<AuthTransportLimits>
+
+  /**
+   * Optional provider-neutral authentication timing. Header values, signatures,
+   * certificate contents, wallet data, and peer identities are never emitted.
+   */
+  telemetry?: TelemetryConfig
 }
 
 class AuthProtocolError extends Error {
@@ -1318,7 +1342,8 @@ export function createAuthMiddleware(
     onCertificatesReceived,
     logger,
     logLevel,
-    transportLimits
+    transportLimits,
+    telemetry: telemetryConfig
   } = options
 
   if (wallet === null || typeof wallet !== 'object') {
@@ -1362,6 +1387,7 @@ export function createAuthMiddleware(
 
   const peer = new Peer(wallet, transport, certificatesToRequest, sessionMgr)
   transport.setPeer(peer)
+  const telemetry = new Telemetry(telemetryConfig)
 
   return (req: AuthRequest, res: Response, next: NextFunction) => {
     if (logger && logLevel && isLogLevelEnabled(logLevel, 'debug')) {
@@ -1371,6 +1397,59 @@ export function createAuthMiddleware(
         hasAuthRequestId: typeof req.headers['x-bsv-auth-request-id'] === 'string'
       })
     }
-    void transport.handleIncomingRequest(req, res, next, onCertificatesReceived).catch(next)
+    if (!telemetry.enabled) {
+      void transport.handleIncomingRequest(req, res, next, onCertificatesReceived).catch(next)
+      return
+    }
+
+    const parent = telemetry.contextFor(req) ?? parseTraceparent(req.headers.traceparent)
+    const span = telemetry.startSpan('wallet.auth.middleware', {
+      component: 'auth-express-middleware',
+      kind: 'server',
+      parent,
+      carrier: req,
+      attributes: {
+        'http.request.method': req.method,
+        'auth.handshake': req.path === WELL_KNOWN_AUTH_PATH,
+        'auth.signed_request': typeof req.headers['x-bsv-auth-request-id'] === 'string'
+      }
+    })
+    span.bind(req)
+    let ended = false
+    const end = (
+      status: 'ok' | 'error' | 'cancelled',
+      error?: unknown,
+      disposition?: string
+    ): void => {
+      if (ended) return
+      ended = true
+      span.end({
+        status,
+        error,
+        attributes: {
+          ...(disposition == null ? {} : { 'auth.disposition': disposition }),
+          ...(res.statusCode > 0 ? { 'http.response.status_code': res.statusCode } : {})
+        }
+      })
+    }
+    const tracedNext = ((argument?: unknown) => {
+      end(argument instanceof Error ? 'error' : 'ok', argument, 'continued')
+      if (argument === undefined) next()
+      else next(argument)
+    }) as NextFunction
+
+    res.once('finish', () => {
+      end(res.statusCode >= 500 ? 'error' : 'ok', undefined, 'responded')
+    })
+    res.once('close', () => {
+      end(res.writableEnded ? 'ok' : 'cancelled', undefined, 'connection_closed')
+    })
+
+    void transport
+      .handleIncomingRequest(req, res, tracedNext, onCertificatesReceived)
+      .catch(error => {
+        end('error', error, 'middleware_error')
+        next(error)
+      })
   }
 }

packages/sdk/docs/telemetry.md

@@ -32,6 +32,66 @@ const resolver = new LookupResolver({ telemetry })
 `capture` never throws into SDK behavior. Synchronous and asynchronous sink
 failures are isolated. Events are sanitized before `beforeSend` and sanitized
 again afterward, so consumer enrichment cannot bypass the redaction boundary.
+Sinks should only enqueue work: telemetry export must not be awaited on a
+wallet, bridge, authentication, or storage request path.
+
+Long-lived hosts can pass `enabled: () => currentDiagnosticsPreference` to
+honor runtime opt-in/opt-out changes without reconstructing the wallet.
+
+## Timed and correlated work
+
+`Telemetry.startSpan()` and `Telemetry.withSpan()` record high-resolution
+monotonic durations while retaining wall-clock start times. Spans use
+W3C-compatible 32-character trace IDs, 16-character span IDs, explicit
+parentage, kinds, and status. Sibling work can run in parallel without losing
+its parent:
+
+```ts
+import { Telemetry, formatTraceparent, instrumentWallet } from '@bsv/sdk'
+
+const dispatcher = new Telemetry(telemetry)
+const walletAtBoundary = instrumentWallet(wallet, dispatcher, {
+  component: 'react-native-bridge',
+  kind: 'server'
+})
+
+await dispatcher.withSpan(
+  'wallet.storage.rpc',
+  { component: 'storage-client', kind: 'client', carrier: args },
+  async span => {
+    const traceparent = formatTraceparent(span.context)
+    await fetch(storageUrl, {
+      method: 'POST',
+      headers: traceparent == null ? {} : { traceparent }
+    })
+  }
+)
+```
+
+The first BRC-100 argument object is an explicit context carrier. Consumers and
+Wallet Toolbox layers can use `contextFor`, `bindContext`, and `linkContext`
+without modifying or serializing wallet arguments. This is the portable
+browser and React Native propagation mechanism. Node hosts should provide a
+`TelemetryContextManager` backed by `AsyncLocalStorage` so database queries and
+other asynchronous descendants inherit the active request automatically.
+
+`formatTraceparent` and `parseTraceparent` propagate context over HTTP using
+W3C Trace Context. Invalid, unsupported-version, oversized, and all-zero
+headers are ignored.
+
+## Runtime measurements
+
+Hosts may provide `TelemetryRuntimeMetrics` to add privacy-safe CPU, garbage
+collection, heap, or event-loop deltas to completed spans. Availability varies
+by runtime: wall-clock and monotonic duration are always available, while a
+browser or React Native runtime must not label inferred waiting time as exact
+CPU or GC time. Metric names and units should be stable scalar attributes, such
+as `runtime.cpu_ms`, `runtime.gc_ms`, and `runtime.event_loop_utilization`.
+
+Telemetry work has its own cost. Exporters should measure queue time, encoding
+time, batch size, dropped-event counts, retry counts, and export latency as
+separate telemetry events. Those exporter events must be prevented from
+recursively reporting themselves.
 
 The event schema intentionally permits scalar attributes only. Names and values
 associated with passwords, private or presentation keys, recovery material,
@@ -43,6 +103,12 @@ Redaction is a defense in depth measure, not permission to attach sensitive
 objects. Producers and consumers must never add request payloads, keys, wallet
 snapshots, encrypted factors, or transaction material to telemetry.
 
+High-cardinality identity, origin, URL, query-text, and transaction attributes
+are not emitted by the generic instrumentation. A wallet vendor may attach a
+bounded origin category or one-way pseudonymous cohort under its own consent,
+retention, and privacy policy, but must not put raw credentials, request
+payloads, SQL, authentication headers, or wallet data into an event.
+
 `LookupResolver.queryDetailed()` returns the ordinary lookup answer together
 with host-settlement evidence. Security-sensitive code can use the counts to
 distinguish an authoritative empty result from timeouts, malformed responses,