diff --git a/clickhouse/src/model/click-house-client.test.ts b/clickhouse/src/model/click-house-client.test.ts new file mode 100644 index 000000000..becfde8e0 --- /dev/null +++ b/clickhouse/src/model/click-house-client.test.ts @@ -0,0 +1,60 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { formatClickHouseDateTime, query, replaceTimeRangePlaceholders } from './click-house-client'; + +describe('ClickHouse client', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should replace time range placeholders', () => { + expect( + replaceTimeRangePlaceholders( + "SELECT * FROM logs WHERE timestamp BETWEEN '{start}' AND '{end}'", + '2025-01-01 00:00:00', + '2025-01-02 00:00:00' + ) + ).toEqual("SELECT * FROM logs WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00'"); + }); + + it('should format time range dates for ClickHouse SQL literals', () => { + expect(formatClickHouseDateTime(new Date('2025-01-01T00:00:00.000Z'))).toBe('2025-01-01 00:00:00'); + }); + + it('should send interpolated query to ClickHouse', async () => { + const fetchMock = jest.fn>, [string, RequestInit?]>(async () => ({ + ok: true, + json: jest.fn(async () => ({ data: [] })), + })); + global.fetch = fetchMock as unknown as typeof fetch; + + await query( + { + query: "SELECT * FROM logs WHERE timestamp BETWEEN '{start}' AND '{end}'", + start: '2025-01-01 00:00:00', + end: '2025-01-02 00:00:00', + }, + { + datasourceUrl: 'http://clickhouse.example.com', + } + ); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [calledUrl] = fetchMock.mock.calls[0]!; + const url = new URL(calledUrl); + expect(url.searchParams.get('query')).toBe( + "SELECT * FROM logs WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' FORMAT JSON" + ); + }); +}); diff --git a/clickhouse/src/model/click-house-client.ts b/clickhouse/src/model/click-house-client.ts index 363611957..51f15ff41 100644 --- a/clickhouse/src/model/click-house-client.ts +++ b/clickhouse/src/model/click-house-client.ts @@ -15,6 +15,8 @@ import { RequestHeaders } from '@perses-dev/core'; export interface ClickHouseQueryParams { query: string; + start?: string; + end?: string; database?: string; } @@ -32,6 +34,17 @@ export interface ClickHouseClient { query: (params: { start: string; end: string; query: string }) => Promise; } +export function replaceTimeRangePlaceholders(query: string, start?: string, end?: string): string { + return query.replaceAll('{start}', start ?? '{start}').replaceAll('{end}', end ?? '{end}'); +} + +export function formatClickHouseDateTime(date: Date): string { + return date + .toISOString() + .replace('T', ' ') + .replace(/\.\d{3}Z$/, ''); +} + export async function query( params: ClickHouseQueryParams, queryOptions: ClickHouseQueryOptions @@ -43,7 +56,7 @@ export async function query( throw new Error('No query provided in params'); } - let finalQuery = params.query.trim(); + let finalQuery = replaceTimeRangePlaceholders(params.query.trim(), params.start, params.end); if (!finalQuery.toUpperCase().includes('FORMAT')) { finalQuery += ' FORMAT JSON'; } diff --git a/clickhouse/src/model/click-house-data-types.ts b/clickhouse/src/model/click-house-data-types.ts index f3b71e886..30495726b 100644 --- a/clickhouse/src/model/click-house-data-types.ts +++ b/clickhouse/src/model/click-house-data-types.ts @@ -19,5 +19,5 @@ export interface ClickHouseTimeSeriesData extends TimeSeriesData { export interface TimeSeriesEntry { time: string; - log_count: number | string; + [key: string]: number | string | null | undefined; } diff --git a/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts b/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts index 65b8f8aa7..87198acb0 100644 --- a/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts +++ b/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts @@ -47,8 +47,8 @@ const createStubContext = (): ClickHouseQueryContext => { setSavedDatasources: jest.fn(), }, timeRange: { - end: new Date('01-01-2025'), - start: new Date('01-02-2025'), + end: new Date('2025-01-02T00:00:00.000Z'), + start: new Date('2025-01-01T00:00:00.000Z'), }, variableState: {}, }; @@ -71,4 +71,23 @@ describe('ClickHouseLogQuery', () => { const initialOptions = ClickHouseLogQuery.createInitialOptions(); expect(initialOptions).toEqual({ query: '' }); }); + + it('should run query with interpolated time range', async () => { + const response = await ClickHouseLogQuery.getLogData( + { + query: "SELECT * FROM application_logs WHERE timestamp >= '{start}' AND timestamp <= '{end}'", + }, + createStubContext() + ); + + expect(clickhouseStubClient.query).toHaveBeenCalledWith({ + start: '2025-01-01 00:00:00', + end: '2025-01-02 00:00:00', + query: + "SELECT * FROM application_logs WHERE timestamp >= '2025-01-01 00:00:00' AND timestamp <= '2025-01-02 00:00:00'", + }); + expect(response.metadata?.executedQueryString).toBe( + "SELECT * FROM application_logs WHERE timestamp >= '2025-01-01 00:00:00' AND timestamp <= '2025-01-02 00:00:00'" + ); + }); }); diff --git a/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts b/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts index 5ea71ec56..01d6c5342 100644 --- a/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts +++ b/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts @@ -13,7 +13,12 @@ import { replaceVariables } from '@perses-dev/plugin-system'; import { LogEntry, LogData } from '@perses-dev/core'; -import { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client'; +import { + ClickHouseClient, + ClickHouseQueryResponse, + formatClickHouseDateTime, + replaceTimeRangePlaceholders, +} from '../../model/click-house-client'; import { DEFAULT_DATASOURCE } from '../constants'; import { ClickHouseLogQuerySpec } from './click-house-log-query-types'; import { LogQueryPlugin } from './log-query-plugin-interface'; @@ -83,18 +88,21 @@ export const getClickHouseLogData: LogQueryPlugin['getLo )) as ClickHouseClient; const { start, end } = context.timeRange; + const startTime = formatClickHouseDateTime(start); + const endTime = formatClickHouseDateTime(end); + const executedQueryString = replaceTimeRangePlaceholders(query, startTime, endTime); const response: ClickHouseQueryResponse = await client.query({ - start: start.getTime().toString(), - end: end.getTime().toString(), - query, + start: startTime, + end: endTime, + query: executedQueryString, }); return { timeRange: { start, end }, logs: convertStreamsToLogs(response.data as LogEntry[]), metadata: { - executedQueryString: query, + executedQueryString, }, }; }; diff --git a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts index ca1e9f809..f6f68de99 100644 --- a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts +++ b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts @@ -32,8 +32,8 @@ clickhouseStubClient.query = jest.fn(async () => { const stubResponse: ClickHouseQueryResponse = { status: 'success', data: [ - { time: '2025-09-09 05:18:00', log_count: '277' }, - { time: '2025-09-09 05:19:00', log_count: '156102' }, + { time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277, service: 'api' }, + { time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102, service: 'api' }, ], }; return stubResponse as ClickHouseQueryResponse; @@ -65,8 +65,8 @@ const createStubContext = (): TimeSeriesQueryContext => { setSavedDatasources: jest.fn(), }, timeRange: { - end: new Date('01-01-2025'), - start: new Date('01-02-2025'), + end: new Date('2025-01-02T00:00:00.000Z'), + start: new Date('2025-01-01T00:00:00.000Z'), }, variableState: {}, }; @@ -90,11 +90,92 @@ describe('ClickHouseTimeSeriesQuery', () => { expect(initialOptions).toEqual({ query: '' }); }); - it('should run query and return ClickHouse data only', async () => { - const client = getDatasourceClient(); - const resp = await client.query('SELECT count(*) FROM otel_logs'); - expect(resp.data.length).toBeGreaterThan(0); - expect(resp.data[0]).toHaveProperty('time'); - expect(resp.data[0]).toHaveProperty('log_count'); + it('should run query with interpolated time range and return one series per numeric column', async () => { + const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData( + { + query: + "SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time", + }, + createStubContext() + ); + + expect(clickhouseStubClient.query).toHaveBeenCalledWith({ + start: '2025-01-01 00:00:00', + end: '2025-01-02 00:00:00', + query: + "SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' GROUP BY time ORDER BY time", + }); + expect(response.series).toEqual([ + { + name: 'avg_cpu', + values: [ + [new Date('2025-09-09 05:18:00').getTime(), 2.5], + [new Date('2025-09-09 05:19:00').getTime(), 3.5], + ], + }, + { + name: 'max_memory', + values: [ + [new Date('2025-09-09 05:18:00').getTime(), 277], + [new Date('2025-09-09 05:19:00').getTime(), 156102], + ], + }, + ]); + expect(response.stepMs).toBe(60 * 1000); + expect(response.metadata?.executedQueryString).toBe( + "SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' GROUP BY time ORDER BY time" + ); + }); + + it('should infer daily query step from returned timestamps', async () => { + (clickhouseStubClient.query as jest.Mock).mockResolvedValueOnce({ + status: 'success', + data: [ + { time: '2026-01-01 22:00:00', flights: 80 }, + { time: '2026-01-02 22:00:00', flights: 56 }, + { time: '2026-01-03 22:00:00', flights: 32 }, + ], + }); + + const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData( + { + query: + "SELECT toStartOfDay(timestamp) as time, sum(flights_count) as flights FROM flight WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time", + }, + createStubContext() + ); + + expect(response.stepMs).toBe(24 * 60 * 60 * 1000); + expect(response.series).toEqual([ + { + name: 'flights', + values: [ + [new Date('2026-01-01 22:00:00').getTime(), 80], + [new Date('2026-01-02 22:00:00').getTime(), 56], + [new Date('2026-01-03 22:00:00').getTime(), 32], + ], + }, + ]); + }); + + it('should keep timezone daily buckets daily across daylight saving time changes', async () => { + (clickhouseStubClient.query as jest.Mock).mockResolvedValueOnce({ + status: 'success', + data: [ + { time: '2026-03-28 22:00:00', flights: 80 }, + { time: '2026-03-29 21:00:00', flights: 56 }, + { time: '2026-03-30 21:00:00', flights: 32 }, + ], + }); + + const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData( + { + query: + "SELECT toStartOfDay(timestamp) as time, sum(flights_count) as flights FROM flight WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time", + }, + createStubContext() + ); + + expect(response.stepMs).toBe(24 * 60 * 60 * 1000); }); }); diff --git a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts index ec21523f8..349b795f4 100644 --- a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts +++ b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts @@ -15,27 +15,91 @@ import { TimeSeries } from '@perses-dev/core'; import { TimeSeriesQueryPlugin, replaceVariables } from '@perses-dev/plugin-system'; import { DEFAULT_DATASOURCE } from '../constants'; import { TimeSeriesEntry } from '../../model/click-house-data-types'; -import { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client'; +import { + ClickHouseClient, + ClickHouseQueryResponse, + formatClickHouseDateTime, + replaceTimeRangePlaceholders, +} from '../../model/click-house-client'; import { ClickHouseTimeSeriesQuerySpec, DatasourceQueryResponse } from './click-house-query-types'; +const DEFAULT_STEP_MS = 30 * 1000; + function buildTimeSeries(response?: DatasourceQueryResponse): TimeSeries[] { const data = response?.data as TimeSeriesEntry[]; if (!response || !data || data.length === 0) { return []; } - const values: Array<[number, number]> = data.map((row: TimeSeriesEntry) => { - const timestamp = new Date(row.time).getTime(); - const value = Number(row.log_count); - return [timestamp, value]; - }); + const metricNames = Object.keys(data[0] ?? {}).filter((key) => key !== 'time'); - return [ - { - name: 'log_count', - values, - }, - ]; + return metricNames + .map((metricName) => { + const values: Array<[number, number | null]> = data.map((row: TimeSeriesEntry) => { + const timestamp = new Date(row.time).getTime(); + const value = toTimeSeriesValue(row[metricName]); + return [timestamp, value]; + }); + + return { + name: metricName, + values, + }; + }) + .filter((series) => series.values.some(([, value]) => value !== null)); +} + +function toTimeSeriesValue(value: number | string | null | undefined): number | null { + if (value === null || value === undefined || value === '') { + return null; + } + + const numericValue = Number(value); + return Number.isFinite(numericValue) ? numericValue : null; +} + +function inferStepMs(response?: DatasourceQueryResponse): number { + const data = response?.data as TimeSeriesEntry[]; + if (!response || !data || data.length < 2) { + return DEFAULT_STEP_MS; + } + + const timestamps = data + .map((row: TimeSeriesEntry) => new Date(row.time).getTime()) + .filter(Number.isFinite) + .sort((a, b) => a - b); + + if (timestamps.length < 2) { + return DEFAULT_STEP_MS; + } + + const deltas: number[] = []; + for (let i = 1; i < timestamps.length; i++) { + const previous = timestamps[i - 1]; + const current = timestamps[i]; + if (previous === undefined || current === undefined || current <= previous) { + continue; + } + deltas.push(current - previous); + } + + if (deltas.length === 0) { + return DEFAULT_STEP_MS; + } + + const deltaCounts = new Map(); + for (const delta of deltas) { + deltaCounts.set(delta, (deltaCounts.get(delta) ?? 0) + 1); + } + + const inferredStep = Array.from(deltaCounts.entries()).sort(([deltaA, countA], [deltaB, countB]) => { + if (countA !== countB) { + return countB - countA; + } + return deltaB - deltaA; + })[0]?.[0]; + + return inferredStep ?? DEFAULT_STEP_MS; } export const getTimeSeriesData: TimeSeriesQueryPlugin['getTimeSeriesData'] = async ( @@ -53,19 +117,22 @@ export const getTimeSeriesData: TimeSeriesQueryPlugin