diff --git a/packages/postgresql/changelog.yml b/packages/postgresql/changelog.yml index 777b9c99d9f..fe35e4edf10 100644 --- a/packages/postgresql/changelog.yml +++ b/packages/postgresql/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.33" + changes: + - description: Add checkpointer data stream + type: enhancement + link: https://github.com/elastic/integrations/pull/18428 - version: "1.32.1" changes: - description: Update README with Alerting Rule Templates. diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/Dockerfile b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/Dockerfile new file mode 100644 index 00000000000..b9c9aa48a9c --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/Dockerfile @@ -0,0 +1,11 @@ +ARG SERVICE_VERSION=${SERVICE_VERSION:-9.5.25} +FROM postgres:${SERVICE_VERSION} + +COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d + +# Runtime fix for host-mounted /var/log/postgresql permissions. +# Runs before the official entrypoint drops privileges to the postgres user. +COPY docker-entrypoint-wrapper.sh /usr/local/bin/docker-entrypoint-wrapper.sh +ENTRYPOINT ["/usr/local/bin/docker-entrypoint-wrapper.sh"] + +HEALTHCHECK --interval=10s --retries=6 CMD pg_isready -U postgres diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-compose.yml b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..194f7c882a3 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,16 @@ +services: + postgresql: + build: + context: . + args: + SERVICE_VERSION: ${SERVICE_VERSION} + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # Enable logging + POSTGRES_INITDB_ARGS: "--data-checksums" + ports: + - 5432 + volumes: + - ${SERVICE_LOGS_DIR}:/var/log/postgresql diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh new file mode 100644 index 00000000000..77f4f9186d4 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +cat <<-EOF >> $PGDATA/postgresql.conf +# Enable some log facilities. +log_duration = 'on' +log_connections = 'on' +log_disconnections = 'on' + +# Ensure that statements are logged, with their durations. +log_statement = 'none' +log_min_duration_statement = 0 + +# Give agent read permissions. In NO case for production usage. +log_file_mode = '0666' + +# Try to imitate logging behaviour in Debian/Ubuntu, but there the logging collector +# is not used. +logging_collector = 'on' +log_directory = '/var/log/postgresql' +log_line_prefix = '%m [%p] %q%u@%d ' +EOF diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh new file mode 100755 index 00000000000..03b17c78789 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +cat <<-EOF >> $PGDATA/postgresql.conf +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.max = 10000 +pg_stat_statements.track = all +EOF diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql new file mode 100644 index 00000000000..41424ded1ef --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql @@ -0,0 +1 @@ +create extension pg_stat_statements; \ No newline at end of file diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-wrapper.sh b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-wrapper.sh new file mode 100755 index 00000000000..8c5ca35e790 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-wrapper.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The official Postgres image entrypoint will drop privileges to the `postgres` user +# before starting the server. If /var/log/postgresql is a host bind-mount, the +# permissions from the host win over whatever we set at build time. +# +# We try a best-effort fix from inside the container: +# - ensure the directory exists +# - if it's not writable, loosen permissions (works for many bind-mount setups) +# +# If the host mount is root-squashed or has restrictive ACLs, no container-side fix +# can override that; in that case you must adjust host perms or mount options. + +LOG_DIR="/var/log/postgresql" + +mkdir -p "${LOG_DIR}" || true + +# If the directory isn't writable, try to make it writable for postgres. +# We avoid chown to a fixed uid/gid (varies across distros / image versions). +# Making it world-writable is acceptable for dev containers (this is under _dev/). +if ! su -s /bin/sh -c "test -w '${LOG_DIR}'" postgres 2>/dev/null; then + chmod 0777 "${LOG_DIR}" 2>/dev/null || true +fi + +# Preserve the original image behavior: when no args are provided, run `postgres`. +if [ "$#" -eq 0 ]; then + set -- postgres +fi + +# Use the original entrypoint and preserve its init logic. +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/packages/postgresql/data_stream/checkpointer/_dev/deploy/variants.yml b/packages/postgresql/data_stream/checkpointer/_dev/deploy/variants.yml new file mode 100644 index 00000000000..dbbdb86d5ff --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v18: + SERVICE_VERSION: 18.1 +default: v18 diff --git a/packages/postgresql/data_stream/checkpointer/_dev/test/system/test-default-config.yml b/packages/postgresql/data_stream/checkpointer/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..1445abc9676 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + hosts: + - postgres://postgres:postgres@{{Hostname}}:{{Port}}?sslmode=disable +data_stream: + vars: ~ diff --git a/packages/postgresql/data_stream/checkpointer/agent/stream/stream.yml.hbs b/packages/postgresql/data_stream/checkpointer/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..5861819e752 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/agent/stream/stream.yml.hbs @@ -0,0 +1,21 @@ +metricsets: ["query"] +period: {{period}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if ssl}} +{{ssl}} +{{/if}} +driver: "postgres" +sql_queries: + - query: "SELECT * FROM pg_stat_checkpointer;" + response_format: "table" \ No newline at end of file diff --git a/packages/postgresql/data_stream/checkpointer/fields/agent.yml b/packages/postgresql/data_stream/checkpointer/fields/agent.yml new file mode 100644 index 00000000000..c8e5134d2e1 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/fields/agent.yml @@ -0,0 +1,94 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + dimension: true + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + dimension: true + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + dimension: true + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + dimension: true + - name: region + level: extended + type: keyword + ignore_above: 1024 + dimension: true + description: Region in which this host is running. + example: us-east-1 + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + dimension: true +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: name + level: core + type: keyword + ignore_above: 1024 + dimension: true + description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: agent + title: Agent + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + dimension: true diff --git a/packages/postgresql/data_stream/checkpointer/fields/base-fields.yml b/packages/postgresql/data_stream/checkpointer/fields/base-fields.yml new file mode 100644 index 00000000000..9d7d1e1029e --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.dataset + type: constant_keyword + description: Event dataset + value: postgresql.checkpointer +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/postgresql/data_stream/checkpointer/fields/ecs.yml b/packages/postgresql/data_stream/checkpointer/fields/ecs.yml new file mode 100644 index 00000000000..4b73f809437 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/fields/ecs.yml @@ -0,0 +1,3 @@ +- external: ecs + name: service.address + dimension: true diff --git a/packages/postgresql/data_stream/checkpointer/fields/fields.yml b/packages/postgresql/data_stream/checkpointer/fields/fields.yml new file mode 100644 index 00000000000..d9840496fc6 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/fields/fields.yml @@ -0,0 +1,87 @@ +- name: sql + type: group + description: > + SQL-related fields. + + fields: + - name: driver + type: keyword + description: > + The SQL driver used (e.g., postgres, mysql, sqlite). + + - name: metrics + type: group + description: > + SQL metrics and performance counters. + + fields: + - name: string.stats_reset + type: keyword + description: > + Time at which these statistics were last reset + + - name: numeric + type: group + description: > + SQL metrics and performance counters. + + fields: + - name: buffers_written + type: long + description: > + Number of buffers written to disk. + + - name: num_requested + type: long + description: > + Number of requests made. + + - name: num_timed + type: long + description: > + Number of timed operations. + + - name: restartpoints_done + type: long + description: > + Number of completed restart points. + + - name: restartpoints_req + type: long + description: > + Number of requested restart points. + + - name: restartpoints_timed + type: long + description: > + Number of timed restart points. + + - name: stats_reset + type: date + description: > + Timestamp when statistics were last reset. + + - name: sync_time + type: double + description: > + Time spent in synchronization operations (in milliseconds). + + - name: write_time + type: double + description: > + Time spent writing data to disk (in milliseconds). + + - name: num_done + type: double + description: > + Number of checkpoints that have been performed + + - name: slru_written + type: double + description: > + Number of SLRU buffers written during checkpoints and restartpoints + + - name: query + type: text + description: >- + The SQL query string. diff --git a/packages/postgresql/data_stream/checkpointer/fields/package-fields.yml b/packages/postgresql/data_stream/checkpointer/fields/package-fields.yml new file mode 100644 index 00000000000..7094d614dd0 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/fields/package-fields.yml @@ -0,0 +1,2 @@ +- name: postgresql + type: group diff --git a/packages/postgresql/data_stream/checkpointer/manifest.yml b/packages/postgresql/data_stream/checkpointer/manifest.yml new file mode 100644 index 00000000000..bbcab2a0413 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/manifest.yml @@ -0,0 +1,19 @@ +title: SQL checkpointer metrics +type: metrics +streams: + - input: sql/metrics + title: PostgreSQL checkpointer metrics + description: Collect PostgreSQL checkpointer metrics + vars: + - name: period + type: text + title: Period + default: 10s + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. diff --git a/packages/postgresql/data_stream/checkpointer/sample_event.json b/packages/postgresql/data_stream/checkpointer/sample_event.json new file mode 100644 index 00000000000..81b8a2961b9 --- /dev/null +++ b/packages/postgresql/data_stream/checkpointer/sample_event.json @@ -0,0 +1,83 @@ +{ + "@timestamp": "2026-01-14T00:33:10.439Z", + "agent": { + "ephemeral_id": "aef0b5f6-28ef-40e6-82f8-bc6bd1ff3f65", + "id": "6c1b550f-d4b8-4388-83fc-b2a18bc59ece", + "name": "elastic-agent-31613", + "type": "metricbeat", + "version": "9.3.0" + }, + "data_stream": { + "dataset": "postgresql.checkpointer", + "namespace": "39524", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "6c1b550f-d4b8-4388-83fc-b2a18bc59ece", + "snapshot": true, + "version": "9.3.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "postgresql.checkpointer", + "duration": 11140095, + "ingested": "2026-01-14T00:33:13Z", + "module": "sql" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "elastic-agent-31613", + "ip": [ + "172.19.0.2", + "172.18.0.7" + ], + "mac": [ + "52-5F-73-53-ED-74", + "72-61-21-12-72-06" + ], + "name": "elastic-agent-31613", + "os": { + "family": "", + "kernel": "6.8.0-64-generic", + "name": "Wolfi", + "platform": "wolfi", + "type": "linux", + "version": "20230201" + } + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "svc-postgresql:5432", + "type": "sql" + }, + "sql": { + "driver": "postgres", + "metrics": { + "numeric": { + "buffers_written": 75, + "num_done": 0, + "num_requested": 1, + "num_timed": 0, + "restartpoints_done": 0, + "restartpoints_req": 0, + "restartpoints_timed": 0, + "slru_written": 3, + "sync_time": 3, + "write_time": 2 + }, + "string": { + "stats_reset": "2026-01-14T00:32:49.36029Z" + } + }, + "query": [ + "SELECT * FROM pg_stat_checkpointer;" + ] + } +} diff --git a/packages/postgresql/manifest.yml b/packages/postgresql/manifest.yml index 4b7023a14ad..fe363bfa57c 100644 --- a/packages/postgresql/manifest.yml +++ b/packages/postgresql/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: postgresql title: PostgreSQL -version: "1.32.1" +version: "1.33" description: Collect logs and metrics from PostgreSQL servers with Elastic Agent. type: integration categories: @@ -73,6 +73,18 @@ policy_templates: multi: false required: false show_user: false + - type: sql/metrics + title: Collect PostgreSQL checkpointer metrics + description: Collect PostgreSQL checkpointer metrics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - postgres://localhost:5432 owner: github: elastic/obs-infraobs-integrations type: elastic