[PostgreSQL] Add checkpointer data stream

packages/postgresql/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.33"
+  changes:
+    - description: Add checkpointer data stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18428
 - version: "1.32.1"
   changes:
     - description: Update README with Alerting Rule Templates.

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,11 @@
+ARG SERVICE_VERSION=${SERVICE_VERSION:-9.5.25}
+FROM postgres:${SERVICE_VERSION}
+
+COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
+
+# Runtime fix for host-mounted /var/log/postgresql permissions.
+# Runs before the official entrypoint drops privileges to the postgres user.
+COPY docker-entrypoint-wrapper.sh /usr/local/bin/docker-entrypoint-wrapper.sh
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint-wrapper.sh"]
+
+HEALTHCHECK --interval=10s --retries=6 CMD pg_isready -U postgres

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  postgresql:
+    build:
+      context: .
+      args:
+        SERVICE_VERSION: ${SERVICE_VERSION}
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+      # Enable logging
+      POSTGRES_INITDB_ARGS: "--data-checksums"
+    ports:
+      - 5432
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/var/log/postgresql

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+cat <<-EOF >> $PGDATA/postgresql.conf
+# Enable some log facilities.
+log_duration = 'on'
+log_connections = 'on'
+log_disconnections = 'on'
+
+# Ensure that statements are logged, with their durations.
+log_statement = 'none'
+log_min_duration_statement = 0
+
+# Give agent read permissions. In NO case for production usage.
+log_file_mode = '0666'
+
+# Try to imitate logging behaviour in Debian/Ubuntu, but there the logging collector
+# is not used.
+logging_collector = 'on'
+log_directory = '/var/log/postgresql'
+log_line_prefix = '%m [%p] %q%u@%d '
+EOF

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+cat <<-EOF >> $PGDATA/postgresql.conf
+shared_preload_libraries = 'pg_stat_statements'
+pg_stat_statements.max = 10000
+pg_stat_statements.track = all
+EOF

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql

@@ -0,0 +1 @@
+create extension pg_stat_statements;

packages/postgresql/data_stream/checkpointer/_dev/deploy/docker/docker-entrypoint-wrapper.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# The official Postgres image entrypoint will drop privileges to the `postgres` user
+# before starting the server. If /var/log/postgresql is a host bind-mount, the
+# permissions from the host win over whatever we set at build time.
+#
+# We try a best-effort fix from inside the container:
+# - ensure the directory exists
+# - if it's not writable, loosen permissions (works for many bind-mount setups)
+#
+# If the host mount is root-squashed or has restrictive ACLs, no container-side fix
+# can override that; in that case you must adjust host perms or mount options.
+
+LOG_DIR="/var/log/postgresql"
+
+mkdir -p "${LOG_DIR}" || true
+
+# If the directory isn't writable, try to make it writable for postgres.
+# We avoid chown to a fixed uid/gid (varies across distros / image versions).
+# Making it world-writable is acceptable for dev containers (this is under _dev/).
+if ! su -s /bin/sh -c "test -w '${LOG_DIR}'" postgres 2>/dev/null; then
+  chmod 0777 "${LOG_DIR}" 2>/dev/null || true
+fi
+
+# Preserve the original image behavior: when no args are provided, run `postgres`.
+if [ "$#" -eq 0 ]; then
+  set -- postgres
+fi
+
+# Use the original entrypoint and preserve its init logic.
+exec /usr/local/bin/docker-entrypoint.sh "$@"

packages/postgresql/data_stream/checkpointer/_dev/deploy/variants.yml

@@ -0,0 +1,4 @@
+variants:
+  v18:
+    SERVICE_VERSION: 18.1
+default: v18

packages/postgresql/data_stream/checkpointer/_dev/test/system/test-default-config.yml

@@ -0,0 +1,5 @@
+vars:
+  hosts:
+    - postgres://postgres:postgres@{{Hostname}}:{{Port}}?sslmode=disable
+data_stream:
+  vars: ~

packages/postgresql/data_stream/checkpointer/agent/stream/stream.yml.hbs

@@ -0,0 +1,21 @@
+metricsets: ["query"]
+period: {{period}}
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+tags:
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+{{#if ssl}}
+{{ssl}}
+{{/if}}
+driver: "postgres"
+sql_queries:
+  - query: "SELECT * FROM pg_stat_checkpointer;"
+    response_format: "table"

packages/postgresql/data_stream/checkpointer/fields/agent.yml

@@ -0,0 +1,94 @@
+- name: cloud
+  title: Cloud
+  group: 2
+  description: Fields related to the cloud or infrastructure the events are coming from.
+  footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.'
+  type: group
+  fields:
+    - name: account.id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      dimension: true
+      description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.  Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
+      example: 666777888999
+    - name: availability_zone
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      dimension: true
+      description: Availability zone in which this host is running.
+      example: us-east-1c
+    - name: instance.id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Instance ID of the host machine.
+      example: i-1234567890abcdef0
+      dimension: true
+    - name: provider
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
+      example: aws
+      dimension: true
+    - name: region
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      dimension: true
+      description: Region in which this host is running.
+      example: us-east-1
+    - name: image.id
+      type: keyword
+      description: Image ID for the cloud instance.
+- name: container
+  title: Container
+  group: 2
+  description: 'Container fields are used for meta information about the specific container that is the source of information.  These fields help correlate data based containers from any runtime.'
+  type: group
+  fields:
+    - name: id
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: Unique container id.
+      dimension: true
+- name: host
+  title: Host
+  group: 2
+  description: 'A host is defined as a general computing instance.  ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
+  type: group
+  fields:
+    - name: name
+      level: core
+      type: keyword
+      ignore_above: 1024
+      dimension: true
+      description: 'Name of the host.  It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
+    - name: containerized
+      type: boolean
+      description: >
+        If the host is a container.
+
+    - name: os.build
+      type: keyword
+      example: "18D109"
+      description: >
+        OS build information.
+
+    - name: os.codename
+      type: keyword
+      example: "stretch"
+      description: >
+        OS codename, if any.
+
+- name: agent
+  title: Agent
+  type: group
+  fields:
+    - name: id
+      type: keyword
+      ignore_above: 1024
+      dimension: true

packages/postgresql/data_stream/checkpointer/fields/base-fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: postgresql.checkpointer
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/postgresql/data_stream/checkpointer/fields/ecs.yml

@@ -0,0 +1,3 @@
+- external: ecs
+  name: service.address
+  dimension: true

packages/postgresql/data_stream/checkpointer/fields/fields.yml

@@ -0,0 +1,87 @@
+- name: sql
+  type: group
+  description: >
+    SQL-related fields.
+
+  fields:
+    - name: driver
+      type: keyword
+      description: >
+        The SQL driver used (e.g., postgres, mysql, sqlite).
+
+    - name: metrics
+      type: group
+      description: >
+        SQL metrics and performance counters.
+
+      fields:
+        - name: string.stats_reset
+          type: keyword
+          description: >
+            Time at which these statistics were last reset
+
+        - name: numeric
+          type: group
+          description: >
+            SQL metrics and performance counters.
+
+          fields:
+            - name: buffers_written
+              type: long
+              description: >
+                Number of buffers written to disk.
+
+            - name: num_requested
+              type: long
+              description: >
+                Number of requests made.
+
+            - name: num_timed
+              type: long
+              description: >
+                Number of timed operations.
+
+            - name: restartpoints_done
+              type: long
+              description: >
+                Number of completed restart points.
+
+            - name: restartpoints_req
+              type: long
+              description: >
+                Number of requested restart points.
+
+            - name: restartpoints_timed
+              type: long
+              description: >
+                Number of timed restart points.
+
+            - name: stats_reset
+              type: date
+              description: >
+                Timestamp when statistics were last reset.
+
+            - name: sync_time
+              type: double
+              description: >
+                Time spent in synchronization operations (in milliseconds).
+
+            - name: write_time
+              type: double
+              description: >
+                Time spent writing data to disk (in milliseconds).
+
+            - name: num_done
+              type: double
+              description: >
+                Number of checkpoints that have been performed
+
+            - name: slru_written
+              type: double
+              description: >
+                Number of SLRU buffers written during checkpoints and restartpoints
+
+    - name: query
+      type: text
+      description: >-
+        The SQL query string.

packages/postgresql/data_stream/checkpointer/fields/package-fields.yml

@@ -0,0 +1,2 @@
+- name: postgresql
+  type: group

packages/postgresql/data_stream/checkpointer/manifest.yml

@@ -0,0 +1,19 @@
+title: SQL checkpointer metrics
+type: metrics
+streams:
+  - input: sql/metrics
+    title: PostgreSQL checkpointer metrics
+    description: Collect PostgreSQL checkpointer metrics
+    vars:
+      - name: period
+        type: text
+        title: Period
+        default: 10s
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details.