PMM-15195 Fixes and MySQL 9.7 changes.

.github/workflows/go.yml

@@ -19,6 +19,7 @@ jobs:
           - mysql/mysql-server:8.0
           - mysql:8.4
           - mysql:9.0
+          - mysql:9.7
           - mariadb:10.5
           - mariadb:10.6
           - mariadb:10.11
@@ -62,3 +63,40 @@ jobs:
           git status
           docker version
           docker compose version
+
+  integration:
+    name: Integration
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: ${{ github.workspace }}/go.mod
+
+      # Kept out of the matrix above on purpose: the integration tests pin the
+      # server images they need and start them through testcontainers, so running
+      # them per matrix entry would start the same containers fourteen times over.
+
+      # Nothing built these files before, so a compile error in an integration
+      # test could not fail the build.
+      - name: Compile-check integration tests
+        run: make test-integration-build
+
+      # Only the table I/O waits test runs for now. TestPScrapeProcesslist brings
+      # up ten database containers in parallel, each server sizing InnoDB from
+      # total host memory, and that has not been validated on a hosted runner.
+      # Widen this to `make test-integration` once it has.
+      - name: Run integration tests
+        run: go test -count 1 -race -tags integration -run TestScrapePerfTableIOWaits ./collector/
+
+      - name: Run debug commands on failure
+        if: ${{ failure() }}
+        run: |
+          env | sort
+          go env | sort
+          docker version
+          free -m

CHANGELOG.md

@@ -6,7 +6,7 @@ Changes:
 
 * [CHANGE]
 * [FEATURE]
-* [ENHANCEMENT]
+* [ENHANCEMENT] Add stable, version-independent InnoDB redo log metrics (`mysql_innodb_redo_log_current_lsn`, `checkpoint_lsn`, `checkpoint_age_bytes`, `capacity_bytes`, `written_bytes_total`, `checkpoint_age_ratio`) with backward-compatible `mysql_global_status_innodb_*` aliases for MySQL 8.4/9.7, plus stable `mysql_innodb_metrics_*` aliases and corrected metric types for selected `INNODB_METRICS` counters and gauges.
 * [BUGFIX]
 
 ## 0.17.2 / 2025-02-25

Makefile

@@ -58,6 +58,19 @@ test:             ## Run all tests
 	@echo ">> running tests"
 	@$(GO) test -count 1 -race $(pkgs)
 
+# Tests behind the integration build tag start their own database containers via
+# testcontainers and pin the images they need, so they ignore MYSQL_IMAGE and must
+# not be run once per entry of the CI matrix.
+test-integration-build: ## Compile-check the tests behind the integration build tag
+	@echo ">> compile-checking integration tests"
+	@$(GO) vet -tags integration $(pkgs)
+
+# Needs a Docker host with room for ten parallel database containers. CI runs a
+# subset instead, see the integration job in .github/workflows/go.yml.
+test-integration: ## Run all tests, including those behind the integration build tag
+	@echo ">> running integration tests"
+	@$(GO) test -count 1 -race -tags integration $(pkgs)
+
 FILES = $(shell find . -type f -name '*.go')
 
 format:           ## Format the code
@@ -111,4 +124,4 @@ release:          ## Build release binary
 # 	docker exec -t --user root pmm-server chown pmm:pmm /usr/local/percona/pmm/exporters/mysqld_exporter
 # 	docker exec -t pmm-server supervisorctl start pmm-agent
 
-.PHONY: all init style format build test vet tarball docker env-up env-down help default
+.PHONY: all init style format build test test-integration test-integration-build vet tarball docker env-up env-down help default

collector/global_status.go

@@ -109,6 +109,7 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha
 
 	var key string
 	var val sql.RawBytes
+	var redoStatus innodbRedoStatus
 	var textItems = map[string]string{
 		"wsrep_local_state_uuid":   "",
 		"wsrep_cluster_state_uuid": "",
@@ -122,6 +123,7 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha
 		}
 		if floatVal, ok := parseStatus(val); ok { // Unparsable values are silently skipped.
 			key = validPrometheusName(key)
+			redoStatus.observe(key, floatVal)
 			match := globalStatusRE.FindStringSubmatch(key)
 			if match == nil {
 				ch <- prometheus.MustNewConstMetric(
@@ -175,6 +177,8 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha
 		}
 	}
 
+	redoStatus.collect(ch)
+
 	// mysql_galera_variables_info metric.
 	if textItems["wsrep_local_state_uuid"] != "" {
 		ch <- prometheus.MustNewConstMetric(

collector/info_schema_innodb_metrics.go

@@ -66,6 +66,94 @@ var (
 	)
 )
 
+type stableInnodbMetric struct {
+	name string
+	help string
+}
+
+// stableInnodbMetrics provides a version-independent API for metrics used by
+// dashboards. MySQL has changed the TYPE reported by INNODB_METRICS for some
+// counters, which changes whether the generic collector appends "_total".
+// Keep the generic metrics for compatibility and emit these stable aliases in
+// addition.
+//
+// Every entry maps to a monotonically increasing counter, so the aliases are
+// always exported as CounterValue regardless of the TYPE MySQL reports for the
+// source row. Only add names here that are genuinely cumulative counters.
+//
+// Entries also make the generic collector emit a "<name>_total" counter next to
+// the historical gauge, so do not add a name whose subsystem already contains a
+// row literally called "<name>_total": the two would collide on one fqName.
+var stableInnodbMetrics = map[string]stableInnodbMetric{
+	"buffer_flush_neighbor": {
+		name: "buffer_flush_neighbor_batches_total",
+		help: "Total number of neighbor page flush batches.",
+	},
+	"buffer_flush_neighbor_total_pages": {
+		name: "buffer_flush_neighbor_pages_total",
+		help: "Total number of pages flushed by neighbor page flushing.",
+	},
+	"purge_invoked": {
+		name: "purge_invocations_total",
+		help: "Total number of times purge was invoked.",
+	},
+	"purge_upd_exist_or_extern_records": {
+		name: "purge_updated_records_total",
+		help: "Total number of updated records processed by purge.",
+	},
+	"purge_del_mark_records": {
+		name: "purge_delete_marked_records_total",
+		help: "Total number of delete-marked records processed by purge.",
+	},
+	"trx_rw_commits": {
+		name: "transactions_read_write_committed_total",
+		help: "Total number of committed read-write transactions.",
+	},
+	"adaptive_hash_rows_added": {
+		name: "adaptive_hash_rows_added_total",
+		help: "Total number of rows added to the adaptive hash index.",
+	},
+	"adaptive_hash_rows_removed": {
+		name: "adaptive_hash_rows_removed_total",
+		help: "Total number of rows removed from the adaptive hash index.",
+	},
+	"adaptive_hash_rows_updated": {
+		name: "adaptive_hash_rows_updated_total",
+		help: "Total number of rows updated in the adaptive hash index.",
+	},
+	"adaptive_hash_pages_added": {
+		name: "adaptive_hash_pages_added_total",
+		help: "Total number of pages added to the adaptive hash index.",
+	},
+	"adaptive_hash_searches": {
+		name: "adaptive_hash_searches_total",
+		help: "Total number of adaptive hash index searches.",
+	},
+	"adaptive_hash_searches_btree": {
+		name: "adaptive_hash_btree_searches_total",
+		help: "Total number of B-tree searches that bypassed the adaptive hash index.",
+	},
+}
+
+// These values are positions or sizes rather than monotonically increasing
+// event counters. Some MySQL versions report them as counters, but dashboards
+// need their unsuffixed gauge names for max_over_time and direct arithmetic.
+// MySQL reports these rows under the "log" subsystem, older versions under
+// "recovery". Both spellings are queried by the dashboards, so both need the
+// override.
+var innodbMetricGaugeOverrides = map[string]struct{}{
+	"log/log_lsn_checkpoint_age":          {},
+	"log/log_lsn_current":                 {},
+	"log/log_lsn_last_checkpoint":         {},
+	"log/log_lsn_last_flush":              {},
+	"log/log_max_modified_age_async":      {},
+	"recovery/log_lsn_checkpoint_age":     {},
+	"recovery/log_lsn_current":            {},
+	"recovery/log_lsn_last_checkpoint":    {},
+	"recovery/log_lsn_last_flush":         {},
+	"recovery/log_max_modified_age_async": {},
+}
+
 // Regexp for matching metric aggregations.
 var (
 	bufferRE     = regexp.MustCompile(`^buffer_(pool_pages)_(.*)$`)
@@ -127,6 +215,16 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch
 		); err != nil {
 			return err
 		}
+		if stable, ok := stableInnodbMetrics[name]; ok && value >= 0 {
+			ch <- prometheus.MustNewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(namespace, "innodb_metrics", stable.name),
+					stable.help, nil, nil,
+				),
+				prometheus.CounterValue,
+				value,
+			)
+		}
 		// Special handling of the "buffer_page_io" subsystem.
 		if subsystem == "buffer_page_io" {
 			match := bufferPageRE.FindStringSubmatch(name)
@@ -171,25 +269,55 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch
 			}
 		}
 		metricName := "innodb_metrics_" + subsystem + "_" + name
-		// MySQL returns counters named two different ways. "counter" and "status_counter"
-		// value >= 0 is necessary due to upstream bugs: http://bugs.mysql.com/bug.php?id=75966
-		if (metricType == "counter" || metricType == "status_counter") && value >= 0 {
-			description := prometheus.NewDesc(
-				prometheus.BuildFQName(namespace, informationSchema, metricName+"_total"),
+		metricDesc := func(suffix string) *prometheus.Desc {
+			return prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, informationSchema, metricName+suffix),
 				comment, nil, nil,
 			)
+		}
+
+		if _, ok := innodbMetricGaugeOverrides[subsystem+"/"+name]; ok {
+			isCounter := metricType == "counter" || metricType == "status_counter"
+			// Some MySQL versions report these overridden metrics as counters
+			// and can emit the -1 sentinel due to an upstream bug
+			// (http://bugs.mysql.com/bug.php?id=75966). A negative value is not
+			// a valid sample for either the gauge or the counter, so skip it.
+			if isCounter && value < 0 {
+				continue
+			}
+			ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value)
+			// Preserve the historical counter name for users that already query
+			// it while also exposing the correctly typed gauge above.
+			if isCounter {
+				ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value)
+			}
+			continue
+		}
+
+		// Only the known cumulative rows listed in stableInnodbMetrics can
+		// safely be exposed as counters. Other set rows hold derived values
+		// such as averages or per-call figures that may decrease.
+		isSetRow := metricType == "set_member" || metricType == "set_owner"
+		if _, ok := stableInnodbMetrics[name]; ok && isSetRow && value >= 0 {
+			// Both row types were historically exported as an unsuffixed
+			// gauge. Preserve that name and add the correctly typed counter so
+			// existing and current dashboards both keep working.
+			ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value)
+			ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value)
+			continue
+		}
+
+		// MySQL returns counters named two different ways. "counter" and "status_counter".
+		// value >= 0 is necessary due to upstream bugs: http://bugs.mysql.com/bug.php?id=75966
+		if (metricType == "counter" || metricType == "status_counter") && value >= 0 {
 			ch <- prometheus.MustNewConstMetric(
-				description,
+				metricDesc("_total"),
 				prometheus.CounterValue,
 				value,
 			)
 		} else {
-			description := prometheus.NewDesc(
-				prometheus.BuildFQName(namespace, informationSchema, metricName),
-				comment, nil, nil,
-			)
 			ch <- prometheus.MustNewConstMetric(
-				description,
+				metricDesc(""),
 				prometheus.GaugeValue,
 				value,
 			)