From 7717de7c737994f015aba02024957dd6bbdc0f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 15 Jul 2026 15:23:51 +0200 Subject: [PATCH 01/11] PMM-15195 Fixies and MySQL 9.7 changes. --- .github/workflows/go.yml | 1 + CHANGELOG.md | 2 +- collector/global_status.go | 4 + collector/info_schema_innodb_metrics.go | 133 +++++++++++++-- collector/info_schema_innodb_metrics_test.go | 83 ++++++++++ collector/innodb_redo.go | 155 ++++++++++++++++++ collector/innodb_redo_test.go | 92 +++++++++++ collector/instance_test.go | 5 + collector/percona_global_status.go | 4 + ...fo_schema_process_list_integration_test.go | 1 + 10 files changed, 468 insertions(+), 12 deletions(-) create mode 100644 collector/innodb_redo.go create mode 100644 collector/innodb_redo_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1e3499cf3..1778e1a89 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -19,6 +19,7 @@ jobs: - mysql/mysql-server:8.0 - mysql:8.4 - mysql:9.0 + - mysql:9.7 - mariadb:10.5 - mariadb:10.6 - mariadb:10.11 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2e3172f..c148bb3e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Changes: * [CHANGE] * [FEATURE] -* [ENHANCEMENT] +* [ENHANCEMENT] Add stable InnoDB redo metrics and normalize internal metric types for MySQL 9.7 compatibility. * [BUGFIX] ## 0.17.2 / 2025-02-25 diff --git a/collector/global_status.go b/collector/global_status.go index aa70bb63b..0fbd2da65 100644 --- a/collector/global_status.go +++ b/collector/global_status.go @@ -109,6 +109,7 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha var key string var val sql.RawBytes + var redoStatus innodbRedoStatus var textItems = map[string]string{ "wsrep_local_state_uuid": "", "wsrep_cluster_state_uuid": "", @@ -122,6 +123,7 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha } if floatVal, ok := parseStatus(val); ok { // Unparsable values are silently skipped. key = validPrometheusName(key) + redoStatus.observe(key, floatVal) match := globalStatusRE.FindStringSubmatch(key) if match == nil { ch <- prometheus.MustNewConstMetric( @@ -175,6 +177,8 @@ func (ScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch cha } } + redoStatus.collect(ch) + // mysql_galera_variables_info metric. if textItems["wsrep_local_state_uuid"] != "" { ch <- prometheus.MustNewConstMetric( diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index 92c803fb4..ff853f75c 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "regexp" + "strings" "github.com/prometheus/client_golang/prometheus" ) @@ -66,6 +67,78 @@ var ( ) ) +type stableInnodbMetric struct { + name string + help string +} + +// stableInnodbMetrics provides a version-independent API for metrics used by +// dashboards. MySQL has changed the TYPE reported by INNODB_METRICS for some +// counters, which changes whether the generic collector appends "_total". +// Keep the generic metrics for compatibility and emit these stable aliases in +// addition. +var stableInnodbMetrics = map[string]stableInnodbMetric{ + "buffer_flush_neighbor": { + name: "buffer_flush_neighbor_batches_total", + help: "Total number of neighbor page flush batches.", + }, + "buffer_flush_neighbor_total_pages": { + name: "buffer_flush_neighbor_pages_total", + help: "Total number of pages flushed by neighbor page flushing.", + }, + "purge_invoked": { + name: "purge_invocations_total", + help: "Total number of times purge was invoked.", + }, + "purge_upd_exist_or_extern_records": { + name: "purge_updated_records_total", + help: "Total number of updated records processed by purge.", + }, + "purge_del_mark_records": { + name: "purge_delete_marked_records_total", + help: "Total number of delete-marked records processed by purge.", + }, + "trx_rw_commits": { + name: "transactions_read_write_committed_total", + help: "Total number of committed read-write transactions.", + }, + "adaptive_hash_rows_added": { + name: "adaptive_hash_rows_added_total", + help: "Total number of rows added to the adaptive hash index.", + }, + "adaptive_hash_rows_removed": { + name: "adaptive_hash_rows_removed_total", + help: "Total number of rows removed from the adaptive hash index.", + }, + "adaptive_hash_rows_updated": { + name: "adaptive_hash_rows_updated_total", + help: "Total number of rows updated in the adaptive hash index.", + }, + "adaptive_hash_pages_added": { + name: "adaptive_hash_pages_added_total", + help: "Total number of pages added to the adaptive hash index.", + }, + "adaptive_hash_searches": { + name: "adaptive_hash_searches_total", + help: "Total number of adaptive hash index searches.", + }, + "adaptive_hash_searches_btree": { + name: "adaptive_hash_btree_searches_total", + help: "Total number of B-tree searches that bypassed the adaptive hash index.", + }, +} + +// These values are positions or sizes rather than monotonically increasing +// event counters. Some MySQL versions report them as counters, but dashboards +// need their unsuffixed gauge names for max_over_time and direct arithmetic. +var innodbMetricGaugeOverrides = map[string]struct{}{ + "log/log_lsn_checkpoint_age": {}, + "log/log_lsn_current": {}, + "log/log_lsn_last_checkpoint": {}, + "log/log_lsn_last_flush": {}, + "log/log_max_modified_age_async": {}, +} + // Regexp for matching metric aggregations. var ( bufferRE = regexp.MustCompile(`^buffer_(pool_pages)_(.*)$`) @@ -127,6 +200,16 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch ); err != nil { return err } + if stable, ok := stableInnodbMetrics[name]; ok && value >= 0 { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_metrics", stable.name), + stable.help, nil, nil, + ), + prometheus.CounterValue, + value, + ) + } // Special handling of the "buffer_page_io" subsystem. if subsystem == "buffer_page_io" { match := bufferPageRE.FindStringSubmatch(name) @@ -171,25 +254,53 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch } } metricName := "innodb_metrics_" + subsystem + "_" + name - // MySQL returns counters named two different ways. "counter" and "status_counter" - // value >= 0 is necessary due to upstream bugs: http://bugs.mysql.com/bug.php?id=75966 - if (metricType == "counter" || metricType == "status_counter") && value >= 0 { - description := prometheus.NewDesc( - prometheus.BuildFQName(namespace, informationSchema, metricName+"_total"), + metricDesc := func(suffix string) *prometheus.Desc { + return prometheus.NewDesc( + prometheus.BuildFQName(namespace, informationSchema, metricName+suffix), comment, nil, nil, ) + } + + if _, ok := innodbMetricGaugeOverrides[subsystem+"/"+name]; ok { + ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) + // Preserve the historical counter name for users that already query + // it while also exposing the correctly typed gauge above. + if (metricType == "counter" || metricType == "status_counter") && value >= 0 { + ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value) + } + continue + } + + // MySQL returns counters as counter/status_counter and set aggregates as + // set_member/set_owner. A set_member needs the normal counter suffix, + // while set_owner names already carry their aggregate suffix. + if metricType == "set_member" && value >= 0 { + // Preserve the historical unsuffixed gauge and add the corrected + // counter so existing and current dashboards both keep working. + ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) + suffix := "_total" + if strings.HasSuffix(name, "_total") { + suffix = "" + } + ch <- prometheus.MustNewConstMetric(metricDesc(suffix), prometheus.CounterValue, value) + continue + } + if metricType == "set_owner" && value >= 0 { + ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.CounterValue, value) + continue + } + + // MySQL returns counters named two different ways. "counter" and "status_counter". + // value >= 0 is necessary due to upstream bugs: http://bugs.mysql.com/bug.php?id=75966 + if (metricType == "counter" || metricType == "status_counter") && value >= 0 { ch <- prometheus.MustNewConstMetric( - description, + metricDesc("_total"), prometheus.CounterValue, value, ) } else { - description := prometheus.NewDesc( - prometheus.BuildFQName(namespace, informationSchema, metricName), - comment, nil, nil, - ) ch <- prometheus.MustNewConstMetric( - description, + metricDesc(""), prometheus.GaugeValue, value, ) diff --git a/collector/info_schema_innodb_metrics_test.go b/collector/info_schema_innodb_metrics_test.go index 10ab6d571..bc2a1aac7 100644 --- a/collector/info_schema_innodb_metrics_test.go +++ b/collector/info_schema_innodb_metrics_test.go @@ -16,6 +16,7 @@ package collector import ( "context" "fmt" + "strings" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -81,3 +82,85 @@ func TestScrapeInnodbMetrics(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestScrapeInnodbMetricsStableAliases(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("error opening a stub database connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + + mock.ExpectQuery(sanitizeQuery(infoSchemaInnodbMetricsEnabledColumnQuery)). + WillReturnRows(sqlmock.NewRows([]string{"COLUMN_NAME"}).AddRow("STATUS")) + + rows := sqlmock.NewRows([]string{"name", "subsystem", "type", "comment", "count"}). + AddRow("buffer_flush_neighbor", "buffer", "set_member", "Neighbor flush batches", 2). + AddRow("buffer_flush_neighbor_total_pages", "buffer", "set_owner", "Neighbor flush pages", 8). + AddRow("trx_rw_commits", "transaction", "status_counter", "Read-write commits", 3). + AddRow("log_lsn_checkpoint_age", "log", "counter", "Checkpoint age", 12) + query := fmt.Sprintf(infoSchemaInnodbMetricsQuery, "status", "enabled") + mock.ExpectQuery(sanitizeQuery(query)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + if scrapeErr := (ScrapeInnodbMetrics{}).Scrape(t.Context(), inst, ch, promslog.NewNopLogger()); scrapeErr != nil { + t.Errorf("error calling function on test: %s", scrapeErr) + } + }() + + type expectedMetric struct { + value float64 + metricType dto.MetricType + } + expected := map[string]expectedMetric{ + "mysql_innodb_metrics_buffer_flush_neighbor_batches_total": { + value: 2, metricType: dto.MetricType_COUNTER, + }, + "mysql_innodb_metrics_buffer_flush_neighbor_pages_total": { + value: 8, metricType: dto.MetricType_COUNTER, + }, + "mysql_innodb_metrics_transactions_read_write_committed_total": { + value: 3, metricType: dto.MetricType_COUNTER, + }, + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_neighbor_total": { + value: 2, metricType: dto.MetricType_COUNTER, + }, + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_neighbor_total_pages": { + value: 8, metricType: dto.MetricType_COUNTER, + }, + "mysql_info_schema_innodb_metrics_log_log_lsn_checkpoint_age": { + value: 12, metricType: dto.MetricType_GAUGE, + }, + "mysql_info_schema_innodb_metrics_log_log_lsn_checkpoint_age_total": { + value: 12, metricType: dto.MetricType_COUNTER, + }, + } + found := make(map[string]bool, len(expected)) + for metric := range ch { + desc := metric.Desc().String() + for name, want := range expected { + if !strings.Contains(desc, `fqName: "`+name+`"`) { + continue + } + got := readMetric(metric) + if got.value != want.value { + t.Errorf("metric %s value = %v, want %v", name, got.value, want.value) + } + if got.metricType != want.metricType { + t.Errorf("metric %s type = %v, want %v", name, got.metricType, want.metricType) + } + found[name] = true + } + } + for name := range expected { + if !found[name] { + t.Errorf("stable metric %s was not collected", name) + } + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled expectations: %s", err) + } +} diff --git a/collector/innodb_redo.go b/collector/innodb_redo.go new file mode 100644 index 000000000..d1ed397d1 --- /dev/null +++ b/collector/innodb_redo.go @@ -0,0 +1,155 @@ +// Copyright 2026 Percona LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import "github.com/prometheus/client_golang/prometheus" + +var ( + innodbRedoCurrentLSNDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "current_lsn"), + "Current InnoDB redo log sequence number.", nil, nil, + ) + innodbRedoCheckpointLSNDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "checkpoint_lsn"), + "InnoDB redo log checkpoint log sequence number.", nil, nil, + ) + innodbRedoCheckpointAgeDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "checkpoint_age_bytes"), + "Bytes of InnoDB redo generated since the last checkpoint.", nil, nil, + ) + innodbRedoCapacityDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "capacity_bytes"), + "Total InnoDB redo log capacity in bytes.", nil, nil, + ) + innodbRedoWrittenDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "written_bytes_total"), + "Total bytes written to the InnoDB redo log files.", nil, nil, + ) + innodbRedoUsageDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "innodb_redo_log", "checkpoint_age_ratio"), + "Ratio of InnoDB checkpoint age to total redo log capacity.", nil, nil, + ) + + // Compatibility descriptors preserve the status names provided by Percona + // Server on Oracle MySQL 8.4 and newer, where Oracle exposes renamed status + // variables. They allow existing dashboards to keep working while migrating + // to the stable mysql_innodb_redo_log_* metrics above. + innodbLSNCurrentCompatDesc = newDesc( + globalStatus, "innodb_lsn_current", "Current InnoDB redo log sequence number.", + ) + innodbLSNLastCheckpointCompatDesc = newDesc( + globalStatus, "innodb_lsn_last_checkpoint", "InnoDB redo log checkpoint log sequence number.", + ) + innodbCheckpointAgeCompatDesc = newDesc( + globalStatus, "innodb_checkpoint_age", "Bytes of InnoDB redo generated since the last checkpoint.", + ) + innodbCheckpointMaxAgeCompatDesc = newDesc( + globalStatus, "innodb_checkpoint_max_age", + "Maximum InnoDB checkpoint age compatibility value based on total redo log capacity.", + ) +) + +type innodbRedoStatus struct { + currentLSN float64 + checkpointLSN float64 + checkpointAge float64 + capacity float64 + written float64 + hasCurrentLSN bool + hasCheckpointLSN bool + hasCheckpointAge bool + hasCapacity bool + hasWritten bool + hasLegacyCurrent bool + hasLegacyCheckpoint bool + hasLegacyCheckpointAge bool + hasLegacyMaxAge bool +} + +func (s *innodbRedoStatus) observe(key string, value float64) { + switch key { + case "innodb_lsn_current": + s.currentLSN = value + s.hasCurrentLSN = true + s.hasLegacyCurrent = true + case "innodb_redo_log_current_lsn": + if !s.hasLegacyCurrent { + s.currentLSN = value + s.hasCurrentLSN = true + } + case "innodb_lsn_last_checkpoint": + s.checkpointLSN = value + s.hasCheckpointLSN = true + s.hasLegacyCheckpoint = true + case "innodb_redo_log_checkpoint_lsn": + if !s.hasLegacyCheckpoint { + s.checkpointLSN = value + s.hasCheckpointLSN = true + } + case "innodb_checkpoint_age": + s.checkpointAge = value + s.hasCheckpointAge = true + s.hasLegacyCheckpointAge = true + case "innodb_checkpoint_max_age": + s.capacity = value + s.hasCapacity = true + s.hasLegacyMaxAge = true + case "innodb_redo_log_capacity_resized": + if !s.hasCapacity { + s.capacity = value + s.hasCapacity = true + } + case "innodb_os_log_written": + s.written = value + s.hasWritten = true + } +} + +func (s *innodbRedoStatus) collect(ch chan<- prometheus.Metric) { + if s.hasCurrentLSN { + ch <- prometheus.MustNewConstMetric(innodbRedoCurrentLSNDesc, prometheus.GaugeValue, s.currentLSN) + if !s.hasLegacyCurrent { + ch <- prometheus.MustNewConstMetric(innodbLSNCurrentCompatDesc, prometheus.GaugeValue, s.currentLSN) + } + } + if s.hasCheckpointLSN { + ch <- prometheus.MustNewConstMetric(innodbRedoCheckpointLSNDesc, prometheus.GaugeValue, s.checkpointLSN) + if !s.hasLegacyCheckpoint { + ch <- prometheus.MustNewConstMetric(innodbLSNLastCheckpointCompatDesc, prometheus.GaugeValue, s.checkpointLSN) + } + } + + if !s.hasCheckpointAge && s.hasCurrentLSN && s.hasCheckpointLSN && s.currentLSN >= s.checkpointLSN { + s.checkpointAge = s.currentLSN - s.checkpointLSN + s.hasCheckpointAge = true + } + if s.hasCheckpointAge { + ch <- prometheus.MustNewConstMetric(innodbRedoCheckpointAgeDesc, prometheus.GaugeValue, s.checkpointAge) + if !s.hasLegacyCheckpointAge { + ch <- prometheus.MustNewConstMetric(innodbCheckpointAgeCompatDesc, prometheus.GaugeValue, s.checkpointAge) + } + if s.hasCapacity && s.capacity > 0 { + ch <- prometheus.MustNewConstMetric(innodbRedoUsageDesc, prometheus.GaugeValue, s.checkpointAge/s.capacity) + } + } + if s.hasCapacity { + ch <- prometheus.MustNewConstMetric(innodbRedoCapacityDesc, prometheus.GaugeValue, s.capacity) + if !s.hasLegacyMaxAge { + ch <- prometheus.MustNewConstMetric(innodbCheckpointMaxAgeCompatDesc, prometheus.GaugeValue, s.capacity) + } + } + if s.hasWritten { + ch <- prometheus.MustNewConstMetric(innodbRedoWrittenDesc, prometheus.CounterValue, s.written) + } +} diff --git a/collector/innodb_redo_test.go b/collector/innodb_redo_test.go new file mode 100644 index 000000000..df921f96f --- /dev/null +++ b/collector/innodb_redo_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 Percona LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +func TestInnodbRedoStatusMySQL97(t *testing.T) { + status := innodbRedoStatus{} + status.observe("innodb_redo_log_current_lsn", 1500) + status.observe("innodb_redo_log_checkpoint_lsn", 1000) + status.observe("innodb_redo_log_capacity_resized", 2000) + status.observe("innodb_os_log_written", 4096) + + ch := make(chan prometheus.Metric, 10) + status.collect(ch) + close(ch) + + type expectedMetric struct { + name string + value float64 + metricType dto.MetricType + } + expected := []expectedMetric{ + {"mysql_innodb_redo_log_current_lsn", 1500, dto.MetricType_GAUGE}, + {"mysql_global_status_innodb_lsn_current", 1500, dto.MetricType_GAUGE}, + {"mysql_innodb_redo_log_checkpoint_lsn", 1000, dto.MetricType_GAUGE}, + {"mysql_global_status_innodb_lsn_last_checkpoint", 1000, dto.MetricType_GAUGE}, + {"mysql_innodb_redo_log_checkpoint_age_bytes", 500, dto.MetricType_GAUGE}, + {"mysql_global_status_innodb_checkpoint_age", 500, dto.MetricType_GAUGE}, + {"mysql_innodb_redo_log_checkpoint_age_ratio", 0.25, dto.MetricType_GAUGE}, + {"mysql_innodb_redo_log_capacity_bytes", 2000, dto.MetricType_GAUGE}, + {"mysql_global_status_innodb_checkpoint_max_age", 2000, dto.MetricType_GAUGE}, + {"mysql_innodb_redo_log_written_bytes_total", 4096, dto.MetricType_COUNTER}, + } + + i := 0 + for metric := range ch { + if i >= len(expected) { + t.Fatal("collected more redo metrics than expected") + } + if !strings.Contains(metric.Desc().String(), `fqName: "`+expected[i].name+`"`) { + t.Errorf("metric %d descriptor %q does not contain expected name %q", i, metric.Desc(), expected[i].name) + } + got := readMetric(metric) + if got.value != expected[i].value { + t.Errorf("metric %s value = %v, want %v", expected[i].name, got.value, expected[i].value) + } + if got.metricType != expected[i].metricType { + t.Errorf("metric %s type = %v, want %v", expected[i].name, got.metricType, expected[i].metricType) + } + i++ + } + if i != len(expected) { + t.Fatalf("collected %d redo metrics, want %d", i, len(expected)) + } +} + +func TestInnodbRedoStatusDoesNotDuplicatePerconaMetrics(t *testing.T) { + status := innodbRedoStatus{} + status.observe("innodb_lsn_current", 1500) + status.observe("innodb_lsn_last_checkpoint", 1000) + status.observe("innodb_checkpoint_age", 500) + status.observe("innodb_checkpoint_max_age", 2000) + + ch := make(chan prometheus.Metric, 5) + status.collect(ch) + close(ch) + + for metric := range ch { + desc := metric.Desc().String() + if strings.Contains(desc, "mysql_global_status_innodb_") { + t.Errorf("unexpected compatibility metric for Percona status variables: %s", desc) + } + } +} diff --git a/collector/instance_test.go b/collector/instance_test.go index ba11bd95d..c5908a2e1 100644 --- a/collector/instance_test.go +++ b/collector/instance_test.go @@ -61,6 +61,11 @@ func TestGetMySQLVersion_Percona(t *testing.T) { semVer, _, err = queryVersion(t.Context(), db) convey.ShouldBeNil(err) convey.So(semVer.String(), convey.ShouldEqual, "5.5.51") + + mock.ExpectQuery(versionQuery).WillReturnRows(sqlmock.NewRows([]string{""}).AddRow("9.7.1")) + semVer, _, err = queryVersion(t.Context(), db) + convey.ShouldBeNil(err) + convey.So(semVer.String(), convey.ShouldEqual, "9.7.1") }) // Ensure all SQL queries were executed diff --git a/collector/percona_global_status.go b/collector/percona_global_status.go index a67d08d3a..e608d396d 100644 --- a/collector/percona_global_status.go +++ b/collector/percona_global_status.go @@ -109,6 +109,7 @@ func (PScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch ch var key string var val sql.RawBytes + var redoStatus innodbRedoStatus var textItems = map[string]string{ "wsrep_local_state_uuid": "", "wsrep_cluster_state_uuid": "", @@ -122,6 +123,7 @@ func (PScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch ch } if floatVal, ok := ParseStatus(val); ok { // Unparsable values are silently skipped. key = ValidPrometheusName(key) + redoStatus.observe(key, floatVal) match := pGlobalStatusRE.FindStringSubmatch(key) if match == nil { ch <- prometheus.MustNewConstMetric( @@ -173,6 +175,8 @@ func (PScrapeGlobalStatus) Scrape(ctx context.Context, instance *instance, ch ch } } + redoStatus.collect(ch) + // mysql_galera_variables_info metric. if textItems["wsrep_local_state_uuid"] != "" { ch <- prometheus.MustNewConstMetric( diff --git a/collector/percona_info_schema_process_list_integration_test.go b/collector/percona_info_schema_process_list_integration_test.go index 0e9ca0c36..10d0f03cb 100644 --- a/collector/percona_info_schema_process_list_integration_test.go +++ b/collector/percona_info_schema_process_list_integration_test.go @@ -54,6 +54,7 @@ func TestPScrapeProcesslist(t *testing.T) { // Forward-compatibility coverage {"MySQL >=8 PS on -> perf_schema", "mysql:8", true, processlistPerfSchema}, {"MySQL >=9 PS on -> perf_schema", "mysql:9", true, processlistPerfSchema}, + {"MySQL 9.7 PS on -> perf_schema", "mysql:9.7", true, processlistPerfSchema}, {"MySQL latest PS on -> perf_schema", "mysql:latest", true, processlistPerfSchema}, } From 2e6970aad0c3b76e0f1216fa57c73162cdee0568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 16 Jul 2026 13:47:17 +0200 Subject: [PATCH 02/11] PMM-15195 Test Community MySQL table activity metrics --- ..._schema_table_io_waits_integration_test.go | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 collector/perf_schema_table_io_waits_integration_test.go diff --git a/collector/perf_schema_table_io_waits_integration_test.go b/collector/perf_schema_table_io_waits_integration_test.go new file mode 100644 index 000000000..20bf015e2 --- /dev/null +++ b/collector/perf_schema_table_io_waits_integration_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Percona LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package collector + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/promslog" + "github.com/testcontainers/testcontainers-go" + tcmysql "github.com/testcontainers/testcontainers-go/modules/mysql" +) + +func TestScrapePerfTableIOWaitsMySQLCompatibility(t *testing.T) { + cases := []struct { + name string + image string + }{ + {"MySQL 8.0", "mysql:8.0"}, + {"MySQL 9.7", "mysql:9.7"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + dsn := startPerfTableIOContainer(t, ctx, tc.image) + + instance, err := newInstance(ctx, dsn) + if err != nil { + t.Fatalf("creating instance: %v", err) + } + t.Cleanup(func() { _ = instance.Close() }) + db := instance.getDB() + + if _, err := db.ExecContext(ctx, "CREATE TABLE table_activity (id INT PRIMARY KEY, value INT)"); err != nil { + t.Fatalf("creating test table: %v", err) + } + if _, err := db.ExecContext(ctx, "INSERT INTO table_activity VALUES (1, 1), (2, 2)"); err != nil { + t.Fatalf("inserting test rows: %v", err) + } + rows, err := db.QueryContext(ctx, "SELECT * FROM table_activity") + if err != nil { + t.Fatalf("reading test rows: %v", err) + } + for rows.Next() { + var id, value int + if err := rows.Scan(&id, &value); err != nil { + _ = rows.Close() + t.Fatalf("scanning test rows: %v", err) + } + } + if err := rows.Close(); err != nil { + t.Fatalf("closing test rows: %v", err) + } + if _, err := db.ExecContext(ctx, "UPDATE table_activity SET value = value + 1 WHERE id = 1"); err != nil { + t.Fatalf("updating test row: %v", err) + } + if _, err := db.ExecContext(ctx, "DELETE FROM table_activity WHERE id = 2"); err != nil { + t.Fatalf("deleting test row: %v", err) + } + + metrics := gatherPerfTableIOWaitMetrics(t, ctx, dsn) + assertTableOperationCounter(t, metrics, "test", "table_activity", "fetch") + assertTableOperationCounter(t, metrics, "test", "table_activity", "insert") + assertTableOperationCounter(t, metrics, "test", "table_activity", "update") + assertTableOperationCounter(t, metrics, "test", "table_activity", "delete") + }) + } +} + +func startPerfTableIOContainer(t *testing.T, ctx context.Context, image string) string { + t.Helper() + + container, err := tcmysql.Run(ctx, image, + tcmysql.WithDatabase("test"), + tcmysql.WithUsername("root"), + tcmysql.WithPassword("test"), + testcontainers.WithCmdArgs("--performance-schema=ON"), + ) + if err != nil { + t.Fatalf("starting %s: %v", image, err) + } + t.Cleanup(func() { + _ = container.Terminate(context.Background()) + }) + + dsn, err := container.ConnectionString(ctx) + if err != nil { + t.Fatalf("getting %s connection string: %v", image, err) + } + return dsn +} + +func gatherPerfTableIOWaitMetrics( + t *testing.T, + ctx context.Context, + dsn string, +) []*dto.Metric { + t.Helper() + + registry := prometheus.NewRegistry() + registry.MustRegister(New(ctx, dsn, []Scraper{ScrapePerfTableIOWaits{}}, promslog.NewNopLogger())) + families, err := registry.Gather() + if err != nil { + t.Fatalf("gathering exporter metrics: %v", err) + } + for _, family := range families { + if family.GetName() == "mysql_perf_schema_table_io_waits_total" { + return family.Metric + } + } + + t.Fatal("mysql_perf_schema_table_io_waits_total metric family was not exported") + return nil +} + +func assertTableOperationCounter( + t *testing.T, + metrics []*dto.Metric, + schema, table, operation string, +) { + t.Helper() + + for _, metric := range metrics { + labels := make(map[string]string, len(metric.Label)) + for _, pair := range metric.Label { + labels[pair.GetName()] = pair.GetValue() + } + if labels["schema"] == schema && + labels["name"] == table && + labels["operation"] == operation && + metric.GetCounter().GetValue() > 0 { + return + } + } + + t.Errorf("missing positive table I/O counter for %s.%s operation %s", schema, table, operation) +} From c01b74675d97bd3ea848a14a025561cdbd9910ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:10:12 +0200 Subject: [PATCH 03/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- collector/info_schema_innodb_metrics.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index ff853f75c..942da88f2 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -277,12 +277,13 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch if metricType == "set_member" && value >= 0 { // Preserve the historical unsuffixed gauge and add the corrected // counter so existing and current dashboards both keep working. - ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) - suffix := "_total" if strings.HasSuffix(name, "_total") { - suffix = "" + // Avoid exporting the same fqName as both a gauge and counter. + ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.CounterValue, value) + continue } - ch <- prometheus.MustNewConstMetric(metricDesc(suffix), prometheus.CounterValue, value) + ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) + ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value) continue } if metricType == "set_owner" && value >= 0 { From 6e64b02f9190b68c7d8c4ae8c9bc8b858d2efc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:10:46 +0200 Subject: [PATCH 04/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- collector/perf_schema_table_io_waits_integration_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/collector/perf_schema_table_io_waits_integration_test.go b/collector/perf_schema_table_io_waits_integration_test.go index 20bf015e2..15d2ba640 100644 --- a/collector/perf_schema_table_io_waits_integration_test.go +++ b/collector/perf_schema_table_io_waits_integration_test.go @@ -64,6 +64,10 @@ func TestScrapePerfTableIOWaitsMySQLCompatibility(t *testing.T) { t.Fatalf("scanning test rows: %v", err) } } + if err := rows.Err(); err != nil { + _ = rows.Close() + t.Fatalf("iterating test rows: %v", err) + } if err := rows.Close(); err != nil { t.Fatalf("closing test rows: %v", err) } From b220677b712b30908e3da06d3780f671b2efc8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 24 Jul 2026 16:25:18 +0200 Subject: [PATCH 05/11] PMM-15195 Improvements. --- CHANGELOG.md | 2 +- collector/info_schema_innodb_metrics.go | 4 ++ collector/innodb_redo.go | 7 +++ collector/innodb_redo_test.go | 60 +++++++++++++------------ 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c148bb3e9..9378899b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Changes: * [CHANGE] * [FEATURE] -* [ENHANCEMENT] Add stable InnoDB redo metrics and normalize internal metric types for MySQL 9.7 compatibility. +* [ENHANCEMENT] Add stable, version-independent InnoDB redo log metrics (`mysql_innodb_redo_log_current_lsn`, `checkpoint_lsn`, `checkpoint_age_bytes`, `capacity_bytes`, `written_bytes_total`, `checkpoint_age_ratio`) with backward-compatible `mysql_global_status_innodb_*` aliases for MySQL 8.4/9.7, plus stable `mysql_innodb_metrics_*` aliases and corrected metric types for selected `INNODB_METRICS` counters and gauges. * [BUGFIX] ## 0.17.2 / 2025-02-25 diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index 942da88f2..d8255ea95 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -77,6 +77,10 @@ type stableInnodbMetric struct { // counters, which changes whether the generic collector appends "_total". // Keep the generic metrics for compatibility and emit these stable aliases in // addition. +// +// Every entry maps to a monotonically increasing counter, so the aliases are +// always exported as CounterValue regardless of the TYPE MySQL reports for the +// source row. Only add names here that are genuinely cumulative counters. var stableInnodbMetrics = map[string]stableInnodbMetric{ "buffer_flush_neighbor": { name: "buffer_flush_neighbor_batches_total", diff --git a/collector/innodb_redo.go b/collector/innodb_redo.go index d1ed397d1..039d019ff 100644 --- a/collector/innodb_redo.go +++ b/collector/innodb_redo.go @@ -140,12 +140,19 @@ func (s *innodbRedoStatus) collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(innodbCheckpointAgeCompatDesc, prometheus.GaugeValue, s.checkpointAge) } if s.hasCapacity && s.capacity > 0 { + // Ratio is relative to the total redo log capacity. On Oracle + // MySQL capacity comes from innodb_redo_log_capacity_resized, + // whereas legacy Percona innodb_checkpoint_max_age is the sync + // flush threshold (a fraction of capacity), so the ratio scale + // may differ slightly between the two sources. ch <- prometheus.MustNewConstMetric(innodbRedoUsageDesc, prometheus.GaugeValue, s.checkpointAge/s.capacity) } } if s.hasCapacity { ch <- prometheus.MustNewConstMetric(innodbRedoCapacityDesc, prometheus.GaugeValue, s.capacity) if !s.hasLegacyMaxAge { + // Best-effort compatibility value: total redo capacity is used in + // place of the historical sync flush threshold. ch <- prometheus.MustNewConstMetric(innodbCheckpointMaxAgeCompatDesc, prometheus.GaugeValue, s.capacity) } } diff --git a/collector/innodb_redo_test.go b/collector/innodb_redo_test.go index df921f96f..55b37bf19 100644 --- a/collector/innodb_redo_test.go +++ b/collector/innodb_redo_test.go @@ -33,42 +33,46 @@ func TestInnodbRedoStatusMySQL97(t *testing.T) { close(ch) type expectedMetric struct { - name string value float64 metricType dto.MetricType } - expected := []expectedMetric{ - {"mysql_innodb_redo_log_current_lsn", 1500, dto.MetricType_GAUGE}, - {"mysql_global_status_innodb_lsn_current", 1500, dto.MetricType_GAUGE}, - {"mysql_innodb_redo_log_checkpoint_lsn", 1000, dto.MetricType_GAUGE}, - {"mysql_global_status_innodb_lsn_last_checkpoint", 1000, dto.MetricType_GAUGE}, - {"mysql_innodb_redo_log_checkpoint_age_bytes", 500, dto.MetricType_GAUGE}, - {"mysql_global_status_innodb_checkpoint_age", 500, dto.MetricType_GAUGE}, - {"mysql_innodb_redo_log_checkpoint_age_ratio", 0.25, dto.MetricType_GAUGE}, - {"mysql_innodb_redo_log_capacity_bytes", 2000, dto.MetricType_GAUGE}, - {"mysql_global_status_innodb_checkpoint_max_age", 2000, dto.MetricType_GAUGE}, - {"mysql_innodb_redo_log_written_bytes_total", 4096, dto.MetricType_COUNTER}, + expected := map[string]expectedMetric{ + "mysql_innodb_redo_log_current_lsn": {1500, dto.MetricType_GAUGE}, + "mysql_global_status_innodb_lsn_current": {1500, dto.MetricType_GAUGE}, + "mysql_innodb_redo_log_checkpoint_lsn": {1000, dto.MetricType_GAUGE}, + "mysql_global_status_innodb_lsn_last_checkpoint": {1000, dto.MetricType_GAUGE}, + "mysql_innodb_redo_log_checkpoint_age_bytes": {500, dto.MetricType_GAUGE}, + "mysql_global_status_innodb_checkpoint_age": {500, dto.MetricType_GAUGE}, + "mysql_innodb_redo_log_checkpoint_age_ratio": {0.25, dto.MetricType_GAUGE}, + "mysql_innodb_redo_log_capacity_bytes": {2000, dto.MetricType_GAUGE}, + "mysql_global_status_innodb_checkpoint_max_age": {2000, dto.MetricType_GAUGE}, + "mysql_innodb_redo_log_written_bytes_total": {4096, dto.MetricType_COUNTER}, } - i := 0 + found := make(map[string]bool, len(expected)) for metric := range ch { - if i >= len(expected) { - t.Fatal("collected more redo metrics than expected") - } - if !strings.Contains(metric.Desc().String(), `fqName: "`+expected[i].name+`"`) { - t.Errorf("metric %d descriptor %q does not contain expected name %q", i, metric.Desc(), expected[i].name) - } - got := readMetric(metric) - if got.value != expected[i].value { - t.Errorf("metric %s value = %v, want %v", expected[i].name, got.value, expected[i].value) - } - if got.metricType != expected[i].metricType { - t.Errorf("metric %s type = %v, want %v", expected[i].name, got.metricType, expected[i].metricType) + desc := metric.Desc().String() + for name, want := range expected { + if !strings.Contains(desc, `fqName: "`+name+`"`) { + continue + } + if found[name] { + t.Errorf("metric %s was collected more than once", name) + } + got := readMetric(metric) + if got.value != want.value { + t.Errorf("metric %s value = %v, want %v", name, got.value, want.value) + } + if got.metricType != want.metricType { + t.Errorf("metric %s type = %v, want %v", name, got.metricType, want.metricType) + } + found[name] = true } - i++ } - if i != len(expected) { - t.Fatalf("collected %d redo metrics, want %d", i, len(expected)) + for name := range expected { + if !found[name] { + t.Errorf("redo metric %s was not collected", name) + } } } From 582c59213de85dca2590170702a4ac04370aaa86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 24 Jul 2026 16:27:16 +0200 Subject: [PATCH 06/11] PMM-15195 Copilot finding. --- collector/info_schema_innodb_metrics.go | 10 +++++- collector/info_schema_innodb_metrics_test.go | 37 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index d8255ea95..33c7227aa 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -266,10 +266,18 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch } if _, ok := innodbMetricGaugeOverrides[subsystem+"/"+name]; ok { + isCounter := metricType == "counter" || metricType == "status_counter" + // Some MySQL versions report these overridden metrics as counters + // and can emit the -1 sentinel due to an upstream bug + // (http://bugs.mysql.com/bug.php?id=75966). A negative value is not + // a valid sample for either the gauge or the counter, so skip it. + if isCounter && value < 0 { + continue + } ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) // Preserve the historical counter name for users that already query // it while also exposing the correctly typed gauge above. - if (metricType == "counter" || metricType == "status_counter") && value >= 0 { + if isCounter { ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value) } continue diff --git a/collector/info_schema_innodb_metrics_test.go b/collector/info_schema_innodb_metrics_test.go index bc2a1aac7..da29e215b 100644 --- a/collector/info_schema_innodb_metrics_test.go +++ b/collector/info_schema_innodb_metrics_test.go @@ -164,3 +164,40 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { t.Errorf("there were unfulfilled expectations: %s", err) } } + +func TestScrapeInnodbMetricsGaugeOverrideSkipsNegative(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("error opening a stub database connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + + mock.ExpectQuery(sanitizeQuery(infoSchemaInnodbMetricsEnabledColumnQuery)). + WillReturnRows(sqlmock.NewRows([]string{"COLUMN_NAME"}).AddRow("STATUS")) + + // The -1 sentinel is reported by some MySQL versions due to an upstream + // INNODB_METRICS bug and must not be exported for gauge-override metrics. + rows := sqlmock.NewRows([]string{"name", "subsystem", "type", "comment", "count"}). + AddRow("log_lsn_checkpoint_age", "log", "counter", "Checkpoint age", -1) + query := fmt.Sprintf(infoSchemaInnodbMetricsQuery, "status", "enabled") + mock.ExpectQuery(sanitizeQuery(query)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + if scrapeErr := (ScrapeInnodbMetrics{}).Scrape(t.Context(), inst, ch, promslog.NewNopLogger()); scrapeErr != nil { + t.Errorf("error calling function on test: %s", scrapeErr) + } + }() + + for metric := range ch { + if strings.Contains(metric.Desc().String(), "log_lsn_checkpoint_age") { + t.Errorf("negative override metric should be skipped, got: %s", metric.Desc()) + } + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled expectations: %s", err) + } +} From 9c8578929fab6ea17032505e5e6a59c5c70887b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 24 Jul 2026 19:49:30 +0200 Subject: [PATCH 07/11] PMM-15195 Derive redo capacity and checkpoint age ratio only from redo. --- collector/innodb_redo.go | 19 +++++----------- collector/innodb_redo_test.go | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/collector/innodb_redo.go b/collector/innodb_redo.go index 039d019ff..26d9c9149 100644 --- a/collector/innodb_redo.go +++ b/collector/innodb_redo.go @@ -74,7 +74,7 @@ type innodbRedoStatus struct { hasLegacyCurrent bool hasLegacyCheckpoint bool hasLegacyCheckpointAge bool - hasLegacyMaxAge bool + hasCheckpointMaxAge bool } func (s *innodbRedoStatus) observe(key string, value float64) { @@ -102,14 +102,12 @@ func (s *innodbRedoStatus) observe(key string, value float64) { s.hasCheckpointAge = true s.hasLegacyCheckpointAge = true case "innodb_checkpoint_max_age": + // This is a checkpoint flush threshold, not the total redo capacity. + // The original metric is emitted by the global status collector. + s.hasCheckpointMaxAge = true + case "innodb_redo_log_capacity_resized": s.capacity = value s.hasCapacity = true - s.hasLegacyMaxAge = true - case "innodb_redo_log_capacity_resized": - if !s.hasCapacity { - s.capacity = value - s.hasCapacity = true - } case "innodb_os_log_written": s.written = value s.hasWritten = true @@ -140,17 +138,12 @@ func (s *innodbRedoStatus) collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(innodbCheckpointAgeCompatDesc, prometheus.GaugeValue, s.checkpointAge) } if s.hasCapacity && s.capacity > 0 { - // Ratio is relative to the total redo log capacity. On Oracle - // MySQL capacity comes from innodb_redo_log_capacity_resized, - // whereas legacy Percona innodb_checkpoint_max_age is the sync - // flush threshold (a fraction of capacity), so the ratio scale - // may differ slightly between the two sources. ch <- prometheus.MustNewConstMetric(innodbRedoUsageDesc, prometheus.GaugeValue, s.checkpointAge/s.capacity) } } if s.hasCapacity { ch <- prometheus.MustNewConstMetric(innodbRedoCapacityDesc, prometheus.GaugeValue, s.capacity) - if !s.hasLegacyMaxAge { + if !s.hasCheckpointMaxAge { // Best-effort compatibility value: total redo capacity is used in // place of the historical sync flush threshold. ch <- prometheus.MustNewConstMetric(innodbCheckpointMaxAgeCompatDesc, prometheus.GaugeValue, s.capacity) diff --git a/collector/innodb_redo_test.go b/collector/innodb_redo_test.go index 55b37bf19..853b9a83f 100644 --- a/collector/innodb_redo_test.go +++ b/collector/innodb_redo_test.go @@ -92,5 +92,47 @@ func TestInnodbRedoStatusDoesNotDuplicatePerconaMetrics(t *testing.T) { if strings.Contains(desc, "mysql_global_status_innodb_") { t.Errorf("unexpected compatibility metric for Percona status variables: %s", desc) } + if strings.Contains(desc, "mysql_innodb_redo_log_capacity_bytes") || + strings.Contains(desc, "mysql_innodb_redo_log_checkpoint_age_ratio") { + t.Errorf("checkpoint max age must not be exported as redo capacity: %s", desc) + } + } +} + +func TestInnodbRedoStatusKeepsCapacitySeparateFromCheckpointMaxAge(t *testing.T) { + status := innodbRedoStatus{} + status.observe("innodb_redo_log_current_lsn", 1500) + status.observe("innodb_redo_log_checkpoint_lsn", 1000) + status.observe("innodb_checkpoint_max_age", 1600) + status.observe("innodb_redo_log_capacity_resized", 2000) + + ch := make(chan prometheus.Metric, 10) + status.collect(ch) + close(ch) + + expected := map[string]float64{ + "mysql_innodb_redo_log_capacity_bytes": 2000, + "mysql_innodb_redo_log_checkpoint_age_ratio": 0.25, + } + found := make(map[string]bool, len(expected)) + for metric := range ch { + desc := metric.Desc().String() + if strings.Contains(desc, "mysql_global_status_innodb_checkpoint_max_age") { + t.Errorf("unexpected compatibility metric when checkpoint max age is present: %s", desc) + } + for name, want := range expected { + if !strings.Contains(desc, `fqName: "`+name+`"`) { + continue + } + if got := readMetric(metric).value; got != want { + t.Errorf("metric %s value = %v, want %v", name, got, want) + } + found[name] = true + } + } + for name := range expected { + if !found[name] { + t.Errorf("redo metric %s was not collected", name) + } } } From e35262ab13b15b94433251b408c9be1acc1edb05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 24 Jul 2026 20:11:24 +0200 Subject: [PATCH 08/11] PMM-15195 Export only known cumulative InnoDB set members as counters. --- collector/info_schema_innodb_metrics.go | 7 +++---- collector/info_schema_innodb_metrics_test.go | 7 +++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index 33c7227aa..6447309b6 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -283,10 +283,9 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch continue } - // MySQL returns counters as counter/status_counter and set aggregates as - // set_member/set_owner. A set_member needs the normal counter suffix, - // while set_owner names already carry their aggregate suffix. - if metricType == "set_member" && value >= 0 { + // Only known cumulative set members can safely be exposed as counters. + // Other set members include derived values that may decrease. + if _, ok := stableInnodbMetrics[name]; ok && metricType == "set_member" && value >= 0 { // Preserve the historical unsuffixed gauge and add the corrected // counter so existing and current dashboards both keep working. if strings.HasSuffix(name, "_total") { diff --git a/collector/info_schema_innodb_metrics_test.go b/collector/info_schema_innodb_metrics_test.go index da29e215b..f8e8411a1 100644 --- a/collector/info_schema_innodb_metrics_test.go +++ b/collector/info_schema_innodb_metrics_test.go @@ -97,6 +97,7 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { rows := sqlmock.NewRows([]string{"name", "subsystem", "type", "comment", "count"}). AddRow("buffer_flush_neighbor", "buffer", "set_member", "Neighbor flush batches", 2). AddRow("buffer_flush_neighbor_total_pages", "buffer", "set_owner", "Neighbor flush pages", 8). + AddRow("buffer_flush_batch_scanned_per_call", "buffer", "set_member", "Pages scanned per flush batch", 4). AddRow("trx_rw_commits", "transaction", "status_counter", "Read-write commits", 3). AddRow("log_lsn_checkpoint_age", "log", "counter", "Checkpoint age", 12) query := fmt.Sprintf(infoSchemaInnodbMetricsQuery, "status", "enabled") @@ -130,6 +131,9 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { "mysql_info_schema_innodb_metrics_buffer_buffer_flush_neighbor_total_pages": { value: 8, metricType: dto.MetricType_COUNTER, }, + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_scanned_per_call": { + value: 4, metricType: dto.MetricType_GAUGE, + }, "mysql_info_schema_innodb_metrics_log_log_lsn_checkpoint_age": { value: 12, metricType: dto.MetricType_GAUGE, }, @@ -140,6 +144,9 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { found := make(map[string]bool, len(expected)) for metric := range ch { desc := metric.Desc().String() + if strings.Contains(desc, `fqName: "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_scanned_per_call_total"`) { + t.Error("derived set member must not be exported as a counter") + } for name, want := range expected { if !strings.Contains(desc, `fqName: "`+name+`"`) { continue From cc79abbdf6203ed403dfab9e21a16ae4894b6d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Sun, 26 Jul 2026 09:08:06 +0200 Subject: [PATCH 09/11] PMM-15195 Scope INNODB_METRICS counter typing to known cumulative rows. --- collector/info_schema_innodb_metrics.go | 45 +++++++++++--------- collector/info_schema_innodb_metrics_test.go | 29 +++++++++++-- collector/innodb_redo_test.go | 34 ++++++++------- 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/collector/info_schema_innodb_metrics.go b/collector/info_schema_innodb_metrics.go index 6447309b6..09d2b56fb 100644 --- a/collector/info_schema_innodb_metrics.go +++ b/collector/info_schema_innodb_metrics.go @@ -21,7 +21,6 @@ import ( "fmt" "log/slog" "regexp" - "strings" "github.com/prometheus/client_golang/prometheus" ) @@ -81,6 +80,10 @@ type stableInnodbMetric struct { // Every entry maps to a monotonically increasing counter, so the aliases are // always exported as CounterValue regardless of the TYPE MySQL reports for the // source row. Only add names here that are genuinely cumulative counters. +// +// Entries also make the generic collector emit a "_total" counter next to +// the historical gauge, so do not add a name whose subsystem already contains a +// row literally called "_total": the two would collide on one fqName. var stableInnodbMetrics = map[string]stableInnodbMetric{ "buffer_flush_neighbor": { name: "buffer_flush_neighbor_batches_total", @@ -135,12 +138,20 @@ var stableInnodbMetrics = map[string]stableInnodbMetric{ // These values are positions or sizes rather than monotonically increasing // event counters. Some MySQL versions report them as counters, but dashboards // need their unsuffixed gauge names for max_over_time and direct arithmetic. +// MySQL reports these rows under the "log" subsystem, older versions under +// "recovery". Both spellings are queried by the dashboards, so both need the +// override. var innodbMetricGaugeOverrides = map[string]struct{}{ - "log/log_lsn_checkpoint_age": {}, - "log/log_lsn_current": {}, - "log/log_lsn_last_checkpoint": {}, - "log/log_lsn_last_flush": {}, - "log/log_max_modified_age_async": {}, + "log/log_lsn_checkpoint_age": {}, + "log/log_lsn_current": {}, + "log/log_lsn_last_checkpoint": {}, + "log/log_lsn_last_flush": {}, + "log/log_max_modified_age_async": {}, + "recovery/log_lsn_checkpoint_age": {}, + "recovery/log_lsn_current": {}, + "recovery/log_lsn_last_checkpoint": {}, + "recovery/log_lsn_last_flush": {}, + "recovery/log_max_modified_age_async": {}, } // Regexp for matching metric aggregations. @@ -283,24 +294,18 @@ func (ScrapeInnodbMetrics) Scrape(ctx context.Context, instance *instance, ch ch continue } - // Only known cumulative set members can safely be exposed as counters. - // Other set members include derived values that may decrease. - if _, ok := stableInnodbMetrics[name]; ok && metricType == "set_member" && value >= 0 { - // Preserve the historical unsuffixed gauge and add the corrected - // counter so existing and current dashboards both keep working. - if strings.HasSuffix(name, "_total") { - // Avoid exporting the same fqName as both a gauge and counter. - ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.CounterValue, value) - continue - } + // Only the known cumulative rows listed in stableInnodbMetrics can + // safely be exposed as counters. Other set rows hold derived values + // such as averages or per-call figures that may decrease. + isSetRow := metricType == "set_member" || metricType == "set_owner" + if _, ok := stableInnodbMetrics[name]; ok && isSetRow && value >= 0 { + // Both row types were historically exported as an unsuffixed + // gauge. Preserve that name and add the correctly typed counter so + // existing and current dashboards both keep working. ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.GaugeValue, value) ch <- prometheus.MustNewConstMetric(metricDesc("_total"), prometheus.CounterValue, value) continue } - if metricType == "set_owner" && value >= 0 { - ch <- prometheus.MustNewConstMetric(metricDesc(""), prometheus.CounterValue, value) - continue - } // MySQL returns counters named two different ways. "counter" and "status_counter". // value >= 0 is necessary due to upstream bugs: http://bugs.mysql.com/bug.php?id=75966 diff --git a/collector/info_schema_innodb_metrics_test.go b/collector/info_schema_innodb_metrics_test.go index f8e8411a1..569d74b65 100644 --- a/collector/info_schema_innodb_metrics_test.go +++ b/collector/info_schema_innodb_metrics_test.go @@ -98,8 +98,10 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { AddRow("buffer_flush_neighbor", "buffer", "set_member", "Neighbor flush batches", 2). AddRow("buffer_flush_neighbor_total_pages", "buffer", "set_owner", "Neighbor flush pages", 8). AddRow("buffer_flush_batch_scanned_per_call", "buffer", "set_member", "Pages scanned per flush batch", 4). + AddRow("buffer_flush_batch_total_pages", "buffer", "set_owner", "Pages flushed by batches", 6). AddRow("trx_rw_commits", "transaction", "status_counter", "Read-write commits", 3). - AddRow("log_lsn_checkpoint_age", "log", "counter", "Checkpoint age", 12) + AddRow("log_lsn_checkpoint_age", "log", "counter", "Checkpoint age", 12). + AddRow("log_lsn_checkpoint_age", "recovery", "counter", "Checkpoint age", 14) query := fmt.Sprintf(infoSchemaInnodbMetricsQuery, "status", "enabled") mock.ExpectQuery(sanitizeQuery(query)).WillReturnRows(rows) @@ -129,23 +131,44 @@ func TestScrapeInnodbMetricsStableAliases(t *testing.T) { value: 2, metricType: dto.MetricType_COUNTER, }, "mysql_info_schema_innodb_metrics_buffer_buffer_flush_neighbor_total_pages": { + value: 8, metricType: dto.MetricType_GAUGE, + }, + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_neighbor_total_pages_total": { value: 8, metricType: dto.MetricType_COUNTER, }, "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_scanned_per_call": { value: 4, metricType: dto.MetricType_GAUGE, }, + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_total_pages": { + value: 6, metricType: dto.MetricType_GAUGE, + }, "mysql_info_schema_innodb_metrics_log_log_lsn_checkpoint_age": { value: 12, metricType: dto.MetricType_GAUGE, }, "mysql_info_schema_innodb_metrics_log_log_lsn_checkpoint_age_total": { value: 12, metricType: dto.MetricType_COUNTER, }, + "mysql_info_schema_innodb_metrics_recovery_log_lsn_checkpoint_age": { + value: 14, metricType: dto.MetricType_GAUGE, + }, + "mysql_info_schema_innodb_metrics_recovery_log_lsn_checkpoint_age_total": { + value: 14, metricType: dto.MetricType_COUNTER, + }, + } + unexpected := []string{ + // Derived set members must not be exported as counters. + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_scanned_per_call_total", + // Set owners outside stableInnodbMetrics must keep their historical + // gauge type instead of silently turning into counters. + "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_total_pages_total", } found := make(map[string]bool, len(expected)) for metric := range ch { desc := metric.Desc().String() - if strings.Contains(desc, `fqName: "mysql_info_schema_innodb_metrics_buffer_buffer_flush_batch_scanned_per_call_total"`) { - t.Error("derived set member must not be exported as a counter") + for _, name := range unexpected { + if strings.Contains(desc, `fqName: "`+name+`"`) { + t.Errorf("metric %s must not be exported", name) + } } for name, want := range expected { if !strings.Contains(desc, `fqName: "`+name+`"`) { diff --git a/collector/innodb_redo_test.go b/collector/innodb_redo_test.go index 853b9a83f..14f48cc59 100644 --- a/collector/innodb_redo_test.go +++ b/collector/innodb_redo_test.go @@ -21,6 +21,22 @@ import ( dto "github.com/prometheus/client_model/go" ) +// collectRedoStatus drains collect() through an unbuffered channel so that the +// tests cannot deadlock when the number of emitted metrics changes. +func collectRedoStatus(status *innodbRedoStatus) []prometheus.Metric { + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + status.collect(ch) + }() + + var metrics []prometheus.Metric + for metric := range ch { + metrics = append(metrics, metric) + } + return metrics +} + func TestInnodbRedoStatusMySQL97(t *testing.T) { status := innodbRedoStatus{} status.observe("innodb_redo_log_current_lsn", 1500) @@ -28,10 +44,6 @@ func TestInnodbRedoStatusMySQL97(t *testing.T) { status.observe("innodb_redo_log_capacity_resized", 2000) status.observe("innodb_os_log_written", 4096) - ch := make(chan prometheus.Metric, 10) - status.collect(ch) - close(ch) - type expectedMetric struct { value float64 metricType dto.MetricType @@ -50,7 +62,7 @@ func TestInnodbRedoStatusMySQL97(t *testing.T) { } found := make(map[string]bool, len(expected)) - for metric := range ch { + for _, metric := range collectRedoStatus(&status) { desc := metric.Desc().String() for name, want := range expected { if !strings.Contains(desc, `fqName: "`+name+`"`) { @@ -83,11 +95,7 @@ func TestInnodbRedoStatusDoesNotDuplicatePerconaMetrics(t *testing.T) { status.observe("innodb_checkpoint_age", 500) status.observe("innodb_checkpoint_max_age", 2000) - ch := make(chan prometheus.Metric, 5) - status.collect(ch) - close(ch) - - for metric := range ch { + for _, metric := range collectRedoStatus(&status) { desc := metric.Desc().String() if strings.Contains(desc, "mysql_global_status_innodb_") { t.Errorf("unexpected compatibility metric for Percona status variables: %s", desc) @@ -106,16 +114,12 @@ func TestInnodbRedoStatusKeepsCapacitySeparateFromCheckpointMaxAge(t *testing.T) status.observe("innodb_checkpoint_max_age", 1600) status.observe("innodb_redo_log_capacity_resized", 2000) - ch := make(chan prometheus.Metric, 10) - status.collect(ch) - close(ch) - expected := map[string]float64{ "mysql_innodb_redo_log_capacity_bytes": 2000, "mysql_innodb_redo_log_checkpoint_age_ratio": 0.25, } found := make(map[string]bool, len(expected)) - for metric := range ch { + for _, metric := range collectRedoStatus(&status) { desc := metric.Desc().String() if strings.Contains(desc, "mysql_global_status_innodb_checkpoint_max_age") { t.Errorf("unexpected compatibility metric when checkpoint max age is present: %s", desc) From 655a0f855159841bcf90dcc0703c78aa145f6f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 29 Jul 2026 17:46:05 +0200 Subject: [PATCH 10/11] PMM-15195 Let the integration tests run on a constrained Docker host. --- ...na_info_schema_process_list_integration_test.go | 14 +++++++++++--- .../perf_schema_table_io_waits_integration_test.go | 14 +++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/collector/percona_info_schema_process_list_integration_test.go b/collector/percona_info_schema_process_list_integration_test.go index 10d0f03cb..4690d2c55 100644 --- a/collector/percona_info_schema_process_list_integration_test.go +++ b/collector/percona_info_schema_process_list_integration_test.go @@ -106,7 +106,15 @@ func startContainerForCase(t *testing.T, ctx context.Context, image string, psEn if psEnabled { psFlag = "ON" } - cmdArg := fmt.Sprintf("--performance-schema=%s", psFlag) + // Ten of these containers come up in parallel and each server would otherwise + // size its buffer pool from total host memory. Pin it small so the suite fits + // on a developer machine and does not thrash a CI runner; the processlist + // source selection under test does not depend on it. innodb_buffer_pool_size + // is the one knob MySQL 5.7, 8.x, 9.x and MariaDB all accept. + cmdArgs := []string{ + fmt.Sprintf("--performance-schema=%s", psFlag), + "--innodb-buffer-pool-size=64M", + } switch { case strings.HasPrefix(image, "mariadb:"): @@ -114,7 +122,7 @@ func startContainerForCase(t *testing.T, ctx context.Context, image string, psEn tcmariadb.WithDatabase("test"), tcmariadb.WithUsername("test"), tcmariadb.WithPassword("test"), - testcontainers.WithCmdArgs(cmdArg), + testcontainers.WithCmdArgs(cmdArgs...), ) if err != nil { t.Fatalf("starting %s: %v", image, err) @@ -133,7 +141,7 @@ func startContainerForCase(t *testing.T, ctx context.Context, image string, psEn tcmysql.WithDatabase("test"), tcmysql.WithUsername("test"), tcmysql.WithPassword("test"), - testcontainers.WithCmdArgs(cmdArg), + testcontainers.WithCmdArgs(cmdArgs...), ) if err != nil { t.Fatalf("starting %s: %v", image, err) diff --git a/collector/perf_schema_table_io_waits_integration_test.go b/collector/perf_schema_table_io_waits_integration_test.go index 15d2ba640..3ae8ad2b3 100644 --- a/collector/perf_schema_table_io_waits_integration_test.go +++ b/collector/perf_schema_table_io_waits_integration_test.go @@ -27,6 +27,10 @@ import ( ) func TestScrapePerfTableIOWaitsMySQLCompatibility(t *testing.T) { + if testing.Short() { + t.Skip("skipping testcontainers integration test in -short mode") + } + cases := []struct { name string image string @@ -94,7 +98,15 @@ func startPerfTableIOContainer(t *testing.T, ctx context.Context, image string) tcmysql.WithDatabase("test"), tcmysql.WithUsername("root"), tcmysql.WithPassword("test"), - testcontainers.WithCmdArgs("--performance-schema=ON"), + // MySQL 9.7 autosizes InnoDB from host memory and gets OOM killed on a + // Docker VM that is already busy, where 8.0 still fits. The counters + // under test do not depend on either size, so pin both low enough that + // the test runs on a developer machine as well as in CI. + testcontainers.WithCmdArgs( + "--performance-schema=ON", + "--innodb-buffer-pool-size=64M", + "--innodb-redo-log-capacity=16M", + ), ) if err != nil { t.Fatalf("starting %s: %v", image, err) From bad6427964fac2205d2f7c4d73bac9cf9d109792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 29 Jul 2026 17:46:05 +0200 Subject: [PATCH 11/11] PMM-15195 Build and run the integration tests in CI. --- .github/workflows/go.yml | 37 +++++++++++++++++++++++++++++++++++++ Makefile | 15 ++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a7ef18381..dd367aaa7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -63,3 +63,40 @@ jobs: git status docker version docker compose version + + integration: + name: Integration + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: ${{ github.workspace }}/go.mod + + # Kept out of the matrix above on purpose: the integration tests pin the + # server images they need and start them through testcontainers, so running + # them per matrix entry would start the same containers fourteen times over. + + # Nothing built these files before, so a compile error in an integration + # test could not fail the build. + - name: Compile-check integration tests + run: make test-integration-build + + # Only the table I/O waits test runs for now. TestPScrapeProcesslist brings + # up ten database containers in parallel, each server sizing InnoDB from + # total host memory, and that has not been validated on a hosted runner. + # Widen this to `make test-integration` once it has. + - name: Run integration tests + run: go test -count 1 -race -tags integration -run TestScrapePerfTableIOWaits ./collector/ + + - name: Run debug commands on failure + if: ${{ failure() }} + run: | + env | sort + go env | sort + docker version + free -m diff --git a/Makefile b/Makefile index 465f57f5d..cd884d70e 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,19 @@ test: ## Run all tests @echo ">> running tests" @$(GO) test -count 1 -race $(pkgs) +# Tests behind the integration build tag start their own database containers via +# testcontainers and pin the images they need, so they ignore MYSQL_IMAGE and must +# not be run once per entry of the CI matrix. +test-integration-build: ## Compile-check the tests behind the integration build tag + @echo ">> compile-checking integration tests" + @$(GO) vet -tags integration $(pkgs) + +# Needs a Docker host with room for ten parallel database containers. CI runs a +# subset instead, see the integration job in .github/workflows/go.yml. +test-integration: ## Run all tests, including those behind the integration build tag + @echo ">> running integration tests" + @$(GO) test -count 1 -race -tags integration $(pkgs) + FILES = $(shell find . -type f -name '*.go') format: ## Format the code @@ -111,4 +124,4 @@ release: ## Build release binary # docker exec -t --user root pmm-server chown pmm:pmm /usr/local/percona/pmm/exporters/mysqld_exporter # docker exec -t pmm-server supervisorctl start pmm-agent -.PHONY: all init style format build test vet tarball docker env-up env-down help default +.PHONY: all init style format build test test-integration test-integration-build vet tarball docker env-up env-down help default