From 6144e69b9a1eeb9aa8ab653168ddd5b7e7752cb2 Mon Sep 17 00:00:00 2001 From: Daniele Rolando Date: Tue, 2 Sep 2025 15:23:47 -0700 Subject: [PATCH 1/9] Emit metric tracking empty responses from prometheus Signed-off-by: Daniele Rolando --- CHANGELOG.md | 1 + pkg/metricscollector/metricscollectors.go | 10 ++++++++++ pkg/metricscollector/opentelemetry.go | 11 +++++++++++ pkg/metricscollector/prommetrics.go | 14 ++++++++++++++ pkg/scalers/prometheus_scaler.go | 3 +++ 5 files changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49206cb4a7d..d522d74fe7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) - **Elasticsearch Scaler**: Add HTTP status check for Elasticsearch errors ([#7480](https://github.com/kedacore/keda/pull/7480)) - **Kubernetes Workload Scaler**: Add `groupByNode` parameter ([#7628](https://github.com/kedacore/keda/issues/7628)) +- **Prometheus Scaler**: Emit metric tracking empty responses from prometheus ([#7060](https://github.com/kedacore/keda/pull/7060)) ### Fixes diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 7a6d1055ffa..55d5332204d 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -79,6 +79,9 @@ type MetricsCollector interface { // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting RecordCloudEventQueueStatus(namespace string, value int) + + // RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result + RecordEmptyPrometheusMetricError() } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -205,6 +208,13 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } } +// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result +func RecordEmptyPrometheusMetricError() { + for _, element := range collectors { + element.RecordEmptyPrometheusMetricError() + } +} + // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts // Currently, only Prometheus metrics are supported. func GetServerMetrics() *grpcprom.ServerMetrics { diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 14e2aa32f47..560a5dbd2ba 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -34,6 +34,7 @@ var ( otCrdTotalsCounterDeprecated api.Int64UpDownCounter otTriggerRegisteredTotalsCounter api.Int64UpDownCounter otCrdRegisteredTotalsCounter api.Int64UpDownCounter + otEmptyPrometheusMetricError api.Int64Counter otelScalerMetricVals []OtelMetricFloat64Val otelScalerMetricsLatencyVals []OtelMetricFloat64Val @@ -135,6 +136,11 @@ func initMeters() { otLog.Error(err, msg) } + otEmptyPrometheusMetricError, err = meter.Int64Counter("keda.prometheus.metrics.empty.error", api.WithDescription("Number of times a prometheus query returns an empty result")) + if err != nil { + otLog.Error(err, msg) + } + _, err = meter.Float64ObservableGauge( "keda.scaler.metrics.value", api.WithDescription("The current value for each scaler's metric that would be used by the HPA in computing the target average"), @@ -506,3 +512,8 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { otCloudEventQueueStatus.measurementOption = opt otCloudEventQueueStatusVals = append(otCloudEventQueueStatusVals, otCloudEventQueueStatus) } + +// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result +func (o *OtelMetrics) RecordEmptyPrometheusMetricError() { + otEmptyPrometheusMetricError.Add(context.Background(), 1, nil) +} diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 92b60c17f26..ee22c8cab60 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -104,6 +104,14 @@ var ( }, []string{"namespace", "scaledJob"}, ) + emptyPrometheusMetricError = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "prometheus", + Name: "metrics_empty_error_total", + Help: "Number of times a prometheus query returns an empty result", + }, + ) triggerRegistered = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, @@ -168,6 +176,7 @@ func NewPromMetrics() *PromMetrics { metrics.Registry.MustRegister(triggerRegistered) metrics.Registry.MustRegister(crdRegistered) metrics.Registry.MustRegister(scaledJobErrors) + metrics.Registry.MustRegister(emptyPrometheusMetricError) metrics.Registry.MustRegister(buildInfo) @@ -328,6 +337,11 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value)) } +// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result +func (p *PromMetrics) RecordEmptyPrometheusMetricError() { + emptyPrometheusMetricError.Inc() +} + // Returns a grpcprom server Metrics object and registers the metrics. The object contains // interceptors to chain to the server so that all requests served are observed. Intended to be called // as part of initialization of metricscollector, hence why this function is not exported diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 033dc2f4f24..5c66ac36de3 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -17,6 +17,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scalers/aws" "github.com/kedacore/keda/v2/pkg/scalers/azure" @@ -258,6 +259,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } + metricscollector.RecordEmptyPrometheusMetricError() return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the result is empty") } else if len(result.Data.Result) > 1 { return -1, fmt.Errorf("prometheus query %s returned multiple elements", s.metadata.Query) @@ -268,6 +270,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } + metricscollector.RecordEmptyPrometheusMetricError() return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the value list is empty") } else if valueLen < 2 { return -1, fmt.Errorf("prometheus query %s didn't return enough values", s.metadata.Query) From 8accfd2187679873c02ec6714f3c6b925fb62791 Mon Sep 17 00:00:00 2001 From: Daniele Rolando Date: Mon, 29 Sep 2025 10:39:22 -0700 Subject: [PATCH 2/9] update comment Signed-off-by: Daniele Rolando --- pkg/metricscollector/metricscollectors.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 55d5332204d..270a083e397 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -208,8 +208,8 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } } -// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result -func RecordEmptyPrometheusMetricError() { +// RecordEmptyPrometheusMetricError counts the number of times a query returns an empty result +func RecordEmptyUpstreamResponse() { for _, element := range collectors { element.RecordEmptyPrometheusMetricError() } From 995d9a9cc3b9b34140960a78853b055202a1742a Mon Sep 17 00:00:00 2001 From: Daniele Rolando Date: Mon, 29 Sep 2025 10:37:35 -0700 Subject: [PATCH 3/9] rename to empty_upstream_responses_total Signed-off-by: Daniele Rolando --- pkg/metricscollector/metricscollectors.go | 6 +++--- pkg/metricscollector/opentelemetry.go | 10 +++++----- pkg/metricscollector/prommetrics.go | 14 +++++++------- pkg/scalers/prometheus_scaler.go | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 270a083e397..5d9c72e3cfb 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -80,8 +80,8 @@ type MetricsCollector interface { // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting RecordCloudEventQueueStatus(namespace string, value int) - // RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result - RecordEmptyPrometheusMetricError() + // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result + RecordEmptyUpstreamResponse() } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -211,7 +211,7 @@ func RecordCloudEventQueueStatus(namespace string, value int) { // RecordEmptyPrometheusMetricError counts the number of times a query returns an empty result func RecordEmptyUpstreamResponse() { for _, element := range collectors { - element.RecordEmptyPrometheusMetricError() + element.RecordEmptyUpstreamResponse() } } diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 560a5dbd2ba..702b387565c 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -34,7 +34,7 @@ var ( otCrdTotalsCounterDeprecated api.Int64UpDownCounter otTriggerRegisteredTotalsCounter api.Int64UpDownCounter otCrdRegisteredTotalsCounter api.Int64UpDownCounter - otEmptyPrometheusMetricError api.Int64Counter + otEmptyUpstreamResponses api.Int64Counter otelScalerMetricVals []OtelMetricFloat64Val otelScalerMetricsLatencyVals []OtelMetricFloat64Val @@ -136,7 +136,7 @@ func initMeters() { otLog.Error(err, msg) } - otEmptyPrometheusMetricError, err = meter.Int64Counter("keda.prometheus.metrics.empty.error", api.WithDescription("Number of times a prometheus query returns an empty result")) + otEmptyUpstreamResponses, err = meter.Int64Counter("keda.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result")) if err != nil { otLog.Error(err, msg) } @@ -513,7 +513,7 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { otCloudEventQueueStatusVals = append(otCloudEventQueueStatusVals, otCloudEventQueueStatus) } -// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result -func (o *OtelMetrics) RecordEmptyPrometheusMetricError() { - otEmptyPrometheusMetricError.Add(context.Background(), 1, nil) +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func (o *OtelMetrics) RecordEmptyUpstreamResponse() { + otEmptyUpstreamResponses.Add(context.Background(), 1, nil) } diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index ee22c8cab60..357e2ec6f98 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -104,12 +104,12 @@ var ( }, []string{"namespace", "scaledJob"}, ) - emptyPrometheusMetricError = prometheus.NewCounter( + emptyUpstreamResponse = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "prometheus", - Name: "metrics_empty_error_total", - Help: "Number of times a prometheus query returns an empty result", + Name: "empty_upstream_responses_total", + Help: "Number of times a query returns an empty result", }, ) triggerRegistered = prometheus.NewGaugeVec( @@ -176,7 +176,7 @@ func NewPromMetrics() *PromMetrics { metrics.Registry.MustRegister(triggerRegistered) metrics.Registry.MustRegister(crdRegistered) metrics.Registry.MustRegister(scaledJobErrors) - metrics.Registry.MustRegister(emptyPrometheusMetricError) + metrics.Registry.MustRegister(emptyUpstreamResponse) metrics.Registry.MustRegister(buildInfo) @@ -337,9 +337,9 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value)) } -// RecordEmptyPrometheusMetricError counts the number of times a prometheus query returns an empty result -func (p *PromMetrics) RecordEmptyPrometheusMetricError() { - emptyPrometheusMetricError.Inc() +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func (p *PromMetrics) RecordEmptyUpstreamResponse() { + emptyUpstreamResponse.Inc() } // Returns a grpcprom server Metrics object and registers the metrics. The object contains diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 5c66ac36de3..52f3a2790ba 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -259,7 +259,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyPrometheusMetricError() + metricscollector.RecordEmptyUpstreamResponse() return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the result is empty") } else if len(result.Data.Result) > 1 { return -1, fmt.Errorf("prometheus query %s returned multiple elements", s.metadata.Query) @@ -270,7 +270,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyPrometheusMetricError() + metricscollector.RecordEmptyUpstreamResponse() return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the value list is empty") } else if valueLen < 2 { return -1, fmt.Errorf("prometheus query %s didn't return enough values", s.metadata.Query) From d80bede51a9811e6d4b45181f74a8aacf0aa232b Mon Sep 17 00:00:00 2001 From: drolando-stripe <102543345+drolando-stripe@users.noreply.github.com> Date: Fri, 3 Oct 2025 12:09:36 -0700 Subject: [PATCH 4/9] Update pkg/metricscollector/prommetrics.go Co-authored-by: Jorge Turrado Ferrero Signed-off-by: drolando-stripe <102543345+drolando-stripe@users.noreply.github.com> --- pkg/metricscollector/prommetrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 357e2ec6f98..29076963a79 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -107,7 +107,7 @@ var ( emptyUpstreamResponse = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, - Subsystem: "prometheus", + Subsystem: "scaler", Name: "empty_upstream_responses_total", Help: "Number of times a query returns an empty result", }, From cc8f1d765267e1f7e9b789d71336a93a907ea85d Mon Sep 17 00:00:00 2001 From: drolando-stripe <102543345+drolando-stripe@users.noreply.github.com> Date: Fri, 3 Oct 2025 12:09:50 -0700 Subject: [PATCH 5/9] Update pkg/metricscollector/opentelemetry.go Co-authored-by: Jorge Turrado Ferrero Signed-off-by: drolando-stripe <102543345+drolando-stripe@users.noreply.github.com> --- pkg/metricscollector/opentelemetry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 702b387565c..dafcaa71936 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -136,7 +136,7 @@ func initMeters() { otLog.Error(err, msg) } - otEmptyUpstreamResponses, err = meter.Int64Counter("keda.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result")) + otEmptyUpstreamResponses, err = meter.Int64Counter("keda.scaler.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result")) if err != nil { otLog.Error(err, msg) } From 728f22e1e3a67589d641ab89717e9af6cd6c59ac Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 20 Apr 2026 03:27:57 +0000 Subject: [PATCH 6/9] Add namespace/scaledObject/triggerName labels to empty upstream response metric Add labels to keda_scaler_empty_upstream_responses_total so operators can identify which scaler is producing empty upstream responses. Also add e2e tests for both Prometheus and OpenTelemetry metric backends. Signed-off-by: Ali Aqel --- pkg/metricscollector/metricscollectors.go | 8 +- pkg/metricscollector/opentelemetry.go | 8 +- pkg/metricscollector/prommetrics.go | 7 +- pkg/scalers/prometheus_scaler.go | 26 ++-- .../opentelemetry_metrics_test.go | 133 ++++++++++++++++- .../prometheus_metrics_test.go | 136 +++++++++++++++++- 6 files changed, 293 insertions(+), 25 deletions(-) diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 5d9c72e3cfb..1b4437b0a44 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -81,7 +81,7 @@ type MetricsCollector interface { RecordCloudEventQueueStatus(namespace string, value int) // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result - RecordEmptyUpstreamResponse() + RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -208,10 +208,10 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } } -// RecordEmptyPrometheusMetricError counts the number of times a query returns an empty result -func RecordEmptyUpstreamResponse() { +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { for _, element := range collectors { - element.RecordEmptyUpstreamResponse() + element.RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName) } } diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index dafcaa71936..9160de4f4fa 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -514,6 +514,10 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { } // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result -func (o *OtelMetrics) RecordEmptyUpstreamResponse() { - otEmptyUpstreamResponses.Add(context.Background(), 1, nil) +func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { + otEmptyUpstreamResponses.Add(context.Background(), 1, api.WithAttributes( + attribute.Key("namespace").String(namespace), + attribute.Key("scaledObject").String(scaledObject), + attribute.Key("triggerName").String(triggerName), + )) } diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 29076963a79..8272fe230a2 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -104,13 +104,14 @@ var ( }, []string{"namespace", "scaledJob"}, ) - emptyUpstreamResponse = prometheus.NewCounter( + emptyUpstreamResponse = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "empty_upstream_responses_total", Help: "Number of times a query returns an empty result", }, + []string{"namespace", "scaledObject", "triggerName"}, ) triggerRegistered = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -338,8 +339,8 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { } // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result -func (p *PromMetrics) RecordEmptyUpstreamResponse() { - emptyUpstreamResponse.Inc() +func (p *PromMetrics) RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { + emptyUpstreamResponse.With(prometheus.Labels{"namespace": namespace, "scaledObject": scaledObject, "triggerName": triggerName}).Inc() } // Returns a grpcprom server Metrics object and registers the metrics. The object contains diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 52f3a2790ba..b38e3944f0e 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -27,10 +27,13 @@ import ( ) type prometheusScaler struct { - metricType v2.MetricTargetType - metadata *prometheusMetadata - httpClient *http.Client - logger logr.Logger + metricType v2.MetricTargetType + metadata *prometheusMetadata + httpClient *http.Client + logger logr.Logger + scalableObjectName string + scalableObjectNS string + triggerName string } // IgnoreNullValues - sometimes should consider there is an error we can accept @@ -139,10 +142,13 @@ func NewPrometheusScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } return &prometheusScaler{ - metricType: metricType, - metadata: meta, - httpClient: httpClient, - logger: logger, + metricType: metricType, + metadata: meta, + httpClient: httpClient, + logger: logger, + scalableObjectName: config.ScalableObjectName, + scalableObjectNS: config.ScalableObjectNamespace, + triggerName: config.TriggerName, }, nil } @@ -259,7 +265,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyUpstreamResponse() + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName) return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the result is empty") } else if len(result.Data.Result) > 1 { return -1, fmt.Errorf("prometheus query %s returned multiple elements", s.metadata.Query) @@ -270,7 +276,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyUpstreamResponse() + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName) return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the value list is empty") } else if valueLen < 2 { return -1, fmt.Errorf("prometheus query %s didn't return enough values", s.metadata.Query) diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index b000f63fb06..d57828ef5d5 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -48,6 +48,7 @@ var ( scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) @@ -73,7 +74,8 @@ type templateData struct { ScaledObjectGrpcName string WrongScaledObjectName string WrongScaledJobName string - WrongScalerName string + WrongScalerName string + EmptyUpstreamScaledObjectName string CronScaledJobName string MonitoredDeploymentName string ClientName string @@ -450,6 +452,89 @@ spec: limits: cpu: "500m" ` + + emptyUpstreamPrometheusConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-empty-upstream-config + namespace: {{.TestNamespace}} +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s +` + + emptyUpstreamPrometheusDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-empty-upstream + template: + metadata: + labels: + app: prometheus-empty-upstream + spec: + containers: + - name: prometheus + image: docker.io/prom/prometheus:v2.47.1 + args: + - --config.file=/etc/config/prometheus.yml + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: prometheus-empty-upstream-config +` + + emptyUpstreamPrometheusServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + selector: + app: prometheus-empty-upstream + ports: + - port: 9090 + targetPort: 9090 +` + + emptyUpstreamResponseScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.EmptyUpstreamScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 0 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: empty-upstream-trigger + metadata: + serverAddress: http://prometheus-empty-upstream.{{.TestNamespace}}.svc.cluster.local:9090 + threshold: '1' + query: 'nonexistent_metric_empty_upstream_response' + ignoreNullValues: 'false' +` ) func TestOpenTelemetryMetrics(t *testing.T) { @@ -487,6 +572,7 @@ func TestOpenTelemetryMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testEmptyUpstreamResponse(t, data) changeOtlpProtocolInOperator(t, kc, "keda-operator", "keda") testScalerGrpcMetricValue(t, kc, data) @@ -505,7 +591,8 @@ func getTemplateData() (templateData, []Template) { ScaledObjectGrpcName: scaledObjectGrpcName, ScaledJobName: scaledJobName, WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, MonitoredDeploymentName: monitoredDeploymentName, ClientName: clientName, CronScaledJobName: cronScaledJobName, @@ -1263,3 +1350,45 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { KubectlDeleteWithTemplate(t, data, "wrongCloudEventSourceTemplate", wrongCloudEventSourceTemplate) KubectlApplyWithTemplate(t, data, "cloudEventSourceTemplate", cloudEventSourceTemplate) } + +func testEmptyUpstreamResponse(t *testing.T, data templateData) { + t.Log("--- testing empty upstream response metric ---") + + kc := GetKubernetesClient(t) + + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, "prometheus-empty-upstream", testNamespace, 1, 60, 3), + "prometheus-empty-upstream deployment should be ready") + + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + }() + + time.Sleep(15 * time.Second) + + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorCollectorPrometheusExportURL)) + val, ok := family["keda_scaler_empty_upstream_responses_total"] + assert.True(t, ok, "keda_scaler_empty_upstream_responses_total not available") + if ok { + var found bool + for _, metric := range val.GetMetric() { + labels := metric.GetLabel() + if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && + ExtractPrometheusLabelValue("scaledObject", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + metric.GetCounter().GetValue() >= 1 { + found = true + } + } + assert.True(t, found, "keda_scaler_empty_upstream_responses_total not found with expected labels") + } +} diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 58152d979d0..7504d13d517 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -43,8 +43,9 @@ var ( wrongScaledObjectName = fmt.Sprintf("%s-so-wrong", testName) scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) - wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) - cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) + wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) + cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) wrongCloudEventSourceName = fmt.Sprintf("%s-ce-w", testName) @@ -65,7 +66,8 @@ type templateData struct { ScaledJobName string WrongScaledObjectName string WrongScaledJobName string - WrongScalerName string + WrongScalerName string + EmptyUpstreamScaledObjectName string CronScaledJobName string MonitoredDeploymentName string ClientName string @@ -421,6 +423,89 @@ spec: limits: cpu: "500m" ` + + emptyUpstreamPrometheusConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-empty-upstream-config + namespace: {{.TestNamespace}} +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s +` + + emptyUpstreamPrometheusDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-empty-upstream + template: + metadata: + labels: + app: prometheus-empty-upstream + spec: + containers: + - name: prometheus + image: docker.io/prom/prometheus:v2.47.1 + args: + - --config.file=/etc/config/prometheus.yml + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: prometheus-empty-upstream-config +` + + emptyUpstreamPrometheusServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + selector: + app: prometheus-empty-upstream + ports: + - port: 9090 + targetPort: 9090 +` + + emptyUpstreamResponseScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.EmptyUpstreamScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 0 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: empty-upstream-trigger + metadata: + serverAddress: http://prometheus-empty-upstream.{{.TestNamespace}}.svc.cluster.local:9090 + threshold: '1' + query: 'nonexistent_metric_empty_upstream_response' + ignoreNullValues: 'false' +` ) func TestPrometheusMetrics(t *testing.T) { @@ -450,6 +535,7 @@ func TestPrometheusMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testEmptyUpstreamResponse(t, data) // cleanup DeleteKubernetesResources(t, testNamespace, data, templates) } @@ -463,7 +549,8 @@ func getTemplateData() (templateData, []Template) { WrongScaledObjectName: wrongScaledObjectName, ScaledJobName: scaledJobName, WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, MonitoredDeploymentName: monitoredDeploymentName, ClientName: clientName, CronScaledJobName: cronScaledJobName, @@ -1430,3 +1517,44 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { assert.True(t, familyValidator(metric)) } + +func testEmptyUpstreamResponse(t *testing.T, data templateData) { + t.Log("--- testing empty upstream response metric ---") + + kc := GetKubernetesClient(t) + + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, "prometheus-empty-upstream", testNamespace, 1, 60, 3), + "prometheus-empty-upstream deployment should be ready") + + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + }() + + familyValidator := func(family *prommodel.MetricFamily) bool { + for _, metric := range family.GetMetric() { + labels := metric.GetLabel() + if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && + ExtractPrometheusLabelValue("scaledObject", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + metric.GetCounter().GetValue() >= 1 { + return true + } + } + return false + } + + families := WaitForPrometheusMetric(t, "keda_scaler_empty_upstream_responses_total", familyValidator) + metric := families["keda_scaler_empty_upstream_responses_total"] + + assert.True(t, familyValidator(metric)) +} From 2edc55e21d77c47adb4eb3476f137f0119877ba7 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Mon, 20 Apr 2026 17:28:43 +0000 Subject: [PATCH 7/9] Fix gci formatting Signed-off-by: Ali Aqel --- pkg/scalers/prometheus_scaler.go | 14 ++-- .../opentelemetry_metrics_test.go | 72 +++++++++--------- .../prometheus_metrics_test.go | 74 +++++++++---------- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index b38e3944f0e..3747e4ed236 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -27,13 +27,13 @@ import ( ) type prometheusScaler struct { - metricType v2.MetricTargetType - metadata *prometheusMetadata - httpClient *http.Client - logger logr.Logger - scalableObjectName string - scalableObjectNS string - triggerName string + metricType v2.MetricTargetType + metadata *prometheusMetadata + httpClient *http.Client + logger logr.Logger + scalableObjectName string + scalableObjectNS string + triggerName string } // IgnoreNullValues - sometimes should consider there is an error we can accept diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index d57828ef5d5..994899ce9d2 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -66,24 +66,24 @@ var ( ) type templateData struct { - TestName string - TestNamespace string - DeploymentName string - ScaledObjectName string - ScaledJobName string - ScaledObjectGrpcName string - WrongScaledObjectName string - WrongScaledJobName string - WrongScalerName string - EmptyUpstreamScaledObjectName string - CronScaledJobName string - MonitoredDeploymentName string - ClientName string - CloudEventSourceName string - WrongCloudEventSourceName string - CloudEventHTTPReceiverName string - CloudEventHTTPServiceName string - CloudEventHTTPServiceURL string + TestName string + TestNamespace string + DeploymentName string + ScaledObjectName string + ScaledJobName string + ScaledObjectGrpcName string + WrongScaledObjectName string + WrongScaledJobName string + WrongScalerName string + EmptyUpstreamScaledObjectName string + CronScaledJobName string + MonitoredDeploymentName string + ClientName string + CloudEventSourceName string + WrongCloudEventSourceName string + CloudEventHTTPReceiverName string + CloudEventHTTPServiceName string + CloudEventHTTPServiceURL string } const ( @@ -583,24 +583,24 @@ func TestOpenTelemetryMetrics(t *testing.T) { func getTemplateData() (templateData, []Template) { return templateData{ - TestName: testName, - TestNamespace: testNamespace, - DeploymentName: deploymentName, - ScaledObjectName: scaledObjectName, - WrongScaledObjectName: wrongScaledObjectName, - ScaledObjectGrpcName: scaledObjectGrpcName, - ScaledJobName: scaledJobName, - WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, - EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, - MonitoredDeploymentName: monitoredDeploymentName, - ClientName: clientName, - CronScaledJobName: cronScaledJobName, - CloudEventSourceName: cloudEventSourceName, - WrongCloudEventSourceName: wrongCloudEventSourceName, - CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, - CloudEventHTTPServiceName: cloudEventHTTPServiceName, - CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, + TestName: testName, + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + WrongScaledObjectName: wrongScaledObjectName, + ScaledObjectGrpcName: scaledObjectGrpcName, + ScaledJobName: scaledJobName, + WrongScaledJobName: wrongScaledJobName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, + MonitoredDeploymentName: monitoredDeploymentName, + ClientName: clientName, + CronScaledJobName: cronScaledJobName, + CloudEventSourceName: cloudEventSourceName, + WrongCloudEventSourceName: wrongCloudEventSourceName, + CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, + CloudEventHTTPServiceName: cloudEventHTTPServiceName, + CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, }, []Template{ {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 7504d13d517..87edbc34f49 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -43,9 +43,9 @@ var ( wrongScaledObjectName = fmt.Sprintf("%s-so-wrong", testName) scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) - wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) - emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) - cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) + wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) + cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) wrongCloudEventSourceName = fmt.Sprintf("%s-ce-w", testName) @@ -59,23 +59,23 @@ var ( ) type templateData struct { - TestName string - TestNamespace string - DeploymentName string - ScaledObjectName string - ScaledJobName string - WrongScaledObjectName string - WrongScaledJobName string - WrongScalerName string - EmptyUpstreamScaledObjectName string - CronScaledJobName string - MonitoredDeploymentName string - ClientName string - CloudEventSourceName string - WrongCloudEventSourceName string - CloudEventHTTPReceiverName string - CloudEventHTTPServiceName string - CloudEventHTTPServiceURL string + TestName string + TestNamespace string + DeploymentName string + ScaledObjectName string + ScaledJobName string + WrongScaledObjectName string + WrongScaledJobName string + WrongScalerName string + EmptyUpstreamScaledObjectName string + CronScaledJobName string + MonitoredDeploymentName string + ClientName string + CloudEventSourceName string + WrongCloudEventSourceName string + CloudEventHTTPReceiverName string + CloudEventHTTPServiceName string + CloudEventHTTPServiceURL string } const ( @@ -542,23 +542,23 @@ func TestPrometheusMetrics(t *testing.T) { func getTemplateData() (templateData, []Template) { return templateData{ - TestName: testName, - TestNamespace: testNamespace, - DeploymentName: deploymentName, - ScaledObjectName: scaledObjectName, - WrongScaledObjectName: wrongScaledObjectName, - ScaledJobName: scaledJobName, - WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, - EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, - MonitoredDeploymentName: monitoredDeploymentName, - ClientName: clientName, - CronScaledJobName: cronScaledJobName, - CloudEventSourceName: cloudEventSourceName, - WrongCloudEventSourceName: wrongCloudEventSourceName, - CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, - CloudEventHTTPServiceName: cloudEventHTTPServiceName, - CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, + TestName: testName, + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + WrongScaledObjectName: wrongScaledObjectName, + ScaledJobName: scaledJobName, + WrongScaledJobName: wrongScaledJobName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, + MonitoredDeploymentName: monitoredDeploymentName, + ClientName: clientName, + CronScaledJobName: cronScaledJobName, + CloudEventSourceName: cloudEventSourceName, + WrongCloudEventSourceName: wrongCloudEventSourceName, + CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, + CloudEventHTTPServiceName: cloudEventHTTPServiceName, + CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, }, []Template{ {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, From ca0a9ec736211ceb251b335a24313266c7ea0ae1 Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Wed, 22 Apr 2026 16:29:17 +0000 Subject: [PATCH 8/9] fix: address PR review feedback on empty upstream response metric - Rename scaledObject label -> scaledResource, add metricName, resourceType, and ignoreNullValues labels to keda_scaler_empty_upstream_responses_total - Record metric unconditionally (before IgnoreNullValues guard) so masked empty responses are also visible, with ignoreNullValues label for filtering - Fix CHANGELOG: reference issue #7062 instead of PR #7060, capitalize Prometheus - Update e2e tests to assert new labels Signed-off-by: Ali Aqel --- CHANGELOG.md | 2 +- pkg/metricscollector/metricscollectors.go | 6 +++--- pkg/metricscollector/opentelemetry.go | 7 +++++-- pkg/metricscollector/prommetrics.go | 13 ++++++++++--- pkg/scalers/prometheus_scaler.go | 8 ++++++-- .../opentelemetry_metrics_test.go | 5 ++++- .../prometheus_metrics/prometheus_metrics_test.go | 5 ++++- 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d31d4588e0f..4c9b9fe97c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,7 +81,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Make APIService cert injections optional ([#7559](https://github.com/kedacore/keda/pull/7559)) - **Elasticsearch Scaler**: Add HTTP status check for Elasticsearch errors ([#7480](https://github.com/kedacore/keda/pull/7480)) - **Kubernetes Workload Scaler**: Add `groupByNode` parameter ([#7628](https://github.com/kedacore/keda/issues/7628)) -- **Prometheus Scaler**: Emit metric tracking empty responses from prometheus ([#7060](https://github.com/kedacore/keda/pull/7060)) +- **Prometheus Scaler**: Emit metric tracking empty responses from Prometheus ([#7062](https://github.com/kedacore/keda/issues/7062)) ### Fixes diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 1b4437b0a44..d734296253a 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -81,7 +81,7 @@ type MetricsCollector interface { RecordCloudEventQueueStatus(namespace string, value int) // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result - RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) + RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -209,9 +209,9 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result -func RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { +func RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { for _, element := range collectors { - element.RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName) + element.RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType, ignoreNullValues) } } diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 9160de4f4fa..60752f71a4f 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -514,10 +514,13 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { } // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result -func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { +func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { otEmptyUpstreamResponses.Add(context.Background(), 1, api.WithAttributes( attribute.Key("namespace").String(namespace), - attribute.Key("scaledObject").String(scaledObject), + attribute.Key("scaledResource").String(scaledResource), attribute.Key("triggerName").String(triggerName), + attribute.Key("metricName").String(metricName), + attribute.Key("resourceType").String(resourceType), + attribute.Key("ignoreNullValues").Bool(ignoreNullValues), )) } diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 8272fe230a2..c84246fb52f 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -111,7 +111,7 @@ var ( Name: "empty_upstream_responses_total", Help: "Number of times a query returns an empty result", }, - []string{"namespace", "scaledObject", "triggerName"}, + []string{"namespace", "scaledResource", "triggerName", "metricName", "resourceType", "ignoreNullValues"}, ) triggerRegistered = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -339,8 +339,15 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { } // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result -func (p *PromMetrics) RecordEmptyUpstreamResponse(namespace, scaledObject, triggerName string) { - emptyUpstreamResponse.With(prometheus.Labels{"namespace": namespace, "scaledObject": scaledObject, "triggerName": triggerName}).Inc() +func (p *PromMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { + emptyUpstreamResponse.With(prometheus.Labels{ + "namespace": namespace, + "scaledResource": scaledResource, + "triggerName": triggerName, + "metricName": metricName, + "resourceType": resourceType, + "ignoreNullValues": strconv.FormatBool(ignoreNullValues), + }).Inc() } // Returns a grpcprom server Metrics object and registers the metrics. The object contains diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 3747e4ed236..ac1fff8f985 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -34,6 +34,8 @@ type prometheusScaler struct { scalableObjectName string scalableObjectNS string triggerName string + metricName string + resourceType string } // IgnoreNullValues - sometimes should consider there is an error we can accept @@ -149,6 +151,8 @@ func NewPrometheusScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { scalableObjectName: config.ScalableObjectName, scalableObjectNS: config.ScalableObjectNamespace, triggerName: config.TriggerName, + metricName: GenerateMetricNameWithIndex(meta.triggerIndex, kedautil.NormalizeString("prometheus")), + resourceType: config.ScalableObjectType, }, nil } @@ -262,10 +266,10 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error // allow for zero element or single element result sets if len(result.Data.Result) == 0 { + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues) if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName) return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the result is empty") } else if len(result.Data.Result) > 1 { return -1, fmt.Errorf("prometheus query %s returned multiple elements", s.metadata.Query) @@ -273,10 +277,10 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error valueLen := len(result.Data.Result[0].Value) if valueLen == 0 { + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues) if s.metadata.IgnoreNullValues { return 0, nil } - metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName) return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the value list is empty") } else if valueLen < 2 { return -1, fmt.Errorf("prometheus query %s didn't return enough values", s.metadata.Query) diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index 994899ce9d2..7340c3609fa 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -1383,8 +1383,11 @@ func testEmptyUpstreamResponse(t *testing.T, data templateData) { for _, metric := range val.GetMetric() { labels := metric.GetLabel() if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && - ExtractPrometheusLabelValue("scaledObject", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("scaledResource", labels) == emptyUpstreamScaledObjectName && ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + ExtractPrometheusLabelValue("metricName", labels) == "s0-prometheus" && + ExtractPrometheusLabelValue("resourceType", labels) == "ScaledObject" && + ExtractPrometheusLabelValue("ignoreNullValues", labels) == "false" && metric.GetCounter().GetValue() >= 1 { found = true } diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 87edbc34f49..3c98e60933c 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -1544,8 +1544,11 @@ func testEmptyUpstreamResponse(t *testing.T, data templateData) { for _, metric := range family.GetMetric() { labels := metric.GetLabel() if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && - ExtractPrometheusLabelValue("scaledObject", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("scaledResource", labels) == emptyUpstreamScaledObjectName && ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + ExtractPrometheusLabelValue("metricName", labels) == "s0-prometheus" && + ExtractPrometheusLabelValue("resourceType", labels) == "ScaledObject" && + ExtractPrometheusLabelValue("ignoreNullValues", labels) == "false" && metric.GetCounter().GetValue() >= 1 { return true } From cab05e634043109db78483b36c1153d55940afaa Mon Sep 17 00:00:00 2001 From: Ali Aqel Date: Tue, 28 Apr 2026 01:03:27 +0000 Subject: [PATCH 9/9] fix: use isScaledObject for OTEL empty upstream metric Signed-off-by: Ali Aqel --- pkg/metricscollector/opentelemetry.go | 2 +- .../opentelemetry_metrics/opentelemetry_metrics_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 60752f71a4f..5c8f8aa028e 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -520,7 +520,7 @@ func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, tri attribute.Key("scaledResource").String(scaledResource), attribute.Key("triggerName").String(triggerName), attribute.Key("metricName").String(metricName), - attribute.Key("resourceType").String(resourceType), + attribute.Key("isScaledObject").Bool(resourceType == "ScaledObject"), attribute.Key("ignoreNullValues").Bool(ignoreNullValues), )) } diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index 7340c3609fa..e65215c3321 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -1386,7 +1386,7 @@ func testEmptyUpstreamResponse(t *testing.T, data templateData) { ExtractPrometheusLabelValue("scaledResource", labels) == emptyUpstreamScaledObjectName && ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && ExtractPrometheusLabelValue("metricName", labels) == "s0-prometheus" && - ExtractPrometheusLabelValue("resourceType", labels) == "ScaledObject" && + ExtractPrometheusLabelValue("isScaledObject", labels) == "true" && ExtractPrometheusLabelValue("ignoreNullValues", labels) == "false" && metric.GetCounter().GetValue() >= 1 { found = true