Emit metric tracking empty responses from prometheus

CHANGELOG.md

@@ -85,6 +85,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
 - **Github Runner Scaler**: Handle rate limit errors by respecting X-RateLimit-Reset and Retry-After headers and returning cached queue length ([#7683](https://github.com/kedacore/keda/issues/7683))
 - **Kubernetes Workload Scaler**: Add `groupByNode` parameter ([#7628](https://github.com/kedacore/keda/issues/7628))
 - **MSSQL Scaler**: Add Azure Workload Identity support for Azure SQL authentication ([#6104](https://github.com/kedacore/keda/issues/6104))
+- **Prometheus Scaler**: Emit metric tracking empty responses from Prometheus ([#7062](https://github.com/kedacore/keda/issues/7062))
 
 ### Fixes
 

pkg/metricscollector/metricscollectors.go

@@ -79,6 +79,9 @@ type MetricsCollector interface {
 
 	// RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting
 	RecordCloudEventQueueStatus(namespace string, value int)
+
+	// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+	RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool)
 }
 
 func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) {
@@ -205,6 +208,13 @@ func RecordCloudEventQueueStatus(namespace string, value int) {
 	}
 }
 
+// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+func RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) {
+	for _, element := range collectors {
+		element.RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType, ignoreNullValues)
+	}
+}
+
 // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts
 // Currently, only Prometheus metrics are supported.
 func GetServerMetrics() *grpcprom.ServerMetrics {

pkg/metricscollector/opentelemetry.go

@@ -34,6 +34,7 @@ var (
 	otCrdTotalsCounterDeprecated     api.Int64UpDownCounter
 	otTriggerRegisteredTotalsCounter api.Int64UpDownCounter
 	otCrdRegisteredTotalsCounter     api.Int64UpDownCounter
+	otEmptyUpstreamResponses         api.Int64Counter
 
 	otelScalerMetricVals                  []OtelMetricFloat64Val
 	otelScalerMetricsLatencyVals          []OtelMetricFloat64Val
@@ -135,6 +136,11 @@ func initMeters() {
 		otLog.Error(err, msg)
 	}
 
+	otEmptyUpstreamResponses, err = meter.Int64Counter("keda.scaler.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result"))
+	if err != nil {
+		otLog.Error(err, msg)
+	}
+
 	_, err = meter.Float64ObservableGauge(
 		"keda.scaler.metrics.value",
 		api.WithDescription("The current value for each scaler's metric that would be used by the HPA in computing the target average"),
@@ -506,3 +512,15 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) {
 	otCloudEventQueueStatus.measurementOption = opt
 	otCloudEventQueueStatusVals = append(otCloudEventQueueStatusVals, otCloudEventQueueStatus)
 }
+
+// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) {
+	otEmptyUpstreamResponses.Add(context.Background(), 1, api.WithAttributes(
+		attribute.Key("namespace").String(namespace),
+		attribute.Key("scaledResource").String(scaledResource),
+		attribute.Key("triggerName").String(triggerName),
+		attribute.Key("metricName").String(metricName),
+		attribute.Key("isScaledObject").Bool(resourceType == "ScaledObject"),
+		attribute.Key("ignoreNullValues").Bool(ignoreNullValues),
+	))
+}

pkg/metricscollector/prommetrics.go

@@ -104,6 +104,15 @@ var (
 		},
 		[]string{"namespace", "scaledJob"},
 	)
+	emptyUpstreamResponse = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: DefaultPromMetricsNamespace,
+			Subsystem: "scaler",
+			Name:      "empty_upstream_responses_total",
+			Help:      "Number of times a query returns an empty result",
+		},
+		[]string{"namespace", "scaledResource", "triggerName", "metricName", "resourceType", "ignoreNullValues"},
+	)
 	triggerRegistered = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: DefaultPromMetricsNamespace,
@@ -168,6 +177,7 @@ func NewPromMetrics() *PromMetrics {
 	metrics.Registry.MustRegister(triggerRegistered)
 	metrics.Registry.MustRegister(crdRegistered)
 	metrics.Registry.MustRegister(scaledJobErrors)
+	metrics.Registry.MustRegister(emptyUpstreamResponse)
 
 	metrics.Registry.MustRegister(buildInfo)
 
@@ -328,6 +338,18 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) {
 	cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value))
 }
 
+// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+func (p *PromMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) {
+	emptyUpstreamResponse.With(prometheus.Labels{
+		"namespace":        namespace,
+		"scaledResource":   scaledResource,
+		"triggerName":      triggerName,
+		"metricName":       metricName,
+		"resourceType":     resourceType,
+		"ignoreNullValues": strconv.FormatBool(ignoreNullValues),
+	}).Inc()
+}
+
 // Returns a grpcprom server Metrics object and registers the metrics. The object contains
 // interceptors to chain to the server so that all requests served are observed. Intended to be called
 // as part of initialization of metricscollector, hence why this function is not exported

pkg/scalers/prometheus_scaler.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/metrics/pkg/apis/external_metrics"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"github.com/kedacore/keda/v2/pkg/metricscollector"
 	"github.com/kedacore/keda/v2/pkg/scalers/authentication"
 	"github.com/kedacore/keda/v2/pkg/scalers/aws"
 	"github.com/kedacore/keda/v2/pkg/scalers/azure"
@@ -26,10 +27,15 @@ import (
 )
 
 type prometheusScaler struct {
-	metricType v2.MetricTargetType
-	metadata   *prometheusMetadata
-	httpClient *http.Client
-	logger     logr.Logger
+	metricType         v2.MetricTargetType
+	metadata           *prometheusMetadata
+	httpClient         *http.Client
+	logger             logr.Logger
+	scalableObjectName string
+	scalableObjectNS   string
+	triggerName        string
+	metricName         string
+	resourceType       string
 }
 
 // IgnoreNullValues - sometimes should consider there is an error we can accept
@@ -138,10 +144,15 @@ func NewPrometheusScaler(config *scalersconfig.ScalerConfig) (Scaler, error) {
 	}
 
 	return &prometheusScaler{
-		metricType: metricType,
-		metadata:   meta,
-		httpClient: httpClient,
-		logger:     logger,
+		metricType:         metricType,
+		metadata:           meta,
+		httpClient:         httpClient,
+		logger:             logger,
+		scalableObjectName: config.ScalableObjectName,
+		scalableObjectNS:   config.ScalableObjectNamespace,
+		triggerName:        config.TriggerName,
+		metricName:         GenerateMetricNameWithIndex(meta.triggerIndex, kedautil.NormalizeString("prometheus")),
+		resourceType:       config.ScalableObjectType,
 	}, nil
 }
 
@@ -255,6 +266,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error
 
 	// allow for zero element or single element result sets
 	if len(result.Data.Result) == 0 {
+		metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues)
 		if s.metadata.IgnoreNullValues {
 			return 0, nil
 		}
@@ -265,6 +277,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error
 
 	valueLen := len(result.Data.Result[0].Value)
 	if valueLen == 0 {
+		metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues)
 		if s.metadata.IgnoreNullValues {
 			return 0, nil
 		}