diff --git a/CHANGELOG.md b/CHANGELOG.md index 048e35cf0bf..53ee72ecbdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **Github Runner Scaler**: Handle rate limit errors by respecting X-RateLimit-Reset and Retry-After headers and returning cached queue length ([#7683](https://github.com/kedacore/keda/issues/7683)) - **Kubernetes Workload Scaler**: Add `groupByNode` parameter ([#7628](https://github.com/kedacore/keda/issues/7628)) - **MSSQL Scaler**: Add Azure Workload Identity support for Azure SQL authentication ([#6104](https://github.com/kedacore/keda/issues/6104)) +- **Prometheus Scaler**: Emit metric tracking empty responses from Prometheus ([#7062](https://github.com/kedacore/keda/issues/7062)) ### Fixes diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 7a6d1055ffa..d734296253a 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -79,6 +79,9 @@ type MetricsCollector interface { // RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting RecordCloudEventQueueStatus(namespace string, value int) + + // RecordEmptyUpstreamResponse counts the number of times a query returns an empty result + RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) } func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) { @@ -205,6 +208,13 @@ func RecordCloudEventQueueStatus(namespace string, value int) { } } +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { + for _, element := range collectors { + element.RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType, ignoreNullValues) + } +} + // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts // Currently, only Prometheus metrics are supported. func GetServerMetrics() *grpcprom.ServerMetrics { diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index 14e2aa32f47..5c8f8aa028e 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -34,6 +34,7 @@ var ( otCrdTotalsCounterDeprecated api.Int64UpDownCounter otTriggerRegisteredTotalsCounter api.Int64UpDownCounter otCrdRegisteredTotalsCounter api.Int64UpDownCounter + otEmptyUpstreamResponses api.Int64Counter otelScalerMetricVals []OtelMetricFloat64Val otelScalerMetricsLatencyVals []OtelMetricFloat64Val @@ -135,6 +136,11 @@ func initMeters() { otLog.Error(err, msg) } + otEmptyUpstreamResponses, err = meter.Int64Counter("keda.scaler.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result")) + if err != nil { + otLog.Error(err, msg) + } + _, err = meter.Float64ObservableGauge( "keda.scaler.metrics.value", api.WithDescription("The current value for each scaler's metric that would be used by the HPA in computing the target average"), @@ -506,3 +512,15 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) { otCloudEventQueueStatus.measurementOption = opt otCloudEventQueueStatusVals = append(otCloudEventQueueStatusVals, otCloudEventQueueStatus) } + +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func (o *OtelMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { + otEmptyUpstreamResponses.Add(context.Background(), 1, api.WithAttributes( + attribute.Key("namespace").String(namespace), + attribute.Key("scaledResource").String(scaledResource), + attribute.Key("triggerName").String(triggerName), + attribute.Key("metricName").String(metricName), + attribute.Key("isScaledObject").Bool(resourceType == "ScaledObject"), + attribute.Key("ignoreNullValues").Bool(ignoreNullValues), + )) +} diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 92b60c17f26..c84246fb52f 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -104,6 +104,15 @@ var ( }, []string{"namespace", "scaledJob"}, ) + emptyUpstreamResponse = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "scaler", + Name: "empty_upstream_responses_total", + Help: "Number of times a query returns an empty result", + }, + []string{"namespace", "scaledResource", "triggerName", "metricName", "resourceType", "ignoreNullValues"}, + ) triggerRegistered = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, @@ -168,6 +177,7 @@ func NewPromMetrics() *PromMetrics { metrics.Registry.MustRegister(triggerRegistered) metrics.Registry.MustRegister(crdRegistered) metrics.Registry.MustRegister(scaledJobErrors) + metrics.Registry.MustRegister(emptyUpstreamResponse) metrics.Registry.MustRegister(buildInfo) @@ -328,6 +338,18 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) { cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value)) } +// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result +func (p *PromMetrics) RecordEmptyUpstreamResponse(namespace, scaledResource, triggerName, metricName, resourceType string, ignoreNullValues bool) { + emptyUpstreamResponse.With(prometheus.Labels{ + "namespace": namespace, + "scaledResource": scaledResource, + "triggerName": triggerName, + "metricName": metricName, + "resourceType": resourceType, + "ignoreNullValues": strconv.FormatBool(ignoreNullValues), + }).Inc() +} + // Returns a grpcprom server Metrics object and registers the metrics. The object contains // interceptors to chain to the server so that all requests served are observed. Intended to be called // as part of initialization of metricscollector, hence why this function is not exported diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 033dc2f4f24..ac1fff8f985 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -17,6 +17,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scalers/aws" "github.com/kedacore/keda/v2/pkg/scalers/azure" @@ -26,10 +27,15 @@ import ( ) type prometheusScaler struct { - metricType v2.MetricTargetType - metadata *prometheusMetadata - httpClient *http.Client - logger logr.Logger + metricType v2.MetricTargetType + metadata *prometheusMetadata + httpClient *http.Client + logger logr.Logger + scalableObjectName string + scalableObjectNS string + triggerName string + metricName string + resourceType string } // IgnoreNullValues - sometimes should consider there is an error we can accept @@ -138,10 +144,15 @@ func NewPrometheusScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { } return &prometheusScaler{ - metricType: metricType, - metadata: meta, - httpClient: httpClient, - logger: logger, + metricType: metricType, + metadata: meta, + httpClient: httpClient, + logger: logger, + scalableObjectName: config.ScalableObjectName, + scalableObjectNS: config.ScalableObjectNamespace, + triggerName: config.TriggerName, + metricName: GenerateMetricNameWithIndex(meta.triggerIndex, kedautil.NormalizeString("prometheus")), + resourceType: config.ScalableObjectType, }, nil } @@ -255,6 +266,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error // allow for zero element or single element result sets if len(result.Data.Result) == 0 { + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues) if s.metadata.IgnoreNullValues { return 0, nil } @@ -265,6 +277,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error valueLen := len(result.Data.Result[0].Value) if valueLen == 0 { + metricscollector.RecordEmptyUpstreamResponse(s.scalableObjectNS, s.scalableObjectName, s.triggerName, s.metricName, s.resourceType, s.metadata.IgnoreNullValues) if s.metadata.IgnoreNullValues { return 0, nil } diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index b000f63fb06..e65215c3321 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -48,6 +48,7 @@ var ( scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) @@ -65,23 +66,24 @@ var ( ) type templateData struct { - TestName string - TestNamespace string - DeploymentName string - ScaledObjectName string - ScaledJobName string - ScaledObjectGrpcName string - WrongScaledObjectName string - WrongScaledJobName string - WrongScalerName string - CronScaledJobName string - MonitoredDeploymentName string - ClientName string - CloudEventSourceName string - WrongCloudEventSourceName string - CloudEventHTTPReceiverName string - CloudEventHTTPServiceName string - CloudEventHTTPServiceURL string + TestName string + TestNamespace string + DeploymentName string + ScaledObjectName string + ScaledJobName string + ScaledObjectGrpcName string + WrongScaledObjectName string + WrongScaledJobName string + WrongScalerName string + EmptyUpstreamScaledObjectName string + CronScaledJobName string + MonitoredDeploymentName string + ClientName string + CloudEventSourceName string + WrongCloudEventSourceName string + CloudEventHTTPReceiverName string + CloudEventHTTPServiceName string + CloudEventHTTPServiceURL string } const ( @@ -450,6 +452,89 @@ spec: limits: cpu: "500m" ` + + emptyUpstreamPrometheusConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-empty-upstream-config + namespace: {{.TestNamespace}} +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s +` + + emptyUpstreamPrometheusDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-empty-upstream + template: + metadata: + labels: + app: prometheus-empty-upstream + spec: + containers: + - name: prometheus + image: docker.io/prom/prometheus:v2.47.1 + args: + - --config.file=/etc/config/prometheus.yml + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: prometheus-empty-upstream-config +` + + emptyUpstreamPrometheusServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + selector: + app: prometheus-empty-upstream + ports: + - port: 9090 + targetPort: 9090 +` + + emptyUpstreamResponseScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.EmptyUpstreamScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 0 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: empty-upstream-trigger + metadata: + serverAddress: http://prometheus-empty-upstream.{{.TestNamespace}}.svc.cluster.local:9090 + threshold: '1' + query: 'nonexistent_metric_empty_upstream_response' + ignoreNullValues: 'false' +` ) func TestOpenTelemetryMetrics(t *testing.T) { @@ -487,6 +572,7 @@ func TestOpenTelemetryMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testEmptyUpstreamResponse(t, data) changeOtlpProtocolInOperator(t, kc, "keda-operator", "keda") testScalerGrpcMetricValue(t, kc, data) @@ -497,23 +583,24 @@ func TestOpenTelemetryMetrics(t *testing.T) { func getTemplateData() (templateData, []Template) { return templateData{ - TestName: testName, - TestNamespace: testNamespace, - DeploymentName: deploymentName, - ScaledObjectName: scaledObjectName, - WrongScaledObjectName: wrongScaledObjectName, - ScaledObjectGrpcName: scaledObjectGrpcName, - ScaledJobName: scaledJobName, - WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, - MonitoredDeploymentName: monitoredDeploymentName, - ClientName: clientName, - CronScaledJobName: cronScaledJobName, - CloudEventSourceName: cloudEventSourceName, - WrongCloudEventSourceName: wrongCloudEventSourceName, - CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, - CloudEventHTTPServiceName: cloudEventHTTPServiceName, - CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, + TestName: testName, + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + WrongScaledObjectName: wrongScaledObjectName, + ScaledObjectGrpcName: scaledObjectGrpcName, + ScaledJobName: scaledJobName, + WrongScaledJobName: wrongScaledJobName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, + MonitoredDeploymentName: monitoredDeploymentName, + ClientName: clientName, + CronScaledJobName: cronScaledJobName, + CloudEventSourceName: cloudEventSourceName, + WrongCloudEventSourceName: wrongCloudEventSourceName, + CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, + CloudEventHTTPServiceName: cloudEventHTTPServiceName, + CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, }, []Template{ {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, @@ -1263,3 +1350,48 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { KubectlDeleteWithTemplate(t, data, "wrongCloudEventSourceTemplate", wrongCloudEventSourceTemplate) KubectlApplyWithTemplate(t, data, "cloudEventSourceTemplate", cloudEventSourceTemplate) } + +func testEmptyUpstreamResponse(t *testing.T, data templateData) { + t.Log("--- testing empty upstream response metric ---") + + kc := GetKubernetesClient(t) + + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, "prometheus-empty-upstream", testNamespace, 1, 60, 3), + "prometheus-empty-upstream deployment should be ready") + + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + }() + + time.Sleep(15 * time.Second) + + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorCollectorPrometheusExportURL)) + val, ok := family["keda_scaler_empty_upstream_responses_total"] + assert.True(t, ok, "keda_scaler_empty_upstream_responses_total not available") + if ok { + var found bool + for _, metric := range val.GetMetric() { + labels := metric.GetLabel() + if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && + ExtractPrometheusLabelValue("scaledResource", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + ExtractPrometheusLabelValue("metricName", labels) == "s0-prometheus" && + ExtractPrometheusLabelValue("isScaledObject", labels) == "true" && + ExtractPrometheusLabelValue("ignoreNullValues", labels) == "false" && + metric.GetCounter().GetValue() >= 1 { + found = true + } + } + assert.True(t, found, "keda_scaler_empty_upstream_responses_total not found with expected labels") + } +} diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index 58152d979d0..3c98e60933c 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -44,6 +44,7 @@ var ( scaledJobName = fmt.Sprintf("%s-sj", testName) wrongScaledJobName = fmt.Sprintf("%s-sj-wrong", testName) wrongScalerName = fmt.Sprintf("%s-wrong-scaler", testName) + emptyUpstreamScaledObjectName = fmt.Sprintf("%s-so-empty-upstream", testName) cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName) clientName = fmt.Sprintf("%s-client", testName) cloudEventSourceName = fmt.Sprintf("%s-ce", testName) @@ -58,22 +59,23 @@ var ( ) type templateData struct { - TestName string - TestNamespace string - DeploymentName string - ScaledObjectName string - ScaledJobName string - WrongScaledObjectName string - WrongScaledJobName string - WrongScalerName string - CronScaledJobName string - MonitoredDeploymentName string - ClientName string - CloudEventSourceName string - WrongCloudEventSourceName string - CloudEventHTTPReceiverName string - CloudEventHTTPServiceName string - CloudEventHTTPServiceURL string + TestName string + TestNamespace string + DeploymentName string + ScaledObjectName string + ScaledJobName string + WrongScaledObjectName string + WrongScaledJobName string + WrongScalerName string + EmptyUpstreamScaledObjectName string + CronScaledJobName string + MonitoredDeploymentName string + ClientName string + CloudEventSourceName string + WrongCloudEventSourceName string + CloudEventHTTPReceiverName string + CloudEventHTTPServiceName string + CloudEventHTTPServiceURL string } const ( @@ -421,6 +423,89 @@ spec: limits: cpu: "500m" ` + + emptyUpstreamPrometheusConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-empty-upstream-config + namespace: {{.TestNamespace}} +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s +` + + emptyUpstreamPrometheusDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-empty-upstream + template: + metadata: + labels: + app: prometheus-empty-upstream + spec: + containers: + - name: prometheus + image: docker.io/prom/prometheus:v2.47.1 + args: + - --config.file=/etc/config/prometheus.yml + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: prometheus-empty-upstream-config +` + + emptyUpstreamPrometheusServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: prometheus-empty-upstream + namespace: {{.TestNamespace}} +spec: + selector: + app: prometheus-empty-upstream + ports: + - port: 9090 + targetPort: 9090 +` + + emptyUpstreamResponseScaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.EmptyUpstreamScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 2 + idleReplicaCount: 0 + minReplicaCount: 0 + maxReplicaCount: 2 + cooldownPeriod: 10 + triggers: + - type: prometheus + name: empty-upstream-trigger + metadata: + serverAddress: http://prometheus-empty-upstream.{{.TestNamespace}}.svc.cluster.local:9090 + threshold: '1' + query: 'nonexistent_metric_empty_upstream_response' + ignoreNullValues: 'false' +` ) func TestPrometheusMetrics(t *testing.T) { @@ -450,28 +535,30 @@ func TestPrometheusMetrics(t *testing.T) { testScaledObjectPausedMetric(t, data) testCloudEventEmitted(t, data) testCloudEventEmittedError(t, data) + testEmptyUpstreamResponse(t, data) // cleanup DeleteKubernetesResources(t, testNamespace, data, templates) } func getTemplateData() (templateData, []Template) { return templateData{ - TestName: testName, - TestNamespace: testNamespace, - DeploymentName: deploymentName, - ScaledObjectName: scaledObjectName, - WrongScaledObjectName: wrongScaledObjectName, - ScaledJobName: scaledJobName, - WrongScaledJobName: wrongScaledJobName, - WrongScalerName: wrongScalerName, - MonitoredDeploymentName: monitoredDeploymentName, - ClientName: clientName, - CronScaledJobName: cronScaledJobName, - CloudEventSourceName: cloudEventSourceName, - WrongCloudEventSourceName: wrongCloudEventSourceName, - CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, - CloudEventHTTPServiceName: cloudEventHTTPServiceName, - CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, + TestName: testName, + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + WrongScaledObjectName: wrongScaledObjectName, + ScaledJobName: scaledJobName, + WrongScaledJobName: wrongScaledJobName, + WrongScalerName: wrongScalerName, + EmptyUpstreamScaledObjectName: emptyUpstreamScaledObjectName, + MonitoredDeploymentName: monitoredDeploymentName, + ClientName: clientName, + CronScaledJobName: cronScaledJobName, + CloudEventSourceName: cloudEventSourceName, + WrongCloudEventSourceName: wrongCloudEventSourceName, + CloudEventHTTPReceiverName: cloudEventHTTPReceiverName, + CloudEventHTTPServiceName: cloudEventHTTPServiceName, + CloudEventHTTPServiceURL: cloudEventHTTPServiceURL, }, []Template{ {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, @@ -1430,3 +1517,47 @@ func testCloudEventEmittedError(t *testing.T, data templateData) { assert.True(t, familyValidator(metric)) } + +func testEmptyUpstreamResponse(t *testing.T, data templateData) { + t.Log("--- testing empty upstream response metric ---") + + kc := GetKubernetesClient(t) + + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, "prometheus-empty-upstream", testNamespace, 1, 60, 3), + "prometheus-empty-upstream deployment should be ready") + + KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + defer func() { + KubectlDeleteWithTemplate(t, data, "emptyUpstreamResponseScaledObjectTemplate", emptyUpstreamResponseScaledObjectTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusServiceTemplate", emptyUpstreamPrometheusServiceTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusDeploymentTemplate", emptyUpstreamPrometheusDeploymentTemplate) + KubectlDeleteWithTemplate(t, data, "emptyUpstreamPrometheusConfigMapTemplate", emptyUpstreamPrometheusConfigMapTemplate) + }() + + familyValidator := func(family *prommodel.MetricFamily) bool { + for _, metric := range family.GetMetric() { + labels := metric.GetLabel() + if ExtractPrometheusLabelValue("namespace", labels) == testNamespace && + ExtractPrometheusLabelValue("scaledResource", labels) == emptyUpstreamScaledObjectName && + ExtractPrometheusLabelValue("triggerName", labels) == "empty-upstream-trigger" && + ExtractPrometheusLabelValue("metricName", labels) == "s0-prometheus" && + ExtractPrometheusLabelValue("resourceType", labels) == "ScaledObject" && + ExtractPrometheusLabelValue("ignoreNullValues", labels) == "false" && + metric.GetCounter().GetValue() >= 1 { + return true + } + } + return false + } + + families := WaitForPrometheusMetric(t, "keda_scaler_empty_upstream_responses_total", familyValidator) + metric := families["keda_scaler_empty_upstream_responses_total"] + + assert.True(t, familyValidator(metric)) +}