Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ require (
sigs.k8s.io/yaml v1.4.0 // indirect
)

replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250715103124-73cbc98c25ff
replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250716123832-f18c798bd04b

// Replace memberlist with our fork which includes some fixes that haven't been
// merged upstream yet:
Expand All @@ -372,7 +372,7 @@ replace github.com/opentracing-contrib/go-stdlib => github.com/grafana/opentraci
replace github.com/opentracing-contrib/go-grpc => github.com/charleskorn/go-grpc v0.0.0-20231024023642-e9298576254f

// Replacing prometheus/alertmanager with our fork.
replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36
replace github.com/prometheus/alertmanager => github.com/juliusmh/alertmanager v0.26.1-0.20250716125725-19a9223bec8c

// Use Mimir fork of prometheus/otlptranslator to allow for higher velocity of upstream development,
// while allowing Mimir to move at a more conservative pace.
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -579,14 +579,12 @@ github.com/grafana/memberlist v0.3.1-0.20250428154222-f7d51a6f6700 h1:0t7iOQ5ZkB
github.com/grafana/memberlist v0.3.1-0.20250428154222-f7d51a6f6700/go.mod h1:Ri9p/tRShbjYnpNf4FFPXG7wxEGY4Nrcn6E7jrVa//4=
github.com/grafana/mimir-otlptranslator v0.0.0-20250703083430-c31a9568ad96 h1:kq5zJVW9LyFOB5xCeQPTON2HNjwwEkefhegZXGIhQPk=
github.com/grafana/mimir-otlptranslator v0.0.0-20250703083430-c31a9568ad96/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
github.com/grafana/mimir-prometheus v1.8.2-0.20250715103124-73cbc98c25ff h1:u+lw876tigZBv4/jGIvDmOJ6uLmscQq8HURg38DaldM=
github.com/grafana/mimir-prometheus v1.8.2-0.20250715103124-73cbc98c25ff/go.mod h1:51sCkEHDpexFCGizYi1W4omiFaKThWThkquNPOxhwxQ=
github.com/grafana/mimir-prometheus v1.8.2-0.20250716123832-f18c798bd04b h1:qZM11g9GR8sibnedD0zt56uASxhxjnx/CMP/JAlCRZQ=
github.com/grafana/mimir-prometheus v1.8.2-0.20250716123832-f18c798bd04b/go.mod h1:51sCkEHDpexFCGizYi1W4omiFaKThWThkquNPOxhwxQ=
github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0=
github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8=
github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls=
github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36 h1:AjZ58JRw1ZieFH/SdsddF5BXtsDKt5kSrKNPWrzYz3Y=
github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36/go.mod h1:O/QP1BCm0HHIzbKvgMzqb5sSyH88rzkFk84F4TfJjBU=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grafana/regexp v0.0.0-20240531075221-3685f1377d7b h1:oMAq12GxTpwo9jxbnG/M4F/HdpwbibTaVoxNA0NZprY=
Expand Down Expand Up @@ -721,6 +719,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/juliusmh/alertmanager v0.26.1-0.20250716125725-19a9223bec8c h1:VaNJy7gIh+FGGaDOmViA7bUsLeTVs10tQ5tCVXHbo6c=
github.com/juliusmh/alertmanager v0.26.1-0.20250716125725-19a9223bec8c/go.mod h1:O/QP1BCm0HHIzbKvgMzqb5sSyH88rzkFk84F4TfJjBU=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
Expand Down
8 changes: 2 additions & 6 deletions pkg/distributor/distributor.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,6 @@ import (

var tracer = otel.Tracer("pkg/distributor")

func init() {
// Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet.
// nolint:staticcheck
model.NameValidationScheme = model.LegacyValidation
}

var (
// Validation errors.
errInvalidTenantShardSize = errors.New("invalid tenant shard size, the value must be greater than or equal to zero")
Expand Down Expand Up @@ -1080,6 +1074,8 @@ func (d *Distributor) prePushHaDedupeMiddleware(next PushFunc) PushFunc {
}
}

// prePushRelabelMiddleware relabels the metric. Validation of those labels is
// handled in prePushValidationMiddleware.
func (d *Distributor) prePushRelabelMiddleware(next PushFunc) PushFunc {
return func(ctx context.Context, pushReq *Request) error {
next, maybeCleanup := NextOrCleanup(next, pushReq)
Expand Down
29 changes: 27 additions & 2 deletions pkg/distributor/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ type labelValidationConfig interface {
MaxLabelNamesPerInfoSeries(userID string) int
MaxLabelNameLength(userID string) int
MaxLabelValueLength(userID string) int
ValidationScheme(userID string) model.ValidationScheme
}

func removeNonASCIIChars(in string) (out string) {
Expand Down Expand Up @@ -421,7 +422,9 @@ func validateLabels(m *sampleValidationMetrics, cfg labelValidationConfig, userI
return errors.New(noMetricNameMsgFormat)
}

if !model.IsValidMetricName(model.LabelValue(unsafeMetricName)) {
validationScheme := cfg.ValidationScheme(userID)

if !isValidMetricName(unsafeMetricName, validationScheme) {
cat.IncrementDiscardedSamples(ls, 1, reasonInvalidMetricName, ts)
m.invalidMetricName.WithLabelValues(userID, group).Inc()
return fmt.Errorf(invalidMetricNameMsgFormat, removeNonASCIIChars(unsafeMetricName))
Expand All @@ -447,7 +450,7 @@ func validateLabels(m *sampleValidationMetrics, cfg labelValidationConfig, userI
maxLabelValueLength := cfg.MaxLabelValueLength(userID)
lastLabelName := ""
for _, l := range ls {
if !skipLabelValidation && !model.LabelName(l.Name).IsValid() {
if !skipLabelValidation && !isValidLabelName(l.Name, validationScheme) {
m.invalidLabel.WithLabelValues(userID, group).Inc()
cat.IncrementDiscardedSamples(ls, 1, reasonInvalidLabel, ts)
return fmt.Errorf(invalidLabelMsgFormat, l.Name, mimirpb.FromLabelAdaptersToString(ls))
Expand All @@ -474,6 +477,28 @@ func validateLabels(m *sampleValidationMetrics, cfg labelValidationConfig, userI
return nil
}

func isValidMetricName(name string, validationScheme model.ValidationScheme) bool {
switch validationScheme {
case model.LegacyValidation:
return model.IsValidLegacyMetricName(name)
case model.UTF8Validation:
return model.IsValidMetricName(model.LabelValue(name))
default:
return false
}
}

func isValidLabelName(name string, validationScheme model.ValidationScheme) bool {
switch validationScheme {
case model.LegacyValidation:
return model.LabelName(name).IsValidLegacy()
case model.UTF8Validation:
return model.LabelName(name).IsValid()
default:
return false
}
}

// metadataValidationMetrics is a collection of metrics used by metadata validation.
type metadataValidationMetrics struct {
missingMetricName *prometheus.CounterVec
Expand Down
81 changes: 73 additions & 8 deletions pkg/distributor/validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type validateLabelsCfg struct {
maxLabelNamesPerInfoSeries int
maxLabelNameLength int
maxLabelValueLength int
validationScheme model.ValidationScheme
}

func (v validateLabelsCfg) MaxLabelNamesPerSeries(_ string) int {
Expand All @@ -55,6 +56,10 @@ func (v validateLabelsCfg) MaxLabelValueLength(_ string) int {
return v.maxLabelValueLength
}

func (v validateLabelsCfg) ValidationScheme(_ string) model.ValidationScheme {
return v.validationScheme
}

type validateMetadataCfg struct {
enforceMetadataMetricName bool
maxMetadataLength int
Expand All @@ -73,13 +78,19 @@ func TestValidateLabels(t *testing.T) {
reg := prometheus.NewPedanticRegistry()
s := newSampleValidationMetrics(reg)

var cfg validateLabelsCfg
userID := "testUser"
legacyConfig := validateLabelsCfg{
maxLabelNamesPerSeries: 3,
maxLabelNamesPerInfoSeries: 4,
maxLabelNameLength: 25,
maxLabelValueLength: 25,
validationScheme: model.LegacyValidation,
}
utf8Config := legacyConfig
utf8Config.validationScheme = model.UTF8Validation
unsetConfig := legacyConfig
unsetConfig.validationScheme = model.UnsetValidation

cfg.maxLabelValueLength = 25
cfg.maxLabelNameLength = 25
cfg.maxLabelNamesPerSeries = 3
cfg.maxLabelNamesPerInfoSeries = 4
limits := catestutils.NewMockCostAttributionLimits(0, userID, "team")
careg := prometheus.NewRegistry()
manager, err := costattribution.NewManager(5*time.Second, 10*time.Second, log.NewNopLogger(), limits, reg, careg)
Expand All @@ -90,30 +101,35 @@ func TestValidateLabels(t *testing.T) {
metric model.Metric
skipLabelNameValidation bool
skipLabelCountValidation bool
config validateLabelsCfg
err error
}{
{
metric: map[model.LabelName]model.LabelValue{"team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: errors.New(noMetricNameMsgFormat),
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: " ", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(invalidMetricNameMsgFormat, " "),
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "metric_name_with_\xb0_invalid_utf8_\xb0", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(invalidMetricNameMsgFormat, "metric_name_with__invalid_utf8_ (non-ascii characters removed)"),
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "valid", "foo ": "bar", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
invalidLabelMsgFormat,
"foo ",
Expand All @@ -130,12 +146,14 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "valid", "team": "c"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: nil,
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "badLabelName", "this_is_a_really_really_long_name_that_should_cause_an_error": "test_value_please_ignore", "team": "biz"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
labelNameTooLongMsgFormat,
"this_is_a_really_really_long_name_that_should_cause_an_error",
Expand All @@ -152,6 +170,7 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "badLabelValue", "much_shorter_name": "test_value_please_ignore_no_really_nothing_to_see_here", "team": "biz"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: LabelValueTooLongError{
Label: mimirpb.LabelAdapter{Name: "much_shorter_name", Value: "test_value_please_ignore_no_really_nothing_to_see_here"},
Limit: 25,
Expand All @@ -166,6 +185,7 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "bar": "baz", "blip": "blop", "team": "plof"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
tooManyLabelsMsgFormat,
tooManyLabelsArgs(
Expand All @@ -184,13 +204,15 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo_info", "bar": "baz", "blip": "blop", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: nil,
},
{
// *_info metrics have higher label limits.
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo_info", "bar": "baz", "blip": "blop", "blap": "blup", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
tooManyInfoLabelsMsgFormat,
tooManyLabelsArgs(
Expand All @@ -209,24 +231,28 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "bar": "baz", "blip": "blop", "team": "a"},
skipLabelNameValidation: false,
skipLabelCountValidation: true,
config: legacyConfig,
err: nil,
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "invalid%label&name": "bar", "team": "biz"},
skipLabelNameValidation: true,
skipLabelCountValidation: false,
config: legacyConfig,
err: nil,
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "你好", "team": "plof"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: nil,
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "abc\xfe\xfddef", "team": "plof"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
invalidLabelValueMsgFormat,
"label1", "abc\ufffddef", "foo",
Expand All @@ -236,10 +262,46 @@ func TestValidateLabels(t *testing.T) {
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "abc\xfe\xfddef"},
skipLabelNameValidation: true,
skipLabelCountValidation: false,
config: legacyConfig,
err: nil,
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "name😀": "value", "team": "b"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: legacyConfig,
err: fmt.Errorf(
invalidLabelMsgFormat,
"name😀",
mimirpb.FromLabelAdaptersToString(
[]mimirpb.LabelAdapter{
{Name: model.MetricNameLabel, Value: "foo"},
{Name: "name😀", Value: "value"},
{Name: "team", Value: "b"},
},
),
),
},
// Unset validation scheme results in invalid metric name error
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "name😀": "value", "team": "b"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: unsetConfig,
err: fmt.Errorf(
invalidMetricNameMsgFormat,
"foo",
),
},
{
metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "name😀": "value", "team": "b"},
skipLabelNameValidation: false,
skipLabelCountValidation: false,
config: utf8Config,
err: nil,
},
} {
err := validateLabels(s, cfg, userID, "custom label", mimirpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, c.skipLabelCountValidation, cast, ts)
err := validateLabels(s, c.config, userID, "custom label", mimirpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, c.skipLabelCountValidation, cast, ts)
assert.Equal(t, c.err, err, "wrong error")
}

Expand All @@ -249,13 +311,13 @@ func TestValidateLabels(t *testing.T) {
require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
# HELP cortex_discarded_samples_total The total number of samples that were discarded.
# TYPE cortex_discarded_samples_total counter
cortex_discarded_samples_total{group="custom label",reason="label_invalid",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="label_invalid",user="testUser"} 2
cortex_discarded_samples_total{group="custom label",reason="label_name_too_long",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="label_value_invalid",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="label_value_too_long",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="max_label_names_per_series",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="max_label_names_per_info_series",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="metric_name_invalid",user="testUser"} 2
cortex_discarded_samples_total{group="custom label",reason="metric_name_invalid",user="testUser"} 3
cortex_discarded_samples_total{group="custom label",reason="missing_metric_name",user="testUser"} 1
cortex_discarded_samples_total{group="custom label",reason="random reason",user="different user"} 1
`), "cortex_discarded_samples_total"))
Expand All @@ -264,12 +326,14 @@ func TestValidateLabels(t *testing.T) {
# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
# TYPE cortex_discarded_attributed_samples_total counter
cortex_discarded_attributed_samples_total{reason="label_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="label_invalid",team="b",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="label_name_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="label_value_invalid",team="plof",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="label_value_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="max_label_names_per_info_series",team="a",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="max_label_names_per_series",team="plof",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="metric_name_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 2
cortex_discarded_attributed_samples_total{reason="metric_name_invalid",team="b",tenant="testUser",tracker="cost-attribution"} 1
cortex_discarded_attributed_samples_total{reason="missing_metric_name",team="a",tenant="testUser",tracker="cost-attribution"} 1
`), "cortex_discarded_attributed_samples_total"))

Expand Down Expand Up @@ -451,6 +515,7 @@ func TestValidateLabelDuplication(t *testing.T) {
cfg.maxLabelNameLength = 10
cfg.maxLabelNamesPerSeries = 10
cfg.maxLabelValueLength = 10
cfg.validationScheme = model.UTF8Validation

userID := "testUser"
actual := validateLabels(newSampleValidationMetrics(nil), cfg, userID, "", []mimirpb.LabelAdapter{
Expand Down
7 changes: 0 additions & 7 deletions pkg/frontend/querymiddleware/request_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,8 @@ import (
"net/http"

"github.com/grafana/dskit/cancellation"
"github.com/prometheus/common/model"
)

func init() {
// Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet.
// nolint:staticcheck
model.NameValidationScheme = model.LegacyValidation
}

const requestValidationFailedFmt = "request validation failed for "

var errMetricsQueryRequestValidationFailed = cancellation.NewErrorf(
Expand Down
Loading
Loading