From 500cba79d4a253134b020434f9011018b8c78cae Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Mon, 17 Jun 2024 12:25:53 +0200 Subject: [PATCH] Update mimir-prometheus to 26c8bac (#8391) * Update mimir-prometheus to 26c8bac Updates to https://github.com/grafana/mimir-prometheus/pull/651 Signed-off-by: Oleg Zaytsev * Update CHANGELOG.md Signed-off-by: Oleg Zaytsev --------- Signed-off-by: Oleg Zaytsev --- CHANGELOG.md | 2 + go.mod | 2 +- go.sum | 4 +- pkg/storage/tsdb/block/index.go | 2 +- .../common/helpers/templates/time.go | 89 +++++++++++++++++++ .../prometheus/prometheus/config/config.go | 2 +- .../prometheus/model/labels/regexp.go | 44 ++++++++- .../prometheus/template/template.go | 48 +--------- .../prometheus/prometheus/tsdb/block.go | 10 +-- .../prometheus/prometheus/tsdb/head_read.go | 19 ++-- .../prometheus/prometheus/tsdb/index/index.go | 9 +- .../prometheus/tsdb/ooo_head_read.go | 2 +- .../prometheus/prometheus/tsdb/querier.go | 11 +-- .../prometheus/prometheus/web/api/v1/api.go | 4 +- vendor/modules.txt | 5 +- 15 files changed, 172 insertions(+), 81 deletions(-) create mode 100644 vendor/github.com/prometheus/common/helpers/templates/time.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e016372e2..c9301e04988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,8 @@ * [BUGFIX] Querier: fix edge case where bucket indexes are sometimes cached forever instead of with the expected TTL. #8343 * [BUGFIX] OTLP handler: fix errors returned by OTLP handler when used via httpgrpc tunneling. #8363 * [BUGFIX] Query-frontend: fix `-querier.max-query-lookback` enforcement when `-compactor.blocks-retention-period` is not set, and viceversa. #8388 +* [BUGFIX] Ingester: fix sporadic `not found` error causing an internal server error if label names are queried with matchers during head compaction. #8391 +* [BUGFIX] Ingester, store-gateway: fix case insensitive regular expressions not matching correctly some Unicode characters. #8391 ### Mixin diff --git a/go.mod b/go.mod index 143ddc7c128..3dc96a1b834 100644 --- a/go.mod +++ b/go.mod @@ -265,7 +265,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240611092206-171cafc36ca7 +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240617090647-26c8bacacf85 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: diff --git a/go.sum b/go.sum index 3670e9767be..c436f809d84 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 h1:X8IKQ0wu40wp github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20240611092206-171cafc36ca7 h1:8FxtzVOzgZrcDylRJekgvSHLCxCcG+ntcvYidkCIRLQ= -github.com/grafana/mimir-prometheus v0.0.0-20240611092206-171cafc36ca7/go.mod h1:hvYx75l+0dZUWI20CXlutLQDnyTnB8SXVSMX2sTF3Xo= +github.com/grafana/mimir-prometheus v0.0.0-20240617090647-26c8bacacf85 h1:L6ZXDQjjUfmvMcPdQvnM/GTzDiNG1St2eE6ZSf+lZrk= +github.com/grafana/mimir-prometheus v0.0.0-20240617090647-26c8bacacf85/go.mod h1:ZrBwbXc+KqeAQT4QXHKfi68+7vaOzoSdrkk90RRgdkE= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/prometheus-alertmanager v0.25.1-0.20240605141526-70d9d63f74fc h1:VoEf4wNiS3hCPxxmFdEvyeZJA3eI4Wb4gAlzYZwh52A= diff --git a/pkg/storage/tsdb/block/index.go b/pkg/storage/tsdb/block/index.go index 45cbd51b038..93a43b9940a 100644 --- a/pkg/storage/tsdb/block/index.go +++ b/pkg/storage/tsdb/block/index.go @@ -626,7 +626,7 @@ type indexReader interface { Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, error) LabelValueFor(ctx context.Context, id storage.SeriesRef, label string) (string, error) - LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) + LabelNamesFor(ctx context.Context, p index.Postings) ([]string, error) Close() error } diff --git a/vendor/github.com/prometheus/common/helpers/templates/time.go b/vendor/github.com/prometheus/common/helpers/templates/time.go new file mode 100644 index 00000000000..266c8c992fd --- /dev/null +++ b/vendor/github.com/prometheus/common/helpers/templates/time.go @@ -0,0 +1,89 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package templates + +import ( + "fmt" + "math" + "strconv" + "time" +) + +func convertToFloat(i interface{}) (float64, error) { + switch v := i.(type) { + case float64: + return v, nil + case string: + return strconv.ParseFloat(v, 64) + case int: + return float64(v), nil + case uint: + return float64(v), nil + case int64: + return float64(v), nil + case uint64: + return float64(v), nil + case time.Duration: + return v.Seconds(), nil + default: + return 0, fmt.Errorf("can't convert %T to float", v) + } +} + +func HumanizeDuration(i interface{}) (string, error) { + v, err := convertToFloat(i) + if err != nil { + return "", err + } + + if math.IsNaN(v) || math.IsInf(v, 0) { + return fmt.Sprintf("%.4g", v), nil + } + if v == 0 { + return fmt.Sprintf("%.4gs", v), nil + } + if math.Abs(v) >= 1 { + sign := "" + if v < 0 { + sign = "-" + v = -v + } + duration := int64(v) + seconds := duration % 60 + minutes := (duration / 60) % 60 + hours := (duration / 60 / 60) % 24 + days := duration / 60 / 60 / 24 + // For days to minutes, we display seconds as an integer. + if days != 0 { + return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil + } + if hours != 0 { + return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil + } + if minutes != 0 { + return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil + } + // For seconds, we display 4 significant digits. + return fmt.Sprintf("%s%.4gs", sign, v), nil + } + prefix := "" + for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} { + if math.Abs(v) >= 1 { + break + } + prefix = p + v *= 1000 + } + return fmt.Sprintf("%.4g%ss", v, prefix), nil +} diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index d02081d2e16..9defa10d485 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -154,7 +154,7 @@ var ( DefaultRuntimeConfig = RuntimeConfig{ // Go runtime tuning. - GoGC: 50, + GoGC: 75, } // DefaultScrapeConfig is the default scrape configuration. diff --git a/vendor/github.com/prometheus/prometheus/model/labels/regexp.go b/vendor/github.com/prometheus/prometheus/model/labels/regexp.go index 69540aca354..f8323deb85c 100644 --- a/vendor/github.com/prometheus/prometheus/model/labels/regexp.go +++ b/vendor/github.com/prometheus/prometheus/model/labels/regexp.go @@ -17,12 +17,14 @@ import ( "slices" "strings" "time" + "unicode" "unicode/utf8" "github.com/DmitriyVTitov/size" "github.com/dgraph-io/ristretto" "github.com/grafana/regexp" "github.com/grafana/regexp/syntax" + "golang.org/x/text/unicode/norm" ) const ( @@ -812,7 +814,7 @@ type equalMultiStringMapMatcher struct { func (m *equalMultiStringMapMatcher) add(s string) { if !m.caseSensitive { - s = strings.ToLower(s) + s = toNormalisedLower(s) } m.values[s] = struct{}{} @@ -832,13 +834,51 @@ func (m *equalMultiStringMapMatcher) setMatches() []string { func (m *equalMultiStringMapMatcher) Matches(s string) bool { if !m.caseSensitive { - s = strings.ToLower(s) + s = toNormalisedLower(s) } _, ok := m.values[s] return ok } +// toNormalisedLower normalise the input string using "Unicode Normalization Form D" and then convert +// it to lower case. +func toNormalisedLower(s string) string { + // Check if the string is all ASCII chars and convert any upper case character to lower case character. + isASCII := true + var ( + b strings.Builder + pos int + ) + b.Grow(len(s)) + for i := 0; i < len(s); i++ { + c := s[i] + if isASCII && c >= utf8.RuneSelf { + isASCII = false + break + } + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + if pos < i { + b.WriteString(s[pos:i]) + } + b.WriteByte(c) + pos = i + 1 + } + } + if pos < len(s) { + b.WriteString(s[pos:]) + } + + // Optimize for ASCII-only strings. In this case we don't have to do any normalization. + if isASCII { + return b.String() + } + + // Normalise and convert to lower. + return strings.Map(unicode.ToLower, norm.NFKD.String(b.String())) +} + // anyStringWithoutNewlineMatcher is a stringMatcher which matches any string // (including an empty one) as far as it doesn't contain any newline character. type anyStringWithoutNewlineMatcher struct{} diff --git a/vendor/github.com/prometheus/prometheus/template/template.go b/vendor/github.com/prometheus/prometheus/template/template.go index 43772805cd1..dbe1607cfa7 100644 --- a/vendor/github.com/prometheus/prometheus/template/template.go +++ b/vendor/github.com/prometheus/prometheus/template/template.go @@ -32,6 +32,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + common_templates "github.com/prometheus/common/helpers/templates" + "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) @@ -263,51 +265,7 @@ func NewTemplateExpander( } return fmt.Sprintf("%.4g%s", v, prefix), nil }, - "humanizeDuration": func(i interface{}) (string, error) { - v, err := convertToFloat(i) - if err != nil { - return "", err - } - if math.IsNaN(v) || math.IsInf(v, 0) { - return fmt.Sprintf("%.4g", v), nil - } - if v == 0 { - return fmt.Sprintf("%.4gs", v), nil - } - if math.Abs(v) >= 1 { - sign := "" - if v < 0 { - sign = "-" - v = -v - } - duration := int64(v) - seconds := duration % 60 - minutes := (duration / 60) % 60 - hours := (duration / 60 / 60) % 24 - days := duration / 60 / 60 / 24 - // For days to minutes, we display seconds as an integer. - if days != 0 { - return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil - } - if hours != 0 { - return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil - } - if minutes != 0 { - return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil - } - // For seconds, we display 4 significant digits. - return fmt.Sprintf("%s%.4gs", sign, v), nil - } - prefix := "" - for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} { - if math.Abs(v) >= 1 { - break - } - prefix = p - v *= 1000 - } - return fmt.Sprintf("%.4g%ss", v, prefix), nil - }, + "humanizeDuration": common_templates.HumanizeDuration, "humanizePercentage": func(i interface{}) (string, error) { v, err := convertToFloat(i) if err != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/block.go b/vendor/github.com/prometheus/prometheus/tsdb/block.go index a3af36ebcde..e25bb181353 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/block.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/block.go @@ -117,9 +117,9 @@ type IndexReader interface { // This is useful for obtaining label values for other postings than the ones you wish to exclude. LabelValuesExcluding(p index.Postings, name string) storage.LabelValues - // LabelNamesFor returns all the label names for the series referred to by IDs. + // LabelNamesFor returns all the label names for the series referred to by the postings. // The names returned are sorted. - LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) + LabelNamesFor(ctx context.Context, postings index.Postings) ([]string, error) // Close releases the underlying resources of the reader. Close() error @@ -590,10 +590,10 @@ func (r blockIndexReader) LabelValueFor(ctx context.Context, id storage.SeriesRe return r.ir.LabelValueFor(ctx, id, label) } -// LabelNamesFor returns all the label names for the series referred to by IDs. +// LabelNamesFor returns all the label names for the series referred to by the postings. // The names returned are sorted. -func (r blockIndexReader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) { - return r.ir.LabelNamesFor(ctx, ids...) +func (r blockIndexReader) LabelNamesFor(ctx context.Context, postings index.Postings) ([]string, error) { + return r.ir.LabelNamesFor(ctx, postings) } type blockTombstoneReader struct { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head_read.go b/vendor/github.com/prometheus/prometheus/tsdb/head_read.go index 6a483202d8e..b8aa574f0e7 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head_read.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head_read.go @@ -282,22 +282,29 @@ func (h *headIndexReader) LabelValueFor(_ context.Context, id storage.SeriesRef, return value, nil } -// LabelNamesFor returns all the label names for the series referred to by IDs. +// LabelNamesFor returns all the label names for the series referred to by the postings. // The names returned are sorted. -func (h *headIndexReader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) { +func (h *headIndexReader) LabelNamesFor(ctx context.Context, series index.Postings) ([]string, error) { namesMap := make(map[string]struct{}) - for _, id := range ids { - if ctx.Err() != nil { + i := 0 + for series.Next() { + i++ + if i%checkContextEveryNIterations == 0 && ctx.Err() != nil { return nil, ctx.Err() } - memSeries := h.head.series.getByID(chunks.HeadSeriesRef(id)) + memSeries := h.head.series.getByID(chunks.HeadSeriesRef(series.At())) if memSeries == nil { - return nil, storage.ErrNotFound + // Series not found, this happens during compaction, + // when series was garbage collected after the caller got the series IDs. + continue } memSeries.lset.Range(func(lbl labels.Label) { namesMap[lbl.Name] = struct{}{} }) } + if err := series.Err(); err != nil { + return nil, err + } names := make([]string, 0, len(namesMap)) for name := range namesMap { names = append(names, name) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go index eb4ddf3ac69..0d8cb163d65 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go @@ -1570,11 +1570,14 @@ func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labe // LabelNamesFor returns all the label names for the series referred to by IDs. // The names returned are sorted. -func (r *Reader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) { +func (r *Reader) LabelNamesFor(ctx context.Context, postings Postings) ([]string, error) { // Gather offsetsMap the name offsetsMap in the symbol table first offsetsMap := make(map[uint32]struct{}) - for _, id := range ids { - if ctx.Err() != nil { + i := 0 + for postings.Next() { + id := postings.At() + + if i%checkContextEveryNIterations == 0 && ctx.Err() != nil { return nil, ctx.Err() } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go index 819d04df4fd..acc0d45f7fb 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go @@ -497,7 +497,7 @@ func (ir *OOOCompactionHeadIndexReader) LabelValueFor(context.Context, storage.S return "", errors.New("not implemented") } -func (ir *OOOCompactionHeadIndexReader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([]string, error) { +func (ir *OOOCompactionHeadIndexReader) LabelNamesFor(ctx context.Context, postings index.Postings) ([]string, error) { return nil, errors.New("not implemented") } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index 2c40153609d..888c239e238 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -557,16 +557,7 @@ func labelNamesWithMatchers(ctx context.Context, r IndexReader, matchers ...*lab if err != nil { return nil, err } - - var postings []storage.SeriesRef - for p.Next() { - postings = append(postings, p.At()) - } - if err := p.Err(); err != nil { - return nil, fmt.Errorf("postings for label names with matchers: %w", err) - } - - return r.LabelNamesFor(ctx, postings...) + return r.LabelNamesFor(ctx, p) } // seriesData, used inside other iterators, are updated when we move from one series to another. diff --git a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go index f0884926e11..b95ff25cf9d 100644 --- a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go +++ b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go @@ -1761,7 +1761,7 @@ func (api *API) respond(w http.ResponseWriter, req *http.Request, data interface b, err := codec.Encode(resp) if err != nil { - level.Error(api.logger).Log("msg", "error marshaling response", "err", err) + level.Error(api.logger).Log("msg", "error marshaling response", "url", req.URL, "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -1769,7 +1769,7 @@ func (api *API) respond(w http.ResponseWriter, req *http.Request, data interface w.Header().Set("Content-Type", codec.ContentType().String()) w.WriteHeader(http.StatusOK) if n, err := w.Write(b); err != nil { - level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err) + level.Error(api.logger).Log("msg", "error writing response", "url", req.URL, "bytesWritten", n, "err", err) } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 58a276fbad7..7cb821f3c61 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -958,6 +958,7 @@ github.com/prometheus/client_model/go ## explicit; go 1.20 github.com/prometheus/common/config github.com/prometheus/common/expfmt +github.com/prometheus/common/helpers/templates github.com/prometheus/common/model github.com/prometheus/common/route github.com/prometheus/common/version @@ -972,7 +973,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240611092206-171cafc36ca7 +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240617090647-26c8bacacf85 ## explicit; go 1.21 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1608,7 +1609,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240611092206-171cafc36ca7 +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240617090647-26c8bacacf85 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240531075221-3685f1377d7b