Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WAL reader changes #8568

Merged
merged 3 commits into from
Jul 1, 2024
Merged

WAL reader changes #8568

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d66816d
Renamed `-series-entries` to `-print-series`
pstibrany Jul 1, 2024
3214468
CHANGELOG.md
pstibrany Jul 1, 2024
4adf622
Update tools/wal-reader/main.go
pstibrany Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@

### Tools

* [CHANGE] `wal-reader`: Renamed `-series-entries` to `-print-series`. Renamed `-print-series-with-samples` to `-print-samples`. #8568
* [ENHANCEMENT] `wal-reader`: References to unknown series from Samples, Exemplars, histogram or tombstones records are now always logged. #8568
* [ENHANCEMENT] `tsdb-series`: added `-stats` option to print min/max time of chunks, total number of samples and DPM for each series. #8420

## v2.13.0-rc.0
Expand Down
57 changes: 35 additions & 22 deletions tools/wal-reader/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ func main() {
}

var (
printSeriesEntries bool
printSeriesWithSampleEntries bool
printSeriesEntries bool
printSamples bool
)

flag.BoolVar(&printSeriesEntries, "series-entries", true, "Print series entries")
flag.BoolVar(&printSeriesWithSampleEntries, "print-series-with-samples", false, "Print series information for each sample")
flag.BoolVar(&printSeriesEntries, "print-series", true, "Print series entries")
flag.BoolVar(&printSamples, "print-samples", false, "Print samples, exemplars, metadata and tombstone.")
pstibrany marked this conversation as resolved.
Show resolved Hide resolved
args, err := flagext.ParseFlagsAndArguments(flag.CommandLine)
if err != nil {
log.Fatalln(err.Error())
Expand All @@ -42,7 +42,7 @@ func main() {
log.SetOutput(os.Stdout)

for _, dir := range args {
err := printWal(dir, printSeriesEntries, printSeriesWithSampleEntries)
err := printWal(dir, printSeriesEntries, printSamples)
if err != nil {
log.Fatalln("failed to print WAL from directory", dir, "due to error:", err)
}
Expand Down Expand Up @@ -124,19 +124,16 @@ func formatTimestamp(ts int64) string {
return time.UnixMilli(ts).UTC().Format(timeFormat)
}

func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string, printSeriesEntries, printSeriesWithSampleEntries bool, minSampleTime, maxSampleTime *int64) error {
func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string, printSeriesEntries, printSamples bool, minSampleTime, maxSampleTime *int64) error {
var dec record.Decoder

seriesInfo := func(ref chunks.HeadSeriesRef) string {
if !printSeriesWithSampleEntries {
return ""
seriesInfo := func(segment int, offset int64, recordType string, ref chunks.HeadSeriesRef) string {
seriesInfo, exists := seriesMap[ref]
if !exists {
seriesInfo = "[series not found]"
log.Println("seg:", segment, "off:", offset, recordType, "record for series:", ref, "series not found")
}

ser, found := seriesMap[ref]
if !found {
ser = "[series not found]"
}
return ser
return seriesInfo
}

for r.Next() {
Expand Down Expand Up @@ -170,7 +167,10 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
}

for _, s := range samples {
log.Println("seg:", seg, "off:", off, "samples record:", s.Ref, s.T, formatTimestamp(s.T), s.V, seriesInfo(s.Ref))
si := seriesInfo(seg, off, "samples", s.Ref)
if printSamples {
log.Println("seg:", seg, "off:", off, "samples record:", s.Ref, s.T, formatTimestamp(s.T), s.V, si)
}

*minSampleTime = util_math.Min(s.T, *minSampleTime)
*maxSampleTime = util_math.Max(s.T, *maxSampleTime)
Expand All @@ -182,8 +182,10 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
return &wlog.CorruptionErr{Err: errors.Wrap(err, "decode tombstones"), Segment: r.Segment(), Offset: r.Offset()}
}

for _, s := range tstones {
log.Println("seg:", seg, "off:", off, "tombstones record:", s.Ref, s.Intervals)
if printSamples {
for _, s := range tstones {
log.Println("seg:", seg, "off:", off, "tombstones record:", s.Ref, s.Intervals)
}
}

case record.Exemplars:
Expand All @@ -193,7 +195,10 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
}

for _, s := range exemplars {
log.Println("seg:", seg, "off:", off, "exemplars record:", s.Ref, s.Labels, s.T, formatTimestamp(s.T), s.V, seriesInfo(s.Ref))
si := seriesInfo(seg, off, "samples", s.Ref)
if printSamples {
log.Println("seg:", seg, "off:", off, "exemplars record:", s.Ref, s.Labels, s.T, formatTimestamp(s.T), s.V, si)
}
}

case record.HistogramSamples:
Expand All @@ -203,7 +208,10 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
}

for _, s := range hists {
log.Println("seg:", seg, "off:", off, "histograms record:", s.Ref, s.T, formatTimestamp(s.T), seriesInfo(s.Ref))
si := seriesInfo(seg, off, "samples", s.Ref)
if printSamples {
log.Println("seg:", seg, "off:", off, "histograms record:", s.Ref, s.T, formatTimestamp(s.T), si)
}

*minSampleTime = util_math.Min(s.T, *minSampleTime)
*maxSampleTime = util_math.Max(s.T, *maxSampleTime)
Expand All @@ -216,7 +224,10 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
}

for _, s := range hists {
log.Println("seg:", seg, "off:", off, "float histograms record:", s.Ref, s.T, formatTimestamp(s.T), seriesInfo(s.Ref))
si := seriesInfo(seg, off, "samples", s.Ref)
if printSamples {
log.Println("seg:", seg, "off:", off, "float histograms record:", s.Ref, s.T, formatTimestamp(s.T), si)
}

*minSampleTime = util_math.Min(s.T, *minSampleTime)
*maxSampleTime = util_math.Max(s.T, *maxSampleTime)
Expand All @@ -229,7 +240,9 @@ func printWalEntries(r *wlog.Reader, seriesMap map[chunks.HeadSeriesRef]string,
}

for _, s := range meta {
log.Println("seg:", seg, "off:", off, "metadata:", s.Ref)
if printSamples {
log.Println("seg:", seg, "off:", off, "metadata:", s.Ref)
}
}
default:
if len(rec) < 1 {
Expand Down
Loading