From 6fb9752dfd399fdee5a460140a4946470e77a672 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 16 Oct 2021 10:15:16 +0530 Subject: [PATCH 01/67] fixed template; export metadata Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 30 ++++++++++++++++++++++++++++++ pkg/compact/compact.go | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index b4ff8f5690..59e6bab24b 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -18,6 +18,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" + "github.com/oklog/ulid" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -458,6 +459,35 @@ func runCompact( return cleanPartialMarked() } + g.Add(func() error { + _ = sy.SyncMetas(context.Background()) + originalMetas := sy.Metas() + + // figure out hasPlan and noPlan + for { + groups, _ := grouper.Groups(originalMetas) + for _, g := range groups { + // parameter should be of type tsdb.BlockMeata.meta + plan, _ := planner.Plan(context.Background(), g.Metadata()) + if len(plan) == 0 { + continue + } + var metas []*tsdb.BlockMeta + for _, p := range plan { + metas = append(metas, &p.BlockMeta) + } + newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) + g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) + } + + // remove 'plan' blocks from 'original metadata' + } + + return nil + }, func(err error) { + cancel() + }) + g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 547eae57f2..77d0072599 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -402,6 +402,10 @@ func (cg *Group) Key() string { return cg.key } +func (cg *Group) Metadata() []*metadata.Meta { + return cg.metasByMinTime +} + // AppendMeta the block with the given meta to the group. func (cg *Group) AppendMeta(meta *metadata.Meta) error { cg.mtx.Lock() From f403fcce11a45c62ddd4e19f6a5764e49b3f484f Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 16 Oct 2021 18:16:17 +0530 Subject: [PATCH 02/67] added some error handling Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 59e6bab24b..a884cb0770 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -460,15 +460,23 @@ func runCompact( } g.Add(func() error { - _ = sy.SyncMetas(context.Background()) + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") + } originalMetas := sy.Metas() // figure out hasPlan and noPlan for { - groups, _ := grouper.Groups(originalMetas) + groups, err := grouper.Groups(originalMetas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } for _, g := range groups { - // parameter should be of type tsdb.BlockMeata.meta - plan, _ := planner.Plan(context.Background(), g.Metadata()) + // parameter should be of type tsdb.BlockMeta.meta + plan, err := planner.Plan(context.Background(), g.Metadata()) + if err != nil { + return errors.Wrapf(err, "could not plan") + } if len(plan) == 0 { continue } @@ -478,9 +486,11 @@ func runCompact( } newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) + + // remove 'plan' blocks from 'original metadata' + } - // remove 'plan' blocks from 'original metadata' } return nil From 02d7037e18710a67452c3eeb1e35d06c7fa87f59 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 16 Oct 2021 20:08:23 +0530 Subject: [PATCH 03/67] remove planned blocks from original Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index a884cb0770..39163c220e 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -458,8 +458,7 @@ func runCompact( return cleanPartialMarked() } - - g.Add(func() error { + g.Add(func() error { if err := sy.SyncMetas(context.Background()); err != nil { return errors.Wrapf(err, "could not sync metas") } @@ -480,17 +479,22 @@ func runCompact( if len(plan) == 0 { continue } + + var toRemove []ulid.ULID var metas []*tsdb.BlockMeta for _, p := range plan { metas = append(metas, &p.BlockMeta) + toRemove = append(toRemove, p.BlockMeta.ULID) } - newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) - g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) - // remove 'plan' blocks from 'original metadata' + // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? + for _, meta := range toRemove { + delete(originalMetas, meta.BlockMeta.ULID) + } + newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) + g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) } - } return nil From ea832503922aa360ddeba86f6bee50b8f3a1ae32 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sun, 17 Oct 2021 18:35:15 +0530 Subject: [PATCH 04/67] moved to pkg/compact Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 48 +++++++------------------------ pkg/compact/compact.go | 65 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 38 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 39163c220e..c932f8455e 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -18,7 +18,6 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" - "github.com/oklog/ulid" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -84,6 +83,9 @@ func (cs compactionSet) maxLevel() int { return len(cs) - 1 } +// add recvr func to planning struct to modify metadata +// move to pkg/compact later + func registerCompact(app *extkingpin.App) { cmd := app.Command(component.Compact.String(), "Continuously compacts blocks in an object store bucket.") conf := &compactConfig{} @@ -458,44 +460,14 @@ func runCompact( return cleanPartialMarked() } - g.Add(func() error { - if err := sy.SyncMetas(context.Background()); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - originalMetas := sy.Metas() - - // figure out hasPlan and noPlan - for { - groups, err := grouper.Groups(originalMetas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } - for _, g := range groups { - // parameter should be of type tsdb.BlockMeta.meta - plan, err := planner.Plan(context.Background(), g.Metadata()) - if err != nil { - return errors.Wrapf(err, "could not plan") - } - if len(plan) == 0 { - continue - } - - var toRemove []ulid.ULID - var metas []*tsdb.BlockMeta - for _, p := range plan { - metas = append(metas, &p.BlockMeta) - toRemove = append(toRemove, p.BlockMeta.ULID) - } - - // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? - for _, meta := range toRemove { - delete(originalMetas, meta.BlockMeta.ULID) - } - - newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) - g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) - } + g.Add(func() error { + ps := compact.NewDefaultPlanSim(grouper, planner, sy) + numberOfIterations, numberOfBlocksMerged, err := ps.PlanProgressCalc(context.Background()) + if err != nil { + return errors.Wrapf(err, "could not simulate planning") } + level.Debug(logger).Log("msg", "number of iterations performed","count", numberOfIterations) + level.Debug(logger).Log("msg", "number of blocks merged", "count",numberOfBlocksMerged) return nil }, func(err error) { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 77d0072599..2f84dbf070 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -474,6 +474,71 @@ func (cg *Group) Resolution() int64 { return cg.resolution } +// should return the results/metrics of the planning simulation +type PlanSim interface { + PlanProgressCalc(ctx context.Context) (int, int, error) +} + +type DefaultPlanSim struct { + grouper Grouper + planner Planner + sy *Syncer +} + +func NewDefaultPlanSim(grouper Grouper, planner Planner, sy *Syncer) *DefaultPlanSim { + return &DefaultPlanSim{ + grouper: grouper, + planner: planner, + sy: sy, + } +} + +func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) (int, int, error) { + numberOfIterations := 0 + numberOfBlocksToMerge := 0 + + if err := ps.sy.SyncMetas(ctx); err != nil { + return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not sync metas") + } + originalMetas := ps.sy.Metas() + + groups, err := ps.grouper.Groups(originalMetas) + if err != nil { + return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not group original metadata") + } + + // figure out hasPlan and noPlan + for { + for _, g := range groups { + // parameter should be of type tsdb.BlockMeta.meta + plan, err := ps.planner.Plan(ctx, g.Metadata()) + if err != nil { + return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not plan") + } + if len(plan) == 0 { + continue + } + numberOfIterations++ + + var toRemove []ulid.ULID + var metas []*tsdb.BlockMeta + for _, p := range plan { + metas = append(metas, &p.BlockMeta) + toRemove = append(toRemove, p.BlockMeta.ULID) + } + numberOfBlocksToMerge += len(plan) + + // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? + // not required to modify originalMeta now + + newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) + g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) + } + } + + return numberOfIterations, numberOfBlocksToMerge, nil +} + // Planner returns blocks to compact. type Planner interface { // Plan returns a list of blocks that should be compacted into single one. From ff2e36a6a31052e4160cafc063644f7050d6ad93 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 08:43:25 +0530 Subject: [PATCH 05/67] draft: export metrics to Prom Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 6 ++---- pkg/compact/compact.go | 44 ++++++++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c932f8455e..48b9bf2164 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -461,13 +461,11 @@ func runCompact( return cleanPartialMarked() } g.Add(func() error { - ps := compact.NewDefaultPlanSim(grouper, planner, sy) - numberOfIterations, numberOfBlocksMerged, err := ps.PlanProgressCalc(context.Background()) + ps := compact.NewDefaultPlanSim(grouper, planner, sy, reg) + err := ps.PlanProgressCalc(context.Background()) if err != nil { return errors.Wrapf(err, "could not simulate planning") } - level.Debug(logger).Log("msg", "number of iterations performed","count", numberOfIterations) - level.Debug(logger).Log("msg", "number of blocks merged", "count",numberOfBlocksMerged) return nil }, func(err error) { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 2f84dbf070..ec4e05c09a 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -476,49 +476,64 @@ func (cg *Group) Resolution() int64 { // should return the results/metrics of the planning simulation type PlanSim interface { - PlanProgressCalc(ctx context.Context) (int, int, error) + PlanProgressCalc(ctx context.Context) error } type DefaultPlanSim struct { - grouper Grouper - planner Planner - sy *Syncer + grouper Grouper + planner Planner + sy *Syncer + numberOfIterations prometheus.Gauge + numberOfBlocksToMerge prometheus.Gauge } -func NewDefaultPlanSim(grouper Grouper, planner Planner, sy *Syncer) *DefaultPlanSim { +func NewDefaultPlanSim(grouper Grouper, planner Planner, sy *Syncer, reg prometheus.Registerer) *DefaultPlanSim { return &DefaultPlanSim{ grouper: grouper, planner: planner, sy: sy, + numberOfIterations: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "thanos_number_of_compaction_iterations", + Help: "The number of compactions to do", + }), + numberOfBlocksToMerge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "thanos_number_of_blocks_to_merge", + Help: "The number of blocks to be merged", + }), } } -func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) (int, int, error) { - numberOfIterations := 0 - numberOfBlocksToMerge := 0 +func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) error { + iterations := 0 + blocksToMerge := 0 if err := ps.sy.SyncMetas(ctx); err != nil { - return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not sync metas") + return errors.Wrapf(err, "could not sync metas") } originalMetas := ps.sy.Metas() groups, err := ps.grouper.Groups(originalMetas) if err != nil { - return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not group original metadata") + return errors.Wrapf(err, "could not group original metadata") } // figure out hasPlan and noPlan for { for _, g := range groups { + if len(g.IDs()) == 1 { + continue + } + // parameter should be of type tsdb.BlockMeta.meta plan, err := ps.planner.Plan(ctx, g.Metadata()) if err != nil { - return numberOfIterations, numberOfBlocksToMerge, errors.Wrapf(err, "could not plan") + return errors.Wrapf(err, "could not plan") } if len(plan) == 0 { continue } - numberOfIterations++ + iterations++ + ps.numberOfIterations.Set(float64(iterations)) var toRemove []ulid.ULID var metas []*tsdb.BlockMeta @@ -526,7 +541,8 @@ func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) (int, int, error metas = append(metas, &p.BlockMeta) toRemove = append(toRemove, p.BlockMeta.ULID) } - numberOfBlocksToMerge += len(plan) + blocksToMerge += len(plan) + ps.numberOfBlocksToMerge.Set(float64(blocksToMerge)) // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? // not required to modify originalMeta now @@ -536,7 +552,7 @@ func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) (int, int, error } } - return numberOfIterations, numberOfBlocksToMerge, nil + return nil } // Planner returns blocks to compact. From 596155286cd7171c5f5a913233250fdd5417a32b Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 09:43:32 +0530 Subject: [PATCH 06/67] update metrics after entire plan sim; change func name Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 48b9bf2164..1e0b686ad8 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -462,7 +462,7 @@ func runCompact( } g.Add(func() error { ps := compact.NewDefaultPlanSim(grouper, planner, sy, reg) - err := ps.PlanProgressCalc(context.Background()) + err := ps.ProgressCalculate(context.Background()) if err != nil { return errors.Wrapf(err, "could not simulate planning") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index ec4e05c09a..ec2084882d 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -406,6 +406,9 @@ func (cg *Group) Metadata() []*metadata.Meta { return cg.metasByMinTime } +func (cg *Group) deleteFromGroup(target ulid.ULID) { +} + // AppendMeta the block with the given meta to the group. func (cg *Group) AppendMeta(meta *metadata.Meta) error { cg.mtx.Lock() @@ -476,7 +479,7 @@ func (cg *Group) Resolution() int64 { // should return the results/metrics of the planning simulation type PlanSim interface { - PlanProgressCalc(ctx context.Context) error + ProgressCalculate(ctx context.Context) error } type DefaultPlanSim struct { @@ -503,7 +506,7 @@ func NewDefaultPlanSim(grouper Grouper, planner Planner, sy *Syncer, reg prometh } } -func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) error { +func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { iterations := 0 blocksToMerge := 0 @@ -533,7 +536,6 @@ func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) error { continue } iterations++ - ps.numberOfIterations.Set(float64(iterations)) var toRemove []ulid.ULID var metas []*tsdb.BlockMeta @@ -542,7 +544,6 @@ func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) error { toRemove = append(toRemove, p.BlockMeta.ULID) } blocksToMerge += len(plan) - ps.numberOfBlocksToMerge.Set(float64(blocksToMerge)) // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? // not required to modify originalMeta now @@ -552,6 +553,11 @@ func (ps *DefaultPlanSim) PlanProgressCalc(ctx context.Context) error { } } + // updated only once here - after the entire planning simulation is completed + // updating the exposed metrics inside the loop will change based on iterations needed for each plan loop + ps.numberOfIterations.Set(float64(iterations)) + ps.numberOfBlocksToMerge.Set(float64(blocksToMerge)) + return nil } From a2b2b97f5768d2d77b8240af8a0c099fb11b7ff6 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 09:52:31 +0530 Subject: [PATCH 07/67] added method to delete metas from group Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index ec2084882d..7be5215150 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -406,7 +406,15 @@ func (cg *Group) Metadata() []*metadata.Meta { return cg.metasByMinTime } -func (cg *Group) deleteFromGroup(target ulid.ULID) { +func (cg *Group) deleteFromGroup(target map[ulid.ULID]bool) { + var newGroupMeta []*metadata.Meta + for _, meta := range cg.metasByMinTime { + if target[meta.BlockMeta.ULID] { + newGroupMeta = append(newGroupMeta, meta) + } + } + + cg.metasByMinTime = newGroupMeta } // AppendMeta the block with the given meta to the group. @@ -537,12 +545,14 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { } iterations++ - var toRemove []ulid.ULID + var toRemove map[ulid.ULID]bool var metas []*tsdb.BlockMeta for _, p := range plan { metas = append(metas, &p.BlockMeta) - toRemove = append(toRemove, p.BlockMeta.ULID) + toRemove[p.BlockMeta.ULID] = true } + g.deleteFromGroup(toRemove) + blocksToMerge += len(plan) // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? From 8f13acc1722c4d05c3c0b841ccfb2a257c35dc42 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 09:56:29 +0530 Subject: [PATCH 08/67] preallocate meta slice Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 7be5215150..0a1d93d71a 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -546,7 +546,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { iterations++ var toRemove map[ulid.ULID]bool - var metas []*tsdb.BlockMeta + metas := make([]*tsdb.BlockMeta, 0, len(plan)) for _, p := range plan { metas = append(metas, &p.BlockMeta) toRemove[p.BlockMeta.ULID] = true From f52e056205e8f8b2dcf6d9f2e609222264e3a6ef Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 10:57:16 +0530 Subject: [PATCH 09/67] changed func signature; started adding metrics to Group Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 19 ++++++++---- pkg/compact/compact.go | 65 ++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1e0b686ad8..0eb09992f7 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -83,9 +83,6 @@ func (cs compactionSet) maxLevel() int { return len(cs) - 1 } -// add recvr func to planning struct to modify metadata -// move to pkg/compact later - func registerCompact(app *extkingpin.App) { cmd := app.Command(component.Compact.String(), "Continuously compacts blocks in an object store bucket.") conf := &compactConfig{} @@ -345,6 +342,7 @@ func runCompact( return errors.Wrap(err, "create working downsample directory") } + var numberOfIterations, numberOfBlocks prometheus.Gauge grouper := compact.NewDefaultGrouper( logger, bkt, @@ -355,6 +353,8 @@ func runCompact( compactMetrics.garbageCollectedBlocks, compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason), metadata.HashFunc(conf.hashFunc), + numberOfIterations, + numberOfBlocks, ) planner := compact.WithLargeTotalIndexSizeFilter( compact.NewPlanner(logger, levels, noCompactMarkerFilter), @@ -461,9 +461,18 @@ func runCompact( return cleanPartialMarked() } g.Add(func() error { - ps := compact.NewDefaultPlanSim(grouper, planner, sy, reg) - err := ps.ProgressCalculate(context.Background()) + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") + } + originalMetas := sy.Metas() + + groups, err := grouper.Groups(originalMetas) if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } + + var ps compact.DefaultPlanSim + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 0a1d93d71a..c53ff111fe 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -242,6 +242,8 @@ type DefaultGrouper struct { blocksMarkedForDeletion prometheus.Counter blocksMarkedForNoCompact prometheus.Counter hashFunc metadata.HashFunc + numberOfIterations prometheus.Gauge + numberOfBlocksToMerge prometheus.Gauge } // NewDefaultGrouper makes a new DefaultGrouper. @@ -255,6 +257,8 @@ func NewDefaultGrouper( garbageCollectedBlocks prometheus.Counter, blocksMarkedForNoCompact prometheus.Counter, hashFunc metadata.HashFunc, + numberOfIterations prometheus.Gauge, + numberOfBlocksToMerge prometheus.Gauge, ) *DefaultGrouper { return &DefaultGrouper{ bkt: bkt, @@ -285,6 +289,14 @@ func NewDefaultGrouper( garbageCollectedBlocks: garbageCollectedBlocks, blocksMarkedForDeletion: blocksMarkedForDeletion, hashFunc: hashFunc, + numberOfIterations: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "thanos_number_of_compaction_iterations", + Help: "The number of compactions to do", + }), + numberOfBlocksToMerge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "thanos_number_of_blocks_to_merge", + Help: "The number of blocks to be merged", + }), } } @@ -314,6 +326,8 @@ func (g *DefaultGrouper) Groups(blocks map[ulid.ULID]*metadata.Meta) (res []*Gro g.blocksMarkedForDeletion, g.blocksMarkedForNoCompact, g.hashFunc, + g.numberOfIterations, + g.numberOfBlocksToMerge, ) if err != nil { return nil, errors.Wrap(err, "create compaction group") @@ -352,6 +366,9 @@ type Group struct { blocksMarkedForDeletion prometheus.Counter blocksMarkedForNoCompact prometheus.Counter hashFunc metadata.HashFunc + + numberOfIterations prometheus.Gauge + numberOfBlocksToMerge prometheus.Gauge } // NewGroup returns a new compaction group. @@ -372,6 +389,8 @@ func NewGroup( blocksMarkedForDeletion prometheus.Counter, blocksMarkedForNoCompact prometheus.Counter, hashFunc metadata.HashFunc, + numberOfIterations prometheus.Gauge, + numberOfBlocksToMerge prometheus.Gauge, ) (*Group, error) { if logger == nil { logger = log.NewNopLogger() @@ -393,6 +412,8 @@ func NewGroup( blocksMarkedForDeletion: blocksMarkedForDeletion, blocksMarkedForNoCompact: blocksMarkedForNoCompact, hashFunc: hashFunc, + numberOfIterations: numberOfIterations, + numberOfBlocksToMerge: numberOfBlocksToMerge, } return g, nil } @@ -402,10 +423,6 @@ func (cg *Group) Key() string { return cg.key } -func (cg *Group) Metadata() []*metadata.Meta { - return cg.metasByMinTime -} - func (cg *Group) deleteFromGroup(target map[ulid.ULID]bool) { var newGroupMeta []*metadata.Meta for _, meta := range cg.metasByMinTime { @@ -487,47 +504,17 @@ func (cg *Group) Resolution() int64 { // should return the results/metrics of the planning simulation type PlanSim interface { - ProgressCalculate(ctx context.Context) error + ProgressCalculate(ctx context.Context, groups []*Group) error } type DefaultPlanSim struct { - grouper Grouper - planner Planner - sy *Syncer - numberOfIterations prometheus.Gauge - numberOfBlocksToMerge prometheus.Gauge + planner Planner } -func NewDefaultPlanSim(grouper Grouper, planner Planner, sy *Syncer, reg prometheus.Registerer) *DefaultPlanSim { - return &DefaultPlanSim{ - grouper: grouper, - planner: planner, - sy: sy, - numberOfIterations: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "thanos_number_of_compaction_iterations", - Help: "The number of compactions to do", - }), - numberOfBlocksToMerge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "thanos_number_of_blocks_to_merge", - Help: "The number of blocks to be merged", - }), - } -} - -func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { +func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { iterations := 0 blocksToMerge := 0 - if err := ps.sy.SyncMetas(ctx); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - originalMetas := ps.sy.Metas() - - groups, err := ps.grouper.Groups(originalMetas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } - // figure out hasPlan and noPlan for { for _, g := range groups { @@ -536,7 +523,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { } // parameter should be of type tsdb.BlockMeta.meta - plan, err := ps.planner.Plan(ctx, g.Metadata()) + plan, err := ps.planner.Plan(ctx, g.metasByMinTime) if err != nil { return errors.Wrapf(err, "could not plan") } @@ -565,8 +552,6 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context) error { // updated only once here - after the entire planning simulation is completed // updating the exposed metrics inside the loop will change based on iterations needed for each plan loop - ps.numberOfIterations.Set(float64(iterations)) - ps.numberOfBlocksToMerge.Set(float64(blocksToMerge)) return nil } From ad639c1e791694ee84f87a8eb6e04e5eecd867a6 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 14:19:47 +0530 Subject: [PATCH 10/67] added termination condition Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index c53ff111fe..f1f6b15d80 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -515,8 +515,8 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group iterations := 0 blocksToMerge := 0 - // figure out hasPlan and noPlan - for { + hasPlan := true + for hasPlan { for _, g := range groups { if len(g.IDs()) == 1 { continue @@ -545,6 +545,13 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? // not required to modify originalMeta now + if len(g.metasByMinTime) == 0 { + hasPlan = false + break + // no plan in case the group is empty after removing the 'planned' metadata + // group size will remain one even after newMeta is added, hence, no plan needed for this case + } + newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) } From 845d49725d0c6c5bdfce5f6902770939b12004c3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 18 Oct 2021 17:24:19 +0530 Subject: [PATCH 11/67] metrics struct added Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 7 ++----- pkg/compact/compact.go | 44 ++++++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 0eb09992f7..c35cce47e0 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -342,7 +342,6 @@ func runCompact( return errors.Wrap(err, "create working downsample directory") } - var numberOfIterations, numberOfBlocks prometheus.Gauge grouper := compact.NewDefaultGrouper( logger, bkt, @@ -353,8 +352,6 @@ func runCompact( compactMetrics.garbageCollectedBlocks, compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason), metadata.HashFunc(conf.hashFunc), - numberOfIterations, - numberOfBlocks, ) planner := compact.WithLargeTotalIndexSizeFilter( compact.NewPlanner(logger, levels, noCompactMarkerFilter), @@ -471,8 +468,8 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - var ps compact.DefaultPlanSim - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + ps := compact.NewDefaultPlanSim(reg) + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index f1f6b15d80..4925c4f8a9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -242,8 +242,6 @@ type DefaultGrouper struct { blocksMarkedForDeletion prometheus.Counter blocksMarkedForNoCompact prometheus.Counter hashFunc metadata.HashFunc - numberOfIterations prometheus.Gauge - numberOfBlocksToMerge prometheus.Gauge } // NewDefaultGrouper makes a new DefaultGrouper. @@ -257,8 +255,6 @@ func NewDefaultGrouper( garbageCollectedBlocks prometheus.Counter, blocksMarkedForNoCompact prometheus.Counter, hashFunc metadata.HashFunc, - numberOfIterations prometheus.Gauge, - numberOfBlocksToMerge prometheus.Gauge, ) *DefaultGrouper { return &DefaultGrouper{ bkt: bkt, @@ -289,14 +285,6 @@ func NewDefaultGrouper( garbageCollectedBlocks: garbageCollectedBlocks, blocksMarkedForDeletion: blocksMarkedForDeletion, hashFunc: hashFunc, - numberOfIterations: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "thanos_number_of_compaction_iterations", - Help: "The number of compactions to do", - }), - numberOfBlocksToMerge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "thanos_number_of_blocks_to_merge", - Help: "The number of blocks to be merged", - }), } } @@ -326,8 +314,6 @@ func (g *DefaultGrouper) Groups(blocks map[ulid.ULID]*metadata.Meta) (res []*Gro g.blocksMarkedForDeletion, g.blocksMarkedForNoCompact, g.hashFunc, - g.numberOfIterations, - g.numberOfBlocksToMerge, ) if err != nil { return nil, errors.Wrap(err, "create compaction group") @@ -366,9 +352,6 @@ type Group struct { blocksMarkedForDeletion prometheus.Counter blocksMarkedForNoCompact prometheus.Counter hashFunc metadata.HashFunc - - numberOfIterations prometheus.Gauge - numberOfBlocksToMerge prometheus.Gauge } // NewGroup returns a new compaction group. @@ -389,8 +372,6 @@ func NewGroup( blocksMarkedForDeletion prometheus.Counter, blocksMarkedForNoCompact prometheus.Counter, hashFunc metadata.HashFunc, - numberOfIterations prometheus.Gauge, - numberOfBlocksToMerge prometheus.Gauge, ) (*Group, error) { if logger == nil { logger = log.NewNopLogger() @@ -412,8 +393,6 @@ func NewGroup( blocksMarkedForDeletion: blocksMarkedForDeletion, blocksMarkedForNoCompact: blocksMarkedForNoCompact, hashFunc: hashFunc, - numberOfIterations: numberOfIterations, - numberOfBlocksToMerge: numberOfBlocksToMerge, } return g, nil } @@ -502,6 +481,13 @@ func (cg *Group) Resolution() int64 { return cg.resolution } +// metrics related to planning and compaction progress +type ProgressMetrics struct { + NumberOfIterations *prometheus.GaugeVec + numberOfBlocksToMerge *prometheus.GaugeVec + // figure out where to add groupKey labels to this +} + // should return the results/metrics of the planning simulation type PlanSim interface { ProgressCalculate(ctx context.Context, groups []*Group) error @@ -509,6 +495,22 @@ type PlanSim interface { type DefaultPlanSim struct { planner Planner + ProgressMetrics +} + +func NewDefaultPlanSim(reg prometheus.Registerer) *DefaultPlanSim { + return &DefaultPlanSim{ + ProgressMetrics: ProgressMetrics{ + NumberOfIterations: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_number_of_iterations", + Help: "number of iterations to be done", + }, []string{"group"}), + numberOfBlocksToMerge: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_number_of_blocks_planned", + Help: "number of blocks planned to be merged", + }, []string{"group"}), + }, + } } func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { From cc4efe4af3d26f8b971562e18f129719fd57735c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 19 Oct 2021 08:38:09 +0530 Subject: [PATCH 12/67] draft: updating progress metric Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 4925c4f8a9..031a84ecc8 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -484,7 +484,7 @@ func (cg *Group) Resolution() int64 { // metrics related to planning and compaction progress type ProgressMetrics struct { NumberOfIterations *prometheus.GaugeVec - numberOfBlocksToMerge *prometheus.GaugeVec + NumberOfBlocksToMerge *prometheus.GaugeVec // figure out where to add groupKey labels to this } @@ -505,7 +505,7 @@ func NewDefaultPlanSim(reg prometheus.Registerer) *DefaultPlanSim { Name: "thanos_number_of_iterations", Help: "number of iterations to be done", }, []string{"group"}), - numberOfBlocksToMerge: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + NumberOfBlocksToMerge: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_blocks_planned", Help: "number of blocks planned to be merged", }, []string{"group"}), @@ -516,6 +516,7 @@ func NewDefaultPlanSim(reg prometheus.Registerer) *DefaultPlanSim { func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { iterations := 0 blocksToMerge := 0 + var groupCompactions, groupBlocks map[string]int hasPlan := true for hasPlan { @@ -533,6 +534,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group continue } iterations++ + groupCompactions[g.key] = iterations var toRemove map[ulid.ULID]bool metas := make([]*tsdb.BlockMeta, 0, len(plan)) @@ -543,6 +545,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group g.deleteFromGroup(toRemove) blocksToMerge += len(plan) + groupBlocks[g.key] = blocksToMerge // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? // not required to modify originalMeta now @@ -560,7 +563,11 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group } // updated only once here - after the entire planning simulation is completed - // updating the exposed metrics inside the loop will change based on iterations needed for each plan loop + // updating the exposed metrics inside the above loop will change based on iterations needed for each plan loop + for _, g := range groups { + ps.ProgressMetrics.NumberOfIterations.WithLabelValues(g.key).Add(float64(groupCompactions[g.key])) + ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(g.key).Add(float64(groupBlocks[g.key])) + } return nil } From 322340bdcf2c0438f99007a3c94c34c366f65d4a Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 19 Oct 2021 09:48:51 +0530 Subject: [PATCH 13/67] added feature flag for compact Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 55 ++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c35cce47e0..e79f2b7d66 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -55,6 +55,10 @@ var ( } ) +const ( + progressMetrics = "compact-progress-metrics" +) + type compactionSet []time.Duration func (cs compactionSet) String() string { @@ -457,26 +461,35 @@ func runCompact( return cleanPartialMarked() } - g.Add(func() error { - if err := sy.SyncMetas(context.Background()); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - originalMetas := sy.Metas() - groups, err := grouper.Groups(originalMetas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } + if conf.progressMetrics { + g.Add(func() error { + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") + } + originalMetas := sy.Metas() - ps := compact.NewDefaultPlanSim(reg) - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { - return errors.Wrapf(err, "could not simulate planning") - } + groups, err := grouper.Groups(originalMetas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } - return nil - }, func(err error) { - cancel() - }) + ps := compact.NewDefaultPlanSim(reg) + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate planning") + } + + for _, meta := range originalMetas { + groupKey := compact.DefaultGroupKey(meta.Thanos) + ps.ProgressMetrics.NumberOfIterations.WithLabelValues(groupKey) + ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(groupKey) + } + + return nil + }, func(err error) { + cancel() + }) + } g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") @@ -610,9 +623,17 @@ type compactConfig struct { enableVerticalCompaction bool dedupFunc string skipBlockWithOutOfOrderChunks bool + progressMetrics bool } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { + featureList := cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+progressMetrics+".").Default("").Strings() + for _, f := range *featureList { + if f == "compact-progress-metrics" { + cc.progressMetrics = true + } + } + cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected."). Hidden().Default("true").BoolVar(&cc.haltOnError) cmd.Flag("debug.accept-malformed-index", From a4642f670a547cc8a3ef512cb04d97f72f65b858 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 20 Oct 2021 08:40:25 +0530 Subject: [PATCH 14/67] use tsdb planner as default Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index e79f2b7d66..1641cb0bce 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -474,7 +474,7 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - ps := compact.NewDefaultPlanSim(reg) + ps := compact.NewDefaultPlanSim(reg, logger) if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 031a84ecc8..f0ee5d38cd 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -498,8 +498,9 @@ type DefaultPlanSim struct { ProgressMetrics } -func NewDefaultPlanSim(reg prometheus.Registerer) *DefaultPlanSim { +func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPlanSim { return &DefaultPlanSim{ + planner: NewTSDBBasedPlanner(logger, []int64{}), ProgressMetrics: ProgressMetrics{ NumberOfIterations: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_iterations", @@ -536,7 +537,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group iterations++ groupCompactions[g.key] = iterations - var toRemove map[ulid.ULID]bool + toRemove := make(map[ulid.ULID]bool, len(plan)) metas := make([]*tsdb.BlockMeta, 0, len(plan)) for _, p := range plan { metas = append(metas, &p.BlockMeta) From 3092479b41fc9f30205bc7cd90d18351b636e160 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 20 Oct 2021 11:33:58 +0530 Subject: [PATCH 15/67] initialised maps with size; changed metrics update map Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1641cb0bce..e16ed9a679 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -474,7 +474,7 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - ps := compact.NewDefaultPlanSim(reg, logger) + ps := compact.NewDefaultPlanSim(reg, planner) if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index f0ee5d38cd..6ba225693a 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -402,10 +402,10 @@ func (cg *Group) Key() string { return cg.key } -func (cg *Group) deleteFromGroup(target map[ulid.ULID]bool) { +func (cg *Group) deleteFromGroup(target map[ulid.ULID]struct{}) { var newGroupMeta []*metadata.Meta for _, meta := range cg.metasByMinTime { - if target[meta.BlockMeta.ULID] { + if _, found := target[meta.BlockMeta.ULID]; found { newGroupMeta = append(newGroupMeta, meta) } } @@ -485,7 +485,6 @@ func (cg *Group) Resolution() int64 { type ProgressMetrics struct { NumberOfIterations *prometheus.GaugeVec NumberOfBlocksToMerge *prometheus.GaugeVec - // figure out where to add groupKey labels to this } // should return the results/metrics of the planning simulation @@ -495,13 +494,13 @@ type PlanSim interface { type DefaultPlanSim struct { planner Planner - ProgressMetrics + *ProgressMetrics } -func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPlanSim { +func NewDefaultPlanSim(reg prometheus.Registerer, planner Planner) *DefaultPlanSim { return &DefaultPlanSim{ - planner: NewTSDBBasedPlanner(logger, []int64{}), - ProgressMetrics: ProgressMetrics{ + planner: planner, + ProgressMetrics: &ProgressMetrics{ NumberOfIterations: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_iterations", Help: "number of iterations to be done", @@ -517,7 +516,8 @@ func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPla func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { iterations := 0 blocksToMerge := 0 - var groupCompactions, groupBlocks map[string]int + groupCompactions := make(map[string]int, len(groups)) + groupBlocks := make(map[string]int, len(groups)) hasPlan := true for hasPlan { @@ -537,11 +537,12 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group iterations++ groupCompactions[g.key] = iterations - toRemove := make(map[ulid.ULID]bool, len(plan)) + // value type in map is struct{} - enpty struct consumes 0 bytes so prefered over bool + toRemove := make(map[ulid.ULID]struct{}, len(plan)) metas := make([]*tsdb.BlockMeta, 0, len(plan)) for _, p := range plan { metas = append(metas, &p.BlockMeta) - toRemove[p.BlockMeta.ULID] = true + toRemove[p.BlockMeta.ULID] = struct{}{} } g.deleteFromGroup(toRemove) @@ -565,9 +566,12 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group // updated only once here - after the entire planning simulation is completed // updating the exposed metrics inside the above loop will change based on iterations needed for each plan loop - for _, g := range groups { - ps.ProgressMetrics.NumberOfIterations.WithLabelValues(g.key).Add(float64(groupCompactions[g.key])) - ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(g.key).Add(float64(groupBlocks[g.key])) + // updating the metrics' maps directly - some keys may not be present in the groups map; also saves the cost of a lookup if done directly + for key, iters := range groupCompactions { + ps.ProgressMetrics.NumberOfIterations.WithLabelValues(key).Add(float64(iters)) + } + for key, blocks := range groupBlocks { + ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(key).Add(float64(blocks)) } return nil From 7f915805e7cdca26edba7c08599d3ef99cba6945 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 20 Oct 2021 14:42:14 +0530 Subject: [PATCH 16/67] removed extra vars; changed prom init location Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 8 ++++---- pkg/compact/compact.go | 12 +++--------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index e16ed9a679..b2e07f3f5a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -475,16 +475,16 @@ func runCompact( } ps := compact.NewDefaultPlanSim(reg, planner) - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { - return errors.Wrapf(err, "could not simulate planning") - } - for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) ps.ProgressMetrics.NumberOfIterations.WithLabelValues(groupKey) ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(groupKey) } + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate planning") + } + return nil }, func(err error) { cancel() diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 6ba225693a..58434b4eb7 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -514,8 +514,6 @@ func NewDefaultPlanSim(reg prometheus.Registerer, planner Planner) *DefaultPlanS } func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { - iterations := 0 - blocksToMerge := 0 groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -534,8 +532,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group if len(plan) == 0 { continue } - iterations++ - groupCompactions[g.key] = iterations + groupCompactions[g.key]++ // value type in map is struct{} - enpty struct consumes 0 bytes so prefered over bool toRemove := make(map[ulid.ULID]struct{}, len(plan)) @@ -546,8 +543,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group } g.deleteFromGroup(toRemove) - blocksToMerge += len(plan) - groupBlocks[g.key] = blocksToMerge + groupBlocks[g.key] += len(plan) // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? // not required to modify originalMeta now @@ -569,9 +565,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group // updating the metrics' maps directly - some keys may not be present in the groups map; also saves the cost of a lookup if done directly for key, iters := range groupCompactions { ps.ProgressMetrics.NumberOfIterations.WithLabelValues(key).Add(float64(iters)) - } - for key, blocks := range groupBlocks { - ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(key).Add(float64(blocks)) + ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(key).Add(float64(groupBlocks[key])) } return nil From 5fd74c693205e938f90ab9f8a204fe135cc605b3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 20 Oct 2021 18:40:40 +0530 Subject: [PATCH 17/67] added labels and downsample resolution to metadata Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 58434b4eb7..ec3e5ca6b2 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -556,7 +556,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group } newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) - g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta}) + g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta, Thanos: metadata.Thanos{Downsample: metadata.ThanosDownsample{Resolution: g.Resolution()}, Labels: g.Labels().Map()}}) } } From 6f0b74a6820ad27312c2c8810322d2b53c35004a Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 21 Oct 2021 08:52:02 +0530 Subject: [PATCH 18/67] added termination condition; reverted planner type Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index b2e07f3f5a..a227d150ee 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -474,7 +474,7 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - ps := compact.NewDefaultPlanSim(reg, planner) + ps := compact.NewDefaultPlanSim(reg, logger) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) ps.ProgressMetrics.NumberOfIterations.WithLabelValues(groupKey) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index ec3e5ca6b2..1199bb6be2 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -497,9 +497,9 @@ type DefaultPlanSim struct { *ProgressMetrics } -func NewDefaultPlanSim(reg prometheus.Registerer, planner Planner) *DefaultPlanSim { +func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPlanSim { return &DefaultPlanSim{ - planner: planner, + planner: NewTSDBBasedPlanner(logger, []int64{}), ProgressMetrics: &ProgressMetrics{ NumberOfIterations: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_iterations", @@ -517,13 +517,12 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) - hasPlan := true - for hasPlan { + for len(groups) > 0 { + tmpGroups := make([]*Group, 0, len(groups)) for _, g := range groups { if len(g.IDs()) == 1 { continue } - // parameter should be of type tsdb.BlockMeta.meta plan, err := ps.planner.Plan(ctx, g.metasByMinTime) if err != nil { @@ -549,15 +548,19 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group // not required to modify originalMeta now if len(g.metasByMinTime) == 0 { - hasPlan = false - break + continue // no plan in case the group is empty after removing the 'planned' metadata // group size will remain one even after newMeta is added, hence, no plan needed for this case } newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) - g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta, Thanos: metadata.Thanos{Downsample: metadata.ThanosDownsample{Resolution: g.Resolution()}, Labels: g.Labels().Map()}}) + if err := g.AppendMeta(&metadata.Meta{BlockMeta: *newMeta, Thanos: metadata.Thanos{Downsample: metadata.ThanosDownsample{Resolution: g.Resolution()}, Labels: g.Labels().Map()}}); err != nil { + return errors.Wrapf(err, "append meta") + } + tmpGroups = append(tmpGroups, g) } + + groups = tmpGroups } // updated only once here - after the entire planning simulation is completed From a3dfae106bd76587ac526d4063d3b3e3724ba61c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 22 Oct 2021 09:45:08 +0530 Subject: [PATCH 19/67] draft: downsampling metrics overview Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 23 +++++++++++++++++++++ pkg/compact/compact.go | 46 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index a227d150ee..e201d4543b 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -491,6 +491,29 @@ func runCompact( }) } + g.Add(func() error { + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") + } + originalMetas := sy.Metas() + + bkt, err := client.NewBucket(logger, confContentYaml, reg, component.String()) + if err != nil { + return err + } + + ds := compact.NewDefaultDownsampleSim(reg) + if err := ds.DownsampleCalculate(context.Background(), logger, bkt, originalMetas); err != nil { + return errors.Wrapf(err, "could not simulate downsampling") + } + + // find source blocks for 5m and 1h - like the above func + + return nil + }, func(err error) { + cancel() + }) + g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 1199bb6be2..169d9e9cf9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -574,6 +574,52 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group return nil } +type DownsampleMetrics struct { + blocksDownsampled *prometheus.GaugeVec + // - number of blocks to be finally downsampled, grouped by resolution - ?? +} + +type DownsampleSim interface { + DownsampleCalculate() +} + +type DefaultDownsampleSim struct { + *DownsampleMetrics +} + +func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { + return &DefaultDownsampleSim{} +} + +func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger log.Logger, bkt objstore.Bucket, metas map[ulid.ULID]*metadata.Meta) error { + + sources5m := map[ulid.ULID]struct{}{} + sources1h := map[ulid.ULID]struct{}{} + + for _, m := range metas { + switch m.Thanos.Downsample.Resolution { + case downsample.ResLevel0: + continue + case downsample.ResLevel1: + for _, id := range m.Compaction.Sources { + sources5m[id] = struct{}{} + } + case downsample.ResLevel2: + for _, id := range m.Compaction.Sources { + sources1h[id] = struct{}{} + } + default: + return errors.Errorf("unexpected downsampling resolution %d", m.Thanos.Downsample.Resolution) + } + } + + level.Info(logger).Log("msg", "number of blocks to be downsampled", "5m", len(sources5m)) + + level.Info(logger).Log("msg", "number of blocks to be downsampled", "1h", len(sources1h)) + + return nil +} + // Planner returns blocks to compact. type Planner interface { // Plan returns a list of blocks that should be compacted into single one. From ad4c70412e8c5de7b57dbb7ed0121a2e9af0a67f Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 22 Oct 2021 12:26:37 +0530 Subject: [PATCH 20/67] removed extra bkt; processing individual blocks Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 10 +++------- pkg/compact/compact.go | 32 +++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index e201d4543b..bc282aeccc 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -497,17 +497,13 @@ func runCompact( } originalMetas := sy.Metas() - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.String()) - if err != nil { - return err - } + testDownsamplingDir := path.Join(conf.dataDir, "testDownsample") ds := compact.NewDefaultDownsampleSim(reg) - if err := ds.DownsampleCalculate(context.Background(), logger, bkt, originalMetas); err != nil { + if err := ds.DownsampleCalculate(context.Background(), logger, bkt, originalMetas, testDownsamplingDir); err != nil { return errors.Wrapf(err, "could not simulate downsampling") } - - // find source blocks for 5m and 1h - like the above func + // delete test downsampling dir after this return nil }, func(err error) { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index af65557007..b42763353b 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -591,7 +591,7 @@ func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { return &DefaultDownsampleSim{} } -func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger log.Logger, bkt objstore.Bucket, metas map[ulid.ULID]*metadata.Meta) error { +func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger log.Logger, bkt objstore.Bucket, metas map[ulid.ULID]*metadata.Meta, dir string) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} @@ -613,10 +613,40 @@ func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger } } + // check for missing blocks here before using len level.Info(logger).Log("msg", "number of blocks to be downsampled", "5m", len(sources5m)) level.Info(logger).Log("msg", "number of blocks to be downsampled", "1h", len(sources1h)) + metasULIDS := make([]ulid.ULID, 0, len(metas)) + for k := range metas { + metasULIDS = append(metasULIDS, k) + } + sort.Slice(metasULIDS, func(i, j int) bool { + return metasULIDS[i].Compare(metasULIDS[j]) < 0 + }) + + // each of these metas is added to the channel and hence, each of these ULIDs should be processed similar to processDownsampling() + for _, mk := range metasULIDS { + m := metas[mk] + + // if block not valid, continue the loop + // else, update the gauge vec. here + bdir := filepath.Join(dir, m.ULID.String()) + + err := block.Download(ctx, logger, bkt, m.ULID, bdir) + if err != nil { + return errors.Wrapf(err, "download block %s", m.ULID) + } + + if err := block.VerifyIndex(logger, filepath.Join(bdir, block.IndexFilename), m.MinTime, m.MaxTime); err != nil { + //return errors.Wrap(err, "input block index not valid") + continue // instead of return so it processes next block + } else { + // increment gauge vector + } + } + return nil } From e6739d728a890266e07db3c533b1c311bfb612d3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 22 Oct 2021 14:53:37 +0530 Subject: [PATCH 21/67] changed downsample sim logic; changed plan metric names Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 8 ++---- pkg/compact/compact.go | 64 +++++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 6acd50d5fc..4177f65d87 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -477,8 +477,8 @@ func runCompact( ps := compact.NewDefaultPlanSim(reg, logger) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) - ps.ProgressMetrics.NumberOfIterations.WithLabelValues(groupKey) - ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(groupKey) + ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(groupKey) + ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(groupKey) } if err = ps.ProgressCalculate(context.Background(), groups); err != nil { @@ -497,10 +497,8 @@ func runCompact( } originalMetas := sy.Metas() - testDownsamplingDir := path.Join(conf.dataDir, "testDownsample") - ds := compact.NewDefaultDownsampleSim(reg) - if err := ds.DownsampleCalculate(context.Background(), logger, bkt, originalMetas, testDownsamplingDir); err != nil { + if err := ds.DownsampleCalculate(context.Background(), originalMetas); err != nil { return errors.Wrapf(err, "could not simulate downsampling") } // delete test downsampling dir after this diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index b42763353b..0d8ce13a23 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -403,6 +403,8 @@ func (cg *Group) Key() string { } func (cg *Group) deleteFromGroup(target map[ulid.ULID]struct{}) { + cg.mtx.Lock() + defer cg.mtx.Unlock() var newGroupMeta []*metadata.Meta for _, meta := range cg.metasByMinTime { if _, found := target[meta.BlockMeta.ULID]; found { @@ -483,8 +485,8 @@ func (cg *Group) Resolution() int64 { // metrics related to planning and compaction progress type ProgressMetrics struct { - NumberOfIterations *prometheus.GaugeVec - NumberOfBlocksToMerge *prometheus.GaugeVec + NumberOfCompactionRuns *prometheus.GaugeVec + NumberOfCompactionBlocks *prometheus.GaugeVec } // should return the results/metrics of the planning simulation @@ -501,11 +503,11 @@ func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPla return &DefaultPlanSim{ planner: NewTSDBBasedPlanner(logger, []int64{}), ProgressMetrics: &ProgressMetrics{ - NumberOfIterations: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_iterations", Help: "number of iterations to be done", }, []string{"group"}), - NumberOfBlocksToMerge: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + NumberOfCompactionBlocks: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_blocks_planned", Help: "number of blocks planned to be merged", }, []string{"group"}), @@ -567,8 +569,8 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group // updating the exposed metrics inside the above loop will change based on iterations needed for each plan loop // updating the metrics' maps directly - some keys may not be present in the groups map; also saves the cost of a lookup if done directly for key, iters := range groupCompactions { - ps.ProgressMetrics.NumberOfIterations.WithLabelValues(key).Add(float64(iters)) - ps.ProgressMetrics.NumberOfBlocksToMerge.WithLabelValues(key).Add(float64(groupBlocks[key])) + ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(key).Add(float64(iters)) + ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(key).Add(float64(groupBlocks[key])) } return nil @@ -591,7 +593,7 @@ func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { return &DefaultDownsampleSim{} } -func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger log.Logger, bkt objstore.Bucket, metas map[ulid.ULID]*metadata.Meta, dir string) error { +func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, metas map[ulid.ULID]*metadata.Meta) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} @@ -614,9 +616,6 @@ func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger } // check for missing blocks here before using len - level.Info(logger).Log("msg", "number of blocks to be downsampled", "5m", len(sources5m)) - - level.Info(logger).Log("msg", "number of blocks to be downsampled", "1h", len(sources1h)) metasULIDS := make([]ulid.ULID, 0, len(metas)) for k := range metas { @@ -632,18 +631,43 @@ func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, logger // if block not valid, continue the loop // else, update the gauge vec. here - bdir := filepath.Join(dir, m.ULID.String()) + switch m.Thanos.Downsample.Resolution { + case downsample.ResLevel2: + continue - err := block.Download(ctx, logger, bkt, m.ULID, bdir) - if err != nil { - return errors.Wrapf(err, "download block %s", m.ULID) - } + case downsample.ResLevel0: + missing := false + for _, id := range m.Compaction.Sources { + if _, ok := sources5m[id]; !ok { + missing = true + break + } + } + if !missing { + continue + } + + if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { + continue + } + //update metrics here - if err := block.VerifyIndex(logger, filepath.Join(bdir, block.IndexFilename), m.MinTime, m.MaxTime); err != nil { - //return errors.Wrap(err, "input block index not valid") - continue // instead of return so it processes next block - } else { - // increment gauge vector + case downsample.ResLevel1: + missing := false + for _, id := range m.Compaction.Sources { + if _, ok := sources1h[id]; !ok { + missing = true + break + } + } + if !missing { + continue + } + + if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { + continue + } + // update metrics here } } From b4a662bb45e69cbff95864742b7677043dc025b2 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 23 Oct 2021 09:23:51 +0530 Subject: [PATCH 22/67] re-used interface; refactoring Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 10 ++-------- pkg/compact/compact.go | 33 ++++++++++++++------------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 4177f65d87..b9b36f761a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -469,11 +469,6 @@ func runCompact( } originalMetas := sy.Metas() - groups, err := grouper.Groups(originalMetas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } - ps := compact.NewDefaultPlanSim(reg, logger) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) @@ -481,7 +476,7 @@ func runCompact( ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(groupKey) } - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + if err = ps.ProgressCalculate(context.Background(), grouper, originalMetas); err != nil { return errors.Wrapf(err, "could not simulate planning") } @@ -498,10 +493,9 @@ func runCompact( originalMetas := sy.Metas() ds := compact.NewDefaultDownsampleSim(reg) - if err := ds.DownsampleCalculate(context.Background(), originalMetas); err != nil { + if err := ds.ProgressCalculate(context.Background(), grouper, originalMetas); err != nil { return errors.Wrapf(err, "could not simulate downsampling") } - // delete test downsampling dir after this return nil }, func(err error) { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 0d8ce13a23..2548e00276 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -491,7 +491,7 @@ type ProgressMetrics struct { // should return the results/metrics of the planning simulation type PlanSim interface { - ProgressCalculate(ctx context.Context, groups []*Group) error + ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error } type DefaultPlanSim struct { @@ -515,7 +515,12 @@ func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPla } } -func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error { + groups, err := grouper.Groups(metas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } + groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -581,10 +586,6 @@ type DownsampleMetrics struct { // - number of blocks to be finally downsampled, grouped by resolution - ?? } -type DownsampleSim interface { - DownsampleCalculate() -} - type DefaultDownsampleSim struct { *DownsampleMetrics } @@ -593,12 +594,16 @@ func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { return &DefaultDownsampleSim{} } -func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, metas map[ulid.ULID]*metadata.Meta) error { +func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} + metasULIDS := make([]ulid.ULID, 0, len(metas)) // change size of map allocation - for _, m := range metas { + for k, m := range metas { + if m.Thanos.Downsample.Resolution != downsample.ResLevel2 { + metasULIDS = append(metasULIDS, k) + } switch m.Thanos.Downsample.Resolution { case downsample.ResLevel0: continue @@ -615,12 +620,6 @@ func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, metas m } } - // check for missing blocks here before using len - - metasULIDS := make([]ulid.ULID, 0, len(metas)) - for k := range metas { - metasULIDS = append(metasULIDS, k) - } sort.Slice(metasULIDS, func(i, j int) bool { return metasULIDS[i].Compare(metasULIDS[j]) < 0 }) @@ -629,12 +628,8 @@ func (ds *DefaultDownsampleSim) DownsampleCalculate(ctx context.Context, metas m for _, mk := range metasULIDS { m := metas[mk] - // if block not valid, continue the loop - // else, update the gauge vec. here switch m.Thanos.Downsample.Resolution { - case downsample.ResLevel2: - continue - + // removed case with ResLevel2 since those aren't included in metasULIDs case downsample.ResLevel0: missing := false for _, id := range m.Compaction.Sources { From cdeb0d71e8f8ce88d195419c3fd2756bdcd1a507 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 23 Oct 2021 09:49:58 +0530 Subject: [PATCH 23/67] draft: added Prom metrics Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 ++ pkg/compact/compact.go | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index b9b36f761a..c09a9dada5 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -493,6 +493,8 @@ func runCompact( originalMetas := sy.Metas() ds := compact.NewDefaultDownsampleSim(reg) + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel0") + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel1") if err := ds.ProgressCalculate(context.Background(), grouper, originalMetas); err != nil { return errors.Wrapf(err, "could not simulate downsampling") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 2548e00276..1873b66d61 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -582,7 +582,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, grouper *Defaul } type DownsampleMetrics struct { - blocksDownsampled *prometheus.GaugeVec + BlocksDownsampled *prometheus.GaugeVec // - number of blocks to be finally downsampled, grouped by resolution - ?? } @@ -591,7 +591,14 @@ type DefaultDownsampleSim struct { } func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { - return &DefaultDownsampleSim{} + return &DefaultDownsampleSim{ + DownsampleMetrics: &DownsampleMetrics{ + BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_blocks_downsampled", + Help: "number of blocks to be downsampled", + }, []string{"resolution"}), + }, + } } func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error { @@ -624,6 +631,10 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * return metasULIDS[i].Compare(metasULIDS[j]) < 0 }) + // count number of blocks downsampled for each res level + resLevel0 := 0 + resLevel1 := 0 + // each of these metas is added to the channel and hence, each of these ULIDs should be processed similar to processDownsampling() for _, mk := range metasULIDS { m := metas[mk] @@ -645,7 +656,7 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { continue } - //update metrics here + resLevel0++ case downsample.ResLevel1: missing := false @@ -662,10 +673,13 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { continue } - // update metrics here + resLevel1++ } } + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel0").Add(float64(resLevel0)) + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel1").Add(float64(resLevel1)) + return nil } From 7a5393595577a53d3d2c57478b416e9d7a1f5098 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 23 Oct 2021 15:28:38 +0530 Subject: [PATCH 24/67] re-used planner; added flag; fixed interface Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 48 ++++++++++++++++++------------------------ pkg/compact/compact.go | 36 +++++++++++++------------------ 2 files changed, 36 insertions(+), 48 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c09a9dada5..79780805cd 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -56,7 +56,7 @@ var ( ) const ( - progressMetrics = "compact-progress-metrics" + compactionProgressMetrics = "compact-progress-metrics" ) type compactionSet []time.Duration @@ -357,8 +357,9 @@ func runCompact( compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason), metadata.HashFunc(conf.hashFunc), ) + tempTSDBPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter) planner := compact.WithLargeTotalIndexSizeFilter( - compact.NewPlanner(logger, levels, noCompactMarkerFilter), + tempTSDBPlanner, bkt, int64(conf.maxBlockIndexSize), compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.IndexSizeExceedingNoCompactReason), @@ -462,48 +463,41 @@ func runCompact( return cleanPartialMarked() } - if conf.progressMetrics { + if conf.compactionProgressMetrics { g.Add(func() error { if err := sy.SyncMetas(context.Background()); err != nil { return errors.Wrapf(err, "could not sync metas") } originalMetas := sy.Metas() - ps := compact.NewDefaultPlanSim(reg, logger) + groups, err := grouper.Groups(originalMetas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } + + ps := compact.NewDefaultPlanSim(reg, tempTSDBPlanner) + ds := compact.NewDefaultDownsampleSim(reg) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(groupKey) ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(groupKey) + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(groupKey) } - if err = ps.ProgressCalculate(context.Background(), grouper, originalMetas); err != nil { + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } + if err := ds.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate downsampling") + } + return nil }, func(err error) { cancel() }) } - g.Add(func() error { - if err := sy.SyncMetas(context.Background()); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - originalMetas := sy.Metas() - - ds := compact.NewDefaultDownsampleSim(reg) - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel0") - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel1") - if err := ds.ProgressCalculate(context.Background(), grouper, originalMetas); err != nil { - return errors.Wrapf(err, "could not simulate downsampling") - } - - return nil - }, func(err error) { - cancel() - }) - g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") @@ -636,14 +630,14 @@ type compactConfig struct { enableVerticalCompaction bool dedupFunc string skipBlockWithOutOfOrderChunks bool - progressMetrics bool + compactionProgressMetrics bool } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { - featureList := cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+progressMetrics+".").Default("").Strings() + featureList := cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+compactionProgressMetrics+".").Default("").Strings() for _, f := range *featureList { - if f == "compact-progress-metrics" { - cc.progressMetrics = true + if f == compactionProgressMetrics { + cc.compactionProgressMetrics = true } } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 1873b66d61..9feef53d53 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -491,7 +491,7 @@ type ProgressMetrics struct { // should return the results/metrics of the planning simulation type PlanSim interface { - ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error + ProgressCalculate(ctx context.Context, groups []*Group) error } type DefaultPlanSim struct { @@ -499,9 +499,9 @@ type DefaultPlanSim struct { *ProgressMetrics } -func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPlanSim { +func NewDefaultPlanSim(reg prometheus.Registerer, planner *tsdbBasedPlanner) *DefaultPlanSim { return &DefaultPlanSim{ - planner: NewTSDBBasedPlanner(logger, []int64{}), + planner: planner, ProgressMetrics: &ProgressMetrics{ NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_number_of_iterations", @@ -515,12 +515,7 @@ func NewDefaultPlanSim(reg prometheus.Registerer, logger log.Logger) *DefaultPla } } -func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error { - groups, err := grouper.Groups(metas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } - +func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -583,7 +578,7 @@ func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, grouper *Defaul type DownsampleMetrics struct { BlocksDownsampled *prometheus.GaugeVec - // - number of blocks to be finally downsampled, grouped by resolution - ?? + // - number of blocks to be finally downsampled, grouped by groupKey } type DefaultDownsampleSim struct { @@ -596,12 +591,20 @@ func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_blocks_downsampled", Help: "number of blocks to be downsampled", - }, []string{"resolution"}), + }, []string{"group"}), }, } } -func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper *DefaultGrouper, metas map[ulid.ULID]*metadata.Meta) error { +func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, groups []*Group) error { + + var metas map[ulid.ULID]*metadata.Meta // pre allocate with size + // mapping ULIDs to meta - reference: https://github.com/thanos-io/thanos/blob/18049504408c5ae09deb76961b92baf7f96b0e93/pkg/compact/compact.go#L425-L436 + for _, group := range groups { + for _, meta := range group.metasByMinTime { + metas[meta.ULID] = meta + } + } sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} @@ -631,10 +634,6 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * return metasULIDS[i].Compare(metasULIDS[j]) < 0 }) - // count number of blocks downsampled for each res level - resLevel0 := 0 - resLevel1 := 0 - // each of these metas is added to the channel and hence, each of these ULIDs should be processed similar to processDownsampling() for _, mk := range metasULIDS { m := metas[mk] @@ -656,7 +655,6 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { continue } - resLevel0++ case downsample.ResLevel1: missing := false @@ -673,13 +671,9 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, grouper * if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { continue } - resLevel1++ } } - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel0").Add(float64(resLevel0)) - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues("resLevel1").Add(float64(resLevel1)) - return nil } From 3b9084643a299fcc35e29c8ef3288890ea0a6398 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sun, 24 Oct 2021 08:25:58 +0530 Subject: [PATCH 25/67] increment exporters; renamed funcs/structs Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 8 ++-- pkg/compact/compact.go | 96 +++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 79780805cd..d425230afa 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -357,9 +357,9 @@ func runCompact( compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason), metadata.HashFunc(conf.hashFunc), ) - tempTSDBPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter) + tsdbPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter) planner := compact.WithLargeTotalIndexSizeFilter( - tempTSDBPlanner, + tsdbPlanner, bkt, int64(conf.maxBlockIndexSize), compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.IndexSizeExceedingNoCompactReason), @@ -475,8 +475,8 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - ps := compact.NewDefaultPlanSim(reg, tempTSDBPlanner) - ds := compact.NewDefaultDownsampleSim(reg) + ps := compact.NewCompactionSimulator(reg, tsdbPlanner) + ds := compact.NewDownsampleSim(reg) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(groupKey) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 9feef53d53..bc27648037 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -490,17 +490,17 @@ type ProgressMetrics struct { } // should return the results/metrics of the planning simulation -type PlanSim interface { +type ProgressCalculator interface { ProgressCalculate(ctx context.Context, groups []*Group) error } -type DefaultPlanSim struct { +type CompactionSimulator struct { planner Planner *ProgressMetrics } -func NewDefaultPlanSim(reg prometheus.Registerer, planner *tsdbBasedPlanner) *DefaultPlanSim { - return &DefaultPlanSim{ +func NewCompactionSimulator(reg prometheus.Registerer, planner *tsdbBasedPlanner) *CompactionSimulator { + return &CompactionSimulator{ planner: planner, ProgressMetrics: &ProgressMetrics{ NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ @@ -515,7 +515,7 @@ func NewDefaultPlanSim(reg prometheus.Registerer, planner *tsdbBasedPlanner) *De } } -func (ps *DefaultPlanSim) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []*Group) error { groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -581,12 +581,12 @@ type DownsampleMetrics struct { // - number of blocks to be finally downsampled, grouped by groupKey } -type DefaultDownsampleSim struct { +type DownsampleSim struct { *DownsampleMetrics } -func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { - return &DefaultDownsampleSim{ +func NewDownsampleSim(reg prometheus.Registerer) *DownsampleSim { + return &DownsampleSim{ DownsampleMetrics: &DownsampleMetrics{ BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_blocks_downsampled", @@ -596,7 +596,7 @@ func NewDefaultDownsampleSim(reg prometheus.Registerer) *DefaultDownsampleSim { } } -func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ds *DownsampleSim) ProgressCalculate(ctx context.Context, groups []*Group) error { var metas map[ulid.ULID]*metadata.Meta // pre allocate with size // mapping ULIDs to meta - reference: https://github.com/thanos-io/thanos/blob/18049504408c5ae09deb76961b92baf7f96b0e93/pkg/compact/compact.go#L425-L436 @@ -608,12 +608,9 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, groups [] sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} - metasULIDS := make([]ulid.ULID, 0, len(metas)) // change size of map allocation + groupBlocks := make(map[string]int, len(groups)) - for k, m := range metas { - if m.Thanos.Downsample.Resolution != downsample.ResLevel2 { - metasULIDS = append(metasULIDS, k) - } + for _, m := range metas { switch m.Thanos.Downsample.Resolution { case downsample.ResLevel0: continue @@ -630,50 +627,51 @@ func (ds *DefaultDownsampleSim) ProgressCalculate(ctx context.Context, groups [] } } - sort.Slice(metasULIDS, func(i, j int) bool { - return metasULIDS[i].Compare(metasULIDS[j]) < 0 - }) - // each of these metas is added to the channel and hence, each of these ULIDs should be processed similar to processDownsampling() - for _, mk := range metasULIDS { - m := metas[mk] - - switch m.Thanos.Downsample.Resolution { - // removed case with ResLevel2 since those aren't included in metasULIDs - case downsample.ResLevel0: - missing := false - for _, id := range m.Compaction.Sources { - if _, ok := sources5m[id]; !ok { - missing = true - break + for _, group := range groups { + for _, m := range group.metasByMinTime { + switch m.Thanos.Downsample.Resolution { + // removed case with ResLevel2 since those aren't included in metasULIDs + case downsample.ResLevel0: + missing := false + for _, id := range m.Compaction.Sources { + if _, ok := sources5m[id]; !ok { + missing = true + break + } + } + if !missing { + continue } - } - if !missing { - continue - } - - if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { - continue - } - case downsample.ResLevel1: - missing := false - for _, id := range m.Compaction.Sources { - if _, ok := sources1h[id]; !ok { - missing = true - break + if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { + continue + } + groupBlocks[group.key]++ + case downsample.ResLevel1: + missing := false + for _, id := range m.Compaction.Sources { + if _, ok := sources1h[id]; !ok { + missing = true + break + } + } + if !missing { + continue } - } - if !missing { - continue - } - if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { - continue + if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { + continue + } + groupBlocks[group.key]++ } } } + for key, blocks := range groupBlocks { + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(key).Add(float64(blocks)) + } + return nil } From da04db8073963e052bb532d05d77e91670f81634 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sun, 24 Oct 2021 08:35:51 +0530 Subject: [PATCH 26/67] check if downsampling is disabled Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index d425230afa..b2d621b07a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -476,20 +476,25 @@ func runCompact( } ps := compact.NewCompactionSimulator(reg, tsdbPlanner) - ds := compact.NewDownsampleSim(reg) for _, meta := range originalMetas { groupKey := compact.DefaultGroupKey(meta.Thanos) ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(groupKey) ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(groupKey) - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(groupKey) } if err = ps.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } - if err := ds.ProgressCalculate(context.Background(), groups); err != nil { - return errors.Wrapf(err, "could not simulate downsampling") + if !conf.disableDownsampling { + ds := compact.NewDownsampleSim(reg) + for _, meta := range originalMetas { + groupKey := compact.DefaultGroupKey(meta.Thanos) + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(groupKey) + } + if err := ds.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate downsampling") + } } return nil From 8922b33124e3509abc0d5a18350c32f563f0a479 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sun, 24 Oct 2021 10:04:25 +0530 Subject: [PATCH 27/67] iterate through groups instead of meta Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 12 +++++------- pkg/compact/compact.go | 41 +++++++++++++++++------------------------ 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index b2d621b07a..9b71b79b5f 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -476,10 +476,9 @@ func runCompact( } ps := compact.NewCompactionSimulator(reg, tsdbPlanner) - for _, meta := range originalMetas { - groupKey := compact.DefaultGroupKey(meta.Thanos) - ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(groupKey) - ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(groupKey) + for _, group := range groups { + ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) + ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) } if err = ps.ProgressCalculate(context.Background(), groups); err != nil { @@ -488,9 +487,8 @@ func runCompact( if !conf.disableDownsampling { ds := compact.NewDownsampleSim(reg) - for _, meta := range originalMetas { - groupKey := compact.DefaultGroupKey(meta.Thanos) - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(groupKey) + for _, group := range groups { + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) } if err := ds.ProgressCalculate(context.Background(), groups); err != nil { return errors.Wrapf(err, "could not simulate downsampling") diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index bc27648037..b91cde0590 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -577,8 +577,7 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* } type DownsampleMetrics struct { - BlocksDownsampled *prometheus.GaugeVec - // - number of blocks to be finally downsampled, grouped by groupKey + BlocksDownsampled *prometheus.GaugeVec // number of blocks to be finally downsampled, grouped by groupKey } type DownsampleSim struct { @@ -597,33 +596,27 @@ func NewDownsampleSim(reg prometheus.Registerer) *DownsampleSim { } func (ds *DownsampleSim) ProgressCalculate(ctx context.Context, groups []*Group) error { - - var metas map[ulid.ULID]*metadata.Meta // pre allocate with size - // mapping ULIDs to meta - reference: https://github.com/thanos-io/thanos/blob/18049504408c5ae09deb76961b92baf7f96b0e93/pkg/compact/compact.go#L425-L436 - for _, group := range groups { - for _, meta := range group.metasByMinTime { - metas[meta.ULID] = meta - } - } - sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} groupBlocks := make(map[string]int, len(groups)) - for _, m := range metas { - switch m.Thanos.Downsample.Resolution { - case downsample.ResLevel0: - continue - case downsample.ResLevel1: - for _, id := range m.Compaction.Sources { - sources5m[id] = struct{}{} - } - case downsample.ResLevel2: - for _, id := range m.Compaction.Sources { - sources1h[id] = struct{}{} + for _, group := range groups { + for _, m := range group.metasByMinTime { + switch m.Thanos.Downsample.Resolution { + case downsample.ResLevel0: + continue + case downsample.ResLevel1: + for _, id := range m.Compaction.Sources { + sources5m[id] = struct{}{} + } + case downsample.ResLevel2: + for _, id := range m.Compaction.Sources { + sources1h[id] = struct{}{} + } + default: + return errors.Errorf("unexpected downsampling resolution %d", m.Thanos.Downsample.Resolution) } - default: - return errors.Errorf("unexpected downsampling resolution %d", m.Thanos.Downsample.Resolution) + } } From 81a5066c9f5271deb45fbadb26c6c3f85e809299 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 25 Oct 2021 09:02:11 +0530 Subject: [PATCH 28/67] run both simulations repeatedly Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 51 +++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 9b71b79b5f..e51d530606 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -465,35 +465,44 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - if err := sy.SyncMetas(context.Background()); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - originalMetas := sy.Metas() - - groups, err := grouper.Groups(originalMetas) - if err != nil { - return errors.Wrapf(err, "could not group original metadata") + // need to create simulator structs only once + var ds *compact.DownsampleSim + if !conf.disableDownsampling { + ds = compact.NewDownsampleSim(reg) } - ps := compact.NewCompactionSimulator(reg, tsdbPlanner) - for _, group := range groups { - ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) - ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") } - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { - return errors.Wrapf(err, "could not simulate planning") - } + return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { + originalMetas := sy.Metas() + groups, err := grouper.Groups(originalMetas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata") + } - if !conf.disableDownsampling { - ds := compact.NewDownsampleSim(reg) + // re-initializing metrics here since new groups are formed each time for _, group := range groups { - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) + ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) + ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) } - if err := ds.ProgressCalculate(context.Background(), groups); err != nil { - return errors.Wrapf(err, "could not simulate downsampling") + + if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate planning") } - } + + if !conf.disableDownsampling { + for _, group := range groups { + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) + } + if err := ds.ProgressCalculate(context.Background(), groups); err != nil { + return errors.Wrapf(err, "could not simulate downsampling") + } + } + + return nil + }) return nil }, func(err error) { From 63ab795b401d9ac7de39b5c146f6b53c6f980bc3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 25 Oct 2021 09:23:14 +0530 Subject: [PATCH 29/67] made naming more consistent Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 4 ++-- pkg/compact/compact.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index e51d530606..c98d8ee7ef 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -466,9 +466,9 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { // need to create simulator structs only once - var ds *compact.DownsampleSim + var ds *compact.DownsampleSimulator if !conf.disableDownsampling { - ds = compact.NewDownsampleSim(reg) + ds = compact.NewDownsampleSimulator(reg) } ps := compact.NewCompactionSimulator(reg, tsdbPlanner) if err := sy.SyncMetas(context.Background()); err != nil { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index b91cde0590..de2117f8cd 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -580,12 +580,12 @@ type DownsampleMetrics struct { BlocksDownsampled *prometheus.GaugeVec // number of blocks to be finally downsampled, grouped by groupKey } -type DownsampleSim struct { +type DownsampleSimulator struct { *DownsampleMetrics } -func NewDownsampleSim(reg prometheus.Registerer) *DownsampleSim { - return &DownsampleSim{ +func NewDownsampleSimulator(reg prometheus.Registerer) *DownsampleSimulator { + return &DownsampleSimulator{ DownsampleMetrics: &DownsampleMetrics{ BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_blocks_downsampled", @@ -595,7 +595,7 @@ func NewDownsampleSim(reg prometheus.Registerer) *DownsampleSim { } } -func (ds *DownsampleSim) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ds *DownsampleSimulator) ProgressCalculate(ctx context.Context, groups []*Group) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} groupBlocks := make(map[string]int, len(groups)) From 235528d86f566ebedf72aa3df5cf00ffff282398 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 25 Oct 2021 17:42:35 +0530 Subject: [PATCH 30/67] added unit test for planning simulation Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 2 +- pkg/compact/compact_test.go | 120 ++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index de2117f8cd..8e18ed6f92 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -407,7 +407,7 @@ func (cg *Group) deleteFromGroup(target map[ulid.ULID]struct{}) { defer cg.mtx.Unlock() var newGroupMeta []*metadata.Meta for _, meta := range cg.metasByMinTime { - if _, found := target[meta.BlockMeta.ULID]; found { + if _, found := target[meta.BlockMeta.ULID]; !found { newGroupMeta = append(newGroupMeta, meta) } } diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 5472c3db3c..52986b7bb4 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -17,6 +17,8 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + promtestutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -174,3 +176,121 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } } + +func TestPlanSimulate(t *testing.T) { + logger := log.NewNopLogger() + planner := NewTSDBBasedPlanner(logger, []int64{ + int64(1 * time.Hour / time.Millisecond), + int64(2 * time.Hour / time.Millisecond), + int64(8 * time.Hour / time.Millisecond), + int64(2 * 24 * time.Hour / time.Millisecond), + }) + reg := prometheus.NewRegistry() + ps := NewCompactionSimulator(reg, planner) + + metas := []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(5, nil), + MinTime: int64(10 * time.Hour / time.Millisecond), + MaxTime: int64(12 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(6, nil), + MinTime: int64(12 * time.Hour / time.Millisecond), + MaxTime: int64(20 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(7, nil), + MinTime: int64(20 * time.Hour / time.Millisecond), + MaxTime: int64(28 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + } + + extLabels := labels.FromMap(map[string]string{"a": "1"}) + groups := []*Group{ + { + labels: extLabels, + resolution: 0, + metasByMinTime: metas, + }, + } + + err := ps.ProgressCalculate(context.Background(), groups) + testutil.Ok(t, err) + metrics := ps.ProgressMetrics + testutil.Equals(t, 2.0, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) + testutil.Equals(t, 6.0, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) +} From 1810ba4ed58c5c28eb599e336279aa9251b521df Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 26 Oct 2021 08:54:07 +0530 Subject: [PATCH 31/67] removed extra comments Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 -- pkg/compact/compact.go | 16 +--------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c98d8ee7ef..cb4c107ed5 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -465,7 +465,6 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - // need to create simulator structs only once var ds *compact.DownsampleSimulator if !conf.disableDownsampling { ds = compact.NewDownsampleSimulator(reg) @@ -482,7 +481,6 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - // re-initializing metrics here since new groups are formed each time for _, group := range groups { ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 8e18ed6f92..2c9a962028 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -483,13 +483,11 @@ func (cg *Group) Resolution() int64 { return cg.resolution } -// metrics related to planning and compaction progress type ProgressMetrics struct { NumberOfCompactionRuns *prometheus.GaugeVec NumberOfCompactionBlocks *prometheus.GaugeVec } -// should return the results/metrics of the planning simulation type ProgressCalculator interface { ProgressCalculate(ctx context.Context, groups []*Group) error } @@ -525,7 +523,6 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* if len(g.IDs()) == 1 { continue } - // parameter should be of type tsdb.BlockMeta.meta plan, err := ps.planner.Plan(ctx, g.metasByMinTime) if err != nil { return errors.Wrapf(err, "could not plan") @@ -535,7 +532,6 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* } groupCompactions[g.key]++ - // value type in map is struct{} - enpty struct consumes 0 bytes so prefered over bool toRemove := make(map[ulid.ULID]struct{}, len(plan)) metas := make([]*tsdb.BlockMeta, 0, len(plan)) for _, p := range plan { @@ -546,13 +542,8 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* groupBlocks[g.key] += len(plan) - // remove 'plan' blocks from 'original metadata' - so that the remaining blocks can now be planned ? - // not required to modify originalMeta now - if len(g.metasByMinTime) == 0 { continue - // no plan in case the group is empty after removing the 'planned' metadata - // group size will remain one even after newMeta is added, hence, no plan needed for this case } newMeta := tsdb.CompactBlockMetas(ulid.MustNew(uint64(time.Now().Unix()), nil), metas...) @@ -565,9 +556,6 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* groups = tmpGroups } - // updated only once here - after the entire planning simulation is completed - // updating the exposed metrics inside the above loop will change based on iterations needed for each plan loop - // updating the metrics' maps directly - some keys may not be present in the groups map; also saves the cost of a lookup if done directly for key, iters := range groupCompactions { ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(key).Add(float64(iters)) ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(key).Add(float64(groupBlocks[key])) @@ -577,7 +565,7 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* } type DownsampleMetrics struct { - BlocksDownsampled *prometheus.GaugeVec // number of blocks to be finally downsampled, grouped by groupKey + BlocksDownsampled *prometheus.GaugeVec } type DownsampleSimulator struct { @@ -620,11 +608,9 @@ func (ds *DownsampleSimulator) ProgressCalculate(ctx context.Context, groups []* } } - // each of these metas is added to the channel and hence, each of these ULIDs should be processed similar to processDownsampling() for _, group := range groups { for _, m := range group.metasByMinTime { switch m.Thanos.Downsample.Resolution { - // removed case with ResLevel2 since those aren't included in metasULIDs case downsample.ResLevel0: missing := false for _, id := range m.Compaction.Sources { From a355886d16fe0e742ba5ad9d0b3b6861900c1b5e Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 26 Oct 2021 10:32:15 +0530 Subject: [PATCH 32/67] fixed flag bug Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index cb4c107ed5..b26a50b7b0 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -93,6 +93,12 @@ func registerCompact(app *extkingpin.App) { conf.registerFlag(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { + for _, f := range conf.enableFeature { + if f == compactionProgressMetrics { + conf.compactionProgressMetrics = true + } + } + return runCompact(g, logger, tracer, reg, component.Compact, *conf, getFlagsMap(cmd.Flags())) }) } @@ -641,15 +647,11 @@ type compactConfig struct { dedupFunc string skipBlockWithOutOfOrderChunks bool compactionProgressMetrics bool + enableFeature []string } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { - featureList := cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+compactionProgressMetrics+".").Default("").Strings() - for _, f := range *featureList { - if f == compactionProgressMetrics { - cc.compactionProgressMetrics = true - } - } + cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+compactionProgressMetrics+".").Default("").StringsVar(&cc.enableFeature) cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected."). Hidden().Default("true").BoolVar(&cc.haltOnError) From 40fa1fcf53a61db59520917024b0979153298cc0 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 26 Oct 2021 11:14:36 +0530 Subject: [PATCH 33/67] added docs for new compact flag Signed-off-by: metonymic-smokey --- docs/components/compact.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/components/compact.md b/docs/components/compact.md index 8b662a88e7..7d66b1d8de 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -348,6 +348,9 @@ Flags: non-downsampled data is not efficient and useful e.g it is not possible to render all samples for a human eye anyway + --enable-feature= ... Comma separated experimental feature names to + enable.The current list of features is + compact-progress-metrics. --hash-func= Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. From 13bec696c5ac038976f0b6a8ce84ed4cae09ce18 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 26 Oct 2021 11:40:26 +0530 Subject: [PATCH 34/67] lint fix Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index b26a50b7b0..824b30969a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -507,8 +507,6 @@ func runCompact( return nil }) - - return nil }, func(err error) { cancel() }) From cec1c58085e14d8600a3d42e030f3958b4436dbc Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 27 Oct 2021 11:26:22 +0530 Subject: [PATCH 35/67] added sync meta to repeat loop Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 824b30969a..1cb9e9aaf1 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -476,11 +476,11 @@ func runCompact( ds = compact.NewDownsampleSimulator(reg) } ps := compact.NewCompactionSimulator(reg, tsdbPlanner) - if err := sy.SyncMetas(context.Background()); err != nil { - return errors.Wrapf(err, "could not sync metas") - } - return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { + if err := sy.SyncMetas(context.Background()); err != nil { + return errors.Wrapf(err, "could not sync metas") + } + originalMetas := sy.Metas() groups, err := grouper.Groups(originalMetas) if err != nil { From d8a1ab8d5ab513204f7579caeeb4bccf28a1f4c5 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 28 Oct 2021 09:23:29 +0530 Subject: [PATCH 36/67] draft: downsampling unit test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 113 ++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 52986b7bb4..05fda7b799 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -22,6 +22,7 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" @@ -294,3 +295,115 @@ func TestPlanSimulate(t *testing.T) { testutil.Equals(t, 2.0, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) testutil.Equals(t, 6.0, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) } + +func TestDownsampleSimulate(t *testing.T) { + + for _, tcase := range []struct { + input []*metadata.Meta + expected float64 + }{ + { + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel1, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel0, + }, + }, + }, + }, + expected: 1.0, + }, + { + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(6, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel2, + }, + }, + }, + }, + expected: 0.0, + }, { + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(9, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(10, nil), ulid.MustNew(11, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel0, + }, + }, + }, + }, + expected: 1.0, + }, + } { + reg := prometheus.NewRegistry() + ds := NewDownsampleSimulator(reg) + + extLabels := labels.FromMap(map[string]string{"a": "1"}) + groups := []*Group{ + { + key: "a", + labels: extLabels, + resolution: downsample.ResLevel1, + metasByMinTime: tcase.input, + }, + } + if ok := t.Run("", func(t *testing.T) { + err := ds.ProgressCalculate(context.Background(), groups) + testutil.Ok(t, err) + metrics := ds.DownsampleMetrics + testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(metrics.BlocksDownsampled.WithLabelValues("a"))) + }); !ok { + return + } + } +} From fc712808f1fef0024c7cdf376f4402d11d5657c3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 28 Oct 2021 17:38:01 +0530 Subject: [PATCH 37/67] new group for downsample; re-used context Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1cb9e9aaf1..3624cc046b 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -477,7 +477,7 @@ func runCompact( } ps := compact.NewCompactionSimulator(reg, tsdbPlanner) return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { - if err := sy.SyncMetas(context.Background()); err != nil { + if err := sy.SyncMetas(ctx); err != nil { return errors.Wrapf(err, "could not sync metas") } @@ -486,21 +486,22 @@ func runCompact( if err != nil { return errors.Wrapf(err, "could not group original metadata") } + downGroups := groups //groups used for downsampling for _, group := range groups { ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) } - if err = ps.ProgressCalculate(context.Background(), groups); err != nil { + if err = ps.ProgressCalculate(ctx, groups); err != nil { return errors.Wrapf(err, "could not simulate planning") } if !conf.disableDownsampling { - for _, group := range groups { + for _, group := range downGroups { ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) } - if err := ds.ProgressCalculate(context.Background(), groups); err != nil { + if err := ds.ProgressCalculate(ctx, downGroups); err != nil { return errors.Wrapf(err, "could not simulate downsampling") } } From 5ea12ef3e00b8b9ac6eddeddd3e5d72719a5387e Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 28 Oct 2021 20:05:55 +0530 Subject: [PATCH 38/67] added test names and comments for downsample unit test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 05fda7b799..b1b111a6f0 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -299,10 +299,13 @@ func TestPlanSimulate(t *testing.T) { func TestDownsampleSimulate(t *testing.T) { for _, tcase := range []struct { + testName string input []*metadata.Meta expected float64 }{ { + // this test case has 1 block to be downsampled out of 2 since for the second block, the difference between MinTime and MaxTime is less than the acceptable threshold, DownsampleRange0 + testName: "min_max_time_diff_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -342,6 +345,8 @@ func TestDownsampleSimulate(t *testing.T) { expected: 1.0, }, { + // this test case returns 0 blocks to be downsampled since the resolution is resLevel2, which is skipped when grouping blocks for downsampling. + testName: "res_level_2_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -363,6 +368,8 @@ func TestDownsampleSimulate(t *testing.T) { }, expected: 0.0, }, { + // this test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel0, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange0. + testName: "res_level_0_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -397,7 +404,7 @@ func TestDownsampleSimulate(t *testing.T) { metasByMinTime: tcase.input, }, } - if ok := t.Run("", func(t *testing.T) { + if ok := t.Run(tcase.testName, func(t *testing.T) { err := ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ds.DownsampleMetrics From 3629986431291306dd42f81e333fd1ef1ab06660 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 28 Oct 2021 20:34:50 +0530 Subject: [PATCH 39/67] fixed copy error - changed to deep copy Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 3624cc046b..a6491efc37 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -486,7 +486,14 @@ func runCompact( if err != nil { return errors.Wrapf(err, "could not group original metadata") } - downGroups := groups //groups used for downsampling + downGroups := make([]*compact.Group, len(groups)) + for ind, group := range groups { + if group == nil { + continue + } + v := *group + downGroups[ind] = &v + } for _, group := range groups { ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) From fa40551ca5eec8e4aa7d73afc9e66071232b98e4 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 29 Oct 2021 09:18:12 +0530 Subject: [PATCH 40/67] changed metric names Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 2c9a962028..f45d9c9c0e 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -502,11 +502,11 @@ func NewCompactionSimulator(reg prometheus.Registerer, planner *tsdbBasedPlanner planner: planner, ProgressMetrics: &ProgressMetrics{ NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_number_of_iterations", + Name: "thanos_compact_todo_compactions", Help: "number of iterations to be done", }, []string{"group"}), NumberOfCompactionBlocks: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_number_of_blocks_planned", + Name: "thanos_compact_todo_compaction_blocks", Help: "number of blocks planned to be merged", }, []string{"group"}), }, @@ -576,7 +576,7 @@ func NewDownsampleSimulator(reg prometheus.Registerer) *DownsampleSimulator { return &DownsampleSimulator{ DownsampleMetrics: &DownsampleMetrics{ BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_blocks_downsampled", + Name: "thanos_compact_todo_downsample_blocks", Help: "number of blocks to be downsampled", }, []string{"group"}), }, From 825a721760a39b988e5605c2ae45020e0342ad21 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 29 Oct 2021 09:34:04 +0530 Subject: [PATCH 41/67] changed to regular flag; recreate simulator obj every run Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 50 ++++++++++++++------------------------ docs/components/compact.md | 6 ++--- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index a6491efc37..1e3f8727ca 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -55,10 +55,6 @@ var ( } ) -const ( - compactionProgressMetrics = "compact-progress-metrics" -) - type compactionSet []time.Duration func (cs compactionSet) String() string { @@ -93,12 +89,6 @@ func registerCompact(app *extkingpin.App) { conf.registerFlag(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { - for _, f := range conf.enableFeature { - if f == compactionProgressMetrics { - conf.compactionProgressMetrics = true - } - } - return runCompact(g, logger, tracer, reg, component.Compact, *conf, getFlagsMap(cmd.Flags())) }) } @@ -471,12 +461,9 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - var ds *compact.DownsampleSimulator - if !conf.disableDownsampling { - ds = compact.NewDownsampleSimulator(reg) - } - ps := compact.NewCompactionSimulator(reg, tsdbPlanner) return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { + ps := compact.NewCompactionSimulator(reg, tsdbPlanner) + if err := sy.SyncMetas(ctx); err != nil { return errors.Wrapf(err, "could not sync metas") } @@ -486,13 +473,22 @@ func runCompact( if err != nil { return errors.Wrapf(err, "could not group original metadata") } - downGroups := make([]*compact.Group, len(groups)) - for ind, group := range groups { - if group == nil { - continue + + if !conf.disableDownsampling { + ds := compact.NewDownsampleSimulator(reg) + + downGroups := make([]*compact.Group, len(groups)) + for ind, group := range groups { + v := *group + downGroups[ind] = &v + } + + for _, group := range downGroups { + ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) + } + if err := ds.ProgressCalculate(ctx, downGroups); err != nil { + return errors.Wrapf(err, "could not simulate downsampling") } - v := *group - downGroups[ind] = &v } for _, group := range groups { @@ -504,15 +500,6 @@ func runCompact( return errors.Wrapf(err, "could not simulate planning") } - if !conf.disableDownsampling { - for _, group := range downGroups { - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) - } - if err := ds.ProgressCalculate(ctx, downGroups); err != nil { - return errors.Wrapf(err, "could not simulate downsampling") - } - } - return nil }) }, func(err error) { @@ -653,11 +640,10 @@ type compactConfig struct { dedupFunc string skipBlockWithOutOfOrderChunks bool compactionProgressMetrics bool - enableFeature []string } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { - cmd.Flag("enable-feature", "Comma separated experimental feature names to enable.The current list of features is "+compactionProgressMetrics+".").Default("").StringsVar(&cc.enableFeature) + cmd.Flag("progress-metrics", "Enables the progress metrics, indicating the progress of the compaction and downsampling processes").Default("false").BoolVar(&cc.compactionProgressMetrics) cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected."). Hidden().Default("true").BoolVar(&cc.haltOnError) diff --git a/docs/components/compact.md b/docs/components/compact.md index 7d66b1d8de..79aa7a6c4b 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -348,9 +348,6 @@ Flags: non-downsampled data is not efficient and useful e.g it is not possible to render all samples for a human eye anyway - --enable-feature= ... Comma separated experimental feature names to - enable.The current list of features is - compact-progress-metrics. --hash-func= Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. @@ -379,6 +376,9 @@ Flags: Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration + --progress-metrics Enables the progress metrics, indicating the + progress of the compaction and downsampling + processes --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 hour) in bucket. Setting this to 0d will retain From 6b9b3269f23c60335249b8286c78d08c33178c7c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 29 Oct 2021 11:55:26 +0530 Subject: [PATCH 42/67] added explanatory comments Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 5 +++++ pkg/compact/compact_test.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index f45d9c9c0e..07be64a1b9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -483,15 +483,18 @@ func (cg *Group) Resolution() int64 { return cg.resolution } +// ProgressMetrics contains Prometheus metrics related to planning and compaction progress. type ProgressMetrics struct { NumberOfCompactionRuns *prometheus.GaugeVec NumberOfCompactionBlocks *prometheus.GaugeVec } +// ProgressCalculator updates the results/metrics from the compaction planning and downsampling simulations. type ProgressCalculator interface { ProgressCalculate(ctx context.Context, groups []*Group) error } +// CompactionSimulator contains a planner and ProgressMetrics, which are updated during the compaction simulation process type CompactionSimulator struct { planner Planner *ProgressMetrics @@ -564,10 +567,12 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* return nil } +// DownsampleMetrics contains Prometheus metrics related to downsampling progress. type DownsampleMetrics struct { BlocksDownsampled *prometheus.GaugeVec } +// DownsampleSimulator contains DownsampleMetrics, which are updated during the downsampling simulation process. type DownsampleSimulator struct { *DownsampleMetrics } diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index b1b111a6f0..6c4d74bddf 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -292,6 +292,8 @@ func TestPlanSimulate(t *testing.T) { err := ps.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ps.ProgressMetrics + // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. + // Hence, a total of 6 blocks are planned for compaction over 2 runs. testutil.Equals(t, 2.0, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) testutil.Equals(t, 6.0, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) } From adba1297fc929e478ea68823aebe00cc761c937c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 29 Oct 2021 12:32:03 +0530 Subject: [PATCH 43/67] removed extra var Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 6c4d74bddf..b356d72c10 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -179,8 +179,7 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } func TestPlanSimulate(t *testing.T) { - logger := log.NewNopLogger() - planner := NewTSDBBasedPlanner(logger, []int64{ + planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{ int64(1 * time.Hour / time.Millisecond), int64(2 * time.Hour / time.Millisecond), int64(8 * time.Hour / time.Millisecond), From 4d7da02d3c94e49aa5827ecf9abd6158a2a75206 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 30 Oct 2021 08:38:31 +0530 Subject: [PATCH 44/67] re-register simulators Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1e3f8727ca..809b1678d7 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -40,6 +40,7 @@ import ( "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" + "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/pkg/runutil" httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/ui" @@ -461,8 +462,13 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { + ps := compact.NewCompactionSimulator(reg, tsdbPlanner) + var ds *compact.DownsampleSimulator + if !conf.disableDownsampling { + ds = compact.NewDownsampleSimulator(reg) + } + return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { - ps := compact.NewCompactionSimulator(reg, tsdbPlanner) if err := sy.SyncMetas(ctx); err != nil { return errors.Wrapf(err, "could not sync metas") @@ -474,15 +480,19 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - if !conf.disableDownsampling { - ds := compact.NewDownsampleSimulator(reg) + u := receive.UnRegisterer{Registerer: reg} + u.MustRegister(ps.ProgressMetrics.NumberOfCompactionRuns) + u.MustRegister(ps.ProgressMetrics.NumberOfCompactionBlocks) + if !conf.disableDownsampling { downGroups := make([]*compact.Group, len(groups)) for ind, group := range groups { v := *group downGroups[ind] = &v } + u.MustRegister(ds.DownsampleMetrics.BlocksDownsampled) + for _, group := range downGroups { ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) } From a0ea1d4cbf6bc6f1ffd5ada067842a4f6f11cbf7 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 30 Oct 2021 09:21:20 +0530 Subject: [PATCH 45/67] fixed names and comments Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 18 +++++++------- pkg/compact/compact.go | 48 ++++++++++++++++++------------------- pkg/compact/compact_test.go | 10 ++++---- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 809b1678d7..a3bc5b755c 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -462,10 +462,10 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - ps := compact.NewCompactionSimulator(reg, tsdbPlanner) - var ds *compact.DownsampleSimulator + ps := compact.NewCompactionProgressCalculator(reg, tsdbPlanner) + var ds *compact.DownsampleProgressCalculator if !conf.disableDownsampling { - ds = compact.NewDownsampleSimulator(reg) + ds = compact.NewDownsampleProgressCalculator(reg) } return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { @@ -481,8 +481,8 @@ func runCompact( } u := receive.UnRegisterer{Registerer: reg} - u.MustRegister(ps.ProgressMetrics.NumberOfCompactionRuns) - u.MustRegister(ps.ProgressMetrics.NumberOfCompactionBlocks) + u.MustRegister(ps.CompactProgressMetrics.NumberOfCompactionRuns) + u.MustRegister(ps.CompactProgressMetrics.NumberOfCompactionBlocks) if !conf.disableDownsampling { downGroups := make([]*compact.Group, len(groups)) @@ -491,10 +491,10 @@ func runCompact( downGroups[ind] = &v } - u.MustRegister(ds.DownsampleMetrics.BlocksDownsampled) + u.MustRegister(ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled) for _, group := range downGroups { - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(group.Key()) + ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(group.Key()) } if err := ds.ProgressCalculate(ctx, downGroups); err != nil { return errors.Wrapf(err, "could not simulate downsampling") @@ -502,8 +502,8 @@ func runCompact( } for _, group := range groups { - ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) - ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) + ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) + ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) } if err = ps.ProgressCalculate(ctx, groups); err != nil { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 07be64a1b9..eb4c22c5e6 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -483,27 +483,27 @@ func (cg *Group) Resolution() int64 { return cg.resolution } -// ProgressMetrics contains Prometheus metrics related to planning and compaction progress. -type ProgressMetrics struct { +// CompactProgressMetrics contains Prometheus metrics related to planning and compaction progress. +type CompactProgressMetrics struct { NumberOfCompactionRuns *prometheus.GaugeVec NumberOfCompactionBlocks *prometheus.GaugeVec } -// ProgressCalculator updates the results/metrics from the compaction planning and downsampling simulations. +// ProgressCalculator calculates the progress of the compaction process for a given slice of Groups. type ProgressCalculator interface { ProgressCalculate(ctx context.Context, groups []*Group) error } -// CompactionSimulator contains a planner and ProgressMetrics, which are updated during the compaction simulation process -type CompactionSimulator struct { +// CompactionProgressCalculator contains a planner and ProgressMetrics, which are updated during the compaction simulation process +type CompactionProgressCalculator struct { planner Planner - *ProgressMetrics + *CompactProgressMetrics } -func NewCompactionSimulator(reg prometheus.Registerer, planner *tsdbBasedPlanner) *CompactionSimulator { - return &CompactionSimulator{ +func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBasedPlanner) *CompactionProgressCalculator { + return &CompactionProgressCalculator{ planner: planner, - ProgressMetrics: &ProgressMetrics{ + CompactProgressMetrics: &CompactProgressMetrics{ NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_compact_todo_compactions", Help: "number of iterations to be done", @@ -516,7 +516,7 @@ func NewCompactionSimulator(reg prometheus.Registerer, planner *tsdbBasedPlanner } } -func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ps *CompactionProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -560,27 +560,27 @@ func (ps *CompactionSimulator) ProgressCalculate(ctx context.Context, groups []* } for key, iters := range groupCompactions { - ps.ProgressMetrics.NumberOfCompactionRuns.WithLabelValues(key).Add(float64(iters)) - ps.ProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(key).Add(float64(groupBlocks[key])) + ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(key).Add(float64(iters)) + ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(key).Add(float64(groupBlocks[key])) } return nil } -// DownsampleMetrics contains Prometheus metrics related to downsampling progress. -type DownsampleMetrics struct { - BlocksDownsampled *prometheus.GaugeVec +// DownsampleProgressMetrics contains Prometheus metrics related to downsampling progress. +type DownsampleProgressMetrics struct { + NumberOfBlocksDownsampled *prometheus.GaugeVec } -// DownsampleSimulator contains DownsampleMetrics, which are updated during the downsampling simulation process. -type DownsampleSimulator struct { - *DownsampleMetrics +// DownsampleProgressCalculator contains DownsampleMetrics, which are updated during the downsampling simulation process. +type DownsampleProgressCalculator struct { + *DownsampleProgressMetrics } -func NewDownsampleSimulator(reg prometheus.Registerer) *DownsampleSimulator { - return &DownsampleSimulator{ - DownsampleMetrics: &DownsampleMetrics{ - BlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ +func NewDownsampleProgressCalculator(reg prometheus.Registerer) *DownsampleProgressCalculator { + return &DownsampleProgressCalculator{ + DownsampleProgressMetrics: &DownsampleProgressMetrics{ + NumberOfBlocksDownsampled: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_compact_todo_downsample_blocks", Help: "number of blocks to be downsampled", }, []string{"group"}), @@ -588,7 +588,7 @@ func NewDownsampleSimulator(reg prometheus.Registerer) *DownsampleSimulator { } } -func (ds *DownsampleSimulator) ProgressCalculate(ctx context.Context, groups []*Group) error { +func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} groupBlocks := make(map[string]int, len(groups)) @@ -653,7 +653,7 @@ func (ds *DownsampleSimulator) ProgressCalculate(ctx context.Context, groups []* } for key, blocks := range groupBlocks { - ds.DownsampleMetrics.BlocksDownsampled.WithLabelValues(key).Add(float64(blocks)) + ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(key).Add(float64(blocks)) } return nil diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index b356d72c10..9e75874475 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -186,7 +186,7 @@ func TestPlanSimulate(t *testing.T) { int64(2 * 24 * time.Hour / time.Millisecond), }) reg := prometheus.NewRegistry() - ps := NewCompactionSimulator(reg, planner) + ps := NewCompactionProgressCalculator(reg, planner) metas := []*metadata.Meta{ { @@ -290,7 +290,7 @@ func TestPlanSimulate(t *testing.T) { err := ps.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) - metrics := ps.ProgressMetrics + metrics := ps.CompactProgressMetrics // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. // Hence, a total of 6 blocks are planned for compaction over 2 runs. testutil.Equals(t, 2.0, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) @@ -394,7 +394,7 @@ func TestDownsampleSimulate(t *testing.T) { }, } { reg := prometheus.NewRegistry() - ds := NewDownsampleSimulator(reg) + ds := NewDownsampleProgressCalculator(reg) extLabels := labels.FromMap(map[string]string{"a": "1"}) groups := []*Group{ @@ -408,8 +408,8 @@ func TestDownsampleSimulate(t *testing.T) { if ok := t.Run(tcase.testName, func(t *testing.T) { err := ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) - metrics := ds.DownsampleMetrics - testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(metrics.BlocksDownsampled.WithLabelValues("a"))) + metrics := ds.DownsampleProgressMetrics + testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(metrics.NumberOfBlocksDownsampled.WithLabelValues("a"))) }); !ok { return } From 5c5d7b6927922c3bff45556ba5538a31361e712d Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 30 Oct 2021 17:44:50 +0530 Subject: [PATCH 46/67] fix simulator obj registration Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index a3bc5b755c..d58300996a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -462,10 +462,11 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - ps := compact.NewCompactionProgressCalculator(reg, tsdbPlanner) + unRegisterer := &receive.UnRegisterer{Registerer: reg} + ps := compact.NewCompactionProgressCalculator(unRegisterer, tsdbPlanner) var ds *compact.DownsampleProgressCalculator if !conf.disableDownsampling { - ds = compact.NewDownsampleProgressCalculator(reg) + ds = compact.NewDownsampleProgressCalculator(unRegisterer) } return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { @@ -480,10 +481,6 @@ func runCompact( return errors.Wrapf(err, "could not group original metadata") } - u := receive.UnRegisterer{Registerer: reg} - u.MustRegister(ps.CompactProgressMetrics.NumberOfCompactionRuns) - u.MustRegister(ps.CompactProgressMetrics.NumberOfCompactionBlocks) - if !conf.disableDownsampling { downGroups := make([]*compact.Group, len(groups)) for ind, group := range groups { @@ -491,8 +488,6 @@ func runCompact( downGroups[ind] = &v } - u.MustRegister(ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled) - for _, group := range downGroups { ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(group.Key()) } From d788f9929f51a3c7282c60a05d3994a774b8ba65 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 2 Nov 2021 08:45:06 +0530 Subject: [PATCH 47/67] added some more test cases; table test for plan simulate Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 332 ++++++++++++++++++++++++------------ 1 file changed, 226 insertions(+), 106 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 9e75874475..91fdee7487 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -26,6 +26,7 @@ import ( "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -178,126 +179,224 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } +type planResult struct { + compactionBlocks, compactionRuns float64 +} + func TestPlanSimulate(t *testing.T) { - planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{ - int64(1 * time.Hour / time.Millisecond), - int64(2 * time.Hour / time.Millisecond), - int64(8 * time.Hour / time.Millisecond), - int64(2 * 24 * time.Hour / time.Millisecond), - }) reg := prometheus.NewRegistry() - ps := NewCompactionProgressCalculator(reg, planner) + unRegisterer := &receive.UnRegisterer{Registerer: reg} + planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{}) - metas := []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, + extLabels := labels.FromMap(map[string]string{"a": "1"}) + + for _, tcase := range []struct { + testName string + testPlannerRanges []int64 + input []*metadata.Meta + expected planResult + }{ + // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. + // Hence, a total of 6 blocks are planned for compaction over 2 runs. { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), + testName: "two_runs", + testPlannerRanges: []int64{ + int64(1 * time.Hour / time.Millisecond), + int64(2 * time.Hour / time.Millisecond), + int64(8 * time.Hour / time.Millisecond), + int64(2 * 24 * time.Hour / time.Millisecond), }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(5, nil), - MinTime: int64(10 * time.Hour / time.Millisecond), - MaxTime: int64(12 * time.Hour / time.Millisecond), + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(5, nil), + MinTime: int64(10 * time.Hour / time.Millisecond), + MaxTime: int64(12 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(6, nil), + MinTime: int64(12 * time.Hour / time.Millisecond), + MaxTime: int64(20 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(7, nil), + MinTime: int64(20 * time.Hour / time.Millisecond), + MaxTime: int64(28 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + expected: planResult{ + compactionRuns: 2.0, + compactionBlocks: 6.0, }, }, { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(6, nil), - MinTime: int64(12 * time.Hour / time.Millisecond), - MaxTime: int64(20 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + // In this test case, the first four blocks are compacted into an 8h block in the first run. + testName: "single_run", + testPlannerRanges: []int64{ + int64(1 * time.Hour / time.Millisecond), + int64(2 * time.Hour / time.Millisecond), + int64(8 * time.Hour / time.Millisecond), }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(7, nil), - MinTime: int64(20 * time.Hour / time.Millisecond), - MaxTime: int64(28 * time.Hour / time.Millisecond), + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + expected: planResult{ + compactionRuns: 1.0, + compactionBlocks: 4.0, }, }, + } { + if ok := t.Run(tcase.testName, func(t *testing.T) { + groups := []*Group{ + { + labels: extLabels, + resolution: 0, + metasByMinTime: tcase.input, + }, + } + planner.ranges = tcase.testPlannerRanges + ps := NewCompactionProgressCalculator(unRegisterer, planner) + err := ps.ProgressCalculate(context.Background(), groups) + testutil.Ok(t, err) + metrics := ps.CompactProgressMetrics + testutil.Equals(t, tcase.expected.compactionBlocks, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) + testutil.Equals(t, tcase.expected.compactionRuns, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) + }); !ok { + return + } } - extLabels := labels.FromMap(map[string]string{"a": "1"}) - groups := []*Group{ - { - labels: extLabels, - resolution: 0, - metasByMinTime: metas, - }, - } - - err := ps.ProgressCalculate(context.Background(), groups) - testutil.Ok(t, err) - metrics := ps.CompactProgressMetrics - // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. - // Hence, a total of 6 blocks are planned for compaction over 2 runs. - testutil.Equals(t, 2.0, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) - testutil.Equals(t, 6.0, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) } func TestDownsampleSimulate(t *testing.T) { + reg := prometheus.NewRegistry() + unRegisterer := &receive.UnRegisterer{Registerer: reg} for _, tcase := range []struct { testName string @@ -305,7 +404,7 @@ func TestDownsampleSimulate(t *testing.T) { expected float64 }{ { - // this test case has 1 block to be downsampled out of 2 since for the second block, the difference between MinTime and MaxTime is less than the acceptable threshold, DownsampleRange0 + // This test case has 1 block to be downsampled out of 2 since for the second block, the difference between MinTime and MaxTime is less than the acceptable threshold, DownsampleRange0 testName: "min_max_time_diff_test", input: []*metadata.Meta{ { @@ -369,7 +468,7 @@ func TestDownsampleSimulate(t *testing.T) { }, expected: 0.0, }, { - // this test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel0, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange0. + // This test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel0, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange0. testName: "res_level_0_test", input: []*metadata.Meta{ { @@ -391,10 +490,31 @@ func TestDownsampleSimulate(t *testing.T) { }, }, expected: 1.0, + }, { + testName: "res_level_1_test", + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(10, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(11, nil), ulid.MustNew(12, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel1, + }, + }, + }, + }, + expected: 1.0, }, } { - reg := prometheus.NewRegistry() - ds := NewDownsampleProgressCalculator(reg) + ds := NewDownsampleProgressCalculator(unRegisterer) extLabels := labels.FromMap(map[string]string{"a": "1"}) groups := []*Group{ From daea94673a7a04704caac9155adbb3c4d18e9a76 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 2 Nov 2021 09:05:22 +0530 Subject: [PATCH 48/67] added comments for some public methods Signed-off-by: metonymic-smokey --- pkg/compact/compact.go | 4 ++++ pkg/compact/compact_test.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index eb4c22c5e6..042a1c3efd 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -500,6 +500,7 @@ type CompactionProgressCalculator struct { *CompactProgressMetrics } +// NewCompactProgressCalculator creates a new CompactionProgressCalculator func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBasedPlanner) *CompactionProgressCalculator { return &CompactionProgressCalculator{ planner: planner, @@ -516,6 +517,7 @@ func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBas } } +// ProgressCalculate calculates the number of blocks and compaction runs in the planning process of the given groups func (ps *CompactionProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -577,6 +579,7 @@ type DownsampleProgressCalculator struct { *DownsampleProgressMetrics } +// NewDownsampleProgressCalculator creates a new DownsampleProgressCalculator func NewDownsampleProgressCalculator(reg prometheus.Registerer) *DownsampleProgressCalculator { return &DownsampleProgressCalculator{ DownsampleProgressMetrics: &DownsampleProgressMetrics{ @@ -588,6 +591,7 @@ func NewDownsampleProgressCalculator(reg prometheus.Registerer) *DownsampleProgr } } +// ProgressCalculate calculates the number of blocks to be downsampled for the given groups func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 91fdee7487..1bbef04a0a 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -183,7 +183,7 @@ type planResult struct { compactionBlocks, compactionRuns float64 } -func TestPlanSimulate(t *testing.T) { +func TestCompactProgressCalculate(t *testing.T) { reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{}) @@ -394,7 +394,7 @@ func TestPlanSimulate(t *testing.T) { } -func TestDownsampleSimulate(t *testing.T) { +func TestDownsampleProgressCalculate(t *testing.T) { reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} From 60bbe9edf80de1af767270fe37ff8817d8a6cc9b Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 2 Nov 2021 10:47:51 +0530 Subject: [PATCH 49/67] lint fixes Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 8 +++----- pkg/compact/compact.go | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index d58300996a..fb46e02e9c 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -482,12 +482,10 @@ func runCompact( } if !conf.disableDownsampling { - downGroups := make([]*compact.Group, len(groups)) - for ind, group := range groups { - v := *group - downGroups[ind] = &v + downGroups, err := grouper.Groups(originalMetas) + if err != nil { + return errors.Wrapf(err, "could not group original metadata into downsample groups") } - for _, group := range downGroups { ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(group.Key()) } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 042a1c3efd..177e80b642 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -500,7 +500,7 @@ type CompactionProgressCalculator struct { *CompactProgressMetrics } -// NewCompactProgressCalculator creates a new CompactionProgressCalculator +// NewCompactProgressCalculator creates a new CompactionProgressCalculator. func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBasedPlanner) *CompactionProgressCalculator { return &CompactionProgressCalculator{ planner: planner, @@ -517,7 +517,7 @@ func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBas } } -// ProgressCalculate calculates the number of blocks and compaction runs in the planning process of the given groups +// ProgressCalculate calculates the number of blocks and compaction runs in the planning process of the given groups. func (ps *CompactionProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { groupCompactions := make(map[string]int, len(groups)) groupBlocks := make(map[string]int, len(groups)) @@ -579,7 +579,7 @@ type DownsampleProgressCalculator struct { *DownsampleProgressMetrics } -// NewDownsampleProgressCalculator creates a new DownsampleProgressCalculator +// NewDownsampleProgressCalculator creates a new DownsampleProgressCalculator. func NewDownsampleProgressCalculator(reg prometheus.Registerer) *DownsampleProgressCalculator { return &DownsampleProgressCalculator{ DownsampleProgressMetrics: &DownsampleProgressMetrics{ @@ -591,7 +591,7 @@ func NewDownsampleProgressCalculator(reg prometheus.Registerer) *DownsampleProgr } } -// ProgressCalculate calculates the number of blocks to be downsampled for the given groups +// ProgressCalculate calculates the number of blocks to be downsampled for the given groups. func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, groups []*Group) error { sources5m := map[ulid.ULID]struct{}{} sources1h := map[ulid.ULID]struct{}{} From 9c3d11ceddfe8120e6b7a58947458abc0b0e13b9 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 2 Nov 2021 12:24:02 +0530 Subject: [PATCH 50/67] fixed names, logs and comments Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 30 +++++++++++++++--------------- pkg/compact/compact.go | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index fb46e02e9c..f2cdc57086 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -475,34 +475,34 @@ func runCompact( return errors.Wrapf(err, "could not sync metas") } - originalMetas := sy.Metas() - groups, err := grouper.Groups(originalMetas) + metas := sy.Metas() + groups, err := grouper.Groups(metas) if err != nil { return errors.Wrapf(err, "could not group original metadata") } + for _, group := range groups { + ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) + ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) + } + + if err = ps.ProgressCalculate(ctx, groups); err != nil { + return errors.Wrapf(err, "could not calculate compaction progress") + } + if !conf.disableDownsampling { - downGroups, err := grouper.Groups(originalMetas) + groups, err = grouper.Groups(metas) if err != nil { return errors.Wrapf(err, "could not group original metadata into downsample groups") } - for _, group := range downGroups { + for _, group := range groups { ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(group.Key()) } - if err := ds.ProgressCalculate(ctx, downGroups); err != nil { - return errors.Wrapf(err, "could not simulate downsampling") + if err := ds.ProgressCalculate(ctx, groups); err != nil { + return errors.Wrapf(err, "could not calculate downsampling progress") } } - for _, group := range groups { - ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) - ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) - } - - if err = ps.ProgressCalculate(ctx, groups); err != nil { - return errors.Wrapf(err, "could not simulate planning") - } - return nil }) }, func(err error) { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 177e80b642..02481583f3 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -494,7 +494,7 @@ type ProgressCalculator interface { ProgressCalculate(ctx context.Context, groups []*Group) error } -// CompactionProgressCalculator contains a planner and ProgressMetrics, which are updated during the compaction simulation process +// CompactionProgressCalculator contains a planner and ProgressMetrics, which are updated during the compaction simulation process. type CompactionProgressCalculator struct { planner Planner *CompactProgressMetrics From afd48bf76a51ced4c01a17aa0dc81976c21fc0a9 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Tue, 2 Nov 2021 17:41:06 +0530 Subject: [PATCH 51/67] refactored test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 1bbef04a0a..c405a8204d 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -179,33 +179,31 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } -type planResult struct { - compactionBlocks, compactionRuns float64 -} - func TestCompactProgressCalculate(t *testing.T) { + type planResult struct { + compactionBlocks, compactionRuns float64 + } + reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} - planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{}) + planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{ + int64(1 * time.Hour / time.Millisecond), + int64(2 * time.Hour / time.Millisecond), + int64(8 * time.Hour / time.Millisecond), + int64(2 * 24 * time.Hour / time.Millisecond), + }) extLabels := labels.FromMap(map[string]string{"a": "1"}) for _, tcase := range []struct { - testName string - testPlannerRanges []int64 - input []*metadata.Meta - expected planResult + testName string + input []*metadata.Meta + expected planResult }{ // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. // Hence, a total of 6 blocks are planned for compaction over 2 runs. { testName: "two_runs", - testPlannerRanges: []int64{ - int64(1 * time.Hour / time.Millisecond), - int64(2 * time.Hour / time.Millisecond), - int64(8 * time.Hour / time.Millisecond), - int64(2 * 24 * time.Hour / time.Millisecond), - }, input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -304,11 +302,6 @@ func TestCompactProgressCalculate(t *testing.T) { { // In this test case, the first four blocks are compacted into an 8h block in the first run. testName: "single_run", - testPlannerRanges: []int64{ - int64(1 * time.Hour / time.Millisecond), - int64(2 * time.Hour / time.Millisecond), - int64(8 * time.Hour / time.Millisecond), - }, input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -380,7 +373,6 @@ func TestCompactProgressCalculate(t *testing.T) { metasByMinTime: tcase.input, }, } - planner.ranges = tcase.testPlannerRanges ps := NewCompactionProgressCalculator(unRegisterer, planner) err := ps.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) From 8f1fff7db519948fed009a9dfec7d9704e46920a Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 3 Nov 2021 09:07:06 +0530 Subject: [PATCH 52/67] added test cases with two groups Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 139 ++++++++++++++++++++++++++++++++++-- 1 file changed, 135 insertions(+), 4 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index c405a8204d..1a8c723c15 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -193,7 +193,7 @@ func TestCompactProgressCalculate(t *testing.T) { int64(2 * 24 * time.Hour / time.Millisecond), }) - extLabels := labels.FromMap(map[string]string{"a": "1"}) + extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) for _, tcase := range []struct { testName string @@ -364,6 +364,94 @@ func TestCompactProgressCalculate(t *testing.T) { compactionBlocks: 4.0, }, }, + { + // In this test case, the metadata is part of two groups. + // The first four blocks are compacted in the first run. + testName: "two_groups", + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(16 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(6, nil), + MinTime: int64(16 * time.Hour / time.Millisecond), + MaxTime: int64(18 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(7, nil), + MinTime: int64(18 * time.Hour / time.Millisecond), + MaxTime: int64(26 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + }, + expected: planResult{ + compactionRuns: 1.0, + compactionBlocks: 4.0, + }, + }, } { if ok := t.Run(tcase.testName, func(t *testing.T) { groups := []*Group{ @@ -389,6 +477,7 @@ func TestCompactProgressCalculate(t *testing.T) { func TestDownsampleProgressCalculate(t *testing.T) { reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} + extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) for _, tcase := range []struct { testName string @@ -483,6 +572,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, expected: 1.0, }, { + // This test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel1, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange1. testName: "res_level_1_test", input: []*metadata.Meta{ { @@ -505,10 +595,49 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, expected: 1.0, }, + { + // This test case has metadata belonging to two input groups. + // It returns two blocks to be downsampled since for both the blocks, the difference between MinTime and MaxTime is above the accepted threshold for their resolution level. + testName: "two_groups", + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel1, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel0, + }, + }, + }, + }, + expected: 2.0, + }, } { - ds := NewDownsampleProgressCalculator(unRegisterer) - - extLabels := labels.FromMap(map[string]string{"a": "1"}) groups := []*Group{ { key: "a", @@ -517,6 +646,8 @@ func TestDownsampleProgressCalculate(t *testing.T) { metasByMinTime: tcase.input, }, } + + ds := NewDownsampleProgressCalculator(unRegisterer) if ok := t.Run(tcase.testName, func(t *testing.T) { err := ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) From ff3ea04db60b7b607f38761f82243ff4b59b1f05 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 3 Nov 2021 09:21:40 +0530 Subject: [PATCH 53/67] added test case with non consecutive blocks Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 77 +++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 1a8c723c15..c21f0065aa 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -299,6 +299,83 @@ func TestCompactProgressCalculate(t *testing.T) { compactionBlocks: 6.0, }, }, + { + // This test case has non-consecutive blocks. + // The first four blocks are planned for compaction, like the previous case. But unlike the previous case, the next two blocks are not planned for compaction since these 6 blocks are not consecutive. + testName: "non_consecutive_blocks", + input: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(10 * time.Hour / time.Millisecond), + MaxTime: int64(12 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(12 * time.Hour / time.Millisecond), + MaxTime: int64(14 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + }, + }, + }, + expected: planResult{ + compactionRuns: 1.0, + compactionBlocks: 4.0, + }, + }, { // In this test case, the first four blocks are compacted into an 8h block in the first run. testName: "single_run", From fa5f03f9d8696e4d774dcd33c87968b442f5ac7e Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 3 Nov 2021 09:29:00 +0530 Subject: [PATCH 54/67] fixed some nits Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 4 ++-- pkg/compact/compact_test.go | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index f2cdc57086..04ed810326 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -646,7 +646,7 @@ type compactConfig struct { } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { - cmd.Flag("progress-metrics", "Enables the progress metrics, indicating the progress of the compaction and downsampling processes").Default("false").BoolVar(&cc.compactionProgressMetrics) + cmd.Flag("progress-metrics", "Enables the progress metrics, indicating the progress of compaction and downsampling").Default("false").BoolVar(&cc.compactionProgressMetrics) cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected."). Hidden().Default("true").BoolVar(&cc.haltOnError) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 02481583f3..17aab3f541 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -483,7 +483,7 @@ func (cg *Group) Resolution() int64 { return cg.resolution } -// CompactProgressMetrics contains Prometheus metrics related to planning and compaction progress. +// CompactProgressMetrics contains Prometheus metrics related to compaction progress. type CompactProgressMetrics struct { NumberOfCompactionRuns *prometheus.GaugeVec NumberOfCompactionBlocks *prometheus.GaugeVec @@ -511,7 +511,7 @@ func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBas }, []string{"group"}), NumberOfCompactionBlocks: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_compact_todo_compaction_blocks", - Help: "number of blocks planned to be merged", + Help: "number of blocks planned to be compacted", }, []string{"group"}), }, } diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index c21f0065aa..47acc531c7 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -203,7 +203,7 @@ func TestCompactProgressCalculate(t *testing.T) { // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. // Hence, a total of 6 blocks are planned for compaction over 2 runs. { - testName: "two_runs", + testName: "two_runs_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -302,7 +302,7 @@ func TestCompactProgressCalculate(t *testing.T) { { // This test case has non-consecutive blocks. // The first four blocks are planned for compaction, like the previous case. But unlike the previous case, the next two blocks are not planned for compaction since these 6 blocks are not consecutive. - testName: "non_consecutive_blocks", + testName: "non_consecutive_blocks_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -378,7 +378,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, { // In this test case, the first four blocks are compacted into an 8h block in the first run. - testName: "single_run", + testName: "single_run_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -444,7 +444,7 @@ func TestCompactProgressCalculate(t *testing.T) { { // In this test case, the metadata is part of two groups. // The first four blocks are compacted in the first run. - testName: "two_groups", + testName: "two_groups_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -563,7 +563,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { }{ { // This test case has 1 block to be downsampled out of 2 since for the second block, the difference between MinTime and MaxTime is less than the acceptable threshold, DownsampleRange0 - testName: "min_max_time_diff_test", + testName: "minTime_maxTime_diff_threshold_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -603,7 +603,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { expected: 1.0, }, { - // this test case returns 0 blocks to be downsampled since the resolution is resLevel2, which is skipped when grouping blocks for downsampling. + // This test case returns 0 blocks to be downsampled since the resolution is resLevel2, which is skipped when grouping blocks for downsampling. testName: "res_level_2_test", input: []*metadata.Meta{ { @@ -675,7 +675,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { { // This test case has metadata belonging to two input groups. // It returns two blocks to be downsampled since for both the blocks, the difference between MinTime and MaxTime is above the accepted threshold for their resolution level. - testName: "two_groups", + testName: "two_groups_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ From 3c5cbcebd0c4982caf95c3fdefb3c4435bda3242 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 3 Nov 2021 13:15:03 +0530 Subject: [PATCH 55/67] fixed two_groups_test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 156 ++++++++++++++++++++++++++++++++++-- 1 file changed, 151 insertions(+), 5 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 47acc531c7..76177e4445 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -184,9 +185,10 @@ func TestCompactProgressCalculate(t *testing.T) { compactionBlocks, compactionRuns float64 } + logger := log.NewNopLogger() reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} - planner := NewTSDBBasedPlanner(log.NewNopLogger(), []int64{ + planner := NewTSDBBasedPlanner(logger, []int64{ int64(1 * time.Hour / time.Millisecond), int64(2 * time.Hour / time.Millisecond), int64(8 * time.Hour / time.Millisecond), @@ -533,17 +535,106 @@ func TestCompactProgressCalculate(t *testing.T) { if ok := t.Run(tcase.testName, func(t *testing.T) { groups := []*Group{ { + key: "b", labels: extLabels, resolution: 0, metasByMinTime: tcase.input, }, } + + if tcase.testName == "two_groups_test" { + groups = append(groups, &Group{ + key: "a", + metasByMinTime: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: int64(0 * time.Hour / time.Millisecond), + MaxTime: int64(2 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + MinTime: int64(2 * time.Hour / time.Millisecond), + MaxTime: int64(4 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + MinTime: int64(4 * time.Hour / time.Millisecond), + MaxTime: int64(6 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: int64(6 * time.Hour / time.Millisecond), + MaxTime: int64(8 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + }, + }, + }, + resolution: 0, + }) + } + ps := NewCompactionProgressCalculator(unRegisterer, planner) err := ps.ProgressCalculate(context.Background(), groups) - testutil.Ok(t, err) metrics := ps.CompactProgressMetrics - testutil.Equals(t, tcase.expected.compactionBlocks, promtestutil.ToFloat64(metrics.NumberOfCompactionBlocks)) - testutil.Equals(t, tcase.expected.compactionRuns, promtestutil.ToFloat64(metrics.NumberOfCompactionRuns)) + testutil.Ok(t, err) + a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("b") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of blocks") + } + testutil.Equals(t, tcase.expected.compactionBlocks, promtestutil.ToFloat64(a)) + + b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("b") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of runs") + } + testutil.Equals(t, tcase.expected.compactionRuns, promtestutil.ToFloat64(b)) + + if tcase.testName == "two_groups_test" { + a, err = metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("a") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of blocks for two groups test") + } + testutil.Equals(t, 4.0, promtestutil.ToFloat64(a)) + b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("a") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of runs") + } + testutil.Equals(t, 1.0, promtestutil.ToFloat64(b)) + + } }); !ok { return } @@ -555,6 +646,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) + logger := log.NewNopLogger() for _, tcase := range []struct { testName string @@ -724,12 +816,66 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, } + if tcase.testName == "two_groups_test" { + groups = append(groups, &Group{ + resolution: 0, + key: "b", + metasByMinTime: []*metadata.Meta{ + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(0, nil), + MinTime: 0, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel1, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(3, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel0, + }, + }, + }, + }, + }) + } + ds := NewDownsampleProgressCalculator(unRegisterer) if ok := t.Run(tcase.testName, func(t *testing.T) { err := ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ds.DownsampleProgressMetrics - testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(metrics.NumberOfBlocksDownsampled.WithLabelValues("a"))) + if tcase.testName == "two_groups_test" { + a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues("b") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of blocks") + } + testutil.Equals(t, 1.0, promtestutil.ToFloat64(a)) + } + a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues("a") + if err != nil { + level.Warn(logger).Log("msg", "could not get number of blocks") + } + testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(a)) }); !ok { return } From 35c4ff7d677610042897b9cb89a2d0bb10b839bc Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 3 Nov 2021 17:18:29 +0530 Subject: [PATCH 56/67] updated docs Signed-off-by: metonymic-smokey --- docs/components/compact.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index 79aa7a6c4b..ee3f9a14a1 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -377,8 +377,7 @@ Flags: configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration --progress-metrics Enables the progress metrics, indicating the - progress of the compaction and downsampling - processes + progress of compaction and downsampling --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 hour) in bucket. Setting this to 0d will retain From 639b964d553d53d996b2a02be74c37dd479a88ef Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 09:35:00 +0530 Subject: [PATCH 57/67] draft: refactor compact progress test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 325 ++++++++++++------------------------ 1 file changed, 104 insertions(+), 221 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 76177e4445..c14fd30649 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -184,6 +184,7 @@ func TestCompactProgressCalculate(t *testing.T) { type planResult struct { compactionBlocks, compactionRuns float64 } + type groupedResult map[string]planResult logger := log.NewNopLogger() reg := prometheus.NewRegistry() @@ -191,21 +192,32 @@ func TestCompactProgressCalculate(t *testing.T) { planner := NewTSDBBasedPlanner(logger, []int64{ int64(1 * time.Hour / time.Millisecond), int64(2 * time.Hour / time.Millisecond), + int64(4 * time.Hour / time.Millisecond), int64(8 * time.Hour / time.Millisecond), - int64(2 * 24 * time.Hour / time.Millisecond), }) - extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) + // pre calculating group keys + keys := make([]string, 3) + m := make([]metadata.Meta, 3) + m[0].Thanos.Labels = map[string]string{"a": "1"} + m[1].Thanos.Labels = map[string]string{"b": "2"} + m[2].Thanos.Labels = map[string]string{"a": "1", "b": "2"} + m[2].Thanos.Downsample.Resolution = 1 + for ind, meta := range m { + keys[ind] = DefaultGroupKey(meta.Thanos) + } + + var bkt objstore.Bucket + temp := prometheus.NewCounter(prometheus.CounterOpts{}) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { testName string input []*metadata.Meta - expected planResult + expected groupedResult }{ - // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. - // Hence, a total of 6 blocks are planned for compaction over 2 runs. { - testName: "two_runs_test", + testName: "first_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -237,7 +249,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -248,7 +260,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -259,7 +271,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -269,8 +281,9 @@ func TestCompactProgressCalculate(t *testing.T) { MaxTime: int64(12 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, { @@ -280,8 +293,9 @@ func TestCompactProgressCalculate(t *testing.T) { MaxTime: int64(20 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, { @@ -291,20 +305,29 @@ func TestCompactProgressCalculate(t *testing.T) { MaxTime: int64(28 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, }, - expected: planResult{ - compactionRuns: 2.0, - compactionBlocks: 6.0, + expected: map[string]planResult{ + keys[0]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + keys[1]: planResult{ + compactionRuns: 1.0, + compactionBlocks: 2.0, + }, + keys[2]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, }, }, { - // This test case has non-consecutive blocks. - // The first four blocks are planned for compaction, like the previous case. But unlike the previous case, the next two blocks are not planned for compaction since these 6 blocks are not consecutive. - testName: "non_consecutive_blocks_test", + testName: "second_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -325,7 +348,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -336,117 +359,63 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(3, nil), MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, { BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(4, nil), MinTime: int64(10 * time.Hour / time.Millisecond), - MaxTime: int64(12 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(12 * time.Hour / time.Millisecond), MaxTime: int64(14 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, - }, - }, - { - // In this test case, the first four blocks are compacted into an 8h block in the first run. - testName: "single_run_test", - input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, { BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), + ULID: ulid.MustNew(5, nil), + MinTime: int64(14 * time.Hour / time.Millisecond), + MaxTime: int64(16 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, + }, + expected: map[string]planResult{ + keys[0]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, + keys[1]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, + keys[2]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, - }, }, { - // In this test case, the metadata is part of two groups. - // The first four blocks are compacted in the first run. - testName: "two_groups_test", + testName: "third_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -467,7 +436,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -488,152 +457,66 @@ func TestCompactProgressCalculate(t *testing.T) { MaxTime: int64(8 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, { BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(4, nil), MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(16 * time.Hour / time.Millisecond), + MaxTime: int64(10 * time.Hour / time.Millisecond), }, Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{Resolution: 1}, }, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(6, nil), - MinTime: int64(16 * time.Hour / time.Millisecond), - MaxTime: int64(18 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, + }, + expected: map[string]planResult{ + keys[0]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(7, nil), - MinTime: int64(18 * time.Hour / time.Millisecond), - MaxTime: int64(26 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, + keys[1]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + keys[2]: planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, }, }, + {}, } { if ok := t.Run(tcase.testName, func(t *testing.T) { - groups := []*Group{ - { - key: "b", - labels: extLabels, - resolution: 0, - metasByMinTime: tcase.input, - }, - } + groups := make([]*Group, 3) - if tcase.testName == "two_groups_test" { - groups = append(groups, &Group{ - key: "a", - metasByMinTime: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: int64(0 * time.Hour / time.Millisecond), - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - }, - resolution: 0, - }) + blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) + for _, meta := range tcase.input { + blocks[meta.ULID] = meta } - + // form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay + groups, _ = grouper.Groups(blocks) ps := NewCompactionProgressCalculator(unRegisterer, planner) err := ps.ProgressCalculate(context.Background(), groups) metrics := ps.CompactProgressMetrics testutil.Ok(t, err) - a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("b") - if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks") - } - testutil.Equals(t, tcase.expected.compactionBlocks, promtestutil.ToFloat64(a)) - - b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("b") - if err != nil { - level.Warn(logger).Log("msg", "could not get number of runs") - } - testutil.Equals(t, tcase.expected.compactionRuns, promtestutil.ToFloat64(b)) - - if tcase.testName == "two_groups_test" { - a, err = metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("a") + for _, key := range keys { + a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues(key) if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks for two groups test") + level.Warn(logger).Log("msg", "could not get number of blocks") } - testutil.Equals(t, 4.0, promtestutil.ToFloat64(a)) - b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("a") + b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues(key) if err != nil { level.Warn(logger).Log("msg", "could not get number of runs") } - testutil.Equals(t, 1.0, promtestutil.ToFloat64(b)) + testutil.Equals(t, tcase.expected[key].compactionBlocks, promtestutil.ToFloat64(a)) + testutil.Equals(t, tcase.expected[key].compactionRuns, promtestutil.ToFloat64(b)) } }); !ok { return From c0caf425d0f852b33b427e965abaf837c0f3d071 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 09:59:18 +0530 Subject: [PATCH 58/67] draft: refactor downsample test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 201 +++++++++++++++++++++--------------- 1 file changed, 116 insertions(+), 85 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index c14fd30649..f6651506fd 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -19,7 +19,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" promtestutil "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -522,29 +521,44 @@ func TestCompactProgressCalculate(t *testing.T) { return } } - } func TestDownsampleProgressCalculate(t *testing.T) { reg := prometheus.NewRegistry() unRegisterer := &receive.UnRegisterer{Registerer: reg} - extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) logger := log.NewNopLogger() + type groupedResult map[string]float64 + + // pre calculating group keys + keys := make([]string, 3) + m := make([]metadata.Meta, 3) + m[0].Thanos.Labels = map[string]string{"a": "1"} + m[0].Thanos.Downsample.Resolution = downsample.ResLevel0 + m[1].Thanos.Labels = map[string]string{"b": "2"} + m[1].Thanos.Downsample.Resolution = downsample.ResLevel1 + m[2].Thanos.Labels = map[string]string{"a": "1", "b": "2"} + m[2].Thanos.Downsample.Resolution = downsample.ResLevel2 + for ind, meta := range m { + keys[ind] = DefaultGroupKey(meta.Thanos) + } + + var bkt objstore.Bucket + temp := prometheus.NewCounter(prometheus.CounterOpts{}) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { testName string input []*metadata.Meta - expected float64 + expected groupedResult }{ { - // This test case has 1 block to be downsampled out of 2 since for the second block, the difference between MinTime and MaxTime is less than the acceptable threshold, DownsampleRange0 - testName: "minTime_maxTime_diff_threshold_test", + testName: "first_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(0, nil), MinTime: 0, - MaxTime: downsample.DownsampleRange1, + MaxTime: downsample.DownsampleRange0, Compaction: tsdb.BlockMetaCompaction{ Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, }, @@ -553,7 +567,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { Version: 1, Labels: map[string]string{"a": "1"}, Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, + Resolution: downsample.ResLevel0, }, }, }, @@ -561,25 +575,45 @@ func TestDownsampleProgressCalculate(t *testing.T) { BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(3, nil), MinTime: 1, - MaxTime: downsample.DownsampleRange0, + MaxTime: downsample.DownsampleRange1, Compaction: tsdb.BlockMetaCompaction{ Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, }, }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, + Resolution: downsample.ResLevel1, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(4, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel2, }, }, }, }, - expected: 1.0, + expected: map[string]float64{ + keys[0]: 1.0, + keys[1]: 0.0, + keys[2]: 0.0, + }, }, { - // This test case returns 0 blocks to be downsampled since the resolution is resLevel2, which is skipped when grouping blocks for downsampling. - testName: "res_level_2_test", + testName: "second_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -593,16 +627,53 @@ func TestDownsampleProgressCalculate(t *testing.T) { Thanos: metadata.Thanos{ Version: 1, Labels: map[string]string{"a": "1"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel0, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(7, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"b": "2"}, + Downsample: metadata.ThanosDownsample{ + Resolution: downsample.ResLevel1, + }, + }, + }, + { + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(8, nil), + MinTime: 1, + MaxTime: downsample.DownsampleRange1, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, + }, + }, + Thanos: metadata.Thanos{ + Version: 1, + Labels: map[string]string{"a": "1", "b": "2"}, Downsample: metadata.ThanosDownsample{ Resolution: downsample.ResLevel2, }, }, }, }, - expected: 0.0, + expected: map[string]float64{ + keys[0]: 0.0, + keys[1]: 0.0, + keys[2]: 0.0, + }, }, { - // This test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel0, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange0. - testName: "res_level_0_test", + testName: "third_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -622,10 +693,13 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, }, }, - expected: 1.0, + expected: map[string]float64{ + keys[0]: 1.0, + keys[1]: 0.0, + keys[2]: 0.0, + }, }, { - // This test case returns 1 block to be downsampled since for this block, which has a resolution of resLevel1, the difference between minTime and maxTime is greater than the acceptable threshold, DownsampleRange1. - testName: "res_level_1_test", + testName: "fourth_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -645,12 +719,14 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, }, }, - expected: 1.0, + expected: map[string]float64{ + keys[0]: 0.0, + keys[1]: 0.0, + keys[2]: 0.0, + }, }, { - // This test case has metadata belonging to two input groups. - // It returns two blocks to be downsampled since for both the blocks, the difference between MinTime and MaxTime is above the accepted threshold for their resolution level. - testName: "two_groups_test", + testName: "fifth_test", input: []*metadata.Meta{ { BlockMeta: tsdb.BlockMeta{ @@ -687,78 +763,33 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, }, }, - expected: 2.0, + expected: map[string]float64{ + keys[0]: 0.0, + keys[1]: 0.0, + keys[2]: 0.0, + }, }, } { - groups := []*Group{ - { - key: "a", - labels: extLabels, - resolution: downsample.ResLevel1, - metasByMinTime: tcase.input, - }, - } + if ok := t.Run(tcase.testName, func(t *testing.T) { + groups := make([]*Group, 3) - if tcase.testName == "two_groups_test" { - groups = append(groups, &Group{ - resolution: 0, - key: "b", - metasByMinTime: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange0, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, - }, - }, - }, - }, - }) - } + blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) + for _, meta := range tcase.input { + blocks[meta.ULID] = meta + } + groups, _ = grouper.Groups(blocks) - ds := NewDownsampleProgressCalculator(unRegisterer) - if ok := t.Run(tcase.testName, func(t *testing.T) { + ds := NewDownsampleProgressCalculator(unRegisterer) err := ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ds.DownsampleProgressMetrics - if tcase.testName == "two_groups_test" { - a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues("b") + for _, key := range keys { + a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues(key) if err != nil { level.Warn(logger).Log("msg", "could not get number of blocks") } - testutil.Equals(t, 1.0, promtestutil.ToFloat64(a)) - } - a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues("a") - if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks") + testutil.Equals(t, tcase.expected[key], promtestutil.ToFloat64(a)) } - testutil.Equals(t, tcase.expected, promtestutil.ToFloat64(a)) }); !ok { return } From a58b63f0dd07dffd2b142bb68ba7278fccc8b35c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 11:36:30 +0530 Subject: [PATCH 59/67] lint fixes and code clean up Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 41 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index f6651506fd..7cf0ffd07e 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -18,6 +18,7 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" promtestutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/tsdb" @@ -207,7 +208,7 @@ func TestCompactProgressCalculate(t *testing.T) { } var bkt objstore.Bucket - temp := prometheus.NewCounter(prometheus.CounterOpts{}) + temp := promauto.NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for compact progress tests"}) grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { @@ -311,15 +312,15 @@ func TestCompactProgressCalculate(t *testing.T) { }, }, expected: map[string]planResult{ - keys[0]: planResult{ + keys[0]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[1]: planResult{ + keys[1]: { compactionRuns: 1.0, compactionBlocks: 2.0, }, - keys[2]: planResult{ + keys[2]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, @@ -399,15 +400,15 @@ func TestCompactProgressCalculate(t *testing.T) { }, }, expected: map[string]planResult{ - keys[0]: planResult{ + keys[0]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[1]: planResult{ + keys[1]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[2]: planResult{ + keys[2]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, @@ -475,15 +476,15 @@ func TestCompactProgressCalculate(t *testing.T) { }, }, expected: map[string]planResult{ - keys[0]: planResult{ + keys[0]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[1]: planResult{ + keys[1]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[2]: planResult{ + keys[2]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, @@ -492,16 +493,17 @@ func TestCompactProgressCalculate(t *testing.T) { {}, } { if ok := t.Run(tcase.testName, func(t *testing.T) { - groups := make([]*Group, 3) - blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) for _, meta := range tcase.input { blocks[meta.ULID] = meta } // form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay - groups, _ = grouper.Groups(blocks) + groups, err := grouper.Groups(blocks) + if err != nil { + level.Warn(logger).Log("msg, unable to form groups") + } ps := NewCompactionProgressCalculator(unRegisterer, planner) - err := ps.ProgressCalculate(context.Background(), groups) + err = ps.ProgressCalculate(context.Background(), groups) metrics := ps.CompactProgressMetrics testutil.Ok(t, err) for _, key := range keys { @@ -543,7 +545,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { } var bkt objstore.Bucket - temp := prometheus.NewCounter(prometheus.CounterOpts{}) + temp := promauto.NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for downsample progress tests"}) grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { @@ -771,16 +773,17 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, } { if ok := t.Run(tcase.testName, func(t *testing.T) { - groups := make([]*Group, 3) - blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) for _, meta := range tcase.input { blocks[meta.ULID] = meta } - groups, _ = grouper.Groups(blocks) + groups, err := grouper.Groups(blocks) + if err != nil { + level.Warn(logger).Log("msg", "unable to form groups") + } ds := NewDownsampleProgressCalculator(unRegisterer) - err := ds.ProgressCalculate(context.Background(), groups) + err = ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ds.DownsampleProgressMetrics for _, key := range keys { From 8ec72e628958070197e2da27d6d88b0015eacfc4 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 14:36:54 +0530 Subject: [PATCH 60/67] draft: refactored compact progress test Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 282 ++++++------------------------------ 1 file changed, 46 insertions(+), 236 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 7cf0ffd07e..ae83c9b088 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -14,7 +14,6 @@ import ( "time" "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -180,6 +179,24 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } +func createBlockMeta(id uint64, minTime, maxTime int64, labels map[string]string, resolution int64) *metadata.Meta { + m := &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(id, nil), + MinTime: minTime, + MaxTime: maxTime, + }, + Thanos: metadata.Thanos{ + Labels: labels, + Downsample: metadata.ThanosDownsample{ + Resolution: resolution, + }, + }, + } + + return m +} + func TestCompactProgressCalculate(t *testing.T) { type planResult struct { compactionBlocks, compactionRuns float64 @@ -219,97 +236,14 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "first_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(5, nil), - MinTime: int64(10 * time.Hour / time.Millisecond), - MaxTime: int64(12 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(6, nil), - MinTime: int64(12 * time.Hour / time.Millisecond), - MaxTime: int64(20 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(7, nil), - MinTime: int64(20 * time.Hour / time.Millisecond), - MaxTime: int64(28 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(5, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(12)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(6, int64(time.Duration(12)*time.Hour/time.Millisecond), int64(time.Duration(20)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(7, int64(time.Duration(20)*time.Hour/time.Millisecond), int64(time.Duration(28)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), }, expected: map[string]planResult{ keys[0]: { @@ -329,75 +263,12 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "second_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(10 * time.Hour / time.Millisecond), - MaxTime: int64(14 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(5, nil), - MinTime: int64(14 * time.Hour / time.Millisecond), - MaxTime: int64(16 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(4, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(14)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(5, int64(time.Duration(14)*time.Hour/time.Millisecond), int64(time.Duration(16)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), }, expected: map[string]planResult{ keys[0]: { @@ -417,63 +288,11 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "third_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{Resolution: 1}, - }, - }, + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), }, expected: map[string]planResult{ keys[0]: { @@ -499,23 +318,17 @@ func TestCompactProgressCalculate(t *testing.T) { } // form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay groups, err := grouper.Groups(blocks) - if err != nil { - level.Warn(logger).Log("msg, unable to form groups") - } + testutil.Ok(t, err) ps := NewCompactionProgressCalculator(unRegisterer, planner) err = ps.ProgressCalculate(context.Background(), groups) + testutil.Ok(t, err) metrics := ps.CompactProgressMetrics testutil.Ok(t, err) - for _, key := range keys { + for key := range tcase.expected { a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues(key) - if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks") - } + testutil.Ok(t, err) b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues(key) - if err != nil { - level.Warn(logger).Log("msg", "could not get number of runs") - } - + testutil.Ok(t, err) testutil.Equals(t, tcase.expected[key].compactionBlocks, promtestutil.ToFloat64(a)) testutil.Equals(t, tcase.expected[key].compactionRuns, promtestutil.ToFloat64(b)) } @@ -778,19 +591,16 @@ func TestDownsampleProgressCalculate(t *testing.T) { blocks[meta.ULID] = meta } groups, err := grouper.Groups(blocks) - if err != nil { - level.Warn(logger).Log("msg", "unable to form groups") - } + testutil.Ok(t, err) ds := NewDownsampleProgressCalculator(unRegisterer) err = ds.ProgressCalculate(context.Background(), groups) testutil.Ok(t, err) metrics := ds.DownsampleProgressMetrics - for _, key := range keys { + for key := range tcase.expected { a, err := metrics.NumberOfBlocksDownsampled.GetMetricWithLabelValues(key) - if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks") - } + + testutil.Ok(t, err) testutil.Equals(t, tcase.expected[key], promtestutil.ToFloat64(a)) } }); !ok { From e10c845dbfbd1eb6b62c6def9f8df3b942cb63bb Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 14:43:13 +0530 Subject: [PATCH 61/67] changed func signature Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 49 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index ae83c9b088..9fba2529c7 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -179,12 +179,20 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) { } -func createBlockMeta(id uint64, minTime, maxTime int64, labels map[string]string, resolution int64) *metadata.Meta { +func createBlockMeta(id uint64, minTime, maxTime int64, labels map[string]string, resolution int64, sources []uint64) *metadata.Meta { + sourceBlocks := make([]ulid.ULID, len(sources)) + for ind, source := range sources { + sourceBlocks[ind] = ulid.MustNew(source, nil) + } + m := &metadata.Meta{ BlockMeta: tsdb.BlockMeta{ ULID: ulid.MustNew(id, nil), MinTime: minTime, MaxTime: maxTime, + Compaction: tsdb.BlockMetaCompaction{ + Sources: sourceBlocks, + }, }, Thanos: metadata.Thanos{ Labels: labels, @@ -236,14 +244,14 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "first_test", input: []*metadata.Meta{ - createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), - createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), - createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(5, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(12)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), - createBlockMeta(6, int64(time.Duration(12)*time.Hour/time.Millisecond), int64(time.Duration(20)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), - createBlockMeta(7, int64(time.Duration(20)*time.Hour/time.Millisecond), int64(time.Duration(28)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(5, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(12)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(6, int64(time.Duration(12)*time.Hour/time.Millisecond), int64(time.Duration(20)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(7, int64(time.Duration(20)*time.Hour/time.Millisecond), int64(time.Duration(28)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), }, expected: map[string]planResult{ keys[0]: { @@ -263,12 +271,12 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "second_test", input: []*metadata.Meta{ - createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), - createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), - createBlockMeta(4, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(14)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), - createBlockMeta(5, int64(time.Duration(14)*time.Hour/time.Millisecond), int64(time.Duration(16)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(4, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(14)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(5, int64(time.Duration(14)*time.Hour/time.Millisecond), int64(time.Duration(16)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), }, expected: map[string]planResult{ keys[0]: { @@ -288,11 +296,11 @@ func TestCompactProgressCalculate(t *testing.T) { { testName: "third_test", input: []*metadata.Meta{ - createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0), - createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0), - createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), - createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1), + createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), }, expected: map[string]planResult{ keys[0]: { @@ -309,7 +317,6 @@ func TestCompactProgressCalculate(t *testing.T) { }, }, }, - {}, } { if ok := t.Run(tcase.testName, func(t *testing.T) { blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) From 3d0d375ae4faa4237bfeb41761ffd32ab6afc796 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 5 Nov 2021 18:26:31 +0530 Subject: [PATCH 62/67] draft: refactored downsample unit test to use helper func Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 185 ++---------------------------------- 1 file changed, 10 insertions(+), 175 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 9fba2529c7..108cd9d5cc 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -376,57 +376,9 @@ func TestDownsampleProgressCalculate(t *testing.T) { { testName: "first_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange0, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel2, - }, - }, - }, + createBlockMeta(0, 0, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{1, 2}), + createBlockMeta(3, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel0, []uint64{4, 5}), + createBlockMeta(6, 1, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{7, 8}), }, expected: map[string]float64{ keys[0]: 1.0, @@ -437,57 +389,9 @@ func TestDownsampleProgressCalculate(t *testing.T) { { testName: "second_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(6, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange0, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(7, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(8, nil), - MinTime: 1, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(7, nil), ulid.MustNew(8, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1", "b": "2"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel2, - }, - }, - }, + createBlockMeta(6, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{7, 8}), + createBlockMeta(7, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{7, 8}), + createBlockMeta(8, 1, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{7, 8}), }, expected: map[string]float64{ keys[0]: 0.0, @@ -497,98 +401,29 @@ func TestDownsampleProgressCalculate(t *testing.T) { }, { testName: "third_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(9, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange0, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(10, nil), ulid.MustNew(11, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, - }, - }, - }, + createBlockMeta(9, 0, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), }, expected: map[string]float64{ keys[0]: 1.0, - keys[1]: 0.0, - keys[2]: 0.0, }, }, { testName: "fourth_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(10, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(11, nil), ulid.MustNew(12, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, - }, - }, - }, + createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1"}, downsample.ResLevel1, []uint64{11, 12}), }, expected: map[string]float64{ keys[0]: 0.0, - keys[1]: 0.0, - keys[2]: 0.0, }, }, { testName: "fifth_test", input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange1, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel1, - }, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: 0, - MaxTime: downsample.DownsampleRange0, - Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{ulid.MustNew(4, nil), ulid.MustNew(5, nil)}, - }, - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - Downsample: metadata.ThanosDownsample{ - Resolution: downsample.ResLevel0, - }, - }, - }, + createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1"}, downsample.ResLevel1, []uint64{1, 2}), + createBlockMeta(3, 0, downsample.DownsampleRange0, map[string]string{"b": "2"}, downsample.ResLevel0, []uint64{4, 5}), }, expected: map[string]float64{ keys[0]: 0.0, keys[1]: 0.0, - keys[2]: 0.0, }, }, } { From 6999f7fdb675a9e0943f43bc2b60690a4f8d938f Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 6 Nov 2021 09:22:24 +0530 Subject: [PATCH 63/67] improved test cases; CircleCI fix Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 112 ++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 108cd9d5cc..878529284d 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -221,7 +221,6 @@ func TestCompactProgressCalculate(t *testing.T) { int64(8 * time.Hour / time.Millisecond), }) - // pre calculating group keys keys := make([]string, 3) m := make([]metadata.Meta, 3) m[0].Thanos.Labels = map[string]string{"a": "1"} @@ -233,7 +232,7 @@ func TestCompactProgressCalculate(t *testing.T) { } var bkt objstore.Bucket - temp := promauto.NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for compact progress tests"}) + temp := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for compact progress tests"}) grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { @@ -242,7 +241,8 @@ func TestCompactProgressCalculate(t *testing.T) { expected groupedResult }{ { - testName: "first_test", + // This test has a single compaction run with two blocks from the second group compacted. + testName: "single_run_test", input: []*metadata.Meta{ createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), @@ -269,51 +269,42 @@ func TestCompactProgressCalculate(t *testing.T) { }, }, { - testName: "second_test", + // This test has three compaction runs, with blocks from the first group getting compacted. + testName: "three_runs_test", input: []*metadata.Meta{ createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(3, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(4, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(5, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), + createBlockMeta(6, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), - createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), - createBlockMeta(4, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(14)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), - createBlockMeta(5, int64(time.Duration(14)*time.Hour/time.Millisecond), int64(time.Duration(16)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), }, expected: map[string]planResult{ keys[0]: { - compactionRuns: 0.0, - compactionBlocks: 0.0, + compactionRuns: 3.0, + compactionBlocks: 6.0, }, keys[1]: { compactionRuns: 0.0, compactionBlocks: 0.0, }, - keys[2]: { - compactionRuns: 0.0, - compactionBlocks: 0.0, - }, }, }, { - testName: "third_test", + // This test case has 4 2-hour blocks, which are non consecutive. + // Hence, only the first two blocks are compacted. + testName: "non_consecutive_blocks_test", input: []*metadata.Meta{ - createBlockMeta(0, 0, int64(time.Duration(2)*time.Hour/time.Millisecond), map[string]string{"a": "1"}, 0, []uint64{}), - createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), - createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"b": "2"}, 0, []uint64{}), + createBlockMeta(1, int64(time.Duration(2)*time.Hour/time.Millisecond), int64(time.Duration(4)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(2, int64(time.Duration(4)*time.Hour/time.Millisecond), int64(time.Duration(6)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), createBlockMeta(3, int64(time.Duration(6)*time.Hour/time.Millisecond), int64(time.Duration(8)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), - createBlockMeta(4, int64(time.Duration(8)*time.Hour/time.Millisecond), int64(time.Duration(10)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), + createBlockMeta(4, int64(time.Duration(10)*time.Hour/time.Millisecond), int64(time.Duration(12)*time.Hour/time.Millisecond), map[string]string{"a": "1", "b": "2"}, 1, []uint64{}), }, expected: map[string]planResult{ - keys[0]: { - compactionRuns: 0.0, - compactionBlocks: 0.0, - }, - keys[1]: { - compactionRuns: 0.0, - compactionBlocks: 0.0, - }, keys[2]: { - compactionRuns: 0.0, - compactionBlocks: 0.0, + compactionRuns: 1.0, + compactionBlocks: 2.0, }, }, }, @@ -323,7 +314,6 @@ func TestCompactProgressCalculate(t *testing.T) { for _, meta := range tcase.input { blocks[meta.ULID] = meta } - // form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay groups, err := grouper.Groups(blocks) testutil.Ok(t, err) ps := NewCompactionProgressCalculator(unRegisterer, planner) @@ -351,7 +341,6 @@ func TestDownsampleProgressCalculate(t *testing.T) { logger := log.NewNopLogger() type groupedResult map[string]float64 - // pre calculating group keys keys := make([]string, 3) m := make([]metadata.Meta, 3) m[0].Thanos.Labels = map[string]string{"a": "1"} @@ -365,7 +354,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { } var bkt objstore.Bucket - temp := promauto.NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for downsample progress tests"}) + temp := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for downsample progress tests"}) grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { @@ -374,32 +363,22 @@ func TestDownsampleProgressCalculate(t *testing.T) { expected groupedResult }{ { - testName: "first_test", - input: []*metadata.Meta{ - createBlockMeta(0, 0, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{1, 2}), - createBlockMeta(3, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel0, []uint64{4, 5}), - createBlockMeta(6, 1, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{7, 8}), - }, - expected: map[string]float64{ - keys[0]: 1.0, - keys[1]: 0.0, - keys[2]: 0.0, - }, - }, - { - testName: "second_test", + // This test case has blocks from multiple groups and resolution levels. Only the second block should be downsampled since the others either have time differences not in the range for their resolution, or a resolution which should not be downsampled. + testName: "multi_group_test", input: []*metadata.Meta{ createBlockMeta(6, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{7, 8}), - createBlockMeta(7, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{7, 8}), - createBlockMeta(8, 1, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{7, 8}), + createBlockMeta(7, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 9}), + createBlockMeta(8, 0, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{9, 10}), }, expected: map[string]float64{ keys[0]: 0.0, - keys[1]: 0.0, + keys[1]: 1.0, keys[2]: 0.0, }, }, { - testName: "third_test", + // This is a test case for resLevel0, with the correct time difference threshold. + // This block should be downsampled. + testName: "res_level0_test", input: []*metadata.Meta{ createBlockMeta(9, 0, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), }, @@ -407,22 +386,45 @@ func TestDownsampleProgressCalculate(t *testing.T) { keys[0]: 1.0, }, }, { - testName: "fourth_test", + // This is a test case for resLevel1, with the correct time difference threshold. + // This block should be downsampled. + testName: "res_level1_test", input: []*metadata.Meta{ - createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1"}, downsample.ResLevel1, []uint64{11, 12}), + createBlockMeta(9, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), }, expected: map[string]float64{ - keys[0]: 0.0, + keys[1]: 1.0, }, }, { - testName: "fifth_test", + // This is a test case for resLevel2. + // Blocks with this resolution should not be downsampled. + testName: "res_level2_test", + input: []*metadata.Meta{ + createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{11, 12}), + }, + expected: map[string]float64{ + keys[2]: 0.0, + }, + }, { + // This is a test case for resLevel0, with incorrect time difference, below the threshold. + // This block should be downsampled. + testName: "res_level0_test_incorrect", input: []*metadata.Meta{ - createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1"}, downsample.ResLevel1, []uint64{1, 2}), - createBlockMeta(3, 0, downsample.DownsampleRange0, map[string]string{"b": "2"}, downsample.ResLevel0, []uint64{4, 5}), + createBlockMeta(9, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), }, expected: map[string]float64{ keys[0]: 0.0, + }, + }, + { + // This is a test case for resLevel1, with incorrect time difference, below the threshold. + // This block should be downsampled. + testName: "res_level1_test", + input: []*metadata.Meta{ + createBlockMeta(9, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), + }, + expected: map[string]float64{ keys[1]: 0.0, }, }, From 37848f118e416d56d39f9ded64126325cea2fe21 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 6 Nov 2021 11:16:01 +0530 Subject: [PATCH 64/67] use reset instead of unregister; minor nits Signed-off-by: metonymic-smokey --- cmd/thanos/compact.go | 20 +++++--------------- pkg/compact/compact.go | 6 +++++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 04ed810326..fba4e31426 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -40,7 +40,6 @@ import ( "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" - "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/pkg/runutil" httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/ui" @@ -462,11 +461,10 @@ func runCompact( if conf.compactionProgressMetrics { g.Add(func() error { - unRegisterer := &receive.UnRegisterer{Registerer: reg} - ps := compact.NewCompactionProgressCalculator(unRegisterer, tsdbPlanner) + ps := compact.NewCompactionProgressCalculator(reg, tsdbPlanner) var ds *compact.DownsampleProgressCalculator if !conf.disableDownsampling { - ds = compact.NewDownsampleProgressCalculator(unRegisterer) + ds = compact.NewDownsampleProgressCalculator(reg) } return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { @@ -478,12 +476,7 @@ func runCompact( metas := sy.Metas() groups, err := grouper.Groups(metas) if err != nil { - return errors.Wrapf(err, "could not group original metadata") - } - - for _, group := range groups { - ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(group.Key()) - ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(group.Key()) + return errors.Wrapf(err, "could not group metadata") } if err = ps.ProgressCalculate(ctx, groups); err != nil { @@ -493,10 +486,7 @@ func runCompact( if !conf.disableDownsampling { groups, err = grouper.Groups(metas) if err != nil { - return errors.Wrapf(err, "could not group original metadata into downsample groups") - } - for _, group := range groups { - ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(group.Key()) + return errors.Wrapf(err, "could not group metadata into downsample groups") } if err := ds.ProgressCalculate(ctx, groups); err != nil { return errors.Wrapf(err, "could not calculate downsampling progress") @@ -646,7 +636,7 @@ type compactConfig struct { } func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { - cmd.Flag("progress-metrics", "Enables the progress metrics, indicating the progress of compaction and downsampling").Default("false").BoolVar(&cc.compactionProgressMetrics) + cmd.Flag("progress-metrics", "Enables the progress metrics, indicating the progress of compaction and downsampling").Default("true").BoolVar(&cc.compactionProgressMetrics) cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected."). Hidden().Default("true").BoolVar(&cc.haltOnError) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 17aab3f541..0beb8c2b4e 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -507,7 +507,7 @@ func NewCompactionProgressCalculator(reg prometheus.Registerer, planner *tsdbBas CompactProgressMetrics: &CompactProgressMetrics{ NumberOfCompactionRuns: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_compact_todo_compactions", - Help: "number of iterations to be done", + Help: "number of compactions to be done", }, []string{"group"}), NumberOfCompactionBlocks: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_compact_todo_compaction_blocks", @@ -561,6 +561,9 @@ func (ps *CompactionProgressCalculator) ProgressCalculate(ctx context.Context, g groups = tmpGroups } + ps.CompactProgressMetrics.NumberOfCompactionRuns.Reset() + ps.CompactProgressMetrics.NumberOfCompactionBlocks.Reset() + for key, iters := range groupCompactions { ps.CompactProgressMetrics.NumberOfCompactionRuns.WithLabelValues(key).Add(float64(iters)) ps.CompactProgressMetrics.NumberOfCompactionBlocks.WithLabelValues(key).Add(float64(groupBlocks[key])) @@ -656,6 +659,7 @@ func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, g } } + ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.Reset() for key, blocks := range groupBlocks { ds.DownsampleProgressMetrics.NumberOfBlocksDownsampled.WithLabelValues(key).Add(float64(blocks)) } From e366232719ac3fd6a80368e0284db5ed27741d19 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 6 Nov 2021 11:26:03 +0530 Subject: [PATCH 65/67] improved downsample test case Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 878529284d..8119a66429 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -363,16 +363,17 @@ func TestDownsampleProgressCalculate(t *testing.T) { expected groupedResult }{ { - // This test case has blocks from multiple groups and resolution levels. Only the second block should be downsampled since the others either have time differences not in the range for their resolution, or a resolution which should not be downsampled. + // This test case has blocks from multiple groups and resolution levels. Only the blocks in the second group should be downsampled since the others either have time differences not in the range for their resolution, or a resolution which should not be downsampled. testName: "multi_group_test", input: []*metadata.Meta{ createBlockMeta(6, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{7, 8}), createBlockMeta(7, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 9}), + createBlockMeta(9, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 11}), createBlockMeta(8, 0, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{9, 10}), }, expected: map[string]float64{ keys[0]: 0.0, - keys[1]: 1.0, + keys[1]: 2.0, keys[2]: 0.0, }, }, { From 50d1be41284460b4864d6b7dd5cc176f3606ed43 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 6 Nov 2021 15:28:35 +0530 Subject: [PATCH 66/67] E2E test fix Signed-off-by: metonymic-smokey --- test/e2e/compact_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 1daede5ea9..49ecdbc9f8 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -715,7 +715,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, err) operationMatcher, err := matchers.NewMatcher(matchers.MatchEqual, "operation", "get") testutil.Ok(t, err) - testutil.Ok(t, c.WaitSumMetricsWithOptions(e2e.Equals(478), + testutil.Ok(t, c.WaitSumMetricsWithOptions(e2e.Equals(538), []string{"thanos_objstore_bucket_operations_total"}, e2e.WithLabelMatchers( bucketMatcher, operationMatcher, From 02bb607abe00907211ae9545bb48dfe2d4d3cc27 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 6 Nov 2021 18:24:57 +0530 Subject: [PATCH 67/67] added CHANGELOG Signed-off-by: metonymic-smokey --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e8055db7..93b57ac0ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4710](https://github.com/thanos-io/thanos/pull/4710) Store: add metric to capture timestamp of the last loaded block. - [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint. - [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas. +- [#4801](https://github.com/thanos-io/thanos/pull/4801) Compactor: added Prometheus metrics for tracking the progress of compaction and downsampling. ### Fixed