Copy labels coming from remote engines (#6957)

* Copy labels coming from remote engines

When running in distributed mode, the remote engine will use an unsafe
cast from ZLabels to Prometheus labels to avoid making new allocations.
This makes it hard to use the new gRPC shared buffer pool for receiving
and decompressing messages since memory gets retained beyond the scope
of a Recv() call.

This commit removes the unsafe cast and makes an explicit memory copy
of received series labels. Since remote queries are already aggregated
series, the amount of data we receive should be small anyway, and the copies
on average should have a small impact.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>

* Use clone on strings

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>

---------

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>

pkg/query/remote_engine.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"io"
 	"math"
+	"strings"
 	"sync"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 	"github.com/prometheus/prometheus/util/stats"
 
 	"github.com/thanos-io/promql-engine/api"
+
 	"github.com/thanos-io/thanos/pkg/api/query/querypb"
 	"github.com/thanos-io/thanos/pkg/info/infopb"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
@@ -258,8 +260,12 @@ func (r *remoteQuery) Exec(ctx context.Context) *promql.Result {
 		if ts == nil {
 			continue
 		}
+		lbls := labels.NewScratchBuilder(len(ts.Labels))
+		for _, l := range ts.Labels {
+			lbls.Add(strings.Clone(l.Name), strings.Clone(l.Value))
+		}
 		series := promql.Series{
-			Metric:     labelpb.ZLabelsToPromLabels(ts.Labels),
+			Metric:     lbls.Labels(),
 			Floats:     make([]promql.FPoint, 0, len(ts.Samples)),
 			Histograms: make([]promql.HPoint, 0, len(ts.Histograms)),
 		}