statistics: fix repetitive selectivity accounting and stabilify the result (#15536)

statistics/selectivity.go

@@ -15,6 +15,8 @@ package statistics
 
 import (
 	"math"
+	"math/bits"
+	"sort"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/ast"
@@ -52,6 +54,27 @@ const (
 	ColType
 )
 
+func compareType(l, r int) int {
+	if l == r {
+		return 0
+	}
+	if l == ColType {
+		return -1
+	}
+	if l == PkType {
+		return 1
+	}
+	if r == ColType {
+		return 1
+	}
+	return -1
+}
+
+// MockStatsNode is only used for test.
+func MockStatsNode(id int64, m int64, num int) *StatsNode {
+	return &StatsNode{ID: id, mask: m, numCols: num}
+}
+
 const unknownColumnID = math.MinInt64
 
 // getConstantColumnID receives two expressions and if one of them is column and another is constant, it returns the
@@ -238,7 +261,7 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp
 			})
 		}
 	}
-	usedSets := getUsableSetsByGreedy(nodes)
+	usedSets := GetUsableSetsByGreedy(nodes)
 	// Initialize the mask with the full set.
 	mask := (int64(1) << uint(len(remainedExprs))) - 1
 	for _, set := range usedSets {
@@ -293,8 +316,14 @@ func getMaskAndRanges(ctx sessionctx.Context, exprs []expression.Expression, ran
 	return mask, ranges, false, nil
 }
 
-// getUsableSetsByGreedy will select the indices and pk used for calculate selectivity by greedy algorithm.
-func getUsableSetsByGreedy(nodes []*StatsNode) (newBlocks []*StatsNode) {
+// GetUsableSetsByGreedy will select the indices and pk used for calculate selectivity by greedy algorithm.
+func GetUsableSetsByGreedy(nodes []*StatsNode) (newBlocks []*StatsNode) {
+	sort.Slice(nodes, func(i int, j int) bool {
+		if r := compareType(nodes[i].Tp, nodes[j].Tp); r != 0 {
+			return r < 0
+		}
+		return nodes[i].ID < nodes[j].ID
+	})
 	marked := make([]bool, len(nodes))
 	mask := int64(math.MaxInt64)
 	for {
@@ -305,9 +334,14 @@ func getUsableSetsByGreedy(nodes []*StatsNode) (newBlocks []*StatsNode) {
 				continue
 			}
 			curMask := set.mask & mask
-			bits := popCount(curMask)
+			if curMask != set.mask {
+				marked[i] = true
+				continue
+			}
+			bits := bits.OnesCount64(uint64(curMask))
 			// This set cannot cover any thing, just skip it.
 			if bits == 0 {
+				marked[i] = true
 				continue
 			}
 			// We greedy select the stats info based on:
@@ -330,14 +364,3 @@ func getUsableSetsByGreedy(nodes []*StatsNode) (newBlocks []*StatsNode) {
 	}
 	return
 }
-
-// popCount is the digit sum of the binary representation of the number x.
-func popCount(x int64) int {
-	ret := 0
-	// x -= x & -x, remove the lowest bit of the x.
-	// e.g. result will be 2 if x is 3.
-	for ; x > 0; x -= x & -x {
-		ret++
-	}
-	return ret
-}

statistics/selectivity_test.go

@@ -560,3 +560,21 @@ func (s *testStatsSuite) TestColumnIndexNullEstimation(c *C) {
 		"  └─TableScan_5 5.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false",
 	))
 }
+
+func (s *testStatsSuite) TestSelectivityGreedyAlgo(c *C) {
+	nodes := make([]*statistics.StatsNode, 3)
+	nodes[0] = statistics.MockStatsNode(1, 3, 2)
+	nodes[1] = statistics.MockStatsNode(2, 5, 2)
+	nodes[2] = statistics.MockStatsNode(3, 9, 2)
+
+	// Sets should not overlap on mask, so only nodes[0] is chosen.
+	usedSets := statistics.GetUsableSetsByGreedy(nodes)
+	c.Assert(len(usedSets), Equals, 1)
+	c.Assert(usedSets[0].ID, Equals, int64(1))
+
+	nodes[0], nodes[1] = nodes[1], nodes[0]
+	// Sets chosen should be stable, so the returned node is still the one with ID 1.
+	usedSets = statistics.GetUsableSetsByGreedy(nodes)
+	c.Assert(len(usedSets), Equals, 1)
+	c.Assert(usedSets[0].ID, Equals, int64(1))
+}