This is an automated cherry-pick of pingcap#46368

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>

planner/core/plan_cost_ver2.go

@@ -386,8 +386,18 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC
 	}
 
 	rows := getCardinality(p.children[0], option.CostFlag)
+<<<<<<< HEAD
 	N := math.Max(1, float64(p.Count+p.Offset))
 	rowSize := getAvgRowSize(p.statsInfo(), p.Schema().Columns)
+=======
+	n := max(1, float64(p.Count+p.Offset))
+	if n > 10000 {
+		// It's only used to prevent some extreme cases, e.g. `select * from t order by a limit 18446744073709551615`.
+		// For normal cases, considering that `rows` may be under-estimated, better to keep `n` unchanged.
+		n = min(n, rows)
+	}
+	rowSize := getAvgRowSize(p.StatsInfo(), p.Schema().Columns)
+>>>>>>> 5fbe25ebe35 (planner: adjust N used in TopN cost formula based on the total number of rows (#46368))
 	cpuFactor := getTaskCPUFactorVer2(p, taskType)
 	memFactor := getTaskMemFactorVer2(p, taskType)
 

planner/core/plan_cost_ver2_test.go

@@ -53,6 +53,18 @@ func testCostQueries(t *testing.T, tk *testkit.TestKit, queries []string) {
 	}
 }
 
+func TestHugeTopNCost(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec(`create table t (a int)`)
+	tk.MustExec(`insert into t values (1)`)
+	tk.MustExec(`analyze table t`)
+	plan1 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1")
+	plan2 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1000000000")
+	require.Equal(t, plan1.Rows()[0][2], plan2.Rows()[0][2]) // should have the same plan cost
+}
+
 func TestCostModelVer2(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)