set config pageSize finding need remove blobs in gc

src/common/const.go

@@ -160,6 +160,8 @@ const (
 
 	// DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours
 	DefaultGCTimeWindowHours = int64(2)
+	// DefaultGCFindBlobsPageSize is the find need remove blob page size used by GC, default is 1000 because ob join slow
+	DefaultGCFindBlobsPageSize = int64(1000)
 
 	// Metric setting items
 	MetricEnable = "metric_enable"

src/controller/gc/controller.go

@@ -80,6 +80,7 @@ func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (
 	para["dry_run"] = policy.DryRun
 	para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
 	para["time_window"] = policy.ExtraAttrs["time_window"]
+	para["find_blobs_page_size"] = policy.ExtraAttrs["find_blobs_page_size"]
 
 	execID, err := c.exeMgr.Create(ctx, job.GarbageCollectionVendorType, -1, trigger, para)
 	if err != nil {

src/jobservice/job/impl/gc/garbage_collection.go

@@ -64,8 +64,9 @@ type GarbageCollector struct {
 	// As table blob has no repositories data, and the repositories are required when to delete a manifest, so use the table ArtifactTrash to capture them.
 	trashedArts map[string][]model.ArtifactTrash
 	// hold all of GC candidates(non-referenced blobs), it's captured by mark and consumed by sweep.
-	deleteSet       []*blobModels.Blob
-	timeWindowHours int64
+	deleteSet         []*blobModels.Blob
+	timeWindowHours   int64
+	findBlobsPageSize int64
 }
 
 // MaxFails implements the interface in job/Interface
@@ -132,6 +133,15 @@ func (gc *GarbageCollector) parseParams(params job.Parameters) {
 		}
 	}
 
+	// find blobs page size: default 1000,and for testing/debugging, it can be set to 100
+	gc.findBlobsPageSize = 1000
+	findBlobsPageSize, exist := params["find_blobs_page_size"]
+	if exist {
+		if pageSize, ok := findBlobsPageSize.(float64); ok {
+			gc.findBlobsPageSize = int64(pageSize)
+		}
+	}
+
 	// dry run: default is false. And for dry run we can have button in the UI.
 	gc.dryRun = false
 	dryRun, exist := params["dry_run"]
@@ -552,7 +562,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo
 		}
 		p := result.Data
 
-		ps := 1000
+		ps := gc.findBlobsPageSize
 		lastBlobID := int64(0)
 		timeRG := q.Range{
 			Max: time.Now().Add(-time.Duration(gc.timeWindowHours) * time.Hour).Format(time.RFC3339),
@@ -573,7 +583,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo
 					"id":          &blobRG,
 				},
 				PageNumber: 1,
-				PageSize:   int64(ps),
+				PageSize:   ps,
 				Sorts: []*q.Sort{
 					q.NewSort("id", false),
 				},
@@ -596,7 +606,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo
 					break
 				}
 			}
-			if len(blobs) < ps {
+			if len(blobs) < int(ps) {
 				break
 			}
 			lastBlobID = blobs[len(blobs)-1].ID

src/jobservice/job/impl/gc/garbage_collection_test.go

@@ -158,9 +158,10 @@ func (suite *gcTestSuite) TestInit() {
 		registryCtlClient: suite.registryCtlClient,
 	}
 	params := map[string]interface{}{
-		"delete_untagged": true,
-		"redis_url_reg":   "redis url",
-		"time_window":     1,
+		"delete_untagged":      true,
+		"redis_url_reg":        "redis url",
+		"time_window":          1,
+		"find_blobs_page_size": 100,
 	}
 	suite.Nil(gc.init(ctx, params))
 	suite.True(gc.deleteUntagged)
@@ -276,9 +277,10 @@ func (suite *gcTestSuite) TestRun() {
 		registryCtlClient: suite.registryCtlClient,
 	}
 	params := map[string]interface{}{
-		"delete_untagged": false,
-		"redis_url_reg":   tests.GetRedisURL(),
-		"time_window":     1,
+		"delete_untagged":      false,
+		"redis_url_reg":        tests.GetRedisURL(),
+		"time_window":          1,
+		"find_blobs_page_size": 100,
 	}
 
 	suite.Nil(gc.Run(ctx, params))

src/lib/config/systemconfig.go

@@ -127,6 +127,18 @@ func GetGCTimeWindow() int64 {
 	return common.DefaultGCTimeWindowHours
 }
 
+// GetGCBlobsFindPageSize returns the find blobs blob.
+func GetGCBlobsFindPageSize() int64 {
+	// the env is for testing/debugging. For production, Do NOT set it.
+	if env, exist := os.LookupEnv("GC_BLOBS_FIND_PAGE_SIZE"); exist {
+		pageSize, err := strconv.ParseInt(env, 10, 64)
+		if err == nil {
+			return pageSize
+		}
+	}
+	return common.DefaultGCFindBlobsPageSize
+}
+
 // GetExecutionStatusRefreshIntervalSeconds returns the interval seconds for the refresh of execution status.
 func GetExecutionStatusRefreshIntervalSeconds() int64 {
 	return DefaultMgr().Get(backgroundCtx, common.ExecutionStatusRefreshIntervalSeconds).GetInt64()

src/lib/config/test/userconfig_test.go

@@ -61,6 +61,7 @@ func TestConfig(t *testing.T) {
 	t.Setenv("TOKEN_PRIVATE_KEY_PATH", "")
 	t.Setenv("JOBSERVICE_URL", "http://myjob:8888")
 	t.Setenv("GC_TIME_WINDOW_HOURS", "0")
+	t.Setenv("GC_BLOBS_FIND_PAGE_SIZE", "100")
 
 	Init()
 	ctx := orm.Context()
@@ -181,7 +182,7 @@ func TestConfig(t *testing.T) {
 	assert.True(NotificationEnable(ctx))
 	assert.Equal(int64(0), GetGCTimeWindow())
 	assert.Equal("robot$", RobotPrefix(ctx))
-
+	assert.Equal(int64(100), GetGCBlobsFindPageSize())
 }
 
 func currPath() string {

src/server/v2.0/handler/gc.go

@@ -85,6 +85,7 @@ func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, paramete
 	// set the required parameters for GC
 	parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
 	parameters["time_window"] = config.GetGCTimeWindow()
+	parameters["find_blobs_page_size"] = config.GetGCBlobsFindPageSize()
 
 	var err error
 	var id int64

tests/ci/api_common_install.sh

@@ -59,6 +59,7 @@ sudo make compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="incl
 
 # set the debugging env
 echo "GC_TIME_WINDOW_HOURS=0" | sudo tee -a ./make/common/config/core/env
+echo "GC_BLOBS_FIND_PAGE_SIZE=100" | sudo tee -a ./make/common/config/core/env
 echo "EXECUTION_STATUS_REFRESH_INTERVAL_SECONDS=5" | sudo tee -a ./make/common/config/core/env
 sudo make start