diff --git a/src/common/const.go b/src/common/const.go index f030dacf1d9..100d066c19c 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -160,6 +160,8 @@ const ( // DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours DefaultGCTimeWindowHours = int64(2) + // DefaultGCFindBlobsPageSize is the find need remove blob page size used by GC, default is 1000 because ob join slow + DefaultGCFindBlobsPageSize = int64(1000) // Metric setting items MetricEnable = "metric_enable" diff --git a/src/controller/gc/controller.go b/src/controller/gc/controller.go index 7734c9d8a61..e2d1a31df3d 100644 --- a/src/controller/gc/controller.go +++ b/src/controller/gc/controller.go @@ -80,6 +80,7 @@ func (c *controller) Start(ctx context.Context, policy Policy, trigger string) ( para["dry_run"] = policy.DryRun para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"] para["time_window"] = policy.ExtraAttrs["time_window"] + para["find_blobs_page_size"] = policy.ExtraAttrs["find_blobs_page_size"] execID, err := c.exeMgr.Create(ctx, job.GarbageCollectionVendorType, -1, trigger, para) if err != nil { diff --git a/src/jobservice/job/impl/gc/garbage_collection.go b/src/jobservice/job/impl/gc/garbage_collection.go index 615f9e6c741..0ad5f078941 100644 --- a/src/jobservice/job/impl/gc/garbage_collection.go +++ b/src/jobservice/job/impl/gc/garbage_collection.go @@ -64,8 +64,9 @@ type GarbageCollector struct { // As table blob has no repositories data, and the repositories are required when to delete a manifest, so use the table ArtifactTrash to capture them. trashedArts map[string][]model.ArtifactTrash // hold all of GC candidates(non-referenced blobs), it's captured by mark and consumed by sweep. - deleteSet []*blobModels.Blob - timeWindowHours int64 + deleteSet []*blobModels.Blob + timeWindowHours int64 + findBlobsPageSize int64 } // MaxFails implements the interface in job/Interface @@ -132,6 +133,15 @@ func (gc *GarbageCollector) parseParams(params job.Parameters) { } } + // find blobs page size: default 1000,and for testing/debugging, it can be set to 100 + gc.findBlobsPageSize = 1000 + findBlobsPageSize, exist := params["find_blobs_page_size"] + if exist { + if pageSize, ok := findBlobsPageSize.(float64); ok { + gc.findBlobsPageSize = int64(pageSize) + } + } + // dry run: default is false. And for dry run we can have button in the UI. gc.dryRun = false dryRun, exist := params["dry_run"] @@ -552,7 +562,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo } p := result.Data - ps := 1000 + ps := gc.findBlobsPageSize lastBlobID := int64(0) timeRG := q.Range{ Max: time.Now().Add(-time.Duration(gc.timeWindowHours) * time.Hour).Format(time.RFC3339), @@ -573,7 +583,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo "id": &blobRG, }, PageNumber: 1, - PageSize: int64(ps), + PageSize: ps, Sorts: []*q.Sort{ q.NewSort("id", false), }, @@ -596,7 +606,7 @@ func (gc *GarbageCollector) markOrSweepUntaggedBlobs(ctx job.Context) ([]*blobMo break } } - if len(blobs) < ps { + if len(blobs) < int(ps) { break } lastBlobID = blobs[len(blobs)-1].ID diff --git a/src/jobservice/job/impl/gc/garbage_collection_test.go b/src/jobservice/job/impl/gc/garbage_collection_test.go index 3c78be42342..1ab8211034d 100644 --- a/src/jobservice/job/impl/gc/garbage_collection_test.go +++ b/src/jobservice/job/impl/gc/garbage_collection_test.go @@ -158,9 +158,10 @@ func (suite *gcTestSuite) TestInit() { registryCtlClient: suite.registryCtlClient, } params := map[string]interface{}{ - "delete_untagged": true, - "redis_url_reg": "redis url", - "time_window": 1, + "delete_untagged": true, + "redis_url_reg": "redis url", + "time_window": 1, + "find_blobs_page_size": 100, } suite.Nil(gc.init(ctx, params)) suite.True(gc.deleteUntagged) @@ -276,9 +277,10 @@ func (suite *gcTestSuite) TestRun() { registryCtlClient: suite.registryCtlClient, } params := map[string]interface{}{ - "delete_untagged": false, - "redis_url_reg": tests.GetRedisURL(), - "time_window": 1, + "delete_untagged": false, + "redis_url_reg": tests.GetRedisURL(), + "time_window": 1, + "find_blobs_page_size": 100, } suite.Nil(gc.Run(ctx, params)) diff --git a/src/lib/config/systemconfig.go b/src/lib/config/systemconfig.go index ac835fff9ce..79898aabb51 100644 --- a/src/lib/config/systemconfig.go +++ b/src/lib/config/systemconfig.go @@ -127,6 +127,18 @@ func GetGCTimeWindow() int64 { return common.DefaultGCTimeWindowHours } +// GetGCBlobsFindPageSize returns the find blobs blob. +func GetGCBlobsFindPageSize() int64 { + // the env is for testing/debugging. For production, Do NOT set it. + if env, exist := os.LookupEnv("GC_BLOBS_FIND_PAGE_SIZE"); exist { + pageSize, err := strconv.ParseInt(env, 10, 64) + if err == nil { + return pageSize + } + } + return common.DefaultGCFindBlobsPageSize +} + // GetExecutionStatusRefreshIntervalSeconds returns the interval seconds for the refresh of execution status. func GetExecutionStatusRefreshIntervalSeconds() int64 { return DefaultMgr().Get(backgroundCtx, common.ExecutionStatusRefreshIntervalSeconds).GetInt64() diff --git a/src/lib/config/test/userconfig_test.go b/src/lib/config/test/userconfig_test.go index 2134ef36801..060da43390d 100644 --- a/src/lib/config/test/userconfig_test.go +++ b/src/lib/config/test/userconfig_test.go @@ -61,6 +61,7 @@ func TestConfig(t *testing.T) { t.Setenv("TOKEN_PRIVATE_KEY_PATH", "") t.Setenv("JOBSERVICE_URL", "http://myjob:8888") t.Setenv("GC_TIME_WINDOW_HOURS", "0") + t.Setenv("GC_BLOBS_FIND_PAGE_SIZE", "100") Init() ctx := orm.Context() @@ -181,7 +182,7 @@ func TestConfig(t *testing.T) { assert.True(NotificationEnable(ctx)) assert.Equal(int64(0), GetGCTimeWindow()) assert.Equal("robot$", RobotPrefix(ctx)) - + assert.Equal(int64(100), GetGCBlobsFindPageSize()) } func currPath() string { diff --git a/src/server/v2.0/handler/gc.go b/src/server/v2.0/handler/gc.go index b52ba215732..5f299ac197d 100644 --- a/src/server/v2.0/handler/gc.go +++ b/src/server/v2.0/handler/gc.go @@ -85,6 +85,7 @@ func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, paramete // set the required parameters for GC parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG") parameters["time_window"] = config.GetGCTimeWindow() + parameters["find_blobs_page_size"] = config.GetGCBlobsFindPageSize() var err error var id int64 diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index 7aa8c002e5f..3963a7a58e1 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -59,6 +59,7 @@ sudo make compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="incl # set the debugging env echo "GC_TIME_WINDOW_HOURS=0" | sudo tee -a ./make/common/config/core/env +echo "GC_BLOBS_FIND_PAGE_SIZE=100" | sudo tee -a ./make/common/config/core/env echo "EXECUTION_STATUS_REFRESH_INTERVAL_SECONDS=5" | sudo tee -a ./make/common/config/core/env sudo make start