Merge branch 'main' into patch-1

.github/dependabot.yml

@@ -13,7 +13,6 @@ updates:
       interval: "daily"
     labels:
       - "release-note/infra"
-      - "release-note/update"
     assignees:
       - "OrlinVasilev"
     reviewers:

.github/workflows/publish_release.yml

@@ -58,7 +58,10 @@ jobs:
         run: |
           tar -zxf ${{ env.OFFLINE_PACKAGE_PATH }}
           docker load -i ./harbor/harbor.${{ env.BASE_TAG }}.tar.gz
-          source tools/release/release_utils.sh && publishImages ${{ env.CUR_TAG }} ${{ env.BASE_TAG }} ${{ secrets.DOCKER_HUB_USERNAME }} ${{ secrets.DOCKER_HUB_PASSWORD }}
+          images="$(docker images --format "{{.Repository}}" --filter=reference='goharbor/*:${{ env.BASE_TAG }}' | xargs)"
+          source tools/release/release_utils.sh
+          publishImages ${{ env.CUR_TAG }} ${{ env.BASE_TAG }} ${{ secrets.DOCKER_HUB_USERNAME }} ${{ secrets.DOCKER_HUB_PASSWORD }} $images
+          publishPackages ${{ env.CUR_TAG }} ${{ env.BASE_TAG }} ${{ github.actor }} ${{ secrets.GITHUB_TOKEN }} $images
       - name: Generate release notes
         run: |
           release_notes_path=$(pwd)/release-notes.txt

CONTRIBUTING.md

@@ -192,7 +192,7 @@ Harbor web UI is built based on [Clarity](https://vmware.github.io/clarity/) and
 |   2.3    |      10.1.2        |       4.0.2        |
 |   2.4    |      12.0.3        |       5.3.0        |
 
-To run the Web UI code, please refer to the UI [start](https://github.com/goharbor/harbor/blob/master/src/portal/README.md) guideline.
+To run the Web UI code, please refer to the UI [start](https://github.com/goharbor/harbor/tree/main/src/portal) guideline.
 
 To run the code, please refer to the [build](https://goharbor.io/docs/edge/build-customize-contribute/compile-guide/) guideline.
 

api/v2.0/swagger.yaml

@@ -8522,7 +8522,7 @@ definitions:
   PayloadFormatType:
     type: string
     description: The type of webhook paylod format.
-    example: 'cloudevent'
+    example: 'CloudEvents'
   PayloadFormat:
     type: object
     description: Webhook supported payload format type collections.

make/harbor.yml.tmpl

@@ -178,6 +178,8 @@ _version: 2.7.0
 #   #  <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
 #   host: redis:6379
 #   password: 
+#   # Redis AUTH command was extended in Redis 6, it is possible to use it in the two-arguments AUTH <username> <password> form.
+#   # username:
 #   # sentinel_master_set must be set to support redis+sentinel
 #   #sentinel_master_set:
 #   # db_index 0 is for core, it's unchangeable

make/migrations/postgresql/0110_2.8.0_schema.up.sql

@@ -80,3 +80,58 @@ UPDATE notification_policy
 SET enabled = false,
     description = 'Chartmuseum is deprecated in Harbor v2.8.0, because this notification policy only has event type about Chartmuseum, so please update or delete this notification policy.'
 WHERE event_types = '[]';
+
+/* migrate the webhook job to execution and task as the webhook refactor since v2.8 */
+DO $$
+DECLARE
+    job_group RECORD;
+    job RECORD;
+    vendor_type varchar;
+    new_status varchar;
+    status_code integer;
+    exec_id integer;
+    extra_attrs varchar;
+BEGIN
+    FOR job_group IN SELECT DISTINCT policy_id,event_type FROM notification_job WHERE event_type NOT IN ('UPLOAD_CHART', 'DOWNLOAD_CHART', 'DELETE_CHART')
+    LOOP
+        SELECT * INTO job FROM notification_job WHERE
+        policy_id=job_group.policy_id
+        AND event_type=job_group.event_type
+        AND status IN ('stopped', 'finished', 'error')
+        ORDER BY creation_time DESC LIMIT 1;
+        /* convert vendor type */
+        IF job.notify_type = 'http' THEN
+            vendor_type = 'WEBHOOK';
+        ELSIF job.notify_type = 'slack' THEN
+            vendor_type = 'SLACK';
+        ELSE
+            vendor_type = 'WEBHOOK';
+        END IF;
+        /* convert status */
+        IF job.status = 'stopped' THEN
+            new_status = 'Stopped';
+            status_code = 3;
+        ELSIF job.status = 'error' THEN
+            new_status = 'Error';
+            status_code = 3;
+        ELSIF job.status = 'finished' THEN
+            new_status = 'Success';
+            status_code = 3;
+        ELSE
+            new_status = '';
+            status_code = 0;
+        END IF;
+
+        SELECT format('{"event_type": "%s", "payload": %s}', job.event_type, to_json(job.job_detail)::TEXT) INTO extra_attrs;
+        INSERT INTO execution (vendor_type,vendor_id,status,trigger,extra_attrs,start_time,end_time,update_time) VALUES (vendor_type,job.policy_id,new_status,'EVENT',to_json(extra_attrs),job.creation_time,job.update_time,job.update_time) RETURNING id INTO exec_id;
+        INSERT INTO task (execution_id,job_id,status,status_code,run_count,creation_time,start_time,update_time,end_time,vendor_type) VALUES (exec_id,job.job_uuid,new_status,status_code,1,job.creation_time,job.update_time,job.update_time,job.update_time,vendor_type);
+    END LOOP;
+END $$;
+/* drop the old notification_job table */
+DROP TABLE IF EXISTS notification_job;
+
+
+
+
+
+

make/photon/prepare/utils/configs.py

@@ -384,8 +384,9 @@ def get_redis_url(db, redis=None):
     kwargs['db_part'] = db and ("/%s" % db) or ""
     kwargs['sentinel_part'] = kwargs.get('sentinel_master_set', None) and ("/" + kwargs['sentinel_master_set']) or ''
     kwargs['password_part'] = kwargs.get('password', None) and (':%s@' % kwargs['password']) or ''
+    kwargs['username_part'] = kwargs.get('username', None) or ''
 
-    return "{scheme}://{password_part}{host}{sentinel_part}{db_part}".format(**kwargs) + get_redis_url_param(kwargs)
+    return "{scheme}://{username_part}{password_part}{host}{sentinel_part}{db_part}".format(**kwargs) + get_redis_url_param(kwargs)
 
 
 def get_redis_url_param(redis=None):

src/controller/artifact/controller.go

@@ -567,8 +567,8 @@ func (c *controller) AddLabel(ctx context.Context, artifactID int64, labelID int
 				LabelID:    labelID,
 				Ctx:        ctx,
 			}
-			if err := e.Build(metaData); err == nil {
-				if err := e.Publish(); err != nil {
+			if err := e.Build(ctx, metaData); err == nil {
+				if err := e.Publish(ctx); err != nil {
 					log.Error(errors.Wrap(err, "mark label to resource handler: event publish"))
 				}
 			} else {

src/controller/event/handler/auditlog/auditlog_test.go

@@ -81,7 +81,7 @@ func (suite *AuditLogHandlerTestSuite) TestSubscribeTagEvent() {
 
 	notifier.Subscribe(event.TopicCreateProject, suite.auditLogHandler)
 	// event data should implement the interface TopicEvent
-	ne.BuildAndPublish(&metadata.CreateProjectEventMetadata{
+	ne.BuildAndPublish(context.TODO(), &metadata.CreateProjectEventMetadata{
 		ProjectID: 1,
 		Project:   "test",
 		Operator:  "admin",

src/controller/event/handler/util/util.go

@@ -1,23 +1,23 @@
 package util
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/goharbor/harbor/src/lib/config"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/distribution"
 	policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model"
 	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model"
 )
 
 // SendHookWithPolicies send hook by publishing topic of specified target type(notify type)
-func SendHookWithPolicies(policies []*policy_model.Policy, payload *notifyModel.Payload, eventType string) error {
+func SendHookWithPolicies(ctx context.Context, policies []*policy_model.Policy, payload *notifyModel.Payload, eventType string) error {
 	// if global notification configured disabled, return directly
-	if !config.NotificationEnable(orm.Context()) {
+	if !config.NotificationEnable(ctx) {
 		log.Debug("notification feature is not enabled")
 		return nil
 	}
@@ -28,14 +28,15 @@ func SendHookWithPolicies(policies []*policy_model.Policy, payload *notifyModel.
 		for _, target := range targets {
 			evt := &event.Event{}
 			hookMetadata := &event.HookMetaData{
+				ProjectID: ply.ProjectID,
 				EventType: eventType,
 				PolicyID:  ply.ID,
 				Payload:   payload,
 				Target:    &target,
 			}
 			// It should never affect evaluating other policies when one is failed, but error should return
-			if err := evt.Build(hookMetadata); err == nil {
-				if err := evt.Publish(); err != nil {
+			if err := evt.Build(ctx, hookMetadata); err == nil {
+				if err := evt.Publish(ctx); err != nil {
 					errRet = true
 					log.Errorf("failed to publish hook notify event: %v", err)
 				}

src/controller/event/handler/webhook/artifact/artifact.go

@@ -88,7 +88,7 @@ func (a *Handler) handle(ctx context.Context, event *event.ArtifactEvent) error
 		return err
 	}
 
-	err = util.SendHookWithPolicies(policies, payload, event.EventType)
+	err = util.SendHookWithPolicies(ctx, policies, payload, event.EventType)
 	if err != nil {
 		return err
 	}

src/controller/event/handler/webhook/artifact/replication.go

@@ -14,7 +14,6 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/config"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier/model"
 	proModels "github.com/goharbor/harbor/src/pkg/project/models"
@@ -46,12 +45,12 @@ func (r *ReplicationHandler) Handle(ctx context.Context, value interface{}) erro
 		return fmt.Errorf("nil replication event")
 	}
 
-	payload, project, err := constructReplicationPayload(rpEvent)
+	payload, project, err := constructReplicationPayload(ctx, rpEvent)
 	if err != nil {
 		return err
 	}
 
-	policies, err := notification.PolicyMgr.GetRelatedPolices(orm.Context(), project.ProjectID, rpEvent.EventType)
+	policies, err := notification.PolicyMgr.GetRelatedPolices(ctx, project.ProjectID, rpEvent.EventType)
 	if err != nil {
 		log.Errorf("failed to find policy for %s event: %v", rpEvent.EventType, err)
 		return err
@@ -60,7 +59,7 @@ func (r *ReplicationHandler) Handle(ctx context.Context, value interface{}) erro
 		log.Debugf("cannot find policy for %s event: %v", rpEvent.EventType, rpEvent)
 		return nil
 	}
-	err = util.SendHookWithPolicies(policies, payload, rpEvent.EventType)
+	err = util.SendHookWithPolicies(ctx, policies, payload, rpEvent.EventType)
 	if err != nil {
 		return err
 	}
@@ -72,8 +71,7 @@ func (r *ReplicationHandler) IsStateful() bool {
 	return false
 }
 
-func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, *proModels.Project, error) {
-	ctx := orm.Context()
+func constructReplicationPayload(ctx context.Context, event *event.ReplicationEvent) (*model.Payload, *proModels.Project, error) {
 	task, err := replication.Ctl.GetTask(ctx, event.ReplicationTaskID)
 	if err != nil {
 		log.Errorf("failed to get replication task %d: error: %v", event.ReplicationTaskID, err)
@@ -191,7 +189,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 		payload.EventData.Replication.FailedArtifact = []*ctlModel.ArtifactInfo{failedArtifact}
 	}
 
-	prj, err := project.Ctl.GetByName(orm.Context(), prjName, project.Metadata(true))
+	prj, err := project.Ctl.GetByName(ctx, prjName, project.Metadata(true))
 	if err != nil {
 		log.Errorf("failed to get project %s, error: %v", prjName, err)
 		return nil, nil, err

src/controller/event/handler/webhook/artifact/replication_test.go

@@ -15,7 +15,6 @@
 package artifact
 
 import (
-	"context"
 	"testing"
 	"time"
 
@@ -28,6 +27,7 @@ import (
 	repctl "github.com/goharbor/harbor/src/controller/replication"
 	repctlmodel "github.com/goharbor/harbor/src/controller/replication/model"
 	"github.com/goharbor/harbor/src/lib/config"
+	"github.com/goharbor/harbor/src/lib/orm"
 	_ "github.com/goharbor/harbor/src/pkg/config/db"
 	_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
 	"github.com/goharbor/harbor/src/pkg/notification"
@@ -109,7 +109,7 @@ func TestReplicationHandler_Handle(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := handler.Handle(context.TODO(), tt.args.data)
+			err := handler.Handle(orm.Context(), tt.args.data)
 			if tt.wantErr {
 				require.NotNil(t, err, "Error: %s", err)
 				return

src/controller/event/handler/webhook/artifact/retention.go

@@ -12,7 +12,6 @@ import (
 	"github.com/goharbor/harbor/src/lib/config"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier/model"
 )
@@ -45,7 +44,7 @@ func (r *RetentionHandler) Handle(ctx context.Context, value interface{}) error
 		return nil
 	}
 
-	payload, dryRun, project, err := r.constructRetentionPayload(trEvent)
+	payload, dryRun, project, err := r.constructRetentionPayload(ctx, trEvent)
 	if err != nil {
 		return err
 	}
@@ -64,7 +63,7 @@ func (r *RetentionHandler) Handle(ctx context.Context, value interface{}) error
 		log.Debugf("cannot find policy for %s event: %v", trEvent.EventType, trEvent)
 		return nil
 	}
-	err = util.SendHookWithPolicies(policies, payload, trEvent.EventType)
+	err = util.SendHookWithPolicies(ctx, policies, payload, trEvent.EventType)
 	if err != nil {
 		return err
 	}
@@ -76,8 +75,7 @@ func (r *RetentionHandler) IsStateful() bool {
 	return false
 }
 
-func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent) (*model.Payload, bool, int64, error) {
-	ctx := orm.Context()
+func (r *RetentionHandler) constructRetentionPayload(ctx context.Context, event *event.RetentionEvent) (*model.Payload, bool, int64, error) {
 	task, err := retention.Ctl.GetRetentionExecTask(ctx, event.TaskID)
 	if err != nil {
 		log.Errorf("failed to get retention task %d: error: %v", event.TaskID, err)

src/controller/event/handler/webhook/artifact/retention_test.go

@@ -1,7 +1,6 @@
 package artifact
 
 import (
-	"context"
 	"os"
 	"testing"
 	"time"
@@ -14,6 +13,7 @@ import (
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/controller/retention"
 	"github.com/goharbor/harbor/src/lib/config"
+	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/selector"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model"
@@ -97,7 +97,7 @@ func TestRetentionHandler_Handle(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := handler.Handle(context.TODO(), tt.args.data)
+			err := handler.Handle(orm.Context(), tt.args.data)
 			if tt.wantErr {
 				require.NotNil(t, err, "Error: %s", err)
 				return

src/controller/event/handler/webhook/quota/quota.go

@@ -23,7 +23,6 @@ import (
 	"github.com/goharbor/harbor/src/controller/event/handler/util"
 	"github.com/goharbor/harbor/src/controller/project"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model"
 	proModels "github.com/goharbor/harbor/src/pkg/project/models"
@@ -48,7 +47,7 @@ func (qp *Handler) Handle(ctx context.Context, value interface{}) error {
 		return fmt.Errorf("nil quota event")
 	}
 
-	prj, err := project.Ctl.GetByName(orm.Context(), quotaEvent.Project.Name, project.Metadata(true))
+	prj, err := project.Ctl.GetByName(ctx, quotaEvent.Project.Name, project.Metadata(true))
 	if err != nil {
 		log.Errorf("failed to get project:%s, error: %v", quotaEvent.Project.Name, err)
 		return err
@@ -69,7 +68,7 @@ func (qp *Handler) Handle(ctx context.Context, value interface{}) error {
 		return err
 	}
 
-	err = util.SendHookWithPolicies(policies, payload, quotaEvent.EventType)
+	err = util.SendHookWithPolicies(ctx, policies, payload, quotaEvent.EventType)
 	if err != nil {
 		return err
 	}

src/controller/event/handler/webhook/quota/quota_test.go

@@ -26,6 +26,7 @@ import (
 	common_dao "github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/lib/config"
+	"github.com/goharbor/harbor/src/lib/orm"
 	_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notification/policy"
@@ -96,7 +97,7 @@ func (suite *QuotaPreprocessHandlerSuite) TearDownSuite() {
 // TestHandle ...
 func (suite *QuotaPreprocessHandlerSuite) TestHandle() {
 	handler := &Handler{}
-	err := handler.Handle(context.TODO(), suite.evt)
+	err := handler.Handle(orm.Context(), suite.evt)
 	suite.NoError(err)
 }