Add configuration for ldap group sync in background

 for better performance with large amount of ldap group

Signed-off-by: stonezdj <stone.zhang@broadcom.com>

api/v2.0/swagger.yaml

@@ -8897,6 +8897,9 @@ definitions:
       ldap_group_search_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
+      ldap_group_sync_in_background:
+        $ref: '#/definitions/BoolConfigItem'
+        description: Sync LDAP group information in background.
       ldap_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'
@@ -9087,6 +9090,11 @@ definitions:
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
         x-omitempty: true
         x-isnullable: true
+      ldap_group_sync_in_background:
+        type: boolean
+        description: Sync LDAP group information in background.
+        x-omitempty: true
+        x-isnullable: true
       ldap_scope:
         type: integer
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'

src/common/const.go

@@ -134,6 +134,7 @@ const (
 	OIDCGroupType                     = 3
 	LDAPGroupAdminDn                  = "ldap_group_admin_dn"
 	LDAPGroupMembershipAttribute      = "ldap_group_membership_attribute"
+	LDAPGroupSyncInBackground         = "ldap_group_sync_in_background"
 	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
 	DefaultPortalURL                  = "http://portal:8080"
 	DefaultRegistryCtlURL             = "http://registryctl:8080"

src/core/auth/ldap/ldap.go

@@ -31,6 +31,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/config"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/ldap"
 	"github.com/goharbor/harbor/src/pkg/ldap/model"
@@ -116,8 +117,17 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m
 	if len(groupCfg.Filter) == 0 {
 		return
 	}
+	if groupCfg.SyncInBackground {
+		l.syncGroupIDFromDB(ctx, ldapUsers, u)
+		return
+	}
+	l.syncAndOnboardGroup(ctx, ldapUsers, u, sess)
+}
+
+func (l *Auth) syncAndOnboardGroup(ctx context.Context, ldapUsers []model.User, u *models.User, sess *ldap.Session) {
 	userGroups := make([]ugModel.UserGroup, 0)
 	for _, dn := range ldapUsers[0].GroupDNList {
+		// verify the group always exist in the ldap server and can be filtered by the ldap group search filter
 		lGroups, err := sess.SearchGroupByDN(dn)
 		if err != nil {
 			log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err)
@@ -129,10 +139,74 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m
 		}
 		userGroups = append(userGroups, ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType})
 	}
-	u.GroupIDs, err = ugCtl.Ctl.Populate(ctx, userGroups)
+	groupIDs, err := ugCtl.Ctl.Populate(ctx, userGroups)
 	if err != nil {
 		log.Warningf("Failed to fetch ldap group configuration:%v", err)
 	}
+	u.GroupIDs = groupIDs
+}
+
+func (l *Auth) syncGroupIDFromDB(ctx context.Context, ldapUsers []model.User, u *models.User) {
+	userGroups := make([]ugModel.UserGroup, 0)
+	groupsToBeSync := make([]string, 0)
+	for _, dn := range ldapUsers[0].GroupDNList {
+		log.Debugf("sync group in background for user %v", u.Username)
+		groups, err := ugCtl.Ctl.List(ctx, q.New(q.KeyWords{"LdapGroupDN": dn}))
+		if err != nil {
+			log.Warningf("Failed to fetch ldap group configuration:%v", err)
+			continue
+		}
+		// if the group does not exist in Harbor, launch a go routine to sync it
+		// when a group not exist in Harbor, usually it should have no actual permission link to it, so we can skip attach the group id to current user
+		if len(groups) == 0 {
+			groupsToBeSync = append(groupsToBeSync, dn)
+			continue
+		}
+		// when there are many groups for a single user, we don't verify the group in ldap server for performance consideration and just attach it
+		// attach the existing group to the user, skip to verify it in the ldap server and do not verify the ldap group search filter
+		userGroups = append(userGroups, *groups[0])
+		log.Debugf("attached group %v to user %v", groups[0].ID, u.Username)
+	}
+	// sync group in background
+	if len(groupsToBeSync) > 0 {
+		go l.onBoardGroups(orm.Context(), groupsToBeSync)
+	}
+	groupIDs, err := ugCtl.Ctl.Populate(ctx, userGroups)
+	if err != nil {
+		log.Warningf("Failed to fetch ldap group configuration:%v", err)
+	}
+	u.GroupIDs = groupIDs
+}
+
+func (l *Auth) onBoardGroups(ctx context.Context, dnList []string) {
+	// use different session because it is background goroutine
+	ldapSession, err := ldapCtl.Ctl.Session(ctx)
+	if err != nil {
+		log.Errorf("can not load system ldap config: %v", err)
+		return
+	}
+	if err = ldapSession.Open(); err != nil {
+		log.Warningf("ldap connection fail: %v", err)
+		return
+	}
+	defer ldapSession.Close()
+
+	for _, dn := range dnList {
+		lGroups, err := ldapSession.SearchGroupByDN(dn) // make sure this group exist in LDAP and matched by filter
+		if err != nil {
+			log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err)
+			continue
+		}
+		if len(lGroups) == 0 {
+			log.Warningf("Can not get the ldap group name with DN %v", dn)
+			continue
+		}
+		u := &ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}
+		if err := l.OnBoardGroup(ctx, u, ""); err != nil {
+			log.Warningf("Failed to onboard group dn: %v, error %v", u.LdapGroupDN, err)
+			continue
+		}
+	}
 }
 
 func (l *Auth) syncUserInfoFromDB(ctx context.Context, u *models.User) {
@@ -274,7 +348,6 @@ func (l *Auth) PostAuthenticate(ctx context.Context, u *models.User) error {
 				}
 			}
 		}
-
 		return nil
 	}
 

src/core/auth/ldap/ldap_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//    http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,6 +16,7 @@ package ldap
 import (
 	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -30,6 +31,7 @@ import (
 	"github.com/goharbor/harbor/src/pkg"
 	_ "github.com/goharbor/harbor/src/pkg/config/db"
 	_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
+	ldapModel "github.com/goharbor/harbor/src/pkg/ldap/model"
 	"github.com/goharbor/harbor/src/pkg/member"
 	memberModels "github.com/goharbor/harbor/src/pkg/member/models"
 	userpkg "github.com/goharbor/harbor/src/pkg/user"
@@ -455,3 +457,74 @@ func TestAddProjectMemberWithLdapGroup(t *testing.T) {
 		t.Errorf("Failed to query project member, %v", queryMember)
 	}
 }
+
+func TestSyncGroupIDFromDB(t *testing.T) {
+	ctx := orm.Context()
+	user := &models.User{
+		Username: "mike01",
+	}
+	ldapUsers := []ldapModel.User{
+		{Username: "mike", GroupDNList: []string{"cn=harbor_users,ou=groups,dc=example,dc=com", "cn=harbor_admin,ou=groups,dc=example,dc=com"}},
+	}
+	userGroups := []ugModel.UserGroup{{GroupName: "harbor_users", LdapGroupDN: "cn=harbor_users,ou=groups,dc=example,dc=com", GroupType: common.LDAPGroupType}}
+	groupIds, err := usergroup.Mgr.Populate(ctx, userGroups)
+	defer func() {
+		for _, id := range groupIds {
+			usergroup.Mgr.Delete(orm.Context(), id)
+		}
+	}()
+	if err != nil {
+		t.Errorf("failed to populate group in the database err: %v", err)
+	}
+	if len(groupIds) < 0 {
+		t.Errorf("failed to populate group in the database")
+	}
+	authHelper.syncGroupIDFromDB(ctx, ldapUsers, user)
+	// only one group id should be attached
+	assert.True(t, len(user.GroupIDs) == 1)
+	assert.Equal(t, user.GroupIDs[0], groupIds[0])
+
+	time.Sleep(2 * time.Second)
+	authHelper.syncGroupIDFromDB(ctx, ldapUsers, user)
+	// both two groups are attached
+	assert.True(t, len(user.GroupIDs) == 2)
+	defer func() {
+		for _, id := range user.GroupIDs {
+			usergroup.Mgr.Delete(orm.Context(), id)
+		}
+	}()
+}
+
+func TestOnBoardGroups(t *testing.T) {
+	groupListToDelete, err := usergroup.Mgr.List(orm.Context(), nil)
+	for _, group := range groupListToDelete {
+		usergroup.Mgr.Delete(orm.Context(), group.ID)
+	}
+	grps := []string{"cn=harbor_root,dc=harbor,dc=example,dc=com", "cn=harbor_admin,ou=groups,dc=example,dc=com"}
+	authHelper.onBoardGroups(orm.Context(), grps)
+	groupList, err := usergroup.Mgr.List(orm.Context(), nil)
+	defer func() {
+		for _, group := range groupList {
+			usergroup.Mgr.Delete(orm.Context(), group.ID)
+		}
+	}()
+	if err != nil {
+		t.Errorf("Failed to list user groups, %v", err)
+	}
+	assert.Equal(t, 2, len(groupList))
+}
+
+func TestSearchGroup(t *testing.T) {
+	ctx := orm.Context()
+	group, err := authHelper.SearchGroup(ctx, "cn=harbor_root,dc=harbor,dc=example,dc=com")
+	if err != nil {
+		t.Errorf("Failed to search group, %v", err)
+	}
+	if group == nil {
+		t.Errorf("Failed to search group")
+	}
+	assert.Equal(t, group.LdapGroupDN, "cn=harbor_root,dc=harbor,dc=example,dc=com")
+	_, err = authHelper.SearchGroup(ctx, "cn=nonexist,dc=harbor,dc=example,dc=com")
+	assert.NotNil(t, err)
+
+}

src/lib/config/metadata/metadatalist.go

@@ -96,6 +96,7 @@ var (
 		{Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`},
 		{Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`},
 		{Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`},
+		{Name: common.LDAPGroupSyncInBackground, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_SYNC_BACKGROUND", DefaultValue: "false", ItemType: &BoolType{}, Editable: true, Description: `Sync LDAP group information to Harbor in background`},
 
 		{Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false},
 		{Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`},

src/lib/config/models/model.go

@@ -94,6 +94,7 @@ type GroupConf struct {
 	SearchScope         int    `json:"ldap_group_search_scope"`
 	AdminDN             string `json:"ldap_group_admin_dn,omitempty"`
 	MembershipAttribute string `json:"ldap_group_membership_attribute,omitempty"`
+	SyncInBackground    bool   `json:"ldap_group_sync_in_background,omitempty"`
 }
 
 type GDPRSetting struct {

src/lib/config/userconfig.go

@@ -81,6 +81,7 @@ func LDAPGroupConf(ctx context.Context) (*cfgModels.GroupConf, error) {
 		SearchScope:         mgr.Get(ctx, common.LDAPGroupSearchScope).GetInt(),
 		AdminDN:             mgr.Get(ctx, common.LDAPGroupAdminDn).GetString(),
 		MembershipAttribute: mgr.Get(ctx, common.LDAPGroupMembershipAttribute).GetString(),
+		SyncInBackground:    mgr.Get(ctx, common.LDAPGroupSyncInBackground).GetBool(),
 	}, nil
 }
 

src/pkg/usergroup/manager.go

@@ -83,11 +83,13 @@ func (m *manager) Get(ctx context.Context, id int) (*model.UserGroup, error) {
 func (m *manager) Populate(ctx context.Context, userGroups []model.UserGroup) ([]int, error) {
 	ugList := make([]int, 0)
 	for _, group := range userGroups {
-		err := m.Onboard(ctx, &group)
-		if err != nil {
-			// log the current error and continue
-			log.Warningf("failed to onboard user group %+v, error %v, continue with other user groups", group, err)
-			continue
+		if group.ID == 0 {
+			err := m.Onboard(ctx, &group)
+			if err != nil {
+				// log the current error and continue
+				log.Warningf("failed to onboard user group %+v, error %v, continue with other user groups", group, err)
+				continue
+			}
 		}
 		if group.ID > 0 {
 			ugList = append(ugList, group.ID)

src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html

@@ -498,6 +498,35 @@
                 </option>
             </select>
         </clr-select-container>
+       <clr-checkbox-container>
+            <label for="ldapGroupSyncInBackground"
+                >{{ 'CONFIG.LDAP.GROUP_SYNC_IN_BACKGROUND' | translate }}
+                <clr-tooltip>
+                    <clr-icon
+                        clrTooltipTrigger
+                        shape="info-circle"
+                        size="24"></clr-icon>
+                    <clr-tooltip-content
+                        clrPosition="top-right"
+                        clrSize="lg"
+                        *clrIfOpen>
+                        <span>{{
+                            'CONFIG.LDAP.GROUP_SYNC_IN_BACKGROUND_INFO' | translate
+                        }}</span>
+                    </clr-tooltip-content>
+                </clr-tooltip>
+            </label>
+            <clr-checkbox-wrapper>
+                <input
+                    type="checkbox"
+                    clrCheckbox
+                    name="ldapGroupSyncInBackground"
+                    id="ldapGroupSyncInBackground"
+                    [ngModel]="currentConfig.ldap_group_sync_in_background.value"
+                    [disabled]="disabled(currentConfig.ldap_group_sync_in_background)"
+                    (ngModelChange)="setLdapGroupSyncInBackgroundValue($event)" />
+            </clr-checkbox-wrapper>
+        </clr-checkbox-container>
     </section>
     <clr-checkbox-container *ngIf="showSelfReg">
         <label for="selfReg"

src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.ts

@@ -132,6 +132,10 @@ export class ConfigurationAuthComponent implements OnInit {
         this.currentConfig.ldap_verify_cert.value = $event;
     }
 
+    setLdapGroupSyncInBackgroundValue($event: any) {
+        this.currentConfig.ldap_group_sync_in_background.value = $event;
+    }
+
     public pingTestServer(): void {
         if (this.testingOnGoing) {
             return; // Should not come here

src/portal/src/app/base/left-side-nav/config/config.ts

@@ -72,6 +72,7 @@ export class Configuration {
     ldap_group_search_scope: NumberValueItem;
     ldap_group_membership_attribute: StringValueItem;
     ldap_group_admin_dn: StringValueItem;
+    ldap_group_sync_in_background: BoolValueItem;
     uaa_client_id: StringValueItem;
     uaa_client_secret?: StringValueItem;
     uaa_endpoint: StringValueItem;

src/portal/src/i18n/lang/de-de-lang.json

@@ -927,8 +927,9 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP Gruppenmitgliedschaft",
             "LDAP_GROUP_MEMBERSHIP_INFO": "Dass Attribut, das die Mitglieder einer LDAP-Gruppe identifiziert. Standardwert ist memberof, in manchen LDAP Servern kann es \"ismemberof\" sein. Das Feld darf nicht leer sein, sofern eine LDAP Gruppen Funktion eingesetzt wird.",
             "GROUP_SCOPE": "LDAP Gruppen Search Scope",
-            "GROUP_SCOPE_INFO": "Der Scope mit dem nach Gruppen gesucht wird. Standard ist Subtree."
-
+            "GROUP_SCOPE_INFO": "Der Scope mit dem nach Gruppen gesucht wird. Standard ist Subtree.",
+            "GROUP_SYNC_IN_BACKGROUND": "LDAP Group Sync in Background",
+            "GROUP_SYNC_IN_BACKGROUND_INFO": "Enable this option to sync LDAP group information in background to avoid timeout when there are too many groups. If disabled, the LDAP group information will be synced when the user logs in."               
         },
         "UAA": {
             "ENDPOINT": "UAA Endpunkt",

src/portal/src/i18n/lang/en-us-lang.json

@@ -928,7 +928,9 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP Group Membership",
             "LDAP_GROUP_MEMBERSHIP_INFO": "The attribute indicates the membership of LDAP group, default value is memberof, in some LDAP server it could be \"ismemberof\". This field cannot be empty if you need to enable the LDAP group related feature.",
             "GROUP_SCOPE": "LDAP Group Search Scope",
-            "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default."
+            "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default.",
+          "GROUP_SYNC_IN_BACKGROUND": "LDAP Group Sync in Background",
+            "GROUP_SYNC_IN_BACKGROUND_INFO": "Enable this option to sync LDAP group information in background to avoid timeout when there are too many groups. If disabled, the LDAP group information will be synced when the user logs in."
 
         },
         "UAA": {

src/portal/src/i18n/lang/es-es-lang.json

@@ -927,7 +927,9 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP Group Membership",
             "LDAP_GROUP_MEMBERSHIP_INFO": "The attribute indicates the membership of LDAP group, default value is memberof, in some LDAP server it could be \"ismemberof\". This field cannot be empty if you need to enable the LDAP group related feature.",
             "GROUP_SCOPE": "LDAP Group Search Scope",
-            "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default."
+            "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default.",
+          "GROUP_SYNC_IN_BACKGROUND": "LDAP Group Sync in Background",
+            "GROUP_SYNC_IN_BACKGROUND_INFO": "Enable this option to sync LDAP group information in background to avoid timeout when there are too many groups. If disabled, the LDAP group information will be synced when the user logs in."                
         },
         "UAA": {
             "ENDPOINT": "UAA Endpoint",

src/portal/src/i18n/lang/fr-fr-lang.json

@@ -927,7 +927,10 @@
             "LDAP_GROUP_MEMBERSHIP": "Appartenance au groupe LDAP",
             "LDAP_GROUP_MEMBERSHIP_INFO": "L'attribut indique l'appartenance au groupe LDAP ; la valeur par défaut est \"memberOf\", dans certains LDAP cela peut être \"isMemberOf\"",
             "GROUP_SCOPE": "Scope des groupes LDAP",
-            "GROUP_SCOPE_INFO": "Scope dans lequel faire la recherche, 'subtree' par défaut."
+            "GROUP_SCOPE_INFO": "Scope dans lequel faire la recherche, 'subtree' par défaut.",
+          "GROUP_SYNC_IN_BACKGROUND": "LDAP Group Sync in Background",
+            "GROUP_SYNC_IN_BACKGROUND_INFO": "Enable this option to sync LDAP group information in background to avoid timeout when there are too many groups. If disabled, the LDAP group information will be synced when the user logs in."      
+
         },
         "UAA": {
             "ENDPOINT": "Endpoint UAA",

src/portal/src/i18n/lang/ko-kr-lang.json

@@ -925,7 +925,9 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP Group 멤버쉽",
             "LDAP_GROUP_MEMBERSHIP_INFO": "속성은 LDAP 그룹의 멤버십을 나타내며 기본값은 memberof이며 일부 LDAP 서버에서는 \"ismemberof\"일 수 있습니다. LDAP 그룹 관련 기능을 활성화해야 하는 경우 이 필드를 비워둘 수 없습니다.",
             "GROUP_SCOPE": "LDAP 그룹 검색 범위",
-            "GROUP_SCOPE_INFO": "그룹을 검색할 범위는 기본적으로 Subtree를 선택합니다."
+            "GROUP_SCOPE_INFO": "그룹을 검색할 범위는 기본적으로 Subtree를 선택합니다.",
+          "GROUP_SYNC_IN_BACKGROUND": "LDAP Group Sync in Background",
+            "GROUP_SYNC_IN_BACKGROUND_INFO": "Enable this option to sync LDAP group information in background to avoid timeout when there are too many groups. If disabled, the LDAP group information will be synced when the user logs in."
 
         },
         "UAA": {