fix: support customize cache db for business

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>

make/harbor.yml.tmpl

@@ -167,6 +167,17 @@ _version: 2.9.0
 #     max_idle_conns: 2
 #     max_open_conns: 0
 
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_db_index: 7
+
 # Uncomment external_redis if using external Redis server
 # external_redis:
 #   # support redis, redis+sentinel
@@ -186,6 +197,10 @@ _version: 2.9.0
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_db_index: 7
 
 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:

make/photon/prepare/templates/core/env.jinja

@@ -1,6 +1,9 @@
 CONFIG_PATH=/etc/core/app.conf
 UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_harbor %}
+_REDIS_URL_HARBOR={{redis_url_harbor}}
+{% endif %}
 SYNC_QUOTA=true
 _REDIS_URL_REG={{redis_url_reg}}
 
@@ -84,6 +87,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 {% endif %}
 
 {% if cache.enabled %}
+{% if redis_url_cache %}
+_REDIS_URL_CACHE={{redis_url_cache}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/templates/jobservice/env.jinja

@@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 
 {% if cache.enabled %}
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_cache %}
+_REDIS_URL_CACHE={{redis_url_cache}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/utils/configs.py

@@ -275,7 +275,7 @@ def parse_yaml_config(config_file_path, with_trivy):
         config_dict['external_database'] = False
 
     # update redis configs
-    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
+    config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
 
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
@@ -371,7 +371,7 @@ def get_redis_url_param(redis=None):
     return ""
 
 
-def get_redis_configs(external_redis=None, with_trivy=True):
+def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
     """Returns configs for redis
 
     >>> get_redis_configs()['external_redis']
@@ -404,6 +404,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
     >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
     True
     """
+
+    internal_redis = internal_redis or {}
     external_redis = external_redis or {}
 
     configs = dict(external_redis=bool(external_redis))
@@ -418,13 +420,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
         'idle_timeout_seconds': 30,
     }
 
-    # overwriting existing keys by external_redis
-    redis.update({key: value for (key, value) in external_redis.items() if value})
+    if len(internal_redis) > 0:
+        # overwriting existing keys by internal_redis
+        redis.update({key: value for (key, value) in internal_redis.items() if value})
+    else:
+        # overwriting existing keys by external_redis
+        redis.update({key: value for (key, value) in external_redis.items() if value})
 
     configs['redis_url_core'] = get_redis_url(0, redis)
     configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
     configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
 
+    if redis.get('harbor_db_index'):
+        configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
+    if redis.get('cache_db_index'):
+        configs['redis_url_cache'] = get_redis_url(redis['cache_db_index'], redis)
+
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
 

src/controller/quota/controller.go

@@ -262,7 +262,7 @@ func (c *controller) updateUsageByRedis(ctx context.Context, reference, referenc
 		return retry.Abort(ctx.Err())
 	}
 
-	client, err := libredis.GetCoreClient()
+	client, err := libredis.GetHarborClient()
 	if err != nil {
 		return retry.Abort(err)
 	}

src/core/main.go

@@ -129,25 +129,35 @@ func main() {
 	web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
 	web.BConfig.MaxMemory = 1 << 35     // (32GB)
 	web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
-
-	redisURL := os.Getenv("_REDIS_URL_CORE")
-	if len(redisURL) > 0 {
-		u, err := url.Parse(redisURL)
+	// the core db used for beego session
+	redisCoreURL := os.Getenv("_REDIS_URL_CORE")
+	if len(redisCoreURL) > 0 {
+		_, err := url.Parse(redisCoreURL)
 		if err != nil {
-			panic("bad _REDIS_URL")
+			panic("bad _REDIS_URL_CORE")
 		}
-
+		// configure the beego session redis
 		web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
-		web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
+		web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
+	}
 
-		log.Info("initializing cache ...")
-		if err := cache.Initialize(u.Scheme, redisURL); err != nil {
-			log.Fatalf("failed to initialize cache: %v", err)
-		}
-		// when config/db init function is called, the cache is not ready,
-		// enable config cache explicitly when the cache is ready
-		dbCfg.EnableConfigCache()
+	log.Info("initializing cache ...")
+	// the harbor db used for harbor business, use core db if not specified
+	redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
+	if redisHarborURL == "" {
+		redisHarborURL = redisCoreURL
 	}
+	u, err := url.Parse(redisHarborURL)
+	if err != nil {
+		panic("bad _REDIS_URL_HARBOR")
+	}
+	if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
+		log.Fatalf("failed to initialize cache: %v", err)
+	}
+	// when config/db init function is called, the cache is not ready,
+	// enable config cache explicitly when the cache is ready
+	dbCfg.EnableConfigCache()
+
 	web.AddTemplateExt("htm")
 
 	log.Info("initializing configurations...")

src/jobservice/main.go

@@ -19,8 +19,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"net/url"
-	"os"
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
@@ -29,7 +27,6 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/job/impl"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	"github.com/goharbor/harbor/src/jobservice/runtime"
-	"github.com/goharbor/harbor/src/lib/cache"
 	cfgLib "github.com/goharbor/harbor/src/lib/config"
 	tracelib "github.com/goharbor/harbor/src/lib/trace"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
@@ -47,21 +44,6 @@ func main() {
 		panic(fmt.Sprintf("failed to load configuration, error: %v", err))
 	}
 
-	// init cache if cache layer enabled
-	// gc needs to delete artifact by artifact manager, but the artifact cache store in
-	// core redis db so here require core redis url and init default cache.
-	if cfgLib.CacheEnabled() {
-		cacheURL := os.Getenv("_REDIS_URL_CORE")
-		u, err := url.Parse(cacheURL)
-		if err != nil {
-			panic("bad _REDIS_URL_CORE")
-		}
-
-		if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
-			panic(fmt.Sprintf("failed to initialize cache: %v", err))
-		}
-	}
-
 	// Get parameters
 	configPath := flag.String("c", "", "Specify the yaml config file path")
 	flag.Parse()

src/lib/cache/cache.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"os"
 	"sync"
 	"time"
 
@@ -137,3 +138,35 @@ func Initialize(typ, addr string) error {
 func Default() Cache {
 	return cache
 }
+
+var (
+	// cacheLayer is the global cache layer cache instance.
+	cacheLayer Cache
+	// cacheLayerOnce is the once condition for initializing instance.
+	cacheLayerOnce sync.Once
+)
+
+// CacheLayer is the global cache instance for cache layer.
+//
+//nolint:revive // ignore the function name check
+func CacheLayer() Cache {
+	// parse the redis url for cache layer, use the default cache if not specify
+	redisCacheURL := os.Getenv("_REDIS_URL_CACHE")
+	if redisCacheURL == "" {
+		return cache
+	}
+
+	u, err := url.Parse(redisCacheURL)
+	if err != nil {
+		log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE")
+	}
+
+	cacheLayerOnce.Do(func() {
+		cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
+		if err != nil {
+			log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
+		}
+	})
+
+	return cacheLayer
+}

src/lib/redis/client.go

@@ -31,9 +31,9 @@ var (
 	registry     *redis.Client
 	registryOnce = &sync.Once{}
 
-	// core is a global redis client for core db
-	core     *redis.Client
-	coreOnce = &sync.Once{}
+	// harbor is a global redis client for harbor db
+	harbor     *redis.Client
+	harborOnce = &sync.Once{}
 )
 
 // GetRegistryClient returns the registry redis client.
@@ -60,26 +60,30 @@ func GetRegistryClient() (*redis.Client, error) {
 	return registry, nil
 }
 
-// GetCoreClient returns the core redis client.
-func GetCoreClient() (*redis.Client, error) {
-	coreOnce.Do(func() {
-		url := os.Getenv("_REDIS_URL_CORE")
+// GetHarborClient returns the harbor redis client.
+func GetHarborClient() (*redis.Client, error) {
+	harborOnce.Do(func() {
+		// parse redis url for harbor business, use core url by default
+		url := os.Getenv("_REDIS_URL_HARBOR")
+		if url == "" {
+			url = os.Getenv("_REDIS_URL_CORE")
+		}
 		c, err := libredis.New(cache.Options{Address: url})
 		if err != nil {
-			log.Errorf("failed to initialize redis client for core, error: %v", err)
+			log.Errorf("failed to initialize redis client for harbor, error: %v", err)
 			// reset the once to support retry if error occurred
-			coreOnce = &sync.Once{}
+			harborOnce = &sync.Once{}
 			return
 		}
 
 		if c != nil {
-			core = c.(*libredis.Cache).Client
+			harbor = c.(*libredis.Cache).Client
 		}
 	})
 
-	if core == nil {
-		return nil, errors.New("no core redis client initialized")
+	if harbor == nil {
+		return nil, errors.New("no harbor redis client initialized")
 	}
 
-	return core, nil
+	return harbor, nil
 }

src/lib/redis/client_test.go

@@ -41,22 +41,22 @@ func TestGetRegistryClient(t *testing.T) {
 	}
 }
 
-func TestGetCoreClient(t *testing.T) {
+func TestGetHarborClient(t *testing.T) {
 	// failure case with invalid address
-	t.Setenv("_REDIS_URL_CORE", "invalid-address")
-	client, err := GetCoreClient()
+	t.Setenv("_REDIS_URL_HARBOR", "invalid-address")
+	client, err := GetHarborClient()
 	assert.Error(t, err)
 	assert.Nil(t, client)
 
 	// normal case with valid address
-	t.Setenv("_REDIS_URL_CORE", "redis://localhost:6379/0")
-	client, err = GetCoreClient()
+	t.Setenv("_REDIS_URL_HARBOR", "redis://localhost:6379/0")
+	client, err = GetHarborClient()
 	assert.NoError(t, err)
 	assert.NotNil(t, client)
 
 	// multiple calls should return the same client
 	for i := 0; i < 10; i++ {
-		newClient, err := GetCoreClient()
+		newClient, err := GetHarborClient()
 		assert.NoError(t, err)
 		assert.Equal(t, client, newClient)
 	}

src/pkg/cached/base_manager.go

@@ -24,27 +24,27 @@ import (
 )
 
 // innerCache is the default cache client,
-// actually it is a wrapper for cache.Default().
+// actually it is a wrapper for cache.CacheLayer().
 var innerCache cache.Cache = &cacheClient{}
 
-// cacheClient is a interceptor for cache.Default, in order to implement specific
+// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
 // case for cache layer.
 type cacheClient struct{}
 
 func (*cacheClient) Contains(ctx context.Context, key string) bool {
-	return cache.Default().Contains(ctx, key)
+	return cache.CacheLayer().Contains(ctx, key)
 }
 
 func (*cacheClient) Delete(ctx context.Context, key string) error {
-	return cache.Default().Delete(ctx, key)
+	return cache.CacheLayer().Delete(ctx, key)
 }
 
 func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
-	return cache.Default().Fetch(ctx, key, value)
+	return cache.CacheLayer().Fetch(ctx, key, value)
 }
 
 func (*cacheClient) Ping(ctx context.Context) error {
-	return cache.Default().Ping(ctx)
+	return cache.CacheLayer().Ping(ctx)
 }
 
 func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
@@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
 		return nil
 	}
 
-	return cache.Default().Save(ctx, key, value, expiration...)
+	return cache.CacheLayer().Save(ctx, key, value, expiration...)
 }
 
 func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
-	return cache.Default().Scan(ctx, match)
+	return cache.CacheLayer().Scan(ctx, match)
 }
 
 var _ Manager = &BaseManager{}