From fd12b0a274648581d441eaed5e40ec6d8783dd39 Mon Sep 17 00:00:00 2001 From: chlins Date: Tue, 15 Aug 2023 11:19:45 +0800 Subject: [PATCH] fix: support customize cache db for business Support to configure the customized redis db for cache layer and other misc business for core, by default the behavior is same with previous(stored in db 0). Signed-off-by: chlins --- make/harbor.yml.tmpl | 15 +++++ .../migrations/version_2_7_0/harbor.yml.jinja | 59 +++++++++++++++++++ make/photon/prepare/templates/core/env.jinja | 6 ++ .../prepare/templates/jobservice/env.jinja | 3 + make/photon/prepare/utils/configs.py | 19 ++++-- src/core/main.go | 38 +++++++----- src/jobservice/main.go | 18 ------ src/lib/cache/cache.go | 35 +++++++++++ src/pkg/cached/base_manager.go | 16 ++--- 9 files changed, 165 insertions(+), 44 deletions(-) diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index 7c953b9c61a..264fa4dd1bc 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -174,6 +174,17 @@ _version: 2.7.0 # password: notary_server_db_password # ssl_mode: disable +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + # Uncomment external_redis if using external Redis server # external_redis: # # support redis, redis+sentinel @@ -190,6 +201,10 @@ _version: 2.7.0 # chartmuseum_db_index: 3 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja index 3be0173c1c6..44aa37d902f 100644 --- a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja @@ -371,6 +371,49 @@ external_database: # ssl_mode: disable {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -387,6 +430,18 @@ external_redis: chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -404,6 +459,10 @@ external_redis: # chartmuseum_db_index: 3 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 5ea5d3734bf..2023f6f74ca 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -1,6 +1,9 @@ CONFIG_PATH=/etc/core/app.conf UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_harbor %} +_REDIS_URL_HARBOR={{redis_url_harbor}} +{% endif %} SYNC_QUOTA=true CHART_CACHE_DRIVER={{chart_cache_driver}} _REDIS_URL_REG={{redis_url_reg}} @@ -88,6 +91,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% endif %} {% if cache.enabled %} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 31e039ff233..24f835841fc 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -50,6 +50,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% if cache.enabled %} _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} \ No newline at end of file diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index d0052b8966c..5e53533aec0 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -312,7 +312,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu config_dict['external_database'] = False # update redis configs - config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy)) + config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -407,7 +407,7 @@ def get_redis_url_param(redis=None): return "" -def get_redis_configs(external_redis=None, with_trivy=True): +def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True): """Returns configs for redis >>> get_redis_configs()['external_redis'] @@ -440,6 +440,8 @@ def get_redis_configs(external_redis=None, with_trivy=True): >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False) True """ + + internal_redis = internal_redis or {} external_redis = external_redis or {} configs = dict(external_redis=bool(external_redis)) @@ -455,14 +457,23 @@ def get_redis_configs(external_redis=None, with_trivy=True): 'idle_timeout_seconds': 30, } - # overwriting existing keys by external_redis - redis.update({key: value for (key, value) in external_redis.items() if value}) + if len(internal_redis) > 0: + # overwriting existing keys by internal_redis + redis.update({key: value for (key, value) in internal_redis.items() if value}) + else: + # overwriting existing keys by external_redis + redis.update({key: value for (key, value) in external_redis.items() if value}) configs['redis_url_core'] = get_redis_url(0, redis) configs['redis_url_chart'] = get_redis_url(redis['chartmuseum_db_index'], redis) configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis) configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis) + if redis.get('harbor_db_index'): + configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis) + if redis.get('cache_layer_db_index'): + configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis) + if with_trivy: configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis) diff --git a/src/core/main.go b/src/core/main.go index d8e68f8852c..4f904e0c32e 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -124,25 +124,35 @@ func main() { web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName web.BConfig.MaxMemory = 1 << 35 // (32GB) web.BConfig.MaxUploadSize = 1 << 35 // (32GB) - - redisURL := os.Getenv("_REDIS_URL_CORE") - if len(redisURL) > 0 { - u, err := url.Parse(redisURL) + // the core db used for beego session + redisCoreURL := os.Getenv("_REDIS_URL_CORE") + if len(redisCoreURL) > 0 { + _, err := url.Parse(redisCoreURL) if err != nil { - panic("bad _REDIS_URL") + panic("bad _REDIS_URL_CORE") } - + // configure the beego session redis web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName - web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL + web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL + } - log.Info("initializing cache ...") - if err := cache.Initialize(u.Scheme, redisURL); err != nil { - log.Fatalf("failed to initialize cache: %v", err) - } - // when config/db init function is called, the cache is not ready, - // enable config cache explicitly when the cache is ready - dbCfg.EnableConfigCache() + log.Info("initializing cache ...") + // the harbor db used for harbor business, use core db if not specified + redisHarborURL := os.Getenv("_REDIS_URL_HARBOR") + if redisHarborURL == "" { + redisHarborURL = redisCoreURL } + u, err := url.Parse(redisHarborURL) + if err != nil { + panic("bad _REDIS_URL_HARBOR") + } + if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil { + log.Fatalf("failed to initialize cache: %v", err) + } + // when config/db init function is called, the cache is not ready, + // enable config cache explicitly when the cache is ready + dbCfg.EnableConfigCache() + web.AddTemplateExt("htm") log.Info("initializing configurations...") diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 318d1129e1f..092190b5027 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -19,8 +19,6 @@ import ( "errors" "flag" "fmt" - "net/url" - "os" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/jobservice/common/utils" @@ -29,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" - "github.com/goharbor/harbor/src/lib/cache" cfgLib "github.com/goharbor/harbor/src/lib/config" tracelib "github.com/goharbor/harbor/src/lib/trace" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base" @@ -44,21 +41,6 @@ func main() { panic(fmt.Sprintf("failed to load configuration, error: %v", err)) } - // init cache if cache layer enabled - // gc needs to delete artifact by artifact manager, but the artifact cache store in - // core redis db so here require core redis url and init default cache. - if cfgLib.CacheEnabled() { - cacheURL := os.Getenv("_REDIS_URL_CORE") - u, err := url.Parse(cacheURL) - if err != nil { - panic("bad _REDIS_URL_CORE") - } - - if err = cache.Initialize(u.Scheme, cacheURL); err != nil { - panic(fmt.Sprintf("failed to initialize cache: %v", err)) - } - } - // Get parameters configPath := flag.String("c", "", "Specify the yaml config file path") flag.Parse() diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 1e0dae56cc6..02a7b09264c 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/url" + "os" "sync" "time" @@ -137,3 +138,37 @@ func Initialize(typ, addr string) error { func Default() Cache { return cache } + +var ( + // cacheLayer is the global cache layer cache instance. + cacheLayer Cache + // cacheLayerOnce is the once condition for initializing instance. + cacheLayerOnce sync.Once +) + +// LayerCache is the global cache instance for cache layer. +func LayerCache() Cache { + // parse the redis url for cache layer, use the default cache if not specify + redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER") + if redisCacheURL == "" { + if cache != nil { + return cache + } + // use the core url if cache layer url not found + redisCacheURL = os.Getenv("_REDIS_URL_CORE") + } + + u, err := url.Parse(redisCacheURL) + if err != nil { + log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER") + } + + cacheLayerOnce.Do(func() { + cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:")) + if err != nil { + log.Fatalf("failed to initialize cache for cache layer, err: %v", err) + } + }) + + return cacheLayer +} diff --git a/src/pkg/cached/base_manager.go b/src/pkg/cached/base_manager.go index bc810372849..d7bf1caddbc 100644 --- a/src/pkg/cached/base_manager.go +++ b/src/pkg/cached/base_manager.go @@ -24,27 +24,27 @@ import ( ) // innerCache is the default cache client, -// actually it is a wrapper for cache.Default(). +// actually it is a wrapper for cache.LayerCache(). var innerCache cache.Cache = &cacheClient{} -// cacheClient is a interceptor for cache.Default, in order to implement specific +// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific // case for cache layer. type cacheClient struct{} func (*cacheClient) Contains(ctx context.Context, key string) bool { - return cache.Default().Contains(ctx, key) + return cache.LayerCache().Contains(ctx, key) } func (*cacheClient) Delete(ctx context.Context, key string) error { - return cache.Default().Delete(ctx, key) + return cache.LayerCache().Delete(ctx, key) } func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error { - return cache.Default().Fetch(ctx, key, value) + return cache.LayerCache().Fetch(ctx, key, value) } func (*cacheClient) Ping(ctx context.Context) error { - return cache.Default().Ping(ctx) + return cache.LayerCache().Ping(ctx) } func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error { @@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp return nil } - return cache.Default().Save(ctx, key, value, expiration...) + return cache.LayerCache().Save(ctx, key, value, expiration...) } func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) { - return cache.Default().Scan(ctx, match) + return cache.LayerCache().Scan(ctx, match) } var _ Manager = &BaseManager{}