From 44284ac6c7105747878cb2cfd3f360338f3784cc Mon Sep 17 00:00:00 2001 From: Maksym Trofimenko Date: Tue, 27 Aug 2024 15:35:31 +0100 Subject: [PATCH 1/6] Feature export Harbor statistics as Prometheus metric (#18679) add statistics metrics collector Signed-off-by: Maksym Trofimenko Co-authored-by: Maksym Trofimenko --- src/pkg/exporter/collector_test.go | 29 +++ src/pkg/exporter/exporter.go | 4 +- src/pkg/exporter/project_collector_test.go | 15 +- src/pkg/exporter/statistics_collector.go | 176 ++++++++++++++++++ src/pkg/exporter/statistics_collector_test.go | 58 ++++++ 5 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 src/pkg/exporter/collector_test.go create mode 100644 src/pkg/exporter/statistics_collector.go create mode 100644 src/pkg/exporter/statistics_collector_test.go diff --git a/src/pkg/exporter/collector_test.go b/src/pkg/exporter/collector_test.go new file mode 100644 index 00000000000..42e8b982a67 --- /dev/null +++ b/src/pkg/exporter/collector_test.go @@ -0,0 +1,29 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "github.com/stretchr/testify/suite" + "testing" +) + +func TestCollectorsTestSuite(t *testing.T) { + setupTest(t) + defer tearDownTest(t) + suite.Run(t, new(ProjectCollectorTestSuite)) + suite.Run(t, &StatisticsCollectorTestSuite{ + collector: NewStatisticsCollector(), + }) +} diff --git a/src/pkg/exporter/exporter.go b/src/pkg/exporter/exporter.go index 824e00f97ff..079071b31eb 100644 --- a/src/pkg/exporter/exporter.go +++ b/src/pkg/exporter/exporter.go @@ -50,7 +50,9 @@ func NewExporter(opt *Opt) *Exporter { err := exporter.RegisterCollector(NewHealthCollect(hbrCli), NewSystemInfoCollector(hbrCli), NewProjectCollector(), - NewJobServiceCollector()) + NewJobServiceCollector(), + NewStatisticsCollector(), + ) if err != nil { log.Warningf("calling RegisterCollector() errored out, error: %v", err) } diff --git a/src/pkg/exporter/project_collector_test.go b/src/pkg/exporter/project_collector_test.go index e4912a3b59c..56edbd2c119 100644 --- a/src/pkg/exporter/project_collector_test.go +++ b/src/pkg/exporter/project_collector_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/suite" - "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" @@ -22,6 +20,7 @@ import ( qtypes "github.com/goharbor/harbor/src/pkg/quota/types" "github.com/goharbor/harbor/src/pkg/repository/model" "github.com/goharbor/harbor/src/pkg/user" + "github.com/stretchr/testify/suite" ) var ( @@ -41,8 +40,8 @@ var ( func setupTest(t *testing.T) { test.InitDatabaseFromEnv() - ctx := orm.Context() + ctx := orm.Context() // register projAdmin and assign project admin role aliceID, err := user.Mgr.Create(ctx, &alice) if err != nil { @@ -137,11 +136,11 @@ func tearDownTest(t *testing.T) { dao.GetOrmer().Raw("delete from harbor_user where user_id in (?, ?, ?)", []int{alice.UserID, bob.UserID, eve.UserID}).Exec() } -type PorjectCollectorTestSuite struct { +type ProjectCollectorTestSuite struct { suite.Suite } -func (c *PorjectCollectorTestSuite) TestProjectCollector() { +func (c *ProjectCollectorTestSuite) TestProjectCollector() { pMap := make(map[int64]*projectInfo) updateProjectBasicInfo(pMap) updateProjectMemberInfo(pMap) @@ -169,9 +168,3 @@ func (c *PorjectCollectorTestSuite) TestProjectCollector() { c.Equalf(pMap[testPro2.ProjectID].Artifact["IMAGE"].ArtifactTotal, float64(1), "pMap %v", pMap) } - -func TestPorjectCollectorTestSuite(t *testing.T) { - setupTest(t) - defer tearDownTest(t) - suite.Run(t, new(PorjectCollectorTestSuite)) -} diff --git a/src/pkg/exporter/statistics_collector.go b/src/pkg/exporter/statistics_collector.go new file mode 100644 index 00000000000..eb36c63736d --- /dev/null +++ b/src/pkg/exporter/statistics_collector.go @@ -0,0 +1,176 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/goharbor/harbor/src/controller/blob" + "github.com/goharbor/harbor/src/controller/project" + "github.com/goharbor/harbor/src/controller/repository" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/systemartifact" +) + +const StatisticsCollectorName = "StatisticsCollector" + +var ( + totalUsage = typedDesc{ + desc: newDescWithLables("", "statistics_total_storage_consumption", "Total storage used"), + valueType: prometheus.GaugeValue, + } + totalProjectAmount = typedDesc{ + desc: newDescWithLables("", "statistics_total_project_amount", "Total amount of projects"), + valueType: prometheus.GaugeValue, + } + publicProjectAmount = typedDesc{ + desc: newDescWithLables("", "statistics_public_project_amount", "Amount of public projects"), + valueType: prometheus.GaugeValue, + } + privateProjectAmount = typedDesc{ + desc: newDescWithLables("", "statistics_private_project_amount", "Amount of private projects"), + valueType: prometheus.GaugeValue, + } + totalRepoAmount = typedDesc{ + desc: newDescWithLables("", "statistics_total_repo_amount", "Total amount of repositories"), + valueType: prometheus.GaugeValue, + } + publicRepoAmount = typedDesc{ + desc: newDescWithLables("", "statistics_public_repo_amount", "Amount of public repositories"), + valueType: prometheus.GaugeValue, + } + privateRepoAmount = typedDesc{ + desc: newDescWithLables("", "statistics_private_repo_amount", "Amount of private repositories"), + valueType: prometheus.GaugeValue, + } +) + +type StatisticsCollector struct { + proCtl project.Controller + repoCtl repository.Controller + blobCtl blob.Controller + systemArtifactMgr systemartifact.Manager +} + +func NewStatisticsCollector() *StatisticsCollector { + return &StatisticsCollector{ + blobCtl: blob.Ctl, + systemArtifactMgr: systemartifact.Mgr, + proCtl: project.Ctl, + repoCtl: repository.Ctl, + } +} + +func (g StatisticsCollector) GetName() string { + return StatisticsCollectorName +} + +func (g StatisticsCollector) Describe(c chan<- *prometheus.Desc) { + c <- totalUsage.Desc() +} + +func (g StatisticsCollector) getTotalUsageMetric(ctx context.Context) prometheus.Metric { + sum, _ := g.blobCtl.CalculateTotalSize(ctx, true) + sysArtifactStorageSize, _ := g.systemArtifactMgr.GetStorageSize(ctx) + return totalUsage.MustNewConstMetric(float64(sum + sysArtifactStorageSize)) +} + +func (g StatisticsCollector) getTotalRepoAmount(ctx context.Context) int64 { + n, err := g.repoCtl.Count(ctx, nil) + if err != nil { + log.Errorf("get total repositories error: %v", err) + return 0 + } + return n +} + +func (g StatisticsCollector) getTotalProjectsAmount(ctx context.Context) int64 { + count, err := g.proCtl.Count(ctx, nil) + if err != nil { + log.Errorf("get total projects error: %v", err) + return 0 + } + return count +} + +func (g StatisticsCollector) getPublicProjectsAndRepositories(ctx context.Context) (int64, int64) { + pubProjects, err := g.proCtl.List(ctx, q.New(q.KeyWords{"public": true}), project.Metadata(false)) + if err != nil { + log.Errorf("get public projects error: %v", err) + } + pubProjectsAmount := int64(len(pubProjects)) + + if pubProjectsAmount == 0 { + return pubProjectsAmount, 0 + } + var ids []interface{} + for _, p := range pubProjects { + ids = append(ids, p.ProjectID) + } + n, err := g.repoCtl.Count(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ProjectID": q.NewOrList(ids), + }, + }) + if err != nil { + log.Errorf("get public repo error: %v", err) + return pubProjectsAmount, 0 + } + return pubProjectsAmount, n +} + +// Collect implements prometheus.Collector +func (g StatisticsCollector) Collect(c chan<- prometheus.Metric) { + for _, m := range g.getStatistics() { + c <- m + } +} + +func (g StatisticsCollector) getStatistics() []prometheus.Metric { + if CacheEnabled() { + value, ok := CacheGet(StatisticsCollectorName) + if ok { + return value.([]prometheus.Metric) + } + } + var ( + result []prometheus.Metric + ctx = orm.Context() + ) + + var ( + publicProjects, publicRepos = g.getPublicProjectsAndRepositories(ctx) + totalProjects = g.getTotalProjectsAmount(ctx) + totalRepos = g.getTotalRepoAmount(ctx) + ) + + result = []prometheus.Metric{ + totalRepoAmount.MustNewConstMetric(float64(totalRepos)), + publicRepoAmount.MustNewConstMetric(float64(publicRepos)), + privateRepoAmount.MustNewConstMetric(float64(totalRepos) - float64(publicRepos)), + totalProjectAmount.MustNewConstMetric(float64(totalProjects)), + publicProjectAmount.MustNewConstMetric(float64(publicProjects)), + privateProjectAmount.MustNewConstMetric(float64(totalProjects) - float64(publicProjects)), + g.getTotalUsageMetric(ctx), + } + if CacheEnabled() { + CachePut(StatisticsCollectorName, result) + } + return result +} diff --git a/src/pkg/exporter/statistics_collector_test.go b/src/pkg/exporter/statistics_collector_test.go new file mode 100644 index 00000000000..67f17d308fa --- /dev/null +++ b/src/pkg/exporter/statistics_collector_test.go @@ -0,0 +1,58 @@ +package exporter + +import ( + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/suite" +) + +type StatisticsCollectorTestSuite struct { + suite.Suite + collector *StatisticsCollector +} + +func (c *StatisticsCollectorTestSuite) TestStatisticsCollector() { + metrics := c.collector.getStatistics() + c.Equalf(7, len(metrics), "statistics collector should return %d metrics", 7) + c.testGaugeMetric(metrics[0], 2, "total repo amount mismatch") // total repo amount + c.testGaugeMetric(metrics[1], 1, "public repo amount mismatch") // only one project is public so its single repo is public too + c.testGaugeMetric(metrics[2], 1, "primate repo amount mismatch") // + c.testGaugeMetric(metrics[3], 3, "total project amount mismatch") // including library, project by default + c.testGaugeMetric(metrics[4], 2, "public project amount mismatch") // including library, project by default + c.testGaugeMetric(metrics[5], 1, "private project amount mismatch") + c.testGaugeMetric(metrics[6], 0, "total storage usage mismatch") // still zero +} + +func (c *StatisticsCollectorTestSuite) getMetricDTO(m prometheus.Metric) *dto.Metric { + d := &dto.Metric{} + c.NoError(m.Write(d)) + return d +} + +func (c *StatisticsCollectorTestSuite) testCounterMetric(m prometheus.Metric, value float64) { + d := c.getMetricDTO(m) + if !c.NotNilf(d, "write metric error") { + return + } + if !c.NotNilf(d.Counter, "counter is nil") { + return + } + if !c.NotNilf(d.Counter.Value, "counter value is nil") { + return + } + c.Equalf(value, *d.Counter.Value, "expected counter value does not match: expected: %v actual: %v", value, *d.Counter.Value) +} + +func (c *StatisticsCollectorTestSuite) testGaugeMetric(m prometheus.Metric, value float64, msg string) { + d := c.getMetricDTO(m) + if !c.NotNilf(d, "write metric error") { + return + } + if !c.NotNilf(d.Gauge, "gauge is nil") { + return + } + if !c.NotNilf(d.Gauge.Value, "gauge value is nil") { + return + } + c.Equalf(value, *d.Gauge.Value, "%s expected: %v actual: %v", msg, value, *d.Gauge.Value) +} From 146443e4d72dfa65d60c6cc50fb24002d1983320 Mon Sep 17 00:00:00 2001 From: kunal Dugar Date: Wed, 28 Aug 2024 08:00:20 +0530 Subject: [PATCH 2/6] Fixed the ui bug of banner coming over the search result container (#20871) Signed-off-by: kunal-511 --- .../components/global-search/search-result.component.html | 2 +- .../components/global-search/search-result.component.ts | 7 +++++++ .../shared/components/global-search/search.component.scss | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.html b/src/portal/src/app/shared/components/global-search/search-result.component.html index 240891d047f..b05652ea6d4 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.html +++ b/src/portal/src/app/shared/components/global-search/search-result.component.html @@ -1,4 +1,4 @@ -
+
Date: Wed, 28 Aug 2024 11:09:31 +0800 Subject: [PATCH 3/6] Fix typos in src/common (#20861) Signed-off-by: BruceAko --- src/common/api/base.go | 6 +++--- src/common/const.go | 2 +- src/common/http/tls.go | 2 +- src/common/job/client.go | 2 +- src/common/job/models/models.go | 4 ++-- src/common/models/job.go | 2 +- src/common/models/uaa.go | 2 +- src/common/rbac/project/evaluator_test.go | 8 ++++---- src/common/secret/request.go | 2 +- src/common/utils/email/mail.go | 4 ++-- src/common/utils/email/mail_test.go | 4 ++-- src/common/utils/encrypt.go | 4 ++-- src/common/utils/passports.go | 4 ++-- src/common/utils/test/config.go | 2 +- src/common/utils/test/test.go | 6 +++--- src/common/utils/utils.go | 4 ++-- src/server/route.go | 2 +- 17 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/common/api/base.go b/src/common/api/base.go index 116883ae772..d2b18c251f7 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -116,9 +116,9 @@ func (b *BaseAPI) DecodeJSONReqAndValidate(v interface{}) (bool, error) { } // Redirect does redirection to resource URI with http header status code. -func (b *BaseAPI) Redirect(statusCode int, resouceID string) { +func (b *BaseAPI) Redirect(statusCode int, resourceID string) { requestURI := b.Ctx.Request.RequestURI - resourceURI := requestURI + "/" + resouceID + resourceURI := requestURI + "/" + resourceID b.Ctx.Redirect(statusCode, resourceURI) } @@ -138,7 +138,7 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) { return id, nil } -// SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request +// SetPaginationHeader set "Link" and "X-Total-Count" header for pagination request func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) { b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10)) diff --git a/src/common/const.go b/src/common/const.go index 224a2e4f35c..a8166cea3a2 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -152,7 +152,7 @@ const ( OIDCCallbackPath = "/c/oidc/callback" OIDCLoginPath = "/c/oidc/login" - AuthProxyRediretPath = "/c/authproxy/redirect" + AuthProxyRedirectPath = "/c/authproxy/redirect" // Global notification enable configuration NotificationEnable = "notification_enable" diff --git a/src/common/http/tls.go b/src/common/http/tls.go index 607529bf838..05a8ad044fc 100644 --- a/src/common/http/tls.go +++ b/src/common/http/tls.go @@ -48,7 +48,7 @@ func GetInternalCertPair() (tls.Certificate, error) { // GetInternalTLSConfig return a tls.Config for internal https communicate func GetInternalTLSConfig() (*tls.Config, error) { - // genrate key pair + // generate key pair cert, err := GetInternalCertPair() if err != nil { return nil, fmt.Errorf("internal TLS enabled but can't get cert file %w", err) diff --git a/src/common/job/client.go b/src/common/job/client.go index 74a95239e36..95a22b1d75f 100644 --- a/src/common/job/client.go +++ b/src/common/job/client.go @@ -151,7 +151,7 @@ func (d *DefaultClient) SubmitJob(jd *models.JobData) (string, error) { return stats.Stats.JobID, nil } -// GetJobLog call jobserivce API to get the log of a job. It only accepts the UUID of the job +// GetJobLog call jobservice API to get the log of a job. It only accepts the UUID of the job func (d *DefaultClient) GetJobLog(uuid string) ([]byte, error) { url := d.endpoint + "/api/v1/jobs/" + uuid + "/log" req, err := http.NewRequest(http.MethodGet, url, nil) diff --git a/src/common/job/models/models.go b/src/common/job/models/models.go index f114971c2f0..8d13bd62822 100644 --- a/src/common/job/models/models.go +++ b/src/common/job/models/models.go @@ -62,7 +62,7 @@ type StatsInfo struct { UpstreamJobID string `json:"upstream_job_id,omitempty"` // Ref the upstream job if existing NumericPID int64 `json:"numeric_policy_id,omitempty"` // The numeric policy ID of the periodic job Parameters Parameters `json:"parameters,omitempty"` - Revision int64 `json:"revision,omitempty"` // For differentiating the each retry of the same job + Revision int64 `json:"revision,omitempty"` // For differentiating each retry of the same job } // JobPoolStats represents the healthy and status of all the running worker pools. @@ -70,7 +70,7 @@ type JobPoolStats struct { Pools []*JobPoolStatsData `json:"worker_pools"` } -// JobPoolStatsData represent the healthy and status of the worker worker. +// JobPoolStatsData represent the healthy and status of the worker. type JobPoolStatsData struct { WorkerPoolID string `json:"worker_pool_id"` StartedAt int64 `json:"started_at"` diff --git a/src/common/models/job.go b/src/common/models/job.go index 3dc0a4f50d1..064563f5846 100644 --- a/src/common/models/job.go +++ b/src/common/models/job.go @@ -29,7 +29,7 @@ const ( JobCanceled string = "canceled" // JobRetrying indicate the job needs to be retried, it will be scheduled to the end of job queue by statemachine after an interval. JobRetrying string = "retrying" - // JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on trasition table. + // JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on transition table. JobContinue string = "_continue" // JobScheduled ... JobScheduled string = "scheduled" diff --git a/src/common/models/uaa.go b/src/common/models/uaa.go index 0047b894cc7..a901da94737 100644 --- a/src/common/models/uaa.go +++ b/src/common/models/uaa.go @@ -14,7 +14,7 @@ package models -// UAASettings wraps the configuraations to access UAA service +// UAASettings wraps the configurations to access UAA service type UAASettings struct { Endpoint string ClientID string diff --git a/src/common/rbac/project/evaluator_test.go b/src/common/rbac/project/evaluator_test.go index bc29b661278..5a995fa8e0c 100644 --- a/src/common/rbac/project/evaluator_test.go +++ b/src/common/rbac/project/evaluator_test.go @@ -87,8 +87,8 @@ func TestProjectRoleAccess(t *testing.T) { Username: "username", } evaluator := NewEvaluator(ctl, NewBuilderForUser(user, ctl)) - resorce := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository) - assert.True(evaluator.HasPermission(context.TODO(), resorce, rbac.ActionPush)) + resource := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository) + assert.True(evaluator.HasPermission(context.TODO(), resource, rbac.ActionPush)) } { @@ -101,8 +101,8 @@ func TestProjectRoleAccess(t *testing.T) { Username: "username", } evaluator := NewEvaluator(ctl, NewBuilderForUser(user, ctl)) - resorce := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository) - assert.False(evaluator.HasPermission(context.TODO(), resorce, rbac.ActionPush)) + resource := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository) + assert.False(evaluator.HasPermission(context.TODO(), resource, rbac.ActionPush)) } } diff --git a/src/common/secret/request.go b/src/common/secret/request.go index f1cb6be021b..36be468dc7e 100644 --- a/src/common/secret/request.go +++ b/src/common/secret/request.go @@ -25,7 +25,7 @@ import ( const HeaderPrefix = "Harbor-Secret " // FromRequest tries to get Harbor Secret from request header. -// It will return empty string if the reqeust is nil. +// It will return empty string if the request is nil. func FromRequest(req *http.Request) string { if req == nil { return "" diff --git a/src/common/utils/email/mail.go b/src/common/utils/email/mail.go index 18006b9b5f4..8f49c254f0d 100644 --- a/src/common/utils/email/mail.go +++ b/src/common/utils/email/mail.go @@ -70,7 +70,7 @@ func Send(addr, identity, username, password string, // Ping tests the connection and authentication with email server // If tls is true, a secure connection is established, or Ping -// trys to upgrate the insecure connection to a secure one if +// trys to upgrade the insecure connection to a secure one if // email server supports it. // Ping doesn't verify the server's certificate and hostname when // needed if the parameter insecure is ture @@ -119,7 +119,7 @@ func newClient(addr, identity, username, password string, return nil, err } - // try to swith to SSL/TLS + // try to switch to SSL/TLS if !tls { if ok, _ := client.Extension("STARTTLS"); ok { log.Debugf("switching the connection with %s to SSL/TLS ...", addr) diff --git a/src/common/utils/email/mail_test.go b/src/common/utils/email/mail_test.go index e9ad10b662f..35e09b33504 100644 --- a/src/common/utils/email/mail_test.go +++ b/src/common/utils/email/mail_test.go @@ -38,7 +38,7 @@ func TestSend(t *testing.T) { err := Send(addr, identity, username, password, timeout, tls, insecure, from, to, subject, message) - // bypass the check due to securty policy change on gmail + // bypass the check due to security policy change on gmail // TODO // assert.Nil(t, err) @@ -78,7 +78,7 @@ func TestPing(t *testing.T) { // tls connection err := Ping(addr, identity, username, password, timeout, tls, insecure) - // bypass the check due to securty policy change on gmail + // bypass the check due to security policy change on gmail // TODO // assert.Nil(t, err) diff --git a/src/common/utils/encrypt.go b/src/common/utils/encrypt.go index 73a7cbec67b..6bfd6b9455d 100644 --- a/src/common/utils/encrypt.go +++ b/src/common/utils/encrypt.go @@ -46,8 +46,8 @@ var HashAlg = map[string]func() hash.Hash{ } // Encrypt encrypts the content with salt -func Encrypt(content string, salt string, encrptAlg string) string { - return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, HashAlg[encrptAlg])) +func Encrypt(content string, salt string, encryptAlg string) string { + return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, HashAlg[encryptAlg])) } // ReversibleEncrypt encrypts the str with aes/base64 diff --git a/src/common/utils/passports.go b/src/common/utils/passports.go index c50cd79bf84..bf08c8ad877 100644 --- a/src/common/utils/passports.go +++ b/src/common/utils/passports.go @@ -72,7 +72,7 @@ func (p *passportsPool) Revoke() bool { type LimitedConcurrentRunner interface { // AddTask adds a task to run AddTask(task func() error) - // Wait waits all the tasks to be finished, returns error if the any of the tasks gets error + // Wait waits all the tasks to be finished, returns error if any of the tasks gets error Wait() (err error) // Cancel cancels all tasks, tasks that already started will continue to run Cancel(err error) @@ -106,7 +106,7 @@ func (r *limitedConcurrentRunner) AddTask(task func() error) { r.wg.Done() }() - // Return false means no passport acquired, and no valid passport will be dispatched any more. + // Return false means no passport acquired, and no valid passport will be dispatched anymore. // For example, some crucial errors happened and all tasks should be cancelled. if ok := r.passportsPool.Apply(); !ok { return diff --git a/src/common/utils/test/config.go b/src/common/utils/test/config.go index cfc0780dcd5..62e557f6b5c 100644 --- a/src/common/utils/test/config.go +++ b/src/common/utils/test/config.go @@ -65,7 +65,7 @@ var defaultConfig = map[string]interface{}{ common.RobotNamePrefix: "robot$", } -// GetDefaultConfigMap returns the defailt config map for easier modification. +// GetDefaultConfigMap returns the default config map for easier modification. func GetDefaultConfigMap() map[string]interface{} { return defaultConfig } diff --git a/src/common/utils/test/test.go b/src/common/utils/test/test.go index ba675813040..bbf8623cffd 100644 --- a/src/common/utils/test/test.go +++ b/src/common/utils/test/test.go @@ -55,11 +55,11 @@ type Response struct { StatusCode int // Headers are the headers of the response Headers map[string]string - // Boby is the body of the response + // Body is the body of the response Body []byte } -// Handler returns a handler function which handle requst according to +// Handler returns a handler function which handle request according to // the response provided func Handler(resp *Response) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { @@ -82,7 +82,7 @@ func Handler(resp *Response) func(http.ResponseWriter, *http.Request) { } } -// NewServer creates a HTTP server for unit test +// NewServer creates an HTTP server for unit test func NewServer(mappings ...*RequestHandlerMapping) *httptest.Server { r := mux.NewRouter() diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index 9a7a1f07c3a..2a3a3728991 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -89,7 +89,7 @@ func GenerateRandomString() string { // TestTCPConn tests TCP connection // timeout: the total time before returning if something is wrong // with the connection, in second -// interval: the interval time for retring after failure, in second +// interval: the interval time for retrying after failure, in second func TestTCPConn(addr string, timeout, interval int) error { success := make(chan int, 1) cancel := make(chan int, 1) @@ -176,7 +176,7 @@ func ParseProjectIDOrName(value interface{}) (int64, string, error) { return id, name, nil } -// SafeCastString -- cast a object to string saftely +// SafeCastString -- cast an object to string safely func SafeCastString(value interface{}) string { if result, ok := value.(string); ok { return result diff --git a/src/server/route.go b/src/server/route.go index 7a9f03b704e..76cbafc18b0 100644 --- a/src/server/route.go +++ b/src/server/route.go @@ -42,7 +42,7 @@ func registerRoutes() { web.Router(common.OIDCLoginPath, &controllers.OIDCController{}, "get:RedirectLogin") web.Router("/c/oidc/onboard", &controllers.OIDCController{}, "post:Onboard") web.Router(common.OIDCCallbackPath, &controllers.OIDCController{}, "get:Callback") - web.Router(common.AuthProxyRediretPath, &controllers.AuthProxyController{}, "get:HandleRedirect") + web.Router(common.AuthProxyRedirectPath, &controllers.AuthProxyController{}, "get:HandleRedirect") web.Router("/api/internal/renameadmin", &api.InternalAPI{}, "post:RenameAdmin") web.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota") From 9d11de9706bcac7fab85a59c7426b1a272606247 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Fri, 30 Aug 2024 11:59:27 +0800 Subject: [PATCH 4/6] refactor: remove useless error check for scan data export job (#20883) Signed-off-by: chlins Co-authored-by: miner --- .../job/impl/scandataexport/scan_data_export.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/jobservice/job/impl/scandataexport/scan_data_export.go b/src/jobservice/job/impl/scandataexport/scan_data_export.go index 93dcb4d9df9..3201d6b417a 100644 --- a/src/jobservice/job/impl/scandataexport/scan_data_export.go +++ b/src/jobservice/job/impl/scandataexport/scan_data_export.go @@ -180,20 +180,17 @@ func (sde *ScanDataExport) updateExecAttributes(ctx job.Context, params job.Para } func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters, fileName string) error { + logger := ctx.GetLogger() csvFile, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm) if err != nil { + logger.Errorf("Failed to create CSV export file %s. Error : %v", fileName, err) return err } - systemContext := ctx.SystemContext() defer csvFile.Close() - logger := ctx.GetLogger() - if err != nil { - logger.Errorf("Failed to create CSV export file %s. Error : %v", fileName, err) - return err - } logger.Infof("Created CSV export file %s", csvFile.Name()) + systemContext := ctx.SystemContext() var exportParams export.Params var artIDGroups [][]int64 From a946447cad5c1ac0b01623fdbe0a16a6c812558e Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Tue, 3 Sep 2024 10:30:18 +0800 Subject: [PATCH 5/6] Escape the - with \- when it is a char of itself (#20892) fixes #20891 Signed-off-by: stonezdj --- .../projects/create-project/create-project.component.html | 2 +- .../create-edit-rule/create-edit-rule.component.html | 2 +- .../system-robot-accounts/new-robot/new-robot.component.html | 2 +- .../project/robot-account/add-robot/add-robot.component.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html b/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html index 51f28e20ab6..e09a4bd6935 100644 --- a/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html +++ b/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html @@ -21,7 +21,7 @@

{{ 'PROJECT.NEW_PROJECT' | translate }}

name="create_project_name" class="clr-input input-width" required - pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" + pattern="^[a-z0-9]+(?:[._\-][a-z0-9]+)*$" #projectName autocomplete="off" /> {{ headerTitle | translate }} type="text" id="ruleName" size="35" - pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" + pattern="^[a-z0-9]+(?:[._\-][a-z0-9]+)*$" required maxlength="255" formControlName="name" diff --git a/src/portal/src/app/base/left-side-nav/system-robot-accounts/new-robot/new-robot.component.html b/src/portal/src/app/base/left-side-nav/system-robot-accounts/new-robot/new-robot.component.html index a3cb1a986f8..0112e887e5a 100644 --- a/src/portal/src/app/base/left-side-nav/system-robot-accounts/new-robot/new-robot.component.html +++ b/src/portal/src/app/base/left-side-nav/system-robot-accounts/new-robot/new-robot.component.html @@ -70,7 +70,7 @@

id="name" [(ngModel)]="systemRobot.name" required - pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" + pattern="^[a-z0-9]+(?:[._\-][a-z0-9]+)*$" maxLengthExt="255" autocomplete="off" size="30" diff --git a/src/portal/src/app/base/project/robot-account/add-robot/add-robot.component.html b/src/portal/src/app/base/project/robot-account/add-robot/add-robot.component.html index a399d6eef34..fd27bd4175c 100644 --- a/src/portal/src/app/base/project/robot-account/add-robot/add-robot.component.html +++ b/src/portal/src/app/base/project/robot-account/add-robot/add-robot.component.html @@ -73,7 +73,7 @@

id="name" [(ngModel)]="robot.name" required - pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" + pattern="^[a-z0-9]+(?:[._\-][a-z0-9]+)*$" maxLengthExt="255" autocomplete="off" size="30" From 9ad8094440370efefe34ab82a5cd4ecdb39e3be2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:26:24 +0000 Subject: [PATCH 6/6] chore(deps): bump go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp from 0.49.0 to 0.54.0 in /src (#20878) chore(deps): bump go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp Bumps [go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.49.0 to 0.54.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.49.0...zpages/v0.54.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shengwen YU Co-authored-by: miner --- src/go.mod | 14 +++++++------- src/go.sum | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/go.mod b/src/go.mod index f33eaa3fafc..9872b76e1f1 100644 --- a/src/go.mod +++ b/src/go.mod @@ -21,12 +21,12 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.7 github.com/go-ldap/ldap/v3 v3.4.6 github.com/go-openapi/errors v0.22.0 - github.com/go-openapi/loads v0.21.2 + github.com/go-openapi/loads v0.21.2 // indirect github.com/go-openapi/runtime v0.26.2 - github.com/go-openapi/spec v0.20.11 + github.com/go-openapi/spec v0.20.11 // indirect github.com/go-openapi/strfmt v0.23.0 github.com/go-openapi/swag v0.23.0 - github.com/go-openapi/validate v0.22.3 + github.com/go-openapi/validate v0.22.3 // indirect github.com/go-redis/redis/v8 v8.11.4 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 github.com/gocraft/work v0.5.1 @@ -56,12 +56,12 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 github.com/volcengine/volcengine-go-sdk v1.0.138 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.51.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 - go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 + go.opentelemetry.io/otel v1.29.0 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.28.0 + go.opentelemetry.io/otel/trace v1.29.0 go.uber.org/ratelimit v0.3.1 golang.org/x/crypto v0.25.0 golang.org/x/net v0.26.0 @@ -166,7 +166,7 @@ require ( github.com/volcengine/volc-sdk-golang v1.0.23 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect diff --git a/src/go.sum b/src/go.sum index 784cde69e6e..d071078b7cb 100644 --- a/src/go.sum +++ b/src/go.sum @@ -569,25 +569,25 @@ go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.51.0 h1:rXpHmgy1pMXlfv3W1T5ctoDA3QeTFjNq/YwCmwrfr8Q= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.51.0/go.mod h1:9uIRD3NZrM7QMQEGeKhr7V4xSDTMku3MPOVs8iZ3VVk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U= go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=