From 336da1e98be0a67f4bde4473b4bb840ff478eff0 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 1 Nov 2023 12:59:20 -0500 Subject: [PATCH 1/9] refactor(types): move worker type to server and nest API object --- api/types/worker.go | 370 +++++++++++++++++++++++ api/types/worker_test.go | 224 ++++++++++++++ api/worker/create.go | 3 +- api/worker/get.go | 17 +- api/worker/list.go | 37 ++- api/worker/update.go | 8 +- database/integration_test.go | 13 +- database/types/sanitize.go | 42 +++ database/types/sanitize_test.go | 83 +++++ database/types/worker.go | 191 ++++++++++++ database/types/worker_test.go | 216 +++++++++++++ database/worker/create.go | 10 +- database/worker/create_test.go | 2 +- database/worker/delete.go | 8 +- database/worker/get.go | 10 +- database/worker/get_hostname.go | 10 +- database/worker/get_hostname_test.go | 5 +- database/worker/get_test.go | 5 +- database/worker/interface.go | 14 +- database/worker/list.go | 12 +- database/worker/list_test.go | 10 +- database/worker/update.go | 10 +- database/worker/update_test.go | 2 +- database/worker/worker.go | 22 ++ database/worker/worker_test.go | 52 +++- go.mod | 4 +- mock/server/worker.go | 67 +++- router/middleware/logger_test.go | 3 +- router/middleware/worker/context.go | 8 +- router/middleware/worker/context_test.go | 6 +- router/middleware/worker/worker.go | 4 +- router/middleware/worker/worker_test.go | 12 +- 32 files changed, 1393 insertions(+), 87 deletions(-) create mode 100644 api/types/worker.go create mode 100644 api/types/worker_test.go create mode 100644 database/types/sanitize.go create mode 100644 database/types/sanitize_test.go create mode 100644 database/types/worker.go create mode 100644 database/types/worker_test.go diff --git a/api/types/worker.go b/api/types/worker.go new file mode 100644 index 000000000..78f86cf1c --- /dev/null +++ b/api/types/worker.go @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + + "github.com/go-vela/types/library" +) + +// Worker is the library representation of a worker. +// +// swagger:model Worker +type Worker struct { + ID *int64 `json:"id,omitempty"` + Hostname *string `json:"hostname,omitempty"` + Address *string `json:"address,omitempty"` + Routes *[]string `json:"routes,omitempty"` + Active *bool `json:"active,omitempty"` + Status *string `json:"status,omitempty"` + LastStatusUpdateAt *int64 `json:"last_status_update_at,omitempty"` + RunningBuilds []*library.Build `json:"running_builds,omitempty"` + LastBuildStartedAt *int64 `json:"last_build_started_at,omitempty"` + LastBuildFinishedAt *int64 `json:"last_build_finished_at,omitempty"` + LastCheckedIn *int64 `json:"last_checked_in,omitempty"` + BuildLimit *int64 `json:"build_limit,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetID() int64 { + // return zero value if Worker type or ID field is nil + if w == nil || w.ID == nil { + return 0 + } + + return *w.ID +} + +// GetHostname returns the Hostname field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetHostname() string { + // return zero value if Worker type or Hostname field is nil + if w == nil || w.Hostname == nil { + return "" + } + + return *w.Hostname +} + +// GetAddress returns the Address field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetAddress() string { + // return zero value if Worker type or Address field is nil + if w == nil || w.Address == nil { + return "" + } + + return *w.Address +} + +// GetRoutes returns the Routes field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRoutes() []string { + // return zero value if Worker type or Routes field is nil + if w == nil || w.Routes == nil { + return []string{} + } + + return *w.Routes +} + +// GetActive returns the Active field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetActive() bool { + // return zero value if Worker type or Active field is nil + if w == nil || w.Active == nil { + return false + } + + return *w.Active +} + +// GetStatus returns the Status field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetStatus() string { + // return zero value if Worker type or Status field is nil + if w == nil || w.Status == nil { + return "" + } + + return *w.Status +} + +// GetLastStatusUpdateAt returns the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastStatusUpdateAt() int64 { + // return zero value if Worker type or LastStatusUpdateAt field is nil + if w == nil || w.LastStatusUpdateAt == nil { + return 0 + } + + return *w.LastStatusUpdateAt +} + +// GetRunningBuilds returns the RunningBuilds field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRunningBuilds() []*library.Build { + // return zero value if Worker type or RunningBuilds field is nil + if w == nil || w.RunningBuilds == nil { + return []*library.Build{} + } + + return w.RunningBuilds +} + +// GetLastBuildStartedAt returns the LastBuildStartedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildStartedAt() int64 { + // return zero value if Worker type or LastBuildStartedAt field is nil + if w == nil || w.LastBuildStartedAt == nil { + return 0 + } + + return *w.LastBuildStartedAt +} + +// GetLastBuildFinishedAt returns the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildFinishedAt() int64 { + // return zero value if Worker type or LastBuildFinishedAt field is nil + if w == nil || w.LastBuildFinishedAt == nil { + return 0 + } + + return *w.LastBuildFinishedAt +} + +// GetLastCheckedIn returns the LastCheckedIn field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastCheckedIn() int64 { + // return zero value if Worker type or LastCheckedIn field is nil + if w == nil || w.LastCheckedIn == nil { + return 0 + } + + return *w.LastCheckedIn +} + +// GetBuildLimit returns the BuildLimit field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetBuildLimit() int64 { + // return zero value if Worker type or BuildLimit field is nil + if w == nil || w.BuildLimit == nil { + return 0 + } + + return *w.BuildLimit +} + +// SetID sets the ID field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetID(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.ID = &v +} + +// SetHostname sets the Hostname field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetHostname(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Hostname = &v +} + +// SetAddress sets the Address field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetAddress(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Address = &v +} + +// SetRoutes sets the Routes field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRoutes(v []string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Routes = &v +} + +// SetActive sets the Active field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetActive(v bool) { + // return if Worker type is nil + if w == nil { + return + } + + w.Active = &v +} + +// SetStatus sets the Status field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetStatus(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Status = &v +} + +// SetLastStatusUpdateAt sets the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastStatusUpdateAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastStatusUpdateAt = &v +} + +// SetRunningBuilds sets the RunningBuilds field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRunningBuilds(builds []*library.Build) { + // return if Worker type is nil + if w == nil { + return + } + + w.RunningBuilds = builds +} + +// SetLastBuildStartedAt sets the LastBuildStartedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildStartedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildStartedAt = &v +} + +// SetLastBuildFinishedAt sets the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildFinishedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildFinishedAt = &v +} + +// SetLastCheckedIn sets the LastCheckedIn field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastCheckedIn(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastCheckedIn = &v +} + +// SetBuildLimit sets the LastBuildLimit field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetBuildLimit(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.BuildLimit = &v +} + +// String implements the Stringer interface for the Worker type. +func (w *Worker) String() string { + return fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) +} diff --git a/api/types/worker_test.go b/api/types/worker_test.go new file mode 100644 index 000000000..19d2defb5 --- /dev/null +++ b/api/types/worker_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/go-vela/types/library" +) + +func TestTypes_Worker_Getters(t *testing.T) { + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: new(Worker), + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + if test.worker.GetID() != test.want.GetID() { + t.Errorf("GetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("GetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("Getaddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("GetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("GetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("GetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("GetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if !reflect.DeepEqual(test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) { + t.Errorf("GetRunningBuildIDs is %v, want %v", test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("GetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("GetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("GetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("GetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_Setters(t *testing.T) { + // setup types + var w *Worker + + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: w, + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + test.worker.SetID(test.want.GetID()) + test.worker.SetHostname(test.want.GetHostname()) + test.worker.SetAddress(test.want.GetAddress()) + test.worker.SetRoutes(test.want.GetRoutes()) + test.worker.SetActive(test.want.GetActive()) + test.worker.SetStatus(test.want.GetStatus()) + test.worker.SetLastStatusUpdateAt(test.want.GetLastStatusUpdateAt()) + test.worker.SetRunningBuilds(test.want.GetRunningBuilds()) + test.worker.SetLastBuildStartedAt(test.want.GetLastBuildStartedAt()) + test.worker.SetLastBuildFinishedAt(test.want.GetLastBuildFinishedAt()) + test.worker.SetLastCheckedIn(test.want.GetLastCheckedIn()) + test.worker.SetBuildLimit(test.want.GetBuildLimit()) + + if test.worker.GetID() != test.want.GetID() { + t.Errorf("SetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("SetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("SetAddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("SetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("SetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("SetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("SetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("SetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("SetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("SetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("SetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_String(t *testing.T) { + // setup types + w := testWorker() + + want := fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) + + // run test + got := w.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testWorker is a test helper function to create a Worker +// type with all fields set to a fake value. +func testWorker() *Worker { + b := new(library.Build) + b.SetID(1) + + w := new(Worker) + + w.SetID(1) + w.SetHostname("worker_0") + w.SetAddress("http://localhost:8080") + w.SetRoutes([]string{"vela"}) + w.SetActive(true) + w.SetStatus("available") + w.SetLastStatusUpdateAt(time.Time{}.UTC().Unix()) + w.SetRunningBuilds([]*library.Build{b}) + w.SetLastBuildStartedAt(time.Time{}.UTC().Unix()) + w.SetLastBuildFinishedAt(time.Time{}.UTC().Unix()) + w.SetLastCheckedIn(time.Time{}.UTC().Unix()) + w.SetBuildLimit(2) + + return w +} diff --git a/api/worker/create.go b/api/worker/create.go index 07edf3acc..d91898885 100644 --- a/api/worker/create.go +++ b/api/worker/create.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/claims" @@ -58,7 +59,7 @@ func CreateWorker(c *gin.Context) { ctx := c.Request.Context() // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { diff --git a/api/worker/get.go b/api/worker/get.go index a8a4d6931..99cb17204 100644 --- a/api/worker/get.go +++ b/api/worker/get.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -55,14 +56,20 @@ func GetWorker(c *gin.Context) { "worker": w.GetHostname(), }).Infof("reading worker %s", w.GetHostname()) - w, err := database.FromContext(c).GetWorkerForHostname(ctx, w.GetHostname()) - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) + var rBs []*library.Build + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) - util.HandleError(c, http.StatusNotFound, retErr) + return + } - return + rBs = append(rBs, build) } + w.SetRunningBuilds(rBs) + c.JSON(http.StatusOK, w) } diff --git a/api/worker/list.go b/api/worker/list.go index 6b24484a6..1bbd9bbc5 100644 --- a/api/worker/list.go +++ b/api/worker/list.go @@ -10,6 +10,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -22,6 +23,12 @@ import ( // - application/json // security: // - ApiKeyAuth: [] +// parameters: +// - in: query +// name: links +// description: Include links to builds currently running +// type: boolean +// default: false // responses: // '200': // description: Successfully retrieved the list of workers @@ -48,7 +55,15 @@ func ListWorkers(c *gin.Context) { "user": u.GetName(), }).Info("reading workers") - w, err := database.FromContext(c).ListWorkers(ctx) + var filters = map[string]interface{}{} + + active := c.Query("active") + + if len(active) > 0 { + filters["active"] = active + } + + workers, err := database.FromContext(c).ListWorkers(ctx) if err != nil { retErr := fmt.Errorf("unable to get workers: %w", err) @@ -57,5 +72,23 @@ func ListWorkers(c *gin.Context) { return } - c.JSON(http.StatusOK, w) + for _, w := range workers { + var rBs []*library.Build + + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + rBs = append(rBs, build) + } + + w.SetRunningBuilds(rBs) + } + + c.JSON(http.StatusOK, workers) } diff --git a/api/worker/update.go b/api/worker/update.go index d95ddf7af..e5b842a0d 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -7,11 +7,11 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -71,7 +71,7 @@ func UpdateWorker(c *gin.Context) { }).Infof("updating worker %s", w.GetHostname()) // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { @@ -97,9 +97,9 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } - if input.RunningBuildIDs != nil { + if input.RunningBuilds != nil { // update runningBuildIDs if set - w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + w.SetRunningBuilds(input.GetRunningBuilds()) } if len(input.GetStatus()) > 0 { diff --git a/database/integration_test.go b/database/integration_test.go index b30176004..0cc6b1659 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" @@ -42,7 +43,7 @@ type Resources struct { Services []*library.Service Steps []*library.Step Users []*library.User - Workers []*library.Worker + Workers []*api.Worker } func TestDatabase_Integration(t *testing.T) { @@ -2313,7 +2314,7 @@ func newResources() *Resources { userTwo.SetActive(true) userTwo.SetAdmin(false) - workerOne := new(library.Worker) + workerOne := new(api.Worker) workerOne.SetID(1) workerOne.SetHostname("worker-1.example.com") workerOne.SetAddress("https://worker-1.example.com") @@ -2321,13 +2322,13 @@ func newResources() *Resources { workerOne.SetActive(true) workerOne.SetStatus("available") workerOne.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerOne.SetRunningBuildIDs([]string{"12345"}) + workerOne.SetRunningBuilds([]*library.Build{buildOne}) workerOne.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerOne.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerOne.SetLastCheckedIn(time.Now().UTC().Unix()) workerOne.SetBuildLimit(1) - workerTwo := new(library.Worker) + workerTwo := new(api.Worker) workerTwo.SetID(2) workerTwo.SetHostname("worker-2.example.com") workerTwo.SetAddress("https://worker-2.example.com") @@ -2335,7 +2336,7 @@ func newResources() *Resources { workerTwo.SetActive(true) workerTwo.SetStatus("available") workerTwo.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerTwo.SetRunningBuildIDs([]string{"12345"}) + workerTwo.SetRunningBuilds([]*library.Build{buildTwo}) workerTwo.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerTwo.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerTwo.SetLastCheckedIn(time.Now().UTC().Unix()) @@ -2354,7 +2355,7 @@ func newResources() *Resources { Services: []*library.Service{serviceOne, serviceTwo}, Steps: []*library.Step{stepOne, stepTwo}, Users: []*library.User{userOne, userTwo}, - Workers: []*library.Worker{workerOne, workerTwo}, + Workers: []*api.Worker{workerOne, workerTwo}, } } diff --git a/database/types/sanitize.go b/database/types/sanitize.go new file mode 100644 index 000000000..c21918194 --- /dev/null +++ b/database/types/sanitize.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "html" + "net/url" + "strings" + + "github.com/microcosm-cc/bluemonday" +) + +// sanitize is a helper function to verify the provided input +// field does not contain HTML content. If the input field +// does contain HTML, then the function will sanitize and +// potentially remove the HTML if deemed malicious. +func sanitize(field string) string { + // create new HTML input microcosm-cc/bluemonday policy + p := bluemonday.StrictPolicy() + + // create a URL query unescaped string from the field + queryUnescaped, err := url.QueryUnescape(field) + if err != nil { + // overwrite URL query unescaped string with field + queryUnescaped = field + } + + // create an HTML escaped string from the field + htmlEscaped := html.EscapeString(queryUnescaped) + + // create a microcosm-cc/bluemonday escaped string from the field + bluemondayEscaped := p.Sanitize(queryUnescaped) + + // check if the field contains html + if !strings.EqualFold(htmlEscaped, bluemondayEscaped) { + // create new HTML input microcosm-cc/bluemonday policy + return bluemondayEscaped + } + + // return the unmodified field + return field +} diff --git a/database/types/sanitize_test.go b/database/types/sanitize_test.go new file mode 100644 index 000000000..a886e982a --- /dev/null +++ b/database/types/sanitize_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "testing" +) + +func TestTypes_Sanitize(t *testing.T) { + // setup tests + tests := []struct { + name string + value string + want string + }{ + { + name: "percent", + value: `%`, + want: `%`, + }, + { + name: "quoted", + value: `"hello"`, + want: `"hello"`, + }, + { + name: "email", + value: `OctoKitty@github.com`, + want: `OctoKitty@github.com`, + }, + { + name: "url", + value: `https://github.com/go-vela`, + want: `https://github.com/go-vela`, + }, + { + name: "encoded", + value: `+ added foo %25 + updated bar %22 +`, + want: `+ added foo %25 + updated bar %22 +`, + }, + { + name: "html with headers", + value: `Merge pull request #1 from me/patch-1\n\n

hello

is now

hello

`, + want: `Merge pull request #1 from me/patch-1\n\nhello is now hello`, + }, + { + name: "html with email", + value: `Co-authored-by: OctoKitty `, + want: `Co-authored-by: OctoKitty `, + }, + { + name: "html with href", + value: `Google`, + want: `Google`, + }, + { + name: "local cross-site script", + value: ``, + want: ``, + }, + { + name: "remote cross-site script", + value: ``, + want: ``, + }, + { + name: "embedded cross-site script", + value: `%3cDIV%20STYLE%3d%22width%3a%20expression(alert('XSS'))%3b%22%3e`, + want: ``, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := sanitize(test.value) + + if got != test.want { + t.Errorf("sanitize is %v, want %v", got, test.want) + } + }) + } +} diff --git a/database/types/worker.go b/database/types/worker.go new file mode 100644 index 000000000..075ce916b --- /dev/null +++ b/database/types/worker.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "errors" + "fmt" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/lib/pq" +) + +var ( + // ErrEmptyWorkerHost defines the error type when a + // Worker type has an empty Host field provided. + ErrEmptyWorkerHost = errors.New("empty worker address provided") + + // ErrEmptyWorkerAddress defines the error type when a + // Worker type has an empty Address field provided. + ErrEmptyWorkerAddress = errors.New("empty worker address provided") + + // ErrExceededRunningBuildIDsLimit defines the error type when a + // Worker type has RunningBuildIDs field provided that exceeds the database limit. + ErrExceededRunningBuildIDsLimit = errors.New("exceeded running build ids limit") +) + +// Worker is the database representation of a worker. +type Worker struct { + ID sql.NullInt64 `sql:"id"` + Hostname sql.NullString `sql:"hostname"` + Address sql.NullString `sql:"address"` + Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` + Active sql.NullBool `sql:"active"` + Status sql.NullString `sql:"status"` + LastStatusUpdateAt sql.NullInt64 `sql:"last_status_update_at"` + RunningBuildIDs pq.StringArray `sql:"running_build_ids" gorm:"type:varchar(500)"` + LastBuildStartedAt sql.NullInt64 `sql:"last_build_started_at"` + LastBuildFinishedAt sql.NullInt64 `sql:"last_build_finished_at"` + LastCheckedIn sql.NullInt64 `sql:"last_checked_in"` + BuildLimit sql.NullInt64 `sql:"build_limit"` +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Build type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (w *Worker) Nullify() *Worker { + if w == nil { + return nil + } + + // check if the ID field should be false + if w.ID.Int64 == 0 { + w.ID.Valid = false + } + + // check if the Hostname field should be false + if len(w.Hostname.String) == 0 { + w.Hostname.Valid = false + } + + // check if the Address field should be false + if len(w.Address.String) == 0 { + w.Address.Valid = false + } + + // check if the Status field should be false + if len(w.Status.String) == 0 { + w.Status.Valid = false + } + + // check if the LastStatusUpdateAt field should be false + if w.LastStatusUpdateAt.Int64 == 0 { + w.LastStatusUpdateAt.Valid = false + } + + // check if the LastBuildStartedAt field should be false + if w.LastBuildStartedAt.Int64 == 0 { + w.LastBuildStartedAt.Valid = false + } + + // check if the LastBuildFinishedAt field should be false + if w.LastBuildFinishedAt.Int64 == 0 { + w.LastBuildFinishedAt.Valid = false + } + + // check if the LastCheckedIn field should be false + if w.LastCheckedIn.Int64 == 0 { + w.LastCheckedIn.Valid = false + } + + if w.BuildLimit.Int64 == 0 { + w.BuildLimit.Valid = false + } + + return w +} + +// ToAPI converts the Worker type +// to an API Worker type. +func (w *Worker) ToAPI(builds []*library.Build) *types.Worker { + worker := new(types.Worker) + + worker.SetID(w.ID.Int64) + worker.SetHostname(w.Hostname.String) + worker.SetAddress(w.Address.String) + worker.SetRoutes(w.Routes) + worker.SetActive(w.Active.Bool) + worker.SetStatus(w.Status.String) + worker.SetLastStatusUpdateAt(w.LastStatusUpdateAt.Int64) + worker.SetRunningBuilds(builds) + worker.SetLastBuildStartedAt(w.LastBuildStartedAt.Int64) + worker.SetLastBuildFinishedAt(w.LastBuildFinishedAt.Int64) + worker.SetLastCheckedIn(w.LastCheckedIn.Int64) + worker.SetBuildLimit(w.BuildLimit.Int64) + + return worker +} + +// Validate verifies the necessary fields for +// the Worker type are populated correctly. +func (w *Worker) Validate() error { + // verify the Host field is populated + if len(w.Hostname.String) == 0 { + return ErrEmptyWorkerHost + } + + // verify the Address field is populated + if len(w.Address.String) == 0 { + return ErrEmptyWorkerAddress + } + + // calculate total size of RunningBuildIds + total := 0 + for _, f := range w.RunningBuildIDs { + total += len(f) + } + + // verify the RunningBuildIds field is within the database constraints + // len is to factor in number of comma separators included in the database field, + // removing 1 due to the last item not having an appended comma + if (total + len(w.RunningBuildIDs) - 1) > constants.RunningBuildIDsMaxSize { + return ErrExceededRunningBuildIDsLimit + } + + // ensure that all Worker string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + w.Hostname = sql.NullString{String: sanitize(w.Hostname.String), Valid: w.Hostname.Valid} + w.Address = sql.NullString{String: sanitize(w.Address.String), Valid: w.Address.Valid} + + // ensure that all Routes are sanitized + // to avoid unsafe HTML content + for i, v := range w.Routes { + w.Routes[i] = sanitize(v) + } + + return nil +} + +// WorkerFromAPI converts the library worker type +// to a database worker type. +func WorkerFromAPI(w *types.Worker) *Worker { + var rBs []string + + for _, b := range w.GetRunningBuilds() { + rBs = append(rBs, fmt.Sprint(b.GetID())) + } + + worker := &Worker{ + ID: sql.NullInt64{Int64: w.GetID(), Valid: true}, + Hostname: sql.NullString{String: w.GetHostname(), Valid: true}, + Address: sql.NullString{String: w.GetAddress(), Valid: true}, + Routes: pq.StringArray(w.GetRoutes()), + Active: sql.NullBool{Bool: w.GetActive(), Valid: true}, + Status: sql.NullString{String: w.GetStatus(), Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: w.GetLastStatusUpdateAt(), Valid: true}, + RunningBuildIDs: pq.StringArray(rBs), + LastBuildStartedAt: sql.NullInt64{Int64: w.GetLastBuildStartedAt(), Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: w.GetLastBuildFinishedAt(), Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: w.GetLastCheckedIn(), Valid: true}, + BuildLimit: sql.NullInt64{Int64: w.GetBuildLimit(), Valid: true}, + } + + return worker.Nullify() +} diff --git a/database/types/worker_test.go b/database/types/worker_test.go new file mode 100644 index 000000000..5a7a65999 --- /dev/null +++ b/database/types/worker_test.go @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "reflect" + "strconv" + "testing" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/types/library" +) + +func TestTypes_Worker_Nullify(t *testing.T) { + // setup types + var w *Worker + + want := &Worker{ + ID: sql.NullInt64{Int64: 0, Valid: false}, + Hostname: sql.NullString{String: "", Valid: false}, + Address: sql.NullString{String: "", Valid: false}, + Active: sql.NullBool{Bool: false, Valid: false}, + Status: sql.NullString{String: "", Valid: false}, + LastStatusUpdateAt: sql.NullInt64{Int64: 0, Valid: false}, + LastBuildStartedAt: sql.NullInt64{Int64: 0, Valid: false}, + LastBuildFinishedAt: sql.NullInt64{Int64: 0, Valid: false}, + LastCheckedIn: sql.NullInt64{Int64: 0, Valid: false}, + BuildLimit: sql.NullInt64{Int64: 0, Valid: false}, + } + + // setup tests + tests := []struct { + repo *Worker + want *Worker + }{ + { + repo: testWorker(), + want: testWorker(), + }, + { + repo: w, + want: nil, + }, + { + repo: new(Worker), + want: want, + }, + } + + // run tests + for _, test := range tests { + got := test.repo.Nullify() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("Nullify is %v, want %v", got, test.want) + } + } +} + +func TestTypes_Worker_ToAPI(t *testing.T) { + // setup types + b := new(library.Build) + b.SetID(12345) + + builds := []*library.Build{b} + + want := new(types.Worker) + + want.SetID(1) + want.SetHostname("worker_0") + want.SetAddress("http://localhost:8080") + want.SetRoutes([]string{"vela"}) + want.SetActive(true) + want.SetStatus("available") + want.SetLastStatusUpdateAt(1563474077) + want.SetRunningBuilds(builds) + want.SetLastBuildStartedAt(1563474077) + want.SetLastBuildFinishedAt(1563474077) + want.SetLastCheckedIn(1563474077) + want.SetBuildLimit(2) + + // run test + got := testWorker().ToAPI(builds) + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestTypes_Worker_Validate(t *testing.T) { + // setup tests + tests := []struct { + failure bool + worker *Worker + }{ + { + failure: false, + worker: testWorker(), + }, + { // no Hostname set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + { // no Address set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + { // invalid RunningBuildIDs set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + RunningBuildIDs: exceededRunningBuildIDs(), + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + } + + // run tests + for _, test := range tests { + err := test.worker.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} + +func TestTypes_WorkerFromLibrary(t *testing.T) { + // setup types + b := new(library.Build) + b.SetID(12345) + + builds := []*library.Build{b} + + w := new(types.Worker) + + w.SetID(1) + w.SetHostname("worker_0") + w.SetAddress("http://localhost:8080") + w.SetRoutes([]string{"vela"}) + w.SetActive(true) + w.SetStatus("available") + w.SetLastStatusUpdateAt(1563474077) + w.SetRunningBuilds(builds) + w.SetLastBuildStartedAt(1563474077) + w.SetLastBuildFinishedAt(1563474077) + w.SetLastCheckedIn(1563474077) + w.SetBuildLimit(2) + + want := testWorker() + + // run test + got := WorkerFromAPI(w) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WorkerFromLibrary is %v, want %v", got, want) + } +} + +// testWorker is a test helper function to create a Worker +// type with all fields set to a fake value. +func testWorker() *Worker { + return &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Routes: []string{"vela"}, + Active: sql.NullBool{Bool: true, Valid: true}, + Status: sql.NullString{String: "available", Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + RunningBuildIDs: []string{"12345"}, + LastBuildStartedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + BuildLimit: sql.NullInt64{Int64: 2, Valid: true}, + } +} + +// exceededRunningBuildIDs returns a list of valid running builds that exceed the maximum size. +func exceededRunningBuildIDs() []string { + // initialize empty runningBuildIDs + runningBuildIDs := []string{} + + // add enough build ids to exceed the character limit + for i := 0; i < 50; i++ { + // construct runningBuildID + // use i to adhere to unique runningBuildIDs + runningBuildID := "1234567890-" + strconv.Itoa(i) + + runningBuildIDs = append(runningBuildIDs, runningBuildID) + } + + return runningBuildIDs +} diff --git a/database/worker/create.go b/database/worker/create.go index 12b386319..7818bd2db 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // CreateWorker creates a new worker in the database. -func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("creating worker %s in the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // validate the necessary fields are populated // @@ -33,5 +33,5 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Create(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/create_test.go b/database/worker/create_test.go index dad81473a..6809bb33e 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -28,7 +28,7 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "workers" ("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/delete.go b/database/worker/delete.go index ba846db91..a2eb6f566 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // DeleteWorker deletes an existing worker from the database. -func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { +func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("deleting worker %s from the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // send query to the database return e.client. diff --git a/database/worker/get.go b/database/worker/get.go index 12201d4d5..7aca60b76 100644 --- a/database/worker/get.go +++ b/database/worker/get.go @@ -5,17 +5,17 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // GetWorker gets a worker by ID from the database. -func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, error) { +func (e *engine) GetWorker(ctx context.Context, id int64) (*api.Worker, error) { e.logger.Tracef("getting worker %d from the database", id) // variable to store query results - w := new(database.Worker) + w := new(types.Worker) // send query to the database and store result in variable err := e.client. @@ -30,5 +30,5 @@ func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, erro // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go index 3370620a0..4b4b54ad2 100644 --- a/database/worker/get_hostname.go +++ b/database/worker/get_hostname.go @@ -5,20 +5,20 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // GetWorkerForHostname gets a worker by hostname from the database. -func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*library.Worker, error) { +func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": hostname, }).Tracef("getting worker %s from the database", hostname) // variable to store query results - w := new(database.Worker) + w := new(types.Worker) // send query to the database and store result in variable err := e.client. @@ -33,5 +33,5 @@ func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*li // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go index fc4802d70..87e9e1e0c 100644 --- a/database/worker/get_hostname_test.go +++ b/database/worker/get_hostname_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorkerForName(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) // sqlmock cannot parse string array values _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/get_test.go b/database/worker/get_test.go index e5249fda1..a5f6f56d7 100644 --- a/database/worker/get_test.go +++ b/database/worker/get_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorker(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/interface.go b/database/worker/interface.go index c594c556a..fbb2433ef 100644 --- a/database/worker/interface.go +++ b/database/worker/interface.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + "github.com/go-vela/server/api/types" ) // WorkerInterface represents the Vela interface for worker @@ -29,15 +29,15 @@ type WorkerInterface interface { // CountWorkers defines a function that gets the count of all workers. CountWorkers(context.Context) (int64, error) // CreateWorker defines a function that creates a new worker. - CreateWorker(context.Context, *library.Worker) (*library.Worker, error) + CreateWorker(context.Context, *types.Worker) (*types.Worker, error) // DeleteWorker defines a function that deletes an existing worker. - DeleteWorker(context.Context, *library.Worker) error + DeleteWorker(context.Context, *types.Worker) error // GetWorker defines a function that gets a worker by ID. - GetWorker(context.Context, int64) (*library.Worker, error) + GetWorker(context.Context, int64) (*types.Worker, error) // GetWorkerForHostname defines a function that gets a worker by hostname. - GetWorkerForHostname(context.Context, string) (*library.Worker, error) + GetWorkerForHostname(context.Context, string) (*types.Worker, error) // ListWorkers defines a function that gets a list of all workers. - ListWorkers(context.Context) ([]*library.Worker, error) + ListWorkers(context.Context) ([]*types.Worker, error) // UpdateWorker defines a function that updates an existing worker. - UpdateWorker(context.Context, *library.Worker) (*library.Worker, error) + UpdateWorker(context.Context, *types.Worker) (*types.Worker, error) } diff --git a/database/worker/list.go b/database/worker/list.go index dbf8ff2a7..23910504b 100644 --- a/database/worker/list.go +++ b/database/worker/list.go @@ -5,19 +5,19 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListWorkers gets a list of all workers from the database. -func (e *engine) ListWorkers(ctx context.Context) ([]*library.Worker, error) { +func (e *engine) ListWorkers(ctx context.Context) ([]*api.Worker, error) { e.logger.Trace("listing all workers from the database") // variables to store query results and return value count := int64(0) - w := new([]database.Worker) - workers := []*library.Worker{} + w := new([]types.Worker) + workers := []*api.Worker{} // count the results count, err := e.CountWorkers(ctx) @@ -47,7 +47,7 @@ func (e *engine) ListWorkers(ctx context.Context) ([]*library.Worker, error) { // convert query result to library type // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - workers = append(workers, tmp.ToLibrary()) + workers = append(workers, tmp.ToAPI(convertToBuilds(tmp.RunningBuildIDs))) } return workers, nil diff --git a/database/worker/list_test.go b/database/worker/list_test.go index 9c5331fe4..0786d8ebf 100644 --- a/database/worker/list_test.go +++ b/database/worker/list_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_ListWorkers(t *testing.T) { @@ -18,12 +18,14 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { _workerOne.SetHostname("worker_0") _workerOne.SetAddress("localhost") _workerOne.SetActive(true) + _workerOne.SetRunningBuilds(nil) _workerTwo := testWorker() _workerTwo.SetID(2) _workerTwo.SetHostname("worker_1") _workerTwo.SetAddress("localhost") _workerTwo.SetActive(true) + _workerTwo.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -61,19 +63,19 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { failure bool name string database *engine - want []*library.Worker + want []*api.Worker }{ { failure: false, name: "postgres", database: _postgres, - want: []*library.Worker{_workerOne, _workerTwo}, + want: []*api.Worker{_workerOne, _workerTwo}, }, { failure: false, name: "sqlite3", database: _sqlite, - want: []*library.Worker{_workerOne, _workerTwo}, + want: []*api.Worker{_workerOne, _workerTwo}, }, } diff --git a/database/worker/update.go b/database/worker/update.go index 3cae0d164..c162af9f9 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // UpdateWorker updates an existing worker in the database. -func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("updating worker %s in the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // validate the necessary fields are populated // @@ -33,5 +33,5 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Save(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/update_test.go b/database/worker/update_test.go index d077bc567..5a483660b 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -25,7 +25,7 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { _mock.ExpectExec(`UPDATE "workers" SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11 WHERE "id" = $12`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker.go b/database/worker/worker.go index 09b69176a..aaeed7f10 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -5,8 +5,10 @@ package worker import ( "context" "fmt" + "strconv" "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" "gorm.io/gorm" @@ -79,3 +81,23 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } + +// convertToBuilds is a helper function that generates build objects with ID fields given a list of IDs. +func convertToBuilds(ids []string) []*library.Build { + // create stripped build objects holding the IDs + var rBs []*library.Build + + for _, b := range ids { + id, err := strconv.ParseInt(b, 10, 64) + if err != nil { + return nil + } + + build := new(library.Build) + build.SetID(id) + + rBs = append(rBs, build) + } + + return rBs +} diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index a219d1ffc..a7f47428d 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" @@ -106,6 +107,48 @@ func TestWorker_New(t *testing.T) { } } +func TestWorker_convertToBuilds(t *testing.T) { + _buildOne := new(library.Build) + _buildOne.SetID(1) + + _buildTwo := new(library.Build) + _buildTwo.SetID(2) + + // setup tests + tests := []struct { + name string + ids []string + want []*library.Build + }{ + { + name: "one id", + ids: []string{"1"}, + want: []*library.Build{_buildOne}, + }, + { + name: "multiple ids", + ids: []string{"1", "2"}, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + name: "not int64", + ids: []string{"1", "foo"}, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := convertToBuilds(test.ids) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("convertToBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + // testPostgres is a helper function to create a Postgres engine for testing. func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { // create the new mock sql database @@ -166,8 +209,11 @@ func testSqlite(t *testing.T) *engine { // testWorker is a test helper function to create a library // Worker type with all fields set to their zero values. -func testWorker() *library.Worker { - return &library.Worker{ +func testWorker() *api.Worker { + b := new(library.Build) + b.SetID(1) + + return &api.Worker{ ID: new(int64), Hostname: new(string), Address: new(string), @@ -175,7 +221,7 @@ func testWorker() *library.Worker { Active: new(bool), Status: new(string), LastStatusUpdateAt: new(int64), - RunningBuildIDs: new([]string), + RunningBuilds: []*library.Build{b}, LastBuildStartedAt: new(int64), LastBuildFinishedAt: new(int64), LastCheckedIn: new(int64), diff --git a/go.mod b/go.mod index d0cf1c469..53e798357 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/vault/api v1.10.0 github.com/joho/godotenv v1.5.1 + github.com/lib/pq v1.10.9 + github.com/microcosm-cc/bluemonday v1.0.25 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/redis/go-redis/v9 v9.2.1 @@ -91,12 +93,10 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/mock/server/worker.go b/mock/server/worker.go index b8e1d724f..c930603db 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/library" ) @@ -27,6 +28,66 @@ const ( ], "active": true, "last_checked_in": 1602612590 + "running_builds": [ + { + "id": 2, + "repo_id": 1, + "number": 2, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474204, + "created": 1563474204, + "started": 1563474204, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "Second commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "ed95dcc0687c", + "runtime": "", + "distribution": "" + }, + { + "id": 1, + "repo_id": 1, + "number": 1, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474077, + "created": 1563474076, + "started": 1563474077, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "First commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "82823eb770b0", + "runtime": "", + "distribution": "" + } + ] }` // WorkersResp represents a JSON return for one to many workers. @@ -87,7 +148,7 @@ const ( func getWorkers(c *gin.Context) { data := []byte(WorkersResp) - var body []library.Worker + var body []api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -107,7 +168,7 @@ func getWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -139,7 +200,7 @@ func updateWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) diff --git a/router/middleware/logger_test.go b/router/middleware/logger_test.go index 4a699520a..6419d601f 100644 --- a/router/middleware/logger_test.go +++ b/router/middleware/logger_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/service" @@ -58,7 +59,7 @@ func TestMiddleware_Logger(t *testing.T) { u.SetName("foo") u.SetToken("bar") - w := new(library.Worker) + w := new(api.Worker) w.SetID(1) w.SetHostname("worker_0") w.SetAddress("localhost") diff --git a/router/middleware/worker/context.go b/router/middleware/worker/context.go index a7780a3e8..2e9772a9e 100644 --- a/router/middleware/worker/context.go +++ b/router/middleware/worker/context.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) const key = "worker" @@ -16,13 +16,13 @@ type Setter interface { } // FromContext returns the Worker associated with this context. -func FromContext(c context.Context) *library.Worker { +func FromContext(c context.Context) *api.Worker { value := c.Value(key) if value == nil { return nil } - w, ok := value.(*library.Worker) + w, ok := value.(*api.Worker) if !ok { return nil } @@ -32,6 +32,6 @@ func FromContext(c context.Context) *library.Worker { // ToContext adds the Worker to this context if it supports // the Setter interface. -func ToContext(c Setter, w *library.Worker) { +func ToContext(c Setter, w *api.Worker) { c.Set(key, w) } diff --git a/router/middleware/worker/context_test.go b/router/middleware/worker/context_test.go index 76efd687b..6fd731e90 100644 --- a/router/middleware/worker/context_test.go +++ b/router/middleware/worker/context_test.go @@ -5,7 +5,7 @@ package worker import ( "testing" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" "github.com/gin-gonic/gin" ) @@ -13,7 +13,7 @@ import ( func TestWorker_FromContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) @@ -72,7 +72,7 @@ func TestWorker_FromContext_Empty(t *testing.T) { func TestWorker_ToContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/worker/worker.go b/router/middleware/worker/worker.go index 11586af9c..5a78d14a4 100644 --- a/router/middleware/worker/worker.go +++ b/router/middleware/worker/worker.go @@ -7,14 +7,14 @@ import ( "net/http" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // Retrieve gets the worker in the given context. -func Retrieve(c *gin.Context) *library.Worker { +func Retrieve(c *gin.Context) *api.Worker { return FromContext(c) } diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index 4de2db921..a5b46b8ff 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -10,13 +10,14 @@ import ( "testing" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) func TestWorker_Retrieve(t *testing.T) { // setup types - want := new(library.Worker) + want := new(api.Worker) want.SetID(1) // setup context @@ -34,7 +35,10 @@ func TestWorker_Retrieve(t *testing.T) { func TestWorker_Establish(t *testing.T) { // setup types - want := new(library.Worker) + b := new(library.Build) + b.SetID(1) + + want := new(api.Worker) want.SetID(1) want.SetHostname("worker_0") want.SetAddress("localhost") @@ -42,13 +46,13 @@ func TestWorker_Establish(t *testing.T) { want.SetActive(true) want.SetStatus("available") want.SetLastStatusUpdateAt(12345) - want.SetRunningBuildIDs([]string{}) + want.SetRunningBuilds([]*library.Build{b}) want.SetLastBuildStartedAt(12345) want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) - got := new(library.Worker) + got := new(api.Worker) // setup database db, err := database.NewTest() From 05d088f82dbf0fbfae8030bf62ca28e5ffac8c78 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 1 Nov 2023 13:15:17 -0500 Subject: [PATCH 2/9] fix integration test --- database/integration_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/database/integration_test.go b/database/integration_test.go index 0cc6b1659..5dfb576a1 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -2314,6 +2314,12 @@ func newResources() *Resources { userTwo.SetActive(true) userTwo.SetAdmin(false) + _bPartialOne := new(library.Build) + _bPartialOne.SetID(1) + + _bPartialTwo := new(library.Build) + _bPartialTwo.SetID(2) + workerOne := new(api.Worker) workerOne.SetID(1) workerOne.SetHostname("worker-1.example.com") @@ -2322,7 +2328,7 @@ func newResources() *Resources { workerOne.SetActive(true) workerOne.SetStatus("available") workerOne.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerOne.SetRunningBuilds([]*library.Build{buildOne}) + workerOne.SetRunningBuilds([]*library.Build{_bPartialOne}) workerOne.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerOne.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerOne.SetLastCheckedIn(time.Now().UTC().Unix()) @@ -2336,7 +2342,7 @@ func newResources() *Resources { workerTwo.SetActive(true) workerTwo.SetStatus("available") workerTwo.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerTwo.SetRunningBuilds([]*library.Build{buildTwo}) + workerTwo.SetRunningBuilds([]*library.Build{_bPartialTwo}) workerTwo.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerTwo.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerTwo.SetLastCheckedIn(time.Now().UTC().Unix()) From 8ae0836dc6771ea2197e2b2ce8f6946e5be6a622 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Mon, 4 Mar 2024 12:19:04 -0600 Subject: [PATCH 3/9] appease linter overlord --- api/worker/get.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/worker/get.go b/api/worker/get.go index 99cb17204..1430f052e 100644 --- a/api/worker/get.go +++ b/api/worker/get.go @@ -57,6 +57,7 @@ func GetWorker(c *gin.Context) { }).Infof("reading worker %s", w.GetHostname()) var rBs []*library.Build + for _, b := range w.GetRunningBuilds() { build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) if err != nil { From c4bcf93df5e441945472e3e1294432914e6249a0 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Mon, 4 Mar 2024 12:29:11 -0600 Subject: [PATCH 4/9] fix double param error --- api/worker/list.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/worker/list.go b/api/worker/list.go index a9af3f852..ea07c5f5f 100644 --- a/api/worker/list.go +++ b/api/worker/list.go @@ -39,12 +39,6 @@ import ( // default: 0 // security: // - ApiKeyAuth: [] -// parameters: -// - in: query -// name: links -// description: Include links to builds currently running -// type: boolean -// default: false // responses: // '200': // description: Successfully retrieved the list of workers From 04eb5bcd380ef4e67cc2b301fc230234caec7738 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 5 Mar 2024 15:38:34 -0600 Subject: [PATCH 5/9] remove database types and be consistent with api types imports --- database/types/sanitize.go | 42 ---- database/types/worker.go | 191 ---------------- database/types/worker_test.go | 216 ------------------ database/worker/create.go | 3 +- database/worker/delete.go | 3 +- database/worker/get.go | 3 +- database/worker/get_hostname.go | 3 +- database/worker/interface.go | 14 +- database/worker/list.go | 7 +- database/worker/update.go | 3 +- database/worker/worker.go | 182 +++++++++++++++ mock/server/worker_test.go | 6 +- util/util.go | 33 +++ .../sanitize_test.go => util/util_test.go | 6 +- 14 files changed, 236 insertions(+), 476 deletions(-) delete mode 100644 database/types/sanitize.go delete mode 100644 database/types/worker.go delete mode 100644 database/types/worker_test.go rename database/types/sanitize_test.go => util/util_test.go (95%) diff --git a/database/types/sanitize.go b/database/types/sanitize.go deleted file mode 100644 index c21918194..000000000 --- a/database/types/sanitize.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package types - -import ( - "html" - "net/url" - "strings" - - "github.com/microcosm-cc/bluemonday" -) - -// sanitize is a helper function to verify the provided input -// field does not contain HTML content. If the input field -// does contain HTML, then the function will sanitize and -// potentially remove the HTML if deemed malicious. -func sanitize(field string) string { - // create new HTML input microcosm-cc/bluemonday policy - p := bluemonday.StrictPolicy() - - // create a URL query unescaped string from the field - queryUnescaped, err := url.QueryUnescape(field) - if err != nil { - // overwrite URL query unescaped string with field - queryUnescaped = field - } - - // create an HTML escaped string from the field - htmlEscaped := html.EscapeString(queryUnescaped) - - // create a microcosm-cc/bluemonday escaped string from the field - bluemondayEscaped := p.Sanitize(queryUnescaped) - - // check if the field contains html - if !strings.EqualFold(htmlEscaped, bluemondayEscaped) { - // create new HTML input microcosm-cc/bluemonday policy - return bluemondayEscaped - } - - // return the unmodified field - return field -} diff --git a/database/types/worker.go b/database/types/worker.go deleted file mode 100644 index 15027a10f..000000000 --- a/database/types/worker.go +++ /dev/null @@ -1,191 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package types - -import ( - "database/sql" - "errors" - "fmt" - - "github.com/go-vela/server/api/types" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/lib/pq" -) - -var ( - // ErrEmptyWorkerHost defines the error type when a - // Worker type has an empty Host field provided. - ErrEmptyWorkerHost = errors.New("empty worker hostname provided") - - // ErrEmptyWorkerAddress defines the error type when a - // Worker type has an empty Address field provided. - ErrEmptyWorkerAddress = errors.New("empty worker address provided") - - // ErrExceededRunningBuildIDsLimit defines the error type when a - // Worker type has RunningBuildIDs field provided that exceeds the database limit. - ErrExceededRunningBuildIDsLimit = errors.New("exceeded running build ids limit") -) - -// Worker is the database representation of a worker. -type Worker struct { - ID sql.NullInt64 `sql:"id"` - Hostname sql.NullString `sql:"hostname"` - Address sql.NullString `sql:"address"` - Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` - Active sql.NullBool `sql:"active"` - Status sql.NullString `sql:"status"` - LastStatusUpdateAt sql.NullInt64 `sql:"last_status_update_at"` - RunningBuildIDs pq.StringArray `sql:"running_build_ids" gorm:"type:varchar(500)"` - LastBuildStartedAt sql.NullInt64 `sql:"last_build_started_at"` - LastBuildFinishedAt sql.NullInt64 `sql:"last_build_finished_at"` - LastCheckedIn sql.NullInt64 `sql:"last_checked_in"` - BuildLimit sql.NullInt64 `sql:"build_limit"` -} - -// Nullify ensures the valid flag for -// the sql.Null types are properly set. -// -// When a field within the Build type is the zero -// value for the field, the valid flag is set to -// false causing it to be NULL in the database. -func (w *Worker) Nullify() *Worker { - if w == nil { - return nil - } - - // check if the ID field should be false - if w.ID.Int64 == 0 { - w.ID.Valid = false - } - - // check if the Hostname field should be false - if len(w.Hostname.String) == 0 { - w.Hostname.Valid = false - } - - // check if the Address field should be false - if len(w.Address.String) == 0 { - w.Address.Valid = false - } - - // check if the Status field should be false - if len(w.Status.String) == 0 { - w.Status.Valid = false - } - - // check if the LastStatusUpdateAt field should be false - if w.LastStatusUpdateAt.Int64 == 0 { - w.LastStatusUpdateAt.Valid = false - } - - // check if the LastBuildStartedAt field should be false - if w.LastBuildStartedAt.Int64 == 0 { - w.LastBuildStartedAt.Valid = false - } - - // check if the LastBuildFinishedAt field should be false - if w.LastBuildFinishedAt.Int64 == 0 { - w.LastBuildFinishedAt.Valid = false - } - - // check if the LastCheckedIn field should be false - if w.LastCheckedIn.Int64 == 0 { - w.LastCheckedIn.Valid = false - } - - if w.BuildLimit.Int64 == 0 { - w.BuildLimit.Valid = false - } - - return w -} - -// ToAPI converts the Worker type -// to an API Worker type. -func (w *Worker) ToAPI(builds []*library.Build) *types.Worker { - worker := new(types.Worker) - - worker.SetID(w.ID.Int64) - worker.SetHostname(w.Hostname.String) - worker.SetAddress(w.Address.String) - worker.SetRoutes(w.Routes) - worker.SetActive(w.Active.Bool) - worker.SetStatus(w.Status.String) - worker.SetLastStatusUpdateAt(w.LastStatusUpdateAt.Int64) - worker.SetRunningBuilds(builds) - worker.SetLastBuildStartedAt(w.LastBuildStartedAt.Int64) - worker.SetLastBuildFinishedAt(w.LastBuildFinishedAt.Int64) - worker.SetLastCheckedIn(w.LastCheckedIn.Int64) - worker.SetBuildLimit(w.BuildLimit.Int64) - - return worker -} - -// Validate verifies the necessary fields for -// the Worker type are populated correctly. -func (w *Worker) Validate() error { - // verify the Host field is populated - if len(w.Hostname.String) == 0 { - return ErrEmptyWorkerHost - } - - // verify the Address field is populated - if len(w.Address.String) == 0 { - return ErrEmptyWorkerAddress - } - - // calculate total size of RunningBuildIds - total := 0 - for _, f := range w.RunningBuildIDs { - total += len(f) - } - - // verify the RunningBuildIds field is within the database constraints - // len is to factor in number of comma separators included in the database field, - // removing 1 due to the last item not having an appended comma - if (total + len(w.RunningBuildIDs) - 1) > constants.RunningBuildIDsMaxSize { - return ErrExceededRunningBuildIDsLimit - } - - // ensure that all Worker string fields - // that can be returned as JSON are sanitized - // to avoid unsafe HTML content - w.Hostname = sql.NullString{String: sanitize(w.Hostname.String), Valid: w.Hostname.Valid} - w.Address = sql.NullString{String: sanitize(w.Address.String), Valid: w.Address.Valid} - - // ensure that all Routes are sanitized - // to avoid unsafe HTML content - for i, v := range w.Routes { - w.Routes[i] = sanitize(v) - } - - return nil -} - -// WorkerFromAPI converts the library worker type -// to a database worker type. -func WorkerFromAPI(w *types.Worker) *Worker { - var rBs []string - - for _, b := range w.GetRunningBuilds() { - rBs = append(rBs, fmt.Sprint(b.GetID())) - } - - worker := &Worker{ - ID: sql.NullInt64{Int64: w.GetID(), Valid: true}, - Hostname: sql.NullString{String: w.GetHostname(), Valid: true}, - Address: sql.NullString{String: w.GetAddress(), Valid: true}, - Routes: pq.StringArray(w.GetRoutes()), - Active: sql.NullBool{Bool: w.GetActive(), Valid: true}, - Status: sql.NullString{String: w.GetStatus(), Valid: true}, - LastStatusUpdateAt: sql.NullInt64{Int64: w.GetLastStatusUpdateAt(), Valid: true}, - RunningBuildIDs: pq.StringArray(rBs), - LastBuildStartedAt: sql.NullInt64{Int64: w.GetLastBuildStartedAt(), Valid: true}, - LastBuildFinishedAt: sql.NullInt64{Int64: w.GetLastBuildFinishedAt(), Valid: true}, - LastCheckedIn: sql.NullInt64{Int64: w.GetLastCheckedIn(), Valid: true}, - BuildLimit: sql.NullInt64{Int64: w.GetBuildLimit(), Valid: true}, - } - - return worker.Nullify() -} diff --git a/database/types/worker_test.go b/database/types/worker_test.go deleted file mode 100644 index 5a7a65999..000000000 --- a/database/types/worker_test.go +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package types - -import ( - "database/sql" - "reflect" - "strconv" - "testing" - - "github.com/go-vela/server/api/types" - "github.com/go-vela/types/library" -) - -func TestTypes_Worker_Nullify(t *testing.T) { - // setup types - var w *Worker - - want := &Worker{ - ID: sql.NullInt64{Int64: 0, Valid: false}, - Hostname: sql.NullString{String: "", Valid: false}, - Address: sql.NullString{String: "", Valid: false}, - Active: sql.NullBool{Bool: false, Valid: false}, - Status: sql.NullString{String: "", Valid: false}, - LastStatusUpdateAt: sql.NullInt64{Int64: 0, Valid: false}, - LastBuildStartedAt: sql.NullInt64{Int64: 0, Valid: false}, - LastBuildFinishedAt: sql.NullInt64{Int64: 0, Valid: false}, - LastCheckedIn: sql.NullInt64{Int64: 0, Valid: false}, - BuildLimit: sql.NullInt64{Int64: 0, Valid: false}, - } - - // setup tests - tests := []struct { - repo *Worker - want *Worker - }{ - { - repo: testWorker(), - want: testWorker(), - }, - { - repo: w, - want: nil, - }, - { - repo: new(Worker), - want: want, - }, - } - - // run tests - for _, test := range tests { - got := test.repo.Nullify() - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("Nullify is %v, want %v", got, test.want) - } - } -} - -func TestTypes_Worker_ToAPI(t *testing.T) { - // setup types - b := new(library.Build) - b.SetID(12345) - - builds := []*library.Build{b} - - want := new(types.Worker) - - want.SetID(1) - want.SetHostname("worker_0") - want.SetAddress("http://localhost:8080") - want.SetRoutes([]string{"vela"}) - want.SetActive(true) - want.SetStatus("available") - want.SetLastStatusUpdateAt(1563474077) - want.SetRunningBuilds(builds) - want.SetLastBuildStartedAt(1563474077) - want.SetLastBuildFinishedAt(1563474077) - want.SetLastCheckedIn(1563474077) - want.SetBuildLimit(2) - - // run test - got := testWorker().ToAPI(builds) - - if !reflect.DeepEqual(got, want) { - t.Errorf("ToAPI is %v, want %v", got, want) - } -} - -func TestTypes_Worker_Validate(t *testing.T) { - // setup tests - tests := []struct { - failure bool - worker *Worker - }{ - { - failure: false, - worker: testWorker(), - }, - { // no Hostname set for worker - failure: true, - worker: &Worker{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - Address: sql.NullString{String: "http://localhost:8080", Valid: true}, - Active: sql.NullBool{Bool: true, Valid: true}, - LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, - }, - }, - { // no Address set for worker - failure: true, - worker: &Worker{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - Hostname: sql.NullString{String: "worker_0", Valid: true}, - Active: sql.NullBool{Bool: true, Valid: true}, - LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, - }, - }, - { // invalid RunningBuildIDs set for worker - failure: true, - worker: &Worker{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - Address: sql.NullString{String: "http://localhost:8080", Valid: true}, - Hostname: sql.NullString{String: "worker_0", Valid: true}, - Active: sql.NullBool{Bool: true, Valid: true}, - RunningBuildIDs: exceededRunningBuildIDs(), - LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, - }, - }, - } - - // run tests - for _, test := range tests { - err := test.worker.Validate() - - if test.failure { - if err == nil { - t.Errorf("Validate should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Validate returned err: %v", err) - } - } -} - -func TestTypes_WorkerFromLibrary(t *testing.T) { - // setup types - b := new(library.Build) - b.SetID(12345) - - builds := []*library.Build{b} - - w := new(types.Worker) - - w.SetID(1) - w.SetHostname("worker_0") - w.SetAddress("http://localhost:8080") - w.SetRoutes([]string{"vela"}) - w.SetActive(true) - w.SetStatus("available") - w.SetLastStatusUpdateAt(1563474077) - w.SetRunningBuilds(builds) - w.SetLastBuildStartedAt(1563474077) - w.SetLastBuildFinishedAt(1563474077) - w.SetLastCheckedIn(1563474077) - w.SetBuildLimit(2) - - want := testWorker() - - // run test - got := WorkerFromAPI(w) - - if !reflect.DeepEqual(got, want) { - t.Errorf("WorkerFromLibrary is %v, want %v", got, want) - } -} - -// testWorker is a test helper function to create a Worker -// type with all fields set to a fake value. -func testWorker() *Worker { - return &Worker{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - Hostname: sql.NullString{String: "worker_0", Valid: true}, - Address: sql.NullString{String: "http://localhost:8080", Valid: true}, - Routes: []string{"vela"}, - Active: sql.NullBool{Bool: true, Valid: true}, - Status: sql.NullString{String: "available", Valid: true}, - LastStatusUpdateAt: sql.NullInt64{Int64: 1563474077, Valid: true}, - RunningBuildIDs: []string{"12345"}, - LastBuildStartedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, - LastBuildFinishedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, - LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, - BuildLimit: sql.NullInt64{Int64: 2, Valid: true}, - } -} - -// exceededRunningBuildIDs returns a list of valid running builds that exceed the maximum size. -func exceededRunningBuildIDs() []string { - // initialize empty runningBuildIDs - runningBuildIDs := []string{} - - // add enough build ids to exceed the character limit - for i := 0; i < 50; i++ { - // construct runningBuildID - // use i to adhere to unique runningBuildIDs - runningBuildID := "1234567890-" + strconv.Itoa(i) - - runningBuildIDs = append(runningBuildIDs, runningBuildID) - } - - return runningBuildIDs -} diff --git a/database/worker/create.go b/database/worker/create.go index 7818bd2db..cdc81d2a7 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" ) @@ -20,7 +19,7 @@ func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := types.WorkerFromAPI(w) + worker := WorkerFromAPI(w) // validate the necessary fields are populated // diff --git a/database/worker/delete.go b/database/worker/delete.go index a2eb6f566..9ca26f88f 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" ) @@ -20,7 +19,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := types.WorkerFromAPI(w) + worker := WorkerFromAPI(w) // send query to the database return e.client. diff --git a/database/worker/get.go b/database/worker/get.go index 7aca60b76..7531fd05a 100644 --- a/database/worker/get.go +++ b/database/worker/get.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" ) @@ -15,7 +14,7 @@ func (e *engine) GetWorker(ctx context.Context, id int64) (*api.Worker, error) { e.logger.Tracef("getting worker %d from the database", id) // variable to store query results - w := new(types.Worker) + w := new(Worker) // send query to the database and store result in variable err := e.client. diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go index 4b4b54ad2..f596850af 100644 --- a/database/worker/get_hostname.go +++ b/database/worker/get_hostname.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" ) @@ -18,7 +17,7 @@ func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*ap }).Tracef("getting worker %s from the database", hostname) // variable to store query results - w := new(types.Worker) + w := new(Worker) // send query to the database and store result in variable err := e.client. diff --git a/database/worker/interface.go b/database/worker/interface.go index 8149bc722..8f8d024a1 100644 --- a/database/worker/interface.go +++ b/database/worker/interface.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/server/api/types" + api "github.com/go-vela/server/api/types" ) // WorkerInterface represents the Vela interface for worker @@ -29,15 +29,15 @@ type WorkerInterface interface { // CountWorkers defines a function that gets the count of all workers. CountWorkers(context.Context) (int64, error) // CreateWorker defines a function that creates a new worker. - CreateWorker(context.Context, *types.Worker) (*types.Worker, error) + CreateWorker(context.Context, *api.Worker) (*api.Worker, error) // DeleteWorker defines a function that deletes an existing worker. - DeleteWorker(context.Context, *types.Worker) error + DeleteWorker(context.Context, *api.Worker) error // GetWorker defines a function that gets a worker by ID. - GetWorker(context.Context, int64) (*types.Worker, error) + GetWorker(context.Context, int64) (*api.Worker, error) // GetWorkerForHostname defines a function that gets a worker by hostname. - GetWorkerForHostname(context.Context, string) (*types.Worker, error) + GetWorkerForHostname(context.Context, string) (*api.Worker, error) // ListWorkers defines a function that gets a list of all workers. - ListWorkers(context.Context, string, int64, int64) ([]*types.Worker, error) + ListWorkers(context.Context, string, int64, int64) ([]*api.Worker, error) // UpdateWorker defines a function that updates an existing worker. - UpdateWorker(context.Context, *types.Worker) (*types.Worker, error) + UpdateWorker(context.Context, *api.Worker) (*api.Worker, error) } diff --git a/database/worker/list.go b/database/worker/list.go index 50a5e5233..daa9a7e06 100644 --- a/database/worker/list.go +++ b/database/worker/list.go @@ -8,7 +8,6 @@ import ( "strconv" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" ) @@ -17,7 +16,7 @@ func (e *engine) ListWorkers(ctx context.Context, active string, before, after i e.logger.Trace("listing all workers from the database") // variables to store query results and return value - w := new([]types.Worker) + results := new([]Worker) workers := []*api.Worker{} // build query with checked in constraints @@ -37,13 +36,13 @@ func (e *engine) ListWorkers(ctx context.Context, active string, before, after i } // send query to the database and store result in variable - err := query.Find(&w).Error + err := query.Find(&results).Error if err != nil { return nil, err } // iterate through all query results - for _, worker := range *w { + for _, worker := range *results { // https://golang.org/doc/faq#closures_and_goroutines tmp := worker diff --git a/database/worker/update.go b/database/worker/update.go index c162af9f9..f97275ed9 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" ) @@ -20,7 +19,7 @@ func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := types.WorkerFromAPI(w) + worker := WorkerFromAPI(w) // validate the necessary fields are populated // diff --git a/database/worker/worker.go b/database/worker/worker.go index aaeed7f10..8fcab41d5 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -4,16 +4,35 @@ package worker import ( "context" + "database/sql" + "errors" "fmt" "strconv" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/util" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" + "github.com/lib/pq" "github.com/sirupsen/logrus" "gorm.io/gorm" ) +var ( + // ErrEmptyWorkerHost defines the error type when a + // Worker type has an empty Host field provided. + ErrEmptyWorkerHost = errors.New("empty worker hostname provided") + + // ErrEmptyWorkerAddress defines the error type when a + // Worker type has an empty Address field provided. + ErrEmptyWorkerAddress = errors.New("empty worker address provided") + + // ErrExceededRunningBuildIDsLimit defines the error type when a + // Worker type has RunningBuildIDs field provided that exceeds the database limit. + ErrExceededRunningBuildIDsLimit = errors.New("exceeded running build ids limit") +) + type ( // config represents the settings required to create the engine that implements the WorkerInterface interface. config struct { @@ -38,6 +57,22 @@ type ( // https://pkg.go.dev/github.com/sirupsen/logrus#Entry logger *logrus.Entry } + + // Worker is the database representation of a worker. + Worker struct { + ID sql.NullInt64 `sql:"id"` + Hostname sql.NullString `sql:"hostname"` + Address sql.NullString `sql:"address"` + Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` + Active sql.NullBool `sql:"active"` + Status sql.NullString `sql:"status"` + LastStatusUpdateAt sql.NullInt64 `sql:"last_status_update_at"` + RunningBuildIDs pq.StringArray `sql:"running_build_ids" gorm:"type:varchar(500)"` + LastBuildStartedAt sql.NullInt64 `sql:"last_build_started_at"` + LastBuildFinishedAt sql.NullInt64 `sql:"last_build_finished_at"` + LastCheckedIn sql.NullInt64 `sql:"last_checked_in"` + BuildLimit sql.NullInt64 `sql:"build_limit"` + } ) // New creates and returns a Vela service for integrating with workers in the database. @@ -82,6 +117,153 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Build type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (w *Worker) Nullify() *Worker { + if w == nil { + return nil + } + + // check if the ID field should be false + if w.ID.Int64 == 0 { + w.ID.Valid = false + } + + // check if the Hostname field should be false + if len(w.Hostname.String) == 0 { + w.Hostname.Valid = false + } + + // check if the Address field should be false + if len(w.Address.String) == 0 { + w.Address.Valid = false + } + + // check if the Status field should be false + if len(w.Status.String) == 0 { + w.Status.Valid = false + } + + // check if the LastStatusUpdateAt field should be false + if w.LastStatusUpdateAt.Int64 == 0 { + w.LastStatusUpdateAt.Valid = false + } + + // check if the LastBuildStartedAt field should be false + if w.LastBuildStartedAt.Int64 == 0 { + w.LastBuildStartedAt.Valid = false + } + + // check if the LastBuildFinishedAt field should be false + if w.LastBuildFinishedAt.Int64 == 0 { + w.LastBuildFinishedAt.Valid = false + } + + // check if the LastCheckedIn field should be false + if w.LastCheckedIn.Int64 == 0 { + w.LastCheckedIn.Valid = false + } + + if w.BuildLimit.Int64 == 0 { + w.BuildLimit.Valid = false + } + + return w +} + +// ToAPI converts the Worker type +// to an API Worker type. +func (w *Worker) ToAPI(builds []*library.Build) *api.Worker { + worker := new(api.Worker) + + worker.SetID(w.ID.Int64) + worker.SetHostname(w.Hostname.String) + worker.SetAddress(w.Address.String) + worker.SetRoutes(w.Routes) + worker.SetActive(w.Active.Bool) + worker.SetStatus(w.Status.String) + worker.SetLastStatusUpdateAt(w.LastStatusUpdateAt.Int64) + worker.SetRunningBuilds(builds) + worker.SetLastBuildStartedAt(w.LastBuildStartedAt.Int64) + worker.SetLastBuildFinishedAt(w.LastBuildFinishedAt.Int64) + worker.SetLastCheckedIn(w.LastCheckedIn.Int64) + worker.SetBuildLimit(w.BuildLimit.Int64) + + return worker +} + +// Validate verifies the necessary fields for +// the Worker type are populated correctly. +func (w *Worker) Validate() error { + // verify the Host field is populated + if len(w.Hostname.String) == 0 { + return ErrEmptyWorkerHost + } + + // verify the Address field is populated + if len(w.Address.String) == 0 { + return ErrEmptyWorkerAddress + } + + // calculate total size of RunningBuildIds + total := 0 + for _, f := range w.RunningBuildIDs { + total += len(f) + } + + // verify the RunningBuildIds field is within the database constraints + // len is to factor in number of comma separators included in the database field, + // removing 1 due to the last item not having an appended comma + if (total + len(w.RunningBuildIDs) - 1) > constants.RunningBuildIDsMaxSize { + return ErrExceededRunningBuildIDsLimit + } + + // ensure that all Worker string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + w.Hostname = sql.NullString{String: util.Sanitize(w.Hostname.String), Valid: w.Hostname.Valid} + w.Address = sql.NullString{String: util.Sanitize(w.Address.String), Valid: w.Address.Valid} + + // ensure that all Routes are sanitized + // to avoid unsafe HTML content + for i, v := range w.Routes { + w.Routes[i] = util.Sanitize(v) + } + + return nil +} + +// WorkerFromAPI converts the library worker type +// to a database worker type. +func WorkerFromAPI(w *api.Worker) *Worker { + var rBs []string + + for _, b := range w.GetRunningBuilds() { + rBs = append(rBs, fmt.Sprint(b.GetID())) + } + + worker := &Worker{ + ID: sql.NullInt64{Int64: w.GetID(), Valid: true}, + Hostname: sql.NullString{String: w.GetHostname(), Valid: true}, + Address: sql.NullString{String: w.GetAddress(), Valid: true}, + Routes: pq.StringArray(w.GetRoutes()), + Active: sql.NullBool{Bool: w.GetActive(), Valid: true}, + Status: sql.NullString{String: w.GetStatus(), Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: w.GetLastStatusUpdateAt(), Valid: true}, + RunningBuildIDs: pq.StringArray(rBs), + LastBuildStartedAt: sql.NullInt64{Int64: w.GetLastBuildStartedAt(), Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: w.GetLastBuildFinishedAt(), Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: w.GetLastCheckedIn(), Valid: true}, + BuildLimit: sql.NullInt64{Int64: w.GetBuildLimit(), Valid: true}, + } + + return worker.Nullify() +} + // convertToBuilds is a helper function that generates build objects with ID fields given a list of IDs. func convertToBuilds(ids []string) []*library.Build { // create stripped build objects holding the IDs diff --git a/mock/server/worker_test.go b/mock/server/worker_test.go index 265c6f7f5..a9a46b810 100644 --- a/mock/server/worker_test.go +++ b/mock/server/worker_test.go @@ -7,11 +7,11 @@ import ( "reflect" "testing" - "github.com/go-vela/server/api/types" + api "github.com/go-vela/server/api/types" ) func TestWorker_ActiveWorkerResp(t *testing.T) { - testWorker := types.Worker{} + testWorker := api.Worker{} err := json.Unmarshal([]byte(WorkerResp), &testWorker) if err != nil { @@ -28,7 +28,7 @@ func TestWorker_ActiveWorkerResp(t *testing.T) { } func TestWorker_ListActiveWorkerResp(t *testing.T) { - testWorkers := []types.Worker{} + testWorkers := []api.Worker{} err := json.Unmarshal([]byte(WorkersResp), &testWorkers) if err != nil { diff --git a/util/util.go b/util/util.go index 840dbc04f..5d9ce644f 100644 --- a/util/util.go +++ b/util/util.go @@ -4,9 +4,11 @@ package util import ( "html" + "net/url" "strings" "github.com/go-vela/types/library" + "github.com/microcosm-cc/bluemonday" "github.com/gin-gonic/gin" "github.com/go-vela/types" @@ -128,3 +130,34 @@ func CheckAllowlist(r *library.Repo, allowlist []string) bool { return false } + +// Sanitize is a helper function to verify the provided input +// field does not contain HTML content. If the input field +// does contain HTML, then the function will sanitize and +// potentially remove the HTML if deemed malicious. +func Sanitize(field string) string { + // create new HTML input microcosm-cc/bluemonday policy + p := bluemonday.StrictPolicy() + + // create a URL query unescaped string from the field + queryUnescaped, err := url.QueryUnescape(field) + if err != nil { + // overwrite URL query unescaped string with field + queryUnescaped = field + } + + // create an HTML escaped string from the field + htmlEscaped := html.EscapeString(queryUnescaped) + + // create a microcosm-cc/bluemonday escaped string from the field + bluemondayEscaped := p.Sanitize(queryUnescaped) + + // check if the field contains html + if !strings.EqualFold(htmlEscaped, bluemondayEscaped) { + // create new HTML input microcosm-cc/bluemonday policy + return bluemondayEscaped + } + + // return the unmodified field + return field +} diff --git a/database/types/sanitize_test.go b/util/util_test.go similarity index 95% rename from database/types/sanitize_test.go rename to util/util_test.go index a886e982a..bf4c458f1 100644 --- a/database/types/sanitize_test.go +++ b/util/util_test.go @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 -package types +package util import ( "testing" ) -func TestTypes_Sanitize(t *testing.T) { +func TestUtil_Sanitize(t *testing.T) { // setup tests tests := []struct { name string @@ -73,7 +73,7 @@ func TestTypes_Sanitize(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := sanitize(test.value) + got := Sanitize(test.value) if got != test.want { t.Errorf("sanitize is %v, want %v", got, test.want) From 167cb5e8dece98adee64e31d90f38ec50c092212 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 5 Mar 2024 15:58:11 -0600 Subject: [PATCH 6/9] allow for exclaim --- .github/workflows/validate-pr-title.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-pr-title.yml b/.github/workflows/validate-pr-title.yml index 462f9be92..ec3674f79 100644 --- a/.github/workflows/validate-pr-title.yml +++ b/.github/workflows/validate-pr-title.yml @@ -14,4 +14,4 @@ jobs: steps: - name: validate title run: | - echo "${{ github.event.pull_request.title }}" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|):\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) + echo "${{ github.event.pull_request.title }}" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|)!?:\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) From 082a404ee6a1b53a1f54903f1f8a10bed88448bb Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 6 Mar 2024 10:47:10 -0600 Subject: [PATCH 7/9] remove library references and remove stutter --- api/types/worker.go | 2 +- database/worker/create.go | 2 +- database/worker/delete.go | 2 +- database/worker/update.go | 2 +- database/worker/worker.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/types/worker.go b/api/types/worker.go index 78f86cf1c..db44ed8f6 100644 --- a/api/types/worker.go +++ b/api/types/worker.go @@ -8,7 +8,7 @@ import ( "github.com/go-vela/types/library" ) -// Worker is the library representation of a worker. +// Worker is the API representation of a worker. // // swagger:model Worker type Worker struct { diff --git a/database/worker/create.go b/database/worker/create.go index cdc81d2a7..131bb393f 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -19,7 +19,7 @@ func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := WorkerFromAPI(w) + worker := FromAPI(w) // validate the necessary fields are populated // diff --git a/database/worker/delete.go b/database/worker/delete.go index 9ca26f88f..ee391567e 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -19,7 +19,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := WorkerFromAPI(w) + worker := FromAPI(w) // send query to the database return e.client. diff --git a/database/worker/update.go b/database/worker/update.go index f97275ed9..0349ffe85 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -19,7 +19,7 @@ func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := WorkerFromAPI(w) + worker := FromAPI(w) // validate the necessary fields are populated // diff --git a/database/worker/worker.go b/database/worker/worker.go index 8fcab41d5..d12055ded 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -237,9 +237,9 @@ func (w *Worker) Validate() error { return nil } -// WorkerFromAPI converts the library worker type +// FromAPI converts the API worker type // to a database worker type. -func WorkerFromAPI(w *api.Worker) *Worker { +func FromAPI(w *api.Worker) *Worker { var rBs []string for _, b := range w.GetRunningBuilds() { From 6dd718a0368260ebc3c79f8cbcff64fd19975a1e Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 18 Mar 2024 14:45:21 -0500 Subject: [PATCH 8/9] fix: use concrete list var --- api/types/worker.go | 28 ++++++++++++++-------------- api/worker/get.go | 2 +- api/worker/list.go | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api/types/worker.go b/api/types/worker.go index db44ed8f6..4c06a0e69 100644 --- a/api/types/worker.go +++ b/api/types/worker.go @@ -12,18 +12,18 @@ import ( // // swagger:model Worker type Worker struct { - ID *int64 `json:"id,omitempty"` - Hostname *string `json:"hostname,omitempty"` - Address *string `json:"address,omitempty"` - Routes *[]string `json:"routes,omitempty"` - Active *bool `json:"active,omitempty"` - Status *string `json:"status,omitempty"` - LastStatusUpdateAt *int64 `json:"last_status_update_at,omitempty"` - RunningBuilds []*library.Build `json:"running_builds,omitempty"` - LastBuildStartedAt *int64 `json:"last_build_started_at,omitempty"` - LastBuildFinishedAt *int64 `json:"last_build_finished_at,omitempty"` - LastCheckedIn *int64 `json:"last_checked_in,omitempty"` - BuildLimit *int64 `json:"build_limit,omitempty"` + ID *int64 `json:"id,omitempty"` + Hostname *string `json:"hostname,omitempty"` + Address *string `json:"address,omitempty"` + Routes *[]string `json:"routes,omitempty"` + Active *bool `json:"active,omitempty"` + Status *string `json:"status,omitempty"` + LastStatusUpdateAt *int64 `json:"last_status_update_at,omitempty"` + RunningBuilds *[]*library.Build `json:"running_builds,omitempty"` + LastBuildStartedAt *int64 `json:"last_build_started_at,omitempty"` + LastBuildFinishedAt *int64 `json:"last_build_finished_at,omitempty"` + LastCheckedIn *int64 `json:"last_checked_in,omitempty"` + BuildLimit *int64 `json:"build_limit,omitempty"` } // GetID returns the ID field. @@ -127,7 +127,7 @@ func (w *Worker) GetRunningBuilds() []*library.Build { return []*library.Build{} } - return w.RunningBuilds + return *w.RunningBuilds } // GetLastBuildStartedAt returns the LastBuildStartedAt field. @@ -283,7 +283,7 @@ func (w *Worker) SetRunningBuilds(builds []*library.Build) { return } - w.RunningBuilds = builds + w.RunningBuilds = &builds } // SetLastBuildStartedAt sets the LastBuildStartedAt field. diff --git a/api/worker/get.go b/api/worker/get.go index 1430f052e..b7bebd149 100644 --- a/api/worker/get.go +++ b/api/worker/get.go @@ -56,7 +56,7 @@ func GetWorker(c *gin.Context) { "worker": w.GetHostname(), }).Infof("reading worker %s", w.GetHostname()) - var rBs []*library.Build + rBs := []*library.Build{} for _, b := range w.GetRunningBuilds() { build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) diff --git a/api/worker/list.go b/api/worker/list.go index ea07c5f5f..82673eccf 100644 --- a/api/worker/list.go +++ b/api/worker/list.go @@ -97,7 +97,7 @@ func ListWorkers(c *gin.Context) { } for _, w := range workers { - var rBs []*library.Build + rBs := []*library.Build{} for _, b := range w.GetRunningBuilds() { build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) From 7cbf3870ddde9a582335a2341672fe29e8b15e76 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 18 Mar 2024 15:12:06 -0500 Subject: [PATCH 9/9] fix: test pointer --- database/worker/worker_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index a7f47428d..8c6896eb7 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -221,7 +221,7 @@ func testWorker() *api.Worker { Active: new(bool), Status: new(string), LastStatusUpdateAt: new(int64), - RunningBuilds: []*library.Build{b}, + RunningBuilds: &[]*library.Build{b}, LastBuildStartedAt: new(int64), LastBuildFinishedAt: new(int64), LastCheckedIn: new(int64),