diff --git a/api/executor.go b/api/executor.go index 84907c93..c4e1cc12 100644 --- a/api/executor.go +++ b/api/executor.go @@ -10,9 +10,9 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/types" "github.com/go-vela/types/library" + "github.com/go-vela/worker/executor" exec "github.com/go-vela/worker/router/middleware/executor" ) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 2d40428f..b0986742 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -8,8 +8,8 @@ import ( "context" "time" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/worker/executor" "github.com/go-vela/worker/version" "github.com/sirupsen/logrus" @@ -45,7 +45,7 @@ func (w *Worker) exec(index int) error { // setup the executor // - // https://godoc.org/github.com/go-vela/pkg-executor/executor#New + // https://godoc.org/github.com/go-vela/worker/executor#New _executor, err := executor.New(&executor.Setup{ Driver: w.Config.Executor.Driver, Client: w.VelaClient, diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index ee282f2b..e47d0b30 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -7,9 +7,9 @@ package main import ( "time" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/pkg-queue/queue" "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/worker/executor" "github.com/urfave/cli/v2" ) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 0cae9f21..6574c62c 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -10,9 +10,9 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/pkg-queue/queue" "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/worker/executor" "github.com/sirupsen/logrus" diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index dae07639..d05af002 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -8,10 +8,10 @@ import ( "net/url" "time" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/pkg-queue/queue" "github.com/go-vela/pkg-runtime/runtime" "github.com/go-vela/sdk-go/vela" + "github.com/go-vela/worker/executor" ) type ( diff --git a/executor/context.go b/executor/context.go new file mode 100644 index 00000000..37750533 --- /dev/null +++ b/executor/context.go @@ -0,0 +1,67 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "context" + + "github.com/gin-gonic/gin" +) + +// key defines the key type for storing +// the executor Engine in the context. +const key = "executor" + +// FromContext retrieves the executor Engine from the context.Context. +func FromContext(c context.Context) Engine { + // get executor value from context.Context + v := c.Value(key) + if v == nil { + return nil + } + + // cast executor value to expected Engine type + e, ok := v.(Engine) + if !ok { + return nil + } + + return e +} + +// FromGinContext retrieves the executor Engine from the gin.Context. +func FromGinContext(c *gin.Context) Engine { + // get executor value from gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Get + v, ok := c.Get(key) + if !ok { + return nil + } + + // cast executor value to expected Engine type + e, ok := v.(Engine) + if !ok { + return nil + } + + return e +} + +// WithContext inserts the executor Engine into the context.Context. +func WithContext(c context.Context, e Engine) context.Context { + // set the executor Engine in the context.Context + // + // nolint: golint,staticcheck // ignore using string with context value + return context.WithValue(c, key, e) +} + +// WithGinContext inserts the executor Engine into the gin.Context. +func WithGinContext(c *gin.Context, e Engine) { + // set the executor Engine in the gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Set + c.Set(key, e) +} diff --git a/executor/context_test.go b/executor/context_test.go new file mode 100644 index 00000000..cb98e237 --- /dev/null +++ b/executor/context_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "context" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/worker/executor/linux" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" +) + +func TestExecutor_FromContext(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _engine, err := linux.New( + linux.WithBuild(_build), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + // setup tests + tests := []struct { + context context.Context + want Engine + }{ + { + // nolint: golint,staticcheck // ignore using string with context value + context: context.WithValue(context.Background(), key, _engine), + want: _engine, + }, + { + context: context.Background(), + want: nil, + }, + { + // nolint: golint,staticcheck // ignore using string with context value + context: context.WithValue(context.Background(), key, "foo"), + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := FromContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_FromGinContext(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _engine, err := linux.New( + linux.WithBuild(_build), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + // setup tests + tests := []struct { + context *gin.Context + value interface{} + want Engine + }{ + { + context: new(gin.Context), + value: _engine, + want: _engine, + }, + { + context: new(gin.Context), + value: nil, + want: nil, + }, + { + context: new(gin.Context), + value: "foo", + want: nil, + }, + } + + // run tests + for _, test := range tests { + if test.value != nil { + test.context.Set(key, test.value) + } + + got := FromGinContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromGinContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_WithContext(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _engine, err := linux.New( + linux.WithBuild(_build), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + // nolint: golint,staticcheck // ignore using string with context value + want := context.WithValue(context.Background(), key, _engine) + + // run test + got := WithContext(context.Background(), _engine) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithContext is %v, want %v", got, want) + } +} + +func TestExecutor_WithGinContext(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _engine, err := linux.New( + linux.WithBuild(_build), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + want := new(gin.Context) + want.Set(key, _engine) + + // run test + got := new(gin.Context) + WithGinContext(got, _engine) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithGinContext is %v, want %v", got, want) + } +} diff --git a/executor/doc.go b/executor/doc.go new file mode 100644 index 00000000..b17464e1 --- /dev/null +++ b/executor/doc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package executor provides the ability for Vela to +// integrate with different supported operating +// systems. +// +// Usage: +// +// import "github.com/go-vela/worker/executor" +package executor diff --git a/executor/engine.go b/executor/engine.go new file mode 100644 index 00000000..0fbd4c88 --- /dev/null +++ b/executor/engine.go @@ -0,0 +1,110 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "context" + "sync" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// Engine represents the interface for Vela integrating +// with the different supported operating systems. +type Engine interface { + + // Engine Interface Functions + + // Driver defines a function that outputs + // the configured executor driver. + Driver() string + + // API interface functions + + // GetBuild defines a function for the API + // that gets the current build in execution. + GetBuild() (*library.Build, error) + // GetPipeline defines a function for the API + // that gets the current pipeline in execution. + GetPipeline() (*pipeline.Build, error) + // GetRepo defines a function for the API + // that gets the current repo in execution. + GetRepo() (*library.Repo, error) + // CancelBuild defines a function for the API + // that Cancels the current build in execution. + CancelBuild() (*library.Build, error) + + // Build Engine interface functions + + // CreateBuild defines a function that + // configures the build for execution. + CreateBuild(context.Context) error + // PlanBuild defines a function that + // handles the resource initialization process + // for the build. + PlanBuild(context.Context) error + // AssembleBuild defines a function that + // prepares the containers within a build + // for execution. + AssembleBuild(context.Context) error + // ExecBuild defines a function that + // runs a pipeline for a build. + ExecBuild(context.Context) error + // DestroyBuild defines a function that + // cleans up the build after execution. + DestroyBuild(context.Context) error + + // Service Engine Interface Functions + + // CreateService defines a function that + // configures the service for execution. + CreateService(context.Context, *pipeline.Container) error + // PlanService defines a function that + // prepares the service for execution. + PlanService(context.Context, *pipeline.Container) error + // ExecService defines a function that + // runs a service. + ExecService(context.Context, *pipeline.Container) error + // StreamService defines a function that + // tails the output for a service. + StreamService(context.Context, *pipeline.Container) error + // DestroyService defines a function that + // cleans up the service after execution. + DestroyService(context.Context, *pipeline.Container) error + + // Stage Engine Interface Functions + + // CreateStage defines a function that + // configures the stage for execution. + CreateStage(context.Context, *pipeline.Stage) error + // PlanStage defines a function that + // prepares the stage for execution. + PlanStage(context.Context, *pipeline.Stage, *sync.Map) error + // ExecStage defines a function that + // runs a stage. + ExecStage(context.Context, *pipeline.Stage, *sync.Map) error + // DestroyStage defines a function that + // cleans up the stage after execution. + DestroyStage(context.Context, *pipeline.Stage) error + + // Step Engine Interface Functions + + // CreateStep defines a function that + // configures the step for execution. + CreateStep(context.Context, *pipeline.Container) error + // PlanStep defines a function that + // prepares the step for execution. + PlanStep(context.Context, *pipeline.Container) error + // ExecStep defines a function that + // runs a step. + ExecStep(context.Context, *pipeline.Container) error + // StreamStep defines a function that + // tails the output for a step. + StreamStep(context.Context, *pipeline.Container) error + // DestroyStep defines a function that + // cleans up the step after execution. + DestroyStep(context.Context, *pipeline.Container) error +} diff --git a/executor/executor.go b/executor/executor.go new file mode 100644 index 00000000..1fd80458 --- /dev/null +++ b/executor/executor.go @@ -0,0 +1,60 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "fmt" + + "github.com/go-vela/types/constants" + + "github.com/sirupsen/logrus" +) + +// nolint: godot // ignore period at end for comment ending in a list +// +// New creates and returns a Vela engine capable of +// integrating with the configured executor. +// +// Currently the following executors are supported: +// +// * linux +// * local +func New(s *Setup) (Engine, error) { + // validate the setup being provided + // + // https://pkg.go.dev/github.com/go-vela/worker/executor?tab=doc#Setup.Validate + err := s.Validate() + if err != nil { + return nil, err + } + + logrus.Debug("creating executor engine from setup") + // process the executor driver being provided + switch s.Driver { + case constants.DriverDarwin: + // handle the Darwin executor driver being provided + // + // https://pkg.go.dev/github.com/go-vela/worker/executor?tab=doc#Setup.Darwin + return s.Darwin() + case constants.DriverLinux: + // handle the Linux executor driver being provided + // + // https://pkg.go.dev/github.com/go-vela/worker/executor?tab=doc#Setup.Linux + return s.Linux() + case constants.DriverLocal: + // handle the Local executor driver being provided + // + // https://pkg.go.dev/github.com/go-vela/worker/executor?tab=doc#Setup.Local + return s.Local() + case constants.DriverWindows: + // handle the Windows executor driver being provided + // + // https://pkg.go.dev/github.com/go-vela/worker/executor?tab=doc#Setup.Windows + return s.Windows() + default: + // handle an invalid executor driver being provided + return nil, fmt.Errorf("invalid executor driver provided: %s", s.Driver) + } +} diff --git a/executor/executor_test.go b/executor/executor_test.go new file mode 100644 index 00000000..ad55ff88 --- /dev/null +++ b/executor/executor_test.go @@ -0,0 +1,279 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/worker/executor/linux" + "github.com/go-vela/worker/executor/local" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestExecutor_New(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _linux, err := linux.New( + linux.WithBuild(_build), + linux.WithHostname("localhost"), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + linux.WithVersion("v1.0.0"), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + _local, err := local.New( + local.WithBuild(_build), + local.WithHostname("localhost"), + local.WithPipeline(_pipeline), + local.WithRepo(_repo), + local.WithRuntime(_runtime), + local.WithUser(_user), + local.WithVelaClient(_client), + local.WithVersion("v1.0.0"), + ) + if err != nil { + t.Errorf("unable to create local engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + setup *Setup + want Engine + }{ + { + failure: true, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverDarwin, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: nil, + }, + { + failure: false, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: _linux, + }, + { + failure: false, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: "local", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: _local, + }, + { + failure: true, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverWindows, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: nil, + }, + { + failure: true, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: "invalid", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: nil, + }, + { + failure: true, + setup: &Setup{ + Build: _build, + Client: _client, + Driver: "", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + }, + want: nil, + }, + } + + // run tests + for _, test := range tests { + got, err := New(test.setup) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New is %v, want %v", got, test.want) + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New is %v, want %v", got, test.want) + } + } +} + +// setup global variables used for testing. +var ( + _build = &library.Build{ + ID: vela.Int64(1), + Number: vela.Int(1), + Parent: vela.Int(1), + Event: vela.String("push"), + Status: vela.String("success"), + Error: vela.String(""), + Enqueued: vela.Int64(1563474077), + Created: vela.Int64(1563474076), + Started: vela.Int64(1563474077), + Finished: vela.Int64(0), + Deploy: vela.String(""), + Clone: vela.String("https://github.com/github/octocat.git"), + Source: vela.String("https://github.com/github/octocat/abcdefghi123456789"), + Title: vela.String("push received from https://github.com/github/octocat"), + Message: vela.String("First commit..."), + Commit: vela.String("48afb5bdc41ad69bf22588491333f7cf71135163"), + Sender: vela.String("OctoKitty"), + Author: vela.String("OctoKitty"), + Branch: vela.String("master"), + Ref: vela.String("refs/heads/master"), + BaseRef: vela.String(""), + Host: vela.String("example.company.com"), + Runtime: vela.String("docker"), + Distribution: vela.String("linux"), + } + + _pipeline = &pipeline.Build{ + Version: "1", + ID: "github_octocat_1", + Steps: pipeline.ContainerSlice{ + { + ID: "step_github_octocat_1_init", + Directory: "/home/github/octocat", + Image: "#init", + Name: "init", + Number: 1, + Pull: "always", + }, + { + ID: "step_github_octocat_1_clone", + Directory: "/home/github/octocat", + Image: "target/vela-git:v0.3.0", + Name: "clone", + Number: 2, + Pull: "always", + }, + { + ID: "step_github_octocat_1_echo", + Commands: []string{"echo hello"}, + Directory: "/home/github/octocat", + Image: "alpine:latest", + Name: "echo", + Number: 3, + Pull: "always", + }, + }, + } + + _repo = &library.Repo{ + ID: vela.Int64(1), + Org: vela.String("github"), + Name: vela.String("octocat"), + FullName: vela.String("github/octocat"), + Link: vela.String("https://github.com/github/octocat"), + Clone: vela.String("https://github.com/github/octocat.git"), + Branch: vela.String("master"), + Timeout: vela.Int64(60), + Visibility: vela.String("public"), + Private: vela.Bool(false), + Trusted: vela.Bool(false), + Active: vela.Bool(true), + AllowPull: vela.Bool(false), + AllowPush: vela.Bool(true), + AllowDeploy: vela.Bool(false), + AllowTag: vela.Bool(false), + } + + _user = &library.User{ + ID: vela.Int64(1), + Name: vela.String("octocat"), + Token: vela.String("superSecretToken"), + Hash: vela.String("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy"), + Favorites: vela.Strings([]string{"github/octocat"}), + Active: vela.Bool(true), + Admin: vela.Bool(false), + } +) diff --git a/executor/flags.go b/executor/flags.go new file mode 100644 index 00000000..f29e4549 --- /dev/null +++ b/executor/flags.go @@ -0,0 +1,45 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "github.com/go-vela/types/constants" + + "github.com/urfave/cli/v2" +) + +// Flags represents all supported command line +// interface (CLI) flags for the executor. +// +// https://pkg.go.dev/github.com/urfave/cli?tab=doc#Flag +var Flags = []cli.Flag{ + + // Logging Flags + + &cli.StringFlag{ + EnvVars: []string{"VELA_LOG_FORMAT", "EXECUTOR_LOG_FORMAT"}, + FilePath: "/vela/executor/log_format", + Name: "executor.log.format", + Usage: "format of logs to output", + Value: "json", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_LOG_LEVEL", "EXECUTOR_LOG_LEVEL"}, + FilePath: "/vela/executor/log_level", + Name: "executor.log.level", + Usage: "level of logs to output", + Value: "info", + }, + + // Executor Flags + + &cli.StringFlag{ + EnvVars: []string{"VELA_EXECUTOR_DRIVER", "EXECUTOR_DRIVER"}, + FilePath: "/vela/executor/driver", + Name: "executor.driver", + Usage: "driver to be used for the executor", + Value: constants.DriverLinux, + }, +} diff --git a/executor/linux/api.go b/executor/linux/api.go new file mode 100644 index 00000000..8c09a521 --- /dev/null +++ b/executor/linux/api.go @@ -0,0 +1,199 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "fmt" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/service" + "github.com/go-vela/worker/internal/step" +) + +// GetBuild gets the current build in execution. +func (c *client) GetBuild() (*library.Build, error) { + // check if the build resource is available + if c.build == nil { + return nil, fmt.Errorf("build resource not found") + } + + return c.build, nil +} + +// GetPipeline gets the current pipeline in execution. +func (c *client) GetPipeline() (*pipeline.Build, error) { + // check if the pipeline resource is available + if c.pipeline == nil { + return nil, fmt.Errorf("pipeline resource not found") + } + + return c.pipeline, nil +} + +// GetRepo gets the current repo in execution. +func (c *client) GetRepo() (*library.Repo, error) { + // check if the repo resource is available + if c.repo == nil { + return nil, fmt.Errorf("repo resource not found") + } + + return c.repo, nil +} + +// CancelBuild cancels the current build in execution. +// nolint: funlen // process of going through steps/services/stages is verbose and could be funcitonalized +func (c *client) CancelBuild() (*library.Build, error) { + // get the current build from the client + b, err := c.GetBuild() + if err != nil { + return nil, err + } + + // set the build status to canceled + b.SetStatus(constants.StatusCanceled) + + // get the current pipeline from the client + pipeline, err := c.GetPipeline() + if err != nil { + return nil, err + } + + // cancel non successful services + // nolint: dupl // false positive, steps/services are different + for _, _service := range pipeline.Services { + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + s, err := service.Load(_service, &c.services) + if err != nil { + // create the library service object + s = new(library.Service) + s.SetName(_service.Name) + s.SetNumber(_service.Number) + s.SetImage(_service.Image) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if service state was not terminal, set it as canceled + switch s.GetStatus() { + // service is in a error state + case constants.StatusError: + break + // service is in a failure state + case constants.StatusFailure: + break + // service is in a killed state + case constants.StatusKilled: + break + // service is in a success state + case constants.StatusSuccess: + break + default: + // update the service with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a service to a map + c.services.Store(_service.ID, s) + } + } + + // cancel non successful steps + // nolint: dupl // false positive, steps/services are different + for _, _step := range pipeline.Steps { + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + s, err := step.Load(_step, &c.steps) + if err != nil { + // create the library step object + s = new(library.Step) + s.SetName(_step.Name) + s.SetNumber(_step.Number) + s.SetImage(_step.Image) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if step state was not terminal, set it as canceled + switch s.GetStatus() { + // step is in a error state + case constants.StatusError: + break + // step is in a failure state + case constants.StatusFailure: + break + // step is in a killed state + case constants.StatusKilled: + break + // step is in a success state + case constants.StatusSuccess: + break + default: + // update the step with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a step to a map + c.steps.Store(_step.ID, s) + } + } + + // cancel non successful stages + for _, _stage := range pipeline.Stages { + // cancel non successful steps for that stage + for _, _step := range _stage.Steps { + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + s, err := step.Load(_step, &c.steps) + if err != nil { + // create the library step object + s = new(library.Step) + s.SetName(_step.Name) + s.SetNumber(_step.Number) + s.SetImage(_step.Image) + s.SetStage(_stage.Name) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if stage state was not terminal, set it as canceled + switch s.GetStatus() { + // stage is in a error state + case constants.StatusError: + break + // stage is in a failure state + case constants.StatusFailure: + break + // stage is in a killed state + case constants.StatusKilled: + break + // stage is in a success state + case constants.StatusSuccess: + break + default: + // update the step with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a step to a map + c.steps.Store(_step.ID, s) + } + } + } + + err = c.DestroyBuild(context.Background()) + if err != nil { + c.logger.Errorf("unable to destroy build: %v", err) + } + + return b, nil +} diff --git a/executor/linux/api_test.go b/executor/linux/api_test.go new file mode 100644 index 00000000..a60d0f11 --- /dev/null +++ b/executor/linux/api_test.go @@ -0,0 +1,154 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "reflect" + "testing" +) + +func TestLinux_GetBuild(t *testing.T) { + // setup types + _build := testBuild() + + _engine, err := New( + WithBuild(_build), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetBuild() + + if test.failure { + if err == nil { + t.Errorf("GetBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetBuild returned err: %v", err) + } + + if !reflect.DeepEqual(got, _build) { + t.Errorf("GetBuild is %v, want %v", got, _build) + } + } +} + +func TestLinux_GetPipeline(t *testing.T) { + // setup types + _steps := testSteps() + + _engine, err := New( + WithPipeline(_steps), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetPipeline() + + if test.failure { + if err == nil { + t.Errorf("GetPipeline should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetPipeline returned err: %v", err) + } + + if !reflect.DeepEqual(got, _steps) { + t.Errorf("GetPipeline is %v, want %v", got, _steps) + } + } +} + +func TestLinux_GetRepo(t *testing.T) { + // setup types + _repo := testRepo() + + _engine, err := New( + WithRepo(_repo), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetRepo() + + if test.failure { + if err == nil { + t.Errorf("GetRepo should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetRepo returned err: %v", err) + } + + if !reflect.DeepEqual(got, _repo) { + t.Errorf("GetRepo is %v, want %v", got, _repo) + } + } +} diff --git a/executor/linux/build.go b/executor/linux/build.go new file mode 100644 index 00000000..9cc36d87 --- /dev/null +++ b/executor/linux/build.go @@ -0,0 +1,581 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/go-vela/types/constants" + "github.com/go-vela/worker/internal/build" + "github.com/go-vela/worker/internal/step" +) + +// CreateBuild configures the build for execution. +func (c *client) CreateBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, c.Vela, c.err, c.logger, c.repo) }() + + // update the build fields + c.build.SetStatus(constants.StatusRunning) + c.build.SetStarted(time.Now().UTC().Unix()) + c.build.SetHost(c.Hostname) + c.build.SetDistribution(c.Driver()) + c.build.SetRuntime(c.Runtime.Driver()) + + c.logger.Info("uploading build state") + // send API call to update the build + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#BuildService.Update + c.build, _, c.err = c.Vela.Build.Update(c.repo.GetOrg(), c.repo.GetName(), c.build) + if c.err != nil { + return fmt.Errorf("unable to upload build state: %v", c.err) + } + + // setup the runtime build + c.err = c.Runtime.SetupBuild(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to setup build %s: %w", c.pipeline.ID, c.err) + } + + // load the init step from the pipeline + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadInit + c.init, c.err = step.LoadInit(c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to load init step from pipeline: %w", c.err) + } + + c.logger.Infof("creating %s step", c.init.Name) + // create the step + c.err = c.CreateStep(ctx, c.init) + if c.err != nil { + return fmt.Errorf("unable to create %s step: %w", c.init.Name, c.err) + } + + c.logger.Infof("planning %s step", c.init.Name) + // plan the step + c.err = c.PlanStep(ctx, c.init) + if c.err != nil { + return fmt.Errorf("unable to plan %s step: %w", c.init.Name, c.err) + } + + return c.err +} + +// PlanBuild prepares the build for execution. +// +// nolint: funlen // ignore function length due to comments and logging messages +func (c *client) PlanBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, c.Vela, c.err, c.logger, c.repo) }() + + // load the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _init, err := step.Load(c.init, &c.steps) + if err != nil { + return err + } + + // load the logs for the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadLogs + _log, err := step.LoadLogs(c.init, &c.stepLogs) + if err != nil { + return err + } + + // defer taking a snapshot of the init step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#SnapshotInit + defer func() { step.SnapshotInit(c.init, c.build, c.Vela, c.logger, c.repo, _init, _log) }() + + c.logger.Info("creating network") + // create the runtime network for the pipeline + c.err = c.Runtime.CreateNetwork(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to create network: %w", c.err) + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Inspecting runtime network...\n")) + + // inspect the runtime network for the pipeline + network, err := c.Runtime.InspectNetwork(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect network: %w", err) + } + + // update the init log with network information + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(network) + + c.logger.Info("creating volume") + // create the runtime volume for the pipeline + c.err = c.Runtime.CreateVolume(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to create volume: %w", c.err) + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Inspecting runtime volume...\n")) + + // inspect the runtime volume for the pipeline + volume, err := c.Runtime.InspectVolume(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect volume: %w", err) + } + + // update the init log with volume information + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(volume) + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Pulling secrets...\n")) + + // iterate through each secret provided in the pipeline + for _, secret := range c.pipeline.Secrets { + // ignore pulling secrets coming from plugins + if !secret.Origin.Empty() { + continue + } + + c.logger.Infof("pulling %s %s secret %s", secret.Engine, secret.Type, secret.Name) + + s, err := c.secret.pull(secret) + if err != nil { + c.err = err + return fmt.Errorf("unable to pull secrets: %w", err) + } + + _log.AppendData([]byte( + fmt.Sprintf("$ vela view secret --secret.engine %s --secret.type %s --org %s --repo %s --name %s \n", + secret.Engine, secret.Type, s.GetOrg(), s.GetRepo(), s.GetName()))) + + sRaw, err := json.MarshalIndent(s.Sanitize(), "", " ") + if err != nil { + c.err = err + return fmt.Errorf("unable to decode secret: %w", err) + } + + _log.AppendData(append(sRaw, "\n"...)) + + // add secret to the map + c.Secrets[secret.Name] = s + } + + return nil +} + +// AssembleBuild prepares the containers within a build for execution. +// +// nolint: funlen // ignore function length due to comments and logging messages +func (c *client) AssembleBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, c.Vela, c.err, c.logger, c.repo) }() + + // load the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _init, err := step.Load(c.init, &c.steps) + if err != nil { + return err + } + + // load the logs for the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadLogs + _log, err := step.LoadLogs(c.init, &c.stepLogs) + if err != nil { + return err + } + + // defer an upload of the init step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload + defer func() { step.Upload(c.init, c.build, c.Vela, c.logger, c.repo, _init) }() + + defer func() { + c.logger.Infof("uploading %s step logs", c.init.Name) + // send API call to update the logs for the step + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.UpdateStep + _log, _, err = c.Vela.Log.UpdateStep(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), c.init.Number, _log) + if err != nil { + c.logger.Errorf("unable to upload %s logs: %v", c.init.Name, err) + } + }() + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Pulling service images...\n")) + + // create the services for the pipeline + for _, s := range c.pipeline.Services { + // TODO: remove this; but we need it for tests + s.Detach = true + + c.logger.Infof("creating %s service", s.Name) + // create the service + c.err = c.CreateService(ctx, s) + if c.err != nil { + return fmt.Errorf("unable to create %s service: %w", s.Name, c.err) + } + + c.logger.Infof("inspecting %s service", s.Name) + // inspect the service image + image, err := c.Runtime.InspectImage(ctx, s) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect %s service: %w", s.Name, err) + } + + // update the init log with service image info + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(image) + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Pulling stage images...\n")) + + // create the stages for the pipeline + for _, s := range c.pipeline.Stages { + // TODO: remove hardcoded reference + // + // nolint: goconst // ignore making a constant for now + if s.Name == "init" { + continue + } + + c.logger.Infof("creating %s stage", s.Name) + // create the stage + c.err = c.CreateStage(ctx, s) + if c.err != nil { + return fmt.Errorf("unable to create %s stage: %w", s.Name, c.err) + } + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Pulling step images...\n")) + + // create the steps for the pipeline + for _, s := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if s.Name == "init" { + continue + } + + c.logger.Infof("creating %s step", s.Name) + // create the step + c.err = c.CreateStep(ctx, s) + if c.err != nil { + return fmt.Errorf("unable to create %s step: %w", s.Name, c.err) + } + + c.logger.Infof("inspecting %s step", s.Name) + // inspect the step image + image, err := c.Runtime.InspectImage(ctx, s) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect %s step: %w", s.Name, c.err) + } + + // update the init log with step image info + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(image) + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Pulling secret images...\n")) + + // create the secrets for the pipeline + for _, s := range c.pipeline.Secrets { + // skip over non-plugin secrets + if s.Origin.Empty() { + continue + } + + c.logger.Infof("creating %s secret", s.Origin.Name) + // create the service + c.err = c.secret.create(ctx, s.Origin) + if c.err != nil { + return fmt.Errorf("unable to create %s secret: %w", s.Origin.Name, c.err) + } + + c.logger.Infof("inspecting %s secret", s.Origin.Name) + // inspect the service image + image, err := c.Runtime.InspectImage(ctx, s.Origin) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect %s secret: %w", s.Origin.Name, err) + } + + // update the init log with secret image info + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(image) + } + + // inspect the runtime build (eg a kubernetes pod) for the pipeline + buildOutput, err := c.Runtime.InspectBuild(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect build: %w", err) + } + + if len(buildOutput) > 0 { + // update the init log with progress + // (an empty value allows the runtime to opt out of providing this) + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(buildOutput) + } + + // assemble runtime build just before any containers execute + c.err = c.Runtime.AssembleBuild(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to assemble runtime build %s: %w", c.pipeline.ID, c.err) + } + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte("> Executing secret images...\n")) + + c.logger.Info("executing secret images") + // execute the secret + c.err = c.secret.exec(ctx, &c.pipeline.Secrets) + if c.err != nil { + return fmt.Errorf("unable to execute secret: %w", c.err) + } + + return c.err +} + +// ExecBuild runs a pipeline for a build. +// +// nolint: funlen // ignore function length due to comments and log messages +func (c *client) ExecBuild(ctx context.Context) error { + // defer an upload of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Upload + defer func() { build.Upload(c.build, c.Vela, c.err, c.logger, c.repo) }() + + // execute the services for the pipeline + for _, _service := range c.pipeline.Services { + c.logger.Infof("planning %s service", _service.Name) + // plan the service + c.err = c.PlanService(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to plan service: %w", c.err) + } + + c.logger.Infof("executing %s service", _service.Name) + // execute the service + c.err = c.ExecService(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to execute service: %w", c.err) + } + } + + // execute the steps for the pipeline + for _, _step := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if _step.Name == "init" { + continue + } + + // check if the step should be skipped + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip + if step.Skip(_step, c.build, c.repo) { + continue + } + + c.logger.Infof("planning %s step", _step.Name) + // plan the step + c.err = c.PlanStep(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to plan step: %w", c.err) + } + + c.logger.Infof("executing %s step", _step.Name) + // execute the step + c.err = c.ExecStep(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to execute step: %w", c.err) + } + } + + // create an error group with the context for each stage + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#WithContext + stages, stageCtx := errgroup.WithContext(ctx) + + // create a map to track the progress of each stage + stageMap := new(sync.Map) + + // iterate through each stage in the pipeline + for _, _stage := range c.pipeline.Stages { + // TODO: remove hardcoded reference + if _stage.Name == "init" { + continue + } + + // https://golang.org/doc/faq#closures_and_goroutines + stage := _stage + + // create a new channel for each stage in the map + stageMap.Store(stage.Name, make(chan error)) + + // spawn errgroup routine for the stage + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#Group.Go + stages.Go(func() error { + c.logger.Infof("planning %s stage", stage.Name) + // plan the stage + c.err = c.PlanStage(stageCtx, stage, stageMap) + if c.err != nil { + return fmt.Errorf("unable to plan stage: %w", c.err) + } + + c.logger.Infof("executing %s stage", stage.Name) + // execute the stage + c.err = c.ExecStage(stageCtx, stage, stageMap) + if c.err != nil { + return fmt.Errorf("unable to execute stage: %w", c.err) + } + + return nil + }) + } + + c.logger.Debug("waiting for stages completion") + // wait for the stages to complete or return an error + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#Group.Wait + c.err = stages.Wait() + if c.err != nil { + return fmt.Errorf("unable to wait for stages: %v", c.err) + } + + return c.err +} + +// DestroyBuild cleans up the build after execution. +func (c *client) DestroyBuild(ctx context.Context) error { + var err error + + defer func() { + c.logger.Info("deleting runtime build") + // remove the runtime build for the pipeline + err = c.Runtime.RemoveBuild(ctx, c.pipeline) + if err != nil { + c.logger.Errorf("unable to remove runtime build: %v", err) + } + }() + + // destroy the steps for the pipeline + for _, _step := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if _step.Name == "init" { + continue + } + + c.logger.Infof("destroying %s step", _step.Name) + // destroy the step + err = c.DestroyStep(ctx, _step) + if err != nil { + c.logger.Errorf("unable to destroy step: %v", err) + } + } + + // destroy the stages for the pipeline + for _, _stage := range c.pipeline.Stages { + // TODO: remove hardcoded reference + if _stage.Name == "init" { + continue + } + + c.logger.Infof("destroying %s stage", _stage.Name) + // destroy the stage + err = c.DestroyStage(ctx, _stage) + if err != nil { + c.logger.Errorf("unable to destroy stage: %v", err) + } + } + + // destroy the services for the pipeline + for _, _service := range c.pipeline.Services { + c.logger.Infof("destroying %s service", _service.Name) + // destroy the service + err = c.DestroyService(ctx, _service) + if err != nil { + c.logger.Errorf("unable to destroy service: %v", err) + } + } + + // destroy the secrets for the pipeline + for _, _secret := range c.pipeline.Secrets { + // skip over non-plugin secrets + if _secret.Origin.Empty() { + continue + } + + c.logger.Infof("destroying %s secret", _secret.Name) + // destroy the secret + err = c.secret.destroy(ctx, _secret.Origin) + if err != nil { + c.logger.Errorf("unable to destroy secret: %v", err) + } + } + + c.logger.Info("deleting volume") + // remove the runtime volume for the pipeline + err = c.Runtime.RemoveVolume(ctx, c.pipeline) + if err != nil { + c.logger.Errorf("unable to remove volume: %v", err) + } + + c.logger.Info("deleting network") + // remove the runtime network for the pipeline + err = c.Runtime.RemoveNetwork(ctx, c.pipeline) + if err != nil { + c.logger.Errorf("unable to remove network: %v", err) + } + + return err +} diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go new file mode 100644 index 00000000..ad937865 --- /dev/null +++ b/executor/linux/build_test.go @@ -0,0 +1,573 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "flag" + "net/http/httptest" + "testing" + + "github.com/go-vela/compiler/compiler/native" + "github.com/go-vela/mock/server" + "github.com/urfave/cli/v2" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + + "github.com/gin-gonic/gin" +) + +func TestLinux_CreateBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + build *library.Build + pipeline string + }{ + { // basic secrets pipeline + failure: false, + build: _build, + pipeline: "testdata/build/secrets/basic.yml", + }, + { // basic services pipeline + failure: false, + build: _build, + pipeline: "testdata/build/services/basic.yml", + }, + { // basic steps pipeline + failure: false, + build: _build, + pipeline: "testdata/build/steps/basic.yml", + }, + { // basic stages pipeline + failure: false, + build: _build, + pipeline: "testdata/build/stages/basic.yml", + }, + { // steps pipeline with empty build + failure: true, + build: new(library.Build), + pipeline: "testdata/build/steps/basic.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(test.build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("CreateBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateBuild returned err: %v", err) + } + } +} + +func TestLinux_PlanBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic secrets pipeline + failure: false, + pipeline: "testdata/build/secrets/basic.yml", + }, + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.PlanBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("PlanBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanBuild returned err: %v", err) + } + } +} + +func TestLinux_AssembleBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic secrets pipeline + failure: false, + pipeline: "testdata/build/secrets/basic.yml", + }, + { // secrets pipeline with image not found + failure: true, + pipeline: "testdata/build/secrets/img_notfound.yml", + }, + { // secrets pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/secrets/img_ignorenotfound.yml", + }, + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with image not found + failure: true, + pipeline: "testdata/build/services/img_notfound.yml", + }, + { // services pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/services/img_ignorenotfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with image not found + failure: true, + pipeline: "testdata/build/steps/img_notfound.yml", + }, + { // steps pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/steps/img_ignorenotfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with image not found + failure: true, + pipeline: "testdata/build/stages/img_notfound.yml", + }, + { // stages pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/stages/img_ignorenotfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.AssembleBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("AssembleBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("AssembleBuild returned err: %v", err) + } + } +} + +func TestLinux_ExecBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with image not found + failure: true, + pipeline: "testdata/build/services/img_notfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with image not found + failure: true, + pipeline: "testdata/build/steps/img_notfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with image not found + failure: true, + pipeline: "testdata/build/stages/img_notfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + // TODO: hack - remove this + // + // When using our Docker mock we default the list of + // Docker images that have privileged access. One of + // these images is target/vela-git which is injected + // by the compiler into every build. + // + // The problem this causes is that we haven't called + // all the necessary functions we do in a real world + // scenario to ensure we can run privileged images. + // + // This is the necessary function to create the + // runtime host config so we can run images + // in a privileged fashion. + err = _runtime.CreateVolume(context.Background(), _pipeline) + if err != nil { + t.Errorf("unable to create runtime volume: %w", err) + } + + // TODO: hack - remove this + // + // When calling CreateBuild(), it will automatically set the + // test build object to a status of `created`. This happens + // because we use a mock for the go-vela/server in our tests + // which only returns dummy based responses. + // + // The problem this causes is that our container.Execute() + // function isn't setup to handle builds in a `created` state. + // + // In a real world scenario, we never would have a build + // in this state when we call ExecBuild() because the + // go-vela/server has logic to set it to an expected state. + _engine.build.SetStatus("running") + + err = _engine.ExecBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("ExecBuild for %s should have returned err", test.pipeline) + } + + continue + } + + if err != nil { + t.Errorf("ExecBuild for %s returned err: %v", test.pipeline, err) + } + } +} + +func TestLinux_DestroyBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic secrets pipeline + failure: false, + pipeline: "testdata/build/secrets/basic.yml", + }, + { // secrets pipeline with name not found + failure: false, + pipeline: "testdata/build/secrets/name_notfound.yml", + }, + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with name not found + failure: false, + pipeline: "testdata/build/services/name_notfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with name not found + failure: false, + pipeline: "testdata/build/steps/name_notfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with name not found + failure: false, + pipeline: "testdata/build/stages/name_notfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.DestroyBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("DestroyBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyBuild returned err: %v", err) + } + } +} diff --git a/executor/linux/doc.go b/executor/linux/doc.go new file mode 100644 index 00000000..cd16e144 --- /dev/null +++ b/executor/linux/doc.go @@ -0,0 +1,11 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package linux provides the ability for Vela to +// integrate with a Linux as an operating system. +// +// Usage: +// +// import "github.com/go-vela/worker/executor/linux" +package linux diff --git a/executor/linux/driver.go b/executor/linux/driver.go new file mode 100644 index 00000000..a1d80eb5 --- /dev/null +++ b/executor/linux/driver.go @@ -0,0 +1,12 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import "github.com/go-vela/types/constants" + +// Driver outputs the configured executor driver. +func (c *client) Driver() string { + return constants.DriverLinux +} diff --git a/executor/linux/driver_test.go b/executor/linux/driver_test.go new file mode 100644 index 00000000..a1e826b3 --- /dev/null +++ b/executor/linux/driver_test.go @@ -0,0 +1,56 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/mock/server" + "github.com/go-vela/pkg-runtime/runtime/docker" + "github.com/go-vela/sdk-go/vela" + "github.com/go-vela/types/constants" +) + +func TestLinux_Driver(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + want := constants.DriverLinux + + _engine, err := New( + WithBuild(testBuild()), + WithHostname("localhost"), + WithPipeline(testSteps()), + WithRepo(testRepo()), + WithRuntime(_runtime), + WithUser(testUser()), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run tes + got := _engine.Driver() + + if !reflect.DeepEqual(got, want) { + t.Errorf("Driver is %v, want %v", got, want) + } +} diff --git a/executor/linux/linux.go b/executor/linux/linux.go new file mode 100644 index 00000000..80d020e8 --- /dev/null +++ b/executor/linux/linux.go @@ -0,0 +1,86 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "sync" + + "github.com/go-vela/pkg-runtime/runtime" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + + "github.com/sirupsen/logrus" +) + +type ( + // client manages communication with the pipeline resources. + client struct { + Vela *vela.Client + Runtime runtime.Engine + Secrets map[string]*library.Secret + Hostname string + Version string + + // clients for build actions + secret *secretSvc + + // private fields + init *pipeline.Container + logger *logrus.Entry + build *library.Build + pipeline *pipeline.Build + repo *library.Repo + // nolint: structcheck,unused // ignore false positives + secrets sync.Map + services sync.Map + serviceLogs sync.Map + steps sync.Map + stepLogs sync.Map + user *library.User + err error + } + + // nolint: structcheck // ignore false positive + svc struct { + client *client + } +) + +// New returns an Executor implementation that integrates with a Linux instance. +// +// nolint: golint // ignore unexported type as it is intentional +func New(opts ...Opt) (*client, error) { + // create new Linux client + c := new(client) + + // create new logger for the client + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#StandardLogger + logger := logrus.StandardLogger() + + // create new logger for the client + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#NewEntry + c.logger = logrus.NewEntry(logger) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(c) + if err != nil { + return nil, err + } + } + + // instantiate map for non-plugin secrets + c.Secrets = make(map[string]*library.Secret) + + // instantiate all client services + c.secret = &secretSvc{client: c} + + return c, nil +} diff --git a/executor/linux/linux_test.go b/executor/linux/linux_test.go new file mode 100644 index 00000000..073ce616 --- /dev/null +++ b/executor/linux/linux_test.go @@ -0,0 +1,245 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + "github.com/go-vela/types" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLinux_New(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + build *library.Build + }{ + { + failure: false, + build: testBuild(), + }, + { + failure: true, + build: nil, + }, + } + + // run tests + for _, test := range tests { + _, err := New( + WithBuild(test.build), + WithHostname("localhost"), + WithPipeline(testSteps()), + WithRepo(testRepo()), + WithRuntime(_runtime), + WithUser(testUser()), + WithVelaClient(_client), + ) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +} + +// testBuild is a test helper function to create a Build +// type with all fields set to a fake value. +func testBuild() *library.Build { + return &library.Build{ + ID: vela.Int64(1), + Number: vela.Int(1), + Parent: vela.Int(1), + Event: vela.String("push"), + Status: vela.String("success"), + Error: vela.String(""), + Enqueued: vela.Int64(1563474077), + Created: vela.Int64(1563474076), + Started: vela.Int64(1563474077), + Finished: vela.Int64(0), + Deploy: vela.String(""), + Clone: vela.String("https://github.com/github/octocat.git"), + Source: vela.String("https://github.com/github/octocat/abcdefghi123456789"), + Title: vela.String("push received from https://github.com/github/octocat"), + Message: vela.String("First commit..."), + Commit: vela.String("48afb5bdc41ad69bf22588491333f7cf71135163"), + Sender: vela.String("OctoKitty"), + Author: vela.String("OctoKitty"), + Branch: vela.String("master"), + Ref: vela.String("refs/heads/master"), + BaseRef: vela.String(""), + Host: vela.String("example.company.com"), + Runtime: vela.String("docker"), + Distribution: vela.String("linux"), + } +} + +// testRepo is a test helper function to create a Repo +// type with all fields set to a fake value. +func testRepo() *library.Repo { + return &library.Repo{ + ID: vela.Int64(1), + Org: vela.String("github"), + Name: vela.String("octocat"), + FullName: vela.String("github/octocat"), + Link: vela.String("https://github.com/github/octocat"), + Clone: vela.String("https://github.com/github/octocat.git"), + Branch: vela.String("master"), + Timeout: vela.Int64(60), + Visibility: vela.String("public"), + Private: vela.Bool(false), + Trusted: vela.Bool(false), + Active: vela.Bool(true), + AllowPull: vela.Bool(false), + AllowPush: vela.Bool(true), + AllowDeploy: vela.Bool(false), + AllowTag: vela.Bool(false), + } +} + +// testUser is a test helper function to create a User +// type with all fields set to a fake value. +func testUser() *library.User { + return &library.User{ + ID: vela.Int64(1), + Name: vela.String("octocat"), + Token: vela.String("superSecretToken"), + Hash: vela.String("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy"), + Favorites: vela.Strings([]string{"github/octocat"}), + Active: vela.Bool(true), + Admin: vela.Bool(false), + } +} + +// testUser is a test helper function to create a metadata +// type with all fields set to a fake value. +func testMetadata() *types.Metadata { + return &types.Metadata{ + Database: &types.Database{ + Driver: "foo", + Host: "foo", + }, + Queue: &types.Queue{ + Channel: "foo", + Driver: "foo", + Host: "foo", + }, + Source: &types.Source{ + Driver: "foo", + Host: "foo", + }, + Vela: &types.Vela{ + Address: "foo", + WebAddress: "foo", + }, + } +} + +// testSteps is a test helper function to create a steps +// pipeline with fake steps. +func testSteps() *pipeline.Build { + return &pipeline.Build{ + Version: "1", + ID: "github_octocat_1", + Services: pipeline.ContainerSlice{ + { + ID: "service_github_octocat_1_postgres", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + Steps: pipeline.ContainerSlice{ + { + ID: "step_github_octocat_1_init", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "always", + }, + { + ID: "step_github_octocat_1_clone", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-git:v0.3.0", + Name: "clone", + Number: 2, + Pull: "always", + }, + { + ID: "step_github_octocat_1_echo", + Commands: []string{"echo hello"}, + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 3, + Pull: "always", + }, + }, + Secrets: pipeline.SecretSlice{ + { + Name: "foo", + Key: "github/octocat/foo", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + }, + { + Name: "foo", + Key: "github/foo", + Engine: "native", + Type: "org", + Origin: &pipeline.Container{}, + }, + { + Name: "foo", + Key: "github/octokitties/foo", + Engine: "native", + Type: "shared", + Origin: &pipeline.Container{}, + }, + }, + } +} diff --git a/executor/linux/opts.go b/executor/linux/opts.go new file mode 100644 index 00000000..179cafdd --- /dev/null +++ b/executor/linux/opts.go @@ -0,0 +1,179 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "fmt" + + "github.com/go-vela/pkg-runtime/runtime" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + + "github.com/sirupsen/logrus" +) + +// Opt represents a configuration option to initialize the client. +type Opt func(*client) error + +// WithBuild sets the library build in the client. +func WithBuild(b *library.Build) Opt { + logrus.Trace("configuring build in linux client") + + return func(c *client) error { + // check if the build provided is empty + if b == nil { + return fmt.Errorf("empty build provided") + } + + // update engine logger with build metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + c.logger = c.logger.WithField("build", b.GetNumber()) + + // set the build in the client + c.build = b + + return nil + } +} + +// WithHostname sets the hostname in the client. +func WithHostname(hostname string) Opt { + logrus.Trace("configuring hostname in linux client") + + return func(c *client) error { + // check if a hostname is provided + if len(hostname) == 0 { + // default the hostname to localhost + hostname = "localhost" + } + + // update engine logger with host metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + c.logger = c.logger.WithField("host", hostname) + + // set the hostname in the client + c.Hostname = hostname + + return nil + } +} + +// WithPipeline sets the pipeline build in the client. +func WithPipeline(p *pipeline.Build) Opt { + logrus.Trace("configuring pipeline in linux client") + + return func(c *client) error { + // check if the pipeline provided is empty + if p == nil { + return fmt.Errorf("empty pipeline provided") + } + + // set the pipeline in the client + c.pipeline = p + + return nil + } +} + +// WithRepo sets the library repo in the client. +func WithRepo(r *library.Repo) Opt { + logrus.Trace("configuring repo in linux client") + + return func(c *client) error { + // check if the repo provided is empty + if r == nil { + return fmt.Errorf("empty repo provided") + } + + // update engine logger with repo metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + c.logger = c.logger.WithField("repo", r.GetFullName()) + + // set the repo in the client + c.repo = r + + return nil + } +} + +// WithRuntime sets the runtime engine in the client. +func WithRuntime(r runtime.Engine) Opt { + logrus.Trace("configuring runtime in linux client") + + return func(c *client) error { + // check if the runtime provided is empty + if r == nil { + return fmt.Errorf("empty runtime provided") + } + + // set the runtime in the client + c.Runtime = r + + return nil + } +} + +// WithUser sets the library user in the client. +func WithUser(u *library.User) Opt { + logrus.Trace("configuring user in linux client") + + return func(c *client) error { + // check if the user provided is empty + if u == nil { + return fmt.Errorf("empty user provided") + } + + // update engine logger with user metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + c.logger = c.logger.WithField("user", u.GetName()) + + // set the user in the client + c.user = u + + return nil + } +} + +// WithVelaClient sets the Vela client in the client. +func WithVelaClient(cli *vela.Client) Opt { + logrus.Trace("configuring Vela client in linux client") + + return func(c *client) error { + // check if the Vela client provided is empty + if cli == nil { + return fmt.Errorf("empty Vela client provided") + } + + // set the Vela client in the client + c.Vela = cli + + return nil + } +} + +// WithVersion sets the version in the client. +func WithVersion(version string) Opt { + logrus.Trace("configuring version in linux client") + + return func(c *client) error { + // check if a version is provided + if len(version) == 0 { + // default the version to localhost + version = "v0.0.0" + } + + // set the version in the client + c.Version = version + + return nil + } +} diff --git a/executor/linux/opts_test.go b/executor/linux/opts_test.go new file mode 100644 index 00000000..1894e0d6 --- /dev/null +++ b/executor/linux/opts_test.go @@ -0,0 +1,353 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLinux_Opt_WithBuild(t *testing.T) { + // setup types + _build := testBuild() + + // setup tests + tests := []struct { + failure bool + build *library.Build + }{ + { + failure: false, + build: _build, + }, + { + failure: true, + build: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(test.build), + ) + + if test.failure { + if err == nil { + t.Errorf("WithBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithBuild returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.build, _build) { + t.Errorf("WithBuild is %v, want %v", _engine.build, _build) + } + } +} + +func TestLinux_Opt_WithHostname(t *testing.T) { + // setup tests + tests := []struct { + hostname string + want string + }{ + { + hostname: "vela.worker.localhost", + want: "vela.worker.localhost", + }, + { + hostname: "", + want: "localhost", + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithHostname(test.hostname), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + if !reflect.DeepEqual(_engine.Hostname, test.want) { + t.Errorf("WithHostname is %v, want %v", _engine.Hostname, test.want) + } + } +} + +func TestLinux_Opt_WithPipeline(t *testing.T) { + // setup types + _steps := testSteps() + + // setup tests + tests := []struct { + failure bool + pipeline *pipeline.Build + }{ + { + failure: false, + pipeline: _steps, + }, + { + failure: true, + pipeline: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithPipeline(test.pipeline), + ) + + if test.failure { + if err == nil { + t.Errorf("WithPipeline should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithPipeline returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.pipeline, _steps) { + t.Errorf("WithPipeline is %v, want %v", _engine.pipeline, _steps) + } + } +} + +func TestLinux_Opt_WithRepo(t *testing.T) { + // setup types + _repo := testRepo() + + // setup tests + tests := []struct { + failure bool + repo *library.Repo + }{ + { + failure: false, + repo: _repo, + }, + { + failure: true, + repo: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithRepo(test.repo), + ) + + if test.failure { + if err == nil { + t.Errorf("WithRepo should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithRepo returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.repo, _repo) { + t.Errorf("WithRepo is %v, want %v", _engine.repo, _repo) + } + } +} + +func TestLinux_Opt_WithRuntime(t *testing.T) { + // setup types + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + runtime runtime.Engine + }{ + { + failure: false, + runtime: _runtime, + }, + { + failure: true, + runtime: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithRuntime(test.runtime), + ) + + if test.failure { + if err == nil { + t.Errorf("WithRuntime should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithRuntime returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.Runtime, _runtime) { + t.Errorf("WithRuntime is %v, want %v", _engine.Runtime, _runtime) + } + } +} + +func TestLinux_Opt_WithUser(t *testing.T) { + // setup types + _user := testUser() + + // setup tests + tests := []struct { + failure bool + user *library.User + }{ + { + failure: false, + user: _user, + }, + { + failure: true, + user: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithUser(test.user), + ) + + if test.failure { + if err == nil { + t.Errorf("WithUser should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithUser returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.user, _user) { + t.Errorf("WithUser is %v, want %v", _engine.user, _user) + } + } +} + +func TestLinux_Opt_WithVelaClient(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + // setup tests + tests := []struct { + failure bool + client *vela.Client + }{ + { + failure: false, + client: _client, + }, + { + failure: true, + client: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithVelaClient(test.client), + ) + + if test.failure { + if err == nil { + t.Errorf("WithVelaClient should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithVelaClient returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.Vela, _client) { + t.Errorf("WithVelaClient is %v, want %v", _engine.Vela, _client) + } + } +} + +func TestLinux_Opt_WithVersion(t *testing.T) { + // setup tests + tests := []struct { + version string + want string + }{ + { + version: "v1.0.0", + want: "v1.0.0", + }, + { + version: "", + want: "v0.0.0", + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithVersion(test.version), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + if !reflect.DeepEqual(_engine.Version, test.want) { + t.Errorf("WithVersion is %v, want %v", _engine.Version, test.want) + } + } +} diff --git a/executor/linux/secret.go b/executor/linux/secret.go new file mode 100644 index 00000000..5db230c5 --- /dev/null +++ b/executor/linux/secret.go @@ -0,0 +1,367 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/step" + + "github.com/sirupsen/logrus" +) + +// secretSvc handles communication with secret processes during a build. +type secretSvc svc + +var ( + // ErrUnrecognizedSecretType defines the error type when the + // SecretType provided to the client is unsupported. + ErrUnrecognizedSecretType = errors.New("unrecognized secret type") + + // ErrUnableToRetrieve defines the error type when the + // secret is not able to be retrieved from the server. + ErrUnableToRetrieve = errors.New("unable to retrieve secret") +) + +// create configures the secret plugin for execution. +func (s *secretSvc) create(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with secret metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := s.client.logger.WithField("secret", ctn.Name) + + ctn.Environment["VELA_DISTRIBUTION"] = s.client.build.GetDistribution() + ctn.Environment["BUILD_HOST"] = s.client.build.GetHost() + ctn.Environment["VELA_HOST"] = s.client.build.GetHost() + ctn.Environment["VELA_RUNTIME"] = s.client.build.GetRuntime() + ctn.Environment["VELA_VERSION"] = s.client.Version + + logger.Debug("setting up container") + // setup the runtime container + err := s.client.Runtime.SetupContainer(ctx, ctn) + if err != nil { + return err + } + + logger.Debug("injecting secrets") + // inject secrets for container + err = injectSecrets(ctn, s.client.Secrets) + if err != nil { + return err + } + + logger.Debug("substituting container configuration") + // substitute container configuration + err = ctn.Substitute() + if err != nil { + return fmt.Errorf("unable to substitute container configuration") + } + + return nil +} + +// destroy cleans up secret plugin after execution. +func (s *secretSvc) destroy(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with secret metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := s.client.logger.WithField("secret", ctn.Name) + + logger.Debug("inspecting container") + // inspect the runtime container + err := s.client.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + logger.Debug("removing container") + // remove the runtime container + err = s.client.Runtime.RemoveContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} + +// exec runs a secret plugins for a pipeline. +func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { + // stream all the logs to the init step + _init, err := step.Load(s.client.init, &s.client.steps) + if err != nil { + return err + } + + defer func() { + _init.SetFinished(time.Now().UTC().Unix()) + + s.client.logger.Infof("uploading %s step state", _init.GetName()) + // send API call to update the build + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#StepService.Update + _, _, err = s.client.Vela.Step.Update(s.client.repo.GetOrg(), s.client.repo.GetName(), s.client.build.GetNumber(), _init) + if err != nil { + s.client.logger.Errorf("unable to upload init state: %v", err) + } + }() + + // execute the secrets for the pipeline + for _, _secret := range *p { + // skip over non-plugin secrets + if _secret.Origin.Empty() { + continue + } + + // update engine logger with secret metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := s.client.logger.WithField("secret", _secret.Origin.Name) + + logger.Debug("running container") + // run the runtime container + err := s.client.Runtime.RunContainer(ctx, _secret.Origin, s.client.pipeline) + if err != nil { + return err + } + + go func() { + logger.Debug("stream logs for container") + // stream logs from container + err = s.client.secret.stream(ctx, _secret.Origin) + if err != nil { + logger.Error(err) + } + }() + + logger.Debug("waiting for container") + // wait for the runtime container + err = s.client.Runtime.WaitContainer(ctx, _secret.Origin) + if err != nil { + return err + } + + logger.Debug("inspecting container") + // inspect the runtime container + err = s.client.Runtime.InspectContainer(ctx, _secret.Origin) + if err != nil { + return err + } + + // check the step exit code + if _secret.Origin.ExitCode != 0 { + // check if we ignore step failures + if !_secret.Origin.Ruleset.Continue { + // set build status to failure + s.client.build.SetStatus(constants.StatusFailure) + } + + // update the step fields + _init.SetExitCode(_secret.Origin.ExitCode) + _init.SetStatus(constants.StatusFailure) + + return fmt.Errorf("%s container exited with non-zero code", _secret.Origin.Name) + } + + // send API call to update the build + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#StepService.Update + _, _, err = s.client.Vela.Step.Update(s.client.repo.GetOrg(), s.client.repo.GetName(), s.client.build.GetNumber(), _init) + if err != nil { + s.client.logger.Errorf("unable to upload init state: %v", err) + } + } + + return nil +} + +// pull defines a function that pulls the secrets from the server for a given pipeline. +func (s *secretSvc) pull(secret *pipeline.Secret) (*library.Secret, error) { + // nolint: staticcheck // reports the value is never used but we return it + _secret := new(library.Secret) + + switch secret.Type { + // handle repo secrets + case constants.SecretOrg: + org, key, err := secret.ParseOrg(s.client.repo.GetOrg()) + if err != nil { + return nil, err + } + + // send API call to capture the org secret + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#SecretService.Get + _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, "*", key) + if err != nil { + return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + } + + secret.Value = _secret.GetValue() + + // handle repo secrets + case constants.SecretRepo: + org, repo, key, err := secret.ParseRepo(s.client.repo.GetOrg(), s.client.repo.GetName()) + if err != nil { + return nil, err + } + + // send API call to capture the repo secret + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#SecretService.Get + _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, repo, key) + if err != nil { + return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + } + + secret.Value = _secret.GetValue() + + // handle shared secrets + case constants.SecretShared: + org, team, key, err := secret.ParseShared() + if err != nil { + return nil, err + } + + // send API call to capture the repo secret + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#SecretService.Get + _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, team, key) + if err != nil { + return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + } + + secret.Value = _secret.GetValue() + + default: + return nil, fmt.Errorf("%s: %s", ErrUnrecognizedSecretType, secret.Type) + } + + return _secret, nil +} + +// stream tails the output for a secret plugin. +func (s *secretSvc) stream(ctx context.Context, ctn *pipeline.Container) error { + // stream all the logs to the init step + _log, err := step.LoadLogs(s.client.init, &s.client.stepLogs) + if err != nil { + return err + } + + // update engine logger with secret metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := s.client.logger.WithField("secret", ctn.Name) + + // create new buffer for uploading logs + logs := new(bytes.Buffer) + + defer func() { + // NOTE: Whenever the stream ends we want to ensure + // that this function makes the call to update + // the step logs + logger.Trace(logs.String()) + + // update the existing log with the last bytes + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(logs.Bytes()) + + logger.Debug("uploading logs") + // send API call to update the logs for the service + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.UpdateService + _log, _, err = s.client.Vela.Log.UpdateStep(s.client.repo.GetOrg(), s.client.repo.GetName(), s.client.build.GetNumber(), ctn.Number, _log) + if err != nil { + logger.Errorf("unable to upload container logs: %v", err) + } + }() + + logger.Debug("tailing container") + // tail the runtime container + rc, err := s.client.Runtime.TailContainer(ctx, ctn) + if err != nil { + return err + } + defer rc.Close() + + // create new scanner from the container output + scanner := bufio.NewScanner(rc) + + // scan entire container output + for scanner.Scan() { + // write all the logs from the scanner + logs.Write(append(scanner.Bytes(), []byte("\n")...)) + + // if we have at least 1000 bytes in our buffer + // + // nolint: gomnd // ignore magic number + if logs.Len() > 1000 { + logger.Trace(logs.String()) + + // update the existing log with the new bytes + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(logs.Bytes()) + + logger.Debug("appending logs") + // send API call to append the logs for the init step + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.UpdateStep + // nolint: lll // skip line length due to variable names + _log, _, err = s.client.Vela.Log.UpdateStep(s.client.repo.GetOrg(), s.client.repo.GetName(), s.client.build.GetNumber(), s.client.init.Number, _log) + if err != nil { + return err + } + + // flush the buffer of logs + logs.Reset() + } + } + + return scanner.Err() +} + +// TODO: Evaluate pulling this into a "bool" types function for injecting +// +// helper function to check secret whitelist before setting value. +func injectSecrets(ctn *pipeline.Container, m map[string]*library.Secret) error { + // inject secrets for step + for _, _secret := range ctn.Secrets { + logrus.Tracef("looking up secret %s from pipeline secrets", _secret.Source) + // lookup container secret in map + s, ok := m[_secret.Source] + if !ok { + continue + } + + logrus.Tracef("matching secret %s to container %s", _secret.Source, ctn.Name) + // ensure the secret matches with the container + if s.Match(ctn) { + ctn.Environment[strings.ToUpper(_secret.Target)] = s.GetValue() + } + } + + return nil +} + +// escapeNewlineSecrets is a helper function to double-escape escaped newlines, +// double-escaped newlines are resolved to newlines during env substitution. +func escapeNewlineSecrets(m map[string]*library.Secret) { + for i, secret := range m { + // only double-escape secrets that have been manually escaped + if !strings.Contains(secret.GetValue(), "\\\\n") { + s := strings.Replace(secret.GetValue(), "\\n", "\\\n", -1) + m[i].Value = &s + } + } +} diff --git a/executor/linux/secret_test.go b/executor/linux/secret_test.go new file mode 100644 index 00000000..a1bcd3b6 --- /dev/null +++ b/executor/linux/secret_test.go @@ -0,0 +1,807 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "flag" + "io/ioutil" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" + + "github.com/go-vela/compiler/compiler/native" + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + + "github.com/google/go-cmp/cmp" +) + +func TestLinux_Secret_create(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + _steps := testSteps() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { + failure: false, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_vault", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "vault", + Number: 1, + Pull: "not_present", + }, + }, + { + failure: true, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_vault", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:notfound", + Name: "vault", + Number: 1, + Pull: "not_present", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(_steps), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.secret.create(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("create should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("create returned err: %v", err) + } + } +} + +func TestLinux_Secret_delete(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + _steps := testSteps() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _step := new(library.Step) + _step.SetName("clone") + _step.SetNumber(2) + _step.SetStatus(constants.StatusPending) + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + step *library.Step + }{ + { + failure: false, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_vault", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "vault", + Number: 1, + Pull: "always", + }, + step: new(library.Step), + }, + { + failure: false, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_vault", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "vault", + Number: 2, + Pull: "always", + }, + step: _step, + }, + { + failure: true, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_notfound", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "notfound", + Number: 2, + Pull: "always", + }, + step: new(library.Step), + }, + { + failure: true, + container: &pipeline.Container{ + ID: "secret_github_octocat_1_ignorenotfound", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "ignorenotfound", + Number: 2, + Pull: "always", + }, + step: new(library.Step), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(_steps), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + _ = _engine.CreateBuild(context.Background()) + + _engine.steps.Store(test.container.ID, test.step) + + err = _engine.secret.destroy(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("destroy should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("destroy returned err: %v", err) + } + } +} + +func TestLinux_Secret_exec(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + pipeline string + }{ + { // basic secrets pipeline + failure: false, + pipeline: "testdata/build/secrets/basic.yml", + }, + { // pipeline with secret name not found + failure: true, + pipeline: "testdata/build/secrets/name_notfound.yml", + }, + } + + // run tests + for _, test := range tests { + file, _ := ioutil.ReadFile(test.pipeline) + + p, _ := compiler. + WithBuild(_build). + WithRepo(_repo). + WithUser(_user). + WithMetadata(_metadata). + Compile(file) + + _engine, err := New( + WithBuild(_build), + WithPipeline(p), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + _engine.build.SetStatus(constants.StatusSuccess) + + // add init container info to client + _ = _engine.CreateBuild(context.Background()) + + err = _engine.secret.exec(context.Background(), &p.Secrets) + + if test.failure { + if err == nil { + t.Errorf("exec should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("exec returned err: %v", err) + } + } +} + +func TestLinux_Secret_pull(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + server := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(server.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + secret *pipeline.Secret + }{ + { // success with org secret + failure: false, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "github/foo", + Engine: "native", + Type: "org", + Origin: &pipeline.Container{}, + }, + }, + { // failure with invalid org secret + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "foo/foo/foo", + Engine: "native", + Type: "org", + Origin: &pipeline.Container{}, + }, + }, + { // failure with org secret key not found + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "not-found", + Engine: "native", + Type: "org", + Origin: &pipeline.Container{}, + }, + }, + { // success with repo secret + failure: false, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "github/octocat/foo", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + }, + }, + { // failure with invalid repo secret + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "foo/foo/foo/foo", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + }, + }, + { // failure with repo secret key not found + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "not-found", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + }, + }, + { // success with shared secret + failure: false, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "github/octokitties/foo", + Engine: "native", + Type: "shared", + Origin: &pipeline.Container{}, + }, + }, + { // failure with shared secret key not found + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "not-found", + Engine: "native", + Type: "shared", + Origin: &pipeline.Container{}, + }, + }, + { // failure with invalid type + failure: true, + secret: &pipeline.Secret{ + Name: "foo", + Value: "bar", + Key: "github/octokitties/foo", + Engine: "native", + Type: "invalid", + Origin: &pipeline.Container{}, + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(testSteps()), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + _, err = _engine.secret.pull(test.secret) + + if test.failure { + if err == nil { + t.Errorf("pull should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("pull returned err: %v", err) + } + } +} + +func TestLinux_Secret_stream(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + _steps := testSteps() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + logs *library.Log + container *pipeline.Container + }{ + { // container step succeeds + failure: false, + logs: new(library.Log), + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "always", + }, + }, + { // container step fails because of invalid container id + failure: true, + logs: new(library.Log), + container: &pipeline.Container{ + ID: "secret_github_octocat_1_notfound", + Directory: "/vela/src/vcs.company.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/secret-vault:latest", + Name: "notfound", + Number: 2, + Pull: "always", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(_steps), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // add init container info to client + _ = _engine.CreateBuild(context.Background()) + + err = _engine.secret.stream(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("stream should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("stream returned err: %v", err) + } + } +} + +func TestLinux_Secret_injectSecret(t *testing.T) { + // name and value of secret + v := "foo" + + // setup types + tests := []struct { + step *pipeline.Container + msec map[string]*library.Secret + want *pipeline.Container + }{ + // Tests for secrets with image ACLs + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Images: &[]string{""}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Images: &[]string{"alpine"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Images: &[]string{"alpine:latest"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Images: &[]string{"centos"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: make(map[string]string), + }, + }, + + // Tests for secrets with event ACLs + { // push event checks + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"push"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo", "BUILD_EVENT": "push"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"deployment"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + }, + }, + { // pull_request event checks + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "pull_request"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"pull_request"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo", "BUILD_EVENT": "pull_request"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "pull_request"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"deployment"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "pull_request"}, + }, + }, + { // tag event checks + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "tag"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"tag"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo", "BUILD_EVENT": "tag"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "tag"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"deployment"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "tag"}, + }, + }, + { // deployment event checks + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "deployment"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"deployment"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo", "BUILD_EVENT": "deployment"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "deployment"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"tag"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "deployment"}, + }, + }, + + // Tests for secrets with event and image ACLs + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"push"}, Images: &[]string{"centos"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + }, + }, + { + step: &pipeline.Container{ + Image: "centos:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"pull_request"}, Images: &[]string{"centos"}}}, + want: &pipeline.Container{ + Image: "centos:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + }, + }, + { + step: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"BUILD_EVENT": "push"}, + Secrets: pipeline.StepSecretSlice{{Source: "FOO", Target: "FOO"}}, + }, + msec: map[string]*library.Secret{"FOO": {Name: &v, Value: &v, Events: &[]string{"push"}, Images: &[]string{"alpine"}}}, + want: &pipeline.Container{ + Image: "alpine:latest", + Environment: map[string]string{"FOO": "foo", "BUILD_EVENT": "push"}, + }, + }, + } + + // run test + for _, test := range tests { + _ = injectSecrets(test.step, test.msec) + got := test.step + + // Preferred use of reflect.DeepEqual(x, y interface) is giving false positives. + // Switching to a Google library for increased clarity. + // https://github.com/google/go-cmp + if diff := cmp.Diff(test.want.Environment, got.Environment); diff != "" { + t.Errorf("injectSecrets mismatch (-want +got):\n%s", diff) + } + } +} + +func TestLinux_Secret_escapeNewlineSecrets(t *testing.T) { + // name and value of secret + n := "foo" + v := "bar\\nbaz" + vEscaped := "bar\\\nbaz" + + // desired secret value + w := "bar\\\nbaz" + + // setup types + tests := []struct { + secretMap map[string]*library.Secret + want map[string]*library.Secret + }{ + + { + secretMap: map[string]*library.Secret{"FOO": {Name: &n, Value: &v}}, + want: map[string]*library.Secret{"FOO": {Name: &n, Value: &w}}, + }, + { + secretMap: map[string]*library.Secret{"FOO": {Name: &n, Value: &vEscaped}}, + want: map[string]*library.Secret{"FOO": {Name: &n, Value: &w}}, + }, + } + + // run test + for _, test := range tests { + escapeNewlineSecrets(test.secretMap) + got := test.secretMap + + // Preferred use of reflect.DeepEqual(x, y interface) is giving false positives. + // Switching to a Google library for increased clarity. + // https://github.com/google/go-cmp + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("escapeNewlineSecrets mismatch (-want +got):\n%s", diff) + } + } +} diff --git a/executor/linux/service.go b/executor/linux/service.go new file mode 100644 index 00000000..7e67bf50 --- /dev/null +++ b/executor/linux/service.go @@ -0,0 +1,273 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/service" + "golang.org/x/sync/errgroup" +) + +// CreateService configures the service for execution. +func (c *client) CreateService(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with service metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("service", ctn.Name) + + logger.Debug("setting up container") + // setup the runtime container + err := c.Runtime.SetupContainer(ctx, ctn) + if err != nil { + return err + } + + // update the service container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Environment + err = service.Environment(ctn, c.build, c.repo, nil, c.Version) + if err != nil { + return err + } + + logger.Debug("injecting secrets") + // inject secrets for container + err = injectSecrets(ctn, c.Secrets) + if err != nil { + return err + } + + logger.Debug("substituting container configuration") + // substitute container configuration + // + // https://pkg.go.dev/github.com/go-vela/types/pipeline#Container.Substitute + err = ctn.Substitute() + if err != nil { + return fmt.Errorf("unable to substitute container configuration") + } + + return nil +} + +// PlanService prepares the service for execution. +func (c *client) PlanService(ctx context.Context, ctn *pipeline.Container) error { + var err error + + // update engine logger with service metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("service", ctn.Name) + + // create the library service object + _service := new(library.Service) + _service.SetName(ctn.Name) + _service.SetNumber(ctn.Number) + _service.SetImage(ctn.Image) + _service.SetStatus(constants.StatusRunning) + _service.SetStarted(time.Now().UTC().Unix()) + _service.SetHost(c.build.GetHost()) + _service.SetRuntime(c.build.GetRuntime()) + _service.SetDistribution(c.build.GetDistribution()) + + logger.Debug("uploading service state") + // send API call to update the service + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#SvcService.Update + _service, _, err = c.Vela.Svc.Update(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), _service) + if err != nil { + return err + } + + // update the service container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Environment + err = service.Environment(ctn, c.build, c.repo, _service, c.Version) + if err != nil { + return err + } + + // add a service to a map + c.services.Store(ctn.ID, _service) + + // get the service log here + logger.Debug("retrieve service log") + // send API call to capture the service log + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.GetService + _log, _, err := c.Vela.Log.GetService(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), _service.GetNumber()) + if err != nil { + return err + } + + // add a service log to a map + c.serviceLogs.Store(ctn.ID, _log) + + return nil +} + +// ExecService runs a service. +func (c *client) ExecService(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with service metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("service", ctn.Name) + + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + _service, err := service.Load(ctn, &c.services) + if err != nil { + return err + } + + // defer taking a snapshot of the service + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Snapshot + defer func() { service.Snapshot(ctn, c.build, c.Vela, c.logger, c.repo, _service) }() + + logger.Debug("running container") + // run the runtime container + err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) + if err != nil { + return err + } + + // create an error group with the parent context + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#WithContext + logs, logCtx := errgroup.WithContext(ctx) + + logs.Go(func() error { + logger.Debug("streaming logs for container") + // stream logs from container + err := c.StreamService(logCtx, ctn) + if err != nil { + logger.Error(err) + } + + return nil + }) + + return nil +} + +// StreamService tails the output for a service. +func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with service metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("service", ctn.Name) + + // load the logs for the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#LoadLogs + _log, err := service.LoadLogs(ctn, &c.serviceLogs) + if err != nil { + return err + } + + // nolint: dupl // ignore similar code + defer func() { + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + logger.Errorf("unable to tail container output for upload: %v", err) + + return + } + defer rc.Close() + + // read all output from the runtime container + data, err := ioutil.ReadAll(rc) + if err != nil { + logger.Errorf("unable to read container output for upload: %v", err) + + return + } + + // overwrite the existing log with all bytes + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.SetData + _log.SetData(data) + + logger.Debug("uploading logs") + // send API call to update the logs for the service + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.UpdateService + _, _, err = c.Vela.Log.UpdateService(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), ctn.Number, _log) + if err != nil { + logger.Errorf("unable to upload container logs: %v", err) + } + }() + + logger.Debug("tailing container") + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + return err + } + defer rc.Close() + + // set the timeout to the repo timeout + // to ensure the stream is not cut off + c.Vela.SetTimeout(time.Minute * time.Duration(c.repo.GetTimeout())) + + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#SvcService.Stream + _, err = c.Vela.Svc.Stream(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), ctn.Number, rc) + if err != nil { + logger.Errorf("unable to stream logs: %v", err) + } + + logger.Info("finished streaming logs") + + return nil +} + +// DestroyService cleans up services after execution. +func (c *client) DestroyService(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with service metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("service", ctn.Name) + + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + _service, err := service.Load(ctn, &c.services) + if err != nil { + // create the service from the container + // + // https://pkg.go.dev/github.com/go-vela/types/library#ServiceFromContainer + _service = library.ServiceFromContainer(ctn) + } + + // defer an upload of the service + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#LoaUploadd + defer func() { service.Upload(ctn, c.build, c.Vela, logger, c.repo, _service) }() + + logger.Debug("inspecting container") + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + logger.Debug("removing container") + // remove the runtime container + err = c.Runtime.RemoveContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} diff --git a/executor/linux/service_test.go b/executor/linux/service_test.go new file mode 100644 index 00000000..5b39eba7 --- /dev/null +++ b/executor/linux/service_test.go @@ -0,0 +1,473 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLinux_CreateService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with image not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:notfound", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("CreateService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateService returned err: %v", err) + } + } +} + +func TestLinux_PlanService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with nil environment + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: nil, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("PlanService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanService returned err: %v", err) + } + } +} + +func TestLinux_ExecService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with image not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:notfound", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.services.Store(test.container.ID, new(library.Service)) + _engine.serviceLogs.Store(test.container.ID, new(library.Log)) + } + + err = _engine.ExecService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("ExecService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecService returned err: %v", err) + } + } +} + +func TestLinux_StreamService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with name not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_notfound", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "notfound", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.services.Store(test.container.ID, new(library.Service)) + _engine.serviceLogs.Store(test.container.ID, new(library.Log)) + } + + err = _engine.StreamService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("StreamService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("StreamService returned err: %v", err) + } + } +} + +func TestLinux_DestroyService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with ignoring name not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_ignorenotfound", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "ignorenotfound", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("DestroyService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyService returned err: %v", err) + } + } +} diff --git a/executor/linux/stage.go b/executor/linux/stage.go new file mode 100644 index 00000000..551fa393 --- /dev/null +++ b/executor/linux/stage.go @@ -0,0 +1,167 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "fmt" + "sync" + + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/step" +) + +// CreateStage prepares the stage for execution. +func (c *client) CreateStage(ctx context.Context, s *pipeline.Stage) error { + // load the logs for the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadLogs + _log, err := step.LoadLogs(c.init, &c.stepLogs) + if err != nil { + return err + } + + // update engine logger with stage metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("stage", s.Name) + + // update the init log with progress + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData([]byte(fmt.Sprintf("> Pulling step images for stage %s...\n", s.Name))) + + // create the steps for the stage + for _, _step := range s.Steps { + // update the container environment with stage name + _step.Environment["VELA_STEP_STAGE"] = s.Name + + logger.Debugf("creating %s step", _step.Name) + // create the step + err := c.CreateStep(ctx, _step) + if err != nil { + return err + } + + logger.Infof("inspecting image for %s step", _step.Name) + // inspect the step image + image, err := c.Runtime.InspectImage(ctx, _step) + if err != nil { + return err + } + + // update the init log with step image info + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData + _log.AppendData(image) + } + + return nil +} + +// PlanStage prepares the stage for execution. +func (c *client) PlanStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) error { + // update engine logger with stage metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("stage", s.Name) + + logger.Debug("gathering stage dependency tree") + // ensure dependent stages have completed + for _, needs := range s.Needs { + logger.Debugf("looking up dependency %s", needs) + // check if a dependency stage has completed + stageErr, ok := m.Load(needs) + if !ok { // stage not found so we continue + continue + } + + logger.Debugf("waiting for dependency %s", needs) + // wait for the stage channel to close + select { + case <-ctx.Done(): + return fmt.Errorf("errgroup context is done") + case err := <-stageErr.(chan error): + if err != nil { + logger.Errorf("%s stage returned error: %v", needs, err) + return err + } + + continue + } + } + + return nil +} + +// ExecStage runs a stage. +func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) error { + // update engine logger with stage metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("stage", s.Name) + + // close the stage channel at the end + defer func() { + // retrieve the error channel for the current stage + errChan, ok := m.Load(s.Name) + if !ok { + logger.Debugf("error channel for stage %s not found", s.Name) + + return + } + + // close the error channel + close(errChan.(chan error)) + }() + + logger.Debug("starting execution of stage") + // execute the steps for the stage + for _, _step := range s.Steps { + // check if the step should be skipped + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip + if step.Skip(_step, c.build, c.repo) { + continue + } + + logger.Debugf("planning %s step", _step.Name) + // plan the step + err := c.PlanStep(ctx, _step) + if err != nil { + return fmt.Errorf("unable to plan step %s: %w", _step.Name, err) + } + + logger.Infof("executing %s step", _step.Name) + // execute the step + err = c.ExecStep(ctx, _step) + if err != nil { + return fmt.Errorf("unable to exec step %s: %w", _step.Name, err) + } + } + + return nil +} + +// DestroyStage cleans up the stage after execution. +func (c *client) DestroyStage(ctx context.Context, s *pipeline.Stage) error { + // update engine logger with stage metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("stage", s.Name) + var err error + + // destroy the steps for the stage + for _, _step := range s.Steps { + logger.Debugf("destroying %s step", _step.Name) + // destroy the step + err = c.DestroyStep(ctx, _step) + if err != nil { + logger.Errorf("unable to destroy step: %v", err) + } + } + + return err +} diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go new file mode 100644 index 00000000..8b7084b5 --- /dev/null +++ b/executor/linux/stage_test.go @@ -0,0 +1,456 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "errors" + "flag" + "net/http/httptest" + "sync" + "testing" + + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" + + "github.com/go-vela/compiler/compiler/native" + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/pipeline" +) + +func TestLinux_CreateStage(t *testing.T) { + // setup types + _file := "testdata/build/stages/basic.yml" + _build := testBuild() + _repo := testRepo() + _user := testUser() + _metadata := testMetadata() + + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithMetadata(_metadata). + WithUser(_user). + Compile(_file) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", _file, err) + } + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // stage with step container with image not found + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // empty stage + failure: true, + stage: new(pipeline.Stage), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if len(test.stage.Name) > 0 { + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + } + + err = _engine.CreateStage(context.Background(), test.stage) + + if test.failure { + if err == nil { + t.Errorf("CreateStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateStage returned err: %v", err) + } + } +} + +func TestLinux_PlanStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + testMap := new(sync.Map) + testMap.Store("foo", make(chan error, 1)) + + tm, _ := testMap.Load("foo") + tm.(chan error) <- nil + close(tm.(chan error)) + + errMap := new(sync.Map) + errMap.Store("foo", make(chan error, 1)) + + em, _ := errMap.Load("foo") + em.(chan error) <- errors.New("bar") + close(em.(chan error)) + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + stageMap *sync.Map + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: new(sync.Map), + }, + { // basic stage with nil stage map + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Needs: []string{"foo"}, + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: testMap, + }, + { // basic stage with error stage map + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Needs: []string{"foo"}, + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: errMap, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanStage(context.Background(), test.stage, test.stageMap) + + if test.failure { + if err == nil { + t.Errorf("PlanStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanStage returned err: %v", err) + } + } +} + +func TestLinux_ExecStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // stage with step container with image not found + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // stage with step container with bad number + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 0, + Pull: "not_present", + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + stageMap := new(sync.Map) + stageMap.Store("echo", make(chan error, 1)) + + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.ExecStage(context.Background(), test.stage, stageMap) + + if test.failure { + if err == nil { + t.Errorf("ExecStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecStage returned err: %v", err) + } + } +} + +func TestLinux_DestroyStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyStage(context.Background(), test.stage) + + if test.failure { + if err == nil { + t.Errorf("DestroyStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyStage returned err: %v", err) + } + } +} diff --git a/executor/linux/step.go b/executor/linux/step.go new file mode 100644 index 00000000..a3200177 --- /dev/null +++ b/executor/linux/step.go @@ -0,0 +1,327 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/step" + "golang.org/x/sync/errgroup" +) + +// CreateStep configures the step for execution. +func (c *client) CreateStep(ctx context.Context, ctn *pipeline.Container) error { + // update engine logger with step metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("step", ctn.Name) + + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + logger.Debug("setting up container") + // setup the runtime container + err := c.Runtime.SetupContainer(ctx, ctn) + if err != nil { + return err + } + + // create a library step object to facilitate injecting environment as early as possible + // (PlanStep is too late to inject environment vars for the kubernetes runtime). + _step := c.newLibraryStep(ctn) + _step.SetStatus(constants.StatusPending) + + // update the step container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Environment + err = step.Environment(ctn, c.build, c.repo, _step, c.Version) + if err != nil { + return err + } + + logger.Debug("escaping newlines in secrets") + escapeNewlineSecrets(c.Secrets) + + logger.Debug("injecting secrets") + // inject secrets for container + err = injectSecrets(ctn, c.Secrets) + if err != nil { + return err + } + + logger.Debug("substituting container configuration") + // substitute container configuration + // + // https://pkg.go.dev/github.com/go-vela/types/pipeline#Container.Substitute + err = ctn.Substitute() + if err != nil { + return fmt.Errorf("unable to substitute container configuration") + } + + return nil +} + +// newLibraryStep creates a library step object. +func (c *client) newLibraryStep(ctn *pipeline.Container) *library.Step { + _step := new(library.Step) + _step.SetName(ctn.Name) + _step.SetNumber(ctn.Number) + _step.SetImage(ctn.Image) + _step.SetStage(ctn.Environment["VELA_STEP_STAGE"]) + _step.SetHost(c.build.GetHost()) + _step.SetRuntime(c.build.GetRuntime()) + _step.SetDistribution(c.build.GetDistribution()) + return _step +} + +// PlanStep prepares the step for execution. +func (c *client) PlanStep(ctx context.Context, ctn *pipeline.Container) error { + var err error + + // update engine logger with step metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("step", ctn.Name) + + // create the library step object + _step := c.newLibraryStep(ctn) + _step.SetStatus(constants.StatusRunning) + _step.SetStarted(time.Now().UTC().Unix()) + + logger.Debug("uploading step state") + // send API call to update the step + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#StepService.Update + _step, _, err = c.Vela.Step.Update(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), _step) + if err != nil { + return err + } + + // update the step container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Environment + err = step.Environment(ctn, c.build, c.repo, _step, c.Version) + if err != nil { + return err + } + + // add a step to a map + c.steps.Store(ctn.ID, _step) + + // get the step log here + logger.Debug("retrieve step log") + // send API call to capture the step log + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.GetStep + _log, _, err := c.Vela.Log.GetStep(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), _step.GetNumber()) + if err != nil { + return err + } + + // add a step log to a map + c.stepLogs.Store(ctn.ID, _log) + + return nil +} + +// ExecStep runs a step. +func (c *client) ExecStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // update engine logger with step metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("step", ctn.Name) + + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _step, err := step.Load(ctn, &c.steps) + if err != nil { + return err + } + + // defer taking a snapshot of the step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Snapshot + defer func() { step.Snapshot(ctn, c.build, c.Vela, c.logger, c.repo, _step) }() + + logger.Debug("running container") + // run the runtime container + err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) + if err != nil { + return err + } + + // create an error group with the parent context + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#WithContext + logs, logCtx := errgroup.WithContext(ctx) + + logs.Go(func() error { + logger.Debug("streaming logs for container") + // stream logs from container + err := c.StreamStep(logCtx, ctn) + if err != nil { + logger.Error(err) + } + + return nil + }) + + // do not wait for detached containers + if ctn.Detach { + return nil + } + + logger.Debug("waiting for container") + // wait for the runtime container + err = c.Runtime.WaitContainer(ctx, ctn) + if err != nil { + return err + } + + logger.Debug("inspecting container") + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} + +// StreamStep tails the output for a step. +func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // update engine logger with step metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("step", ctn.Name) + + // load the logs for the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadLogs + _log, err := step.LoadLogs(ctn, &c.stepLogs) + if err != nil { + return err + } + + // nolint: dupl // ignore similar code + defer func() { + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + logger.Errorf("unable to tail container output for upload: %v", err) + + return + } + defer rc.Close() + + // read all output from the runtime container + data, err := ioutil.ReadAll(rc) + if err != nil { + logger.Errorf("unable to read container output for upload: %v", err) + + return + } + + // overwrite the existing log with all bytes + // + // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.SetData + _log.SetData(data) + + logger.Debug("uploading logs") + // send API call to update the logs for the step + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#LogService.UpdateStep + _, _, err = c.Vela.Log.UpdateStep(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), ctn.Number, _log) + if err != nil { + logger.Errorf("unable to upload container logs: %v", err) + } + }() + + logger.Debug("tailing container") + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + return err + } + defer rc.Close() + + // set the timeout to the repo timeout + // to ensure the stream is not cut off + c.Vela.SetTimeout(time.Minute * time.Duration(c.repo.GetTimeout())) + + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#StepService.Stream + _, err = c.Vela.Step.Stream(c.repo.GetOrg(), c.repo.GetName(), c.build.GetNumber(), ctn.Number, rc) + if err != nil { + logger.Errorf("unable to stream logs: %v", err) + } + + logger.Info("finished streaming logs") + + return nil +} + +// DestroyStep cleans up steps after execution. +func (c *client) DestroyStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // update engine logger with step metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithField + logger := c.logger.WithField("step", ctn.Name) + + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _step, err := step.Load(ctn, &c.steps) + if err != nil { + // create the step from the container + // + // https://pkg.go.dev/github.com/go-vela/types/library#StepFromContainer + _step = library.StepFromContainer(ctn) + } + + // defer an upload of the step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload + defer func() { step.Upload(ctn, c.build, c.Vela, logger, c.repo, _step) }() + + logger.Debug("inspecting container") + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + logger.Debug("removing container") + // remove the runtime container + err = c.Runtime.RemoveContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} diff --git a/executor/linux/step_test.go b/executor/linux/step_test.go new file mode 100644 index 00000000..314ca550 --- /dev/null +++ b/executor/linux/step_test.go @@ -0,0 +1,515 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package linux + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLinux_CreateStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with image not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("CreateStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateStep returned err: %v", err) + } + } +} + +func TestLinux_PlanStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with nil environment + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: nil, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("PlanStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanStep returned err: %v", err) + } + } +} + +func TestLinux_ExecStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // detached step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with image not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.steps.Store(test.container.ID, new(library.Step)) + _engine.stepLogs.Store(test.container.ID, new(library.Log)) + } + + err = _engine.ExecStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("ExecStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecStep returned err: %v", err) + } + } +} + +func TestLinux_StreamStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + logs *library.Log + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with name not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_notfound", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "notfound", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.steps.Store(test.container.ID, new(library.Step)) + _engine.stepLogs.Store(test.container.ID, new(library.Log)) + } + + err = _engine.StreamStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("StreamStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("StreamStep returned err: %v", err) + } + } +} + +func TestLinux_DestroyStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with ignoring name not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_ignorenotfound", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "ignorenotfound", + Number: 1, + Pull: "not_present", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + WithVelaClient(_client), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("DestroyStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyStep returned err: %v", err) + } + } +} diff --git a/executor/linux/testdata/build/empty.yml b/executor/linux/testdata/build/empty.yml new file mode 100644 index 00000000..73b314ff --- /dev/null +++ b/executor/linux/testdata/build/empty.yml @@ -0,0 +1 @@ +--- \ No newline at end of file diff --git a/executor/linux/testdata/build/secrets/basic.yml b/executor/linux/testdata/build/secrets/basic.yml new file mode 100644 index 00000000..27a2336a --- /dev/null +++ b/executor/linux/testdata/build/secrets/basic.yml @@ -0,0 +1,23 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foob + origin: + name: vault + environment: + FOO: bar + image: vault:latest + parameters: + foo: bar + pull: true + + \ No newline at end of file diff --git a/executor/linux/testdata/build/secrets/img_ignorenotfound.yml b/executor/linux/testdata/build/secrets/img_ignorenotfound.yml new file mode 100644 index 00000000..99172227 --- /dev/null +++ b/executor/linux/testdata/build/secrets/img_ignorenotfound.yml @@ -0,0 +1,23 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foob + origin: + name: vault + environment: + FOO: bar + image: vault:ignorenotfound + parameters: + foo: bar + pull: true + + \ No newline at end of file diff --git a/executor/linux/testdata/build/secrets/img_notfound.yml b/executor/linux/testdata/build/secrets/img_notfound.yml new file mode 100644 index 00000000..9107fa9e --- /dev/null +++ b/executor/linux/testdata/build/secrets/img_notfound.yml @@ -0,0 +1,23 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foob + origin: + name: vault + environment: + FOO: bar + image: vault:notfound + parameters: + foo: bar + pull: true + + \ No newline at end of file diff --git a/executor/linux/testdata/build/secrets/name_notfound.yml b/executor/linux/testdata/build/secrets/name_notfound.yml new file mode 100644 index 00000000..69178bc3 --- /dev/null +++ b/executor/linux/testdata/build/secrets/name_notfound.yml @@ -0,0 +1,23 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foob + origin: + name: notfound + environment: + FOO: bar + image: vault:latest + parameters: + foo: bar + pull: true + + \ No newline at end of file diff --git a/executor/linux/testdata/build/services/basic.yml b/executor/linux/testdata/build/services/basic.yml new file mode 100644 index 00000000..0c0f8699 --- /dev/null +++ b/executor/linux/testdata/build/services/basic.yml @@ -0,0 +1,18 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:latest + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/services/img_ignorenotfound.yml b/executor/linux/testdata/build/services/img_ignorenotfound.yml new file mode 100644 index 00000000..324248ca --- /dev/null +++ b/executor/linux/testdata/build/services/img_ignorenotfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:ignorenotfound + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/linux/testdata/build/services/img_notfound.yml b/executor/linux/testdata/build/services/img_notfound.yml new file mode 100644 index 00000000..5378fe7f --- /dev/null +++ b/executor/linux/testdata/build/services/img_notfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:notfound + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/linux/testdata/build/services/name_notfound.yml b/executor/linux/testdata/build/services/name_notfound.yml new file mode 100644 index 00000000..3dd1998b --- /dev/null +++ b/executor/linux/testdata/build/services/name_notfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: notfound + environment: + FOO: bar + image: postgres:latest + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/linux/testdata/build/stages/basic.yml b/executor/linux/testdata/build/stages/basic.yml new file mode 100644 index 00000000..f49e1750 --- /dev/null +++ b/executor/linux/testdata/build/stages/basic.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/stages/img_ignorenotfound.yml b/executor/linux/testdata/build/stages/img_ignorenotfound.yml new file mode 100644 index 00000000..e261e316 --- /dev/null +++ b/executor/linux/testdata/build/stages/img_ignorenotfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:ignorenotfound + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/stages/img_notfound.yml b/executor/linux/testdata/build/stages/img_notfound.yml new file mode 100644 index 00000000..1639a4f6 --- /dev/null +++ b/executor/linux/testdata/build/stages/img_notfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:notfound + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/stages/name_notfound.yml b/executor/linux/testdata/build/stages/name_notfound.yml new file mode 100644 index 00000000..69216319 --- /dev/null +++ b/executor/linux/testdata/build/stages/name_notfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: notfound + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/steps/basic.yml b/executor/linux/testdata/build/steps/basic.yml new file mode 100644 index 00000000..10852530 --- /dev/null +++ b/executor/linux/testdata/build/steps/basic.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/steps/img_ignorenotfound.yml b/executor/linux/testdata/build/steps/img_ignorenotfound.yml new file mode 100644 index 00000000..539fac96 --- /dev/null +++ b/executor/linux/testdata/build/steps/img_ignorenotfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:ignorenotfound + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/steps/img_notfound.yml b/executor/linux/testdata/build/steps/img_notfound.yml new file mode 100644 index 00000000..20d1b53a --- /dev/null +++ b/executor/linux/testdata/build/steps/img_notfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:notfound + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/build/steps/name_notfound.yml b/executor/linux/testdata/build/steps/name_notfound.yml new file mode 100644 index 00000000..735fce7c --- /dev/null +++ b/executor/linux/testdata/build/steps/name_notfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: notfound + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/linux/testdata/secret/basic.yml b/executor/linux/testdata/secret/basic.yml new file mode 100644 index 00000000..80b57c5f --- /dev/null +++ b/executor/linux/testdata/secret/basic.yml @@ -0,0 +1,25 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foo + + - name: foob + origin: + name: vault + environment: + FOO: bar + image: vault:latest + parameters: + addr: vault.company.com + pull: true + + \ No newline at end of file diff --git a/executor/linux/testdata/secret/name_notfound.yml b/executor/linux/testdata/secret/name_notfound.yml new file mode 100644 index 00000000..c7e5cdec --- /dev/null +++ b/executor/linux/testdata/secret/name_notfound.yml @@ -0,0 +1,25 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + +secrets: + - name: foo + + - name: foob + origin: + name: notfound + environment: + FOO: bar + image: vault:latest + parameters: + foo: bar + pull: true + + \ No newline at end of file diff --git a/executor/local/api.go b/executor/local/api.go new file mode 100644 index 00000000..0ed25483 --- /dev/null +++ b/executor/local/api.go @@ -0,0 +1,200 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/service" + "github.com/go-vela/worker/internal/step" +) + +// GetBuild gets the current build in execution. +func (c *client) GetBuild() (*library.Build, error) { + // check if the build resource is available + if c.build == nil { + return nil, fmt.Errorf("build resource not found") + } + + return c.build, nil +} + +// GetPipeline gets the current pipeline in execution. +func (c *client) GetPipeline() (*pipeline.Build, error) { + // check if the pipeline resource is available + if c.pipeline == nil { + return nil, fmt.Errorf("pipeline resource not found") + } + + return c.pipeline, nil +} + +// GetRepo gets the current repo in execution. +func (c *client) GetRepo() (*library.Repo, error) { + // check if the repo resource is available + if c.repo == nil { + return nil, fmt.Errorf("repo resource not found") + } + + return c.repo, nil +} + +// CancelBuild cancels the current build in execution. +// nolint: funlen // process of going through steps/services/stages is verbose and could be funcitonalized +func (c *client) CancelBuild() (*library.Build, error) { + // get the current build from the client + b, err := c.GetBuild() + if err != nil { + return nil, err + } + + // set the build status to canceled + b.SetStatus(constants.StatusCanceled) + + // get the current pipeline from the client + pipeline, err := c.GetPipeline() + if err != nil { + return nil, err + } + + // cancel non successful services + // nolint: dupl // false positive, steps/services are different + for _, _service := range pipeline.Services { + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + s, err := service.Load(_service, &c.services) + if err != nil { + // create the library service object + s = new(library.Service) + s.SetName(_service.Name) + s.SetNumber(_service.Number) + s.SetImage(_service.Image) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if service state was not terminal, set it as canceled + switch s.GetStatus() { + // service is in a error state + case constants.StatusError: + break + // service is in a failure state + case constants.StatusFailure: + break + // service is in a killed state + case constants.StatusKilled: + break + // service is in a success state + case constants.StatusSuccess: + break + default: + // update the service with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a service to a map + c.services.Store(_service.ID, s) + } + } + + // cancel non successful steps + // nolint: dupl // false positive, steps/services are different + for _, _step := range pipeline.Steps { + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + s, err := step.Load(_step, &c.steps) + if err != nil { + // create the library step object + s = new(library.Step) + s.SetName(_step.Name) + s.SetNumber(_step.Number) + s.SetImage(_step.Image) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if step state was not terminal, set it as canceled + switch s.GetStatus() { + // step is in a error state + case constants.StatusError: + break + // step is in a failure state + case constants.StatusFailure: + break + // step is in a killed state + case constants.StatusKilled: + break + // step is in a success state + case constants.StatusSuccess: + break + default: + // update the step with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a step to a map + c.steps.Store(_step.ID, s) + } + } + + // cancel non successful stages + for _, _stage := range pipeline.Stages { + // cancel non successful steps for that stage + for _, _step := range _stage.Steps { + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + s, err := step.Load(_step, &c.steps) + if err != nil { + // create the library step object + s = new(library.Step) + s.SetName(_step.Name) + s.SetNumber(_step.Number) + s.SetImage(_step.Image) + s.SetStage(_stage.Name) + s.SetStarted(time.Now().UTC().Unix()) + s.SetHost(c.build.GetHost()) + s.SetRuntime(c.build.GetRuntime()) + s.SetDistribution(c.build.GetDistribution()) + } + + // if stage state was not terminal, set it as canceled + switch s.GetStatus() { + // stage is in a error state + case constants.StatusError: + break + // stage is in a failure state + case constants.StatusFailure: + break + // stage is in a killed state + case constants.StatusKilled: + break + // stage is in a success state + case constants.StatusSuccess: + break + default: + // update the step with a canceled state + s.SetStatus(constants.StatusCanceled) + // add a step to a map + c.steps.Store(_step.ID, s) + } + } + } + + err = c.DestroyBuild(context.Background()) + if err != nil { + fmt.Fprintln(os.Stdout, "unable to destroy build:", err) + } + + return b, nil +} diff --git a/executor/local/api_test.go b/executor/local/api_test.go new file mode 100644 index 00000000..97ebc3ab --- /dev/null +++ b/executor/local/api_test.go @@ -0,0 +1,154 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "reflect" + "testing" +) + +func TestLocal_GetBuild(t *testing.T) { + // setup types + _build := testBuild() + + _engine, err := New( + WithBuild(_build), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetBuild() + + if test.failure { + if err == nil { + t.Errorf("GetBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetBuild returned err: %v", err) + } + + if !reflect.DeepEqual(got, _build) { + t.Errorf("GetBuild is %v, want %v", got, _build) + } + } +} + +func TestLocal_GetPipeline(t *testing.T) { + // setup types + _steps := testSteps() + + _engine, err := New( + WithPipeline(_steps), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetPipeline() + + if test.failure { + if err == nil { + t.Errorf("GetPipeline should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetPipeline returned err: %v", err) + } + + if !reflect.DeepEqual(got, _steps) { + t.Errorf("GetPipeline is %v, want %v", got, _steps) + } + } +} + +func TestLocal_GetRepo(t *testing.T) { + // setup types + _repo := testRepo() + + _engine, err := New( + WithRepo(_repo), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + engine *client + }{ + { + failure: false, + engine: _engine, + }, + { + failure: true, + engine: new(client), + }, + } + + // run tests + for _, test := range tests { + got, err := test.engine.GetRepo() + + if test.failure { + if err == nil { + t.Errorf("GetRepo should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("GetRepo returned err: %v", err) + } + + if !reflect.DeepEqual(got, _repo) { + t.Errorf("GetRepo is %v, want %v", got, _repo) + } + } +} diff --git a/executor/local/build.go b/executor/local/build.go new file mode 100644 index 00000000..0511dacb --- /dev/null +++ b/executor/local/build.go @@ -0,0 +1,417 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/go-vela/types/constants" + "github.com/go-vela/worker/internal/build" + "github.com/go-vela/worker/internal/step" +) + +// CreateBuild configures the build for execution. +func (c *client) CreateBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, nil, c.err, nil, nil) }() + + // update the build fields + c.build.SetStatus(constants.StatusRunning) + c.build.SetStarted(time.Now().UTC().Unix()) + c.build.SetHost(c.Hostname) + c.build.SetDistribution(c.Driver()) + c.build.SetRuntime(c.Runtime.Driver()) + + // setup the runtime build + c.err = c.Runtime.SetupBuild(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to setup build %s: %w", c.pipeline.ID, c.err) + } + + // load the init step from the pipeline + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#LoadInit + c.init, c.err = step.LoadInit(c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to load init step from pipeline: %w", c.err) + } + + // create the step + c.err = c.CreateStep(ctx, c.init) + if c.err != nil { + return fmt.Errorf("unable to create %s step: %w", c.init.Name, c.err) + } + + // plan the step + c.err = c.PlanStep(ctx, c.init) + if c.err != nil { + return fmt.Errorf("unable to plan %s step: %w", c.init.Name, c.err) + } + + return c.err +} + +// PlanBuild prepares the build for execution. +func (c *client) PlanBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, nil, c.err, nil, nil) }() + + // load the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _init, err := step.Load(c.init, &c.steps) + if err != nil { + return err + } + + // defer taking a snapshot of the init step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#SnapshotInit + defer func() { step.SnapshotInit(c.init, c.build, nil, nil, nil, _init, nil) }() + + // create a step pattern for log output + _pattern := fmt.Sprintf(stepPattern, c.init.Name) + + // check if the pipeline provided has stages + if len(c.pipeline.Stages) > 0 { + // create a stage pattern for log output + _pattern = fmt.Sprintf(stagePattern, c.init.Name, c.init.Name) + } + + // create the runtime network for the pipeline + c.err = c.Runtime.CreateNetwork(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to create network: %w", c.err) + } + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Inspecting runtime network...") + + // inspect the runtime network for the pipeline + network, err := c.Runtime.InspectNetwork(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect network: %w", err) + } + + // output the network information to stdout + fmt.Fprintln(os.Stdout, _pattern, string(network)) + + // create the runtime volume for the pipeline + err = c.Runtime.CreateVolume(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to create volume: %w", err) + } + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Inspecting runtime volume...") + + // inspect the runtime volume for the pipeline + volume, err := c.Runtime.InspectVolume(ctx, c.pipeline) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect volume: %w", err) + } + + // output the volume information to stdout + fmt.Fprintln(os.Stdout, _pattern, string(volume)) + + return c.err +} + +// AssembleBuild prepares the containers within a build for execution. +// +// nolint: funlen // ignore function length due to comments +func (c *client) AssembleBuild(ctx context.Context) error { + // defer taking a snapshot of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot + defer func() { build.Snapshot(c.build, nil, c.err, nil, nil) }() + + // load the init step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _init, err := step.Load(c.init, &c.steps) + if err != nil { + return err + } + + // defer an upload of the init step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload + defer func() { step.Upload(c.init, c.build, nil, nil, nil, _init) }() + + // create a step pattern for log output + _pattern := fmt.Sprintf(stepPattern, c.init.Name) + + // check if the pipeline provided has stages + if len(c.pipeline.Stages) > 0 { + // create a stage pattern for log output + _pattern = fmt.Sprintf(stagePattern, c.init.Name, c.init.Name) + } + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Pulling service images...") + + // create the services for the pipeline + for _, _service := range c.pipeline.Services { + // TODO: remove this; but we need it for tests + _service.Detach = true + + // create the service + c.err = c.CreateService(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to create %s service: %w", _service.Name, c.err) + } + + // inspect the service image + image, err := c.Runtime.InspectImage(ctx, _service) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect %s service: %w", _service.Name, err) + } + + // output the image information to stdout + fmt.Fprintln(os.Stdout, _pattern, string(image)) + } + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Pulling stage images...") + + // create the stages for the pipeline + for _, _stage := range c.pipeline.Stages { + // TODO: remove hardcoded reference + // + // nolint: goconst // ignore making a constant for now + if _stage.Name == "init" { + continue + } + + // create the stage + c.err = c.CreateStage(ctx, _stage) + if c.err != nil { + return fmt.Errorf("unable to create %s stage: %w", _stage.Name, c.err) + } + } + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Pulling step images...") + + // create the steps for the pipeline + for _, _step := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if _step.Name == "init" { + continue + } + + // create the step + c.err = c.CreateStep(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to create %s step: %w", _step.Name, c.err) + } + + // inspect the step image + image, err := c.Runtime.InspectImage(ctx, _step) + if err != nil { + c.err = err + return fmt.Errorf("unable to inspect %s step: %w", _step.Name, err) + } + + // output the image information to stdout + fmt.Fprintln(os.Stdout, _pattern, string(image)) + } + + // output a new line for readability to stdout + fmt.Fprintln(os.Stdout, "") + + // assemble runtime build just before any containers execute + c.err = c.Runtime.AssembleBuild(ctx, c.pipeline) + if c.err != nil { + return fmt.Errorf("unable to assemble runtime build %s: %w", c.pipeline.ID, c.err) + } + + return c.err +} + +// ExecBuild runs a pipeline for a build. +func (c *client) ExecBuild(ctx context.Context) error { + // defer an upload of the build + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Upload + defer func() { build.Upload(c.build, nil, c.err, nil, nil) }() + + // execute the services for the pipeline + for _, _service := range c.pipeline.Services { + // plan the service + c.err = c.PlanService(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to plan service: %w", c.err) + } + + // execute the service + c.err = c.ExecService(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to execute service: %w", c.err) + } + } + + // execute the steps for the pipeline + for _, _step := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if _step.Name == "init" { + continue + } + + // check if the step should be skipped + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip + if step.Skip(_step, c.build, c.repo) { + continue + } + + // plan the step + c.err = c.PlanStep(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to plan step: %w", c.err) + } + + // execute the step + c.err = c.ExecStep(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to execute step: %w", c.err) + } + } + + // create an error group with the context for each stage + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#WithContext + stages, stageCtx := errgroup.WithContext(ctx) + // create a map to track the progress of each stage + stageMap := new(sync.Map) + + // iterate through each stage in the pipeline + for _, _stage := range c.pipeline.Stages { + // TODO: remove hardcoded reference + if _stage.Name == "init" { + continue + } + + // https://golang.org/doc/faq#closures_and_goroutines + stage := _stage + + // create a new channel for each stage in the map + stageMap.Store(stage.Name, make(chan error)) + + // spawn errgroup routine for the stage + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#Group.Go + stages.Go(func() error { + // plan the stage + c.err = c.PlanStage(stageCtx, stage, stageMap) + if c.err != nil { + return fmt.Errorf("unable to plan stage: %w", c.err) + } + + // execute the stage + c.err = c.ExecStage(stageCtx, stage, stageMap) + if c.err != nil { + return fmt.Errorf("unable to execute stage: %w", c.err) + } + + return nil + }) + } + + // wait for the stages to complete or return an error + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#Group.Wait + c.err = stages.Wait() + if c.err != nil { + return fmt.Errorf("unable to wait for stages: %v", c.err) + } + + return c.err +} + +// DestroyBuild cleans up the build after execution. +func (c *client) DestroyBuild(ctx context.Context) error { + var err error + + defer func() { + // remove the runtime build for the pipeline + err = c.Runtime.RemoveBuild(ctx, c.pipeline) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy runtime build:", err) + } + }() + + // destroy the steps for the pipeline + for _, _step := range c.pipeline.Steps { + // TODO: remove hardcoded reference + if _step.Name == "init" { + continue + } + + // destroy the step + err = c.DestroyStep(ctx, _step) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy step:", err) + } + } + + // destroy the stages for the pipeline + for _, _stage := range c.pipeline.Stages { + // TODO: remove hardcoded reference + if _stage.Name == "init" { + continue + } + + // destroy the stage + err = c.DestroyStage(ctx, _stage) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy stage:", err) + } + } + + // destroy the services for the pipeline + for _, _service := range c.pipeline.Services { + // destroy the service + err = c.DestroyService(ctx, _service) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy service:", err) + } + } + + // remove the runtime volume for the pipeline + err = c.Runtime.RemoveVolume(ctx, c.pipeline) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy runtime volume:", err) + } + + // remove the runtime network for the pipeline + err = c.Runtime.RemoveNetwork(ctx, c.pipeline) + if err != nil { + // output the error information to stdout + fmt.Fprintln(os.Stdout, "unable to destroy runtime network:", err) + } + + return err +} diff --git a/executor/local/build_test.go b/executor/local/build_test.go new file mode 100644 index 00000000..527f1748 --- /dev/null +++ b/executor/local/build_test.go @@ -0,0 +1,438 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "flag" + "testing" + + "github.com/go-vela/compiler/compiler/native" + "github.com/urfave/cli/v2" + + "github.com/go-vela/pkg-runtime/runtime/docker" +) + +func TestLocal_CreateBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("CreateBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateBuild returned err: %v", err) + } + } +} + +func TestLocal_PlanBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.PlanBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("PlanBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanBuild returned err: %v", err) + } + } +} + +func TestLocal_AssembleBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with image not found + failure: true, + pipeline: "testdata/build/services/img_notfound.yml", + }, + { // services pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/services/img_ignorenotfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with image not found + failure: true, + pipeline: "testdata/build/steps/img_notfound.yml", + }, + { // steps pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/steps/img_ignorenotfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with image not found + failure: true, + pipeline: "testdata/build/stages/img_notfound.yml", + }, + { // stages pipeline with ignoring image not found + failure: true, + pipeline: "testdata/build/stages/img_ignorenotfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.AssembleBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("AssembleBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("AssembleBuild returned err: %v", err) + } + } +} + +func TestLocal_ExecBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with image not found + failure: true, + pipeline: "testdata/build/services/img_notfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with image not found + failure: true, + pipeline: "testdata/build/steps/img_notfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with image not found + failure: true, + pipeline: "testdata/build/stages/img_notfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.ExecBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("ExecBuild for %s should have returned err", test.pipeline) + } + + continue + } + + if err != nil { + t.Errorf("ExecBuild for %s returned err: %v", test.pipeline, err) + } + } +} + +func TestLocal_DestroyBuild(t *testing.T) { + // setup types + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + tests := []struct { + failure bool + pipeline string + }{ + { // basic services pipeline + failure: false, + pipeline: "testdata/build/services/basic.yml", + }, + { // services pipeline with name not found + failure: false, + pipeline: "testdata/build/services/name_notfound.yml", + }, + { // basic steps pipeline + failure: false, + pipeline: "testdata/build/steps/basic.yml", + }, + { // steps pipeline with name not found + failure: false, + pipeline: "testdata/build/steps/name_notfound.yml", + }, + { // basic stages pipeline + failure: false, + pipeline: "testdata/build/stages/basic.yml", + }, + { // stages pipeline with name not found + failure: false, + pipeline: "testdata/build/stages/name_notfound.yml", + }, + } + + // run test + for _, test := range tests { + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(test.pipeline) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", test.pipeline, err) + } + + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.DestroyBuild(context.Background()) + + if test.failure { + if err == nil { + t.Errorf("DestroyBuild should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyBuild returned err: %v", err) + } + } +} diff --git a/executor/local/doc.go b/executor/local/doc.go new file mode 100644 index 00000000..0f487a8f --- /dev/null +++ b/executor/local/doc.go @@ -0,0 +1,11 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package local provides the ability for Vela to +// integrate with the local system. +// +// Usage: +// +// import "github.com/go-vela/worker/executor/local" +package local diff --git a/executor/local/driver.go b/executor/local/driver.go new file mode 100644 index 00000000..e02eb0f1 --- /dev/null +++ b/executor/local/driver.go @@ -0,0 +1,12 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import "github.com/go-vela/types/constants" + +// Driver outputs the configured executor driver. +func (c *client) Driver() string { + return constants.DriverLocal +} diff --git a/executor/local/driver_test.go b/executor/local/driver_test.go new file mode 100644 index 00000000..23a43c92 --- /dev/null +++ b/executor/local/driver_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "reflect" + "testing" + + "github.com/go-vela/pkg-runtime/runtime/docker" + "github.com/go-vela/types/constants" +) + +func TestLocal_Driver(t *testing.T) { + // setup types + want := constants.DriverLocal + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _engine, err := New( + WithBuild(testBuild()), + WithHostname("localhost"), + WithPipeline(testSteps()), + WithRuntime(_runtime), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run tes + got := _engine.Driver() + + if !reflect.DeepEqual(got, want) { + t.Errorf("Driver is %v, want %v", got, want) + } +} diff --git a/executor/local/local.go b/executor/local/local.go new file mode 100644 index 00000000..67668744 --- /dev/null +++ b/executor/local/local.go @@ -0,0 +1,52 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "sync" + + "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/sdk-go/vela" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +type ( + // client manages communication with the pipeline resources. + client struct { + Vela *vela.Client + Runtime runtime.Engine + Hostname string + Version string + + // private fields + init *pipeline.Container + build *library.Build + pipeline *pipeline.Build + repo *library.Repo + services sync.Map + steps sync.Map + user *library.User + err error + } +) + +// New returns an Executor implementation that integrates with the local system. +// +// nolint: golint // ignore unexported type as it is intentional +func New(opts ...Opt) (*client, error) { + // create new local client + c := new(client) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(c) + if err != nil { + return nil, err + } + } + + return c, nil +} diff --git a/executor/local/local_test.go b/executor/local/local_test.go new file mode 100644 index 00000000..b21ed525 --- /dev/null +++ b/executor/local/local_test.go @@ -0,0 +1,220 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLocal_New(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + pipeline *pipeline.Build + }{ + { + failure: false, + pipeline: testSteps(), + }, + { + failure: true, + pipeline: nil, + }, + } + + // run tests + for _, test := range tests { + _, err := New( + WithBuild(testBuild()), + WithHostname("localhost"), + WithPipeline(test.pipeline), + WithRepo(testRepo()), + WithRuntime(_runtime), + WithUser(testUser()), + WithVelaClient(_client), + ) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +} + +// testBuild is a test helper function to create a Build +// type with all fields set to a fake value. +func testBuild() *library.Build { + return &library.Build{ + ID: vela.Int64(1), + Number: vela.Int(1), + Parent: vela.Int(1), + Event: vela.String("push"), + Status: vela.String("success"), + Error: vela.String(""), + Enqueued: vela.Int64(1563474077), + Created: vela.Int64(1563474076), + Started: vela.Int64(1563474077), + Finished: vela.Int64(0), + Deploy: vela.String(""), + Clone: vela.String("https://github.com/github/octocat.git"), + Source: vela.String("https://github.com/github/octocat/abcdefghi123456789"), + Title: vela.String("push received from https://github.com/github/octocat"), + Message: vela.String("First commit..."), + Commit: vela.String("48afb5bdc41ad69bf22588491333f7cf71135163"), + Sender: vela.String("OctoKitty"), + Author: vela.String("OctoKitty"), + Branch: vela.String("master"), + Ref: vela.String("refs/heads/master"), + BaseRef: vela.String(""), + Host: vela.String("example.company.com"), + Runtime: vela.String("docker"), + Distribution: vela.String("Local"), + } +} + +// testRepo is a test helper function to create a Repo +// type with all fields set to a fake value. +func testRepo() *library.Repo { + return &library.Repo{ + ID: vela.Int64(1), + Org: vela.String("github"), + Name: vela.String("octocat"), + FullName: vela.String("github/octocat"), + Link: vela.String("https://github.com/github/octocat"), + Clone: vela.String("https://github.com/github/octocat.git"), + Branch: vela.String("master"), + Timeout: vela.Int64(60), + Visibility: vela.String("public"), + Private: vela.Bool(false), + Trusted: vela.Bool(false), + Active: vela.Bool(true), + AllowPull: vela.Bool(false), + AllowPush: vela.Bool(true), + AllowDeploy: vela.Bool(false), + AllowTag: vela.Bool(false), + } +} + +// testUser is a test helper function to create a User +// type with all fields set to a fake value. +func testUser() *library.User { + return &library.User{ + ID: vela.Int64(1), + Name: vela.String("octocat"), + Token: vela.String("superSecretToken"), + Hash: vela.String("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy"), + Favorites: vela.Strings([]string{"github/octocat"}), + Active: vela.Bool(true), + Admin: vela.Bool(false), + } +} + +// testSteps is a test helper function to create a steps +// pipeline with fake steps. +func testSteps() *pipeline.Build { + return &pipeline.Build{ + Version: "1", + ID: "github_octocat_1", + Services: pipeline.ContainerSlice{ + { + ID: "service_github_octocat_1_postgres", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + Steps: pipeline.ContainerSlice{ + { + ID: "step_github_octocat_1_init", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "always", + }, + { + ID: "step_github_octocat_1_clone", + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-git:v0.3.0", + Name: "clone", + Number: 2, + Pull: "always", + }, + { + ID: "step_github_octocat_1_echo", + Commands: []string{"echo hello"}, + Directory: "/home/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 3, + Pull: "always", + }, + }, + Secrets: pipeline.SecretSlice{ + { + Name: "foo", + Key: "github/octocat/foo", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + }, + { + Name: "foo", + Key: "github/foo", + Engine: "native", + Type: "org", + Origin: &pipeline.Container{}, + }, + { + Name: "foo", + Key: "github/octokitties/foo", + Engine: "native", + Type: "shared", + Origin: &pipeline.Container{}, + }, + }, + } +} diff --git a/executor/local/opts.go b/executor/local/opts.go new file mode 100644 index 00000000..b3e69be2 --- /dev/null +++ b/executor/local/opts.go @@ -0,0 +1,121 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "fmt" + + "github.com/go-vela/pkg-runtime/runtime" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// Opt represents a configuration option to initialize the client. +type Opt func(*client) error + +// WithBuild sets the library build in the client. +func WithBuild(b *library.Build) Opt { + return func(c *client) error { + // set the build in the client + c.build = b + + return nil + } +} + +// WithHostname sets the hostname in the client. +func WithHostname(hostname string) Opt { + return func(c *client) error { + // check if a hostname is provided + if len(hostname) == 0 { + // default the hostname to localhost + hostname = "localhost" + } + + // set the hostname in the client + c.Hostname = hostname + + return nil + } +} + +// WithPipeline sets the pipeline build in the client. +func WithPipeline(p *pipeline.Build) Opt { + return func(c *client) error { + // check if the pipeline provided is empty + if p == nil { + return fmt.Errorf("empty pipeline provided") + } + + // set the pipeline in the client + c.pipeline = p + + return nil + } +} + +// WithRepo sets the library repo in the client. +func WithRepo(r *library.Repo) Opt { + return func(c *client) error { + // set the repo in the client + c.repo = r + + return nil + } +} + +// WithRuntime sets the runtime engine in the client. +func WithRuntime(r runtime.Engine) Opt { + return func(c *client) error { + // check if the runtime provided is empty + if r == nil { + return fmt.Errorf("empty runtime provided") + } + + // set the runtime in the client + c.Runtime = r + + return nil + } +} + +// WithUser sets the library user in the client. +func WithUser(u *library.User) Opt { + return func(c *client) error { + // set the user in the client + c.user = u + + return nil + } +} + +// WithVelaClient sets the Vela client in the client. +func WithVelaClient(cli *vela.Client) Opt { + return func(c *client) error { + // set the Vela client in the client + c.Vela = cli + + return nil + } +} + +// WithVersion sets the version in the client. +func WithVersion(version string) Opt { + return func(c *client) error { + // check if a version is provided + if len(version) == 0 { + // default the version + version = "v0.0.0" + } + + // set the version in the client + c.Version = version + + return nil + } +} diff --git a/executor/local/opts_test.go b/executor/local/opts_test.go new file mode 100644 index 00000000..2d5b57ee --- /dev/null +++ b/executor/local/opts_test.go @@ -0,0 +1,297 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/pkg-runtime/runtime" + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLocal_Opt_WithBuild(t *testing.T) { + // setup types + _build := testBuild() + + // setup tests + tests := []struct { + build *library.Build + }{ + { + build: _build, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(test.build), + ) + + if err != nil { + t.Errorf("WithBuild returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.build, _build) { + t.Errorf("WithBuild is %v, want %v", _engine.build, _build) + } + } +} + +func TestLocal_Opt_WithHostname(t *testing.T) { + // setup tests + tests := []struct { + hostname string + want string + }{ + { + hostname: "vela.worker.localhost", + want: "vela.worker.localhost", + }, + { + hostname: "", + want: "localhost", + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithHostname(test.hostname), + ) + if err != nil { + t.Errorf("unable to create local engine: %v", err) + } + + if !reflect.DeepEqual(_engine.Hostname, test.want) { + t.Errorf("WithHostname is %v, want %v", _engine.Hostname, test.want) + } + } +} + +func TestLocal_Opt_WithPipeline(t *testing.T) { + // setup types + _steps := testSteps() + + // setup tests + tests := []struct { + failure bool + pipeline *pipeline.Build + }{ + { + failure: false, + pipeline: _steps, + }, + { + failure: true, + pipeline: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithPipeline(test.pipeline), + ) + + if test.failure { + if err == nil { + t.Errorf("WithPipeline should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithPipeline returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.pipeline, _steps) { + t.Errorf("WithPipeline is %v, want %v", _engine.pipeline, _steps) + } + } +} + +func TestLocal_Opt_WithRepo(t *testing.T) { + // setup types + _repo := testRepo() + + // setup tests + tests := []struct { + repo *library.Repo + }{ + { + repo: _repo, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithRepo(test.repo), + ) + + if err != nil { + t.Errorf("WithRepo returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.repo, _repo) { + t.Errorf("WithRepo is %v, want %v", _engine.repo, _repo) + } + } +} + +func TestLocal_Opt_WithRuntime(t *testing.T) { + // setup types + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + runtime runtime.Engine + }{ + { + failure: false, + runtime: _runtime, + }, + { + failure: true, + runtime: nil, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithRuntime(test.runtime), + ) + + if test.failure { + if err == nil { + t.Errorf("WithRuntime should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithRuntime returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.Runtime, _runtime) { + t.Errorf("WithRuntime is %v, want %v", _engine.Runtime, _runtime) + } + } +} + +func TestLocal_Opt_WithUser(t *testing.T) { + // setup types + _user := testUser() + + // setup tests + tests := []struct { + user *library.User + }{ + { + user: _user, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithUser(test.user), + ) + + if err != nil { + t.Errorf("WithUser returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.user, _user) { + t.Errorf("WithUser is %v, want %v", _engine.user, _user) + } + } +} + +func TestLocal_Opt_WithVelaClient(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + // setup tests + tests := []struct { + client *vela.Client + }{ + { + client: _client, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithVelaClient(test.client), + ) + + if err != nil { + t.Errorf("WithVelaClient returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.Vela, _client) { + t.Errorf("WithVelaClient is %v, want %v", _engine.Vela, _client) + } + } +} + +func TestLocal_Opt_WithVersion(t *testing.T) { + // setup tests + tests := []struct { + version string + want string + }{ + { + version: "v1.0.0", + want: "v1.0.0", + }, + { + version: "", + want: "v0.0.0", + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithVersion(test.version), + ) + if err != nil { + t.Errorf("unable to create local engine: %v", err) + } + + if !reflect.DeepEqual(_engine.Version, test.want) { + t.Errorf("WithVersion is %v, want %v", _engine.Version, test.want) + } + } +} diff --git a/executor/local/service.go b/executor/local/service.go new file mode 100644 index 00000000..7248502f --- /dev/null +++ b/executor/local/service.go @@ -0,0 +1,165 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "bufio" + "context" + "fmt" + "os" + "time" + + "github.com/go-vela/worker/internal/service" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// create a service logging pattern. +const servicePattern = "[service: %s]" + +// CreateService configures the service for execution. +func (c *client) CreateService(ctx context.Context, ctn *pipeline.Container) error { + // setup the runtime container + err := c.Runtime.SetupContainer(ctx, ctn) + if err != nil { + return err + } + + // update the service container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Environment + err = service.Environment(ctn, c.build, c.repo, nil, c.Version) + if err != nil { + return err + } + + // substitute container configuration + // + // https://pkg.go.dev/github.com/go-vela/types/pipeline#Container.Substitute + err = ctn.Substitute() + if err != nil { + return err + } + + return nil +} + +// PlanService prepares the service for execution. +func (c *client) PlanService(ctx context.Context, ctn *pipeline.Container) error { + // update the engine service object + _service := new(library.Service) + _service.SetName(ctn.Name) + _service.SetNumber(ctn.Number) + _service.SetImage(ctn.Image) + _service.SetStatus(constants.StatusRunning) + _service.SetStarted(time.Now().UTC().Unix()) + _service.SetHost(c.build.GetHost()) + _service.SetRuntime(c.build.GetRuntime()) + _service.SetDistribution(c.build.GetDistribution()) + + // update the service container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Environment + err := service.Environment(ctn, c.build, c.repo, _service, c.Version) + if err != nil { + return err + } + + // add a service to a map + c.services.Store(ctn.ID, _service) + + return nil +} + +// ExecService runs a service. +func (c *client) ExecService(ctx context.Context, ctn *pipeline.Container) error { + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + _service, err := service.Load(ctn, &c.services) + if err != nil { + return err + } + + // defer taking a snapshot of the service + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Snapshot + defer func() { service.Snapshot(ctn, c.build, nil, nil, nil, _service) }() + + // run the runtime container + err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) + if err != nil { + return err + } + + go func() { + // stream logs from container + err := c.StreamService(context.Background(), ctn) + if err != nil { + fmt.Fprintln(os.Stdout, "unable to stream logs for service:", err) + } + }() + + return nil +} + +// StreamService tails the output for a service. +func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) error { + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + return err + } + defer rc.Close() + + // create a service pattern for log output + _pattern := fmt.Sprintf(servicePattern, ctn.Name) + + // create new scanner from the container output + scanner := bufio.NewScanner(rc) + + // scan entire container output + for scanner.Scan() { + // ensure we output to stdout + fmt.Fprintln(os.Stdout, _pattern, scanner.Text()) + } + + return scanner.Err() +} + +// DestroyService cleans up services after execution. +func (c *client) DestroyService(ctx context.Context, ctn *pipeline.Container) error { + // load the service from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load + _service, err := service.Load(ctn, &c.services) + if err != nil { + // create the service from the container + // + // https://pkg.go.dev/github.com/go-vela/types/library#ServiceFromContainer + _service = library.ServiceFromContainer(ctn) + } + + // defer an upload of the service + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Upload + defer func() { service.Upload(ctn, c.build, nil, nil, nil, _service) }() + + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + // remove the runtime container + err = c.Runtime.RemoveContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} diff --git a/executor/local/service_test.go b/executor/local/service_test.go new file mode 100644 index 00000000..581a1ac2 --- /dev/null +++ b/executor/local/service_test.go @@ -0,0 +1,368 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "testing" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLocal_CreateService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with image not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:notfound", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("CreateService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateService returned err: %v", err) + } + } +} + +func TestLocal_PlanService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("PlanService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanService returned err: %v", err) + } + } +} + +func TestLocal_ExecService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // service container with image not found + failure: true, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:notfound", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.services.Store(test.container.ID, new(library.Service)) + } + + err = _engine.ExecService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("ExecService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecService returned err: %v", err) + } + } +} + +func TestLocal_StreamService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + { // empty service container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.StreamService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("StreamService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("StreamService returned err: %v", err) + } + } +} + +func TestLocal_DestroyService(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic service container + failure: false, + container: &pipeline.Container{ + ID: "service_github_octocat_1_postgres", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Number: 1, + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyService(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("DestroyService should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyService returned err: %v", err) + } + } +} diff --git a/executor/local/stage.go b/executor/local/stage.go new file mode 100644 index 00000000..6b807590 --- /dev/null +++ b/executor/local/stage.go @@ -0,0 +1,129 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/step" +) + +// create a stage logging pattern. +const stagePattern = "[stage: %s][step: %s]" + +// CreateStage prepares the stage for execution. +func (c *client) CreateStage(ctx context.Context, s *pipeline.Stage) error { + // create a stage pattern for log output + _pattern := fmt.Sprintf(stagePattern, c.init.Name, c.init.Name) + + // output init progress to stdout + fmt.Fprintln(os.Stdout, _pattern, "> Pulling step images for stage", s.Name, "...") + + // create the steps for the stage + for _, _step := range s.Steps { + // update the container environment with stage name + _step.Environment["VELA_STEP_STAGE"] = s.Name + + // create the step + err := c.CreateStep(ctx, _step) + if err != nil { + return err + } + + // inspect the step image + image, err := c.Runtime.InspectImage(ctx, _step) + if err != nil { + return err + } + + // output the image information to stdout + fmt.Fprintln(os.Stdout, _pattern, string(image)) + } + + return nil +} + +// PlanStage prepares the stage for execution. +func (c *client) PlanStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) error { + // ensure dependent stages have completed + for _, needs := range s.Needs { + // check if a dependency stage has completed + stageErr, ok := m.Load(needs) + if !ok { // stage not found so we continue + continue + } + + // wait for the stage channel to close + select { + case <-ctx.Done(): + return fmt.Errorf("errgroup context is done") + case err := <-stageErr.(chan error): + if err != nil { + return err + } + + continue + } + } + + return nil +} + +// ExecStage runs a stage. +func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) error { + // close the stage channel at the end + defer func() { + errChan, ok := m.Load(s.Name) + if !ok { + return + } + + close(errChan.(chan error)) + }() + + // execute the steps for the stage + for _, _step := range s.Steps { + // check if the step should be skipped + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip + if step.Skip(_step, c.build, c.repo) { + continue + } + + // plan the step + err := c.PlanStep(ctx, _step) + if err != nil { + return fmt.Errorf("unable to plan step %s: %w", _step.Name, err) + } + + // execute the step + err = c.ExecStep(ctx, _step) + if err != nil { + return fmt.Errorf("unable to exec step %s: %w", _step.Name, err) + } + } + + return nil +} + +// DestroyStage cleans up the stage after execution. +func (c *client) DestroyStage(ctx context.Context, s *pipeline.Stage) error { + var err error + + // destroy the steps for the stage + for _, _step := range s.Steps { + // destroy the step + err = c.DestroyStep(ctx, _step) + if err != nil { + fmt.Fprintln(os.Stdout, "unable to destroy step: ", err) + } + } + + return err +} diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go new file mode 100644 index 00000000..ec1f04c9 --- /dev/null +++ b/executor/local/stage_test.go @@ -0,0 +1,387 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "errors" + "flag" + "sync" + "testing" + + "github.com/urfave/cli/v2" + + "github.com/go-vela/compiler/compiler/native" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/types/pipeline" +) + +func TestLocal_CreateStage(t *testing.T) { + // setup types + _file := "testdata/build/stages/basic.yml" + _build := testBuild() + _repo := testRepo() + _user := testUser() + + compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + + _pipeline, err := compiler. + WithBuild(_build). + WithRepo(_repo). + WithLocal(true). + WithUser(_user). + Compile(_file) + if err != nil { + t.Errorf("unable to compile pipeline %s: %v", _file, err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // stage with step container with image not found + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(_pipeline), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + // run create to init steps to be created properly + err = _engine.CreateBuild(context.Background()) + if err != nil { + t.Errorf("unable to create build: %v", err) + } + + err = _engine.CreateStage(context.Background(), test.stage) + + if test.failure { + if err == nil { + t.Errorf("CreateStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateStage returned err: %v", err) + } + } +} + +func TestLocal_PlanStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + testMap := new(sync.Map) + testMap.Store("foo", make(chan error, 1)) + + tm, _ := testMap.Load("foo") + tm.(chan error) <- nil + close(tm.(chan error)) + + errMap := new(sync.Map) + errMap.Store("foo", make(chan error, 1)) + + em, _ := errMap.Load("foo") + em.(chan error) <- errors.New("bar") + close(em.(chan error)) + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + stageMap *sync.Map + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: new(sync.Map), + }, + { // basic stage with nil stage map + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Needs: []string{"foo"}, + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: testMap, + }, + { // basic stage with error stage map + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Needs: []string{"foo"}, + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + stageMap: errMap, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanStage(context.Background(), test.stage, test.stageMap) + + if test.failure { + if err == nil { + t.Errorf("PlanStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanStage returned err: %v", err) + } + } +} + +func TestLocal_ExecStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + { // stage with step container with image not found + failure: true, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + stageMap := new(sync.Map) + stageMap.Store("echo", make(chan error)) + + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.ExecStage(context.Background(), test.stage, stageMap) + + if test.failure { + if err == nil { + t.Errorf("ExecStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecStage returned err: %v", err) + } + } +} + +func TestLocal_DestroyStage(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + stage *pipeline.Stage + }{ + { // basic stage + failure: false, + stage: &pipeline.Stage{ + Name: "echo", + Steps: pipeline.ContainerSlice{ + { + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyStage(context.Background(), test.stage) + + if test.failure { + if err == nil { + t.Errorf("DestroyStage should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyStage returned err: %v", err) + } + } +} diff --git a/executor/local/step.go b/executor/local/step.go new file mode 100644 index 00000000..3c5d54a2 --- /dev/null +++ b/executor/local/step.go @@ -0,0 +1,224 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "bufio" + "context" + "fmt" + "os" + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/step" +) + +// create a step logging pattern. +const stepPattern = "[step: %s]" + +// CreateStep configures the step for execution. +func (c *client) CreateStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // setup the runtime container + err := c.Runtime.SetupContainer(ctx, ctn) + if err != nil { + return err + } + + // create a library step object to facilitate injecting environment as early as possible + // (PlanStep is too late to inject environment vars for the kubernetes runtime). + _step := c.newLibraryStep(ctn) + _step.SetStatus(constants.StatusPending) + + // update the step container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Environment + err = step.Environment(ctn, c.build, c.repo, _step, c.Version) + if err != nil { + return err + } + + // substitute container configuration + // + // https://pkg.go.dev/github.com/go-vela/types/pipeline#Container.Substitute + err = ctn.Substitute() + if err != nil { + return err + } + + return nil +} + +// newLibraryStep creates a library step object. +func (c *client) newLibraryStep(ctn *pipeline.Container) *library.Step { + _step := new(library.Step) + _step.SetName(ctn.Name) + _step.SetNumber(ctn.Number) + _step.SetImage(ctn.Image) + _step.SetStage(ctn.Environment["VELA_STEP_STAGE"]) + _step.SetHost(c.build.GetHost()) + _step.SetRuntime(c.build.GetRuntime()) + _step.SetDistribution(c.build.GetDistribution()) + return _step +} + +// PlanStep prepares the step for execution. +func (c *client) PlanStep(ctx context.Context, ctn *pipeline.Container) error { + // create the library step object + _step := c.newLibraryStep(ctn) + _step.SetStatus(constants.StatusRunning) + _step.SetStarted(time.Now().UTC().Unix()) + + // update the step container environment + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Environment + err := step.Environment(ctn, c.build, c.repo, _step, c.Version) + if err != nil { + return err + } + + // add the step to the client map + c.steps.Store(ctn.ID, _step) + + return nil +} + +// ExecStep runs a step. +func (c *client) ExecStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _step, err := step.Load(ctn, &c.steps) + if err != nil { + return err + } + + // defer taking a snapshot of the step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Snapshot + defer func() { step.Snapshot(ctn, c.build, nil, nil, nil, _step) }() + + // run the runtime container + err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) + if err != nil { + return err + } + + go func() { + // stream logs from container + err := c.StreamStep(context.Background(), ctn) + if err != nil { + // TODO: Should this be changed or removed? + fmt.Println(err) + } + }() + + // do not wait for detached containers + if ctn.Detach { + return nil + } + + // wait for the runtime container + err = c.Runtime.WaitContainer(ctx, ctn) + if err != nil { + return err + } + + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} + +// StreamStep tails the output for a step. +func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // tail the runtime container + rc, err := c.Runtime.TailContainer(ctx, ctn) + if err != nil { + return err + } + defer rc.Close() + + // create a step pattern for log output + _pattern := fmt.Sprintf(stepPattern, ctn.Name) + + // check if the container provided is for stages + _stage, ok := ctn.Environment["VELA_STEP_STAGE"] + if ok { + // check if the stage name is set + if len(_stage) > 0 { + // create a stage pattern for log output + _pattern = fmt.Sprintf(stagePattern, _stage, ctn.Name) + } + } + + // create new scanner from the container output + scanner := bufio.NewScanner(rc) + + // scan entire container output + for scanner.Scan() { + // ensure we output to stdout + fmt.Fprintln(os.Stdout, _pattern, scanner.Text()) + } + + return scanner.Err() +} + +// DestroyStep cleans up steps after execution. +func (c *client) DestroyStep(ctx context.Context, ctn *pipeline.Container) error { + // TODO: remove hardcoded reference + if ctn.Name == "init" { + return nil + } + + // load the step from the client + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load + _step, err := step.Load(ctn, &c.steps) + if err != nil { + // create the step from the container + // + // https://pkg.go.dev/github.com/go-vela/types/library#StepFromContainer + _step = library.StepFromContainer(ctn) + } + + // defer an upload of the step + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload + defer func() { step.Upload(ctn, c.build, nil, nil, nil, _step) }() + + // inspect the runtime container + err = c.Runtime.InspectContainer(ctx, ctn) + if err != nil { + return err + } + + // remove the runtime container + err = c.Runtime.RemoveContainer(ctx, ctn) + if err != nil { + return err + } + + return nil +} diff --git a/executor/local/step_test.go b/executor/local/step_test.go new file mode 100644 index 00000000..e6f08ae8 --- /dev/null +++ b/executor/local/step_test.go @@ -0,0 +1,427 @@ +// Copyright (c) 1011 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package local + +import ( + "context" + "testing" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +func TestLocal_CreateStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with image not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.CreateStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("CreateStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("CreateStep returned err: %v", err) + } + } +} + +func TestLocal_PlanStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.PlanStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("PlanStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("PlanStep returned err: %v", err) + } + } +} + +func TestLocal_ExecStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // detached step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Detach: true, + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // step container with image not found + failure: true, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:notfound", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + if !test.container.Empty() { + _engine.steps.Store(test.container.ID, new(library.Step)) + } + + err = _engine.ExecStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("ExecStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("ExecStep returned err: %v", err) + } + } +} + +func TestLocal_StreamStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // basic stage container + failure: false, + container: &pipeline.Container{ + ID: "github_octocat_1_echo_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"VELA_STEP_STAGE": "foo"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + { // empty step container + failure: true, + container: new(pipeline.Container), + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.StreamStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("StreamStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("StreamStep returned err: %v", err) + } + } +} + +func TestLocal_DestroyStep(t *testing.T) { + // setup types + _build := testBuild() + _repo := testRepo() + _user := testUser() + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + failure bool + container *pipeline.Container + }{ + { // init step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // basic step container + failure: false, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + }, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithBuild(_build), + WithPipeline(new(pipeline.Build)), + WithRepo(_repo), + WithRuntime(_runtime), + WithUser(_user), + ) + if err != nil { + t.Errorf("unable to create executor engine: %v", err) + } + + err = _engine.DestroyStep(context.Background(), test.container) + + if test.failure { + if err == nil { + t.Errorf("DestroyStep should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("DestroyStep returned err: %v", err) + } + } +} diff --git a/executor/local/testdata/build/empty.yml b/executor/local/testdata/build/empty.yml new file mode 100644 index 00000000..73b314ff --- /dev/null +++ b/executor/local/testdata/build/empty.yml @@ -0,0 +1 @@ +--- \ No newline at end of file diff --git a/executor/local/testdata/build/services/basic.yml b/executor/local/testdata/build/services/basic.yml new file mode 100644 index 00000000..0c0f8699 --- /dev/null +++ b/executor/local/testdata/build/services/basic.yml @@ -0,0 +1,18 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:latest + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/services/img_ignorenotfound.yml b/executor/local/testdata/build/services/img_ignorenotfound.yml new file mode 100644 index 00000000..324248ca --- /dev/null +++ b/executor/local/testdata/build/services/img_ignorenotfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:ignorenotfound + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/local/testdata/build/services/img_notfound.yml b/executor/local/testdata/build/services/img_notfound.yml new file mode 100644 index 00000000..5378fe7f --- /dev/null +++ b/executor/local/testdata/build/services/img_notfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: postgres + environment: + FOO: bar + image: postgres:notfound + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/local/testdata/build/services/name_notfound.yml b/executor/local/testdata/build/services/name_notfound.yml new file mode 100644 index 00000000..3dd1998b --- /dev/null +++ b/executor/local/testdata/build/services/name_notfound.yml @@ -0,0 +1,17 @@ +--- +version: "1" +services: + - name: notfound + environment: + FOO: bar + image: postgres:latest + pull: true + +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true \ No newline at end of file diff --git a/executor/local/testdata/build/stages/basic.yml b/executor/local/testdata/build/stages/basic.yml new file mode 100644 index 00000000..f49e1750 --- /dev/null +++ b/executor/local/testdata/build/stages/basic.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/stages/img_ignorenotfound.yml b/executor/local/testdata/build/stages/img_ignorenotfound.yml new file mode 100644 index 00000000..e261e316 --- /dev/null +++ b/executor/local/testdata/build/stages/img_ignorenotfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:ignorenotfound + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/stages/img_notfound.yml b/executor/local/testdata/build/stages/img_notfound.yml new file mode 100644 index 00000000..1639a4f6 --- /dev/null +++ b/executor/local/testdata/build/stages/img_notfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:notfound + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/stages/name_notfound.yml b/executor/local/testdata/build/stages/name_notfound.yml new file mode 100644 index 00000000..69216319 --- /dev/null +++ b/executor/local/testdata/build/stages/name_notfound.yml @@ -0,0 +1,13 @@ +--- +version: "1" +stages: + test: + steps: + - name: notfound + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/steps/basic.yml b/executor/local/testdata/build/steps/basic.yml new file mode 100644 index 00000000..10852530 --- /dev/null +++ b/executor/local/testdata/build/steps/basic.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/steps/img_ignorenotfound.yml b/executor/local/testdata/build/steps/img_ignorenotfound.yml new file mode 100644 index 00000000..539fac96 --- /dev/null +++ b/executor/local/testdata/build/steps/img_ignorenotfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:ignorenotfound + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/steps/img_notfound.yml b/executor/local/testdata/build/steps/img_notfound.yml new file mode 100644 index 00000000..20d1b53a --- /dev/null +++ b/executor/local/testdata/build/steps/img_notfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: test + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:notfound + pull: true + \ No newline at end of file diff --git a/executor/local/testdata/build/steps/name_notfound.yml b/executor/local/testdata/build/steps/name_notfound.yml new file mode 100644 index 00000000..735fce7c --- /dev/null +++ b/executor/local/testdata/build/steps/name_notfound.yml @@ -0,0 +1,11 @@ +--- +version: "1" +steps: + - name: notfound + commands: + - echo ${FOO} + environment: + FOO: bar + image: alpine:latest + pull: true + \ No newline at end of file diff --git a/executor/setup.go b/executor/setup.go new file mode 100644 index 00000000..bedbd415 --- /dev/null +++ b/executor/setup.go @@ -0,0 +1,159 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "fmt" + "strings" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/worker/executor/linux" + "github.com/go-vela/worker/executor/local" + + "github.com/go-vela/pkg-runtime/runtime" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + + "github.com/sirupsen/logrus" +) + +// Setup represents the configuration necessary for +// creating a Vela engine capable of integrating +// with a configured executor. +type Setup struct { + // Executor Configuration + + // specifies the executor driver to use + Driver string + // specifies the executor hostname + Hostname string + // specifies the executor version + Version string + // API client for sending requests to Vela + Client *vela.Client + // engine used for creating runtime resources + Runtime runtime.Engine + + // Vela Resource Configuration + + // resource for storing build information in Vela + Build *library.Build + // resource for storing pipeline information in Vela + Pipeline *pipeline.Build + // resource for storing repo information in Vela + Repo *library.Repo + // resource for storing user information in Vela + User *library.User +} + +// Darwin creates and returns a Vela engine capable of +// integrating with a Darwin executor. +func (s *Setup) Darwin() (Engine, error) { + logrus.Trace("creating darwin executor client from setup") + + return nil, fmt.Errorf("unsupported executor driver: %s", constants.DriverDarwin) +} + +// Linux creates and returns a Vela engine capable of +// integrating with a Linux executor. +func (s *Setup) Linux() (Engine, error) { + logrus.Trace("creating linux executor client from setup") + + // create new Linux executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/linux?tab=doc#New + return linux.New( + linux.WithBuild(s.Build), + linux.WithHostname(s.Hostname), + linux.WithPipeline(s.Pipeline), + linux.WithRepo(s.Repo), + linux.WithRuntime(s.Runtime), + linux.WithUser(s.User), + linux.WithVelaClient(s.Client), + linux.WithVersion(s.Version), + ) +} + +// Local creates and returns a Vela engine capable of +// integrating with a local executor. +func (s *Setup) Local() (Engine, error) { + logrus.Trace("creating local executor client from setup") + + // create new Local executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/local?tab=doc#New + return local.New( + local.WithBuild(s.Build), + local.WithHostname(s.Hostname), + local.WithPipeline(s.Pipeline), + local.WithRepo(s.Repo), + local.WithRuntime(s.Runtime), + local.WithUser(s.User), + local.WithVelaClient(s.Client), + local.WithVersion(s.Version), + ) +} + +// Windows creates and returns a Vela engine capable of +// integrating with a Windows executor. +func (s *Setup) Windows() (Engine, error) { + logrus.Trace("creating windows executor client from setup") + + return nil, fmt.Errorf("unsupported executor driver: %s", constants.DriverWindows) +} + +// Validate verifies the necessary fields for the +// provided configuration are populated correctly. +func (s *Setup) Validate() error { + logrus.Trace("validating executor setup for client") + + // check if an executor driver was provided + if len(s.Driver) == 0 { + return fmt.Errorf("no executor driver provided in setup") + } + + // check if a Vela pipeline was provided + if s.Pipeline == nil { + return fmt.Errorf("no Vela pipeline provided in setup") + } + + // check if a runtime engine was provided + if s.Runtime == nil { + return fmt.Errorf("no runtime engine provided in setup") + } + + // check if the local driver is provided + if strings.EqualFold(constants.DriverLocal, s.Driver) { + // all other fields are not required + // for the local executor + return nil + } + + // check if a Vela client was provided + if s.Client == nil { + return fmt.Errorf("no Vela client provided in setup") + } + + // check if a Vela build was provided + if s.Build == nil { + return fmt.Errorf("no Vela build provided in setup") + } + + // check if a Vela repo was provided + if s.Repo == nil { + return fmt.Errorf("no Vela repo provided in setup") + } + + // check if a Vela user was provided + if s.User == nil { + return fmt.Errorf("no Vela user provided in setup") + } + + // setup is valid + return nil +} diff --git a/executor/setup_test.go b/executor/setup_test.go new file mode 100644 index 00000000..c56496cc --- /dev/null +++ b/executor/setup_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executor + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/mock/server" + + "github.com/go-vela/worker/executor/linux" + "github.com/go-vela/worker/executor/local" + + "github.com/go-vela/pkg-runtime/runtime/docker" + + "github.com/go-vela/sdk-go/vela" + + "github.com/go-vela/types/constants" +) + +func TestExecutor_Setup_Darwin(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _setup := &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverDarwin, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + } + + got, err := _setup.Darwin() + if err == nil { + t.Errorf("Darwin should have returned err") + } + + if got != nil { + t.Errorf("Darwin is %v, want nil", got) + } +} + +func TestExecutor_Setup_Linux(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + want, err := linux.New( + linux.WithBuild(_build), + linux.WithHostname("localhost"), + linux.WithPipeline(_pipeline), + linux.WithRepo(_repo), + linux.WithRuntime(_runtime), + linux.WithUser(_user), + linux.WithVelaClient(_client), + linux.WithVersion("v1.0.0"), + ) + if err != nil { + t.Errorf("unable to create linux engine: %v", err) + } + + _setup := &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Hostname: "localhost", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + } + + // run test + got, err := _setup.Linux() + if err != nil { + t.Errorf("Linux returned err: %v", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Linux is %v, want %v", got, want) + } +} + +func TestExecutor_Setup_Local(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + want, err := local.New( + local.WithBuild(_build), + local.WithHostname("localhost"), + local.WithPipeline(_pipeline), + local.WithRepo(_repo), + local.WithRuntime(_runtime), + local.WithUser(_user), + local.WithVelaClient(_client), + local.WithVersion("v1.0.0"), + ) + if err != nil { + t.Errorf("unable to create local engine: %v", err) + } + + _setup := &Setup{ + Build: _build, + Client: _client, + Driver: "local", + Hostname: "localhost", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", + } + + // run test + got, err := _setup.Local() + if err != nil { + t.Errorf("Local returned err: %v", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Local is %v, want %v", got, want) + } +} + +func TestExecutor_Setup_Windows(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + _setup := &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverWindows, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + } + + got, err := _setup.Windows() + if err == nil { + t.Errorf("Windows should have returned err") + } + + if got != nil { + t.Errorf("Windows is %v, want nil", got) + } +} + +func TestExecutor_Setup_Validate(t *testing.T) { + // setup types + gin.SetMode(gin.TestMode) + + s := httptest.NewServer(server.FakeHandler()) + + _client, err := vela.NewClient(s.URL, "", nil) + if err != nil { + t.Errorf("unable to create Vela API client: %v", err) + } + + _runtime, err := docker.NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + setup *Setup + failure bool + }{ + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + }, + failure: false, + }, + { + setup: &Setup{ + Build: nil, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: nil, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: "", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: nil, + Repo: _repo, + Runtime: _runtime, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: nil, + Runtime: _runtime, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: nil, + User: _user, + }, + failure: true, + }, + { + setup: &Setup{ + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: nil, + }, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err = test.setup.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} diff --git a/go.mod b/go.mod index 48e88967..8d6af872 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,13 @@ go 1.16 require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/gin-gonic/gin v1.7.4 + github.com/go-vela/compiler v0.10.0 github.com/go-vela/mock v0.10.0 - github.com/go-vela/pkg-executor v0.10.0 github.com/go-vela/pkg-queue v0.10.0 - github.com/go-vela/pkg-runtime v0.10.0 + github.com/go-vela/pkg-runtime v0.10.1-0.20211025172651-7d29320dd785 github.com/go-vela/sdk-go v0.10.0 github.com/go-vela/types v0.10.0 + github.com/google/go-cmp v0.5.6 github.com/joho/godotenv v1.4.0 github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index dd3a123b..86a720f7 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -164,12 +165,10 @@ github.com/go-vela/compiler v0.10.0 h1:dFilpf5A+tiJWibILE2kHnAa+UEFDZgv/9lDwd0+n github.com/go-vela/compiler v0.10.0/go.mod h1:Zq1L6qXsV/h5kWO5A3boGzFWvXk7he6FO2hcMVuSupw= github.com/go-vela/mock v0.10.0 h1:ZJs40xElnB4DNiQc+nEEeZS4Z0K/uXl6kGRpPlccuMY= github.com/go-vela/mock v0.10.0/go.mod h1:TihYvb+NBiKXgcsBIpARU9H00rzrLAhFQvsRkzUqDxc= -github.com/go-vela/pkg-executor v0.10.0 h1:7zPysuDj7oi6E056IEkU66KOYA2SIc32Kqgm1lnn0pE= -github.com/go-vela/pkg-executor v0.10.0/go.mod h1:voWCsAoHZb8HUSEcrTGYNHRh1IUBXWlovFVAWnF8xXo= github.com/go-vela/pkg-queue v0.10.0 h1:cxpkyVuX+ZJuF9t7XEQuHOFBa776SNgraEsFpnWI03E= github.com/go-vela/pkg-queue v0.10.0/go.mod h1:ZtkPoazVfpKK/ePdea/2s2LpNWDrc19nqmn1hPI3jxY= -github.com/go-vela/pkg-runtime v0.10.0 h1:ycjK0mSrM+sAwOtENKjKMGMTGkx4xkc/zMA0JIr9T0k= -github.com/go-vela/pkg-runtime v0.10.0/go.mod h1:ffabnvUkIEY/5r1XFep40HdCqrHp4OxmzbbY3W7g6C4= +github.com/go-vela/pkg-runtime v0.10.1-0.20211025172651-7d29320dd785 h1:VByvQACNppX/P74CfgSr+E0+HuOCMXnDHZYFiF9HdCs= +github.com/go-vela/pkg-runtime v0.10.1-0.20211025172651-7d29320dd785/go.mod h1:7KV1aAufGNpGbbLLsD7W7z8gc50fXdIK03NPM+SClg8= github.com/go-vela/sdk-go v0.10.0 h1:monESdM738WeY2MKlj0COGK0W/f1PIGwp8K4tClfLlo= github.com/go-vela/sdk-go v0.10.0/go.mod h1:LGHpZezP0+KBb3OX9Mf5rGXK1dS7Ms8kWCHb8bWzOYc= github.com/go-vela/types v0.10.0-rc3/go.mod h1:6taTlivaC0wDwDJVlc8sBaVZToyzkyDMtGUYIAfgA9M= @@ -266,6 +265,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.10.0 h1:b86HUuA126IcSHyC55WjPo7KtCOVeTCKIjr+3lBhPxI= github.com/hashicorp/go-hclog v0.10.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= @@ -312,6 +312,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= diff --git a/router/middleware/executor.go b/router/middleware/executor.go index e36027c8..c5f53eb1 100644 --- a/router/middleware/executor.go +++ b/router/middleware/executor.go @@ -7,7 +7,7 @@ package middleware import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" + "github.com/go-vela/worker/executor" ) // Executors is a middleware function that attaches the diff --git a/router/middleware/executor/executor.go b/router/middleware/executor/executor.go index 5aa4bcc2..3b558b27 100644 --- a/router/middleware/executor/executor.go +++ b/router/middleware/executor/executor.go @@ -11,8 +11,8 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/types" + "github.com/go-vela/worker/executor" "github.com/sirupsen/logrus" ) diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 18298d8d..cf703562 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -12,12 +12,12 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" "github.com/go-vela/pkg-runtime/runtime/docker" "github.com/go-vela/sdk-go/vela" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/executor" ) func TestExecutor_Retrieve(t *testing.T) { diff --git a/router/middleware/executor_test.go b/router/middleware/executor_test.go index 187b2538..7e4d40a0 100644 --- a/router/middleware/executor_test.go +++ b/router/middleware/executor_test.go @@ -12,7 +12,7 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/pkg-executor/executor" + "github.com/go-vela/worker/executor" ) func TestMiddleware_Executors(t *testing.T) {