diff --git a/cmd/all.go b/cmd/all.go index 4e87ec93..26b70a95 100644 --- a/cmd/all.go +++ b/cmd/all.go @@ -19,7 +19,6 @@ func init() { log.Fatal().Err(err).Msg("Unable to initialize myks's globe") } - log.Info().Msg("Syncing and rendering environments") if err := g.SyncAndRender(); err != nil { log.Fatal().Err(err).Msg("Unable to sync vendir configs") } diff --git a/cmd/render.go b/cmd/render.go index 0f9379c6..ac99af07 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -13,7 +13,6 @@ func init() { Short: "Render manifests", Long: "Render manifests", Run: func(cmd *cobra.Command, args []string) { - log.Info().Msg("Rendering manifests") g := myks.New(".") diff --git a/cmd/root.go b/cmd/root.go index c7580f56..28624849 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -67,7 +67,6 @@ var rootCmd = &cobra.Command{ func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringP("log-level", "l", "info", "Set the logging level") if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil { diff --git a/go.mod b/go.mod index 48f856d3..fc9e5711 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/spf13/afero v1.9.5 // indirect diff --git a/go.sum b/go.sum index 07563a6d..6a3accb3 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,9 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= @@ -323,6 +324,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/myks/application.go b/internal/myks/application.go index 9e06d7f0..057a9577 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -2,10 +2,23 @@ package myks import ( "errors" + "fmt" + "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v3" + "io" "os" "path/filepath" +) - yaml "gopkg.in/yaml.v3" +const ( + renderStepName = "render" + syncStepName = "sync" + globalYttStepName = "global-ytt" + yttStepName = "ytt" + yttPkgStepName = "ytt-pkg" + helmStepName = "helm" + sliceStepName = "slice" + initStepName = "init" ) type Application struct { @@ -28,6 +41,7 @@ type HelmConfig struct { } var ErrNoVendirConfig = errors.New("no vendir config found") +var ApplicationLogFormat = "\033[1m[%s > %s > %s]\033[0m %s" func NewApplication(e *Environment, name string, prototypeName string) (*Application, error) { if prototypeName == "" { @@ -61,7 +75,7 @@ func (a *Application) Init() error { a.collectDataFiles() - dataYaml, err := renderDataYaml(a.Name, append(a.e.g.extraYttPaths, a.yttDataFiles...)) + dataYaml, err := a.renderDataYaml(append(a.e.g.extraYttPaths, a.yttDataFiles...)) if err != nil { return err } @@ -122,3 +136,49 @@ func (a *Application) collectDataFiles() { overrideDataFiles := a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.ApplicationDataFileName)) a.yttDataFiles = append(a.yttDataFiles, overrideDataFiles...) } + +func (a *Application) Msg(step string, msg string) string { + formattedMessage := fmt.Sprintf(ApplicationLogFormat, a.e.Id, a.Name, step, msg) + return formattedMessage +} + +func (a *Application) runCmd(purpose string, cmd string, stdin io.Reader, args []string) (CmdResult, error) { + return runCmd(cmd, stdin, args, func(cmd string, args []string) { + log.Debug().Msg(msgRunCmd(purpose, cmd, args)) + }) +} + +func (a *Application) renderDataYaml(dataFiles []string) ([]byte, error) { + if len(dataFiles) == 0 { + return nil, errors.New("No data files found") + } + res, err := runYttWithFilesAndStdin(dataFiles, nil, func(name string, args []string) { + log.Debug().Msg(a.Msg("init", msgRunCmd("render application data values file", name, args))) + }, "--data-values-inspect") + if err != nil { + log.Error().Err(err).Str("stderr", res.Stderr).Msg(a.Msg("init", "Unable to render data")) + return nil, err + } + if res.Stdout == "" { + return nil, errors.New("Empty output from ytt") + } + + dataYaml := []byte(res.Stdout) + return dataYaml, nil +} + +func (a *Application) mergeValuesYaml(valueFilesYaml string) (CmdResult, error) { + return runYttWithFilesAndStdin(nil, nil, func(name string, args []string) { + log.Debug().Msg(msgRunCmd("merge data values file", name, args)) + }, "--data-values-file="+valueFilesYaml, "--data-values-inspect") +} + +func (a *Application) ytt(step string, purpose string, paths []string, args ...string) (CmdResult, error) { + return a.yttS(step, purpose, paths, nil, args...) +} + +func (a *Application) yttS(step string, purpose string, paths []string, stdin io.Reader, args ...string) (CmdResult, error) { + return runYttWithFilesAndStdin(append(a.e.g.extraYttPaths, paths...), stdin, func(name string, args []string) { + log.Debug().Msg(a.Msg(step, msgRunCmd(purpose, name, args))) + }, args...) +} diff --git a/internal/myks/application_test.go b/internal/myks/application_test.go new file mode 100644 index 00000000..9ac82c88 --- /dev/null +++ b/internal/myks/application_test.go @@ -0,0 +1,36 @@ +package myks + +import ( + "os" + "reflect" + "testing" +) + +func TestApplication_renderDataYaml(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("Skipping in pipeline since ytt is not installed") + } + type args struct { + dataFiles []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"happy path", args{[]string{"../../testData/ytt/data-file-schema.yaml", "../../testData/ytt/data-file-schema-2.yaml", "../../testData/ytt/data-file-values.yaml"}}, "application:\n cache:\n enabled: true\n name: cert-manager\n", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := testApp.renderDataYaml(tt.args.dataFiles) + if (err != nil) != tt.wantErr { + t.Errorf("renderDataYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("renderDataYaml() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 172a076c..92c38d8c 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -3,6 +3,7 @@ package myks import ( "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -11,6 +12,8 @@ import ( yaml "gopkg.in/yaml.v3" ) +var EnvLogFormat = "\033[1m[%s > %s]\033[0m %s" + type ManifestApplication struct { Name string Prototype string @@ -63,11 +66,15 @@ func NewEnvironment(g *Globe, dir string) *Environment { func (e *Environment) Init(applicationNames []string) error { if err := e.initEnvData(); err != nil { - log.Warn().Err(err).Str("dir", e.Dir).Msg("Unable to initialize environment data") + log.Warn().Err(err).Str("dir", e.Dir).Msg(e.Msg("Unable to initialize environment data")) return err } - e.initApplications(applicationNames) + err := e.initApplications(applicationNames) + if err != nil { + log.Error().Err(err).Msg(e.Msg("Unable to initialize applications")) + return err + } return nil } @@ -120,7 +127,7 @@ func (e *Environment) SyncAndRender() error { func (e *Environment) setId() error { yamlBytes, err := os.ReadFile(e.EnvironmentDataFile) if err != nil { - log.Debug().Err(err).Msg("Unable to read environment data file") + log.Debug().Err(err).Msg(e.Msg("Unable to read environment data file")) return err } @@ -131,12 +138,10 @@ func (e *Environment) setId() error { } err = yaml.Unmarshal(yamlBytes, &envData) if err != nil { - log.Debug().Err(err).Msg("Unable to unmarshal environment data file") + log.Debug().Err(err).Msg(e.Msg("Unable to unmarshal environment data file")) return err } - log.Debug().Interface("envData", envData).Msg("Environment data") - if envData.Environment.Id == "" { err = errors.New("Environment data file missing id") log.Debug().Err(err).Str("file", e.EnvironmentDataFile).Msg("Unable to set environment id") @@ -145,6 +150,8 @@ func (e *Environment) setId() error { e.Id = envData.Environment.Id + log.Debug().Interface("envData", envData).Msg(e.Msg("Environment data")) + return nil } @@ -152,17 +159,17 @@ func (e *Environment) initEnvData() error { envDataFiles := e.collectBySubpath(e.g.EnvironmentDataFileName) envDataYaml, err := e.renderEnvData(envDataFiles) if err != nil { - log.Warn().Err(err).Str("dir", e.Dir).Msg("Unable to render environment data") + log.Warn().Err(err).Str("dir", e.Dir).Msg(e.Msg("Unable to render environment data")) return err } err = e.saveRenderedEnvData(envDataYaml) if err != nil { - log.Warn().Err(err).Str("dir", e.Dir).Msg("Unable to save rendered environment data") + log.Warn().Err(err).Str("dir", e.Dir).Msg(e.Msg("Unable to save rendered environment data")) return err } err = e.setEnvDataFromYaml(envDataYaml) if err != nil { - log.Warn().Err(err).Str("dir", e.Dir).Msg("Unable to set environment data") + log.Warn().Err(err).Str("dir", e.Dir).Msg(e.Msg("Unable to set environment data")) return err } @@ -173,9 +180,9 @@ func (e *Environment) renderEnvData(envDataFiles []string) ([]byte, error) { if len(envDataFiles) == 0 { return nil, errors.New("No environment data files found") } - res, err := e.g.ytt(envDataFiles, "--data-values-inspect") + res, err := e.ytt("render environment data values file", envDataFiles, "--data-values-inspect") if err != nil { - log.Error().Err(err).Str("stderr", res.Stderr).Msg("Unable to render environment data") + log.Error().Err(err).Str("stderr", res.Stderr).Msg(e.Msg("Unable to render environment data")) return nil, err } if res.Stdout == "" { @@ -190,15 +197,14 @@ func (e *Environment) saveRenderedEnvData(envDataYaml []byte) error { dir := filepath.Dir(e.renderedEnvDataFilePath) err := os.MkdirAll(dir, 0o750) if err != nil { - log.Error().Err(err).Str("dir", dir).Msg("Unable to create directory for rendered envData file") + log.Error().Err(err).Str("dir", dir).Msg(e.Msg("Unable to create directory for rendered envData file")) return err } err = os.WriteFile(e.renderedEnvDataFilePath, envDataYaml, 0o600) if err != nil { - log.Error().Err(err).Msg("Unable to write rendered envData file") + log.Error().Err(err).Msg(e.Msg("Unable to write rendered envData file")) return err } - log.Debug().Str("file", e.renderedEnvDataFilePath).Msg("Wrote rendered envData file") return nil } @@ -213,14 +219,14 @@ func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { } err := yaml.Unmarshal(envDataYaml, &envDataStruct) if err != nil { - log.Error().Err(err).Msg("Unable to unmarshal environment data yaml") + log.Error().Err(err).Msg(e.Msg("Unable to unmarshal environment data yaml")) return err } for _, app := range envDataStruct.Environment.Applications { proto := app.Proto if len(proto) == 0 { - log.Error().Interface("app", app).Msg("Application prototype is not set") + log.Error().Interface("app", app).Msg(e.Msg("Application prototype is not set")) continue } @@ -230,7 +236,7 @@ func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { } if _, ok := e.foundApplications[name]; ok { - log.Error().Str("app_name", name).Msg("Duplicated application") + log.Error().Str("app_name", name).Msg(e.Msg("Duplicated application")) continue } @@ -241,24 +247,36 @@ func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { if len(e.foundApplications) == 0 { log.Warn().Str("dir", e.Dir).Msg("No applications found") } else { - log.Debug().Interface("apps", e.foundApplications).Msg("Found applications") + log.Debug().Interface("apps", e.foundApplications).Msg(e.Msg("Found applications")) } return nil } -func (e *Environment) initApplications(applicationNames []string) { - for name, proto := range e.foundApplications { - if len(applicationNames) == 0 || contains(applicationNames, name) { +func (e *Environment) initApplications(applicationNames []string) error { + if len(applicationNames) == 0 { + for name, proto := range e.foundApplications { app, err := NewApplication(e, name, proto) if err != nil { - log.Warn().Err(err).Str("dir", e.Dir).Interface("app", name).Msg("Unable to initialize application") + log.Warn().Err(err).Str("dir", e.Dir).Interface("app", name).Msg(e.Msg("Unable to initialize application")) } else { e.Applications = append(e.Applications, app) } } } - log.Debug().Interface("applications", e.Applications).Msg("Applications") + for _, appName := range applicationNames { + proto := e.foundApplications[appName] + if proto == "" { + return errors.New("Application not found: " + appName) + } + app, err := NewApplication(e, appName, proto) + if err != nil { + log.Warn().Err(err).Str("dir", e.Dir).Interface("app", appName).Msg(e.Msg("Unable to initialize application")) + } else { + e.Applications = append(e.Applications, app) + } + } + return nil } func (e *Environment) collectBySubpath(subpath string) []string { @@ -275,3 +293,18 @@ func (e *Environment) collectBySubpath(subpath string) []string { } return items } + +func (e *Environment) Msg(msg string) string { + formattedMessage := fmt.Sprintf(EnvLogFormat, e.Id, initStepName, msg) + return formattedMessage +} + +func (e *Environment) ytt(purpose string, paths []string, args ...string) (CmdResult, error) { + return e.yttS(purpose, paths, nil, args...) +} + +func (e *Environment) yttS(purpose string, paths []string, stdin io.Reader, args ...string) (CmdResult, error) { + return runYttWithFilesAndStdin(append(e.g.extraYttPaths, paths...), stdin, func(name string, args []string) { + log.Debug().Msg(e.Msg(msgRunCmd(purpose, name, args))) + }, args...) +} diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 780787fd..40a5d456 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -4,7 +4,6 @@ import ( "bytes" "embed" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -27,6 +26,8 @@ var prototypesFs embed.FS //go:embed all:assets/envs var environmentsFs embed.FS +var GlobalLogFormat = "\033[1m[global]\033[0m %s" + // Define the main structure type Globe struct { /// Globe configuration @@ -303,7 +304,7 @@ func (g *Globe) collectEnvironments(searchPaths []string) { g.collectEnvironmentsInPath(searchPath) } - log.Debug().Interface("environments", g.environments).Msg("Collected environments") + log.Debug().Interface("environments", g.environments).Msg(g.Msg("Collected environments")) } func (g *Globe) collectEnvironmentsInPath(searchPath string) { @@ -331,7 +332,9 @@ func (g *Globe) collectEnvironmentsInPath(searchPath string) { func (g *Globe) setGitRepoUrl() error { if g.GitRepoUrl == "" { - result, err := runCmd("git", nil, []string{"remote", "get-url", "origin"}) + result, err := runCmd("git", nil, []string{"remote", "get-url", "origin"}, func(name string, args []string) { + log.Debug().Msg(msgRunCmd("set git repository url", name, args)) + }) if err != nil { return err } @@ -342,7 +345,9 @@ func (g *Globe) setGitRepoUrl() error { func (g *Globe) setGitRepoBranch() error { if g.GitRepoBranch == "" { - result, err := runCmd("git", nil, []string{"rev-parse", "--abbrev-ref", "HEAD"}) + result, err := runCmd("git", nil, []string{"rev-parse", "--abbrev-ref", "HEAD"}, func(name string, args []string) { + log.Debug().Msg(msgRunCmd("set git repository branch", name, args)) + }) if err != nil { return err } @@ -351,10 +356,7 @@ func (g *Globe) setGitRepoBranch() error { return nil } -func (g *Globe) ytt(paths []string, args ...string) (CmdResult, error) { - return g.yttS(paths, nil, args...) -} - -func (g *Globe) yttS(paths []string, stdin io.Reader, args ...string) (CmdResult, error) { - return runYttWithFilesAndStdin(append(g.extraYttPaths, paths...), stdin, args...) +func (g *Globe) Msg(msg string) string { + formattedMessage := fmt.Sprintf(GlobalLogFormat, msg) + return formattedMessage } diff --git a/internal/myks/render.go b/internal/myks/render.go index d1199bac..fc92ff71 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -22,24 +22,26 @@ func (a *Application) RenderAndSlice(yamlTemplatingTools []YamlTemplatingTool) e var lastStepOutputFile string var err error if lastStepOutputFile, err = a.Render(yamlTemplatingTools); err != nil { - log.Error().Str("app", a.Name).Str("env", a.e.Id).Err(err).Msg("Failed to render") + log.Error().Str("env", a.e.Id).Err(err).Msg("Failed to render") } err = a.runSliceFormatStore(lastStepOutputFile) if err != nil { log.Error().Err(err).Msg("Failed to slice the output yaml") return err } + log.Info().Msg(a.Msg(renderStepName, "Completed")) return nil } func (a *Application) Render(yamlTemplatingTools []YamlTemplatingTool) (string, error) { + log.Debug().Msg(a.Msg(renderStepName, "Starting")) outputYaml := "" lastStepOutputFile := "" for _, yamlTool := range yamlTemplatingTools { - log.Debug().Str("app", a.Name).Str("env", a.e.Id).Msg("Rendering: " + yamlTool.Ident()) + log.Debug().Msg(a.Msg(yamlTool.Ident(), "Starting")) stepOutputYaml, err := yamlTool.Render(lastStepOutputFile) if err != nil { - log.Error().Err(err).Msg("Failed during render step: " + yamlTool.Ident()) + log.Error().Err(err).Msg(a.Msg(yamlTool.Ident(), "Failed during render step: "+yamlTool.Ident())) } if yamlTool.IsAdditive() { outputYaml = outputYaml + "\n---\n" + stepOutputYaml @@ -48,7 +50,7 @@ func (a *Application) Render(yamlTemplatingTools []YamlTemplatingTool) (string, } lastStepOutputFile, err = a.storeStepResult(outputYaml, yamlTool.Ident(), 1) if err != nil { - log.Error().Str("app", a.Name).Err(err).Msg("Failed to store step result for: " + yamlTool.Ident()) + log.Error().Err(err).Msg(a.Msg(yamlTool.Ident(), "Failed to store step result for: "+yamlTool.Ident())) return "", err } } @@ -56,9 +58,10 @@ func (a *Application) Render(yamlTemplatingTools []YamlTemplatingTool) (string, } func (a *Application) runSliceFormatStore(previousStepFile string) error { + log.Debug().Msg(a.Msg(renderStepName, "Slicing")) data, err := os.ReadFile(filepath.Clean(previousStepFile)) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("file", previousStepFile).Msg("Unable to read previous step file") + log.Warn().Err(err).Str("file", previousStepFile).Msg(a.Msg(sliceStepName, "Unable to read previous step file")) return err } @@ -67,12 +70,12 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { // Cleanup the destination directory before writing new files err = os.RemoveAll(destinationDir) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("dir", destinationDir).Msg("Unable to remove destination directory") + log.Warn().Err(err).Str("dir", destinationDir).Msg(a.Msg(sliceStepName, "Unable to remove destination directory")) return err } err = os.MkdirAll(destinationDir, os.ModePerm) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("dir", destinationDir).Msg("Unable to create destination directory") + log.Warn().Err(err).Str("dir", destinationDir).Msg(a.Msg(sliceStepName, "Unable to create destination directory")) return err } @@ -88,7 +91,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { var obj map[string]interface{} err := yaml.Unmarshal([]byte(document), &obj) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("file", previousStepFile).Msg("Unable to unmarshal yaml") + log.Warn().Err(err).Str("file", previousStepFile).Msg(a.Msg(sliceStepName, "Unable to unmarshal yaml")) return err } @@ -97,7 +100,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { enc.SetIndent(2) err = enc.Encode(obj) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("file", previousStepFile).Msg("Unable to marshal yaml") + log.Warn().Err(err).Str("file", previousStepFile).Msg(a.Msg(sliceStepName, "Unable to marshal yaml")) return err } @@ -106,11 +109,11 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { // FIXME: If a file already exists, we should merge the two documents (probably). // For now, we just overwrite the file and log a warning. if _, err := os.Stat(filePath); err == nil { - log.Warn().Str("app", a.Name).Str("file", filePath).Msg("File already exists, check duplicated resources") + log.Warn().Str("file", filePath).Msg(a.Msg(sliceStepName, "File already exists, check duplicated resources")) } err = writeFile(filePath, data.Bytes()) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("file", filePath).Msg("Unable to write file") + log.Warn().Err(err).Str("file", filePath).Msg(a.Msg(renderStepName, "Unable to write file")) return err } } @@ -143,11 +146,10 @@ func (a *Application) getVendoredDir(dirname string) (string, error) { resourceDir := a.expandPath(filepath.Join(a.e.g.VendorDirName, dirname)) if _, err := os.Stat(resourceDir); err != nil { if os.IsNotExist(err) { - log.Debug().Str("app", a.Name).Msg("Vendored directory directory does not exist: " + resourceDir) return "", nil } - log.Warn().Err(err).Str("app", a.Name).Msg("Unable to stat helm charts directory: " + resourceDir) + log.Warn().Err(err).Msg(a.Msg(renderStepName, "Unable to find vendor directory: "+resourceDir)) return "", err } @@ -169,37 +171,35 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st valuesFiles = append(valuesFiles, a.e.collectBySubpath(filepath.Join("_apps", a.Name, valuesFileName))...) if len(valuesFiles) == 0 { - log.Debug().Str("app", a.Name).Str("resource", resourceName).Msg("No values files found") + log.Debug().Str("resource", resourceName).Msg(a.Msg(renderStepName, "No values files found")) return "", nil } - log.Debug().Strs("files", valuesFiles).Str("app", a.Name).Str("resourceName", resourceName).Msg("Collected resource values templates") - - resourceValuesYaml, err := a.e.g.ytt(append(a.yttDataFiles, valuesFiles...)) + resourceValuesYaml, err := a.ytt(renderStepName, "collect data values file", append(a.yttDataFiles, valuesFiles...)) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Msg("Unable to render resource values templates") + log.Warn().Err(err).Msg(a.Msg(renderStepName, "Unable to render resource values templates")) return "", err } if resourceValuesYaml.Stdout == "" { - log.Warn().Str("app", a.Name).Msg("Empty resource values") + log.Warn().Msg(a.Msg(renderStepName, "Empty resource values")) return "", nil } err = a.writeTempFile(valuesFileName, resourceValuesYaml.Stdout) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Msg("Unable to write resource values file") + log.Warn().Err(err).Msg(a.Msg(renderStepName, "Unable to write resource values file")) return "", err } - resourceValues, err := mergeValuesYaml(a.expandTempPath(valuesFileName)) + resourceValues, err := a.mergeValuesYaml(a.expandTempPath(valuesFileName)) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Msg("Unable to render resource values") + log.Warn().Err(err).Msg(a.Msg(renderStepName, "Unable to render resource values")) return "", err } if resourceValues.Stdout == "" { - log.Warn().Str("app", a.Name).Msg("Empty resource values") + log.Warn().Msg(a.Msg(renderStepName, "Empty resource values")) return "", nil } diff --git a/internal/myks/render_global_ytt.go b/internal/myks/render_global_ytt.go index 429deba5..fe204747 100644 --- a/internal/myks/render_global_ytt.go +++ b/internal/myks/render_global_ytt.go @@ -32,22 +32,22 @@ func (g *GlobalYtt) Render(previousStepFile string) (string, error) { yttFiles = append(yttFiles, g.app.e.collectBySubpath(filepath.Join("_env", g.app.e.g.YttPkgStepDirName))...) if len(yttFiles) == 0 { - log.Debug().Str("app", g.app.Name).Msg("No ytt files found") + log.Debug().Msg(g.app.Msg(globalYttStepName, "No ytt files found")) return "", nil } - log.Debug().Str("step", "global-ytt").Strs("files", yttFiles).Str("app", g.app.Name).Msg("Collected ytt files") - - yttOutput, err := g.app.e.g.ytt(yttFiles) + yttOutput, err := g.app.ytt(globalYttStepName, "render global ytt directory", yttFiles) if err != nil { - log.Warn().Err(err).Str("app", g.app.Name).Msg("Unable to render ytt files") + log.Warn().Err(err).Msg(g.app.Msg(globalYttStepName, "Unable to render ytt files")) return "", err } if yttOutput.Stdout == "" { - log.Warn().Str("app", g.app.Name).Msg("Empty ytt output") + log.Warn().Msg(g.app.Msg(globalYttStepName, "Empty ytt output")) return "", nil } + log.Debug().Msg(g.app.Msg(helmStepName, "Global ytt applied")) + return yttOutput.Stdout, nil } diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 27ea6e12..307106af 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -25,18 +25,18 @@ func (h *Helm) Ident() string { func (h *Helm) Render(_ string) (string, error) { chartDir, err := h.app.getVendoredDir(h.app.e.g.HelmChartsDirName) if err != nil { - log.Err(err).Str("app", h.app.Name).Msg("Unable to get helm charts dir") + log.Err(err).Msg(h.app.Msg(helmStepName, "Unable to get helm charts dir")) return "", err } chartDirs := getSubDirs(chartDir) if len(chartDirs) == 0 { - log.Debug().Str("app", h.app.Name).Msg("No charts to process") + log.Debug().Msg(h.app.Msg(helmStepName, "No Helm charts found")) return "", nil } helmConfig, err := h.getHelmConfig() if err != nil { - log.Warn().Err(err).Str("app", h.app.Name).Msg("Unable to get helm config") + log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to get helm config")) return "", err } @@ -63,7 +63,7 @@ func (h *Helm) Render(_ string) (string, error) { chartName := filepath.Base(chartDir) var helmValuesFile string if helmValuesFile, err = h.app.prepareValuesFile("helm", chartName); err != nil { - log.Warn().Err(err).Str("app", h.app.Name).Msg("Unable to prepare helm values") + log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to prepare helm values")) return "", err } @@ -79,14 +79,14 @@ func (h *Helm) Render(_ string) (string, error) { helmArgs = append(helmArgs, "--values", helmValuesFile) } - res, err := runCmd("helm", nil, append(helmArgs, commonHelmArgs...)) + res, err := h.app.runCmd("helm template chart", "helm", nil, append(helmArgs, commonHelmArgs...)) if err != nil { - log.Warn().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to run helm") + log.Warn().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg(h.app.Msg(helmStepName, "Unable to run helm")) return "", err } if res.Stdout == "" { - log.Warn().Str("app", h.app.Name).Str("chart", chartName).Msg("No helm output") + log.Warn().Str("chart", chartName).Msg(h.app.Msg(helmStepName, "No helm output")) continue } @@ -94,13 +94,15 @@ func (h *Helm) Render(_ string) (string, error) { } + log.Info().Msg(h.app.Msg(helmStepName, "Helm chart rendered")) + return strings.Join(outputs, "---\n"), nil } func (h *Helm) getHelmConfig() (HelmConfig, error) { - dataValuesYaml, err := h.app.e.g.ytt(h.app.yttDataFiles, "--data-values-inspect") + dataValuesYaml, err := h.app.ytt(helmStepName, "get helm config", h.app.yttDataFiles, "--data-values-inspect") if err != nil { - log.Warn().Err(err).Str("app", h.app.Name).Msg("Unable to inspect data values") + log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to inspect data values")) return HelmConfig{}, err } @@ -109,7 +111,7 @@ func (h *Helm) getHelmConfig() (HelmConfig, error) { } err = yaml.Unmarshal([]byte(dataValuesYaml.Stdout), &helmConfig) if err != nil { - log.Warn().Err(err).Str("app", h.app.Name).Msg("Unable to unmarshal data values") + log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to unmarshal data values")) return HelmConfig{}, err } diff --git a/internal/myks/render_ytt.go b/internal/myks/render_ytt.go index 7427f88b..ee9b82f3 100644 --- a/internal/myks/render_ytt.go +++ b/internal/myks/render_ytt.go @@ -1,6 +1,7 @@ package myks import ( + "errors" "os" "path/filepath" @@ -42,22 +43,20 @@ func (y *Ytt) Render(previousStepFile string) (string, error) { yttFiles = append(yttFiles, y.app.e.collectBySubpath(filepath.Join("_apps", y.app.Name, y.app.e.g.YttStepDirName))...) if len(yttFiles) == 0 { - log.Debug().Str("app", y.app.Name).Msg("No yaml files found") + log.Debug().Msg(y.app.Msg(yttStepName, "No local ytt directory found")) return "", nil } - log.Debug().Strs("files", yttFiles).Str("app", y.app.Name).Msg("Collected ytt files") - - yamlOutput, err := y.app.e.g.ytt(yttFiles) + yamlOutput, err := y.app.ytt(yttStepName, "render local ytt", yttFiles) if err != nil { - log.Warn().Err(err).Str("app", y.app.Name).Msg("Unable to render ytt files") return "", err } if yamlOutput.Stdout == "" { - log.Warn().Str("app", y.app.Name).Msg("Empty ytt output") - return "", nil + return "", errors.New("empty ytt output") } + log.Info().Msg(y.app.Msg(yttStepName, "Local YTT rendered")) + return yamlOutput.Stdout, nil } diff --git a/internal/myks/render_ytt_pkg.go b/internal/myks/render_ytt_pkg.go index f9b6dd3a..84c14ccd 100644 --- a/internal/myks/render_ytt_pkg.go +++ b/internal/myks/render_ytt_pkg.go @@ -24,12 +24,12 @@ func (y *YttPkg) Ident() string { func (y *YttPkg) Render(_ string) (string, error) { yttPkgRootDir, err := y.app.getVendoredDir(y.app.e.g.YttPkgStepDirName) if err != nil { - log.Err(err).Str("app", y.app.Name).Msg("Unable to get ytt package dir") + log.Err(err).Msg(y.app.Msg(yttPkgStepName, "Unable to get ytt package dir")) return "", err } yttPkgSubDirs := getSubDirs(yttPkgRootDir) if len(yttPkgSubDirs) == 0 { - log.Debug().Str("app", y.app.Name).Msg("No packages to process") + log.Debug().Msg(y.app.Msg(yttPkgStepName, "No ytt package found")) return "", nil } @@ -39,7 +39,7 @@ func (y *YttPkg) Render(_ string) (string, error) { pkgName := filepath.Base(pkgDir) var pkgValuesFile string if pkgValuesFile, err = y.app.prepareValuesFile("ytt-pkg", pkgName); err != nil { - log.Warn().Err(err).Str("app", y.app.Name).Msg("Unable to prepare vendir packages value files") + log.Warn().Err(err).Msg(y.app.Msg(globalYttStepName, "Unable to prepare vendir packages value files")) return "", err } @@ -56,19 +56,24 @@ func (y *YttPkg) Render(_ string) (string, error) { yttArgs = append(yttArgs, "--data-values-file="+pkgValuesFile) } - res, err := runYttWithFilesAndStdin(yttFiles, nil, yttArgs...) + res, err := runYttWithFilesAndStdin(yttFiles, nil, func(name string, args []string) { + // make this copy-n-pastable + log.Debug().Msg(msgRunCmd("ytt-pkg render step", name, args)) + }, yttArgs...) if err != nil { - log.Error().Err(err).Str("app", y.app.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to run ytt") + log.Error().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg(y.app.Msg(globalYttStepName, "Unable to run ytt")) return "", err } if res.Stdout == "" { - log.Warn().Str("app", y.app.Name).Str("pkgName", pkgName).Msg("No ytt package output") + log.Warn().Str("pkgName", pkgName).Msg(y.app.Msg(globalYttStepName, "No ytt package output")) continue } outputs = append(outputs, res.Stdout) } + log.Info().Msg(y.app.Msg(helmStepName, "Ytt package rendered")) + return strings.Join(outputs, "---\n"), nil } diff --git a/internal/myks/sync.go b/internal/myks/sync.go index 31aae745..a3bc825c 100644 --- a/internal/myks/sync.go +++ b/internal/myks/sync.go @@ -19,8 +19,10 @@ type Directory struct { } func (a *Application) Sync() error { + log.Debug().Msg(a.Msg(syncStepName, "Starting")) if err := a.prepareSync(); err != nil { if err == ErrNoVendirConfig { + log.Info().Msg(a.Msg(syncStepName, "No vendir config found")) return nil } return err @@ -43,7 +45,6 @@ func (a *Application) prepareSync() error { protoVendirDir := filepath.Join(a.Prototype, "vendir") if _, err := os.Stat(protoVendirDir); err == nil { yttFiles = append(yttFiles, protoVendirDir) - log.Debug().Str("dir", protoVendirDir).Msg("Using prototype vendir directory") } appVendirDirs := a.e.collectBySubpath(filepath.Join("_apps", a.Name, "vendir")) @@ -51,19 +52,17 @@ func (a *Application) prepareSync() error { if len(yttFiles) == 0 { err := ErrNoVendirConfig - log.Warn().Err(err).Str("app", a.Name).Msg("") return err } - vendirConfig, err := a.e.g.ytt(yttFiles) + vendirConfig, err := a.ytt(syncStepName, "creating vendir config", yttFiles) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Msg("Unable to render vendir config") + log.Warn().Err(err).Msg(a.Msg(syncStepName, "Unable to render vendir config")) return err } if vendirConfig.Stdout == "" { err = errors.New("Empty vendir config") - log.Warn().Err(err).Msg("") return err } @@ -71,15 +70,14 @@ func (a *Application) prepareSync() error { // Create directory if it does not exist err = os.MkdirAll(filepath.Dir(vendirConfigFilePath), 0o750) if err != nil { - log.Warn().Err(err).Msg("Unable to create directory for vendir config file") + log.Warn().Err(err).Msg(a.Msg(syncStepName, "Unable to create directory for vendir config file")) return err } err = os.WriteFile(vendirConfigFilePath, []byte(vendirConfig.Stdout), 0o600) if err != nil { - log.Warn().Err(err).Msg("Unable to write vendir config file") + log.Warn().Err(err).Msg(a.Msg(syncStepName, "Unable to write vendir config file")) return err } - log.Debug().Str("app", a.Name).Str("file", vendirConfigFilePath).Msg("Wrote vendir config file") return nil } @@ -95,28 +93,25 @@ func (a *Application) doSync() error { vendirDirs, err := readVendirConfig(vendirConfigFilePath) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Error while trying to find directories in vendir config: " + vendirConfigFilePath) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Error while trying to find directories in vendir config: "+vendirConfigFilePath)) return err } syncFileDirs, err := readSyncFile(vendirSyncFilePath) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncFilePath) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to read Vendir Sync file: "+vendirSyncFilePath)) return err } - if len(syncFileDirs) == 0 { - log.Debug().Str("app", a.Name).Msg("Vendir sync file not found. First sync..") - } lockFileDirs, err := readLockFile(vendirLockFilePath) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Lock file: " + vendirLockFilePath) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to read Vendir Lock file: "+vendirLockFilePath)) return err } err = createDirectory(vendorDir) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to create vendor dir: " + vendorDir) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to create vendor dir: "+vendorDir)) return err } @@ -126,10 +121,9 @@ func (a *Application) doSync() error { if a.cached && checkLockFileMatch(vendirDirs, lockFileDirs) { for _, dir := range vendirDirs { if checkVersionMatch(dir.Path, dir.ContentHash, syncFileDirs) { - log.Debug().Str("app", a.Name).Msg("Skipping vendir sync for: " + dir.Path) + log.Info().Msg(a.Msg(syncStepName, "Resource already synced")) continue } - log.Info().Str("app", a.Name).Msg("Syncing vendir for: " + dir.Path) args := []string{ "sync", "--chdir=" + vendorDir, @@ -137,23 +131,23 @@ func (a *Application) doSync() error { "--file=" + vendirConfigFileRelativePath, "--lock-file=" + vendirLockFileRelativePath, } - args, secretFilePath, err := handleVendirSecret(a.Name, dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) + args, secretFilePath, err := handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to create secret for: "+dir.Path)) return err } if secretFilePath != "" { secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath) } - res, err := runCmd("vendir", nil, args) + res, err := a.runCmd("vendir directory sync", "vendir", nil, args) if err != nil { - log.Warn().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") + log.Warn().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg(a.Msg(syncStepName, "Unable to sync vendir")) return err } + log.Info().Msg(a.Msg(syncStepName, "Synced")) } } else { - log.Info().Str("app", a.Name).Msg("Syncing vendir completely for: " + vendirConfigFilePath) args := []string{ "sync", "--chdir=" + vendorDir, @@ -162,9 +156,9 @@ func (a *Application) doSync() error { } for _, dir := range vendirDirs { var secretFilePath string - args, secretFilePath, err = handleVendirSecret(a.Name, dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) + args, secretFilePath, err = handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path) + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to create secret for: "+dir.Path)) return err } if secretFilePath != "" { @@ -172,33 +166,31 @@ func (a *Application) doSync() error { } } - res, err := runCmd("vendir", nil, args) + res, err := a.runCmd("vendir full sync", "vendir", nil, args) if err != nil { - log.Error().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") + log.Error().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg(a.Msg(syncStepName, "Unable to sync vendir")) return err } + log.Info().Msg(a.Msg(syncStepName, "Synced")) } // make sure secrets do not linger on disk for _, secretFilePath := range secretFilePaths { defer func(name string) { - log.Debug().Str("app", a.Name).Msg("delete secret file: " + name) + log.Debug().Msg(a.Msg(syncStepName, "delete secret file: "+name)) err := os.Remove(name) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("unable to delete secret file: " + name) + log.Error().Err(err).Msg(a.Msg(syncStepName, "unable to delete secret file: "+name)) } }(secretFilePath) } err = writeSyncFile(a.expandTempPath(a.e.g.VendirSyncFileName), vendirDirs) if err != nil { - log.Error().Str("app", a.Name).Err(err).Msg("Unable to write sync file") + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to write sync file")) return err } - log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncFilePath) - log.Info().Str("app", a.Name).Msg("Vendir sync completed!") - return nil } @@ -215,7 +207,7 @@ func writeSyncFile(syncFilePath string, directories []Directory) error { return nil } -func handleVendirSecret(app string, dir Directory, tempPath string, tempRelativePath string, vendirArgs []string) ([]string, string, error) { +func handleVendirSecret(dir Directory, tempPath string, tempRelativePath string, vendirArgs []string) ([]string, string, error) { if dir.Secret != "" { username, password, err := getEnvCreds(dir.Secret) if err != nil { diff --git a/internal/myks/sync_test.go b/internal/myks/sync_test.go index 1ff975e8..42a1c8af 100644 --- a/internal/myks/sync_test.go +++ b/internal/myks/sync_test.go @@ -341,7 +341,7 @@ func Test_handleVendirSecret(t *testing.T) { t.Run(tt.name, func(t *testing.T) { setEnvSafely(tt.args.envUsernameKey, tt.args.envUsernameValue, t) setEnvSafely(tt.args.envPasswordKey, tt.args.envPasswordValue, t) - got, secretPath, err := handleVendirSecret("", tt.args.dir, tt.args.tempPath, tt.args.tempRelativePath, tt.args.vendirArgs) + got, secretPath, err := handleVendirSecret(tt.args.dir, tt.args.tempPath, tt.args.tempRelativePath, tt.args.vendirArgs) if (err != nil) != tt.wantErr { t.Errorf("handleVendirSecret() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/internal/myks/template.go b/internal/myks/template.go index 6f63659e..747c10c5 100644 --- a/internal/myks/template.go +++ b/internal/myks/template.go @@ -2,6 +2,7 @@ package myks import ( "embed" + "github.com/rs/zerolog/log" "os" "path/filepath" ) @@ -14,8 +15,9 @@ func writeSecretFile(secretName string, secretFilePath string, username string, if err != nil { return err } - - res, err := runYttWithFilesAndStdin([]string{filepath.Join(os.TempDir(), "templates", "secret.yaml")}, nil, "--data-value=secret_name="+secretName, "--data-value=username="+username, "--data-value=password="+password) + res, err := runYttWithFilesAndStdin([]string{filepath.Join(os.TempDir(), "templates", "secret.yaml")}, nil, func(name string, args []string) { + log.Debug().Msg(msgRunCmd("render vendir secret yaml", name, args)) + }, "--data-value=secret_name="+secretName, "--data-value=username="+username, "--data-value=password="+password) if err != nil { return err } diff --git a/internal/myks/util.go b/internal/myks/util.go index 4ce1a587..04967636 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "errors" "fmt" "io" "io/fs" @@ -24,30 +23,6 @@ type CmdResult struct { Stderr string } -func runCmd(name string, stdin io.Reader, args []string) (CmdResult, error) { - if log.Debug().Enabled() { - logArgs := reductSecrets(args) - // make this copy-n-pastable - log.Debug().Msg("Running command:\n" + name + " " + strings.Join(logArgs, " ")) - } - cmd := exec.Command(name, args...) - - if stdin != nil { - cmd.Stdin = stdin - } - - var stdoutBs, stderrBs bytes.Buffer - cmd.Stdout = &stdoutBs - cmd.Stderr = &stderrBs - - err := cmd.Run() - - return CmdResult{ - Stdout: stdoutBs.String(), - Stderr: stderrBs.String(), - }, err -} - func reductSecrets(args []string) []string { sensitiveFields := []string{"password", "secret", "token"} var logArgs []string @@ -59,34 +34,6 @@ func reductSecrets(args []string) []string { return logArgs } -func runYttWithFilesAndStdin(paths []string, stdin io.Reader, args ...string) (CmdResult, error) { - if stdin != nil { - paths = append(paths, "-") - } - - cmdArgs := []string{} - for _, path := range paths { - cmdArgs = append(cmdArgs, "--file="+path) - } - - cmdArgs = append(cmdArgs, args...) - res, err := runCmd("ytt", stdin, cmdArgs) - if err != nil { - log.Warn().Str("cmd", "ytt").Interface("args", cmdArgs).Msg("Failed to run command\n" + res.Stderr) - } - - return res, err -} - -func contains(list []string, item string) bool { - for _, listItem := range list { - if listItem == item { - return true - } - } - return false -} - func processItemsInParallel(collection interface{}, fn func(interface{}) error) error { var items []interface{} @@ -224,27 +171,6 @@ func hash(s string) string { return hex.EncodeToString(hash[:]) } -func renderDataYaml(app string, dataFiles []string) ([]byte, error) { - if len(dataFiles) == 0 { - return nil, errors.New("No data files found") - } - res, err := runYttWithFilesAndStdin(dataFiles, nil, "--data-values-inspect") - if err != nil { - log.Error().Err(err).Str("stderr", res.Stderr).Msg("Unable to render data") - return nil, err - } - if res.Stdout == "" { - return nil, errors.New("Empty output from ytt") - } - - dataYaml := []byte(res.Stdout) - return dataYaml, nil -} - -func mergeValuesYaml(valueFilesYaml string) (CmdResult, error) { - return runYttWithFilesAndStdin(nil, nil, "--data-values-file="+valueFilesYaml, "--data-values-inspect") -} - func createDirectory(dir string) error { if _, err := os.Stat(dir); err != nil { err := os.MkdirAll(dir, 0o750) @@ -303,3 +229,50 @@ func getSubDirs(rootDir string) []string { return resourceDirs } + +func runCmd(name string, stdin io.Reader, args []string, log func(name string, args []string)) (CmdResult, error) { + cmd := exec.Command(name, args...) + + if stdin != nil { + cmd.Stdin = stdin + } + + var stdoutBs, stderrBs bytes.Buffer + cmd.Stdout = &stdoutBs + cmd.Stderr = &stderrBs + + err := cmd.Run() + + if log != nil { + log(name, args) + } + + return CmdResult{ + Stdout: stdoutBs.String(), + Stderr: stderrBs.String(), + }, err +} + +func msgRunCmd(purpose string, cmd string, args []string) string { + msg := cmd + " " + strings.Join(reductSecrets(args), " ") + return "Running \u001B[34m" + cmd + "\u001B[0m to: \u001B[3m" + purpose + "\u001B[0m\n\u001B[37m" + msg + "\u001B[0m" +} + +func runYttWithFilesAndStdin(paths []string, stdin io.Reader, log func(name string, args []string), args ...string) (CmdResult, error) { + if stdin != nil { + paths = append(paths, "-") + } + + cmdArgs := []string{} + for _, path := range paths { + cmdArgs = append(cmdArgs, "--file="+path) + } + + cmdArgs = append(cmdArgs, args...) + res, err := runCmd("ytt", stdin, cmdArgs, log) + if err != nil { + return res, err + } + + return res, err +} diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index 26bb94ad..0c0b44ba 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -1,6 +1,7 @@ package myks import ( + "io" "os" "reflect" "testing" @@ -90,35 +91,6 @@ func Test_unmarshalYaml(t *testing.T) { } } -func Test_renderDataYaml(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in pipeline since ytt is not installed") - } - type args struct { - dataFiles []string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - {"happy path", args{[]string{"../../testData/ytt/data-file-schema.yaml", "../../testData/ytt/data-file-schema-2.yaml", "../../testData/ytt/data-file-values.yaml"}}, "application:\n cache:\n enabled: true\n name: cert-manager\n", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := renderDataYaml("", tt.args.dataFiles) - if (err != nil) != tt.wantErr { - t.Errorf("renderDataYaml() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(string(got), tt.want) { - t.Errorf("renderDataYaml() got = %v, wantArgs %v", string(got), tt.want) - } - }) - } -} - func Test_createDirectory(t *testing.T) { type args struct { dir string @@ -235,3 +207,65 @@ func Test_getSubDirs(t *testing.T) { }) } } + +func Test_runCmd(t *testing.T) { + type args struct { + name string + stdin io.Reader + args []string + log func(name string, args []string) + } + tests := []struct { + name string + args args + want CmdResult + wantErr bool + }{ + {"happy path", args{"echo", nil, []string{"test"}, nil}, CmdResult{"test\n", ""}, false}, + {"said path", args{"sure-to-fail", nil, []string{}, nil}, CmdResult{"", ""}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := runCmd(tt.args.name, tt.args.stdin, tt.args.args, tt.args.log) + if (err != nil) != tt.wantErr { + t.Errorf("runCmd() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("runCmd() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_runYttWithFilesAndStdin(t *testing.T) { + type args struct { + paths []string + stdin io.Reader + log func(name string, args []string) + args []string + } + tests := []struct { + name string + args args + want CmdResult + wantErr bool + }{ + {"happy path", args{[]string{"../../testData/ytt/simple.yaml"}, nil, nil, []string{}}, CmdResult{"key1: A\n", ""}, false}, + {"said path", args{[]string{"does-not-exist.yaml"}, nil, nil, []string{}}, CmdResult{"", ""}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := runYttWithFilesAndStdin(tt.args.paths, tt.args.stdin, tt.args.log, tt.args.args...) + if (err != nil) != tt.wantErr { + t.Errorf("runYttWithFilesAndStdin() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil { + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("runYttWithFilesAndStdin() got = %v, want %v", got, tt.want) + } + } + }) + } +} diff --git a/main.go b/main.go index dfb6cde6..428fba02 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,9 @@ package main import ( - "os" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "os" "github.com/mykso/myks/cmd" ) diff --git a/testData/ytt/simple.yaml b/testData/ytt/simple.yaml new file mode 100644 index 00000000..4280bd0c --- /dev/null +++ b/testData/ytt/simple.yaml @@ -0,0 +1 @@ +key1: A