From 62a2eb501cd265d38151a08b8d3eb082626463a3 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 20:03:41 +0100 Subject: [PATCH 01/40] refactor(cli): move findBaseDirs to tanka package --- cmd/tk/args.go | 16 +++++++++++++++- cmd/tk/env.go | 12 ++++++++++-- cmd/tk/other.go | 36 ------------------------------------ pkg/tanka/environments.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 39 deletions(-) delete mode 100644 cmd/tk/other.go create mode 100644 pkg/tanka/environments.go diff --git a/cmd/tk/args.go b/cmd/tk/args.go index 24f39b474..4c95a63ec 100644 --- a/cmd/tk/args.go +++ b/cmd/tk/args.go @@ -1,14 +1,28 @@ package main import ( + "os" + "github.com/go-clix/cli" "github.com/posener/complete" + + "github.com/grafana/tanka/pkg/tanka" ) var workflowArgs = cli.Args{ Validator: cli.ValidateExact(1), Predictor: cli.PredictFunc(func(args complete.Args) []string { - if dirs := findBaseDirs(); len(dirs) != 0 { + pwd, err := os.Getwd() + if err != nil { + return nil + } + + dirs, err := tanka.FindBaseDirs(pwd) + if err != nil { + return nil + } + + if len(dirs) != 0 { return dirs } diff --git a/cmd/tk/env.go b/cmd/tk/env.go index b21f648b9..d0e2bcafd 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes/client" "github.com/grafana/tanka/pkg/spec/v1alpha1" + "github.com/grafana/tanka/pkg/tanka" "github.com/grafana/tanka/pkg/term" ) @@ -227,9 +228,16 @@ func envListCmd() *cli.Command { cmd.Run = func(cmd *cli.Command, args []string) error { envs := []v1alpha1.Environment{} - dirs := findBaseDirs() + pwd, err := os.Getwd() + if err != nil { + return err + } + dirs, err := tanka.FindBaseDirs(pwd) + if err != nil { + return err + } + var selector labels.Selector - var err error if *labelSelector != "" { selector, err = labels.Parse(*labelSelector) diff --git a/cmd/tk/other.go b/cmd/tk/other.go deleted file mode 100644 index 97558abe4..000000000 --- a/cmd/tk/other.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "log" - "os" - "path/filepath" - - "github.com/grafana/tanka/pkg/jsonnet/jpath" -) - -// findBaseDirs searches for possible environments -func findBaseDirs() (dirs []string) { - pwd, err := os.Getwd() - if err != nil { - return - } - _, _, _, err = jpath.Resolve(pwd) - if err == jpath.ErrorNoRoot { - return - } - - if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { - requiredFiles := []string{"main.jsonnet"} - for _, name := range requiredFiles { - if _, err := os.Stat(filepath.Join(path, name)); err != nil { - // missing file, not a valid environment directory - return nil - } - } - dirs = append(dirs, path) - return nil - }); err != nil { - log.Fatalln(err) - } - return dirs -} diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go new file mode 100644 index 000000000..ee2805399 --- /dev/null +++ b/pkg/tanka/environments.go @@ -0,0 +1,30 @@ +package tanka + +import ( + "os" + "path/filepath" + + "github.com/grafana/tanka/pkg/jsonnet/jpath" +) + +const BASEDIR_INDICATOR = "main.jsonnet" + +// FindBaseDirs searches for possible environments +func FindBaseDirs(workdir string) (dirs []string, err error) { + _, _, _, err = jpath.Resolve(workdir) + if err == jpath.ErrorNoRoot { + return nil, err + } + + if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if _, err := os.Stat(filepath.Join(path, BASEDIR_INDICATOR)); err != nil { + // missing file, not a valid environment directory + return nil + } + dirs = append(dirs, path) + return nil + }); err != nil { + return nil, err + } + return dirs, nil +} From 809bbcd24cde0566ffd9a27946df7332fa4d5776 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 20:30:14 +0100 Subject: [PATCH 02/40] refactor(cli): use tanka.FindEnvironments() --- cmd/tk/env.go | 19 +++---------------- pkg/tanka/environments.go | 32 +++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index d0e2bcafd..61b73f532 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "log" "os" "path/filepath" "text/tabwriter" @@ -227,18 +226,12 @@ func envListCmd() *cli.Command { useNames := cmd.Flags().Bool("names", false, "plain names output") cmd.Run = func(cmd *cli.Command, args []string) error { - envs := []v1alpha1.Environment{} pwd, err := os.Getwd() if err != nil { return err } - dirs, err := tanka.FindBaseDirs(pwd) - if err != nil { - return err - } var selector labels.Selector - if *labelSelector != "" { selector, err = labels.Parse(*labelSelector) if err != nil { @@ -246,15 +239,9 @@ func envListCmd() *cli.Command { } } - for _, dir := range dirs { - env := setupConfiguration(dir) - if env == nil { - log.Printf("Could not setup configuration from %q", dir) - continue - } - if selector == nil || selector.Empty() || selector.Matches(env.Metadata) { - envs = append(envs, *env) - } + envs, err := tanka.FindEnvironments(pwd, selector) + if err != nil { + return err } if *useJSON { diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index ee2805399..acb3cd4e3 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -4,7 +4,10 @@ import ( "os" "path/filepath" + "k8s.io/apimachinery/pkg/labels" + "github.com/grafana/tanka/pkg/jsonnet/jpath" + "github.com/grafana/tanka/pkg/spec/v1alpha1" ) const BASEDIR_INDICATOR = "main.jsonnet" @@ -12,7 +15,7 @@ const BASEDIR_INDICATOR = "main.jsonnet" // FindBaseDirs searches for possible environments func FindBaseDirs(workdir string) (dirs []string, err error) { _, _, _, err = jpath.Resolve(workdir) - if err == jpath.ErrorNoRoot { + if err != nil { return nil, err } @@ -28,3 +31,30 @@ func FindBaseDirs(workdir string) (dirs []string, err error) { } return dirs, nil } + +// FindEnvironments searches for actual environments +// ignores main.jsonnet if no environments found +func FindEnvironments(workdir string, selector labels.Selector) (envs []v1alpha1.Environment, err error) { + dirs, err := FindBaseDirs(workdir) + if err != nil { + return nil, err + } + + for _, dir := range dirs { + _, env, err := ParseEnv(dir, ParseOpts{Evaluator: EnvsOnlyEvaluator}) + if err != nil { + switch err.(type) { + case ErrNoEnv: + continue + default: + return nil, err + } + } + + if selector == nil || selector.Empty() || selector.Matches(env.Metadata) { + envs = append(envs, *env) + } + } + + return envs, nil +} From c96ff106702ad0c58d3e0e0e198df3cb0bf891c9 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 20:31:15 +0100 Subject: [PATCH 03/40] refactor(cli): remove ErrNoSpec logic setupConfiguration is only used in env.go:envSetCmd() which should fail in case it is missing a spec.json file. --- cmd/tk/main.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cmd/tk/main.go b/cmd/tk/main.go index f0de64993..0945b6a0b 100644 --- a/cmd/tk/main.go +++ b/cmd/tk/main.go @@ -77,18 +77,6 @@ func setupConfiguration(baseDir string) *v1alpha1.Environment { if verbose { fmt.Print(err) } - // no spec.json is found, try parsing main.jsonnet - case spec.ErrNoSpec: - _, config, err := tanka.ParseEnv(baseDir, tanka.JsonnetOpts{EvalScript: tanka.EnvsOnlyEvalScript}) - if err != nil { - switch err.(type) { - case tanka.ErrNoEnv: - return nil - default: - log.Fatalf("Reading main.jsonnet: %s", err) - } - } - return config // some other error default: log.Fatalf("Reading spec.json: %s", err) From 652ba037a4b26db47daa0ee9986ecbf5e2c8ce7f Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 20:47:55 +0100 Subject: [PATCH 04/40] refactor(cli): move setupConfiguration to env.go --- cmd/tk/env.go | 28 ++++++++++++++++++++++++++++ cmd/tk/main.go | 34 +--------------------------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index 61b73f532..f46f08792 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "log" "os" "path/filepath" "text/tabwriter" @@ -15,6 +16,7 @@ import ( "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes/client" + "github.com/grafana/tanka/pkg/spec" "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/grafana/tanka/pkg/tanka" "github.com/grafana/tanka/pkg/term" @@ -50,6 +52,32 @@ var kubectlContexts = cli.PredictFunc( }, ) +func setupConfiguration(baseDir string) *v1alpha1.Environment { + _, baseDir, rootDir, err := jpath.Resolve(baseDir) + if err != nil { + log.Fatalln("Resolving jpath:", err) + } + + // name of the environment: relative path from rootDir + name, _ := filepath.Rel(rootDir, baseDir) + + config, err := spec.ParseDir(baseDir, name) + if err != nil { + switch err.(type) { + // the config includes deprecated fields + case spec.ErrDeprecated: + if verbose { + fmt.Print(err) + } + // some other error + default: + log.Fatalf("Reading spec.json: %s", err) + } + } + + return config +} + func envSetCmd() *cli.Command { cmd := &cli.Command{ Use: "set", diff --git a/cmd/tk/main.go b/cmd/tk/main.go index 0945b6a0b..67e416bf9 100644 --- a/cmd/tk/main.go +++ b/cmd/tk/main.go @@ -1,18 +1,12 @@ package main import ( - "fmt" "log" "os" - "path/filepath" - - "golang.org/x/crypto/ssh/terminal" "github.com/go-clix/cli" + "golang.org/x/crypto/ssh/terminal" - "github.com/grafana/tanka/pkg/jsonnet/jpath" - "github.com/grafana/tanka/pkg/spec" - "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/grafana/tanka/pkg/tanka" ) @@ -59,29 +53,3 @@ func main() { log.Fatalln(err) } } - -func setupConfiguration(baseDir string) *v1alpha1.Environment { - _, baseDir, rootDir, err := jpath.Resolve(baseDir) - if err != nil { - log.Fatalln("Resolving jpath:", err) - } - - // name of the environment: relative path from rootDir - name, _ := filepath.Rel(rootDir, baseDir) - - config, err := spec.ParseDir(baseDir, name) - if err != nil { - switch err.(type) { - // the config includes deprecated fields - case spec.ErrDeprecated: - if verbose { - fmt.Print(err) - } - // some other error - default: - log.Fatalf("Reading spec.json: %s", err) - } - } - - return config -} From a468ebd0595d23c471e750122f1a0314384cf6e9 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 21:26:17 +0100 Subject: [PATCH 05/40] feat(cli): go relative with `tk env list ` --- cmd/tk/env.go | 14 +++++++++----- pkg/tanka/environments.go | 8 ++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index f46f08792..abbcb0a2c 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -242,10 +242,10 @@ func envRemoveCmd() *cli.Command { func envListCmd() *cli.Command { cmd := &cli.Command{ - Use: "list", + Use: "list ", Aliases: []string{"ls"}, - Short: "list environments", - Args: cli.ArgsNone(), + Short: "list environments relative to ", + Args: workflowArgs, } useJSON := cmd.Flags().Bool("json", false, "json output") @@ -254,10 +254,14 @@ func envListCmd() *cli.Command { useNames := cmd.Flags().Bool("names", false, "plain names output") cmd.Run = func(cmd *cli.Command, args []string) error { - pwd, err := os.Getwd() + dir := args[0] + stat, err := os.Stat(dir) if err != nil { return err } + if !stat.IsDir() { + return fmt.Errorf("Not a directory: %s", dir) + } var selector labels.Selector if *labelSelector != "" { @@ -267,7 +271,7 @@ func envListCmd() *cli.Command { } } - envs, err := tanka.FindEnvironments(pwd, selector) + envs, err := tanka.FindEnvironments(dir, selector) if err != nil { return err } diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index acb3cd4e3..6695cd8ed 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -15,11 +15,11 @@ const BASEDIR_INDICATOR = "main.jsonnet" // FindBaseDirs searches for possible environments func FindBaseDirs(workdir string) (dirs []string, err error) { _, _, _, err = jpath.Resolve(workdir) - if err != nil { + if err == jpath.ErrorNoRoot { return nil, err } - if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(workdir, func(path string, info os.FileInfo, err error) error { if _, err := os.Stat(filepath.Join(path, BASEDIR_INDICATOR)); err != nil { // missing file, not a valid environment directory return nil @@ -34,7 +34,7 @@ func FindBaseDirs(workdir string) (dirs []string, err error) { // FindEnvironments searches for actual environments // ignores main.jsonnet if no environments found -func FindEnvironments(workdir string, selector labels.Selector) (envs []v1alpha1.Environment, err error) { +func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha1.Environment, err error) { dirs, err := FindBaseDirs(workdir) if err != nil { return nil, err @@ -52,7 +52,7 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []v1alpha1 } if selector == nil || selector.Empty() || selector.Matches(env.Metadata) { - envs = append(envs, *env) + envs = append(envs, env) } } From cb7bd94db0623309c0caf0307bb6a108f069bf4e Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 21:27:54 +0100 Subject: [PATCH 06/40] chore(git): don't ignore cmd/tk as a dir --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 517ab3db8..18e9056fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ dist -tk +/tk +/cmd/tk/tk From 72869d7e5385815c2ad2fd1bbe990245b5e52381 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 19 Dec 2020 21:38:59 +0100 Subject: [PATCH 07/40] fix(cli): relative path for predict --- cmd/tk/args.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/tk/args.go b/cmd/tk/args.go index 4c95a63ec..46f477601 100644 --- a/cmd/tk/args.go +++ b/cmd/tk/args.go @@ -2,6 +2,7 @@ package main import ( "os" + "path/filepath" "github.com/go-clix/cli" "github.com/posener/complete" @@ -22,8 +23,16 @@ var workflowArgs = cli.Args{ return nil } - if len(dirs) != 0 { - return dirs + var reldirs []string + for _, dir := range dirs { + reldir, err := filepath.Rel(pwd, dir) + if err == nil { + reldirs = append(reldirs, reldir) + } + } + + if len(reldirs) != 0 { + return reldirs } return complete.PredictDirs("*").Predict(args) From 79f53335ee3a601e75788fc71e80038aa6e091da Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 12:29:50 +0100 Subject: [PATCH 08/40] fix: EnvsOnlyEvaluator explicitly removes Data, we should not add it --- pkg/tanka/parse.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/tanka/parse.go b/pkg/tanka/parse.go index 5f6adc137..d4e3ee050 100644 --- a/pkg/tanka/parse.go +++ b/pkg/tanka/parse.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "path/filepath" + "reflect" "github.com/Masterminds/semver" "github.com/pkg/errors" @@ -189,9 +190,11 @@ func ParseEnv(path string, opts JsonnetOpts) (interface{}, *v1alpha1.Environment return nil, nil, err } return data, env, nil - } else if specEnv != nil { - // if no environments found, fallback to original behavior - specEnv.Data = data + } else if specEnv != nil { // if no environments found, fallback to original behavior + // EnvsOnlyEvaluator explicitly removes Data, we should not add it + if reflect.ValueOf(opts.Evaluator).Pointer() != reflect.ValueOf(EnvsOnlyEvaluator).Pointer() { + specEnv.Data = data + } return data, specEnv, nil } // if no environments or spec found, behave as jsonnet interpreter From fc096896fb59ba120047fd0412f827a356731100 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 12:30:56 +0100 Subject: [PATCH 09/40] feat(api): ParseEnvs evaluates multiple envs in parallel --- pkg/tanka/environments.go | 93 ++++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 11 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index 6695cd8ed..c4fe3fd71 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -1,8 +1,10 @@ package tanka import ( + "fmt" "os" "path/filepath" + "sync" "k8s.io/apimachinery/pkg/labels" @@ -11,6 +13,7 @@ import ( ) const BASEDIR_INDICATOR = "main.jsonnet" +const PARALLEL = 8 // FindBaseDirs searches for possible environments func FindBaseDirs(workdir string) (dirs []string, err error) { @@ -33,28 +36,96 @@ func FindBaseDirs(workdir string) (dirs []string, err error) { } // FindEnvironments searches for actual environments -// ignores main.jsonnet if no environments found +// ignores directories if no environments are found func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha1.Environment, err error) { dirs, err := FindBaseDirs(workdir) if err != nil { return nil, err } + opts := ParseOpts{ + Evaluator: EnvsOnlyEvaluator, + Selector: selector, + } + envs, errs := ParseEnvs(dirs, opts, PARALLEL) - for _, dir := range dirs { - _, env, err := ParseEnv(dir, ParseOpts{Evaluator: EnvsOnlyEvaluator}) - if err != nil { - switch err.(type) { - case ErrNoEnv: - continue - default: - return nil, err + var returnErrs string + for _, err := range errs { + switch err.(type) { + case ErrNoEnv: + continue + default: + fmt.Print(err) + returnErrs = fmt.Sprintf("%s\n%s", returnErrs, err) + } + } + if len(returnErrs) != 0 { + return nil, fmt.Errorf("Unable to parse selected Environments: \n%s", returnErrs) + } + + return envs, nil +} + +// ParseEnvs evaluates multiple environments in parallel +func ParseEnvs(paths []string, opts ParseOpts, numParallel int) (envs []*v1alpha1.Environment, errs []error) { + wg := sync.WaitGroup{} + envsChan := make(chan parseEnvsRoutineOpts) + var allErrors []error + + for i := 0; i < numParallel; i++ { + wg.Add(1) + go func() { + err := parseEnvsRoutine(envsChan) + if err != nil { + allErrors = append(allErrors, err) } + wg.Done() + }() + } + + results := make([]*v1alpha1.Environment, len(paths)) + currentIndex := 0 + + for _, path := range paths { + env := &v1alpha1.Environment{} + results[currentIndex] = env + currentIndex++ + envsChan <- parseEnvsRoutineOpts{ + path: path, + opts: opts, + env: env, } + } + close(envsChan) + wg.Wait() - if selector == nil || selector.Empty() || selector.Matches(env.Metadata) { - envs = append(envs, env) + for _, env := range results { + if env != nil { + if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(env.Metadata) { + envs = append(envs, env) + } } } + if len(allErrors) != 0 { + return envs, allErrors + } + return envs, nil } + +type parseEnvsRoutineOpts struct { + path string + opts ParseOpts + env *v1alpha1.Environment +} + +func parseEnvsRoutine(envsChan <-chan parseEnvsRoutineOpts) error { + for req := range envsChan { + _, env, err := ParseEnv(req.path, req.opts) + if err != nil { + return err + } + *req.env = *env + } + return nil +} From 2b5757e03eba70b75493c4a80d140d8bceb76eb1 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 12:52:01 +0100 Subject: [PATCH 10/40] fix: make Parallel an option with sane default --- pkg/tanka/environments.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index c4fe3fd71..b5018c291 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -46,7 +46,7 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha Evaluator: EnvsOnlyEvaluator, Selector: selector, } - envs, errs := ParseEnvs(dirs, opts, PARALLEL) + envs, errs := ParseEnvs(dirs, opts) var returnErrs string for _, err := range errs { @@ -66,11 +66,15 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha } // ParseEnvs evaluates multiple environments in parallel -func ParseEnvs(paths []string, opts ParseOpts, numParallel int) (envs []*v1alpha1.Environment, errs []error) { +func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, errs []error) { wg := sync.WaitGroup{} envsChan := make(chan parseEnvsRoutineOpts) var allErrors []error + numParallel := PARALLEL + if opts.Parallel > 0 { + numParallel = opts.Parallel + } for i := 0; i < numParallel; i++ { wg.Add(1) go func() { From 3f6653fb52d0ac488af86d8ec8fde46d507c9734 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 13:17:15 +0100 Subject: [PATCH 11/40] refactor(cli): load manifests for export in separate block --- cmd/tk/export.go | 10 ++++++---- pkg/tanka/parse.go | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 8ce4a6acb..f8e001915 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -63,11 +63,13 @@ func exportCmd() *cli.Command { return fmt.Errorf("Parsing name format: %s", err) } + _, env, err := tanka.ParseEnv(args[0], tanka.ParseOpts{JsonnetOpts: getJsonnetOpts()}) + if err != nil { + return err + } + // get the manifests - res, err := tanka.Show(args[0], tanka.Opts{ - JsonnetOpts: getJsonnetOpts(), - Filters: stringsToRegexps(vars.targets), - }) + res, err := tanka.LoadManifests(env, stringsToRegexps(vars.targets)) if err != nil { return err } diff --git a/pkg/tanka/parse.go b/pkg/tanka/parse.go index d4e3ee050..8ce029445 100644 --- a/pkg/tanka/parse.go +++ b/pkg/tanka/parse.go @@ -74,23 +74,32 @@ func load(path string, opts Opts) (*loaded, error) { return nil, fmt.Errorf("no Tanka environment found") } + rec, err := LoadManifests(env, opts.Filters) + if err != nil { + return nil, err + } + + return &loaded{ + Resources: rec, + Env: env, + }, nil +} + +func LoadManifests(env *v1alpha1.Environment, filters process.Matchers) (manifest.List, error) { if env.Metadata.Name == "" { - return nil, fmt.Errorf("Environment has no metadata.name set in %s", path) + return nil, fmt.Errorf("Environment has no metadata.name set in %s", env.Metadata.Namespace) } if err := checkVersion(env.Spec.ExpectVersions.Tanka); err != nil { return nil, err } - rec, err := process.Process(env.Data, *env, opts.Filters) + resources, err := process.Process(env.Data, *env, filters) if err != nil { return nil, err } - return &loaded{ - Resources: rec, - Env: env, - }, nil + return resources, nil } // parseSpec parses the `spec.json` of the environment and returns a From f6acfaff1742ce39ff4c70aa68647686016d76ee Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 14:09:07 +0100 Subject: [PATCH 12/40] feat(cli): export with dir template from Environment object --- cmd/tk/export.go | 49 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index f8e001915..3a8e4d470 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -15,6 +16,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/go-clix/cli" + "github.com/grafana/tanka/pkg/kubernetes/manifest" "github.com/grafana/tanka/pkg/tanka" ) @@ -34,6 +36,8 @@ func exportCmd() *cli.Command { } format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames") + dirFormat := cmd.Flags().String("dirformat", "{{.spec.namespace}}/{{.metadata.name}}", "based on tanka.dev/Environment object") + extension := cmd.Flags().String("extension", "yaml", "File extension") merge := cmd.Flags().Bool("merge", false, "Allow merging with existing directory") @@ -54,13 +58,23 @@ func exportCmd() *cli.Command { // exit early if the template is bad // Replace all os.path separators in string with BelRune for creating subfolders - replacedFormat := strings.Replace(*format, string(os.PathSeparator), BelRune, -1) + manifestReplaceFormat := strings.Replace(*format, string(os.PathSeparator), BelRune, -1) - tmpl, err := template.New(""). - Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig - Parse(replacedFormat) // parse template + manifestTemplate, err := template.New(""). + Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig + Parse(manifestReplaceFormat) // parse template if err != nil { - return fmt.Errorf("Parsing name format: %s", err) + return fmt.Errorf("Parsing filename format: %s", err) + } + + // Replace all os.path separators in string with BelRune for creating subfolders + directoryReplaceFormat := strings.Replace(*dirFormat, string(os.PathSeparator), BelRune, -1) + + directoryTemplate, err := template.New(""). + Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig + Parse(directoryReplaceFormat) // parse template + if err != nil { + return fmt.Errorf("Parsing directory format: %s", err) } _, env, err := tanka.ParseEnv(args[0], tanka.ParseOpts{JsonnetOpts: getJsonnetOpts()}) @@ -74,10 +88,33 @@ func exportCmd() *cli.Command { return err } + raw, err := json.Marshal(env) + if err != nil { + return err + } + + var m manifest.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return err + } + + buf := bytes.Buffer{} + if err := directoryTemplate.Execute(&buf, m); err != nil { + log.Fatalln("executing directory template:", err) + } + + // Replace all os.path separators in string in order to not accidentally create subfolders + dir := strings.Replace(buf.String(), string(os.PathSeparator), "-", -1) + // Replace the BEL character inserted with a path separator again in order to create a subfolder + dir = strings.Replace(dir, BelRune, string(os.PathSeparator), -1) + + // Create all subfolders in path + to = filepath.Join(to, dir) + // write each to a file for _, m := range res { buf := bytes.Buffer{} - if err := tmpl.Execute(&buf, m); err != nil { + if err := manifestTemplate.Execute(&buf, m); err != nil { log.Fatalln("executing name template:", err) } From 3fde4b59514b329a788914f09b69632266e58df0 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sun, 20 Dec 2020 18:05:52 +0100 Subject: [PATCH 13/40] refactor(cli): move export logic to pkg/tanka --- cmd/tk/export.go | 127 ++++-------------------------- cmd/tk/workflow.go | 17 +--- pkg/tanka/export.go | 186 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 126 deletions(-) create mode 100644 pkg/tanka/export.go diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 3a8e4d470..f6ae6bd47 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -1,22 +1,11 @@ package main import ( - "bytes" - "encoding/json" - "fmt" "io" - "io/ioutil" - "log" "os" - "path/filepath" - "strings" - "text/template" - - "github.com/Masterminds/sprig/v3" "github.com/go-clix/cli" - "github.com/grafana/tanka/pkg/kubernetes/manifest" "github.com/grafana/tanka/pkg/tanka" ) @@ -35,115 +24,27 @@ func exportCmd() *cli.Command { Args: args, } - format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames") - dirFormat := cmd.Flags().String("dirformat", "{{.spec.namespace}}/{{.metadata.name}}", "based on tanka.dev/Environment object") + defaultOpts := tanka.DefaultExportEnvOpts() + + format := cmd.Flags().String("format", defaultOpts.Format, "https://tanka.dev/exporting#filenames") + dirFormat := cmd.Flags().String("dirformat", defaultOpts.DirFormat, "based on tanka.dev/Environment object") - extension := cmd.Flags().String("extension", "yaml", "File extension") - merge := cmd.Flags().Bool("merge", false, "Allow merging with existing directory") + extension := cmd.Flags().String("extension", defaultOpts.Extension, "File extension") + merge := cmd.Flags().Bool("merge", defaultOpts.Merge, "Allow merging with existing directory") vars := workflowFlags(cmd.Flags()) getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - // dir must be empty - to := args[1] - empty, err := dirEmpty(to) - if err != nil { - return fmt.Errorf("Checking target dir: %s", err) - } - if !empty && !*merge { - return fmt.Errorf("Output dir `%s` not empty. Pass --merge to ignore this", to) - } - - // exit early if the template is bad - - // Replace all os.path separators in string with BelRune for creating subfolders - manifestReplaceFormat := strings.Replace(*format, string(os.PathSeparator), BelRune, -1) - - manifestTemplate, err := template.New(""). - Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig - Parse(manifestReplaceFormat) // parse template - if err != nil { - return fmt.Errorf("Parsing filename format: %s", err) + opts := tanka.ExportEnvOpts{ + Format: *format, + DirFormat: *dirFormat, + Extension: *extension, + Targets: vars.targets, + Merge: *merge, + JsonnetOpts: getJsonnetOpts(), } - - // Replace all os.path separators in string with BelRune for creating subfolders - directoryReplaceFormat := strings.Replace(*dirFormat, string(os.PathSeparator), BelRune, -1) - - directoryTemplate, err := template.New(""). - Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig - Parse(directoryReplaceFormat) // parse template - if err != nil { - return fmt.Errorf("Parsing directory format: %s", err) - } - - _, env, err := tanka.ParseEnv(args[0], tanka.ParseOpts{JsonnetOpts: getJsonnetOpts()}) - if err != nil { - return err - } - - // get the manifests - res, err := tanka.LoadManifests(env, stringsToRegexps(vars.targets)) - if err != nil { - return err - } - - raw, err := json.Marshal(env) - if err != nil { - return err - } - - var m manifest.Manifest - if err := json.Unmarshal(raw, &m); err != nil { - return err - } - - buf := bytes.Buffer{} - if err := directoryTemplate.Execute(&buf, m); err != nil { - log.Fatalln("executing directory template:", err) - } - - // Replace all os.path separators in string in order to not accidentally create subfolders - dir := strings.Replace(buf.String(), string(os.PathSeparator), "-", -1) - // Replace the BEL character inserted with a path separator again in order to create a subfolder - dir = strings.Replace(dir, BelRune, string(os.PathSeparator), -1) - - // Create all subfolders in path - to = filepath.Join(to, dir) - - // write each to a file - for _, m := range res { - buf := bytes.Buffer{} - if err := manifestTemplate.Execute(&buf, m); err != nil { - log.Fatalln("executing name template:", err) - } - - // Replace all os.path separators in string in order to not accidentally create subfolders - name := strings.Replace(buf.String(), string(os.PathSeparator), "-", -1) - // Replace the BEL character inserted with a path separator again in order to create a subfolder - name = strings.Replace(name, BelRune, string(os.PathSeparator), -1) - - // Create all subfolders in path - path := filepath.Join(to, name+"."+*extension) - - // Abort if already exists - if exists, err := fileExists(path); err != nil { - return err - } else if exists { - return fmt.Errorf("File '%s' already exists. Aborting", path) - } - - // Write file - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) - } - data := m.String() - if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { - return fmt.Errorf("writing manifest: %s", err) - } - } - - return nil + return tanka.ExportEnvironment(args[0], args[1], &opts) } return cmd } diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index 4f1d46a61..482c5bcdd 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -10,7 +10,6 @@ import ( "github.com/go-clix/cli" - "github.com/grafana/tanka/pkg/process" "github.com/grafana/tanka/pkg/tanka" "github.com/grafana/tanka/pkg/term" ) @@ -49,7 +48,7 @@ func applyCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = stringsToRegexps(vars.targets) + opts.Filters = tanka.StringsToRegexps(vars.targets) opts.JsonnetOpts = getJsonnetOpts() return tanka.Apply(args[0], opts) @@ -94,7 +93,7 @@ func deleteCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = stringsToRegexps(vars.targets) + opts.Filters = tanka.StringsToRegexps(vars.targets) opts.JsonnetOpts = getJsonnetOpts() return tanka.Delete(args[0], opts) @@ -120,7 +119,7 @@ func diffCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = stringsToRegexps(vars.targets) + opts.Filters = tanka.StringsToRegexps(vars.targets) opts.JsonnetOpts = getJsonnetOpts() changes, err := tanka.Diff(args[0], opts) @@ -167,7 +166,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) pretty, err := tanka.Show(args[0], tanka.Opts{ JsonnetOpts: getJsonnetOpts(), - Filters: stringsToRegexps(vars.targets), + Filters: tanka.StringsToRegexps(vars.targets), }) if err != nil { @@ -178,11 +177,3 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) } return cmd } - -func stringsToRegexps(exps []string) process.Matchers { - regexs, err := process.StrExps(exps...) - if err != nil { - log.Fatalln(err) - } - return regexs -} diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go new file mode 100644 index 000000000..3b250b82c --- /dev/null +++ b/pkg/tanka/export.go @@ -0,0 +1,186 @@ +package tanka + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "text/template" + + "github.com/Masterminds/sprig/v3" + + "github.com/grafana/tanka/pkg/kubernetes/manifest" + "github.com/grafana/tanka/pkg/process" +) + +// BelRune is a string of the Ascii character BEL which made computers ring in ancient times +// We use it as "magic" char for the subfolder creation as it is a non printable character and thereby will never be +// in a valid filepath by accident. Only when we include it. +const BelRune = string(rune(7)) + +type ExportEnvOpts struct { + Format string + DirFormat string + Extension string + Targets []string + Merge bool + JsonnetOpts JsonnetOpts +} + +func DefaultExportEnvOpts() ExportEnvOpts { + return ExportEnvOpts{ + Format: "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", + DirFormat: "{{.spec.namespace}}/{{.metadata.name}}", + Extension: "yaml", + Merge: false, + } +} + +func ExportEnvironment(path, to string, opts *ExportEnvOpts) error { + // dir must be empty + empty, err := dirEmpty(to) + if err != nil { + return fmt.Errorf("Checking target dir: %s", err) + } + if !empty && !opts.Merge { + return fmt.Errorf("Output dir `%s` not empty. Pass --merge to ignore this", to) + } + + // exit early if the template is bad + + manifestTemplate, err := createTemplate(opts.Format) + if err != nil { + return fmt.Errorf("Parsing filename format: %s", err) + } + + directoryTemplate, err := createTemplate(opts.DirFormat) + if err != nil { + return fmt.Errorf("Parsing directory format: %s", err) + } + + _, env, err := ParseEnv(path, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) + if err != nil { + return err + } + + // get the manifests + res, err := LoadManifests(env, StringsToRegexps(opts.Targets)) + if err != nil { + return err + } + + raw, err := json.Marshal(env) + if err != nil { + return err + } + + var m manifest.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return err + } + + dir, err := applyTemplate(directoryTemplate, m) + if err != nil { + log.Fatalln("executing directory template:", err) + } + + // Create all subfolders in path + to = filepath.Join(to, dir) + + // write each to a file + for _, m := range res { + name, err := applyTemplate(manifestTemplate, m) + if err != nil { + log.Fatalln("executing name template:", err) + } + + // Create all subfolders in path + path := filepath.Join(to, name+"."+opts.Extension) + + // Abort if already exists + if exists, err := fileExists(path); err != nil { + return err + } else if exists { + return fmt.Errorf("File '%s' already exists. Aborting", path) + } + + // Write file + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) + } + data := m.String() + if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { + return fmt.Errorf("writing manifest: %s", err) + } + } + + return nil +} + +func fileExists(name string) (bool, error) { + _, err := os.Stat(name) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func dirEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if os.IsNotExist(err) { + return true, os.MkdirAll(dir, os.ModePerm) + } else if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +func createTemplate(format string) (*template.Template, error) { + // Replace all os.path separators in string with BelRune for creating subfolders + replaceFormat := strings.Replace(format, string(os.PathSeparator), BelRune, -1) + + template, err := template.New(""). + Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig + Parse(replaceFormat) // parse template + if err != nil { + return nil, err + } + return template, nil +} + +func applyTemplate(template *template.Template, m manifest.Manifest) (path string, err error) { + buf := bytes.Buffer{} + if err := template.Execute(&buf, m); err != nil { + return "", err + } + + // Replace all os.path separators in string in order to not accidentally create subfolders + path = strings.Replace(buf.String(), string(os.PathSeparator), "-", -1) + // Replace the BEL character inserted with a path separator again in order to create a subfolder + path = strings.Replace(path, BelRune, string(os.PathSeparator), -1) + + return path, nil +} + +func StringsToRegexps(exps []string) process.Matchers { + regexs, err := process.StrExps(exps...) + if err != nil { + log.Fatalln(err) + } + return regexs +} From a963666d31794c5d76cdb156337db3a8f0c78574 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 09:06:13 +0100 Subject: [PATCH 14/40] feat(api): export multiple paths/envs --- cmd/tk/export.go | 2 +- cmd/tk/workflow.go | 25 +++++++-- pkg/tanka/environments.go | 3 +- pkg/tanka/export.go | 106 +++++++++++++++++++++----------------- 4 files changed, 81 insertions(+), 55 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index f6ae6bd47..d2cb51796 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -44,7 +44,7 @@ func exportCmd() *cli.Command { Merge: *merge, JsonnetOpts: getJsonnetOpts(), } - return tanka.ExportEnvironment(args[0], args[1], &opts) + return tanka.ExportEnvironments([]string{args[0]}, args[1], &opts) } return cmd } diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index 482c5bcdd..3863b6c76 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -48,7 +48,11 @@ func applyCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = tanka.StringsToRegexps(vars.targets) + filters, err := tanka.StringsToRegexps(vars.targets) + if err != nil { + return err + } + opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() return tanka.Apply(args[0], opts) @@ -93,7 +97,11 @@ func deleteCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = tanka.StringsToRegexps(vars.targets) + filters, err := tanka.StringsToRegexps(vars.targets) + if err != nil { + return err + } + opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() return tanka.Delete(args[0], opts) @@ -119,7 +127,11 @@ func diffCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Filters = tanka.StringsToRegexps(vars.targets) + filters, err := tanka.StringsToRegexps(vars.targets) + if err != nil { + return err + } + opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() changes, err := tanka.Diff(args[0], opts) @@ -164,9 +176,14 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) return nil } + filters, err := tanka.StringsToRegexps(vars.targets) + if err != nil { + return err + } + pretty, err := tanka.Show(args[0], tanka.Opts{ JsonnetOpts: getJsonnetOpts(), - Filters: tanka.StringsToRegexps(vars.targets), + Filters: filters, }) if err != nil { diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index b5018c291..5ab6c890b 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -54,7 +54,6 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha case ErrNoEnv: continue default: - fmt.Print(err) returnErrs = fmt.Sprintf("%s\n%s", returnErrs, err) } } @@ -127,7 +126,7 @@ func parseEnvsRoutine(envsChan <-chan parseEnvsRoutineOpts) error { for req := range envsChan { _, env, err := ParseEnv(req.path, req.opts) if err != nil { - return err + return fmt.Errorf("%w: %s", err, req.path) } *req.env = *env } diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 3b250b82c..412a9d95e 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -6,14 +6,13 @@ import ( "fmt" "io" "io/ioutil" - "log" "os" "path/filepath" "strings" - "text/template" "github.com/Masterminds/sprig/v3" + "github.com/pkg/errors" "github.com/grafana/tanka/pkg/kubernetes/manifest" "github.com/grafana/tanka/pkg/process" @@ -42,7 +41,7 @@ func DefaultExportEnvOpts() ExportEnvOpts { } } -func ExportEnvironment(path, to string, opts *ExportEnvOpts) error { +func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { // dir must be empty empty, err := dirEmpty(to) if err != nil { @@ -64,59 +63,70 @@ func ExportEnvironment(path, to string, opts *ExportEnvOpts) error { return fmt.Errorf("Parsing directory format: %s", err) } - _, env, err := ParseEnv(path, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) - if err != nil { - return err - } - - // get the manifests - res, err := LoadManifests(env, StringsToRegexps(opts.Targets)) - if err != nil { - return err - } - - raw, err := json.Marshal(env) - if err != nil { - return err - } - - var m manifest.Manifest - if err := json.Unmarshal(raw, &m); err != nil { - return err - } - - dir, err := applyTemplate(directoryTemplate, m) - if err != nil { - log.Fatalln("executing directory template:", err) + envs, errs := ParseEnvs(paths, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) + if len(errs) != 0 { + returnErr := errors.New("Unable to parse selected Environments") + for _, err := range errs { + returnErr = errors.Wrap(returnErr, err.Error()) + } + return returnErr } - // Create all subfolders in path - to = filepath.Join(to, dir) + for _, env := range envs { + filter, err := StringsToRegexps(opts.Targets) + if err != nil { + return err + } - // write each to a file - for _, m := range res { - name, err := applyTemplate(manifestTemplate, m) + // get the manifests + res, err := LoadManifests(env, filter) if err != nil { - log.Fatalln("executing name template:", err) + return err } - // Create all subfolders in path - path := filepath.Join(to, name+"."+opts.Extension) + raw, err := json.Marshal(env) + if err != nil { + return err + } - // Abort if already exists - if exists, err := fileExists(path); err != nil { + var m manifest.Manifest + if err := json.Unmarshal(raw, &m); err != nil { return err - } else if exists { - return fmt.Errorf("File '%s' already exists. Aborting", path) } - // Write file - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) + dir, err := applyTemplate(directoryTemplate, m) + if err != nil { + return fmt.Errorf("executing directory template: %w", err) } - data := m.String() - if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { - return fmt.Errorf("writing manifest: %s", err) + + // Create all subfolders in path + to = filepath.Join(to, dir) + + // write each to a file + for _, m := range res { + name, err := applyTemplate(manifestTemplate, m) + if err != nil { + return fmt.Errorf("executing name template: %w", err) + } + + // Create all subfolders in path + path := filepath.Join(to, name+"."+opts.Extension) + + // Abort if already exists + if exists, err := fileExists(path); err != nil { + return err + } else if exists { + return fmt.Errorf("File '%s' already exists. Aborting", path) + } + + // Write file + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) + } + data := m.String() + if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { + return fmt.Errorf("writing manifest: %s", err) + } } } @@ -177,10 +187,10 @@ func applyTemplate(template *template.Template, m manifest.Manifest) (path strin return path, nil } -func StringsToRegexps(exps []string) process.Matchers { +func StringsToRegexps(exps []string) (process.Matchers, error) { regexs, err := process.StrExps(exps...) if err != nil { - log.Fatalln(err) + return nil, err } - return regexs + return regexs, nil } From a033202364d829e63471dcf853cec9aa4162d8f5 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 09:33:11 +0100 Subject: [PATCH 15/40] fix(api): cleaner ParseEnvs error handling --- pkg/tanka/environments.go | 35 +++++++++++++++++++++-------------- pkg/tanka/errors.go | 15 +++++++++++++++ pkg/tanka/export.go | 11 +++-------- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index 5ab6c890b..2ee04ff30 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -46,26 +46,33 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha Evaluator: EnvsOnlyEvaluator, Selector: selector, } - envs, errs := ParseEnvs(dirs, opts) - - var returnErrs string - for _, err := range errs { - switch err.(type) { - case ErrNoEnv: - continue - default: - returnErrs = fmt.Sprintf("%s\n%s", returnErrs, err) + envs, err = ParseEnvs(dirs, opts) + + switch err.(type) { + case ErrParseEnvs: + // ignore ErrNoEnv errors + e := err.(ErrParseEnvs) + var errors []error + for _, err := range e.errors { + switch err.(type) { + case ErrNoEnv: + continue + default: + errors = append(errors, err) + } } - } - if len(returnErrs) != 0 { - return nil, fmt.Errorf("Unable to parse selected Environments: \n%s", returnErrs) + if len(errors) != 0 { + return nil, ErrParseEnvs{errors: errors} + } + default: + return nil, err } return envs, nil } // ParseEnvs evaluates multiple environments in parallel -func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, errs []error) { +func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, err error) { wg := sync.WaitGroup{} envsChan := make(chan parseEnvsRoutineOpts) var allErrors []error @@ -110,7 +117,7 @@ func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, er } if len(allErrors) != 0 { - return envs, allErrors + return envs, ErrParseEnvs{errors: allErrors} } return envs, nil diff --git a/pkg/tanka/errors.go b/pkg/tanka/errors.go index 546285b09..7cca9a0ed 100644 --- a/pkg/tanka/errors.go +++ b/pkg/tanka/errors.go @@ -3,6 +3,8 @@ package tanka import ( "fmt" "strings" + + "github.com/pkg/errors" ) // ErrNoEnv means that the given jsonnet has no Environment object @@ -24,3 +26,16 @@ type ErrMultipleEnvs struct { func (e ErrMultipleEnvs) Error() string { return fmt.Sprintf("found multiple Environments (%s) in '%s'", strings.Join(e.names, ", "), e.path) } + +// ErrParseEnvs is an array of errors collected while parsing environments in parallel +type ErrParseEnvs struct { + errors []error +} + +func (e ErrParseEnvs) Error() string { + returnErr := errors.New("Unable to parse selected Environments") + for _, err := range e.errors { + returnErr = errors.Wrap(returnErr, err.Error()) + } + return returnErr.Error() +} diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 412a9d95e..fbee4a9b6 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -12,7 +12,6 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/pkg/errors" "github.com/grafana/tanka/pkg/kubernetes/manifest" "github.com/grafana/tanka/pkg/process" @@ -63,13 +62,9 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { return fmt.Errorf("Parsing directory format: %s", err) } - envs, errs := ParseEnvs(paths, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) - if len(errs) != 0 { - returnErr := errors.New("Unable to parse selected Environments") - for _, err := range errs { - returnErr = errors.Wrap(returnErr, err.Error()) - } - return returnErr + envs, err := ParseEnvs(paths, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) + if err != nil { + return err } for _, env := range envs { From 57937743a98f6abb085d18ce0067372e52749df8 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 13:13:54 +0100 Subject: [PATCH 16/40] chore(cli): remove unused code --- cmd/tk/export.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index d2cb51796..7f6e450a5 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -1,19 +1,10 @@ package main import ( - "io" - "os" - "github.com/go-clix/cli" - "github.com/grafana/tanka/pkg/tanka" ) -// BelRune is a string of the Ascii character BEL which made computers ring in ancient times -// We use it as "magic" char for the subfolder creation as it is a non printable character and thereby will never be -// in a valid filepath by accident. Only when we include it. -const BelRune = string(rune(7)) - func exportCmd() *cli.Command { args := workflowArgs args.Validator = cli.ValidateExact(2) @@ -48,30 +39,3 @@ func exportCmd() *cli.Command { } return cmd } - -func fileExists(name string) (bool, error) { - _, err := os.Stat(name) - if os.IsNotExist(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil -} - -func dirEmpty(dir string) (bool, error) { - f, err := os.Open(dir) - if os.IsNotExist(err) { - return true, os.MkdirAll(dir, os.ModePerm) - } else if err != nil { - return false, err - } - defer f.Close() - - _, err = f.Readdirnames(1) - if err == io.EOF { - return true, nil - } - return false, err -} From 5aeeb7590e41199626616619cba53d22c04bdaa2 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 14:59:13 +0100 Subject: [PATCH 17/40] feat(cli): export multiple environments --- cmd/tk/export.go | 50 ++++++++++++++++++++++++++++++++++++--------- pkg/tanka/export.go | 18 ++++++++-------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 7f6e450a5..326f90964 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -1,17 +1,26 @@ package main import ( + "fmt" + "github.com/go-clix/cli" + "k8s.io/apimachinery/pkg/labels" + "github.com/grafana/tanka/pkg/tanka" ) func exportCmd() *cli.Command { args := workflowArgs - args.Validator = cli.ValidateExact(2) + args.Validator = cli.ValidateFunc(func(args []string) error { + if len(args) < 2 { + return fmt.Errorf("expects at least 2 args, received %v", len(args)) + } + return nil + }) cmd := &cli.Command{ - Use: "export ", - Short: "write each resources as a YAML file", + Use: "export [...]", + Short: "export environments found in path(s)", Args: args, } @@ -22,20 +31,41 @@ func exportCmd() *cli.Command { extension := cmd.Flags().String("extension", defaultOpts.Extension, "File extension") merge := cmd.Flags().Bool("merge", defaultOpts.Merge, "Allow merging with existing directory") + labelSelector := cmd.Flags().StringP("selector", "l", "", "Label selector. Uses the same syntax as kubectl does") vars := workflowFlags(cmd.Flags()) getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { + var selector labels.Selector + var err error + if *labelSelector != "" { + selector, err = labels.Parse(*labelSelector) + if err != nil { + return err + } + } + opts := tanka.ExportEnvOpts{ - Format: *format, - DirFormat: *dirFormat, - Extension: *extension, - Targets: vars.targets, - Merge: *merge, - JsonnetOpts: getJsonnetOpts(), + Format: *format, + DirFormat: *dirFormat, + Extension: *extension, + Targets: vars.targets, + Merge: *merge, + ParseOpts: tanka.ParseOpts{ + Selector: selector, + JsonnetOpts: getJsonnetOpts(), + }, + } + var paths []string + for _, path := range args[1:] { + dirs, err := tanka.FindBaseDirs(path) + if err != nil { + return err + } + paths = append(paths, dirs...) } - return tanka.ExportEnvironments([]string{args[0]}, args[1], &opts) + return tanka.ExportEnvironments(paths, args[0], &opts) } return cmd } diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index fbee4a9b6..b71237e84 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -23,12 +23,12 @@ import ( const BelRune = string(rune(7)) type ExportEnvOpts struct { - Format string - DirFormat string - Extension string - Targets []string - Merge bool - JsonnetOpts JsonnetOpts + Format string + DirFormat string + Extension string + Targets []string + Merge bool + ParseOpts ParseOpts } func DefaultExportEnvOpts() ExportEnvOpts { @@ -62,7 +62,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { return fmt.Errorf("Parsing directory format: %s", err) } - envs, err := ParseEnvs(paths, ParseOpts{JsonnetOpts: opts.JsonnetOpts}) + envs, err := ParseEnvs(paths, opts.ParseOpts) if err != nil { return err } @@ -95,7 +95,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // Create all subfolders in path - to = filepath.Join(to, dir) + toDir := filepath.Join(to, dir) // write each to a file for _, m := range res { @@ -105,7 +105,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // Create all subfolders in path - path := filepath.Join(to, name+"."+opts.Extension) + path := filepath.Join(toDir, name+"."+opts.Extension) // Abort if already exists if exists, err := fileExists(path); err != nil { From fbb193e8908349853bba9be3f4c67b59a1de6ab6 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 15:07:20 +0100 Subject: [PATCH 18/40] feat(cli): manifest file to map exported file to environment --- pkg/tanka/export.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index b71237e84..725f22a76 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -22,6 +22,11 @@ import ( // in a valid filepath by accident. Only when we include it. const BelRune = string(rune(7)) +// When exporting manifests to files, it becomes increasingly hard to map manifests back to its environment, this file +// can be used to map the files back to their environment. This is aimed to be used by CI/CD but can also be used for +// debugging purposes. +const manifestFile = "manifest.json" + type ExportEnvOpts struct { Format string DirFormat string @@ -41,6 +46,9 @@ func DefaultExportEnvOpts() ExportEnvOpts { } func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { + // Keep track of which file maps to which environment + fileToEnv := map[string]string{} + // dir must be empty empty, err := dirEmpty(to) if err != nil { @@ -107,6 +115,8 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { // Create all subfolders in path path := filepath.Join(toDir, name+"."+opts.Extension) + fileToEnv[path] = env.Metadata.Namespace + // Abort if already exists if exists, err := fileExists(path); err != nil { return err @@ -125,6 +135,20 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } } + if len(fileToEnv) != 0 { + data, err := json.MarshalIndent(fileToEnv, "", " ") + if err != nil { + return err + } + path := filepath.Join(to, manifestFile) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) + } + if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { + return err + } + } + return nil } From d9ae6377b64bd67061debef06d4e006cd3671d2b Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 21 Dec 2020 15:39:45 +0100 Subject: [PATCH 19/40] fix(api): return envs if err nil in FindEnvironments --- pkg/tanka/environments.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index 2ee04ff30..efd01219f 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -48,24 +48,26 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha } envs, err = ParseEnvs(dirs, opts) - switch err.(type) { - case ErrParseEnvs: - // ignore ErrNoEnv errors - e := err.(ErrParseEnvs) - var errors []error - for _, err := range e.errors { - switch err.(type) { - case ErrNoEnv: - continue - default: - errors = append(errors, err) + if err != nil { + switch err.(type) { + case ErrParseEnvs: + // ignore ErrNoEnv errors + e := err.(ErrParseEnvs) + var errors []error + for _, err := range e.errors { + switch err.(type) { + case ErrNoEnv: + continue + default: + errors = append(errors, err) + } } + if len(errors) != 0 { + return nil, ErrParseEnvs{errors: errors} + } + default: + return nil, err } - if len(errors) != 0 { - return nil, ErrParseEnvs{errors: errors} - } - default: - return nil, err } return envs, nil From 638c2535c6ace4a4111ed0b8e1b2f06252c9add0 Mon Sep 17 00:00:00 2001 From: Duologic Date: Tue, 22 Dec 2020 16:23:20 +0100 Subject: [PATCH 20/40] fix(cli): make recursive export optional --- cmd/tk/export.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 326f90964..91693be69 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -31,6 +31,7 @@ func exportCmd() *cli.Command { extension := cmd.Flags().String("extension", defaultOpts.Extension, "File extension") merge := cmd.Flags().Bool("merge", defaultOpts.Merge, "Allow merging with existing directory") + recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") labelSelector := cmd.Flags().StringP("selector", "l", "", "Label selector. Uses the same syntax as kubectl does") vars := workflowFlags(cmd.Flags()) @@ -46,6 +47,29 @@ func exportCmd() *cli.Command { } } + var paths []string + for _, path := range args[1:] { + if *recursive { + envs, err := tanka.FindEnvironments(path, selector) + if err != nil { + return err + } + for _, env := range envs { + paths = append(paths, env.Metadata.Namespace) + } + } else { + opts := tanka.ParseOpts{ + Evaluator: tanka.EnvsOnlyEvaluator, + Selector: selector, + } + _, _, err := tanka.ParseEnv(path, opts) + if err != nil { + return err + } + paths = append(paths, path) + } + } + opts := tanka.ExportEnvOpts{ Format: *format, DirFormat: *dirFormat, @@ -57,14 +81,6 @@ func exportCmd() *cli.Command { JsonnetOpts: getJsonnetOpts(), }, } - var paths []string - for _, path := range args[1:] { - dirs, err := tanka.FindBaseDirs(path) - if err != nil { - return err - } - paths = append(paths, dirs...) - } return tanka.ExportEnvironments(paths, args[0], &opts) } return cmd From 26de70f5b426d2b0930d93444e00c01a303c3562 Mon Sep 17 00:00:00 2001 From: Duologic Date: Tue, 22 Dec 2020 21:39:49 +0100 Subject: [PATCH 21/40] refactor: group flags in flags.go --- cmd/tk/env.go | 8 ----- cmd/tk/flags.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/tk/jsonnet.go | 50 ------------------------------- cmd/tk/workflow.go | 14 +-------- 4 files changed, 76 insertions(+), 71 deletions(-) create mode 100644 cmd/tk/flags.go diff --git a/cmd/tk/env.go b/cmd/tk/env.go index abbcb0a2c..61a0748bc 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -11,7 +11,6 @@ import ( "github.com/go-clix/cli" "github.com/pkg/errors" "github.com/posener/complete" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/jsonnet/jpath" @@ -38,13 +37,6 @@ func envCmd() *cli.Command { return cmd } -func envSettingsFlags(env *v1alpha1.Environment, fs *pflag.FlagSet) { - fs.StringVar(&env.Spec.APIServer, "server", env.Spec.APIServer, "endpoint of the Kubernetes API") - fs.StringVar(&env.Spec.APIServer, "server-from-context", env.Spec.APIServer, "set the server to a known one from $KUBECONFIG") - fs.StringVar(&env.Spec.Namespace, "namespace", env.Spec.Namespace, "namespace to create objects in") - fs.StringVar(&env.Spec.DiffStrategy, "diff-strategy", env.Spec.DiffStrategy, "specify diff-strategy. Automatically detected otherwise.") -} - var kubectlContexts = cli.PredictFunc( func(complete.Args) []string { c, _ := client.Contexts() diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go new file mode 100644 index 000000000..e9e6f43af --- /dev/null +++ b/cmd/tk/flags.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "log" + "strings" + + "github.com/spf13/pflag" + + "github.com/grafana/tanka/pkg/spec/v1alpha1" + "github.com/grafana/tanka/pkg/tanka" +) + +type workflowFlagVars struct { + targets []string +} + +func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars { + v := workflowFlagVars{} + fs.StringSliceVarP(&v.targets, "target", "t", nil, "only use the specified objects (Format: /)") + return &v +} + +func jsonnetFlags(fs *pflag.FlagSet) func() tanka.JsonnetOpts { + getExtCode, getTLACode := cliCodeParser(fs) + + return func() tanka.JsonnetOpts { + return tanka.JsonnetOpts{ + ExtCode: getExtCode(), + TLACode: getTLACode(), + } + } +} + +func cliCodeParser(fs *pflag.FlagSet) (func() map[string]string, func() map[string]string) { + // need to use StringArray instead of StringSlice, because pflag attempts to + // parse StringSlice using the csv parser, which breaks when passing objects + extCode := fs.StringArray("ext-code", nil, "Set code value of extVar (Format: key=)") + extStr := fs.StringArrayP("ext-str", "V", nil, "Set string value of extVar (Format: key=value)") + + tlaCode := fs.StringArray("tla-code", nil, "Set code value of top level function (Format: key=)") + tlaStr := fs.StringArrayP("tla-str", "A", nil, "Set string value of top level function (Format: key=value)") + + newParser := func(kind string, code, str *[]string) func() map[string]string { + return func() map[string]string { + m := make(map[string]string) + for _, s := range *code { + split := strings.SplitN(s, "=", 2) + if len(split) != 2 { + log.Fatalf(kind+"-code argument has wrong format: `%s`. Expected `key=`", s) + } + m[split[0]] = split[1] + } + + for _, s := range *str { + split := strings.SplitN(s, "=", 2) + if len(split) != 2 { + log.Fatalf(kind+"-str argument has wrong format: `%s`. Expected `key=`", s) + } + m[split[0]] = fmt.Sprintf(`"%s"`, split[1]) + } + return m + } + } + + return newParser("ext", extCode, extStr), + newParser("tla", tlaCode, tlaStr) +} + +func envSettingsFlags(env *v1alpha1.Environment, fs *pflag.FlagSet) { + fs.StringVar(&env.Spec.APIServer, "server", env.Spec.APIServer, "endpoint of the Kubernetes API") + fs.StringVar(&env.Spec.APIServer, "server-from-context", env.Spec.APIServer, "set the server to a known one from $KUBECONFIG") + fs.StringVar(&env.Spec.Namespace, "namespace", env.Spec.Namespace, "namespace to create objects in") + fs.StringVar(&env.Spec.DiffStrategy, "diff-strategy", env.Spec.DiffStrategy, "specify diff-strategy. Automatically detected otherwise.") +} diff --git a/cmd/tk/jsonnet.go b/cmd/tk/jsonnet.go index 16fb5f085..8e9f91d45 100644 --- a/cmd/tk/jsonnet.go +++ b/cmd/tk/jsonnet.go @@ -2,12 +2,8 @@ package main import ( "encoding/json" - "fmt" - "log" - "strings" "github.com/go-clix/cli" - "github.com/spf13/pflag" "github.com/grafana/tanka/pkg/tanka" ) @@ -48,49 +44,3 @@ func evalCmd() *cli.Command { return cmd } - -func jsonnetFlags(fs *pflag.FlagSet) func() tanka.JsonnetOpts { - getExtCode, getTLACode := cliCodeParser(fs) - - return func() tanka.JsonnetOpts { - return tanka.JsonnetOpts{ - ExtCode: getExtCode(), - TLACode: getTLACode(), - } - } -} - -func cliCodeParser(fs *pflag.FlagSet) (func() map[string]string, func() map[string]string) { - // need to use StringArray instead of StringSlice, because pflag attempts to - // parse StringSlice using the csv parser, which breaks when passing objects - extCode := fs.StringArray("ext-code", nil, "Set code value of extVar (Format: key=)") - extStr := fs.StringArrayP("ext-str", "V", nil, "Set string value of extVar (Format: key=value)") - - tlaCode := fs.StringArray("tla-code", nil, "Set code value of top level function (Format: key=)") - tlaStr := fs.StringArrayP("tla-str", "A", nil, "Set string value of top level function (Format: key=value)") - - newParser := func(kind string, code, str *[]string) func() map[string]string { - return func() map[string]string { - m := make(map[string]string) - for _, s := range *code { - split := strings.SplitN(s, "=", 2) - if len(split) != 2 { - log.Fatalf(kind+"-code argument has wrong format: `%s`. Expected `key=`", s) - } - m[split[0]] = split[1] - } - - for _, s := range *str { - split := strings.SplitN(s, "=", 2) - if len(split) != 2 { - log.Fatalf(kind+"-str argument has wrong format: `%s`. Expected `key=`", s) - } - m[split[0]] = fmt.Sprintf(`"%s"`, split[1]) - } - return m - } - } - - return newParser("ext", extCode, extStr), - newParser("tla", tlaCode, tlaStr) -} diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index 3863b6c76..b71ac2d8d 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -5,10 +5,8 @@ import ( "log" "os" - "github.com/posener/complete" - "github.com/spf13/pflag" - "github.com/go-clix/cli" + "github.com/posener/complete" "github.com/grafana/tanka/pkg/tanka" "github.com/grafana/tanka/pkg/term" @@ -22,16 +20,6 @@ const ( ExitStatusDiff = 16 ) -type workflowFlagVars struct { - targets []string -} - -func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars { - v := workflowFlagVars{} - fs.StringSliceVarP(&v.targets, "target", "t", nil, "only use the specified objects (Format: /)") - return &v -} - func applyCmd() *cli.Command { cmd := &cli.Command{ Use: "apply ", From d007048b53143ae94ff766cb0d428bb4396c98fc Mon Sep 17 00:00:00 2001 From: Duologic Date: Tue, 22 Dec 2020 21:53:18 +0100 Subject: [PATCH 22/40] refactor: directly assign flags to opts --- cmd/tk/env.go | 13 ++----------- cmd/tk/export.go | 46 +++++++++++++++------------------------------- cmd/tk/flags.go | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 42 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index 61a0748bc..01642434d 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -11,7 +11,6 @@ import ( "github.com/go-clix/cli" "github.com/pkg/errors" "github.com/posener/complete" - "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes/client" @@ -241,7 +240,7 @@ func envListCmd() *cli.Command { } useJSON := cmd.Flags().Bool("json", false, "json output") - labelSelector := cmd.Flags().StringP("selector", "l", "", "Label selector. Uses the same syntax as kubectl does") + getLabelSelector := labelSelectorFlag(cmd.Flags()) useNames := cmd.Flags().Bool("names", false, "plain names output") @@ -255,15 +254,7 @@ func envListCmd() *cli.Command { return fmt.Errorf("Not a directory: %s", dir) } - var selector labels.Selector - if *labelSelector != "" { - selector, err = labels.Parse(*labelSelector) - if err != nil { - return err - } - } - - envs, err := tanka.FindEnvironments(dir, selector) + envs, err := tanka.FindEnvironments(dir, getLabelSelector()) if err != nil { return err } diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 91693be69..264134008 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/go-clix/cli" - "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/tanka" ) @@ -24,33 +23,29 @@ func exportCmd() *cli.Command { Args: args, } - defaultOpts := tanka.DefaultExportEnvOpts() + opts := tanka.DefaultExportEnvOpts() - format := cmd.Flags().String("format", defaultOpts.Format, "https://tanka.dev/exporting#filenames") - dirFormat := cmd.Flags().String("dirformat", defaultOpts.DirFormat, "based on tanka.dev/Environment object") + opts.Format = *cmd.Flags().String("format", opts.Format, "https://tanka.dev/exporting#filenames") + opts.DirFormat = *cmd.Flags().String("dirformat", opts.DirFormat, "based on tanka.dev/Environment object") - extension := cmd.Flags().String("extension", defaultOpts.Extension, "File extension") - merge := cmd.Flags().Bool("merge", defaultOpts.Merge, "Allow merging with existing directory") - recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") - labelSelector := cmd.Flags().StringP("selector", "l", "", "Label selector. Uses the same syntax as kubectl does") + opts.Extension = *cmd.Flags().String("extension", opts.Extension, "File extension") + opts.Merge = *cmd.Flags().Bool("merge", opts.Merge, "Allow merging with existing directory") vars := workflowFlags(cmd.Flags()) getJsonnetOpts := jsonnetFlags(cmd.Flags()) + getLabelSelector := labelSelectorFlag(cmd.Flags()) + + recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") cmd.Run = func(cmd *cli.Command, args []string) error { - var selector labels.Selector - var err error - if *labelSelector != "" { - selector, err = labels.Parse(*labelSelector) - if err != nil { - return err - } - } + opts.Targets = vars.targets + opts.ParseOpts.JsonnetOpts = getJsonnetOpts() + opts.ParseOpts.Selector = getLabelSelector() var paths []string for _, path := range args[1:] { if *recursive { - envs, err := tanka.FindEnvironments(path, selector) + envs, err := tanka.FindEnvironments(path, opts.ParseOpts.Selector) if err != nil { return err } @@ -58,11 +53,11 @@ func exportCmd() *cli.Command { paths = append(paths, env.Metadata.Namespace) } } else { - opts := tanka.ParseOpts{ + parseOpts := tanka.ParseOpts{ Evaluator: tanka.EnvsOnlyEvaluator, - Selector: selector, + Selector: opts.ParseOpts.Selector, } - _, _, err := tanka.ParseEnv(path, opts) + _, _, err := tanka.ParseEnv(path, parseOpts) if err != nil { return err } @@ -70,17 +65,6 @@ func exportCmd() *cli.Command { } } - opts := tanka.ExportEnvOpts{ - Format: *format, - DirFormat: *dirFormat, - Extension: *extension, - Targets: vars.targets, - Merge: *merge, - ParseOpts: tanka.ParseOpts{ - Selector: selector, - JsonnetOpts: getJsonnetOpts(), - }, - } return tanka.ExportEnvironments(paths, args[0], &opts) } return cmd diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go index e9e6f43af..aab5af384 100644 --- a/cmd/tk/flags.go +++ b/cmd/tk/flags.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/grafana/tanka/pkg/tanka" @@ -21,6 +22,21 @@ func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars { return &v } +func labelSelectorFlag(fs *pflag.FlagSet) func() labels.Selector { + labelSelector := fs.StringP("selector", "l", "", "Label selector. Uses the same syntax as kubectl does") + + return func() labels.Selector { + if *labelSelector != "" { + selector, err := labels.Parse(*labelSelector) + if err != nil { + log.Fatalf("Could not parse selector (-l) %s", *labelSelector) + } + return selector + } + return nil + } +} + func jsonnetFlags(fs *pflag.FlagSet) func() tanka.JsonnetOpts { getExtCode, getTLACode := cliCodeParser(fs) From f72c57149e519b5c81f2aa45b2dc4e620111f3dd Mon Sep 17 00:00:00 2001 From: Duologic Date: Tue, 22 Dec 2020 22:12:05 +0100 Subject: [PATCH 23/40] doc(api): add docstrings on ExportEnvOpts --- pkg/tanka/export.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 725f22a76..538f7df9f 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -28,12 +28,12 @@ const BelRune = string(rune(7)) const manifestFile = "manifest.json" type ExportEnvOpts struct { - Format string - DirFormat string - Extension string - Targets []string - Merge bool - ParseOpts ParseOpts + Format string // formatting the filename based on the exported Kubernetes manifest + DirFormat string // formatting the directory based on the exported Environment + Extension string // extension of the filename + Merge bool // merge export with existing directory + Targets []string // optional: only export specified Kubernetes manifests + ParseOpts ParseOpts // optional: options for parsing Environments } func DefaultExportEnvOpts() ExportEnvOpts { From f72a09ae2944c274ec48e3d9ebb1ba59998799e1 Mon Sep 17 00:00:00 2001 From: Duologic Date: Wed, 23 Dec 2020 11:07:32 +0100 Subject: [PATCH 24/40] fix(cli): single --format option --- cmd/tk/export.go | 7 +++--- pkg/tanka/export.go | 56 ++++++++++++++++++++------------------------- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 264134008..fce410590 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -25,11 +25,10 @@ func exportCmd() *cli.Command { opts := tanka.DefaultExportEnvOpts() - opts.Format = *cmd.Flags().String("format", opts.Format, "https://tanka.dev/exporting#filenames") - opts.DirFormat = *cmd.Flags().String("dirformat", opts.DirFormat, "based on tanka.dev/Environment object") + opts.Format = cmd.Flags().String("format", *opts.Format, "https://tanka.dev/exporting#filenames") - opts.Extension = *cmd.Flags().String("extension", opts.Extension, "File extension") - opts.Merge = *cmd.Flags().Bool("merge", opts.Merge, "Allow merging with existing directory") + opts.Extension = cmd.Flags().String("extension", *opts.Extension, "File extension") + opts.Merge = cmd.Flags().Bool("merge", *opts.Merge, "Allow merging with existing directory") vars := workflowFlags(cmd.Flags()) getJsonnetOpts := jsonnetFlags(cmd.Flags()) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 538f7df9f..9c0892ab8 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -28,20 +28,21 @@ const BelRune = string(rune(7)) const manifestFile = "manifest.json" type ExportEnvOpts struct { - Format string // formatting the filename based on the exported Kubernetes manifest - DirFormat string // formatting the directory based on the exported Environment - Extension string // extension of the filename - Merge bool // merge export with existing directory + Format *string // formatting the filename based on the exported Kubernetes manifest + Extension *string // extension of the filename + Merge *bool // merge export with existing directory Targets []string // optional: only export specified Kubernetes manifests ParseOpts ParseOpts // optional: options for parsing Environments } func DefaultExportEnvOpts() ExportEnvOpts { + format := "{{env.spec.namespace}}/{{env.metadata.name}}/{{.apiVersion}}.{{.kind}}-{{.metadata.name}}" + extension := "yaml" + merge := false return ExportEnvOpts{ - Format: "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", - DirFormat: "{{.spec.namespace}}/{{.metadata.name}}", - Extension: "yaml", - Merge: false, + Format: &format, + Extension: &extension, + Merge: &merge, } } @@ -54,28 +55,18 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { if err != nil { return fmt.Errorf("Checking target dir: %s", err) } - if !empty && !opts.Merge { + if !empty && !*opts.Merge { return fmt.Errorf("Output dir `%s` not empty. Pass --merge to ignore this", to) } - // exit early if the template is bad - - manifestTemplate, err := createTemplate(opts.Format) - if err != nil { - return fmt.Errorf("Parsing filename format: %s", err) - } - - directoryTemplate, err := createTemplate(opts.DirFormat) - if err != nil { - return fmt.Errorf("Parsing directory format: %s", err) - } - + // get all environments for paths envs, err := ParseEnvs(paths, opts.ParseOpts) if err != nil { return err } for _, env := range envs { + // select targets to export filter, err := StringsToRegexps(opts.Targets) if err != nil { return err @@ -87,33 +78,32 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { return err } + // create raw manifest version of env for templating raw, err := json.Marshal(env) if err != nil { return err } - - var m manifest.Manifest - if err := json.Unmarshal(raw, &m); err != nil { + var menv manifest.Manifest + if err := json.Unmarshal(raw, &menv); err != nil { return err } - dir, err := applyTemplate(directoryTemplate, m) + // create template + manifestTemplate, err := createTemplate(*opts.Format, menv) if err != nil { - return fmt.Errorf("executing directory template: %w", err) + return fmt.Errorf("Parsing format: %s", err) } - // Create all subfolders in path - toDir := filepath.Join(to, dir) - // write each to a file for _, m := range res { + // apply template name, err := applyTemplate(manifestTemplate, m) if err != nil { return fmt.Errorf("executing name template: %w", err) } // Create all subfolders in path - path := filepath.Join(toDir, name+"."+opts.Extension) + path := filepath.Join(to, name+"."+*opts.Extension) fileToEnv[path] = env.Metadata.Namespace @@ -135,6 +125,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } } + // create manifest file if len(fileToEnv) != 0 { data, err := json.MarshalIndent(fileToEnv, "", " ") if err != nil { @@ -179,12 +170,15 @@ func dirEmpty(dir string) (bool, error) { return false, err } -func createTemplate(format string) (*template.Template, error) { +func createTemplate(format string, env manifest.Manifest) (*template.Template, error) { // Replace all os.path separators in string with BelRune for creating subfolders replaceFormat := strings.Replace(format, string(os.PathSeparator), BelRune, -1) + envMap := template.FuncMap{"env": func() manifest.Manifest { return env }} + template, err := template.New(""). Funcs(sprig.TxtFuncMap()). // register Masterminds/sprig + Funcs(envMap). // register environment mapping Parse(replaceFormat) // parse template if err != nil { return nil, err From 269e6e34650d664dcc31207fd1cb1ac9b4718207 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 14:29:19 +0100 Subject: [PATCH 25/40] fix: prune in EnvsOnlyEvaluator --- pkg/tanka/parse.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/tanka/parse.go b/pkg/tanka/parse.go index 8ce029445..9955f36d5 100644 --- a/pkg/tanka/parse.go +++ b/pkg/tanka/parse.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "path/filepath" - "reflect" "github.com/Masterminds/semver" "github.com/pkg/errors" @@ -199,11 +198,9 @@ func ParseEnv(path string, opts JsonnetOpts) (interface{}, *v1alpha1.Environment return nil, nil, err } return data, env, nil - } else if specEnv != nil { // if no environments found, fallback to original behavior - // EnvsOnlyEvaluator explicitly removes Data, we should not add it - if reflect.ValueOf(opts.Evaluator).Pointer() != reflect.ValueOf(EnvsOnlyEvaluator).Pointer() { - specEnv.Data = data - } + } else if specEnv != nil { + // if no environments found, fallback to original behavior + specEnv.Data = data return data, specEnv, nil } // if no environments or spec found, behave as jsonnet interpreter From 8d61a2c5fb3e711fb6d19ffe3748291b0fcceb6f Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 14:55:20 +0100 Subject: [PATCH 26/40] fix: naming is hard --- cmd/tk/export.go | 7 +++---- pkg/tanka/environments.go | 42 ++++++++++++++++++++------------------- pkg/tanka/errors.go | 6 +++--- pkg/tanka/export.go | 17 ++++++++++------ pkg/tanka/tanka.go | 8 ++++++++ 5 files changed, 47 insertions(+), 33 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index fce410590..ce95b5a9d 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -38,13 +38,13 @@ func exportCmd() *cli.Command { cmd.Run = func(cmd *cli.Command, args []string) error { opts.Targets = vars.targets - opts.ParseOpts.JsonnetOpts = getJsonnetOpts() - opts.ParseOpts.Selector = getLabelSelector() + opts.ParseParallelOpts.ParseOpts.JsonnetOpts = getJsonnetOpts() + opts.ParseParallelOpts.Selector = getLabelSelector() var paths []string for _, path := range args[1:] { if *recursive { - envs, err := tanka.FindEnvironments(path, opts.ParseOpts.Selector) + envs, err := tanka.FindEnvironments(path, opts.ParseParallelOpts.Selector) if err != nil { return err } @@ -54,7 +54,6 @@ func exportCmd() *cli.Command { } else { parseOpts := tanka.ParseOpts{ Evaluator: tanka.EnvsOnlyEvaluator, - Selector: opts.ParseOpts.Selector, } _, _, err := tanka.ParseEnv(path, parseOpts) if err != nil { diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index efd01219f..ddc1400f4 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -12,8 +12,8 @@ import ( "github.com/grafana/tanka/pkg/spec/v1alpha1" ) -const BASEDIR_INDICATOR = "main.jsonnet" -const PARALLEL = 8 +const baseDirIndicator = "main.jsonnet" +const parallel = 8 // FindBaseDirs searches for possible environments func FindBaseDirs(workdir string) (dirs []string, err error) { @@ -23,7 +23,7 @@ func FindBaseDirs(workdir string) (dirs []string, err error) { } if err := filepath.Walk(workdir, func(path string, info os.FileInfo, err error) error { - if _, err := os.Stat(filepath.Join(path, BASEDIR_INDICATOR)); err != nil { + if _, err := os.Stat(filepath.Join(path, baseDirIndicator)); err != nil { // missing file, not a valid environment directory return nil } @@ -42,17 +42,19 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha if err != nil { return nil, err } - opts := ParseOpts{ - Evaluator: EnvsOnlyEvaluator, - Selector: selector, + opts := ParseParallelOpts{ + ParseOpts: ParseOpts{ + Evaluator: EnvsOnlyEvaluator, + }, + Selector: selector, } - envs, err = ParseEnvs(dirs, opts) + envs, err = ParseParallel(dirs, opts) if err != nil { switch err.(type) { - case ErrParseEnvs: + case ErrParseParallel: // ignore ErrNoEnv errors - e := err.(ErrParseEnvs) + e := err.(ErrParseParallel) var errors []error for _, err := range e.errors { switch err.(type) { @@ -63,7 +65,7 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha } } if len(errors) != 0 { - return nil, ErrParseEnvs{errors: errors} + return nil, ErrParseParallel{errors: errors} } default: return nil, err @@ -73,20 +75,20 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha return envs, nil } -// ParseEnvs evaluates multiple environments in parallel -func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, err error) { +// ParseParallel evaluates multiple environments in parallel +func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Environment, err error) { wg := sync.WaitGroup{} - envsChan := make(chan parseEnvsRoutineOpts) + envsChan := make(chan parseJob) var allErrors []error - numParallel := PARALLEL + numParallel := parallel if opts.Parallel > 0 { numParallel = opts.Parallel } for i := 0; i < numParallel; i++ { wg.Add(1) go func() { - err := parseEnvsRoutine(envsChan) + err := parseWorker(envsChan) if err != nil { allErrors = append(allErrors, err) } @@ -101,9 +103,9 @@ func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, er env := &v1alpha1.Environment{} results[currentIndex] = env currentIndex++ - envsChan <- parseEnvsRoutineOpts{ + envsChan <- parseJob{ path: path, - opts: opts, + opts: opts.ParseOpts, env: env, } } @@ -119,19 +121,19 @@ func ParseEnvs(paths []string, opts ParseOpts) (envs []*v1alpha1.Environment, er } if len(allErrors) != 0 { - return envs, ErrParseEnvs{errors: allErrors} + return envs, ErrParseParallel{errors: allErrors} } return envs, nil } -type parseEnvsRoutineOpts struct { +type parseJob struct { path string opts ParseOpts env *v1alpha1.Environment } -func parseEnvsRoutine(envsChan <-chan parseEnvsRoutineOpts) error { +func parseWorker(envsChan <-chan parseJob) error { for req := range envsChan { _, env, err := ParseEnv(req.path, req.opts) if err != nil { diff --git a/pkg/tanka/errors.go b/pkg/tanka/errors.go index 7cca9a0ed..f4e9616a2 100644 --- a/pkg/tanka/errors.go +++ b/pkg/tanka/errors.go @@ -27,12 +27,12 @@ func (e ErrMultipleEnvs) Error() string { return fmt.Sprintf("found multiple Environments (%s) in '%s'", strings.Join(e.names, ", "), e.path) } -// ErrParseEnvs is an array of errors collected while parsing environments in parallel -type ErrParseEnvs struct { +// ErrParseParallel is an array of errors collected while parsing environments in parallel +type ErrParseParallel struct { errors []error } -func (e ErrParseEnvs) Error() string { +func (e ErrParseParallel) Error() string { returnErr := errors.New("Unable to parse selected Environments") for _, err := range e.errors { returnErr = errors.Wrap(returnErr, err.Error()) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 9c0892ab8..7bbaa3c68 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -28,11 +28,16 @@ const BelRune = string(rune(7)) const manifestFile = "manifest.json" type ExportEnvOpts struct { - Format *string // formatting the filename based on the exported Kubernetes manifest - Extension *string // extension of the filename - Merge *bool // merge export with existing directory - Targets []string // optional: only export specified Kubernetes manifests - ParseOpts ParseOpts // optional: options for parsing Environments + // formatting the filename based on the exported Kubernetes manifest + Format *string + // extension of the filename + Extension *string + // merge export with existing directory + Merge *bool + // optional: only export specified Kubernetes manifests + Targets []string + // optional: options for parsing Environments + ParseParallelOpts ParseParallelOpts } func DefaultExportEnvOpts() ExportEnvOpts { @@ -60,7 +65,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // get all environments for paths - envs, err := ParseEnvs(paths, opts.ParseOpts) + envs, err := ParseParallel(paths, opts.ParseParallelOpts) if err != nil { return err } diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go index 5564a5d42..71e909f21 100644 --- a/pkg/tanka/tanka.go +++ b/pkg/tanka/tanka.go @@ -5,6 +5,8 @@ package tanka import ( + "k8s.io/apimachinery/pkg/labels" + "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/process" ) @@ -18,3 +20,9 @@ type Opts struct { // Filters are used to optionally select a subset of the resources Filters process.Matchers } + +type ParseParallelOpts struct { + JsonnetOpts JsonnetOpts + Selector labels.Selector + Parallel int +} From 31bbf27235ad8a532df841d0be17c3a275160a8c Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:05:48 +0100 Subject: [PATCH 27/40] fix: remove DefaultExportEnvOpts --- cmd/tk/export.go | 27 +++++++++++++++++++-------- pkg/tanka/export.go | 24 +++++++----------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index ce95b5a9d..13856fe22 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -23,12 +23,14 @@ func exportCmd() *cli.Command { Args: args, } - opts := tanka.DefaultExportEnvOpts() + format := cmd.Flags().String( + "format", + "{{env.spec.namespace}}/{{env.metadata.name}}/{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", + "https://tanka.dev/exporting#filenames", + ) - opts.Format = cmd.Flags().String("format", *opts.Format, "https://tanka.dev/exporting#filenames") - - opts.Extension = cmd.Flags().String("extension", *opts.Extension, "File extension") - opts.Merge = cmd.Flags().Bool("merge", *opts.Merge, "Allow merging with existing directory") + extension := cmd.Flags().String("extension", "yaml", "File extension") + merge := cmd.Flags().Bool("merge", false, "Allow merging with existing directory") vars := workflowFlags(cmd.Flags()) getJsonnetOpts := jsonnetFlags(cmd.Flags()) @@ -37,9 +39,18 @@ func exportCmd() *cli.Command { recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") cmd.Run = func(cmd *cli.Command, args []string) error { - opts.Targets = vars.targets - opts.ParseParallelOpts.ParseOpts.JsonnetOpts = getJsonnetOpts() - opts.ParseParallelOpts.Selector = getLabelSelector() + opts := tanka.ExportEnvOpts{ + Format: *format, + Extension: *extension, + Merge: *merge, + Targets: vars.targets, + ParseParallelOpts: tanka.ParseParallelOpts{ + ParseOpts: tanka.ParseOpts{ + JsonnetOpts: getJsonnetOpts(), + }, + Selector: getLabelSelector(), + }, + } var paths []string for _, path := range args[1:] { diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 7bbaa3c68..4fa01efaf 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -27,30 +27,20 @@ const BelRune = string(rune(7)) // debugging purposes. const manifestFile = "manifest.json" +// ExportEnvOpts specify options on how to export environments type ExportEnvOpts struct { // formatting the filename based on the exported Kubernetes manifest - Format *string + Format string // extension of the filename - Extension *string + Extension string // merge export with existing directory - Merge *bool + Merge bool // optional: only export specified Kubernetes manifests Targets []string // optional: options for parsing Environments ParseParallelOpts ParseParallelOpts } -func DefaultExportEnvOpts() ExportEnvOpts { - format := "{{env.spec.namespace}}/{{env.metadata.name}}/{{.apiVersion}}.{{.kind}}-{{.metadata.name}}" - extension := "yaml" - merge := false - return ExportEnvOpts{ - Format: &format, - Extension: &extension, - Merge: &merge, - } -} - func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { // Keep track of which file maps to which environment fileToEnv := map[string]string{} @@ -60,7 +50,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { if err != nil { return fmt.Errorf("Checking target dir: %s", err) } - if !empty && !*opts.Merge { + if !empty && !opts.Merge { return fmt.Errorf("Output dir `%s` not empty. Pass --merge to ignore this", to) } @@ -94,7 +84,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // create template - manifestTemplate, err := createTemplate(*opts.Format, menv) + manifestTemplate, err := createTemplate(opts.Format, menv) if err != nil { return fmt.Errorf("Parsing format: %s", err) } @@ -108,7 +98,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // Create all subfolders in path - path := filepath.Join(to, name+"."+*opts.Extension) + path := filepath.Join(to, name+"."+opts.Extension) fileToEnv[path] = env.Metadata.Namespace From 0ed15063ce35f24354de4fd83f573352d3701735 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:13:23 +0100 Subject: [PATCH 28/40] fix: use process.StrExps --- cmd/tk/workflow.go | 9 +++++---- pkg/tanka/export.go | 10 +--------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index b71ac2d8d..c66c31ab4 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -8,6 +8,7 @@ import ( "github.com/go-clix/cli" "github.com/posener/complete" + "github.com/grafana/tanka/pkg/process" "github.com/grafana/tanka/pkg/tanka" "github.com/grafana/tanka/pkg/term" ) @@ -36,7 +37,7 @@ func applyCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - filters, err := tanka.StringsToRegexps(vars.targets) + filters, err := process.StrExps(vars.targets...) if err != nil { return err } @@ -85,7 +86,7 @@ func deleteCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - filters, err := tanka.StringsToRegexps(vars.targets) + filters, err := process.StrExps(vars.targets...) if err != nil { return err } @@ -115,7 +116,7 @@ func diffCmd() *cli.Command { getJsonnetOpts := jsonnetFlags(cmd.Flags()) cmd.Run = func(cmd *cli.Command, args []string) error { - filters, err := tanka.StringsToRegexps(vars.targets) + filters, err := process.StrExps(vars.targets...) if err != nil { return err } @@ -164,7 +165,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) return nil } - filters, err := tanka.StringsToRegexps(vars.targets) + filters, err := process.StrExps(vars.targets...) if err != nil { return err } diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 4fa01efaf..bbd71ba94 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -62,7 +62,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { for _, env := range envs { // select targets to export - filter, err := StringsToRegexps(opts.Targets) + filter, err := process.StrExps(opts.Targets...) if err != nil { return err } @@ -194,11 +194,3 @@ func applyTemplate(template *template.Template, m manifest.Manifest) (path strin return path, nil } - -func StringsToRegexps(exps []string) (process.Matchers, error) { - regexs, err := process.StrExps(exps...) - if err != nil { - return nil, err - } - return regexs, nil -} From 340c77675ae47de8f06bc28d5acea58beaea30d1 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:21:50 +0100 Subject: [PATCH 29/40] fix: include ParseOpts from cli --- cmd/tk/export.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 13856fe22..5e910bf39 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -63,9 +63,8 @@ func exportCmd() *cli.Command { paths = append(paths, env.Metadata.Namespace) } } else { - parseOpts := tanka.ParseOpts{ - Evaluator: tanka.EnvsOnlyEvaluator, - } + parseOpts := opts.ParseParallelOpts.ParseOpts + parseOpts.Evaluator = tanka.EnvsOnlyEvaluator _, _, err := tanka.ParseEnv(path, parseOpts) if err != nil { return err From 88af061f5f7a7d8686384af05969b9f34a33149b Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:24:27 +0100 Subject: [PATCH 30/40] fix: coding nits --- cmd/tk/export.go | 17 +++++++++-------- pkg/tanka/environments.go | 15 +++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 5e910bf39..768738058 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -62,15 +62,16 @@ func exportCmd() *cli.Command { for _, env := range envs { paths = append(paths, env.Metadata.Namespace) } - } else { - parseOpts := opts.ParseParallelOpts.ParseOpts - parseOpts.Evaluator = tanka.EnvsOnlyEvaluator - _, _, err := tanka.ParseEnv(path, parseOpts) - if err != nil { - return err - } - paths = append(paths, path) + continue + } + + parseOpts := opts.ParseParallelOpts.ParseOpts + parseOpts.Evaluator = tanka.EnvsOnlyEvaluator + _, _, err := tanka.ParseEnv(path, parseOpts) + if err != nil { + return err } + paths = append(paths, path) } return tanka.ExportEnvironments(paths, args[0], &opts) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index ddc1400f4..cd5126788 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -96,13 +96,11 @@ func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Env }() } - results := make([]*v1alpha1.Environment, len(paths)) - currentIndex := 0 + results := make([]*v1alpha1.Environment, 0, len(paths)) for _, path := range paths { env := &v1alpha1.Environment{} - results[currentIndex] = env - currentIndex++ + results = append(results, env) envsChan <- parseJob{ path: path, opts: opts.ParseOpts, @@ -113,10 +111,11 @@ func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Env wg.Wait() for _, env := range results { - if env != nil { - if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(env.Metadata) { - envs = append(envs, env) - } + if env == nil { + continue + } + if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(env.Metadata) { + envs = append(envs, env) } } From 7441ccc7ccce7ac330bdb8df24ea357c2b87c23c Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:28:47 +0100 Subject: [PATCH 31/40] fix: error formatting --- pkg/tanka/errors.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/tanka/errors.go b/pkg/tanka/errors.go index f4e9616a2..5d0c72a4d 100644 --- a/pkg/tanka/errors.go +++ b/pkg/tanka/errors.go @@ -3,8 +3,6 @@ package tanka import ( "fmt" "strings" - - "github.com/pkg/errors" ) // ErrNoEnv means that the given jsonnet has no Environment object @@ -33,9 +31,9 @@ type ErrParseParallel struct { } func (e ErrParseParallel) Error() string { - returnErr := errors.New("Unable to parse selected Environments") + returnErr := fmt.Sprintf("Unable to parse selected Environments:\n\n") for _, err := range e.errors { - returnErr = errors.Wrap(returnErr, err.Error()) + returnErr = fmt.Sprintf("%s- %s\n", returnErr, err.Error()) } - return returnErr.Error() + return returnErr } From 5db036ed289b75d724acf3da537e2ce8e983b938 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:36:04 +0100 Subject: [PATCH 32/40] fix: remove Data before re-serialization --- pkg/tanka/export.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index bbd71ba94..ba57d721e 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -74,6 +74,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } // create raw manifest version of env for templating + env.Data = nil raw, err := json.Marshal(env) if err != nil { return err From 70c1f4a33ceb49be8148b86742f945e58b6e063c Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:43:02 +0100 Subject: [PATCH 33/40] refactor: consolidate writing files in export.go --- pkg/tanka/export.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index ba57d721e..f6f2cbe16 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -110,28 +110,22 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { return fmt.Errorf("File '%s' already exists. Aborting", path) } - // Write file - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) - } + // Write manifest data := m.String() - if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { - return fmt.Errorf("writing manifest: %s", err) + if err := writeExportFile(path, []byte(data)); err != nil { + return err } } } - // create manifest file + // Write manifest file if len(fileToEnv) != 0 { data, err := json.MarshalIndent(fileToEnv, "", " ") if err != nil { return err } path := filepath.Join(to, manifestFile) - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) - } - if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil { + if err := writeExportFile(path, data); err != nil { return err } } @@ -166,6 +160,14 @@ func dirEmpty(dir string) (bool, error) { return false, err } +func writeExportFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err) + } + + return ioutil.WriteFile(path, data, 0644) +} + func createTemplate(format string, env manifest.Manifest) (*template.Template, error) { // Replace all os.path separators in string with BelRune for creating subfolders replaceFormat := strings.Replace(format, string(os.PathSeparator), BelRune, -1) From 80eec4437c0481dd547c49b51f2dbe63ff2ff5e0 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 24 Dec 2020 15:57:43 +0100 Subject: [PATCH 34/40] fix: tk env list with optional path (backwards compat) --- cmd/tk/env.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index 01642434d..5cf5f8186 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -232,11 +232,19 @@ func envRemoveCmd() *cli.Command { } func envListCmd() *cli.Command { + args := workflowArgs + args.Validator = cli.ValidateFunc(func(args []string) error { + if len(args) > 1 { + return fmt.Errorf("expects at most 1 arg, received %v", len(args)) + } + return nil + }) + cmd := &cli.Command{ - Use: "list ", + Use: "list []", Aliases: []string{"ls"}, - Short: "list environments relative to ", - Args: workflowArgs, + Short: "list environments relative to current dir or ", + Args: args, } useJSON := cmd.Flags().Bool("json", false, "json output") @@ -245,7 +253,17 @@ func envListCmd() *cli.Command { useNames := cmd.Flags().Bool("names", false, "plain names output") cmd.Run = func(cmd *cli.Command, args []string) error { - dir := args[0] + var dir string + var err error + if len(args) == 1 { + dir = args[0] + } else { + dir, err = os.Getwd() + if err != nil { + return nil + } + } + stat, err := os.Stat(dir) if err != nil { return err From a583ca84d6fa6d889d309790fcf253f4010195b5 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 26 Dec 2020 19:51:02 +0100 Subject: [PATCH 35/40] fix(api): prevent race condition --- pkg/tanka/parse.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/tanka/parse.go b/pkg/tanka/parse.go index 9955f36d5..2a2cacd08 100644 --- a/pkg/tanka/parse.go +++ b/pkg/tanka/parse.go @@ -9,6 +9,7 @@ import ( "github.com/Masterminds/semver" "github.com/pkg/errors" + "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes" "github.com/grafana/tanka/pkg/kubernetes/manifest" @@ -148,7 +149,13 @@ func ParseEnv(path string, opts JsonnetOpts) (interface{}, *v1alpha1.Environment if err != nil { return nil, nil, errors.Wrap(err, "marshalling environment config") } - opts.ExtCode.Set(spec.APIGroup+"/environment", string(jsonEnv)) + // make a copy to prevent race condition in parallel execution + injectCode := jsonnet.InjectedCode{} + for k, v := range opts.ExtCode { + injectCode.Set(k, v) + } + injectCode.Set(spec.APIGroup+"/environment", string(jsonEnv)) + opts.ExtCode = injectCode } raw, err := EvalJsonnet(path, opts) From 79fc752c2bfed9a55ceb01516330218e5c115d26 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 26 Dec 2020 19:52:46 +0100 Subject: [PATCH 36/40] fix(api): prevent another race condition + hanging worker --- pkg/tanka/environments.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index cd5126788..3ab4bd3aa 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -2,6 +2,7 @@ package tanka import ( "fmt" + "log" "os" "path/filepath" "sync" @@ -88,11 +89,11 @@ func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Env for i := 0; i < numParallel; i++ { wg.Add(1) go func() { - err := parseWorker(envsChan) - if err != nil { - allErrors = append(allErrors, err) + defer wg.Done() + errs := parseWorker(envsChan) + if errs != nil { + allErrors = append(allErrors, errs...) } - wg.Done() }() } @@ -132,13 +133,18 @@ type parseJob struct { env *v1alpha1.Environment } -func parseWorker(envsChan <-chan parseJob) error { +func parseWorker(envsChan <-chan parseJob) (errs []error) { for req := range envsChan { + log.Printf("Parsing %s\n", req.path) _, env, err := ParseEnv(req.path, req.opts) if err != nil { - return fmt.Errorf("%w: %s", err, req.path) + errs = append(errs, fmt.Errorf("%s:\n %w", req.path, err)) + continue } *req.env = *env } + if len(errs) != 0 { + return errs + } return nil } From af5fd23f5d4363e8c5ca3082f2eceb201fc7d239 Mon Sep 17 00:00:00 2001 From: Duologic Date: Sat, 26 Dec 2020 19:53:57 +0100 Subject: [PATCH 37/40] fix(cli): ensure relative paths are absolute from root --- cmd/tk/export.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 768738058..fc4f28ef1 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -2,9 +2,12 @@ package main import ( "fmt" + "path/filepath" "github.com/go-clix/cli" + "github.com/pkg/errors" + "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/tanka" ) @@ -55,12 +58,19 @@ func exportCmd() *cli.Command { var paths []string for _, path := range args[1:] { if *recursive { + rootDir, err := jpath.FindRoot(path) + if err != nil { + return errors.Wrap(err, "resolving jpath") + } + + // get absolute path to Environment envs, err := tanka.FindEnvironments(path, opts.ParseParallelOpts.Selector) if err != nil { return err } + for _, env := range envs { - paths = append(paths, env.Metadata.Namespace) + paths = append(paths, filepath.Join(rootDir, env.Metadata.Namespace)) } continue } From 25161b9dc73a7bd17a4025337dea11056e2de16c Mon Sep 17 00:00:00 2001 From: Duologic Date: Wed, 30 Dec 2020 21:23:07 +0100 Subject: [PATCH 38/40] post rebase --- cmd/tk/export.go | 12 +++++------- pkg/tanka/environments.go | 8 ++++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index fc4f28ef1..3c20067b1 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -48,10 +48,8 @@ func exportCmd() *cli.Command { Merge: *merge, Targets: vars.targets, ParseParallelOpts: tanka.ParseParallelOpts{ - ParseOpts: tanka.ParseOpts{ - JsonnetOpts: getJsonnetOpts(), - }, - Selector: getLabelSelector(), + JsonnetOpts: getJsonnetOpts(), + Selector: getLabelSelector(), }, } @@ -75,9 +73,9 @@ func exportCmd() *cli.Command { continue } - parseOpts := opts.ParseParallelOpts.ParseOpts - parseOpts.Evaluator = tanka.EnvsOnlyEvaluator - _, _, err := tanka.ParseEnv(path, parseOpts) + jsonnetOpts := opts.ParseParallelOpts.JsonnetOpts + jsonnetOpts.EvalScript = tanka.EnvsOnlyEvalScript + _, _, err := tanka.ParseEnv(path, jsonnetOpts) if err != nil { return err } diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index 3ab4bd3aa..0ba27671e 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -44,8 +44,8 @@ func FindEnvironments(workdir string, selector labels.Selector) (envs []*v1alpha return nil, err } opts := ParseParallelOpts{ - ParseOpts: ParseOpts{ - Evaluator: EnvsOnlyEvaluator, + JsonnetOpts: JsonnetOpts{ + EvalScript: EnvsOnlyEvalScript, }, Selector: selector, } @@ -104,7 +104,7 @@ func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Env results = append(results, env) envsChan <- parseJob{ path: path, - opts: opts.ParseOpts, + opts: opts.JsonnetOpts, env: env, } } @@ -129,7 +129,7 @@ func ParseParallel(paths []string, opts ParseParallelOpts) (envs []*v1alpha1.Env type parseJob struct { path string - opts ParseOpts + opts JsonnetOpts env *v1alpha1.Environment } From a83bb20822441cf5d264871aaacc537a680b5ee4 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 31 Dec 2020 00:32:35 +0100 Subject: [PATCH 39/40] fix: remove noisy log statement --- pkg/tanka/environments.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/tanka/environments.go b/pkg/tanka/environments.go index 0ba27671e..459df6dce 100644 --- a/pkg/tanka/environments.go +++ b/pkg/tanka/environments.go @@ -2,7 +2,6 @@ package tanka import ( "fmt" - "log" "os" "path/filepath" "sync" @@ -135,7 +134,6 @@ type parseJob struct { func parseWorker(envsChan <-chan parseJob) (errs []error) { for req := range envsChan { - log.Printf("Parsing %s\n", req.path) _, env, err := ParseEnv(req.path, req.opts) if err != nil { errs = append(errs, fmt.Errorf("%s:\n %w", req.path, err)) From 9085e6c13e951d1f5679f380f8ba336c2e3f72d5 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 31 Dec 2020 11:26:20 +0100 Subject: [PATCH 40/40] fix(cli): don't change original export format --- cmd/tk/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 3c20067b1..19eef18a8 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -28,7 +28,7 @@ func exportCmd() *cli.Command { format := cmd.Flags().String( "format", - "{{env.spec.namespace}}/{{env.metadata.name}}/{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", + "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames", )