diff --git a/SMARTMODE.md b/SMARTMODE.md new file mode 100644 index 00000000..7b5d3ab1 --- /dev/null +++ b/SMARTMODE.md @@ -0,0 +1,34 @@ +# Smart Mode + +Smart Mode looks at the changes you made and figures out which environments and applications need re-rendering. + +It is activated by default. + +## How it works + +### No changes relative to rendering + +If you make changes that have no impact on the rendered output of your workloads, e.g. modify `Dockerfile`, myks +will exit immediately, before any syncing or rendering happens. + +### Re-rendering of an application + +An application of a specific environment get re-rendered when: +- The `app-data.ytt.yaml` of that application has changed, e.g. `envs/env-1/_apps/app-1/app-data.ytt.yaml` +- The prototype application has changed, e.g. `prototypes/app-1/helm/app-1.yaml`, in which case all environments that use is re-render that application. + +### Re-rendering of an environment + +All applications of an environment get re-rendered when: +- The `env-data.ytt.yaml` of that environment has changed. +- The `env-data.ytt.yaml` of a parent environment of that environment has changed. +- **Edge case:** If you have made changes to an application in env-1, but at the same time have modified the `env-data.ytt.yaml` of env-2, smart-mode will re-render all applications of env-1 AND env-2, even though this is not strictly required for env-1. + +### Complete rendering + +A complete rendering of all environments and all applications is required when: +- A file in the common lib has changed, e.g. `/lib/common.lib.star` +- A file in the global ytt folder has changed, e.g. `/envs/_env/ytt/annotate_crds.yaml` +- The base `env-data.ytt.yaml` has changed: `/envs/env-data.ytt.yaml` + + diff --git a/Taskfile.yaml b/Taskfile.yaml index b5fa0b72..d35a022b 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -26,7 +26,7 @@ tasks: go:lint:sec: cmds: - - gosec ./... + - gosec -exclude=G304 ./... # G304: Potential file inclusion via variable is a false positive go:lint: desc: Lint the code diff --git a/cmd/root.go b/cmd/root.go index fcfc68c0..ec7a1a62 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -12,6 +12,8 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/mykso/myks/internal/myks" ) var ( @@ -24,8 +26,18 @@ var ( var rootCmd = &cobra.Command{ Use: "myks", Short: "Myks helps to manage configuration for kubernetes clusters", - Long: "Myks TBD", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + Long: `Myks fetches K8s workloads from a variety of sources, e.g. Helm charts or Git Repositories. It renders their respective yaml files to the file system in a structure of environments and their applications. + +It supports prototype applications that can be shared between environments and inheritance of configuration from parent environments to their "children". + +Myks supports two positional arguments: + +- A comma-separated list of environments to render. If you provide "ALL", all environments will be rendered. +- A comma-separated list of applications to render. If you provide "ALL", all applications will be rendered. + +If you do not provide any positional arguments, myks will run in "Smart Mode". In Smart Mode, myks will only render environments and applications that have changed since the last run. +`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // Check positional arguments: // 1. Comma-separated list of environment search paths or ALL to search everywhere (default: ALL) // 2. Comma-separated list of application names or none to process all applications (default: none) @@ -44,7 +56,17 @@ var rootCmd = &cobra.Command{ switch len(onlyArgs) { case 0: - // No positional arguments + // smart mode requires instantiation of globe object to get the list of environments + // the globe object will not be used later in the process. It is only used to get the list of all environments and their apps. + globeAllEnvsAndApps := myks.New(".") + targetEnvironments, targetApplications, err = globeAllEnvsAndApps.InitSmartMode() + if err != nil { + log.Warn().Err(err).Msg("Unable to run Smart Mode. Rendering everything.") + } + if targetEnvironments == nil && targetApplications == nil { + log.Warn().Msg("Smart Mode did not find any changes. Existing.") + os.Exit(0) + } case 1: if onlyArgs[0] != "ALL" { targetEnvironments = strings.Split(onlyArgs[0], ",") diff --git a/go.mod b/go.mod index aa70ceb1..517b8829 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,8 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.9.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 8d0ea93d..6d8960e3 100644 --- a/go.sum +++ b/go.sum @@ -212,6 +212,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -329,6 +331,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 5ffa85f2..a4345928 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -49,7 +49,6 @@ func NewEnvironment(g *Globe, dir string) *Environment { // Read an environment id from an environment data file. // The environment data file must exist and contain an .environment.id field. if err := env.setId(); err != nil { - log.Warn().Err(err).Str("dir", dir).Msg("Unable to set environment id") return nil } diff --git a/internal/myks/git.go b/internal/myks/git.go new file mode 100644 index 00000000..a8a27fd2 --- /dev/null +++ b/internal/myks/git.go @@ -0,0 +1,112 @@ +package myks + +import ( + "fmt" + "os" + "strings" + + "github.com/rs/zerolog/log" +) + +type ChangedFile struct { + path string + status string +} + +// return all files paths that were changed since the revision +func getChangedFiles(revision string) ([]ChangedFile, error) { + log := func(name string, args []string) { + log.Debug().Msg(msgRunCmd("get diff for smart-mode", name, args)) + } + _, err := runCmd("git", nil, []string{"add", ".", "--intent-to-add"}, log) + if err != nil { + return nil, err + } + result, err := runCmd("git", nil, []string{"diff", "--ignore-blank-lines", "--name-status", revision}, log) + if err != nil { + return nil, err + } + if result.Stdout == "" { + return nil, nil + } + return convertToChangedFiles(result.Stdout), err +} + +func convertToChangedFiles(changes string) []ChangedFile { + var cfs []ChangedFile + for _, str := range strings.Split(changes, "\n") { + if str != "" { + parts := strings.Split(str, "\t") + cf := ChangedFile{path: parts[1], status: parts[0]} + cfs = append(cfs, cf) + } + } + return cfs +} + +func extractChangedFilePaths(cfs []ChangedFile) []string { + var paths []string + for _, cf := range cfs { + paths = append(paths, cf.path) + } + return paths +} + +func extractChangedFilePathsWithStatus(cfs []ChangedFile, status string) []string { + filter := func(cf ChangedFile) bool { + if status == "" || cf.status == status { + return true + } + return false + } + return extractChangedFilePaths(extract(cfs, filter)) +} + +func extractChangedFilePathsWithoutStatus(cfs []ChangedFile, status string) []string { + filter := func(cf ChangedFile) bool { + if status == "" || cf.status != status { + return true + } + return false + } + return extractChangedFilePaths(extract(cfs, filter)) +} + +// get head revision of main branch +func getMainBranchHeadRevision(mainBranch string) (string, error) { + log := func(name string, args []string) { + log.Debug().Msg(msgRunCmd("get main branch head revision for smart-mode", name, args)) + } + _, err := runCmd("git", nil, []string{"fetch", "origin", mainBranch}, log) + if err != nil { + return "", err + } + cmdResult, err := runCmd("git", nil, []string{"merge-base", "origin/" + mainBranch, "HEAD"}, log) + if err != nil { + return "", err + } + // git adds new line to output which messes up the result + headRevision := strings.TrimRight(cmdResult.Stdout, "\n") + return headRevision, nil +} + +// get head revision +func getCurrentBranchHeadRevision() (string, error) { + log := func(name string, args []string) { + log.Debug().Msg(msgRunCmd("get current head revision for smart-mode", name, args)) + } + cmdResult, err := runCmd("git", nil, []string{"rev-parse", "HEAD"}, log) + if err != nil { + return "", fmt.Errorf("failed to get current branch head revision: %v", err) + } + return strings.TrimRight(cmdResult.Stdout, "\n"), nil +} + +func getDiffRevision(mainBranch string) (string, error) { + if os.Getenv("CI") != "" { + log.Debug().Msg("Pipeline mode: comparing with HEAD revision on main") + return getMainBranchHeadRevision(mainBranch) + } + log.Debug().Msg("Local mode: comparing with HEAD revision on current branch") + return getCurrentBranchHeadRevision() +} diff --git a/internal/myks/git_test.go b/internal/myks/git_test.go new file mode 100644 index 00000000..eda8cc6a --- /dev/null +++ b/internal/myks/git_test.go @@ -0,0 +1,199 @@ +package myks + +import ( + "reflect" + "sort" + "testing" +) + +func Test_getChangedFiles(t *testing.T) { + type args struct { + revision string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"happy path", args{"HEAD"}, false}, + {"sad path", args{"unknown-revision"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := getChangedFiles(tt.args.revision) + if (err != nil) != tt.wantErr { + t.Errorf("getChangedFiles() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func Test_getMainBranchHeadRevision(t *testing.T) { + type args struct { + mainBranch string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"happy path", args{"main"}, false}, + {"sad path", args{"unknown-branch"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getMainBranchHeadRevision(tt.args.mainBranch) + if (err != nil) != tt.wantErr { + t.Errorf("getMainBranchHeadRevision() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got == "" { + t.Errorf("getMainBranchHeadRevision() must not be empty") + } + }) + } +} + +func Test_getCurrentBranchHeadRevision(t *testing.T) { + tests := []struct { + name string + wantErr bool + }{ + {"happy path", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getCurrentBranchHeadRevision() + if (err != nil) != tt.wantErr { + t.Errorf("getCurrentBranchHeadRevision() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got == "" { + t.Errorf("getCurrentBranchHeadRevision() must not be empty") + } + }) + } +} + +func Test_convertToChangedFiles(t *testing.T) { + type args struct { + changes string + } + tests := []struct { + name string + args args + want []ChangedFile + }{ + { + "happy path", + args{ + "A\tfile1\nM\tfile2\nD\tfile3\n", + }, + []ChangedFile{ + {path: "file1", status: "A"}, + {path: "file2", status: "M"}, + {path: "file3", status: "D"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := convertToChangedFiles(tt.args.changes); !reflect.DeepEqual(got, tt.want) { + t.Errorf("convertToChangedFiles() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_extractChangedFilePathsWithStatus(t *testing.T) { + type args struct { + cfs []ChangedFile + status string + } + tests := []struct { + name string + args args + want []string + }{ + { + "filter out deletions", + args{ + []ChangedFile{ + {"file1", "M"}, + {"file2", "D"}, + {"file3", "D"}, + }, + "D", + }, + []string{"file2", "file3"}, + }, + { + "filter out noting", + args{ + []ChangedFile{ + {"file1", "M"}, + {"file2", "D"}, + {"file3", "D"}, + }, + "", + }, + []string{"file1", "file2", "file3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractChangedFilePathsWithStatus(tt.args.cfs, tt.args.status) + sort.Strings(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("extractChangedFilePathsWithStatus() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_extractChangedFilePathsWithoutStatus(t *testing.T) { + type args struct { + cfs []ChangedFile + status string + } + tests := []struct { + name string + args args + want []string + }{ + { + "filter out deletions", + args{ + []ChangedFile{ + {"file1", "M"}, + {"file2", "D"}, + {"file3", "D"}, + }, + "D", + }, + []string{"file1"}, + }, + { + "filter out noting", + args{ + []ChangedFile{ + {"file1", "M"}, + {"file2", "D"}, + {"file3", "D"}, + }, + "", + }, + []string{"file1", "file2", "file3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractChangedFilePathsWithoutStatus(tt.args.cfs, tt.args.status) + sort.Strings(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("extractChangedFilePathsWithStatus() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 84795aaf..9261446e 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -86,6 +86,8 @@ type Globe struct { YttPkgStepDirName string `default:"ytt-pkg"` // Ytt step directory name YttStepDirName string `default:"ytt"` + // Main branch name + MainBranchName string `default:"main"` /// User input @@ -141,7 +143,6 @@ func New(rootDir string) *Globe { if err := g.setGitRepoBranch(); err != nil { log.Warn().Err(err).Msg("Unable to set git repo branch") } - log.Debug().Interface("globe", g).Msg("Globe config") return g } @@ -156,8 +157,14 @@ func (g *Globe) Init(asyncLevel int, searchPaths []string, applicationNames []st } dataSchemaFileName := filepath.Join(g.RootDir, g.ServiceDirName, g.TempDirName, g.DataSchemaFileName) - if err := writeFile(dataSchemaFileName, dataSchema); err != nil { - log.Fatal().Err(err).Msg("Unable to write data schema file") + if _, err := os.Stat(dataSchemaFileName); err != nil { + log.Warn().Msg("Unable to find data schema file, creating one") + if err := os.MkdirAll(filepath.Dir(dataSchemaFileName), 0o750); err != nil { + log.Fatal().Err(err).Msg("Unable to create data schema file directory") + } + if err := os.WriteFile(dataSchemaFileName, dataSchema, 0o600); err != nil { + log.Fatal().Err(err).Msg("Unable to create data schema file") + } } g.extraYttPaths = append(g.extraYttPaths, dataSchemaFileName) @@ -346,7 +353,7 @@ func (g *Globe) collectEnvironmentsInPath(searchPath string) { if env != nil { g.environments[path] = env } else { - log.Warn().Str("path", path).Msg("Unable to collect environment, skipping") + log.Debug().Str("path", path).Msg("Unable to collect environment, might be base or parent environment. Skipping") } } } diff --git a/internal/myks/path.go b/internal/myks/path.go new file mode 100644 index 00000000..32e1a0b9 --- /dev/null +++ b/internal/myks/path.go @@ -0,0 +1,47 @@ +package myks + +import ( + "strings" + + _ "golang.org/x/exp/slices" +) + +// if changes are: +// /path1/path2/path3 +// /path1/path3/path3 +// then return /path1 +func removeSubPaths(paths []string) []string { + var results []string + for _, path := range paths { + if !isSubPath(path, paths) { + results = append(results, path) + } + } + return results +} + +// checks whether path is sub path of any of the paths within paths +func isSubPath(path string, paths []string) bool { + for _, curPath := range paths { + if curPath != path { + if strings.HasPrefix(path, curPath) { + return true + } + } + } + return false +} + +func removeDuplicates(paths []string) []string { + seen := make(map[string]bool) + var result []string + + for _, item := range paths { + if _, exists := seen[item]; !exists { + seen[item] = true + result = append(result, item) + } + } + + return result +} diff --git a/internal/myks/path_test.go b/internal/myks/path_test.go new file mode 100644 index 00000000..faf4ed99 --- /dev/null +++ b/internal/myks/path_test.go @@ -0,0 +1,103 @@ +package myks + +import ( + "reflect" + "testing" +) + +func Test_findCommonPath(t *testing.T) { + type args struct { + changes []string + } + tests := []struct { + name string + args args + want []string + }{ + { + "happy path", + args{ + []string{ + "/path1/path3", + "/path1/path3/file2", + }, + }, + []string{ + "/path1/path3", + }, + }, + { + "reverse order", + args{ + []string{ + "/path1/path3/file2", + "/path1/path3", + }, + }, + []string{ + "/path1/path3", + }, + }, + { + "no common path", + args{ + []string{ + "/path1/path3/file2", + "/path1/path4", + }, + }, + []string{ + "/path1/path3/file2", + "/path1/path4", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := removeSubPaths(tt.args.changes); !reflect.DeepEqual(got, tt.want) { + t.Errorf("removeSubPaths() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isSubPath(t *testing.T) { + type args struct { + path string + paths []string + } + tests := []struct { + name string + args args + want bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSubPath(tt.args.path, tt.args.paths); got != tt.want { + t.Errorf("isSubPath() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_removeDuplicates(t *testing.T) { + type args struct { + paths []string + } + tests := []struct { + name string + args args + want []string + }{ + {"happy path", args{[]string{"path1", "path2", "path3", "path2"}}, []string{"path1", "path2", "path3"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := removeDuplicates(tt.args.paths); !reflect.DeepEqual(got, tt.want) { + t.Errorf("removeDuplicates() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/myks/smart_mode.go b/internal/myks/smart_mode.go new file mode 100644 index 00000000..665c041f --- /dev/null +++ b/internal/myks/smart_mode.go @@ -0,0 +1,179 @@ +package myks + +import ( + "fmt" + "golang.org/x/exp/slices" + "regexp" + "sort" + "strings" + + "github.com/rs/zerolog/log" +) + +func (g *Globe) getGlobalLibDirExpr() string { + return "^" + g.YttLibraryDirName + "/.*$" +} + +func (g *Globe) getGlobalYttDirExpr() string { + return "^" + g.EnvironmentBaseDir + "/_env/" + g.YttStepDirName + "/.*$" +} + +func (g *Globe) getGlobalEnvExpr() string { + return "^" + g.EnvironmentBaseDir + "/" + g.EnvironmentDataFileName + "$" +} + +func (g *Globe) getBaseAppExpr() string { + return "^" + g.PrototypesDir + "/(?:.*?/)?(.*?)/[(?:/" + g.YttStepDirName + "),(?:/helm),(?:/vendir),(?:/ytt\\-pkg)].*$" +} + +func (g *Globe) getBaseAppDataFileExpr() string { + return "^" + g.PrototypesDir + "/(?:.*?/)?(.*?)/" + g.ApplicationDataFileName + "$" +} + +func (g *Globe) getEnvsExpr() string { + return "^(" + g.EnvironmentBaseDir + "/.+)/" + g.EnvironmentDataFileName + "$" +} + +func (g *Globe) getAppsExpr() string { + return "^(" + g.EnvironmentBaseDir + "/.*?)/_apps/(.*?)/.*$" +} + +func (g *Globe) InitSmartMode() ([]string, []string, error) { + g.collectEnvironments(nil) + + err := process(0, g.environments, func(item interface{}) error { + env, ok := item.(*Environment) + if !ok { + return fmt.Errorf("unable to cast item to *Environment") + } + return env.initEnvData() + }) + if err != nil { + log.Err(err).Msg(g.Msg("Failed to collect environments")) + return nil, nil, err + } + + curRev, err := getDiffRevision(g.MainBranchName) + if err != nil { + log.Err(err).Msg(g.Msg("Failed to get current revision")) + return nil, nil, err + } + changedFiles, err := getChangedFiles(curRev) + if err != nil { + log.Err(err).Msg(g.Msg("Failed to get diff")) + return nil, nil, err + } + envs, apps := g.runSmartMode(changedFiles) + log.Info().Msg(g.Msg(fmt.Sprintf("Smart mode detected changes in environments: %v, applications: %v", envs, apps))) + return envs, apps, err +} + +func (g *Globe) runSmartMode(changedFiles []ChangedFile) ([]string, []string) { + allChangedFilePaths := extractChangedFilePathsWithStatus(changedFiles, "") + allDeletions := extractChangedFilePathsWithStatus(changedFiles, "D") + allChangedFilesExceptDeletions := extractChangedFilePathsWithoutStatus(changedFiles, "D") + + if g.checkGlobalConfigChanged(allChangedFilePaths) { + return nil, nil + } + modifiedEnvs := g.getModifiedEnvs(allChangedFilesExceptDeletions) + modifiedEnvsFromApp, modifiedApps := g.getModifiedApps(allChangedFilePaths, g.getModifiedEnvs(allDeletions)) + modifiedBaseApps := g.getModifiedBaseApps(allChangedFilePaths) + modifiedEnvsFromBase, modifiedAppsFromBase := g.findBaseAppUsage(modifiedBaseApps) + + // Once envs have been modified globally, we can no longer render individual apps, since we don't know which apps are affected. + // This goes for editing of env-data.ytt.yaml, global ytt files as well as manifests. + if len(modifiedEnvs) > 0 { + envs := append(modifiedEnvs, modifiedEnvsFromBase...) + envs = append(envs, modifiedEnvsFromApp...) + envs = removeDuplicates(envs) + sort.Strings(envs) + return envs, nil + } else { + envs := removeDuplicates(append(modifiedEnvsFromBase, modifiedEnvsFromApp...)) + sort.Strings(envs) + apps := removeDuplicates(append(modifiedApps, modifiedAppsFromBase...)) + sort.Strings(apps) + return envs, apps + } +} + +func (g *Globe) findBaseAppUsage(baseApps []string) ([]string, []string) { + var envs []string + var apps []string + for _, baseApp := range baseApps { + for envPath, env := range g.environments { + for proto, appName := range env.foundApplications { + if proto == baseApp || strings.HasSuffix(proto, "/"+baseApp) { + envs = append(envs, envPath) + apps = append(apps, appName) + } + } + } + } + return removeDuplicates(envs), removeDuplicates(apps) +} + +func (g *Globe) checkGlobalConfigChanged(changedFiles []string) bool { + return checkFileChanged(changedFiles, g.getGlobalLibDirExpr(), g.getGlobalYttDirExpr(), g.getGlobalEnvExpr()) +} + +func (g *Globe) getModifiedBaseApps(changedFiles []string) []string { + changes, _ := getChanges(changedFiles, g.getBaseAppDataFileExpr(), g.getBaseAppExpr()) + return changes +} + +func (g *Globe) getModifiedApps(changedFiles []string, deletedEnvs []string) ([]string, []string) { + envs, apps := getChanges(changedFiles, g.getAppsExpr()) + return filterDeletedEnvs(envs, apps, deletedEnvs) +} + +func (g *Globe) getModifiedEnvs(changedFiles []string) []string { + modifiedEnvs, _ := getChanges(changedFiles, g.getEnvsExpr()) + return removeSubPaths(modifiedEnvs) +} + +func checkFileChanged(changedFiles []string, regExps ...string) bool { + for _, expr := range regExps { + changes, _ := getChanges(changedFiles, expr) + if len(changes) > 0 { + return true + } + } + return false +} + +func getChanges(changedFilePaths []string, regExps ...string) ([]string, []string) { + var matches1 []string + var matches2 []string + for _, expr := range regExps { + for _, line := range changedFilePaths { + expr := regexp.MustCompile(expr) + matches := expr.FindStringSubmatch(line) + if matches != nil { + if len(matches) == 1 { + matches1 = append(matches1, matches[0]) + } else if len(matches) == 2 { + matches1 = append(matches1, matches[1]) + } else { + matches1 = append(matches1, matches[1]) + matches2 = append(matches2, matches[2]) + } + } + } + } + return matches1, matches2 +} + +func filterDeletedEnvs(envs []string, apps []string, deletedEnvs []string) ([]string, []string) { + var resultEnvs []string + var resultApps []string + for i, env := range envs { + if !slices.Contains(deletedEnvs, env) { + resultEnvs = append(resultEnvs, env) + resultApps = append(resultApps, apps[i]) + } + } + + return resultEnvs, resultApps +} diff --git a/internal/myks/smart_mode_test.go b/internal/myks/smart_mode_test.go new file mode 100644 index 00000000..d51cb8c1 --- /dev/null +++ b/internal/myks/smart_mode_test.go @@ -0,0 +1,442 @@ +package myks + +import ( + "reflect" + "sort" + "testing" + + "github.com/creasty/defaults" +) + +func Test_getChanges(t *testing.T) { + type args struct { + diff []string + regExPattern string + } + tests := []struct { + name string + args args + want []string + }{ + { + "happy path", + args{ + []string{ + "path1/file1", + "path1/file2", + }, + "^path1/(.*)$", + }, + []string{ + "file1", + "file2", + }, + }, + { + "no capture group", + args{ + []string{ + "path1/file1", + "path1/file2", + }, + "^path1/.*$", + }, + []string{ + "path1/file1", + "path1/file2", + }, + }, + { + "no match", + args{ + []string{ + "nothing-to-match", + }, + "^path1/.*$", + }, + nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := getChanges(tt.args.diff, tt.args.regExPattern); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getChanges() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkFileChanged(t *testing.T) { + type args struct { + changedFiles []string + regExps []string + } + tests := []struct { + name string + args args + want bool + }{ + {"happy path", args{[]string{"path1/file1"}, []string{"^path1/(.*)$"}}, true}, + {"no match", args{[]string{"path1/file1"}, []string{"no-match"}}, false}, + {"empty", args{[]string{}, []string{"no-match"}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := checkFileChanged(tt.args.changedFiles, tt.args.regExps...); got != tt.want { + t.Errorf("checkFileChanged() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGlobe_getModifiedEnvs(t *testing.T) { + type args struct { + changedFiles []string + } + tests := []struct { + name string + args args + want []string + }{ + {"cross check", args{[]string{"some-irrelevant-path.yaml"}}, nil}, + {"happy path", args{[]string{ + "envs/env1/env-data.ytt.yaml", + "envs/sub-env/env2/env-data.ytt.yaml", + "envs/sub-env/env4/some-file.ytt.yaml", + }}, []string{ + "envs/env1", + "envs/sub-env/env2", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := createGlobe(t) + envs := g.getModifiedEnvs(tt.args.changedFiles) + sort.Strings(envs) + if got := envs; !reflect.DeepEqual(got, tt.want) { + t.Errorf("getChanges() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGlobe_getModifiedBaseApps(t *testing.T) { + type args struct { + changedFiles []string + } + tests := []struct { + name string + args args + want []string + }{ + {"cross check", args{[]string{"some-irrelevant-path.yaml"}}, nil}, + {"happy path", args{[]string{ + "prototypes/app1/app-data.ytt.yaml", + "prototypes/app2/vendir/app.yaml", + "prototypes/app3/ytt/app.yaml", + "prototypes/app4/ytt-pkg/app.yaml", + "prototypes/app5/helm/app.yaml", + "prototypes/app5/any/app.yaml", + }}, []string{ + "app1", + "app2", + "app3", + "app4", + "app5", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := createGlobe(t) + apps := g.getModifiedBaseApps(tt.args.changedFiles) + sort.Strings(apps) + if got := apps; !reflect.DeepEqual(got, tt.want) { + t.Errorf("getChanges() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGlobe_getModifiedApps(t *testing.T) { + type args struct { + changedFiles []string + deletedEnvs []string + } + tests := []struct { + name string + args args + wantEnvs []string + wantApps []string + }{ + {"cross check", args{[]string{"some-irrelevant-path.yaml"}, nil}, nil, nil}, + { + "happy path", + args{[]string{ + "envs/env1/_apps/app1/app.yaml", + "envs/env1/env2/_apps/app2/app.yaml", + "envs/env1/no-app/test.yaml", + "base/env1/env2/_apps/app2/app.yaml", + }, nil}, + []string{ + "envs/env1", + "envs/env1/env2", + }, + []string{ + "app1", + "app2", + }, + }, + { + "exclude deleted env", + args{[]string{ + "envs/env1/_apps/app1/app.yaml", + "envs/env2/_apps/app2/app.yaml", + }, []string{ + "envs/env1", + }}, + []string{ + "envs/env2", + }, + []string{ + "app2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := createGlobe(t) + gotEnvs, gotApps := g.getModifiedApps(tt.args.changedFiles, tt.args.deletedEnvs) + sort.Strings(gotEnvs) + sort.Strings(gotApps) + if !reflect.DeepEqual(gotEnvs, tt.wantEnvs) { + t.Errorf("getChanges() = %v, want %v", gotEnvs, tt.wantEnvs) + } + if !reflect.DeepEqual(gotApps, tt.wantApps) { + t.Errorf("getChanges() = %v, want %v", gotApps, tt.wantApps) + } + }) + } +} + +func TestGlobe_checkGlobalConfigChanged(t *testing.T) { + type args struct { + changedFiles []string + } + tests := []struct { + name string + args args + want bool + }{ + {"cross check", args{[]string{"envs/some-env/env-data.ytt.yaml"}}, false}, + {"common lib", args{[]string{"lib/file1"}}, true}, + {"common lib sub", args{[]string{"lib/sub/file1"}}, true}, + {"match with additional file", args{[]string{"lib/sub/file1", "some-irrelevant-file"}}, true}, + {"common ytt lib", args{[]string{"envs/_env/ytt/file1"}}, true}, + {"common ytt lib sub", args{[]string{"envs/_env/ytt/sub/file1"}}, true}, + {"root env", args{[]string{"envs/env-data.ytt.yaml"}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := createGlobe(t) + if got := g.checkGlobalConfigChanged(tt.args.changedFiles); got != tt.want { + t.Errorf("checkGlobalConfigChanged() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGlobe_findBaseAppUsage(t *testing.T) { + type args struct { + baseApps []string + globe Globe + } + tests := []struct { + name string + args args + wantEnvs []string + wantApps []string + }{ + { + "happy path", + args{ + []string{"app1"}, + Globe{ + environments: map[string]*Environment{ + "env1": { + Id: "env1", + foundApplications: map[string]string{ + "app1": "app1", + "app2": "app2", + }, + }, + }, + }, + }, + []string{"env1"}, + []string{"app1"}, + }, + { + "prototype ref", + args{ + []string{"app1", "app2"}, + Globe{ + environments: map[string]*Environment{ + "env1": { + Id: "env1", + foundApplications: map[string]string{ + "app1": "my-app-1", + "root/app2": "my-app-2", + }, + }, + }, + }, + }, + []string{"env1"}, + []string{"my-app-1", "my-app-2"}, + }, + { + "duplicates", + args{ + []string{"app1", "app2"}, + Globe{ + environments: map[string]*Environment{ + "env1": { + Id: "env1", + foundApplications: map[string]string{ + "app1": "my-app-1", + }, + }, + "env2": { + Id: "env2", + foundApplications: map[string]string{ + "app1": "my-app-1", + }, + }, + }, + }, + }, + []string{"env1", "env2"}, + []string{"my-app-1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotEnvs, gotApps := tt.args.globe.findBaseAppUsage(tt.args.baseApps) + sort.Strings(gotEnvs) + sort.Strings(gotApps) + if !reflect.DeepEqual(gotEnvs, tt.wantEnvs) { + t.Errorf("findBaseAppUsage() got = %v, want %v", gotEnvs, tt.wantEnvs) + } + if !reflect.DeepEqual(gotApps, tt.wantApps) { + t.Errorf("findBaseAppUsage() got1 = %v, want %v", gotApps, tt.wantApps) + } + }) + } +} + +func TestGlobe_runSmartMode(t *testing.T) { + g := createGlobe(t) + g.environments = map[string]*Environment{ + "envs/env1": { + Id: "env1", + foundApplications: map[string]string{ + "app1": "app1", + "app2": "app2", + }, + }, + "envs/env2": { + Id: "env2", + foundApplications: map[string]string{ + "app3": "app3", + "app2": "app2", + }, + }, + } + type args struct { + changedFiles []ChangedFile + } + tests := []struct { + name string + args args + wantEnvs []string + wantApps []string + }{ + { + "change to global lib", + args{ + []ChangedFile{{"lib/file1", "M"}}, + }, + nil, + nil, + }, + { + "change to base app", + args{ + []ChangedFile{{"prototypes/app1/app-data.ytt.yaml", "M"}}, + }, + []string{"envs/env1"}, + []string{"app1"}, + }, + { + "change to app", + args{ + []ChangedFile{{"envs/env1/_apps/app1/app-data.ytt.yaml", "M"}}, + }, + []string{"envs/env1"}, + []string{"app1"}, + }, + { + "change to env", + args{ + []ChangedFile{ + {"envs/env1/env-data.ytt.yaml", "M"}, + {"envs/env1/_apps/app1/app-data.ytt.yaml", "M"}, + }, + }, + []string{"envs/env1"}, + nil, + }, + { + "ignore env deletion", + args{ + []ChangedFile{ + {"envs/env1/env-data.ytt.yaml", "D"}, + }, + }, + nil, + nil, + }, + { + "changes to all multiple envs and apps", + args{ + []ChangedFile{ + {"prototypes/app2/app-data.ytt.yaml", "M"}, + {"envs/env2/_apps/app3/some-file.yaml", "M"}, + }, + }, + []string{"envs/env1", "envs/env2"}, + []string{"app2", "app3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotEnvs, gotApps := g.runSmartMode(tt.args.changedFiles) + sort.Strings(gotEnvs) + sort.Strings(gotApps) + if !reflect.DeepEqual(gotEnvs, tt.wantEnvs) { + t.Errorf("runSmartMode() got = %v, want %v", gotEnvs, tt.wantEnvs) + } + if !reflect.DeepEqual(gotApps, tt.wantApps) { + t.Errorf("runSmartMode() got1 = %v, want %v", gotApps, tt.wantApps) + } + }) + } +} + +func createGlobe(t *testing.T) *Globe { + g := &Globe{} + if err := defaults.Set(g); err != nil { + t.Errorf("failed to create Globe object") + } + return g +} diff --git a/internal/myks/util.go b/internal/myks/util.go index 9b4bf9d3..ac1ddc47 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -267,3 +267,13 @@ func runYttWithFilesAndStdin(paths []string, stdin io.Reader, log func(name stri cmdArgs = append(cmdArgs, args...) return runCmd("ytt", stdin, cmdArgs, log) } + +func extract[T any](items []T, filterFunc func(cf T) bool) []T { + var result []T + for _, item := range items { + if filterFunc(item) { + result = append(result, item) + } + } + return result +} diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index 75a472d7..2a072354 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -417,3 +417,42 @@ func diff(a, b interface{}) string { text, _ := difflib.GetUnifiedDiffString(diff) return text } + +func Test_extract(t *testing.T) { + type TestMe struct { + Name string + } + type args[T any] struct { + items []T + filterFunc func(cf T) bool + } + type testCase[T any] struct { + name string + args args[T] + want []T + } + tests := []testCase[TestMe]{ + { + name: "happy path", + args: args[TestMe]{ + []TestMe{ + {Name: "test1"}, + {Name: "test2"}, + }, + func(cf TestMe) bool { + return cf.Name == "test1" + }, + }, + want: []TestMe{ + {Name: "test1"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := extract(tt.args.items, tt.args.filterFunc); !reflect.DeepEqual(got, tt.want) { + t.Errorf("extract() = %v, want %v", got, tt.want) + } + }) + } +}