From b0c50c265348e9b62d064d61740573bf9df1b4f6 Mon Sep 17 00:00:00 2001 From: Fritz Durchardt Date: Thu, 20 Jul 2023 11:51:11 +0200 Subject: [PATCH] feat: Add vendir authentication via environment --- .gitignore | 1 + internal/myks/application.go | 38 ++---- internal/myks/sync.go | 134 ++++++++++++++---- internal/myks/sync_test.go | 203 ++++++++++++++++++++-------- internal/myks/template.go | 29 ++++ internal/myks/template_test.go | 42 ++++++ internal/myks/templates/secret.yaml | 9 ++ internal/myks/util.go | 42 +++++- internal/myks/util_test.go | 83 +++++++++++- testData/sync/vendir-oci.yaml | 10 ++ testData/sync/vendir-simple.yaml | 2 + 11 files changed, 478 insertions(+), 115 deletions(-) create mode 100644 internal/myks/template.go create mode 100644 internal/myks/template_test.go create mode 100644 internal/myks/templates/secret.yaml create mode 100644 testData/sync/vendir-oci.yaml diff --git a/.gitignore b/.gitignore index 3982db88..64c021e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ bin/* dist .idea +.run diff --git a/internal/myks/application.go b/internal/myks/application.go index 1f17d655..85c9981a 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -31,7 +31,7 @@ type HelmConfig struct { IncludeCRDs bool `yaml:"includeCRDs"` } -var ErrNoVendirConfig = errors.New("No vendir config found") +var ErrNoVendirConfig = errors.New("no vendir config found") func NewApplication(e *Environment, name string, prototypeName string) (*Application, error) { if prototypeName == "" { @@ -41,7 +41,7 @@ func NewApplication(e *Environment, name string, prototypeName string) (*Applica prototype := filepath.Join(e.g.PrototypesDir, prototypeName) if _, err := os.Stat(prototype); err != nil { - return nil, errors.New("Application prototype does not exist") + return nil, errors.New("application prototype does not exist") } app := &Application{ @@ -164,7 +164,7 @@ func (a *Application) prepareSync() error { // 1. If exists, use the `apps//vendir` directory. // 2. If exists, for every level of environments use `/_apps//vendir` directory. - yttFiles := []string{} + var yttFiles []string protoVendirDir := filepath.Join(a.Prototype, "vendir") if _, err := os.Stat(protoVendirDir); err == nil { @@ -222,26 +222,12 @@ func (a *Application) expandTempPath(path string) string { return a.expandServicePath(filepath.Join(a.e.g.TempDirName, path)) } -// TODO: for content, use []byte instead of string -func (a *Application) writeFile(path string, content string) error { - dir := filepath.Dir(path) - if _, err := os.Stat(dir); err != nil { - err := os.MkdirAll(dir, 0o750) - if err != nil { - log.Warn().Err(err).Msg("Unable to create directory") - return err - } - } - - return os.WriteFile(path, []byte(content), 0o600) -} - func (a *Application) writeServiceFile(name string, content string) error { - return a.writeFile(a.expandServicePath(name), content) + return writeFile(a.expandServicePath(name), []byte(content)) } func (a *Application) writeTempFile(name string, content string) error { - return a.writeFile(a.expandTempPath(name), content) + return writeFile(a.expandTempPath(name), []byte(content)) } func (a *Application) collectDataFiles() { @@ -270,7 +256,7 @@ func (a *Application) runHelm() (string, error) { return "", err } - commonHelmArgs := []string{} + var commonHelmArgs []string // FIXME: move Namespace to a per-chart config if helmConfig.Namespace == "" { @@ -287,7 +273,7 @@ func (a *Application) runHelm() (string, error) { commonHelmArgs = append(commonHelmArgs, "--include-crds") } - outputs := []string{} + var outputs []string for _, chartDir := range chartDirs { chartName := filepath.Base(chartDir) @@ -349,7 +335,7 @@ func (a *Application) getHelmChartDirs() []string { return []string{} } - chartDirs := []string{} + var chartDirs []string err = filepath.Walk(chartsDir, func(path string, info os.FileInfo, err error) error { if err != nil { log.Warn().Err(err).Str("path", path).Msg("Unable to walk helm charts directory") @@ -377,7 +363,7 @@ func (a *Application) getHelmChartDirs() []string { func (a *Application) prepareHelm(chartName string) error { helmValuesFileName := a.getHelmValuesFileName(chartName) - helmYttFiles := []string{} + var helmYttFiles []string prototypeHelmValues := filepath.Join(a.Prototype, helmValuesFileName) if _, err := os.Stat(prototypeHelmValues); err == nil { @@ -448,7 +434,7 @@ func (a *Application) getHelmValuesFileName(chartName string) string { } func (a *Application) runYtt(previousStepFile string) (string, error) { - yttFiles := []string{} + var yttFiles []string yttFiles = append(yttFiles, a.yttDataFiles...) @@ -490,7 +476,7 @@ func (a *Application) runYtt(previousStepFile string) (string, error) { } func (a *Application) runGlobalYtt(previousStepFile string) (string, error) { - yttFiles := []string{} + var yttFiles []string yttFiles = append(yttFiles, a.yttDataFiles...) @@ -574,7 +560,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { if _, err := os.Stat(filePath); err == nil { log.Warn().Str("app", a.Name).Str("file", filePath).Msg("File already exists, check duplicated resources") } - err = a.writeFile(filePath, data.String()) + err = writeFile(filePath, data.Bytes()) if err != nil { log.Warn().Err(err).Str("app", a.Name).Str("file", filePath).Msg("Unable to write file") return err diff --git a/internal/myks/sync.go b/internal/myks/sync.go index ca56d06d..63825e7c 100644 --- a/internal/myks/sync.go +++ b/internal/myks/sync.go @@ -6,22 +6,24 @@ import ( "gopkg.in/yaml.v3" "os" "path/filepath" + "strings" ) +const secretEnvPrefix = "VENDIR_SECRET_" + type Directory struct { Path string ContentHash string `yaml:"contentHash"` + Secret string `yaml:"-"` } func (a *Application) doSync() error { - // TODO: implement secrets-from-env extraction - // Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`) vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName) vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName) vendirConfigFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirConfigFileName) vendirLockFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirLockFileName) - vendirSyncPath := a.expandTempPath(a.e.g.VendirSyncFileName) + vendirSyncFilePath := a.expandTempPath(a.e.g.VendirSyncFileName) vendorDir := a.expandPath(a.e.g.VendorDirName) vendirDirs, err := readVendirConfig(vendirConfigFilePath) @@ -30,9 +32,9 @@ func (a *Application) doSync() error { return err } - syncFileDirs, err := readSyncFile(vendirSyncPath) + syncFileDirs, err := readSyncFile(vendirSyncFilePath) if err != nil { - log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncPath) + log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncFilePath) return err } if len(syncFileDirs) == 0 { @@ -51,6 +53,7 @@ func (a *Application) doSync() error { return err } + var secretFilePaths []string //TODO sync retry // only sync vendir with directory flag, if the lock file matches the vendir config file and caching is enabled if a.cached && checkLockFileMatch(vendirDirs, lockFileDirs) { @@ -60,51 +63,86 @@ func (a *Application) doSync() error { continue } log.Info().Str("app", a.Name).Msg("Syncing vendir for: " + dir.Path) - res, err := runCmd("vendir", nil, []string{ + args := []string{ "sync", "--chdir=" + vendorDir, "--directory=" + dir.Path, "--file=" + vendirConfigFileRelativePath, "--lock-file=" + vendirLockFileRelativePath, - }) + } + args, secretFilePath, err := handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path) + return err + } + if secretFilePath != "" { + secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath) + } + + res, err := runCmd("vendir", nil, args) if err != nil { log.Warn().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") return err } } + } else { log.Info().Str("app", a.Name).Msg("Syncing vendir completely for: " + vendirConfigFilePath) - res, err := runCmd("vendir", nil, []string{ + args := []string{ "sync", "--chdir=" + vendorDir, "--file=" + vendirConfigFileRelativePath, "--lock-file=" + vendirLockFileRelativePath, - }) + } + for _, dir := range vendirDirs { + var secretFilePath string + args, secretFilePath, err = handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path) + return err + } + if secretFilePath != "" { + secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath) + } + + } + res, err := runCmd("vendir", nil, args) if err != nil { log.Error().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") return err } } - err = a.writeSyncFile(vendirDirs) + // make sure secrets do not linger on disk + for _, secretFilePath := range secretFilePaths { + defer func(name string) { + log.Debug().Str("app", a.Name).Msg("delete secret file: " + name) + err := os.Remove(name) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("unable to delete secret file: " + name) + } + }(secretFilePath) + } + + err = writeSyncFile(a.expandTempPath(a.e.g.VendirSyncFileName), vendirDirs) if err != nil { log.Error().Str("app", a.Name).Err(err).Msg("Unable to write sync file") return err } - log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncPath) + log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncFilePath) log.Info().Str("app", a.Name).Msg("Vendir sync completed!") return nil } -func (a *Application) writeSyncFile(directories []Directory) error { +func writeSyncFile(syncFilePath string, directories []Directory) error { bytes, err := yaml.Marshal(directories) if err != nil { return err } - err = a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes)) + err = writeFile(syncFilePath, bytes) if err != nil { return err } @@ -112,6 +150,30 @@ func (a *Application) writeSyncFile(directories []Directory) error { return nil } +func handleVendirSecret(dir Directory, tempPath string, tempRelativePath string, vendirArgs []string) ([]string, string, error) { + if dir.Secret != "" { + username, password, err := getEnvCreds(dir.Secret) + if err != nil { + return vendirArgs, "", err + } + secretFileName := dir.Secret + ".yaml" + secretFilePath := filepath.Join(tempPath, secretFileName) + err = writeSecretFile(dir.Secret, secretFilePath, username, password) + if err != nil { + return vendirArgs, "", err + } + secretRelativePath := filepath.Join(tempRelativePath, secretFileName) + var addedSecret bool + vendirArgs, addedSecret = appendIfNotExists(vendirArgs, "--file="+secretRelativePath) + if addedSecret { + return vendirArgs, secretFilePath, nil + } else { + return vendirArgs, "", nil + } + } + return vendirArgs, "", nil +} + func readVendirConfig(vendirConfigFilePath string) ([]Directory, error) { config, err := unmarshalYamlToMap(vendirConfigFilePath) if err != nil { @@ -170,7 +232,8 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) { if _, ok := config["directories"]; !ok { return nil, errors.New("no directories found in vendir config") } - var directories = make(map[string]string) + var syncDirs []Directory + for _, dir := range config["directories"].([]interface{}) { dirMap := dir.(map[string]interface{}) path := dirMap["path"].(string) @@ -183,24 +246,38 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) { if subPath != "." { path += "/" + subPath } + + secret := "" + if contents["imgpkgBundle"] != nil { + imgpkgBundle := contents["imgpkgBundle"].(map[string]interface{}) + if imgpkgBundle["secretRef"] != nil { + secretRef := imgpkgBundle["secretRef"].(map[string]interface{}) + secret = secretRef["name"].(string) + } + } + + if contents["helmChart"] != nil { + helmChart := contents["helmChart"].(map[string]interface{}) + if helmChart["repository"] != nil { + repository := helmChart["repository"].(map[string]interface{}) + if repository["secretRef"] != nil { + secretRef := repository["secretRef"].(map[string]interface{}) + secret = secretRef["name"].(string) + } + } + } + sortedYaml, err := sortYaml(contents) if err != nil { return nil, err } - directories[path] = sortedYaml - } - return convertDirectoryMapToHashedStruct(directories), nil -} - -func convertDirectoryMapToHashedStruct(directories map[string]string) []Directory { - var syncDirs []Directory - for path, contents := range directories { syncDirs = append(syncDirs, Directory{ Path: path, - ContentHash: hash(contents), + ContentHash: hash(sortedYaml), + Secret: secret, }) } - return syncDirs + return syncDirs, nil } func checkVersionMatch(path string, contentHash string, syncDirs []Directory) bool { @@ -231,3 +308,12 @@ func checkLockFileMatch(vendirDirs []Directory, lockFileDirs []Directory) bool { } return true } + +func getEnvCreds(secretName string) (string, string, error) { + username := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_USERNAME") + password := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_PASSWORD") + if username == "" || password == "" { + return "", "", errors.New("no credentials found in environment for secret: " + secretName) + } + return username, password, nil +} diff --git a/internal/myks/sync_test.go b/internal/myks/sync_test.go index ed5e91ed..b30915bb 100644 --- a/internal/myks/sync_test.go +++ b/internal/myks/sync_test.go @@ -2,58 +2,11 @@ package myks import ( "os" + "path/filepath" "reflect" "testing" ) -func TestApplication_writeSyncFile(t *testing.T) { - type fields struct { - Name string - Prototype string - e *Environment - yttDataFiles []string - } - type args struct { - directories []Directory - } - tests := []struct { - name string - fields fields - args args - want string - wantErr bool - }{ - { - "happy path", - fields{"name", "proto", &Environment{Dir: "/tmp", g: &Globe{VendirSyncFileName: "TestFile"}}, []string{}}, - args{[]Directory{{"path", "hash"}, {"path2", "hash2"}}}, - "- path: path\n contentHash: hash\n- path: path2\n contentHash: hash2\n", - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := &Application{ - Name: tt.fields.Name, - Prototype: tt.fields.Prototype, - e: tt.fields.e, - yttDataFiles: tt.fields.yttDataFiles, - } - // write sync file - if err := a.writeSyncFile(tt.args.directories); (err != nil) != tt.wantErr { - t.Errorf("writeSyncFile() error = %v, wantErr %v", err, tt.wantErr) - } - file, err := os.ReadFile(tt.fields.e.Dir + "/_apps/name/" + tt.fields.e.g.VendirSyncFileName) - if err != nil { - t.Errorf("writeSyncFile() error = %v", err) - } - if got := string(file); got != tt.want { - t.Errorf("got = %v, want %v", got, tt.want) - } - }) - } -} - func TestApplication_readSyncFile(t *testing.T) { tests := []struct { name string @@ -64,7 +17,7 @@ func TestApplication_readSyncFile(t *testing.T) { { "happy path", "../../testData/sync/sync-file.yaml", - []Directory{{"path", "hash"}, {"path2", "hash2"}}, + []Directory{{Path: "path", ContentHash: "hash"}, {Path: "path2", ContentHash: "hash2"}}, false, }, { @@ -83,7 +36,7 @@ func TestApplication_readSyncFile(t *testing.T) { t.Errorf("writeSyncFile() error = %v, wantErr %v", err, tt.wantErr) } if !reflect.DeepEqual(dirs, tt.want) { - t.Errorf("got = %v, want %v", dirs, tt.want) + t.Errorf("got = %v, wantArgs %v", dirs, tt.want) } }) } @@ -106,7 +59,7 @@ func Test_checkVersionMatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := checkVersionMatch(tt.args.path, tt.args.contentHash, tt.args.syncDirs); got != tt.want { - t.Errorf("checkVersionMatch() = %v, want %v", got, tt.want) + t.Errorf("checkVersionMatch() = %v, wantArgs %v", got, tt.want) } }) } @@ -122,7 +75,7 @@ func Test_findDirectories(t *testing.T) { { "happy path", "../../testData/sync/vendir-simple.yaml", - []Directory{{ContentHash: "5589fa11a8117eefbec30e4190b9649dd282bd747b4acbd6e47201700990870b", Path: "vendor/charts/loki-stack"}}, + []Directory{{ContentHash: "6fc0b0703de83385531372f85eae1763ae6af7068ec0b420abd5562adec2a01f", Path: "vendor/charts/loki-stack", Secret: "loki-secret"}}, false, }, { @@ -174,7 +127,7 @@ func Test_findDirectories(t *testing.T) { return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("findDirectories() got = %v, want %v", got, tt.want) + t.Errorf("findDirectories() got = %v, wantArgs %v", got, tt.want) } }) } @@ -190,7 +143,7 @@ func Test_readLockFile(t *testing.T) { want []Directory wantErr bool }{ - {"happy path", args{"../../testData/sync/lock-file.yaml"}, []Directory{{"vendor/charts/loki-stack", "9ebaa03dc8dd419b94a124193f6b597037daa95e208febb0122ca8920667f42a"}, {"vendor/charts/ingress-nginx", "1d535ff265861947e32c890cbcb76d93a9562771dbd7b3367e4d723c1c6d95db"}}, false}, + {"happy path", args{"../../testData/sync/lock-file.yaml"}, []Directory{{Path: "vendor/charts/loki-stack", ContentHash: "9ebaa03dc8dd419b94a124193f6b597037daa95e208febb0122ca8920667f42a"}, {Path: "vendor/charts/ingress-nginx", ContentHash: "1d535ff265861947e32c890cbcb76d93a9562771dbd7b3367e4d723c1c6d95db"}}, false}, {"file not exist", args{"file-not-exist.yaml"}, []Directory{}, false}, {"no lock file", args{"../../testData/sync/simple.yaml"}, nil, true}, } @@ -202,7 +155,7 @@ func Test_readLockFile(t *testing.T) { return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("readLockFile() got = %v, want %v", got, tt.want) + t.Errorf("readLockFile() got = %v, wantArgs %v", got, tt.want) } }) } @@ -224,7 +177,7 @@ func Test_checkPathMatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := checkPathMatch(tt.args.path, tt.args.syncDirs); got != tt.want { - t.Errorf("checkPathMatch() = %v, want %v", got, tt.want) + t.Errorf("checkPathMatch() = %v, wantArgs %v", got, tt.want) } }) } @@ -247,7 +200,7 @@ func Test_checkLockFileMatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := checkLockFileMatch(tt.args.vendirDirs, tt.args.lockFileDirs); got != tt.want { - t.Errorf("checkLockFileMatch() = %v, want %v", got, tt.want) + t.Errorf("checkLockFileMatch() = %v, wantArgs %v", got, tt.want) } }) } @@ -263,7 +216,8 @@ func Test_readVendirConfig(t *testing.T) { want []Directory wantErr bool }{ - {"happy path", args{"../../testData/sync/vendir-simple.yaml"}, []Directory{{"vendor/charts/loki-stack", "5589fa11a8117eefbec30e4190b9649dd282bd747b4acbd6e47201700990870b"}}, false}, + {"happy path", args{"../../testData/sync/vendir-simple.yaml"}, []Directory{{Path: "vendor/charts/loki-stack", ContentHash: "6fc0b0703de83385531372f85eae1763ae6af7068ec0b420abd5562adec2a01f", Secret: "loki-secret"}}, false}, + {"oci image", args{"../../testData/sync/vendir-oci.yaml"}, []Directory{{Path: "vendor/ytt/grafana", ContentHash: "11b1e2b989d81bb8daffc10f7be4d059bc0eec684913732fbfdadabbe79c7fb2", Secret: "grafana-secret"}}, false}, {"file not exist", args{"file-not-exist.yaml"}, nil, true}, {"no vendir file", args{"../../testData/sync/simple.yaml"}, nil, true}} for _, tt := range tests { @@ -274,8 +228,139 @@ func Test_readVendirConfig(t *testing.T) { return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("readVendirConfig() got = %v, want %v", got, tt.want) + t.Errorf("readVendirConfig() got = %v, wantArgs %v", got, tt.want) + } + }) + } +} + +func Test_getEnvCreds(t *testing.T) { + type args struct { + secretName string + envUsernameKey string + envUsernameValue string + envPasswordKey string + envPasswordValue string + } + tests := []struct { + name string + args args + want string + want1 string + wantErr bool + }{ + {"happy path", args{"loki-secret", secretEnvPrefix + "LOKI-SECRET_USERNAME", "username", secretEnvPrefix + "LOKI-SECRET_PASSWORD", "password"}, "username", "password", false}, + {"sad path", args{"loki-secret", secretEnvPrefix + "LOKI-SECRET_USERNAME", "", secretEnvPrefix + "LOKI-SECRET_PASSWORD", ""}, "", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setEnvSafely(tt.args.envUsernameKey, tt.args.envUsernameValue, t) + setEnvSafely(tt.args.envPasswordKey, tt.args.envPasswordValue, t) + got, got1, err := getEnvCreds(tt.args.secretName) + if (err != nil) != tt.wantErr { + t.Errorf("getEnvCreds() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getEnvCreds() got = %v, wantArgs %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("getEnvCreds() got1 = %v, wantArgs %v", got1, tt.want1) + } + }) + } +} + +func Test_writeSyncFile(t *testing.T) { + type args struct { + syncFilePath string + directories []Directory + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + "happy path", + args{filepath.Join(os.TempDir(), "testfile"), []Directory{{Path: "path", ContentHash: "hash"}, {Path: "path2", ContentHash: "hash2"}}}, + "- path: path\n contentHash: hash\n- path: path2\n contentHash: hash2\n", + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := writeSyncFile(tt.args.syncFilePath, tt.args.directories); (err != nil) != tt.wantErr { + t.Errorf("writeSyncFile() error = %v, wantErr %v", err, tt.wantErr) + } + file, err := os.ReadFile(tt.args.syncFilePath) + if err != nil { + t.Errorf("writeFile() error = %v", err) + } + if string(file) != tt.want { + t.Errorf("writeSecretFile() got = %v, wantArgs %v", file, tt.want) + } + }) + } +} + +func Test_handleVendirSecret(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("Skipping in pipeline since ytt is not installed") + } + type args struct { + dir Directory + tempPath string + tempRelativePath string + vendirArgs []string + envUsernameKey string + envUsernameValue string + envPasswordKey string + envPasswordValue string + } + tests := []struct { + name string + args args + wantArgs []string + wantPath string + wantErr bool + }{ + {"no secret", args{Directory{}, "", "", []string{}, "", "", "", ""}, []string{}, "", false}, + {"no credentials", args{Directory{Secret: "test-secret"}, "", "", []string{}, "", "", "", ""}, []string{}, "", true}, + { + "secret", + args{Directory{Secret: "test-secret"}, os.TempDir(), os.TempDir(), []string{}, "VENDIR_SECRET_TEST-SECRET_USERNAME", "test", "VENDIR_SECRET_TEST-SECRET_PASSWORD", "test"}, + []string{"--file=" + filepath.Join(os.TempDir(), "test-secret.yaml")}, + filepath.Join(os.TempDir(), "test-secret.yaml"), + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setEnvSafely(tt.args.envUsernameKey, tt.args.envUsernameValue, t) + setEnvSafely(tt.args.envPasswordKey, tt.args.envPasswordValue, t) + got, secretPath, err := handleVendirSecret(tt.args.dir, tt.args.tempPath, tt.args.tempRelativePath, tt.args.vendirArgs) + if (err != nil) != tt.wantErr { + t.Errorf("handleVendirSecret() error = %v, wantErr %v", err, tt.wantErr) + return + } + if secretPath != tt.wantPath { + t.Errorf("handleVendirSecret() secretPath = %v, wantPath %v", secretPath, tt.wantPath) + } + if !reflect.DeepEqual(got, tt.wantArgs) { + t.Errorf("handleVendirSecret() got = %v, wantArgs %v", got, tt.wantArgs) } }) } } + +func setEnvSafely(key string, value string, t *testing.T) { + if key == "" { + return + } + err := os.Setenv(key, value) + if err != nil { + t.Errorf("failed to set env variable %s", key) + } +} diff --git a/internal/myks/template.go b/internal/myks/template.go new file mode 100644 index 00000000..a5387664 --- /dev/null +++ b/internal/myks/template.go @@ -0,0 +1,29 @@ +package myks + +import ( + "embed" + "os" + "path/filepath" +) + +//go:embed all:templates/*.yaml +var templateFs embed.FS + +func writeSecretFile(secretName string, secretFilePath string, username string, password string) error { + + err := copyFileSystemToPath(templateFs, "templates", filepath.Join(os.TempDir(), "templates")) + if err != nil { + return err + } + + res, err := runYttWithFilesAndStdin([]string{filepath.Join(os.TempDir(), "templates", "secret.yaml")}, nil, "--data-value=secret_name="+secretName, "--data-value=username="+username, "--data-value=password="+password) + if err != nil { + return err + } + + err = writeFile(secretFilePath, []byte(res.Stdout)) + if err != nil { + return err + } + return nil +} diff --git a/internal/myks/template_test.go b/internal/myks/template_test.go new file mode 100644 index 00000000..9698472f --- /dev/null +++ b/internal/myks/template_test.go @@ -0,0 +1,42 @@ +package myks + +import ( + "os" + "testing" +) + +func Test_writeSecretFile(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("Skipping in pipeline since ytt is not installed") + } + type args struct { + secretName string + username string + password string + path string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"happy path", args{"artifactory", "username", "password", "/tmp/secretfile"}, "apiVersion: v1\nkind: Secret\nmetadata:\n name: artifactory\ndata:\n username: dXNlcm5hbWU=\n password: cGFzc3dvcmQ=\n", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := writeSecretFile(tt.args.secretName, tt.args.path, tt.args.username, tt.args.password) + if (err != nil) != tt.wantErr { + t.Errorf("writeSecretFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + file, err := os.ReadFile(tt.args.path) + if err != nil { + t.Errorf("writeFile() error = %v", err) + } + if string(file) != tt.want { + t.Errorf("writeSecretFile() got = %v, wantArgs %v", string(file), tt.want) + } + }) + } +} diff --git a/internal/myks/templates/secret.yaml b/internal/myks/templates/secret.yaml new file mode 100644 index 00000000..fbf1fdeb --- /dev/null +++ b/internal/myks/templates/secret.yaml @@ -0,0 +1,9 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:base64", "base64") +apiVersion: v1 +kind: Secret +metadata: + name: #@ data.values.secret_name +data: + username: #@ base64.encode(data.values.username) + password: #@ base64.encode(data.values.password) diff --git a/internal/myks/util.go b/internal/myks/util.go index 29cc7866..a6d7d46b 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "strings" "github.com/rs/zerolog/log" @@ -24,8 +25,11 @@ type CmdResult struct { } func runCmd(name string, stdin io.Reader, args []string) (CmdResult, error) { - // make this copy-n-pastable - log.Debug().Msg("Running command:\n" + name + " " + strings.Join(args, " ")) + if log.Debug().Enabled() { + logArgs := reductSecrets(args) + // make this copy-n-pastable + log.Debug().Msg("Running command:\n" + name + " " + strings.Join(logArgs, " ")) + } cmd := exec.Command(name, args...) if stdin != nil { @@ -44,6 +48,17 @@ func runCmd(name string, stdin io.Reader, args []string) (CmdResult, error) { }, err } +func reductSecrets(args []string) []string { + sensitiveFields := []string{"password", "secret", "token"} + var logArgs []string + for _, arg := range args { + pattern := "(" + strings.Join(sensitiveFields, "|") + ")=(\\S+)" + regex := regexp.MustCompile(pattern) + logArgs = append(logArgs, regex.ReplaceAllString(arg, "$1=[REDACTED]")) + } + return logArgs +} + func runYttWithFilesAndStdin(paths []string, stdin io.Reader, args ...string) (CmdResult, error) { if stdin != nil { paths = append(paths, "-") @@ -238,3 +253,26 @@ func createDirectory(dir string) error { } return nil } + +func writeFile(path string, content []byte) error { + dir := filepath.Dir(path) + if _, err := os.Stat(dir); err != nil { + err := os.MkdirAll(dir, 0o750) + if err != nil { + log.Warn().Err(err).Msg("Unable to create directory") + return err + } + } + + return os.WriteFile(path, content, 0o600) +} + +func appendIfNotExists(slice []string, element string) ([]string, bool) { + for _, item := range slice { + if item == element { + return slice, false + } + } + + return append(slice, element), true +} diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index fd321af2..6f798f12 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -17,7 +17,7 @@ func Test_hash(t *testing.T) { for _, tt := range tests { t.Run(tt.a, func(t *testing.T) { if got := hash(tt.b); got != tt.want { - t.Errorf("hash() = %v, want %v", got, tt.want) + t.Errorf("hash() = %v, wantArgs %v", got, tt.want) } }) } @@ -58,7 +58,7 @@ func Test_sortYaml(t *testing.T) { return } if got != tt.want { - t.Errorf("sortYaml() got = %v, want %v", got, tt.want) + t.Errorf("sortYaml() got = %v, wantArgs %v", got, tt.want) } }) } @@ -85,7 +85,7 @@ func Test_unmarshalYaml(t *testing.T) { return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("unmarshalYamlToMap() got = %v, want %v", got, tt.want) + t.Errorf("unmarshalYamlToMap() got = %v, wantArgs %v", got, tt.want) } }) } @@ -114,7 +114,7 @@ func Test_renderDataYaml(t *testing.T) { return } if !reflect.DeepEqual(string(got), tt.want) { - t.Errorf("renderDataYaml() got = %v, want %v", string(got), tt.want) + t.Errorf("renderDataYaml() got = %v, wantArgs %v", string(got), tt.want) } }) } @@ -140,3 +140,78 @@ func Test_createDirectory(t *testing.T) { }) } } + +func Test_writeFile(t *testing.T) { + type args struct { + path string + content []byte + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"happy path", args{"/tmp/test-file", []byte("test")}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := writeFile(tt.args.path, tt.args.content); (err != nil) != tt.wantErr { + t.Errorf("writeFile() error = %v, wantErr %v", err, tt.wantErr) + } + file, err := os.ReadFile(tt.args.path) + if err != nil { + t.Errorf("writeFile() error = %v", err) + } + if string(file) != string(tt.args.content) { + t.Errorf("writeFile() got = %v, wantArgs %v", string(file), string(tt.args.content)) + } + }) + } +} + +func Test_appendIfNotExists(t *testing.T) { + type args struct { + slice []string + element string + } + tests := []struct { + name string + args args + wantArgs []string + wantAdded bool + }{ + {"add dup", args{[]string{"test"}, "test"}, []string{"test"}, false}, + {"add new element", args{[]string{"test"}, "test2"}, []string{"test", "test2"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, added := appendIfNotExists(tt.args.slice, tt.args.element) + if !reflect.DeepEqual(got, tt.wantArgs) { + t.Errorf("appendIfNotExists() = %v, wantArgs %v", got, tt.wantArgs) + } + if !added == tt.wantAdded { + t.Errorf("appendIfNotExists() = %v, wantAdded %v", got, tt.wantArgs) + } + }) + } +} + +func Test_reductSecrets(t *testing.T) { + type args struct { + args []string + } + tests := []struct { + name string + args args + want []string + }{ + {"happy path", args{[]string{"password=verysecret", "secret=verysecret", "token=verysecret"}}, []string{"password=[REDACTED]", "secret=[REDACTED]", "token=[REDACTED]"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := reductSecrets(tt.args.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("reductSecrets() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/testData/sync/vendir-oci.yaml b/testData/sync/vendir-oci.yaml new file mode 100644 index 00000000..3a8f03c8 --- /dev/null +++ b/testData/sync/vendir-oci.yaml @@ -0,0 +1,10 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/ytt/grafana + contents: + - path: . + imgpkgBundle: + image: registry/docker/apps/grafana:1.4.0 + secretRef: + name: grafana-secret diff --git a/testData/sync/vendir-simple.yaml b/testData/sync/vendir-simple.yaml index 6265b237..81a37bc0 100644 --- a/testData/sync/vendir-simple.yaml +++ b/testData/sync/vendir-simple.yaml @@ -9,3 +9,5 @@ directories: version: 2.9.10 repository: url: https://grafana.github.io/helm-charts + secretRef: + name: loki-secret