Skip to content

Commit

Permalink
feat: Add vendir authentication via environment
Browse files Browse the repository at this point in the history
  • Loading branch information
Fritz Durchardt committed Jul 21, 2023
1 parent 2dbbd55 commit 118ae37
Show file tree
Hide file tree
Showing 11 changed files with 477 additions and 115 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
bin/*
dist
.idea
.run
38 changes: 12 additions & 26 deletions internal/myks/application.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type HelmConfig struct {
IncludeCRDs bool `yaml:"includeCRDs"`
}

var ErrNoVendirConfig = errors.New("No vendir config found")
var ErrNoVendirConfig = errors.New("no vendir config found")

func NewApplication(e *Environment, name string, prototypeName string) (*Application, error) {
if prototypeName == "" {
Expand All @@ -41,7 +41,7 @@ func NewApplication(e *Environment, name string, prototypeName string) (*Applica
prototype := filepath.Join(e.g.PrototypesDir, prototypeName)

if _, err := os.Stat(prototype); err != nil {
return nil, errors.New("Application prototype does not exist")
return nil, errors.New("application prototype does not exist")
}

app := &Application{
Expand Down Expand Up @@ -164,7 +164,7 @@ func (a *Application) prepareSync() error {
// 1. If exists, use the `apps/<prototype>/vendir` directory.
// 2. If exists, for every level of environments use `<env>/_apps/<app>/vendir` directory.

yttFiles := []string{}
var yttFiles []string

protoVendirDir := filepath.Join(a.Prototype, "vendir")
if _, err := os.Stat(protoVendirDir); err == nil {
Expand Down Expand Up @@ -222,26 +222,12 @@ func (a *Application) expandTempPath(path string) string {
return a.expandServicePath(filepath.Join(a.e.g.TempDirName, path))
}

// TODO: for content, use []byte instead of string
func (a *Application) writeFile(path string, content string) error {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); err != nil {
err := os.MkdirAll(dir, 0o750)
if err != nil {
log.Warn().Err(err).Msg("Unable to create directory")
return err
}
}

return os.WriteFile(path, []byte(content), 0o600)
}

func (a *Application) writeServiceFile(name string, content string) error {
return a.writeFile(a.expandServicePath(name), content)
return writeFile(a.expandServicePath(name), []byte(content))
}

func (a *Application) writeTempFile(name string, content string) error {
return a.writeFile(a.expandTempPath(name), content)
return writeFile(a.expandTempPath(name), []byte(content))
}

func (a *Application) collectDataFiles() {
Expand Down Expand Up @@ -270,7 +256,7 @@ func (a *Application) runHelm() (string, error) {
return "", err
}

commonHelmArgs := []string{}
var commonHelmArgs []string

// FIXME: move Namespace to a per-chart config
if helmConfig.Namespace == "" {
Expand All @@ -287,7 +273,7 @@ func (a *Application) runHelm() (string, error) {
commonHelmArgs = append(commonHelmArgs, "--include-crds")
}

outputs := []string{}
var outputs []string

for _, chartDir := range chartDirs {
chartName := filepath.Base(chartDir)
Expand Down Expand Up @@ -349,7 +335,7 @@ func (a *Application) getHelmChartDirs() []string {
return []string{}
}

chartDirs := []string{}
var chartDirs []string
err = filepath.Walk(chartsDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Warn().Err(err).Str("path", path).Msg("Unable to walk helm charts directory")
Expand Down Expand Up @@ -377,7 +363,7 @@ func (a *Application) getHelmChartDirs() []string {
func (a *Application) prepareHelm(chartName string) error {
helmValuesFileName := a.getHelmValuesFileName(chartName)

helmYttFiles := []string{}
var helmYttFiles []string

prototypeHelmValues := filepath.Join(a.Prototype, helmValuesFileName)
if _, err := os.Stat(prototypeHelmValues); err == nil {
Expand Down Expand Up @@ -448,7 +434,7 @@ func (a *Application) getHelmValuesFileName(chartName string) string {
}

func (a *Application) runYtt(previousStepFile string) (string, error) {
yttFiles := []string{}
var yttFiles []string

yttFiles = append(yttFiles, a.yttDataFiles...)

Expand Down Expand Up @@ -490,7 +476,7 @@ func (a *Application) runYtt(previousStepFile string) (string, error) {
}

func (a *Application) runGlobalYtt(previousStepFile string) (string, error) {
yttFiles := []string{}
var yttFiles []string

yttFiles = append(yttFiles, a.yttDataFiles...)

Expand Down Expand Up @@ -574,7 +560,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error {
if _, err := os.Stat(filePath); err == nil {
log.Warn().Str("app", a.Name).Str("file", filePath).Msg("File already exists, check duplicated resources")
}
err = a.writeFile(filePath, data.String())
err = writeFile(filePath, data.Bytes())
if err != nil {
log.Warn().Err(err).Str("app", a.Name).Str("file", filePath).Msg("Unable to write file")
return err
Expand Down
134 changes: 110 additions & 24 deletions internal/myks/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,24 @@ import (
"gopkg.in/yaml.v3"
"os"
"path/filepath"
"strings"
)

const secretEnvPrefix = "VENDIR_SECRET_"

type Directory struct {
Path string
ContentHash string `yaml:"contentHash"`
Secret string `yaml:"-"`
}

func (a *Application) doSync() error {
// TODO: implement secrets-from-env extraction

// Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`)
vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName)
vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName)
vendirConfigFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirConfigFileName)
vendirLockFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirLockFileName)
vendirSyncPath := a.expandTempPath(a.e.g.VendirSyncFileName)
vendirSyncFilePath := a.expandTempPath(a.e.g.VendirSyncFileName)
vendorDir := a.expandPath(a.e.g.VendorDirName)

vendirDirs, err := readVendirConfig(vendirConfigFilePath)
Expand All @@ -30,9 +32,9 @@ func (a *Application) doSync() error {
return err
}

syncFileDirs, err := readSyncFile(vendirSyncPath)
syncFileDirs, err := readSyncFile(vendirSyncFilePath)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncPath)
log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncFilePath)
return err
}
if len(syncFileDirs) == 0 {
Expand All @@ -51,6 +53,7 @@ func (a *Application) doSync() error {
return err
}

var secretFilePaths []string
//TODO sync retry
// only sync vendir with directory flag, if the lock file matches the vendir config file and caching is enabled
if a.cached && checkLockFileMatch(vendirDirs, lockFileDirs) {
Expand All @@ -60,58 +63,117 @@ func (a *Application) doSync() error {
continue
}
log.Info().Str("app", a.Name).Msg("Syncing vendir for: " + dir.Path)
res, err := runCmd("vendir", nil, []string{
args := []string{
"sync",
"--chdir=" + vendorDir,
"--directory=" + dir.Path,
"--file=" + vendirConfigFileRelativePath,
"--lock-file=" + vendirLockFileRelativePath,
})
}
args, secretFilePath, err := handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path)
return err
}
if secretFilePath != "" {
secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath)
}

res, err := runCmd("vendir", nil, args)
if err != nil {
log.Warn().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir")
return err
}
}

} else {
log.Info().Str("app", a.Name).Msg("Syncing vendir completely for: " + vendirConfigFilePath)
res, err := runCmd("vendir", nil, []string{
args := []string{
"sync",
"--chdir=" + vendorDir,
"--file=" + vendirConfigFileRelativePath,
"--lock-file=" + vendirLockFileRelativePath,
})
}
for _, dir := range vendirDirs {
var secretFilePath string
args, secretFilePath, err = handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path)
return err
}
if secretFilePath != "" {
secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath)
}

}
res, err := runCmd("vendir", nil, args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir")
return err
}
}

err = a.writeSyncFile(vendirDirs)
// make sure secrets do not linger on disk
for _, secretFilePath := range secretFilePaths {
defer func(name string) {
log.Debug().Str("app", a.Name).Msg("delete secret file: " + name)
err := os.Remove(name)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("unable to delete secret file: " + name)
}
}(secretFilePath)
}

err = writeSyncFile(a.expandTempPath(a.e.g.VendirSyncFileName), vendirDirs)
if err != nil {
log.Error().Str("app", a.Name).Err(err).Msg("Unable to write sync file")
return err
}

log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncPath)
log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncFilePath)
log.Info().Str("app", a.Name).Msg("Vendir sync completed!")

return nil
}

func (a *Application) writeSyncFile(directories []Directory) error {
func writeSyncFile(syncFilePath string, directories []Directory) error {

bytes, err := yaml.Marshal(directories)
if err != nil {
return err
}
err = a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes))
err = writeFile(syncFilePath, bytes)
if err != nil {
return err
}

return nil
}

func handleVendirSecret(dir Directory, tempPath string, tempRelativePath string, vendirArgs []string) ([]string, string, error) {
if dir.Secret != "" {
username, password, err := getEnvCreds(dir.Secret)
if err != nil {
return vendirArgs, "", err
}
secretFileName := dir.Secret + ".yaml"
secretFilePath := filepath.Join(tempPath, secretFileName)
err = writeSecretFile(dir.Secret, secretFilePath, username, password)
if err != nil {
return vendirArgs, "", err
}
secretRelativePath := filepath.Join(tempRelativePath, secretFileName)
var addedSecret bool
vendirArgs, addedSecret = appendIfNotExists(vendirArgs, "--file="+secretRelativePath)
if addedSecret {
return vendirArgs, secretFilePath, nil
} else {
return vendirArgs, "", nil
}
}
return vendirArgs, "", nil
}

func readVendirConfig(vendirConfigFilePath string) ([]Directory, error) {
config, err := unmarshalYamlToMap(vendirConfigFilePath)
if err != nil {
Expand Down Expand Up @@ -170,7 +232,8 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) {
if _, ok := config["directories"]; !ok {
return nil, errors.New("no directories found in vendir config")
}
var directories = make(map[string]string)
var syncDirs []Directory

for _, dir := range config["directories"].([]interface{}) {
dirMap := dir.(map[string]interface{})
path := dirMap["path"].(string)
Expand All @@ -183,24 +246,38 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) {
if subPath != "." {
path += "/" + subPath
}

secret := ""
if contents["imgpkgBundle"] != nil {
imgpkgBundle := contents["imgpkgBundle"].(map[string]interface{})
if imgpkgBundle["secretRef"] != nil {
secretRef := imgpkgBundle["secretRef"].(map[string]interface{})
secret = secretRef["name"].(string)
}
}

if contents["helmChart"] != nil {
helmChart := contents["helmChart"].(map[string]interface{})
if helmChart["repository"] != nil {
repository := helmChart["repository"].(map[string]interface{})
if repository["secretRef"] != nil {
secretRef := repository["secretRef"].(map[string]interface{})
secret = secretRef["name"].(string)
}
}
}

sortedYaml, err := sortYaml(contents)
if err != nil {
return nil, err
}
directories[path] = sortedYaml
}
return convertDirectoryMapToHashedStruct(directories), nil
}

func convertDirectoryMapToHashedStruct(directories map[string]string) []Directory {
var syncDirs []Directory
for path, contents := range directories {
syncDirs = append(syncDirs, Directory{
Path: path,
ContentHash: hash(contents),
ContentHash: hash(sortedYaml),
Secret: secret,
})
}
return syncDirs
return syncDirs, nil
}

func checkVersionMatch(path string, contentHash string, syncDirs []Directory) bool {
Expand Down Expand Up @@ -231,3 +308,12 @@ func checkLockFileMatch(vendirDirs []Directory, lockFileDirs []Directory) bool {
}
return true
}

func getEnvCreds(secretName string) (string, string, error) {
username := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_USERNAME")
password := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_PASSWORD")
if username == "" || password == "" {
return "", "", errors.New("no credentials found in environment for secret: " + secretName)
}
return username, password, nil
}
Loading

0 comments on commit 118ae37

Please sign in to comment.