Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add vendir authentication via the enviroment #35

Merged
merged 1 commit into from
Jul 26, 2023
Merged

feat: Add vendir authentication via the enviroment #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
bin/*
dist
.idea
.run
38 changes: 12 additions & 26 deletions internal/myks/application.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type HelmConfig struct {
IncludeCRDs bool `yaml:"includeCRDs"`
}

var ErrNoVendirConfig = errors.New("No vendir config found")
var ErrNoVendirConfig = errors.New("no vendir config found")

func NewApplication(e *Environment, name string, prototypeName string) (*Application, error) {
if prototypeName == "" {
Expand All @@ -41,7 +41,7 @@ func NewApplication(e *Environment, name string, prototypeName string) (*Applica
prototype := filepath.Join(e.g.PrototypesDir, prototypeName)

if _, err := os.Stat(prototype); err != nil {
return nil, errors.New("Application prototype does not exist")
return nil, errors.New("application prototype does not exist")
}

app := &Application{
Expand Down Expand Up @@ -164,7 +164,7 @@ func (a *Application) prepareSync() error {
// 1. If exists, use the `apps/<prototype>/vendir` directory.
// 2. If exists, for every level of environments use `<env>/_apps/<app>/vendir` directory.

yttFiles := []string{}
var yttFiles []string

protoVendirDir := filepath.Join(a.Prototype, "vendir")
if _, err := os.Stat(protoVendirDir); err == nil {
Expand Down Expand Up @@ -222,26 +222,12 @@ func (a *Application) expandTempPath(path string) string {
return a.expandServicePath(filepath.Join(a.e.g.TempDirName, path))
}

// TODO: for content, use []byte instead of string
func (a *Application) writeFile(path string, content string) error {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); err != nil {
err := os.MkdirAll(dir, 0o750)
if err != nil {
log.Warn().Err(err).Msg("Unable to create directory")
return err
}
}

return os.WriteFile(path, []byte(content), 0o600)
}

func (a *Application) writeServiceFile(name string, content string) error {
return a.writeFile(a.expandServicePath(name), content)
return writeFile(a.expandServicePath(name), []byte(content))
}

func (a *Application) writeTempFile(name string, content string) error {
return a.writeFile(a.expandTempPath(name), content)
return writeFile(a.expandTempPath(name), []byte(content))
}

func (a *Application) collectDataFiles() {
Expand Down Expand Up @@ -270,7 +256,7 @@ func (a *Application) runHelm() (string, error) {
return "", err
}

commonHelmArgs := []string{}
var commonHelmArgs []string

// FIXME: move Namespace to a per-chart config
if helmConfig.Namespace == "" {
Expand All @@ -287,7 +273,7 @@ func (a *Application) runHelm() (string, error) {
commonHelmArgs = append(commonHelmArgs, "--include-crds")
}

outputs := []string{}
var outputs []string

for _, chartDir := range chartDirs {
chartName := filepath.Base(chartDir)
Expand Down Expand Up @@ -349,7 +335,7 @@ func (a *Application) getHelmChartDirs() []string {
return []string{}
}

chartDirs := []string{}
var chartDirs []string
err = filepath.Walk(chartsDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Warn().Err(err).Str("path", path).Msg("Unable to walk helm charts directory")
Expand Down Expand Up @@ -377,7 +363,7 @@ func (a *Application) getHelmChartDirs() []string {
func (a *Application) prepareHelm(chartName string) error {
helmValuesFileName := a.getHelmValuesFileName(chartName)

helmYttFiles := []string{}
var helmYttFiles []string

prototypeHelmValues := filepath.Join(a.Prototype, helmValuesFileName)
if _, err := os.Stat(prototypeHelmValues); err == nil {
Expand Down Expand Up @@ -448,7 +434,7 @@ func (a *Application) getHelmValuesFileName(chartName string) string {
}

func (a *Application) runYtt(previousStepFile string) (string, error) {
yttFiles := []string{}
var yttFiles []string

yttFiles = append(yttFiles, a.yttDataFiles...)

Expand Down Expand Up @@ -490,7 +476,7 @@ func (a *Application) runYtt(previousStepFile string) (string, error) {
}

func (a *Application) runGlobalYtt(previousStepFile string) (string, error) {
yttFiles := []string{}
var yttFiles []string

yttFiles = append(yttFiles, a.yttDataFiles...)

Expand Down Expand Up @@ -574,7 +560,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error {
if _, err := os.Stat(filePath); err == nil {
log.Warn().Str("app", a.Name).Str("file", filePath).Msg("File already exists, check duplicated resources")
}
err = a.writeFile(filePath, data.String())
err = writeFile(filePath, data.Bytes())
if err != nil {
log.Warn().Err(err).Str("app", a.Name).Str("file", filePath).Msg("Unable to write file")
return err
Expand Down
134 changes: 110 additions & 24 deletions internal/myks/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,24 @@ import (
"gopkg.in/yaml.v3"
"os"
"path/filepath"
"strings"
)

const secretEnvPrefix = "VENDIR_SECRET_"

type Directory struct {
Path string
ContentHash string `yaml:"contentHash"`
Secret string `yaml:"-"`
}

func (a *Application) doSync() error {
// TODO: implement secrets-from-env extraction

// Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`)
vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName)
vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName)
vendirConfigFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirConfigFileName)
vendirLockFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirLockFileName)
vendirSyncPath := a.expandTempPath(a.e.g.VendirSyncFileName)
vendirSyncFilePath := a.expandTempPath(a.e.g.VendirSyncFileName)
vendorDir := a.expandPath(a.e.g.VendorDirName)

vendirDirs, err := readVendirConfig(vendirConfigFilePath)
Expand All @@ -30,9 +32,9 @@ func (a *Application) doSync() error {
return err
}

syncFileDirs, err := readSyncFile(vendirSyncPath)
syncFileDirs, err := readSyncFile(vendirSyncFilePath)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncPath)
log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncFilePath)
return err
}
if len(syncFileDirs) == 0 {
Expand All @@ -51,6 +53,7 @@ func (a *Application) doSync() error {
return err
}

var secretFilePaths []string
//TODO sync retry
// only sync vendir with directory flag, if the lock file matches the vendir config file and caching is enabled
if a.cached && checkLockFileMatch(vendirDirs, lockFileDirs) {
Expand All @@ -60,58 +63,117 @@ func (a *Application) doSync() error {
continue
}
log.Info().Str("app", a.Name).Msg("Syncing vendir for: " + dir.Path)
res, err := runCmd("vendir", nil, []string{
args := []string{
"sync",
"--chdir=" + vendorDir,
"--directory=" + dir.Path,
"--file=" + vendirConfigFileRelativePath,
"--lock-file=" + vendirLockFileRelativePath,
})
}
args, secretFilePath, err := handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path)
return err
}
if secretFilePath != "" {
secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath)
}

res, err := runCmd("vendir", nil, args)
if err != nil {
log.Warn().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir")
return err
}
}

} else {
log.Info().Str("app", a.Name).Msg("Syncing vendir completely for: " + vendirConfigFilePath)
res, err := runCmd("vendir", nil, []string{
args := []string{
"sync",
"--chdir=" + vendorDir,
"--file=" + vendirConfigFileRelativePath,
"--lock-file=" + vendirLockFileRelativePath,
})
}
for _, dir := range vendirDirs {
var secretFilePath string
args, secretFilePath, err = handleVendirSecret(dir, a.expandTempPath(""), filepath.Join("..", a.e.g.ServiceDirName, a.e.g.TempDirName), args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("Unable to create secret for: " + dir.Path)
return err
}
if secretFilePath != "" {
secretFilePaths, _ = appendIfNotExists(secretFilePaths, secretFilePath)
}

}
res, err := runCmd("vendir", nil, args)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir")
return err
}
}

err = a.writeSyncFile(vendirDirs)
// make sure secrets do not linger on disk
for _, secretFilePath := range secretFilePaths {
defer func(name string) {
log.Debug().Str("app", a.Name).Msg("delete secret file: " + name)
err := os.Remove(name)
if err != nil {
log.Error().Err(err).Str("app", a.Name).Msg("unable to delete secret file: " + name)
}
}(secretFilePath)
}

err = writeSyncFile(a.expandTempPath(a.e.g.VendirSyncFileName), vendirDirs)
if err != nil {
log.Error().Str("app", a.Name).Err(err).Msg("Unable to write sync file")
return err
}

log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncPath)
log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncFilePath)
log.Info().Str("app", a.Name).Msg("Vendir sync completed!")

return nil
}

func (a *Application) writeSyncFile(directories []Directory) error {
func writeSyncFile(syncFilePath string, directories []Directory) error {

bytes, err := yaml.Marshal(directories)
if err != nil {
return err
}
err = a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes))
err = writeFile(syncFilePath, bytes)
if err != nil {
return err
}

return nil
}

func handleVendirSecret(dir Directory, tempPath string, tempRelativePath string, vendirArgs []string) ([]string, string, error) {
if dir.Secret != "" {
username, password, err := getEnvCreds(dir.Secret)
if err != nil {
return vendirArgs, "", err
}
secretFileName := dir.Secret + ".yaml"
secretFilePath := filepath.Join(tempPath, secretFileName)
err = writeSecretFile(dir.Secret, secretFilePath, username, password)
if err != nil {
return vendirArgs, "", err
}
secretRelativePath := filepath.Join(tempRelativePath, secretFileName)
var addedSecret bool
vendirArgs, addedSecret = appendIfNotExists(vendirArgs, "--file="+secretRelativePath)
if addedSecret {
return vendirArgs, secretFilePath, nil
} else {
return vendirArgs, "", nil
}
}
return vendirArgs, "", nil
}

func readVendirConfig(vendirConfigFilePath string) ([]Directory, error) {
config, err := unmarshalYamlToMap(vendirConfigFilePath)
if err != nil {
Expand Down Expand Up @@ -170,7 +232,8 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) {
if _, ok := config["directories"]; !ok {
return nil, errors.New("no directories found in vendir config")
}
var directories = make(map[string]string)
var syncDirs []Directory

for _, dir := range config["directories"].([]interface{}) {
dirMap := dir.(map[string]interface{})
path := dirMap["path"].(string)
Expand All @@ -183,24 +246,38 @@ func findDirectories(config map[string]interface{}) ([]Directory, error) {
if subPath != "." {
path += "/" + subPath
}

secret := ""
if contents["imgpkgBundle"] != nil {
imgpkgBundle := contents["imgpkgBundle"].(map[string]interface{})
if imgpkgBundle["secretRef"] != nil {
secretRef := imgpkgBundle["secretRef"].(map[string]interface{})
secret = secretRef["name"].(string)
}
}

if contents["helmChart"] != nil {
helmChart := contents["helmChart"].(map[string]interface{})
if helmChart["repository"] != nil {
repository := helmChart["repository"].(map[string]interface{})
if repository["secretRef"] != nil {
secretRef := repository["secretRef"].(map[string]interface{})
secret = secretRef["name"].(string)
}
}
}

sortedYaml, err := sortYaml(contents)
if err != nil {
return nil, err
}
directories[path] = sortedYaml
}
return convertDirectoryMapToHashedStruct(directories), nil
}

func convertDirectoryMapToHashedStruct(directories map[string]string) []Directory {
var syncDirs []Directory
for path, contents := range directories {
syncDirs = append(syncDirs, Directory{
Path: path,
ContentHash: hash(contents),
ContentHash: hash(sortedYaml),
Secret: secret,
})
}
return syncDirs
return syncDirs, nil
}

func checkVersionMatch(path string, contentHash string, syncDirs []Directory) bool {
Expand Down Expand Up @@ -231,3 +308,12 @@ func checkLockFileMatch(vendirDirs []Directory, lockFileDirs []Directory) bool {
}
return true
}

func getEnvCreds(secretName string) (string, string, error) {
username := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_USERNAME")
password := os.Getenv(secretEnvPrefix + strings.ToUpper(secretName) + "_PASSWORD")
if username == "" || password == "" {
return "", "", errors.New("no credentials found in environment for secret: " + secretName)
}
return username, password, nil
}
Loading