Fix unable to use SecretEncryption and Attachment features at the sam…

…e time (#4855)

* Add test to mention error with go templating

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Reimplement sourceprosser logic

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* SourceProssesor without processor should be marked as error

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Add test

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Enable perform template processing in chain

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Fix typo

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

---------

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

pkg/app/piped/deploysource/deploysource.go

@@ -171,21 +171,24 @@ func (p *provider) prepare(ctx context.Context, lw io.Writer) (*DeploySource, er
 	}
 	fmt.Fprintln(lw, "Successfully loaded the application configuration file")
 
+	var templProcessors []sourceprocesser.SourceTemplateProcessor
 	// Decrypt the sealed secrets if needed.
-	if gac.Encryption != nil && p.secretDecrypter != nil && len(gac.Encryption.DecryptionTargets) > 0 {
-		if err := sourceprocesser.DecryptSecrets(appDir, *gac.Encryption, p.secretDecrypter); err != nil {
-			fmt.Fprintf(lw, "Unable to decrypt the secrets (%v)\n", err)
-			return nil, err
-		}
-		fmt.Fprintf(lw, "Successfully decrypted secrets: %v\n", gac.Encryption.DecryptionTargets)
+	if gac.Encryption != nil && p.secretDecrypter != nil {
+		templProcessors = append(templProcessors, sourceprocesser.NewSecretDecrypterProcessor(gac.Encryption, p.secretDecrypter))
+	}
+	// Attach the data if needed.
+	if gac.Attachment != nil {
+		templProcessors = append(templProcessors, sourceprocesser.NewAttachmentProcessor(gac.Attachment))
 	}
 
-	if gac.Attachment != nil && len(gac.Attachment.Targets) > 0 {
-		if err := sourceprocesser.AttachData(appDir, *gac.Attachment); err != nil {
-			fmt.Fprintf(lw, "Unable to attach the data (%v)\n", err)
+	// Process templating source files.
+	if len(templProcessors) > 0 {
+		sp := sourceprocesser.NewSourceProcessor(appDir, templProcessors...)
+		if err := sp.Process(); err != nil {
+			fmt.Fprintf(lw, "Unable to process the source files (%v)\n", err)
 			return nil, err
 		}
-		fmt.Fprintf(lw, "Successfully attached data: %v\n", gac.Attachment.Targets)
+		fmt.Fprintln(lw, "Successfully processed the source files")
 	}
 
 	return &DeploySource{

pkg/app/piped/driftdetector/cloudrun/detector.go

@@ -267,16 +267,19 @@ func (d *detector) loadHeadServiceManifest(app *model.Application, repo git.Repo
 			appDir = filepath.Join(repoDir, app.GitPath.Path)
 		}
 
+		var templProcessors []sourceprocesser.SourceTemplateProcessor
 		// Decrypting secrets to manifests.
 		if encryptionUsed {
-			if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
-				return provider.ServiceManifest{}, fmt.Errorf("failed to decrypt secrets (%w)", err)
-			}
+			templProcessors = append(templProcessors, sourceprocesser.NewSecretDecrypterProcessor(gds.Encryption, d.secretDecrypter))
 		}
 		// Then attaching configurated files to manifests.
 		if attachmentUsed {
-			if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
-				return provider.ServiceManifest{}, fmt.Errorf("failed to attach files (%w)", err)
+			templProcessors = append(templProcessors, sourceprocesser.NewAttachmentProcessor(gds.Attachment))
+		}
+		if len(templProcessors) > 0 {
+			sp := sourceprocesser.NewSourceProcessor(appDir, templProcessors...)
+			if err := sp.Process(); err != nil {
+				return provider.ServiceManifest{}, fmt.Errorf("failed to process source files: %w", err)
 			}
 		}
 

pkg/app/piped/driftdetector/kubernetes/detector.go

@@ -265,16 +265,19 @@ func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application
 			appDir = filepath.Join(repoDir, app.GitPath.Path)
 		}
 
+		var templProcessors []sourceprocesser.SourceTemplateProcessor
 		// Decrypting secrets to manifests.
 		if encryptionUsed {
-			if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
-				return nil, fmt.Errorf("failed to decrypt secrets (%w)", err)
-			}
+			templProcessors = append(templProcessors, sourceprocesser.NewSecretDecrypterProcessor(gds.Encryption, d.secretDecrypter))
 		}
 		// Then attaching configurated files to manifests.
 		if attachmentUsed {
-			if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
-				return nil, fmt.Errorf("failed to attach files (%w)", err)
+			templProcessors = append(templProcessors, sourceprocesser.NewAttachmentProcessor(gds.Attachment))
+		}
+		if len(templProcessors) > 0 {
+			sp := sourceprocesser.NewSourceProcessor(appDir, templProcessors...)
+			if err := sp.Process(); err != nil {
+				return nil, fmt.Errorf("failed to process source files: %w", err)
 			}
 		}
 

pkg/app/piped/driftdetector/terraform/detector.go

@@ -210,16 +210,19 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 		appDir = filepath.Join(repoDir, app.GitPath.Path)
 	}
 
+	var templProcessors []sourceprocesser.SourceTemplateProcessor
 	// Decrypting secrets to manifests.
 	if encryptionUsed {
-		if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
-			return fmt.Errorf("failed to decrypt secrets (%w)", err)
-		}
+		templProcessors = append(templProcessors, sourceprocesser.NewSecretDecrypterProcessor(gds.Encryption, d.secretDecrypter))
 	}
 	// Then attaching configurated files to manifests.
 	if attachmentUsed {
-		if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
-			return fmt.Errorf("failed to attach files (%w)", err)
+		templProcessors = append(templProcessors, sourceprocesser.NewAttachmentProcessor(gds.Attachment))
+	}
+	if len(templProcessors) > 0 {
+		sp := sourceprocesser.NewSourceProcessor(appDir, templProcessors...)
+		if err := sp.Process(); err != nil {
+			return fmt.Errorf("failed to process source files: %w", err)
 		}
 	}
 

pkg/app/piped/sourceprocesser/attacher.go

@@ -25,53 +25,61 @@ import (
 	"github.com/pipe-cd/pipecd/pkg/config"
 )
 
-func AttachData(appDir string, atc config.Attachment) error {
-	if len(atc.Targets) == 0 {
-		return nil
+type attachmentProcessor struct {
+	atc *config.Attachment
+}
+
+func NewAttachmentProcessor(atc *config.Attachment) *attachmentProcessor {
+	return &attachmentProcessor{
+		atc: atc,
 	}
-	if len(atc.Sources) == 0 {
-		return fmt.Errorf("no source data to attach")
+}
+
+func (p *attachmentProcessor) BuildTemplateData(appDir string) (map[string]string, error) {
+	if len(p.atc.Sources) == 0 {
+		// Skip building no error.
+		return nil, nil
 	}
 
-	src := make(map[string]string, len(atc.Sources))
-	for k, v := range atc.Sources {
+	src := make(map[string]string, len(p.atc.Sources))
+	for k, v := range p.atc.Sources {
 		srcPath := filepath.Join(appDir, v)
 		buff, err := os.ReadFile(srcPath)
 		if err != nil {
-			return fmt.Errorf("failed to read data source to attach from file %s (%w)", v, err)
+			return nil, fmt.Errorf("failed to read data source to attach from file %s (%w)", v, err)
 		}
 		src[k] = string(buff)
 	}
-	data := map[string](map[string]string){
-		"attachment": src,
-	}
+	return src, nil
+}
+
+func (p *attachmentProcessor) TemplateKey() string {
+	return "attachment"
+}
 
-	for _, t := range atc.Targets {
+func (p *attachmentProcessor) TemplateSource(appDir string, data map[string]map[string]string) error {
+	for _, t := range p.atc.Targets {
 		targetPath := filepath.Join(appDir, t)
 		fileName := filepath.Base(targetPath)
-		tmpl := template.
-			New(fileName).
-			Funcs(sprig.TxtFuncMap()).
-			Option("missingkey=error")
+		tmpl := template.New(fileName).Funcs(sprig.TxtFuncMap()).Option("missingkey=error")
 		tmpl, err := tmpl.ParseFiles(targetPath)
 		if err != nil {
-			return fmt.Errorf("failed to parse attaching target %s (%w)", t, err)
+			return fmt.Errorf("failed to parse target file %s (%w)", t, err)
 		}
 
 		f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_TRUNC, 0644)
 		if err != nil {
-			return fmt.Errorf("failed to open attaching target %s (%w)", t, err)
+			return fmt.Errorf("failed to open target file %s (%w)", t, err)
 		}
 
 		if err := tmpl.Execute(f, data); err != nil {
 			f.Close()
-			return fmt.Errorf("failed to render attaching target %s (%w)", t, err)
+			return fmt.Errorf("failed to render target file %s (%w)", t, err)
 		}
 
 		if err := f.Close(); err != nil {
-			return fmt.Errorf("failed to close attached target %s (%w)", t, err)
+			return fmt.Errorf("failed to close target file %s (%w)", t, err)
 		}
 	}
-
 	return nil
 }

pkg/app/piped/sourceprocesser/attacher_test.go

@@ -43,39 +43,6 @@ func TestAttachData(t *testing.T) {
 		expected            map[string]string
 		expectedErrorPrefix string
 	}{
-		{
-			name: "target not found",
-			fileData: map[string]string{
-				"config.yaml": "config-data",
-			},
-			attachConfig: config.Attachment{
-				Sources: map[string]string{
-					"config": "config.yaml",
-				},
-				Targets: []string{
-					"not-found-resource.yaml",
-				},
-			},
-			expectedErrorPrefix: "failed to parse attaching target not-found-resource.yaml",
-		},
-		{
-			name: "the target is not using any attachment",
-			fileData: map[string]string{
-				"config.yaml":   "config-data",
-				"resource.yaml": "resource-data",
-			},
-			attachConfig: config.Attachment{
-				Sources: map[string]string{
-					"config": "config.yaml",
-				},
-				Targets: []string{
-					"resource.yaml",
-				},
-			},
-			expected: map[string]string{
-				"resource.yaml": "resource-data",
-			},
-		},
 		{
 			name: "single target",
 			fileData: map[string]string{
@@ -91,25 +58,7 @@ func TestAttachData(t *testing.T) {
 				},
 			},
 			expected: map[string]string{
-				"resource.yaml": "echo config-data",
-			},
-		},
-		{
-			name: "sprig function",
-			fileData: map[string]string{
-				"config.yaml":   "config-data",
-				"resource.yaml": "echo {{ .attachment.config | b64enc }}",
-			},
-			attachConfig: config.Attachment{
-				Sources: map[string]string{
-					"config": "config.yaml",
-				},
-				Targets: []string{
-					"resource.yaml",
-				},
-			},
-			expected: map[string]string{
-				"resource.yaml": "echo Y29uZmlnLWRhdGE=", // base64 encode of "config-data"
+				"config": "config-data",
 			},
 		},
 		{
@@ -131,8 +80,8 @@ func TestAttachData(t *testing.T) {
 				},
 			},
 			expected: map[string]string{
-				"resource1.yaml": "echo Y29uZmlnLWRhdGE=", // base64 encode of "config-data"
-				"resource2.yaml": "echo config-data-2",
+				"config1": "config-data",
+				"config2": "config-data-2",
 			},
 		},
 		{
@@ -150,24 +99,8 @@ func TestAttachData(t *testing.T) {
 				},
 			},
 			expected: map[string]string{
-				"resource.yaml": "echo config-data",
-			},
-		},
-		{
-			name: "target is using a nonexistent attachment",
-			fileData: map[string]string{
-				"config.yaml":   "config-data",
-				"resource.yaml": "echo {{ .attachment.config }} && echo {{ .attachment.nonexistent }}",
-			},
-			attachConfig: config.Attachment{
-				Sources: map[string]string{
-					"config": "config.yaml",
-				},
-				Targets: []string{
-					"resource.yaml",
-				},
+				"config": "config-data",
 			},
-			expectedErrorPrefix: "failed to render attaching target resource.yaml",
 		},
 	}
 
@@ -185,20 +118,16 @@ func TestAttachData(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			err = AttachData(appDir, tc.attachConfig)
+			atp := NewAttachmentProcessor(&tc.attachConfig)
+			data, err := atp.BuildTemplateData(appDir)
 			if tc.expectedErrorPrefix != "" {
 				require.Error(t, err)
 				assert.True(t, strings.HasPrefix(err.Error(), tc.expectedErrorPrefix), fmt.Sprintf("Error: %v", err))
 			} else {
 				require.NoError(t, err)
 			}
 
-			for p, c := range tc.expected {
-				p = filepath.Join(appDir, p)
-				data, err := os.ReadFile(p)
-				require.NoError(t, err)
-				assert.Equal(t, c, string(data))
-			}
+			assert.Equal(t, tc.expected, data)
 		})
 	}
 }