Add --delete-previous option to tk export

Documented in `exporting.md`. This option allows us to delete the previously exported manifests for an environment.

This is useful when exporting a single environment and merging the result with an existing GitOps repository, rather than re-exporting all environments.

I also added benchmark and a test for the export code.

cmd/tk/export.go

@@ -39,6 +39,7 @@ func exportCmd() *cli.Command {
 	cachePath := cmd.Flags().StringP("cache-path", "c", "", "Local file path where cached evaluations should be stored")
 	cacheEnvs := cmd.Flags().StringArrayP("cache-envs", "e", nil, "Regexes which define which environment should be cached (if caching is enabled)")
 	ballastBytes := cmd.Flags().Int("mem-ballast-size-bytes", 0, "Size of memory ballast to allocate. This may improve performance for large environments.")
+	deletePrevious := cmd.Flags().Bool("delete-previous", false, "If set, before exporting, delete files previously exported by the targeted envs, leaving untargeted envs intact. To be used with --merge.")
 
 	vars := workflowFlags(cmd.Flags())
 	getJsonnetOpts := jsonnetFlags(cmd.Flags())
@@ -57,9 +58,10 @@ func exportCmd() *cli.Command {
 		}
 
 		opts := tanka.ExportEnvOpts{
-			Format:    *format,
-			Extension: *extension,
-			Merge:     *merge,
+			Format:         *format,
+			Extension:      *extension,
+			Merge:          *merge,
+			DeletePrevious: *deletePrevious,
 			Opts: tanka.Opts{
 				JsonnetOpts: getJsonnetOpts(),
 				Filters:     filters,

docs/docs/exporting.md

@@ -99,15 +99,29 @@ $ tk export exportDir environments/ --recursive
 $ tk export exportDir environments/ -r -l team=infra
 ```
 
-## Using a memory ballast
+## Performance features
+
+When exporting a large amount of environments, jsonnet evaluation can become a bottleneck. To speed up the process, Tanka provides a few optional features.
+
+### Partial export (in a GitOps context)
+
+Given multiple environments, one may want to only export the environments that were modified since the last export. This is enabled by passing both the `--merge` and `--delete-previous` flags.
+
+When these flags are passed, Tanka will:
+
+1. Delete the manifests that were previously exported by the environments that are being exported. This is done by looking at the `manifest.json` file that is generated by Tanka when exporting. The related entries are also removed from the `manifest.json` file.
+2. Generate the manifests for the targeted environments into the output directory.
+3. Add in the new manifests entries into the `manifest.json` file and re-export it.
+
+### Using a memory ballast
 
 _Read [this blog post](https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap/) for more information about memory ballasts._
 
 For large environments that load lots of data into memory on evaluation, a memory ballast can dramatically improve performance. This feature is exposed through the `--mem-ballast-size-bytes` flag on the export command.
 
 Anecdotally (Grafana Labs), environments that took around a minute to load were able to load in around 45 secs with a ballast of 5GB (`--mem-ballast-size-bytes=5368709120`). Decreasing the ballast size resulted in negative impact on performance, and increasing it more did not result in any noticeable impact.
 
-## Caching
+### Caching
 
 Tanka can also cache the results of the export. This is useful if you often export the same files and want to avoid recomputing them. The cache key is calculated from the main file and all of its transitive imports, so any change to any file possibly used in an environment will invalidate the cache.
 

pkg/tanka/export.go

@@ -5,12 +5,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
@@ -41,6 +44,8 @@ type ExportEnvOpts struct {
 	Selector labels.Selector
 	// optional: number of environments to process in parallel
 	Parallelism int
+	// optional: If set, target an existing export directory, delete files previously exported by the targeted envs and re-export targeted envs.
+	DeletePrevious bool
 }
 
 func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnvOpts) error {
@@ -56,6 +61,13 @@ func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnv
 		return fmt.Errorf("output dir `%s` not empty. Pass --merge to ignore this", to)
 	}
 
+	// delete files previously exported by the targeted envs.
+	if opts.DeletePrevious {
+		if err := deletePreviouslyExportedManifests(to, envs); err != nil {
+			return fmt.Errorf("deleting previously exported manifests: %w", err)
+		}
+	}
+
 	// get all environments for paths
 	loadedEnvs, err := parallelLoadEnvironments(envs, parallelOpts{
 		Opts:        opts.Opts,
@@ -122,19 +134,7 @@ func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnv
 		}
 	}
 
-	// Write manifest file
-	if len(fileToEnv) != 0 {
-		data, err := json.MarshalIndent(fileToEnv, "", "    ")
-		if err != nil {
-			return err
-		}
-		path := filepath.Join(to, manifestFile)
-		if err := writeExportFile(path, data); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return exportManifestFile(to, fileToEnv, nil)
 }
 
 func fileExists(name string) (bool, error) {
@@ -164,6 +164,74 @@ func dirEmpty(dir string) (bool, error) {
 	return false, err
 }
 
+func deletePreviouslyExportedManifests(path string, envs []*v1alpha1.Environment) error {
+	fileToEnvMap := make(map[string]string)
+
+	manifestFilePath := filepath.Join(path, manifestFile)
+	manifestContent, err := os.ReadFile(manifestFilePath)
+	if err != nil && errors.Is(err, fs.ErrNotExist) {
+		log.Printf("Warning: No manifest file found at %s, skipping deletion of previously exported manifests\n", manifestFilePath)
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(manifestContent, &fileToEnvMap); err != nil {
+		return err
+	}
+
+	envNames := make(map[string]struct{})
+	for _, env := range envs {
+		envNames[env.Metadata.Namespace] = struct{}{}
+	}
+
+	var deletedManifestKeys []string
+	for exportedManifest, manifestEnv := range fileToEnvMap {
+		if _, ok := envNames[manifestEnv]; ok {
+			deletedManifestKeys = append(deletedManifestKeys, exportedManifest)
+			if err := os.Remove(filepath.Join(path, exportedManifest)); err != nil {
+				return err
+			}
+		}
+	}
+
+	return exportManifestFile(path, nil, deletedManifestKeys)
+}
+
+// exportManifestFile writes a manifest file that maps the exported files to their environment.
+// If the file already exists, the new entries will be merged with the existing ones.
+func exportManifestFile(path string, newFileToEnvMap map[string]string, deletedKeys []string) error {
+	if len(newFileToEnvMap) == 0 && len(deletedKeys) == 0 {
+		return nil
+	}
+
+	manifestFilePath := filepath.Join(path, manifestFile)
+	manifestContent, err := os.ReadFile(manifestFilePath)
+	currentFileToEnvMap := make(map[string]string)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("reading existing manifest file: %w", err)
+	} else if err == nil {
+		if err := json.Unmarshal(manifestContent, &currentFileToEnvMap); err != nil {
+			return fmt.Errorf("unmarshalling existing manifest file: %w", err)
+		}
+	}
+
+	for k, v := range newFileToEnvMap {
+		currentFileToEnvMap[k] = v
+	}
+	for _, k := range deletedKeys {
+		delete(currentFileToEnvMap, k)
+	}
+
+	// Write manifest file
+	data, err := json.MarshalIndent(currentFileToEnvMap, "", "    ")
+	if err != nil {
+		return fmt.Errorf("marshalling manifest file: %w", err)
+	}
+
+	return writeExportFile(manifestFilePath, data)
+}
+
 func writeExportFile(path string, data []byte) error {
 	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
 		return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err)

pkg/tanka/export_test.go

@@ -1,6 +1,18 @@
 package tanka
 
-import "testing"
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/grafana/tanka/pkg/jsonnet"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/labels"
+)
 
 func Test_replaceTmplText(t *testing.T) {
 	type args struct {
@@ -31,3 +43,133 @@ func Test_replaceTmplText(t *testing.T) {
 		})
 	}
 }
+
+func TestExportEnvironments(t *testing.T) {
+	tempDir := t.TempDir()
+	require.NoError(t, os.Chdir("testdata"))
+	defer os.Chdir("..")
+
+	// Find envs
+	envs, err := FindEnvs("test-export-envs", FindOpts{Selector: labels.Everything()})
+	require.NoError(t, err)
+
+	// Export all envs
+	opts := &ExportEnvOpts{
+		Format:    "{{.metadata.namespace}}/{{.metadata.name}}",
+		Extension: "yaml",
+	}
+	opts.Opts.ExtCode = jsonnet.InjectedCode{
+		"deploymentName": "'initial-deployment'",
+		"serviceName":    "'initial-service'",
+	}
+	require.NoError(t, ExportEnvironments(envs, tempDir, opts))
+	checkFiles(t, tempDir, []string{
+		filepath.Join(tempDir, "inline-namespace1", "my-configmap.yaml"),
+		filepath.Join(tempDir, "inline-namespace1", "my-deployment.yaml"),
+		filepath.Join(tempDir, "inline-namespace1", "my-service.yaml"),
+		filepath.Join(tempDir, "inline-namespace2", "my-deployment.yaml"),
+		filepath.Join(tempDir, "inline-namespace2", "my-service.yaml"),
+		filepath.Join(tempDir, "static", "initial-deployment.yaml"),
+		filepath.Join(tempDir, "static", "initial-service.yaml"),
+		filepath.Join(tempDir, "manifest.json"),
+	})
+	manifestContent, err := os.ReadFile(filepath.Join(tempDir, "manifest.json"))
+	require.NoError(t, err)
+	assert.Equal(t, string(manifestContent), `{
+    "inline-namespace1/my-configmap.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace1/my-deployment.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace1/my-service.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace2/my-deployment.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace2/my-service.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "static/initial-deployment.yaml": "test-export-envs/static-env/main.jsonnet",
+    "static/initial-service.yaml": "test-export-envs/static-env/main.jsonnet"
+}`)
+
+	// Try to re-export
+	assert.EqualError(t, ExportEnvironments(envs, tempDir, opts), fmt.Sprintf("Output dir `%s` not empty. Pass --merge to ignore this", tempDir))
+
+	// Try to re-export with the --merge flag. Will still fail because Tanka will not overwrite manifests silently
+	opts.Merge = true
+	assert.ErrorContains(t, ExportEnvironments(envs, tempDir, opts), "already exists. Aborting")
+
+	// Re-export only one env with --delete-previous flag
+	opts.Opts.ExtCode = jsonnet.InjectedCode{
+		"deploymentName": "'updated-deployment'",
+		"serviceName":    "'updated-service'",
+	}
+	opts.DeletePrevious = true
+	staticEnv, err := FindEnvs("test-export-envs", FindOpts{Selector: labels.SelectorFromSet(labels.Set{"type": "static"})})
+	require.NoError(t, err)
+	require.NoError(t, ExportEnvironments(staticEnv, tempDir, opts))
+	checkFiles(t, tempDir, []string{
+		filepath.Join(tempDir, "inline-namespace1", "my-configmap.yaml"),
+		filepath.Join(tempDir, "inline-namespace1", "my-deployment.yaml"),
+		filepath.Join(tempDir, "inline-namespace1", "my-service.yaml"),
+		filepath.Join(tempDir, "inline-namespace2", "my-deployment.yaml"),
+		filepath.Join(tempDir, "inline-namespace2", "my-service.yaml"),
+		filepath.Join(tempDir, "static", "updated-deployment.yaml"),
+		filepath.Join(tempDir, "static", "updated-service.yaml"),
+		filepath.Join(tempDir, "manifest.json"),
+	})
+	manifestContent, err = os.ReadFile(filepath.Join(tempDir, "manifest.json"))
+	require.NoError(t, err)
+	assert.Equal(t, string(manifestContent), `{
+    "inline-namespace1/my-configmap.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace1/my-deployment.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace1/my-service.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace2/my-deployment.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "inline-namespace2/my-service.yaml": "test-export-envs/inline-envs/main.jsonnet",
+    "static/updated-deployment.yaml": "test-export-envs/static-env/main.jsonnet",
+    "static/updated-service.yaml": "test-export-envs/static-env/main.jsonnet"
+}`)
+}
+
+func BenchmarkExportEnvironmentsWithDeletePrevious(b *testing.B) {
+	log.SetOutput(ioutil.Discard)
+	tempDir := b.TempDir()
+	require.NoError(b, os.Chdir("testdata"))
+	defer os.Chdir("..")
+
+	// Find envs
+	envs, err := FindEnvs("test-export-envs", FindOpts{Selector: labels.Everything()})
+	require.NoError(b, err)
+
+	// Export all envs
+	opts := &ExportEnvOpts{
+		Format:         "{{.metadata.namespace}}/{{.metadata.name}}",
+		Extension:      "yaml",
+		Merge:          true,
+		DeletePrevious: true,
+	}
+	opts.Opts.ExtCode = jsonnet.InjectedCode{
+		"deploymentName": "'initial-deployment'",
+		"serviceName":    "'initial-service'",
+	}
+	// Export a first time so that the benchmark loops are identical
+	require.NoError(b, ExportEnvironments(envs, tempDir, opts))
+
+	// On every loop, delete manifests from previous envs + reexport all envs
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		require.NoError(b, ExportEnvironments(envs, tempDir, opts), "failed on iteration %d", i)
+	}
+}
+
+func checkFiles(t testing.TB, dir string, files []string) {
+	t.Helper()
+
+	var existingFiles []string
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		existingFiles = append(existingFiles, path)
+		return nil
+	})
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, files, existingFiles)
+}

pkg/tanka/testdata/test-export-envs/inline-envs/main.jsonnet

@@ -0,0 +1,53 @@
+[
+  {
+    apiVersion: 'tanka.dev/v1alpha1',
+    kind: 'Environment',
+    metadata: {
+      name: env.namespace,
+      labels: {
+        type: 'inline',
+      },
+    },
+    spec: {
+      apiServer: 'https://localhost',
+      namespace: env.namespace,
+    },
+    data:
+      {
+        deployment: {
+          apiVersion: 'apps/v1',
+          kind: 'Deployment',
+          metadata: {
+            name: 'my-deployment',
+          },
+        },
+        service: {
+          apiVersion: 'v1',
+          kind: 'Service',
+          metadata: {
+            name: 'my-service',
+          },
+        },
+      } +
+      (if env.hasConfigMap then {
+         configMap: {
+           apiVersion: 'v1',
+           kind: 'ConfigMap',
+           metadata: {
+             name: 'my-configmap',
+           },
+         },
+       } else {}),
+  }
+
+  for env in [
+    {
+      namespace: 'inline-namespace1',
+      hasConfigMap: true,
+    },
+    {
+      namespace: 'inline-namespace2',
+      hasConfigMap: false,
+    },
+  ]
+]