diff --git a/go.mod b/go.mod index d5890930f..434a16991 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,10 @@ require ( github.com/Masterminds/semver v1.4.2 github.com/alecthomas/chroma v0.6.6 github.com/fatih/color v1.7.0 - github.com/google/go-jsonnet v0.13.0 + github.com/google/go-jsonnet v0.14.1-0.20191006203837-42cb19ef24fb github.com/kr/pretty v0.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/posener/complete v1.2.1 - github.com/sh0rez/go-jsonnet v0.14.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.2 diff --git a/go.sum b/go.sum index 0f380ec54..75b17305b 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/google/go-jsonnet v0.13.0 h1:Ul0FtJiQl705JIyGKaBZug/W2LBY5p0xwY08Q69eOAg= -github.com/google/go-jsonnet v0.13.0/go.mod h1:gNwctc8xrpXNs749bjRLO58rjIBVrWz+pgsRoOCh5Vs= +github.com/google/go-jsonnet v0.14.1-0.20191006203837-42cb19ef24fb h1:6GyG7yy0z1foZBWyfRa9CZYdcLbSPHR19PW70fJBedI= +github.com/google/go-jsonnet v0.14.1-0.20191006203837-42cb19ef24fb/go.mod h1:zPGC9lj/TbjkBtUACIvYR/ILHrFqKRhxeEA+bLyeMnY= github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -81,8 +81,6 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sh0rez/go-jsonnet v0.14.2 h1:xt6UJNVUR9blFgVrOrRqYwv5Qncp3xgM18fz1t2WjPQ= -github.com/sh0rez/go-jsonnet v0.14.2/go.mod h1:OC3U+HWq8EVB5nvJDJAWvgVb43HFEXc2VqCUipW9/BE= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go new file mode 100644 index 000000000..2f601e798 --- /dev/null +++ b/pkg/jsonnet/eval.go @@ -0,0 +1,41 @@ +package jsonnet + +import ( + "io/ioutil" + "path/filepath" + + jsonnet "github.com/google/go-jsonnet" + "github.com/pkg/errors" + + "github.com/grafana/tanka/pkg/jpath" + "github.com/grafana/tanka/pkg/native" +) + +// EvaluateFile opens the file, reads it into memory and evaluates it afterwards (`Evaluate()`) +func EvaluateFile(jsonnetFile string) (string, error) { + bytes, err := ioutil.ReadFile(jsonnetFile) + if err != nil { + return "", err + } + + jpath, _, _, err := jpath.Resolve(filepath.Dir(jsonnetFile)) + if err != nil { + return "", errors.Wrap(err, "resolving jpath") + } + return Evaluate(string(bytes), jpath) +} + +// Evaluate renders the given jsonnet into a string +func Evaluate(sonnet string, jpath []string) (string, error) { + importer := jsonnet.FileImporter{ + JPaths: jpath, + } + + vm := jsonnet.MakeVM() + vm.Importer(&importer) + for _, nf := range native.Funcs() { + vm.NativeFunction(nf) + } + + return vm.EvaluateSnippet("main.jsonnet", sonnet) +} diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go new file mode 100644 index 000000000..e20af5532 --- /dev/null +++ b/pkg/jsonnet/imports.go @@ -0,0 +1,102 @@ +package jsonnet + +import ( + "io/ioutil" + "path/filepath" + + jsonnet "github.com/google/go-jsonnet" + "github.com/google/go-jsonnet/ast" + "github.com/google/go-jsonnet/toolutils" + "github.com/pkg/errors" + + "github.com/grafana/tanka/pkg/jpath" + "github.com/grafana/tanka/pkg/native" +) + +// TransitiveImports returns all recursive imports of a file +func TransitiveImports(filename string) ([]string, error) { + sonnet, err := ioutil.ReadFile(filename) + if err != nil { + return nil, errors.Wrap(err, "opening file") + } + + jpath, _, _, err := jpath.Resolve(filepath.Dir(filename)) + if err != nil { + return nil, errors.Wrap(err, "resolving JPATH") + } + importer := jsonnet.FileImporter{ + JPaths: jpath, + } + + vm := jsonnet.MakeVM() + vm.Importer(&importer) + for _, nf := range native.Funcs() { + vm.NativeFunction(nf) + } + + node, err := jsonnet.SnippetToAST("main.jsonnet", string(sonnet)) + if err != nil { + return nil, errors.Wrap(err, "creating Jsonnet AST") + } + + imports := make([]string, 0, 0) + err = importRecursive(&imports, vm, node, "main.jsonnet") + + return uniqueStringSlice(imports), err +} + +// importRecursive takes a Jsonnet VM and recursively imports the AST. Every +// found import is added to the `list` string slice, which will ultimately +// contain all recursive imports +func importRecursive(list *[]string, vm *jsonnet.VM, node ast.Node, currentPath string) error { + switch node := node.(type) { + // we have an `import` + case *ast.Import: + p := node.File.Value + + contents, foundAt, err := vm.ImportAST(currentPath, p) + if err != nil { + return errors.Wrap(err, "importing jsonnet") + } + + *list = append(*list, foundAt) + + if err := importRecursive(list, vm, contents, foundAt); err != nil { + return err + } + + // we have an `importstr` + case *ast.ImportStr: + p := node.File.Value + + foundAt, err := vm.ResolveImport(currentPath, p) + if err != nil { + return errors.Wrap(err, "importing string") + } + + *list = append(*list, foundAt) + + // neither `import` nor `importstr`, probably object or similar: try children + default: + for _, child := range toolutils.Children(node) { + if err := importRecursive(list, vm, child, currentPath); err != nil { + return err + } + } + } + return nil +} + +func uniqueStringSlice(s []string) []string { + seen := make(map[string]struct{}, len(s)) + j := 0 + for _, v := range s { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + s[j] = v + j++ + } + return s[:j] +} diff --git a/pkg/jsonnet/imports_test.go b/pkg/jsonnet/imports_test.go new file mode 100644 index 000000000..d5d2499bf --- /dev/null +++ b/pkg/jsonnet/imports_test.go @@ -0,0 +1,24 @@ +package jsonnet + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestTransitiveImports checks that TransitiveImports is able to report all +// recursive imports of a file +func TestTransitiveImports(t *testing.T) { + imports, err := TransitiveImports("testdata/main.jsonnet") + require.NoError(t, err) + assert.ElementsMatch(t, []string{ + "testdata/trees.jsonnet", + + "testdata/trees/apple.jsonnet", + "testdata/trees/cherry.jsonnet", + "testdata/trees/peach.jsonnet", + + "testdata/trees/generic.libsonnet", + }, imports) +} diff --git a/pkg/jsonnet/jsonnet.go b/pkg/jsonnet/jsonnet.go deleted file mode 100644 index 61e9f1483..000000000 --- a/pkg/jsonnet/jsonnet.go +++ /dev/null @@ -1,96 +0,0 @@ -package jsonnet - -import ( - "io/ioutil" - "path/filepath" - - "github.com/grafana/tanka/pkg/jpath" - "github.com/grafana/tanka/pkg/native" - "github.com/pkg/errors" - - jsonnet "github.com/sh0rez/go-jsonnet" -) - -// EvaluateFile opens the file, reads it into memory and evaluates it afterwards (`Evaluate()`) -func EvaluateFile(jsonnetFile string) (string, error) { - bytes, err := ioutil.ReadFile(jsonnetFile) - if err != nil { - return "", err - } - - jpath, _, _, err := jpath.Resolve(filepath.Dir(jsonnetFile)) - if err != nil { - return "", errors.Wrap(err, "resolving jpath") - } - return Evaluate(string(bytes), jpath) -} - -// Evaluate renders the given jssonet into a string -func Evaluate(sonnet string, jpath []string) (string, error) { - importer := jsonnet.FileImporter{ - JPaths: jpath, - } - - vm := jsonnet.MakeVM() - vm.Importer(&importer) - for _, nf := range native.Funcs() { - vm.NativeFunction(nf) - } - - return vm.EvaluateSnippet("main.jsonnet", sonnet) -} - -type ImportVisitor func(who, what string) error - -func VisitImportsFile(jsonnetFile string, v ImportVisitor) error { - bytes, err := ioutil.ReadFile(jsonnetFile) - if err != nil { - return err - } - - jpath, _, _, err := jpath.Resolve(filepath.Dir(jsonnetFile)) - if err != nil { - return err - } - return VisitImports(string(bytes), jpath, v) -} - -func VisitImports(sonnet string, jpath []string, v ImportVisitor) error { - importer := TraceImporter{ - JPaths: jpath, - Visitor: v, - } - - vm := jsonnet.MakeVM() - vm.Importer(&importer) - for _, nf := range native.Funcs() { - vm.NativeFunction(nf) - } - - // This method does not exist in google/go-jsonnet. It has been patched in sh0rez/go-jsonnet. - // Basically it aborts the evaluation after the imports are done. This is much faster (7s vs 0.5s) - if err := vm.EvaluateSnippetWithoutManifestation("main.jsonnet", sonnet); err != nil { - return err - } - return nil -} - -type TraceImporter struct { - JPaths []string - Visitor ImportVisitor - importer *jsonnet.FileImporter -} - -func (t *TraceImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { - if t.importer == nil { - t.importer = &jsonnet.FileImporter{ - JPaths: t.JPaths, - } - } - - contents, foundAt, err = t.importer.Import(importedFrom, importedPath) - if err := t.Visitor(importedFrom, foundAt); err != nil { - return jsonnet.Contents{}, "", err - } - return -} diff --git a/pkg/jsonnet/testdata/README.md b/pkg/jsonnet/testdata/README.md new file mode 100644 index 000000000..22808f847 --- /dev/null +++ b/pkg/jsonnet/testdata/README.md @@ -0,0 +1,5 @@ +# Importing testdata + +This directory contains some jsonnet files importing each other, to test if the import analysis tooling works correctly. + +![](test.svg) diff --git a/pkg/jsonnet/testdata/jsonnetfile.json b/pkg/jsonnet/testdata/jsonnetfile.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/jsonnetfile.json @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/main.jsonnet b/pkg/jsonnet/testdata/main.jsonnet new file mode 100644 index 000000000..ebf096ba5 --- /dev/null +++ b/pkg/jsonnet/testdata/main.jsonnet @@ -0,0 +1,8 @@ +local trees = import 'trees.jsonnet'; + +// a list of trees +[ + trees.apple, + trees.cherry, + trees.peach, +] diff --git a/pkg/jsonnet/testdata/test.svg b/pkg/jsonnet/testdata/test.svg new file mode 100644 index 000000000..f733b608f --- /dev/null +++ b/pkg/jsonnet/testdata/test.svg @@ -0,0 +1,91 @@ + + + + + + +importtree + + + +0 + +trees/cherry.jsonnet + + + +1 + +trees/generic.libsonnet + + + +0->1 + + + + + +2 + +main.jsonnet + + + +3 + +trees.jsonnet + + + +2->3 + + + + + +3->0 + + + + + +4 + +trees/apple.jsonnet + + + +3->4 + + + + + +5 + +trees/peach.jsonnet + + + +3->5 + + + + + +4->1 + + + + + +5->1 + + + + + diff --git a/pkg/jsonnet/testdata/trees.jsonnet b/pkg/jsonnet/testdata/trees.jsonnet new file mode 100644 index 000000000..4c5f5886b --- /dev/null +++ b/pkg/jsonnet/testdata/trees.jsonnet @@ -0,0 +1,5 @@ +{ + apple: import 'trees/apple.jsonnet', + peach: import 'trees/peach.jsonnet', + cherry: import 'trees/cherry.jsonnet', +} diff --git a/pkg/jsonnet/testdata/trees/apple.jsonnet b/pkg/jsonnet/testdata/trees/apple.jsonnet new file mode 100644 index 000000000..e348477a5 --- /dev/null +++ b/pkg/jsonnet/testdata/trees/apple.jsonnet @@ -0,0 +1,2 @@ +local t = import 'generic.libsonnet'; +t.new('apple', 'red') diff --git a/pkg/jsonnet/testdata/trees/cherry.jsonnet b/pkg/jsonnet/testdata/trees/cherry.jsonnet new file mode 100644 index 000000000..708ea5812 --- /dev/null +++ b/pkg/jsonnet/testdata/trees/cherry.jsonnet @@ -0,0 +1,2 @@ +local t = import 'generic.libsonnet'; +t.new('cherry', 'red', 'xs') diff --git a/pkg/jsonnet/testdata/trees/generic.libsonnet b/pkg/jsonnet/testdata/trees/generic.libsonnet new file mode 100644 index 000000000..14f61c3a5 --- /dev/null +++ b/pkg/jsonnet/testdata/trees/generic.libsonnet @@ -0,0 +1,14 @@ +{ + new(breed, color, size="m"):: { + kind: "tree", + + breed: breed, + color: color, + size: size, + + needs: "water", + eats: "co2", + creates: "o2", + keeps: "the world healthy" + } +} diff --git a/pkg/jsonnet/testdata/trees/peach.jsonnet b/pkg/jsonnet/testdata/trees/peach.jsonnet new file mode 100644 index 000000000..4d311f0b0 --- /dev/null +++ b/pkg/jsonnet/testdata/trees/peach.jsonnet @@ -0,0 +1,2 @@ +local t = import 'generic.libsonnet'; +t.new('peach', 'orange', 's') diff --git a/pkg/jsonnet/transitive.go b/pkg/jsonnet/transitive.go deleted file mode 100644 index e75cc0624..000000000 --- a/pkg/jsonnet/transitive.go +++ /dev/null @@ -1,79 +0,0 @@ -package jsonnet - -import ( - "path/filepath" -) - -// TransitiveImports returns a slice with all files this file imports plus downstream imports -func TransitiveImports(filename string) ([]string, error) { - imports := map[string][]string{} - if err := VisitImportsFile(filename, func(who, what string) error { - if imports[who] == nil { - imports[who] = []string{} - } - imports[who] = append(imports[who], what) - return nil - }); err != nil { - return nil, err - } - - deps := map[string]*File{} - for k, v := range imports { - deps[k] = &File{Imports: v} - } - - for _, d := range deps { - resolveTransitives(d, deps) - } - - for _, d := range deps { - d.Dependencies = uniqueStringSlice(d.Dependencies) - } - - return deps[filepath.Base(filename)].Dependencies, nil -} - -// File represents a jsonnet file that may import other files -type File struct { - // List of files this file imports - Imports []string - // Full list of transitive imports - Dependencies []string -} - -func resolveTransitives(f *File, deps map[string]*File) { - // already resolved - if len(f.Dependencies) != 0 { - return - } - - for _, i := range f.Imports { - f.Dependencies = append(f.Dependencies, i) - - // import has no dependencies - if deps[i] == nil { - continue - } - - // import dependencies have not yet been resolved - if len(deps[i].Dependencies) == 0 { - resolveTransitives(deps[i], deps) - } - - f.Dependencies = append(f.Dependencies, deps[i].Dependencies...) - } -} - -func uniqueStringSlice(s []string) []string { - seen := make(map[string]struct{}, len(s)) - j := 0 - for _, v := range s { - if _, ok := seen[v]; ok { - continue - } - seen[v] = struct{}{} - s[j] = v - j++ - } - return s[:j] -} diff --git a/pkg/native/funcs.go b/pkg/native/funcs.go index b63c952e4..dbdaa876e 100644 --- a/pkg/native/funcs.go +++ b/pkg/native/funcs.go @@ -7,10 +7,9 @@ import ( "regexp" "strings" + jsonnet "github.com/google/go-jsonnet" "github.com/google/go-jsonnet/ast" yaml "gopkg.in/yaml.v2" - - jsonnet "github.com/sh0rez/go-jsonnet" ) // Funcs returns a slice of native Go functions that shall be available