diff --git a/.github/workflows/benchstat-pr.yml b/.github/workflows/benchstat-pr.yml new file mode 100644 index 000000000..08871dc78 --- /dev/null +++ b/.github/workflows/benchstat-pr.yml @@ -0,0 +1,75 @@ +name: benchstat + +on: [pull_request] + +jobs: + benchstat: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: '1.19' + + # Generate benchmark report for main branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + - name: Benchmark + run: go test ./... -count=10 -run="^$" -bench=. -benchmem | tee -a bench.txt + - name: Upload Benchmark + uses: actions/upload-artifact@v3 + with: + name: bench-current + path: bench.txt + + # Generate benchmark report for the PR + - name: Checkout + uses: actions/checkout@v3 + - name: Benchmark + run: go test ./... -count=10 -run="^$" -bench=. -benchmem | tee -a bench.txt + - name: Upload Benchmark + uses: actions/upload-artifact@v3 + with: + name: bench-incoming + path: bench.txt + + # Compare the two reports + - name: Checkout + uses: actions/checkout@v3 + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + - name: Download Incoming + uses: actions/download-artifact@v3 + with: + name: bench-incoming + path: bench-incoming + - name: Download Current + uses: actions/download-artifact@v3 + with: + name: bench-current + path: bench-current + - name: Benchstat Results + run: benchstat bench-current/bench.txt bench-incoming/bench.txt | tee -a benchstat.txt + - name: Upload benchstat results + uses: actions/upload-artifact@v3 + with: + name: benchstat + path: benchstat.txt + - name: Read benchstat.txt + id: benchstat_content + uses: juliangruber/read-file-action@v1 + with: + path: ./benchstat.txt + - name: Post PR Comment + uses: thollander/actions-comment-pull-request@v1 + with: + message: | + Benchstat (compared to main): + + ``` + ${{ steps.benchstat_content.outputs.content }} + ``` + comment_includes: 'Benchstat (compared to main):' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pkg/jsonnet/imports_test.go b/pkg/jsonnet/imports_test.go index a7c2f05e8..fe940c91b 100644 --- a/pkg/jsonnet/imports_test.go +++ b/pkg/jsonnet/imports_test.go @@ -2,6 +2,10 @@ package jsonnet import ( "fmt" + "os" + "path/filepath" + "strings" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -23,3 +27,82 @@ func TestTransitiveImports(t *testing.T) { "trees/peach.jsonnet", }, imports) } + +const testFile = ` +local localImport = ; +local myFunc = function() ; + +{ + local this = self, + + attribute: { + name: 'test', + value: self.name, + otherValue: 'other ' + self.value, + }, + nested: { + nested: { + nested: { + nested: { + nested1: { + nested: { + nested1: { + nested: { + attribute: , + }, + }, + nested2: { + strValue: this.nested.nested.nested, + }, + }, + }, + nested2: { + intValue: 1, + importValue: , + }, + }, + }, + }, + }, + + other: myFunc(), + useLocal: localImport, +}` + +func BenchmarkGetSnippetHash(b *testing.B) { + // Create a very large and complex project + tempDir := b.TempDir() + + var mainContentSplit []string + for i := 0; i < 1000; i++ { + mainContentSplit = append(mainContentSplit, fmt.Sprintf("(import 'file%d.libsonnet')", i)) + } + require.NoError(b, os.WriteFile(filepath.Join(tempDir, "main.jsonnet"), []byte(strings.Join(mainContentSplit, " + ")), 0644)) + for i := 0; i < 1000; i++ { + err := os.WriteFile( + filepath.Join(tempDir, fmt.Sprintf("file%d.libsonnet", i)), + []byte(strings.ReplaceAll(testFile, "", fmt.Sprintf("import 'file%d.libsonnet'", i+1))), + 0644, + ) + require.NoError(b, err) + } + require.NoError(b, os.WriteFile(filepath.Join(tempDir, "file1000.libsonnet"), []byte(`"a string"`), 0644)) + + // Create a VM. It's important to reuse the same VM + // While there is a caching mechanism that normally shouldn't be shared in a benchmark iteration, + // it's useful to evaluate its impact here, because the caching will also improve the evaluation performance afterwards. + vm := MakeVM(Opts{ImportPaths: []string{tempDir}}) + content, err := os.ReadFile(filepath.Join(tempDir, "main.jsonnet")) + require.NoError(b, err) + + // Run the benchmark + mainPath := filepath.Join(tempDir, "main.jsonnet") + c := string(content) + b.ResetTimer() + for i := 0; i < b.N; i++ { + fileHashes = sync.Map{} + hash, err := getSnippetHash(vm, mainPath, c) + require.NoError(b, err) + require.Equal(b, "XrkW8N2EvkFMvdIuHTsGsQespVUl9_xiFmM7v1mqX5s=", hash) + } +}