Replace tar hashing with golang.org/x/mod and add options

config-example.json

@@ -83,7 +83,14 @@
         },
         "FakeRepo": {
           "url": "file:///absolute/path/to/directory",
-          "vcs": "fake"
+          "vcs": "fake",
+          "url-pattern" : {
+              "base-url" : "{url}/{path}"
+          },
+          "vcs-config": {
+              "detect-changes": false,
+              "ignored-files": []
+          }
         },
         "BitbucketServerUrl" : {
             "url" : "git@bitbucket.internal.org:7999:organization/project.git",

go.mod

@@ -2,4 +2,7 @@ module github.com/hound-search/hound
 
 go 1.16
 
-require github.com/blang/semver/v4 v4.0.0
+require (
+	github.com/blang/semver/v4 v4.0.0
+	golang.org/x/mod v0.10.0
+)

go.sum

@@ -1,2 +1,27 @@
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

vcs/fake.go

@@ -2,36 +2,83 @@ package vcs
 
 import (
 	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"os"
-	"os/exec"
+	"sort"
 	"strings"
+
+	"golang.org/x/mod/sumdb/dirhash"
 )
 
 func init() {
 	Register(newFake, "fake")
 }
 
-type FakeDriver struct{}
+type FakeDriver struct {
+	DetectChanges bool     `json:"detect-changes"`
+	IgnoredFiles  []string `json:"ignored-files"`
+}
 
 func newFake(b []byte) (Driver, error) {
-	return &FakeDriver{}, nil
+	d := FakeDriver{}
+
+	if b != nil {
+		if err := json.Unmarshal(b, &d); err != nil {
+			return nil, err
+		}
+	}
+
+	return &d, nil
+}
+
+// Adapted from: https://github.com/golang/mod/blob/ad6fd61f94f8fdf6926f5dee6e45bdd13add2f9f/sumdb/dirhash/hash.go#L44
+func myHash(files []string, open func(string) (io.ReadCloser, error)) (string, error) {
+	h := sha1.New()
+	files = append([]string(nil), files...)
+	sort.Strings(files)
+	for _, file := range files {
+		if strings.Contains(file, "\n") {
+			return "", errors.New("dirhash: filenames with newlines are not supported")
+		}
+		r, err := open(file)
+		if err != nil {
+			return "", err
+		}
+		hf := sha1.New()
+		_, err = io.Copy(hf, r)
+		r.Close()
+		if err != nil {
+			return "", err
+		}
+		fmt.Fprintf(h, "%x  %s\n", hf.Sum(nil), file)
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
 func (g *FakeDriver) HeadRev(dir string) (string, error) {
-	hsh := sha1.New()
-	cmd := exec.Command(
-		"tar",
-		"cf", "-",
-		"./",
-	)
-	cmd.Dir = dir
-	cmd.Stdout = hsh
-	if err := cmd.Run(); err != nil {
-		return "", fmt.Errorf("%v: %+v", cmd.Args, err)
+	if !g.DetectChanges {
+		idx := strings.LastIndex(dir, "vcs-")
+		if idx == -1 {
+			return "", fmt.Errorf("could not find 'vcs-' in path: %s", dir)
+		}
+		return dir[idx+4:], nil
+	}
+
+	// Resolve the symbolic link
+	if fi, err := os.Stat(dir); err == nil && fi.Mode()|os.ModeSymlink != 0 {
+		if s, err := os.Readlink(dir); err == nil {
+			dir = s
+		}
 	}
 
-	return fmt.Sprintf("%x", hsh.Sum(nil)), nil
+	// Hash all files in the directory
+	// TODO: implement logic from indexAllFiles to skip ignored files?
+	hash, err := dirhash.HashDir(dir, "", myHash)
+	return hash, err
 }
 
 func (g *FakeDriver) Pull(dir string) (string, error) {
@@ -40,14 +87,17 @@ func (g *FakeDriver) Pull(dir string) (string, error) {
 
 func (g *FakeDriver) Clone(dir, url string) (string, error) {
 	src := strings.TrimPrefix(url, "file://")
+	if !strings.HasPrefix(url, "file://") {
+		return "", fmt.Errorf("expected 'file://' prefix in url: %s", url)
+	}
 	if err := os.Symlink(src, dir); err != nil {
 		return "", err
 	}
 	return g.Pull(dir)
 }
 
 func (g *FakeDriver) SpecialFiles() []string {
-	return []string{}
+	return g.IgnoredFiles
 }
 
 func (g *FakeDriver) AutoGeneratedFiles(dir string) []string {