Git VCS driver: Support non-default target refs (hound-search#345)

Currently, only the default ref (`master`) is supported.

- Allow target non-default target refs to be specified via configs.
- Add config-driven target ref auto-detection using the `HEAD` branch.
- Update default repo base URL pattern to link to the content revision,
  accounting for potential non-default target ref usage.
- Add global-level VCS configs which are merged into the repo-level configs
  during startup (repo-level vals take precedence).

.gitignore

@@ -3,3 +3,5 @@
 /node_modules
 .DS_Store
 *.exe
+/db
+/data

config-example.json

@@ -3,6 +3,11 @@
     "dbpath" : "data",
     "title" : "Hound",
     "health-check-uri" : "/healthz",
+    "vcs-config" : {
+        "git": {
+            "detect-ref" : true
+        }
+    },
     "repos" : {
         "SomeGitRepo" : {
             "url" : "https://www.github.com/YourOrganization/RepoOne.git"
@@ -12,13 +17,19 @@
             "ms-between-poll": 10000,
             "exclude-dot-files": true
         },
+        "GitRepoWithDetectRefDisabled" : {
+            "url" : "https://www.github.com/YourOrganization/RepoOne.git",
+            "vcs-config" : {
+                "detect-ref" : false
+            }
+        },
         "SomeMercurialRepo" : {
             "url" : "https://www.example.com/foo/hg",
             "vcs" : "hg"
         },
         "Subversion" : {
             "url" : "http://my-svn.com/repo",
-            "url-pattern" : { 
+            "url-pattern" : {
                 "base-url" : "{url}/{path}{anchor}"
             },
             "vcs" : "svn"
@@ -49,6 +60,9 @@
             "url-pattern" : {
                 "base-url" : "https://{url}/src/master/{path}{anchor}",
                 "anchor" : "#{filename}-{line}"
+            },
+            "vcs-config" : {
+                "ref" : "main"
             }
         },
         "RepoWithPollingDisabled" : {

config/config.go

@@ -14,7 +14,7 @@ const (
 	defaultPollEnabled           = true
 	defaultTitle                 = "Hound"
 	defaultVcs                   = "git"
-	defaultBaseUrl               = "{url}/blob/master/{path}{anchor}"
+	defaultBaseUrl               = "{url}/blob/{rev}/{path}{anchor}"
 	defaultAnchor                = "#L{line}"
 	defaultHealthCheckURI        = "/healthz"
 )
@@ -55,11 +55,12 @@ func (r *Repo) PushUpdatesEnabled() bool {
 }
 
 type Config struct {
-	DbPath                string           `json:"dbpath"`
-	Title                 string           `json:"title"`
-	Repos                 map[string]*Repo `json:"repos"`
-	MaxConcurrentIndexers int              `json:"max-concurrent-indexers"`
-	HealthCheckURI        string           `json:"health-check-uri"`
+	DbPath                string                    `json:"dbpath"`
+	Title                 string                    `json:"title"`
+	Repos                 map[string]*Repo          `json:"repos"`
+	MaxConcurrentIndexers int                       `json:"max-concurrent-indexers"`
+	HealthCheckURI        string                    `json:"health-check-uri"`
+	VCSConfigMessages     map[string]*SecretMessage `json:"vcs-config"`
 }
 
 // SecretMessage is just like json.RawMessage but it will not
@@ -116,15 +117,67 @@ func initRepo(r *Repo) {
 	}
 }
 
-// Populate missing config values with default values.
-func initConfig(c *Config) {
+// Populate missing config values with default values and
+// merge global VCS configs into repo level configs.
+func initConfig(c *Config) error {
 	if c.MaxConcurrentIndexers == 0 {
 		c.MaxConcurrentIndexers = defaultMaxConcurrentIndexers
 	}
 
 	if c.HealthCheckURI == "" {
 		c.HealthCheckURI = defaultHealthCheckURI
 	}
+
+	return mergeVCSConfigs(c)
+}
+
+func mergeVCSConfigs(cfg *Config) error {
+	globalConfigLen := len(cfg.VCSConfigMessages)
+	if globalConfigLen == 0 {
+		return nil
+	}
+
+	globalConfigVals := make(map[string]map[string]interface{}, globalConfigLen)
+	for vcs, configBytes := range cfg.VCSConfigMessages {
+		var configVals map[string]interface{}
+		if err := json.Unmarshal(*configBytes, &configVals); err != nil {
+			return err
+		}
+
+		globalConfigVals[vcs] = configVals
+	}
+
+	for _, repo := range cfg.Repos {
+		var globalVals map[string]interface{}
+		globalVals, valsExist := globalConfigVals[repo.Vcs]
+		if !valsExist {
+			continue
+		}
+
+		repoBytes := repo.VcsConfig()
+		var repoVals map[string]interface{}
+		if len(repoBytes) == 0 {
+			repoVals = make(map[string]interface{}, len(globalVals))
+		} else if err := json.Unmarshal(repoBytes, &repoVals); err != nil {
+			return err
+		}
+
+		for name, val := range globalVals {
+			if _, ok := repoVals[name]; !ok {
+				repoVals[name] = val
+			}
+		}
+
+		repoBytes, err := json.Marshal(&repoVals)
+		if err != nil {
+			return err
+		}
+
+		repoMessage := SecretMessage(repoBytes)
+		repo.VcsConfigMessage = &repoMessage
+	}
+
+	return nil
 }
 
 func (c *Config) LoadFromFile(filename string) error {
@@ -155,9 +208,7 @@ func (c *Config) LoadFromFile(filename string) error {
 		initRepo(repo)
 	}
 
-	initConfig(c)
-
-	return nil
+	return initConfig(c)
 }
 
 func (c *Config) ToJsonString() (string, error) {

config/config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/json"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -34,4 +35,21 @@ func TestExampleConfigsAreValid(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+
+	// Ensure that global VCS config vals are merged
+	repo := cfg.Repos["SomeGitRepo"]
+	vcsConfigBytes := repo.VcsConfig()
+	var vcsConfigVals map[string]interface{}
+	json.Unmarshal(vcsConfigBytes, &vcsConfigVals)
+	if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || !detectRef.(bool) {
+		t.Error("global detectRef vcs config setting not set for repo")
+	}
+
+	repo = cfg.Repos["GitRepoWithDetectRefDisabled"]
+	vcsConfigBytes = repo.VcsConfig()
+	json.Unmarshal(vcsConfigBytes, &vcsConfigVals)
+	if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || detectRef.(bool) {
+		t.Error("global detectRef vcs config setting not overriden by repo-level setting")
+	}
+
 }

vcs/git.go

@@ -2,24 +2,49 @@ package vcs
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 )
 
 const defaultRef = "master"
 
+var headBranchRegexp = regexp.MustCompile(`HEAD branch: (?P<branch>.+)`)
+
 func init() {
 	Register(newGit, "git")
 }
 
-type GitDriver struct{}
+type GitDriver struct {
+	DetectRef     bool   `json:"detect-ref"`
+	Ref           string `json:"ref"`
+	refDetetector refDetetector
+}
+
+type refDetetector interface {
+	detectRef(dir string) string
+}
+
+type headBranchDetector struct {
+}
 
 func newGit(b []byte) (Driver, error) {
-	return &GitDriver{}, nil
+	var d GitDriver
+
+	if b != nil {
+		if err := json.Unmarshal(b, &d); err != nil {
+			return nil, err
+		}
+	}
+
+	d.refDetetector = &headBranchDetector{}
+
+	return &d, nil
 }
 
 func (g *GitDriver) HeadRev(dir string) (string, error) {
@@ -47,43 +72,61 @@ func (g *GitDriver) HeadRev(dir string) (string, error) {
 	return strings.TrimSpace(buf.String()), cmd.Wait()
 }
 
-func run(desc, dir, cmd string, args ...string) error {
+func run(desc, dir, cmd string, args ...string) (string, error) {
 	c := exec.Command(cmd, args...)
 	c.Dir = dir
-	if out, err := c.CombinedOutput(); err != nil {
+	out, err := c.CombinedOutput()
+	if err != nil {
 		log.Printf(
 			"Failed to %s %s, see output below\n%sContinuing...",
 			desc,
 			dir,
 			out)
-		return err
 	}
-	return nil
+
+	return string(out), nil
 }
 
 func (g *GitDriver) Pull(dir string) (string, error) {
-	if err := run("git fetch", dir,
+	targetRef := g.targetRef(dir)
+
+	if _, err := run("git fetch", dir,
 		"git",
 		"fetch",
 		"--prune",
 		"--no-tags",
 		"--depth", "1",
 		"origin",
-		fmt.Sprintf("+%s:remotes/origin/%s", defaultRef, defaultRef)); err != nil {
+		fmt.Sprintf("+%s:remotes/origin/%s", targetRef, targetRef)); err != nil {
 		return "", err
 	}
 
-	if err := run("git reset", dir,
+	if _, err := run("git reset", dir,
 		"git",
 		"reset",
 		"--hard",
-		fmt.Sprintf("origin/%s", defaultRef)); err != nil {
+		fmt.Sprintf("origin/%s", targetRef)); err != nil {
 		return "", err
 	}
 
 	return g.HeadRev(dir)
 }
 
+func (g *GitDriver) targetRef(dir string) string {
+	var targetRef string
+	if g.Ref != "" {
+		targetRef = g.Ref
+	} else if g.DetectRef {
+		targetRef = g.refDetetector.detectRef(dir)
+	}
+
+	if targetRef == "" {
+		targetRef = defaultRef
+	}
+
+	return targetRef
+}
+
 func (g *GitDriver) Clone(dir, url string) (string, error) {
 	par, rep := filepath.Split(dir)
 	cmd := exec.Command(
@@ -99,11 +142,42 @@ func (g *GitDriver) Clone(dir, url string) (string, error) {
 		return "", err
 	}
 
-	return g.HeadRev(dir)
+	return g.Pull(dir)
 }
 
 func (g *GitDriver) SpecialFiles() []string {
 	return []string{
 		".git",
 	}
 }
+
+func (d *headBranchDetector) detectRef(dir string) string {
+	output, err := run("git show remote info", dir,
+		"git",
+		"remote",
+		"show",
+		"origin",
+	)
+
+	if err != nil {
+		log.Printf(
+			"error occured when fetching info to determine target ref in %s: %s. Will fall back to default ref %s",
+			dir,
+			err,
+			defaultRef,
+		)
+		return ""
+	}
+
+	matches := headBranchRegexp.FindStringSubmatch(output)
+	if len(matches) != 2 {
+		log.Printf(
+			"could not determine target ref in %s. Will fall back to default ref %s",
+			dir,
+			defaultRef,
+		)
+		return ""
+	}
+
+	return matches[1]
+}