Add graceful shutdown on SIGTERM

cmds/houndd/main.go

@@ -6,15 +6,19 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
 	"runtime"
 	"strings"
+	"syscall"
 
 	"github.com/etsy/hound/api"
 	"github.com/etsy/hound/config"
 	"github.com/etsy/hound/searcher"
 	"github.com/etsy/hound/ui"
 )
 
+const gracefulShutdownSignal = syscall.SIGTERM
+
 var (
 	info_log  *log.Logger
 	error_log *log.Logger
@@ -46,6 +50,28 @@ func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, err
 	return searchers, true, nil
 }
 
+func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) {
+	go func() {
+		<-shutdownCh
+		info_log.Printf("Graceful shutdown requested...")
+		for _, s := range searchers {
+			s.Stop()
+		}
+
+		for _, s := range searchers {
+			s.Wait()
+		}
+
+		os.Exit(0)
+	}()
+}
+
+func registerShutdownSignal() <-chan os.Signal {
+	shutdownCh := make(chan os.Signal, 1)
+	signal.Notify(shutdownCh, gracefulShutdownSignal)
+	return shutdownCh
+}
+
 func makeTemplateData(cfg *config.Config) (interface{}, error) {
 	var data struct {
 		ReposAsJson string
@@ -98,6 +124,9 @@ func main() {
 		panic(err)
 	}
 
+	// It's not safe to be killed during makeSearchers, so register the
+	// shutdown signal here and defer processing it until we are ready.
+	shutdownCh := registerShutdownSignal()
 	idx, ok, err := makeSearchers(&cfg)
 	if err != nil {
 		log.Panic(err)
@@ -108,6 +137,8 @@ func main() {
 		info_log.Println("All indexes built!")
 	}
 
+	handleShutdown(shutdownCh, idx)
+
 	host := *flagAddr
 	if strings.HasPrefix(host, ":") {
 		host = "localhost" + host

searcher/searcher.go

@@ -29,8 +29,13 @@ type Searcher struct {
 	// It has a buffer size of 1 to allow at most one pending
 	// update at a time.
 	updateCh chan time.Time
+
+	shutdownRequested bool
+	shutdownCh        chan empty
+	doneCh            chan empty
 }
 
+type empty struct{}
 type limiter chan bool
 
 /**
@@ -141,6 +146,23 @@ func (s *Searcher) Update() bool {
 	return true
 }
 
+// Shut down the searcher cleanly, waiting for any indexing operations to complete.
+func (s *Searcher) Stop() {
+	select {
+	case s.shutdownCh <- empty{}:
+	default:
+	}
+}
+
+// Blocks until the searcher's associated goroutine is stopped.
+func (s *Searcher) Wait() {
+	<-s.doneCh
+}
+
+func (s *Searcher) completeShutdown() {
+	close(s.doneCh)
+}
+
 // Wait for either the delay period to expire or an update request to
 // arrive. Note that an empty delay will result in an infinite timeout.
 func (s *Searcher) waitForUpdate(delay time.Duration) {
@@ -153,6 +175,8 @@ func (s *Searcher) waitForUpdate(delay time.Duration) {
 	select {
 	case <-s.updateCh:
 	case <-tch:
+	case <-s.shutdownCh:
+		s.shutdownRequested = true
 	}
 }
 
@@ -393,9 +417,11 @@ func newSearcher(
 	}
 
 	s := &Searcher{
-		idx:      idx,
-		updateCh: make(chan time.Time, 1),
-		Repo:     repo,
+		idx:        idx,
+		updateCh:   make(chan time.Time, 1),
+		Repo:       repo,
+		doneCh:     make(chan empty),
+		shutdownCh: make(chan empty, 1),
 	}
 
 	go func() {
@@ -405,6 +431,7 @@ func newSearcher(
 
 		// if all forms of updating are turned off, we're done here.
 		if !repo.PollUpdatesEnabled() && !repo.PushUpdatesEnabled() {
+			s.completeShutdown()
 			return
 		}
 
@@ -414,10 +441,14 @@ func newSearcher(
 		}
 
 		for {
-
 			// Wait for a signal to proceed
 			s.waitForUpdate(delay)
 
+			if s.shutdownRequested {
+				s.completeShutdown()
+				return
+			}
+
 			// attempt to update and reindex this searcher
 			newRev, ok := updateAndReindex(s, dbpath, vcsDir, name, rev, wd, opt, lim)
 			if !ok {