Parse files in parallel and only parse buildable files

This commit introduces two changes.  The first is to parse other package
files in parallel.  The second is to only parse files that would be
considered buildable by go/build.

Using go/build.Import() is relatively slow and cannot be parallelized
(it must run before the parallel parse).  To get around using go/build,
this commit adds a package 'buildutil', which is a slimmed down version
of github.com/charlievieth/buildutil, that can quickly determine if a
package should be included.  In order for 'buildutil' to respect the
PackedContext passed in by the client - it was added as a field to
suggest.Config, which is populated by the server.

internal/buildutil/.gitignore

@@ -0,0 +1 @@
+*.txt

internal/buildutil/LICENSE

@@ -0,0 +1,30 @@
+# This package includes portions of the Go standard library.  The Go license
+# is reproduced in full below:
+
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

internal/buildutil/buildutil.go

@@ -0,0 +1,186 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package buildutil exposes some useful internal functions of the go/build.
+package buildutil
+
+import (
+	"bytes"
+	"go/build"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode"
+)
+
+// ShouldBuild reports whether it is okay to use this file with build.Context
+// ctxt and returns the package name.
+func ShouldBuild(ctxt *build.Context, path string) (string, bool) {
+	if !goodOSArchFile(ctxt, filepath.Base(path), nil) {
+		return "", false
+	}
+	var f io.ReadCloser
+	var err error
+	if fn := ctxt.OpenFile; fn != nil {
+		f, err = fn(path)
+	} else {
+		f, err = os.Open(path)
+	}
+	if err != nil {
+		return "", false
+	}
+	data, err := readImportsFast(f)
+	f.Close()
+	if err != nil {
+		return "", false
+	}
+	if !shouldBuild(ctxt, data, nil) {
+		return "", false
+	}
+	name, err := readPackageName(data)
+	return name, err == nil
+}
+
+var slashslash = []byte("//")
+
+// shouldBuild reports whether it is okay to use this file,
+// The rule is that in the file's leading run of // comments
+// and blank lines, which must be followed by a blank line
+// (to avoid including a Go package clause doc comment),
+// lines beginning with '// +build' are taken as build directives.
+//
+// The file is accepted only if each such line lists something
+// matching the file.  For example:
+//
+//	// +build windows linux
+//
+// marks the file as applicable only on Windows and Linux.
+//
+func shouldBuild(ctxt *build.Context, content []byte, allTags map[string]bool) bool {
+	// Pass 1. Identify leading run of // comments and blank lines,
+	// which must be followed by a blank line.
+	end := 0
+	p := content
+	for len(p) > 0 {
+		line := p
+		if i := bytes.IndexByte(line, '\n'); i >= 0 {
+			line, p = line[:i], p[i+1:]
+		} else {
+			p = p[len(p):]
+		}
+		line = bytes.TrimSpace(line)
+		if len(line) == 0 { // Blank line
+			end = len(content) - len(p)
+			continue
+		}
+		if !bytes.HasPrefix(line, slashslash) { // Not comment line
+			break
+		}
+	}
+	content = content[:end]
+
+	// Pass 2.  Process each line in the run.
+	p = content
+	allok := true
+	for len(p) > 0 {
+		line := p
+		if i := bytes.IndexByte(line, '\n'); i >= 0 {
+			line, p = line[:i], p[i+1:]
+		} else {
+			p = p[len(p):]
+		}
+		line = bytes.TrimSpace(line)
+		if bytes.HasPrefix(line, slashslash) {
+			line = bytes.TrimSpace(line[len(slashslash):])
+			if len(line) > 0 && line[0] == '+' {
+				// Looks like a comment +line.
+				f := strings.Fields(string(line))
+				if f[0] == "+build" {
+					ok := false
+					for _, tok := range f[1:] {
+						if match(ctxt, tok, allTags, false) {
+							ok = true
+						}
+					}
+					if !ok {
+						allok = false
+					}
+				}
+			}
+		}
+	}
+
+	return allok
+}
+
+// match reports whether the name is one of:
+//
+//	$GOOS
+//	$GOARCH
+//	cgo (if cgo is enabled)
+//	!cgo (if cgo is disabled)
+//	ctxt.Compiler
+//	!ctxt.Compiler
+//	tag (if tag is listed in ctxt.BuildTags or ctxt.ReleaseTags)
+//	!tag (if tag is not listed in ctxt.BuildTags or ctxt.ReleaseTags)
+//	a comma-separated list of any of these
+//
+func match(ctxt *build.Context, name string, allTags map[string]bool, negated bool) bool {
+	if name == "" {
+		if allTags != nil {
+			allTags[name] = true
+		}
+		return false
+	}
+	if i := strings.IndexByte(name, ','); i >= 0 {
+		// comma-separated list
+		ok1 := match(ctxt, name[:i], allTags, false)
+		ok2 := match(ctxt, name[i+1:], allTags, false)
+		return ok1 && ok2
+	}
+	if strings.HasPrefix(name, "!!") { // bad syntax, reject always
+		return false
+	}
+	if strings.HasPrefix(name, "!") { // negation
+		return len(name) > 1 && !match(ctxt, name[1:], allTags, true)
+	}
+
+	if allTags != nil {
+		allTags[name] = !negated
+	}
+
+	// Tags must be letters, digits, underscores or dots.
+	// Unlike in Go identifiers, all digits are fine (e.g., "386").
+	for _, c := range name {
+		if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' {
+			return false
+		}
+	}
+
+	// special tags
+	if ctxt.CgoEnabled && name == "cgo" {
+		return true
+	}
+	if name == ctxt.GOOS || name == ctxt.GOARCH || name == ctxt.Compiler {
+		return true
+	}
+	if ctxt.GOOS == "android" && name == "linux" {
+		return true
+	}
+
+	// other tags
+	for _, tag := range ctxt.BuildTags {
+		if tag == name {
+			return true
+		}
+	}
+	for _, tag := range ctxt.ReleaseTags {
+		if tag == name {
+			return true
+		}
+	}
+
+	return false
+}