vgo: Loader initial commit

README.md

@@ -1,2 +1,13 @@
 ### `myitcv.io/...` mono-repo
 
+<!-- __TEMPLATE: go list -f "{{${DOLLAR}ip := .ImportPath}}{{range .Deps}}{{if (eq \"myitcv.io/vgo\" .)}}{{${DOLLAR}ip}}{{end}}{{end}}" ./...
+{{ with . }}
+Please note the following packages current rely on `vgo` with https://go-review.googlesource.com/c/vgo/+/105855 applied:
+
+```
+{{. -}}
+```
+{{end -}}
+-->
+<!-- END -->
+

_scripts/run_tests.sh

@@ -10,8 +10,11 @@ if [ "${CI:-}" == "true" ]
 then
 	go get -u golang.org/x/vgo
 	pushd $(go list -f "{{.Dir}}" golang.org/x/vgo) > /dev/null
-	git checkout -qf $VGO_COMMIT
+
+	# git checkout -qf $VGO_COMMIT
+	git fetch -q https://go.googlesource.com/vgo refs/changes/55/105855/3 && git checkout -qf FETCH_HEAD
 	go install
+
 	popd > /dev/null
 
 	# so we can access Github without hitting rate limits
@@ -63,17 +66,33 @@ do
 	then
 		./_scripts/run_tests.sh
 	else
+		if [ -f ./_scripts/pre_run_tests.sh ]
+		then
+			./_scripts/pre_run_tests.sh
+		fi
+
 		$go generate ./...
 		$go test ./...
 
-		# we can remove this once we resolve https://github.com/golang/go/issues/24661
-		$go install ./...
+		if [ -f ./_scripts/post_run_tests.sh ]
+		then
+			./_scripts/post_run_tests.sh
+		fi
 	fi
 	popd > /dev/null
 	echo "----"
 	echo ""
 done
 
+# we use regular go to list here because of https://github.com/golang/go/issues/24749;
+# this is also the reason why we need to change to the directory to do the vgo install
+for i in $(go list -f "{{if eq .Name \"main\"}}{{.Dir}}{{end}}" ./...)
+do
+	pushd $i > /dev/null
+	$go install
+	popd > /dev/null
+done
+
 echo Checking markdown files are current
 # by this point we will have mdreplace installed. Hence check that
 # committed .md files are "fresh"

cmd/mdreplace/README.md

@@ -55,7 +55,7 @@ _To see this in action, look at the [source of the
 ### Usage
 
 ```
-{{.}}
+{{. -}}
 ```
 -->
 ### Usage
@@ -69,12 +69,13 @@ Usage:
 When called with no file arguments, mdreplace works with stdin
 
 Flags:
+  -debug
+    	whether to print debug information of not
   -strip
     	whether to strip special comments from the file
   -w	whether to write back to input files (cannot be used when reading from 
   stdin)
 
-
 ```
 <!-- END -->
 

cmd/mdreplace/mdreplace.go

@@ -50,6 +50,7 @@ import (
 var (
 	fWrite = flag.Bool("w", false, "whether to write back to input files (cannot be used when reading from stdin)")
 	fStrip = flag.Bool("strip", false, "whether to strip special comments from the file")
+	fDebug = flag.Bool("debug", false, "whether to print debug information of not")
 )
 
 //go:generate pkgconcat -out gen_cliflag.go myitcv.io/_tmpls/cliflag
@@ -150,7 +151,7 @@ func infof(format string, args ...interface{}) {
 }
 
 func debugf(format string, args ...interface{}) {
-	if debug {
+	if debug || *fDebug {
 		fmt.Fprintf(os.Stderr, format, args...)
 	}
 }

vgo/go.mod

@@ -0,0 +1,3 @@
+module "myitcv.io/vgo"
+
+require "golang.org/x/net" v0.0.0-20180406214816-61147c48b25b

vgo/loader.go

@@ -0,0 +1,155 @@
+// package vgo provides some utility types, functions etc to support vgo
+package vgo // import "myitcv.io/vgo"
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"go/build"
+	"go/importer"
+	"go/types"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+// Loader supports loading of vgo-build cached packages. NewLoader returns a
+// correctly initialised *Loader.  A Loader must not be copied once created.
+type Loader struct {
+	mu sync.Mutex
+
+	dir       string
+	compiler  string
+	resCache  map[string]map[string]*types.Package
+	importers map[string]types.ImporterFrom
+	test      bool
+}
+
+func NewLoader(dir string) *Loader {
+	res := &Loader{
+		dir:       dir,
+		compiler:  "gc",
+		resCache:  make(map[string]map[string]*types.Package),
+		importers: make(map[string]types.ImporterFrom),
+	}
+
+	return res
+}
+
+func NewTestLoader(dir string) *Loader {
+	res := NewLoader(dir)
+	res.test = true
+	return res
+}
+
+var _ types.ImporterFrom = new(Loader)
+
+func (l *Loader) Import(path string) (*types.Package, error) {
+	return nil, fmt.Errorf("did not expect this method to be used; we implement types.ImporterFrom")
+}
+
+func (l *Loader) ImportFrom(path, dir string, mode types.ImportMode) (*types.Package, error) {
+	if mode != 0 {
+		panic(fmt.Errorf("unknown types.ImportMode %v", mode))
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	// TODO optimise mutex usage later... keep it simple for now
+	dirCache, ok := l.resCache[dir]
+	if ok {
+		if p, ok := dirCache[path]; ok {
+			return p, nil
+		}
+	} else {
+		// ensures dirCache is now set
+		dirCache = make(map[string]*types.Package)
+		l.resCache[dir] = dirCache
+	}
+
+	// res cache miss
+	imp, ok := l.importers[dir]
+	if !ok {
+		// we need to load the results for this dir and build an importer
+
+		// resolve the package found in dir
+		bpkg, err := build.ImportDir(dir, 0)
+		if err != nil {
+			return nil, fmt.Errorf("unable to resolve %v to a package: %v", dir, err)
+		}
+
+		// now run vgo depbuildlist with the import path
+		args := []string{"vgo", "deplist", "-build"}
+
+		if l.test {
+			args = append(args, "-test")
+		}
+
+		args = append(args, bpkg.ImportPath)
+
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Dir = l.dir
+
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return nil, fmt.Errorf("unable to run %v: %v [%q]", strings.Join(cmd.Args, " "), err, string(out))
+		}
+
+		// parse the JSON
+
+		lookup := make(map[string]string)
+
+		dec := json.NewDecoder(bytes.NewBuffer(out))
+
+		for {
+			var d struct {
+				ImportPath  string
+				PackageFile string
+			}
+
+			if err := dec.Decode(&d); err != nil {
+				if err == io.EOF {
+					break
+				}
+
+				return nil, fmt.Errorf("failed to parse vgo output: %v\noutput was:\n%v", err, string(out))
+			}
+
+			lookup[d.ImportPath] = d.PackageFile
+		}
+
+		i := importer.For(l.compiler, func(path string) (io.ReadCloser, error) {
+			file, ok := lookup[path]
+			if !ok {
+				return nil, fmt.Errorf("failed to resolve import path %q", path)
+			}
+
+			f, err := os.Open(file)
+			if err != nil {
+				return nil, fmt.Errorf("failed to open file %v: %v", file, err)
+			}
+
+			return f, nil
+		})
+
+		from, ok := i.(types.ImporterFrom)
+		if !ok {
+			return nil, fmt.Errorf("failed to get an importer that implements go/types.ImporterFrom; got %T", i)
+		}
+
+		imp = from
+		l.importers[dir] = imp
+	}
+
+	p, err := imp.ImportFrom(path, dir, mode)
+	if err != nil {
+		return nil, fmt.Errorf("failed to import: %v", err)
+	}
+
+	dirCache[path] = p
+
+	return p, nil
+}

vgo/loader_test.go

@@ -0,0 +1,49 @@
+package vgo_test
+
+import (
+	"os"
+	"testing"
+
+	// import a non-standard library package for its side effects.
+	// vgo will then detect this
+	_ "golang.org/x/net/html"
+	"myitcv.io/vgo"
+)
+
+func TestLoader(t *testing.T) {
+	// given the side-effect import above, we can now create a Loader
+	// to load "golang.org/x/net/html" in the context of the current
+	// directory
+
+	l := vgo.NewTestLoader(".")
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("failed to get cwd: %v", cwd)
+	}
+
+	cases := []string{
+		"golang.org/x/net/html",
+
+		// this is a dependency of x/net/html; hence an indirect
+		// test dependency of vgo_test
+		"golang.org/x/net/html/atom",
+	}
+
+	for _, c := range cases {
+		t.Run(c, func(t *testing.T) {
+			p, err := l.ImportFrom(c, cwd, 0)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if p == nil {
+				t.Fatal("expected response; got nil")
+			}
+
+			if v := p.Path(); v != c {
+				t.Fatalf("expected ImportPath %q; got %q", c, v)
+			}
+		})
+	}
+}