From 64a837bbbd94a672302d6c4ede4b110ff4e11ab5 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 1 Aug 2018 09:31:11 -0700 Subject: [PATCH 1/4] use retool for linters Also fix goword and errcheck which were not being run correctly. These are now under the check-fail task. --- .gitignore | 1 + Makefile | 73 ++++++++++++++++++++++++++++++++++++++++++------------ tools.json | 41 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 tools.json diff --git a/.gitignore b/.gitignore index c92d409d2cba1..feedc554fd2dd 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ y.output profile.coverprofile explain_test cmd/explaintest/explain-test.out +_tools/ diff --git a/Makefile b/Makefile index 2ba7bc2669111..8555a0e2f49b3 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,11 @@ +PROJECT=tidb GOPATH ?= $(shell go env GOPATH) # Ensure GOPATH is set before running build process. ifeq "$(GOPATH)" "" $(error Please set the environment variable GOPATH before running `make`) endif +FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }' CURDIR := $(shell pwd) path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH))) @@ -18,9 +20,10 @@ GOVERALLS := goveralls ARCH := "`uname -s`" LINUX := "Linux" MAC := "Darwin" -PACKAGES := $$(go list ./...| grep -vE "vendor") -FILES := $$(find . -name "*.go" | grep -vE "vendor") -TOPDIRS := $$(ls -d */ | grep -vE "vendor") +PACKAGE_LIST := go list ./...| grep -vE "vendor" +PACKAGES := $$($(PACKAGE_LIST)) +PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||' +FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go" | grep -vE "vendor") GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs gofail enable) GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs gofail disable) @@ -71,30 +74,68 @@ parserlib: parser/parser.go parser/parser.go: parser/parser.y make parser -check: fmt errcheck lint vet +# This is how tools.json is generated +tool-install: + @which retool >/dev/null || go get github.com/twitchtv/retool + + # This tool can run other checks in a standardized way + retool add gopkg.in/alecthomas/gometalinter.v2 v2.0.5 + + # check spelling + # misspell works with gometalinter + retool add github.com/client9/misspell/cmd/misspell v0.3.4 + # goword adds additional capability to checking comments + retool add github.com/chzchzchz/goword a9744cb52b033fe5c269df48eeef2c954526cd79 + + # checks correctness + retool add github.com/gordonklaus/ineffassign 7bae11eba15a3285c75e388f77eb6357a2d73ee2 + retool add honnef.co/go/tools/cmd/megacheck master + retool add github.com/dnephin/govet 4a96d43e39d340b63daa8bc5576985aa599885f6 + + # slow checks + retool add github.com/kisielk/errcheck v1.1.0 + retool add github.com/securego/gosec/cmd/gosec 1.0.0 + + # linter + retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b + +check-setup: + @which retool >/dev/null 2>&1 || go get github.com/twitchtv/retool + @retool sync + +check: check-setup fmt lint vet + +# These need to be fixed before they can be ran regularly +check-fail: goword check-static check-slow fmt: @echo "gofmt (simplify)" - @ gofmt -s -l -w $(FILES) 2>&1 | grep -v "vendor|parser/parser.go" | awk '{print} END{if(NR>0) {exit 1}}' + @gofmt -s -l -w $(FILES) 2>&1 | grep -v "vendor|parser/parser.go" | $(FAIL_ON_STDOUT) goword: - go get github.com/chzchzchz/goword - @echo "goword" - @ goword $(FILES) | awk '{print} END{if(NR>0) {exit 1}}' - -errcheck: - go get github.com/kisielk/errcheck - @echo "errcheck" - @ GOPATH=$(GOPATH) errcheck -exclude errcheck_excludes.txt -blank $(PACKAGES) | grep -v "_test\.go" | awk '{print} END{if(NR>0) {exit 1}}' + retool do goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT) + +check-static: + @ # vet and fmt have problems with vendor when ran through metalinter + CGO_ENABLED=0 retool do gometalinter.v2 --disable-all --deadline 120s \ + --enable misspell \ + --enable megacheck \ + --enable ineffassign \ + $$($(PACKAGE_DIRECTORIES)) + +check-slow: + CGO_ENABLED=0 retool do gometalinter.v2 --disable-all \ + --enable errcheck \ + $$($(PACKAGE_DIRECTORIES)) + CGO_ENABLED=0 retool do gosec $$($(PACKAGE_DIRECTORIES)) lint: - go get github.com/mgechev/revive @echo "linting" - CGO_ENABLED=0 revive -formatter friendly -config revive.toml $(PACKAGES) + @CGO_ENABLED=0 retool do revive -formatter friendly -config revive.toml $(PACKAGES) vet: @echo "vet" - @ go tool vet -all -shadow $(TOPDIRS) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}' + @retool do govet -all -shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(FAIL_ON_STDOUT) clean: $(GO) clean -i ./... diff --git a/tools.json b/tools.json new file mode 100644 index 0000000000000..d3317dac96026 --- /dev/null +++ b/tools.json @@ -0,0 +1,41 @@ +{ + "Tools": [ + { + "Repository": "gopkg.in/alecthomas/gometalinter.v2", + "Commit": "46cc1ea3778b247666c2949669a3333c532fa9c6" + }, + { + "Repository": "github.com/client9/misspell/cmd/misspell", + "Commit": "7888c6b6ce89353cd98e196bce3c3f9e4cdf31f6" + }, + { + "Repository": "github.com/chzchzchz/goword", + "Commit": "a9744cb52b033fe5c269df48eeef2c954526cd79" + }, + { + "Repository": "github.com/gordonklaus/ineffassign", + "Commit": "7bae11eba15a3285c75e388f77eb6357a2d73ee2" + }, + { + "Repository": "honnef.co/go/tools/cmd/megacheck", + "Commit": "88497007e8588ea5b6baee991f74a1607e809487" + }, + { + "Repository": "github.com/dnephin/govet", + "Commit": "4a96d43e39d340b63daa8bc5576985aa599885f6" + }, + { + "Repository": "github.com/securego/gosec/cmd/gosec", + "Commit": "5fb530cda357c16175f2c049577d2030de735b28" + }, + { + "Repository": "github.com/kisielk/errcheck", + "Commit": "55d8f507faff4d6eddd0c41a3e713e2567fca4e5" + }, + { + "Repository": "github.com/mgechev/revive", + "Commit": "7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b" + } + ], + "RetoolVersion": "1.3.7" +} \ No newline at end of file From 02baf848811504f6a8a8f0ecbf811cbce79e6ff5 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 1 Aug 2018 09:32:41 -0700 Subject: [PATCH 2/4] fix goword wants proper package comments --- ast/functions_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ast/functions_test.go b/ast/functions_test.go index af1b728ebea1e..f54120c717fd2 100644 --- a/ast/functions_test.go +++ b/ast/functions_test.go @@ -10,6 +10,7 @@ // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package ast_test import ( From a5a5f4182a06b6f1dd4b87408639f7066469469e Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 1 Aug 2018 21:17:41 -0700 Subject: [PATCH 3/4] grammar --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8555a0e2f49b3..283b1abd46c1f 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ tool-install: # check spelling # misspell works with gometalinter retool add github.com/client9/misspell/cmd/misspell v0.3.4 - # goword adds additional capability to checking comments + # goword adds additional capability to check comments retool add github.com/chzchzchz/goword a9744cb52b033fe5c269df48eeef2c954526cd79 # checks correctness From c9fedb6eac7cc52acac7ba9c9503f852facb06ad Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 1 Aug 2018 21:19:54 -0700 Subject: [PATCH 4/4] tools: move into the hack/ folder --- Makefile | 26 +------------------------- hack/retool-install.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 25 deletions(-) create mode 100755 hack/retool-install.sh diff --git a/Makefile b/Makefile index 283b1abd46c1f..cb09a38c9a0ab 100644 --- a/Makefile +++ b/Makefile @@ -74,31 +74,7 @@ parserlib: parser/parser.go parser/parser.go: parser/parser.y make parser -# This is how tools.json is generated -tool-install: - @which retool >/dev/null || go get github.com/twitchtv/retool - - # This tool can run other checks in a standardized way - retool add gopkg.in/alecthomas/gometalinter.v2 v2.0.5 - - # check spelling - # misspell works with gometalinter - retool add github.com/client9/misspell/cmd/misspell v0.3.4 - # goword adds additional capability to check comments - retool add github.com/chzchzchz/goword a9744cb52b033fe5c269df48eeef2c954526cd79 - - # checks correctness - retool add github.com/gordonklaus/ineffassign 7bae11eba15a3285c75e388f77eb6357a2d73ee2 - retool add honnef.co/go/tools/cmd/megacheck master - retool add github.com/dnephin/govet 4a96d43e39d340b63daa8bc5576985aa599885f6 - - # slow checks - retool add github.com/kisielk/errcheck v1.1.0 - retool add github.com/securego/gosec/cmd/gosec 1.0.0 - - # linter - retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b - +# The retool tools.json is setup from hack/retool-install.sh check-setup: @which retool >/dev/null 2>&1 || go get github.com/twitchtv/retool @retool sync diff --git a/hack/retool-install.sh b/hack/retool-install.sh new file mode 100755 index 0000000000000..fd124f9ad3901 --- /dev/null +++ b/hack/retool-install.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script generates tools.json +# It helps record what releases/branches are being used +which retool >/dev/null || go get github.com/twitchtv/retool + +# This tool can run other checks in a standardized way +retool add gopkg.in/alecthomas/gometalinter.v2 v2.0.5 + +# check spelling +# misspell works with gometalinter +retool add github.com/client9/misspell/cmd/misspell v0.3.4 +# goword adds additional capability to check comments +retool add github.com/chzchzchz/goword a9744cb52b033fe5c269df48eeef2c954526cd79 + +# checks correctness +retool add github.com/gordonklaus/ineffassign 7bae11eba15a3285c75e388f77eb6357a2d73ee2 +retool add honnef.co/go/tools/cmd/megacheck master +retool add github.com/dnephin/govet 4a96d43e39d340b63daa8bc5576985aa599885f6 + +# slow checks +retool add github.com/kisielk/errcheck v1.1.0 +retool add github.com/securego/gosec/cmd/gosec 1.0.0 + +# linter +retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b