diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c899443..71bf277b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,10 +33,14 @@ jobs: restore-keys: ${{ runner.os }}-protocompile-ci- - name: Test run: make test + - name: Benchmarks + # no need to run benchmarks for every Go version + if: matrix.go-version == '1.19.x' + run: make benchmarks - name: Lint # Often, lint & gofmt guidelines depend on the Go version. To prevent # conflicting guidance, run only on the most recent supported version. # For the same reason, only check generated code on the most recent # supported version. if: matrix.go-version == '1.19.x' - run: make checkgenerate && make lint + run: make checkgenerate lint diff --git a/.golangci.yml b/.golangci.yml index 006366b1..8ec8d758 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -85,3 +85,8 @@ issues: # Don't ban use of fmt.Errorf to create new errors, but the remaining # checks from err113 are useful. - "err113: do not define dynamic errors.*" + exclude-rules: + # Benchmarks can't be run in parallel + - path: benchmark_test\.go + linters: + - paralleltest \ No newline at end of file diff --git a/Makefile b/Makefile index cf98def8..47d87c7d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ TOOLS_MOD_DIR := ./internal/tools UNAME_OS := $(shell uname -s) UNAME_ARCH := $(shell uname -m) +# NB: this must be kept in sync with constant in internal/benchmarks. PROTOC_VERSION ?= 21.7 PROTOC_DIR := $(abspath ./internal/testdata/protoc/$(PROTOC_VERSION)) PROTOC := $(PROTOC_DIR)/bin/protoc @@ -50,6 +51,10 @@ clean: ## Delete intermediate build artifacts test: build ## Run unit tests $(GO) test -vet=off -race -cover ./... +.PHONY: benchmarks +benchmarks: build ## Run benchmarks + cd internal/benchmarks && $(GO) test -bench=. -benchmem -v ./... + .PHONY: build build: generate ## Build all packages $(GO) build ./... @@ -60,12 +65,14 @@ install: ## Install all binaries .PHONY: lint lint: $(BIN)/golangci-lint ## Lint Go - $(GO) vet ./... + $(GO) vet ./... ./internal/benchmarks/... $(BIN)/golangci-lint run + cd internal/benchmarks && $(BIN)/golangci-lint run .PHONY: lintfix lintfix: $(BIN)/golangci-lint ## Automatically fix some lint errors $(BIN)/golangci-lint run --fix + cd internal/benchmarks && $(BIN)/golangci-lint run --fix .PHONY: generate generate: $(BIN)/license-header $(BIN)/goyacc test-descriptors ## Regenerate code and licenses diff --git a/compiler.go b/compiler.go index c97b95cd..e188f46d 100644 --- a/compiler.go +++ b/compiler.go @@ -80,6 +80,13 @@ type Compiler struct { // concludes. Similarly, if they already have source code info but this flag // is false, existing info will be left in place. SourceInfoMode SourceInfoMode + + // If true, ASTs are retained in compilation results for which an AST was + // constructed. So any linker.Result value in the resulting compiled files + // will have an AST, in addition to descriptors. If left false, the AST + // will be removed as soon as it's no longer needed. This can help reduce + // total memory usage for operations involving a large number of files. + RetainASTs bool } // SourceInfoMode indicates how source code info is generated by a Compiler. @@ -518,6 +525,10 @@ func (t *task) link(parseRes parser.Result, deps linker.Files) (linker.File, err } file.PopulateSourceCodeInfo() } + + if !t.e.c.RetainASTs { + file.RemoveAST() + } return file, nil } diff --git a/go.work b/go.work new file mode 100644 index 00000000..de1c41a8 --- /dev/null +++ b/go.work @@ -0,0 +1,7 @@ +go 1.19 + +use ( + . + ./internal/benchmarks + ./internal/tools +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 00000000..8cf92817 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,183 @@ +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/benchmarks/benchmark_test.go b/internal/benchmarks/benchmark_test.go new file mode 100644 index 00000000..278b9289 --- /dev/null +++ b/internal/benchmarks/benchmark_test.go @@ -0,0 +1,524 @@ +// Copyright 2020-2022 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmarks + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jhump/protoreflect/desc" + "github.com/jhump/protoreflect/desc/protoparse" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" + + "github.com/bufbuild/protocompile" + "github.com/bufbuild/protocompile/ast" + "github.com/bufbuild/protocompile/linker" + "github.com/bufbuild/protocompile/parser" + "github.com/bufbuild/protocompile/protoutil" + "github.com/bufbuild/protocompile/reporter" +) + +const ( + // NB: this must be kept in sync with PROTOC_VERSION in Makefile. + protocVersion = "21.7" + + googleapisCommit = "cb6fbe8784479b22af38c09a5039d8983e894566" +) + +var ( + protocPath = fmt.Sprintf("../testdata/protoc/%s/bin/protoc", protocVersion) + googleapisURI = fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s.tar.gz", googleapisCommit) + googleapisDir string + googleapisSources []string +) + +func TestMain(m *testing.M) { + if runtime.GOOS == "windows" { + protocPath += ".exe" + } + if info, err := os.Stat(protocPath); err != nil { + if os.IsNotExist(err) { + _, _ = fmt.Fprintf(os.Stderr, "Path %s not found. Run `make generate` in the project root first.\n", protocPath) + } else { + _, _ = fmt.Fprintf(os.Stderr, "Error querying for path %s: %v\n", protocPath, err) + } + os.Exit(1) + } else if info.IsDir() { + _, _ = fmt.Fprintf(os.Stderr, "Path %s is a directory but expecting an executable file.\n", protocPath) + os.Exit(1) + } + + var stat int + defer func() { + os.Exit(stat) + }() + // After this point, we can set stat and return instead of directly calling os.Exit. + // That allows deferred functions to execute, to perform cleanup, before exiting. + + dir, err := os.MkdirTemp("", "testdownloads") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Could not create temporary directory: %v\n", err) + stat = 1 + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Failed to cleanup temp directory %s: %v\n", dir, err) + } + }() + + if err := downloadAndExpand(googleapisURI, dir); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Failed to download and expand googleapis: %v\n", err) + stat = 1 + return + } + + googleapisDir = filepath.Join(dir, fmt.Sprintf("googleapis-%s", googleapisCommit)) + "/" + var sourceSize int64 + err = filepath.Walk(googleapisDir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".proto") { + relPath := strings.TrimPrefix(path, googleapisDir) + googleapisSources = append(googleapisSources, relPath) + sourceSize += info.Size() + } + return nil + }) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Failed to enumerate googleapis source files: %v\n", err) + stat = 1 + return + } + sort.Strings(googleapisSources) + fmt.Printf("%d total source files found in googleapis (%d bytes).\n", len(googleapisSources), sourceSize) + + stat = m.Run() +} + +func downloadAndExpand(url, targetDir string) (e error) { + start := time.Now() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if resp.Body != nil { + defer func() { + if err = resp.Body.Close(); err != nil && e == nil { + e = err + } + }() + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading %s resulted in status code %s", url, resp.Status) + } + if err := os.MkdirAll(targetDir, 0777); err != nil { + return err + } + f, err := os.CreateTemp(targetDir, "testdownload.*.tar.gz") + if err != nil { + return err + } + defer func() { + if f != nil { + if err := f.Close(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "warning: failed to close %s: %v\n", f.Name(), err) + } + } + }() + n, err := io.Copy(f, resp.Body) + if err != nil { + return err + } + fmt.Printf("Downloaded %v; %d bytes (%v).\n", url, n, time.Since(start)) + archiveName := f.Name() + if err := f.Close(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "warning: failed to close %s: %v\n", f.Name(), err) + } + f = nil + + f, err = os.OpenFile(archiveName, os.O_RDONLY, 0) + if err != nil { + return err + } + + gzr, err := gzip.NewReader(f) + if err != nil { + return err + } + defer func() { + if err = gzr.Close(); err != nil && e == nil { + e = err + } + }() + + tr := tar.NewReader(gzr) + count := 0 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if hdr == nil { + continue + } + target := filepath.Join(targetDir, hdr.Name) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0777); err != nil { + return err + } + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + return err + } + count++ + default: + // skip anything else + } + } + fmt.Printf("Expanded archive into %d files.\n", count) + + return nil +} + +func BenchmarkGoogleapisProtocompile(b *testing.B) { + benchmarkGoogleapisProtocompile(b, false, func() *protocompile.Compiler { + return &protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoExtraComments, + // leave MaxParallelism unset to let it use all cores available + } + }) +} + +func BenchmarkGoogleapisProtocompileCanonical(b *testing.B) { + benchmarkGoogleapisProtocompile(b, true, func() *protocompile.Compiler { + return &protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoStandard, + // leave MaxParallelism unset to let it use all cores available + } + }) +} + +func BenchmarkGoogleapisProtocompileNoSourceInfo(b *testing.B) { + benchmarkGoogleapisProtocompile(b, false, func() *protocompile.Compiler { + return &protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoNone, + // leave MaxParallelism unset to let it use all cores available + } + }) +} + +func benchmarkGoogleapisProtocompile(b *testing.B, canonicalBytes bool, factory func() *protocompile.Compiler) { + for i := 0; i < b.N; i++ { + benchmarkProtocompile(b, factory(), googleapisSources, canonicalBytes) + } +} + +func benchmarkProtocompile(b *testing.B, c *protocompile.Compiler, sources []string, canonicalBytes bool) { + fds, err := c.Compile(context.Background(), sources...) + require.NoError(b, err) + var fdSet descriptorpb.FileDescriptorSet + fdSet.File = make([]*descriptorpb.FileDescriptorProto, len(fds)) + for i, fd := range fds { + if canonicalBytes { + fdSet.File[i] = fd.(linker.Result).CanonicalProto() + } else { + fdSet.File[i] = protoutil.ProtoFromFileDescriptor(fd) + } + } + // protoc is writing output to file descriptor set, so we should, too + writeToNull(b, &fdSet) +} + +func BenchmarkGoogleapisProtoparse(b *testing.B) { + benchmarkGoogleapisProtoparse(b, func() *protoparse.Parser { + return &protoparse.Parser{ + ImportPaths: []string{googleapisDir}, + IncludeSourceCodeInfo: true, + } + }) +} + +func BenchmarkGoogleapisProtoparseNoSourceInfo(b *testing.B) { + benchmarkGoogleapisProtoparse(b, func() *protoparse.Parser { + return &protoparse.Parser{ + ImportPaths: []string{googleapisDir}, + IncludeSourceCodeInfo: false, + } + }) +} + +func benchmarkGoogleapisProtoparse(b *testing.B, factory func() *protoparse.Parser) { + par := runtime.GOMAXPROCS(-1) + cpus := runtime.NumCPU() + if par > cpus { + par = cpus + } + for i := 0; i < b.N; i++ { + // Buf currently batches files into chunks and then runs all chunks in parallel + chunks := make([][]string, par) + j := 0 + total := 0 + for ch := 0; ch < par; ch++ { + chunkStart := j + chunkEnd := (ch + 1) * len(googleapisSources) / par + chunks[ch] = googleapisSources[chunkStart:chunkEnd] + j = chunkEnd + total += len(chunks[ch]) + } + require.Equal(b, total, len(googleapisSources)) + var wg sync.WaitGroup + results := make([][]*desc.FileDescriptor, par) + errors := make([]error, par) + for ch, chunk := range chunks { + ch, chunk := ch, chunk + wg.Add(1) + go func() { + defer wg.Done() + p := factory() + results[ch], errors[ch] = p.ParseFiles(chunk...) + }() + } + wg.Wait() + for _, err := range errors { + require.NoError(b, err) + } + var fdSet descriptorpb.FileDescriptorSet + fdSet.File = make([]*descriptorpb.FileDescriptorProto, 0, len(googleapisSources)) + for _, chunk := range results { + for _, fd := range chunk { + fdSet.File = append(fdSet.File, fd.AsFileDescriptorProto()) + } + } + writeToNull(b, &fdSet) + } +} + +func BenchmarkGoogleapisProtoc(b *testing.B) { + benchmarkGoogleapisProtoc(b, "--include_source_info") +} + +func BenchmarkGoogleapisProtocNoSourceInfo(b *testing.B) { + benchmarkGoogleapisProtoc(b) +} + +func benchmarkGoogleapisProtoc(b *testing.B, extraArgs ...string) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + args := make([]string, 0, len(googleapisSources)+5) + args = append(args, "-I", googleapisDir, "-o", os.DevNull) + args = append(args, extraArgs...) + args = append(args, googleapisSources...) + cmd := exec.Command(protocPath, args...) + cmd.Stdin = nil + cmd.Stdout = nil + var errBuffer bytes.Buffer + cmd.Stderr = &errBuffer + + err := cmd.Run() + if err != nil { + _, _ = os.Stderr.Write(errBuffer.Bytes()) + b.Fatalf("failed to invoke protoc: %v", err) + } + } + }) +} + +func BenchmarkGoogleapisProtocompileSingleThreaded(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + c := &protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoExtraComments, + // to really test performance compared to protoc and protoparse, we + // need to run a single-threaded compile + MaxParallelism: 1, + } + benchmarkProtocompile(b, c, googleapisSources, false) + } + }) +} + +func BenchmarkGoogleapisProtoparseSingleThreaded(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + p := protoparse.Parser{ + ImportPaths: []string{googleapisDir}, + IncludeSourceCodeInfo: true, + } + fds, err := p.ParseFiles(googleapisSources...) + require.NoError(b, err) + var fdSet descriptorpb.FileDescriptorSet + fdSet.File = make([]*descriptorpb.FileDescriptorProto, len(fds)) + for i, fd := range fds { + fdSet.File[i] = fd.AsFileDescriptorProto() + } + writeToNull(b, &fdSet) + } + }) +} + +func writeToNull(b *testing.B, fds *descriptorpb.FileDescriptorSet) { + f, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + b.Fatalf("failed to open output file %s: %v", os.DevNull, err) + } + defer func() { + if err := f.Close(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "warning: failed to close %s: %v\n", f.Name(), err) + } + }() + data, err := proto.Marshal(fds) + if err != nil { + b.Fatalf("failed to marshal file descriptor set: %v", err) + } + _, err = f.Write(data) + if err != nil { + b.Fatalf("failed to write file descriptor set to file: %v", err) + } +} + +func TestGoogleapisProtocompileResultMemory(t *testing.T) { + c := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoExtraComments, + } + fds, err := c.Compile(context.Background(), googleapisSources...) + require.NoError(t, err) + measure(t, fds) +} + +func TestGoogleapisProtocompileResultMemoryNoSourceInfo(t *testing.T) { + c := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{googleapisDir}, + }), + SourceInfoMode: protocompile.SourceInfoNone, + } + fds, err := c.Compile(context.Background(), googleapisSources...) + require.NoError(t, err) + measure(t, fds) +} + +func TestGoogleapisProtocompileASTMemory(t *testing.T) { + var asts []*ast.FileNode + for _, file := range googleapisSources { + func() { + f, err := os.OpenFile(filepath.Join(googleapisDir, file), os.O_RDONLY, 0) + require.NoError(t, err) + defer func() { + if err := f.Close(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "warning: failed to close %s: %v\n", f.Name(), err) + } + }() + h := reporter.NewHandler(nil) + ast, err := parser.Parse(file, f, h) + require.NoError(t, err) + asts = append(asts, ast) + }() + } + measure(t, asts) +} + +func TestGoogleapisProtoparseResultMemory(t *testing.T) { + p := protoparse.Parser{ + ImportPaths: []string{googleapisDir}, + IncludeSourceCodeInfo: true, + } + fds, err := p.ParseFiles(googleapisSources...) + require.NoError(t, err) + measure(t, fds) +} + +func TestGoogleapisProtoparseResultMemoryNoSourceInfo(t *testing.T) { + p := protoparse.Parser{ + ImportPaths: []string{googleapisDir}, + IncludeSourceCodeInfo: false, + } + fds, err := p.ParseFiles(googleapisSources...) + require.NoError(t, err) + measure(t, fds) +} + +func TestGoogleapisProtoparseASTMemory(t *testing.T) { + p := protoparse.Parser{ + IncludeSourceCodeInfo: true, + } + // NB: ParseToAST fails to respect import paths, so we have to pass full names + filenames := make([]string, len(googleapisSources)) + for i := range googleapisSources { + filenames[i] = filepath.Join(googleapisDir, googleapisSources[i]) + } + asts, err := p.ParseToAST(filenames...) + require.NoError(t, err) + measure(t, asts) +} + +func measure(t *testing.T, v any) { + // log heap allocations + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + t.Logf("(heap used: %d bytes)", m.Alloc) + + // and then try to directly measure just the given value + mt := newMeasuringTape() + mt.measure(reflect.ValueOf(v)) + t.Logf("memory used: %d bytes", mt.memoryUsed()) +} diff --git a/internal/benchmarks/go.mod b/internal/benchmarks/go.mod new file mode 100644 index 00000000..7a6fbbf3 --- /dev/null +++ b/internal/benchmarks/go.mod @@ -0,0 +1,21 @@ +module github.com/jhump/protocompile/internal/benchmarks + +go 1.19 + +require ( + github.com/bufbuild/protocompile v0.0.0-20221004230924-06a336f5b6be + github.com/igrmk/treemap/v2 v2.0.1 + github.com/jhump/protoreflect v1.13.0 + github.com/stretchr/testify v1.8.0 + google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/exp v0.0.0-20220317015231-48e79f11773a // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/internal/benchmarks/go.sum b/internal/benchmarks/go.sum new file mode 100644 index 00000000..c2ad6fe6 --- /dev/null +++ b/internal/benchmarks/go.sum @@ -0,0 +1,128 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bufbuild/protocompile v0.0.0-20221004230924-06a336f5b6be h1:nj/vRT4P6Kz2Hr8lvyYe3EgTKfg9fNMJKHaZZXHvt9g= +github.com/bufbuild/protocompile v0.0.0-20221004230924-06a336f5b6be/go.mod h1:ix/MMMdsT3fzxfw91dvbfzKW3fRRnuPCP47kpAm5m/4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/igrmk/treemap/v2 v2.0.1 h1:Jhy4z3yhATvYZMWCmxsnHO5NnNZBdueSzvxh6353l+0= +github.com/igrmk/treemap/v2 v2.0.1/go.mod h1:PkTPvx+8OHS8/41jnnyVY+oVsfkaOUZGcr+sfonosd4= +github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= +github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= +github.com/jhump/protoreflect v1.13.0 h1:zrrZqa7JAc2YGgPSzZZkmUXJ5G6NRPdxOg/9t7ISImA= +github.com/jhump/protoreflect v1.13.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220317015231-48e79f11773a h1:DAzrdbxsb5tXNOhMCSwF7ZdfMbW46hE9fSVO6BsmUZM= +golang.org/x/exp v0.0.0-20220317015231-48e79f11773a/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk= +google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/benchmarks/measure.go b/internal/benchmarks/measure.go new file mode 100644 index 00000000..c474bd76 --- /dev/null +++ b/internal/benchmarks/measure.go @@ -0,0 +1,214 @@ +// Copyright 2020-2022 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmarks + +import ( + "math/bits" + "reflect" + "unsafe" + + "github.com/igrmk/treemap/v2" +) + +type measuringTape struct { + bst *treemap.TreeMap[uintptr, uint64] + other uint64 +} + +func newMeasuringTape() *measuringTape { + return &measuringTape{ + bst: treemap.New[uintptr, uint64](), + } +} + +func (t *measuringTape) insert(start uintptr, length uint64) bool { + if start == 0 { + // nil ptr + return false + } + end := start + uintptr(length) + iter := t.bst.LowerBound(start) + if !iter.Valid() { + // tree is empty or all entries are too low to overlap + t.bst.Set(end, length) + return true + } + entryEnd := iter.Key() + entryStart := entryEnd - uintptr(iter.Value()) + if entryStart > end { + // range does not exist; add it + t.bst.Set(end, length) + return true + } + if entryStart <= start && entryEnd >= end { + // range is entirely encompassed in existing entry + return false + } + + // navigate back to find the first overlapping range and push + // start out if needed to encompass all overlaps + first := t.bst.Iterator().Key() + for entryStart > start { + if iter.Key() == first { + // can go no further + break + } + iter.Prev() + if iter.Key() < start { + // gone back too far + break + } + entryStart = iter.Key() - uintptr(iter.Value()) + } + if entryStart < start { + start = entryStart + } + + // find last overlapping range + if entryEnd < end { + for entryEnd < end { + // remove overlaps that will be replaced with + // new, larger, encompassing range + t.bst.Del(entryEnd) + + // Iterator doesn't like concurrent removal of node. So after + // Del above, we can't call Next; we have to re-search the tree + // for the next node. + iter = t.bst.LowerBound(entryEnd) + if !iter.Valid() { + // can go no further + break + } + st := iter.Key() - uintptr(iter.Value()) + if st > end { + // gone too far + break + } + entryEnd = iter.Key() + } + } + if entryEnd > end { + end = entryEnd + } + + t.bst.Set(end, uint64(end-start)) + return true +} + +func (t *measuringTape) memoryUsed() uint64 { + iter := t.bst.Iterator() + var total uint64 + for iter.Valid() { + total += iter.Value() + iter.Next() + } + return total + t.other +} + +func (t *measuringTape) measure(value reflect.Value) { + // We only need to measure outbound references. So we don't care about the size of the pointer itself + // if value is a pointer, since that is either passed by value (not on heap) or accounted for in the + // type that contains the pointer (which we'll have already measured). + + switch value.Kind() { + case reflect.Pointer: + if !t.insert(value.Pointer(), uint64(value.Type().Elem().Size())) { + return + } + t.measure(value.Elem()) + + case reflect.Slice: + if !t.insert(value.Pointer(), uint64(value.Cap())*uint64(value.Type().Elem().Size())) { + return + } + for i := 0; i < value.Len(); i++ { + t.measure(value.Index(i)) + } + + case reflect.Chan: + if !t.insert(value.Pointer(), uint64(value.Cap())*uint64(value.Type().Elem().Size())) { + return + } + // no way to query for objects in the channel's buffer :( + + case reflect.Map: + const mapHdrSz = 48 // estimate based on struct hmap in runtime/map.go + if !t.insert(value.Pointer(), mapHdrSz) { + return + } + + // Can't really get pointers to bucket arrays, + // so we estimate their size and add them via t.other. + buckets := numBuckets(value.Len()) + // estimate based on struct bmap in runtime/map.go + bucketSz := uint64(8 * (value.Type().Key().Size() + value.Type().Elem().Size() + 1)) + t.other += uint64(buckets) * bucketSz + + for iter := value.MapRange(); iter.Next(); { + t.measure(iter.Key()) + t.measure(iter.Value()) + } + + case reflect.Interface: + v := value.Elem() + if v.IsValid() { + if !isReference(v.Kind()) { + t.other += uint64(v.Type().Size()) + } + t.measure(v) + } + + case reflect.String: + str := value.String() + hdr := (*reflect.StringHeader)(unsafe.Pointer(&str)) + t.insert(hdr.Data, uint64(hdr.Len)) + + case reflect.Struct: + for i := 0; i < value.NumField(); i++ { + t.measure(value.Field(i)) + } + + default: + // nothing to do + } +} + +func numBuckets(mapSize int) int { + // each bucket holds 8 entries + buckets := mapSize / 8 + if mapSize > buckets*8 { + buckets++ + } + // Number of buckets is a power of two (map doubles each + // time it grows). + highestBit := 63 - bits.LeadingZeros64(uint64(buckets)) + if highestBit >= 0 { + powerOf2 := 1 << highestBit + if buckets > powerOf2 { + powerOf2 <<= 1 + } + buckets = powerOf2 + } + return buckets +} + +func isReference(k reflect.Kind) bool { + switch k { + case reflect.Pointer, reflect.Chan, reflect.Map, reflect.Func: + return true + default: + return false + } +} diff --git a/internal/benchmarks/measure_test.go b/internal/benchmarks/measure_test.go new file mode 100644 index 00000000..517909dc --- /dev/null +++ b/internal/benchmarks/measure_test.go @@ -0,0 +1,157 @@ +// Copyright 2020-2022 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmarks + +import ( + "math/bits" + "reflect" + "testing" + + "github.com/igrmk/treemap/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMeasuringTapeInsert(t *testing.T) { + t.Parallel() + + mt := newMeasuringTape() + assert.True(t, mt.insert(100, 300)) // 100 -> 400 + verifyMap(t, mt.bst, 100, 400) + + // wholly contained + assert.False(t, mt.insert(100, 300)) + assert.False(t, mt.insert(150, 200)) + + // extends range start + assert.True(t, mt.insert(50, 300)) // 50 -> 350 + verifyMap(t, mt.bst, 50, 400) + + // extends range end + assert.True(t, mt.insert(300, 175)) // 300 -> 475 + verifyMap(t, mt.bst, 50, 475) + + // new range above + assert.True(t, mt.insert(1500, 100)) // 1500 -> 1600 + verifyMap(t, mt.bst, 50, 475, 1500, 1600) + + // new range below + assert.True(t, mt.insert(10, 10)) // 10 -> 20 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 1600) + + // new range above + assert.True(t, mt.insert(25000, 50000)) // 25,000 -> 75,000 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 1600, 25000, 75000) + + // new interior range + assert.True(t, mt.insert(1700, 300)) // 1700 -> 2000 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 1600, 1700, 2000, 25000, 75000) + + // new interior range + assert.True(t, mt.insert(2100, 300)) // 2100 -> 2400 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 1600, 1700, 2000, 2100, 2400, 25000, 75000) + + // matches range boundary, extends end + assert.True(t, mt.insert(2400, 100)) // 2400 -> 2500 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 1600, 1700, 2000, 2100, 2500, 25000, 75000) + + // matches both adjacent range boundaries, collapses + assert.True(t, mt.insert(1600, 100)) // 1600 -> 1700 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 2000, 2100, 2500, 25000, 75000) + + // matches range boundary, extends start + assert.True(t, mt.insert(24000, 1000)) // 24,000 -> 25,000 + verifyMap(t, mt.bst, 10, 20, 50, 475, 1500, 2000, 2100, 2500, 24000, 75000) + + // encompasses many ranges, collapses + assert.True(t, mt.insert(10, 3000)) // 10 -> 3010 + verifyMap(t, mt.bst, 10, 3010, 24000, 75000) + + // wholly contained + assert.False(t, mt.insert(1500, 1510)) // 1500 -> 3010 + + mt.other = 99 + assert.Equal(t, 54099, int(mt.memoryUsed())) +} + +func TestMeasuringTapeMeasure(t *testing.T) { + t.Parallel() + + mt := newMeasuringTape() + bytes := make([]byte, 1000000) + mt.measure(reflect.ValueOf(bytes)) + require.Equal(t, uint64(1000000), mt.memoryUsed()) + // these do nothing since they are part of already-measured slice + mt.measure(reflect.ValueOf(bytes[0:10])) + mt.measure(reflect.ValueOf(bytes[1000:10000])) + require.Equal(t, uint64(1000000), mt.memoryUsed()) + + int64s := make([]int64, 1000000) + mt.measure(reflect.ValueOf(int64s)) + require.Equal(t, uint64(9000000), mt.memoryUsed()) + + int64ptrs := make([]*int64, 1000000) + for i := range int64ptrs { + int64ptrs[i] = &int64s[i] + } + mt.measure(reflect.ValueOf(int64ptrs)) + // increase is only the size of slice, not pointed-to values, since all pointers + // point to locations in already-measured slice above + ptrsSz := uint64(1000000 * reflect.TypeOf(uintptr(0)).Size()) + require.Equal(t, 9000000+ptrsSz, mt.memoryUsed()) +} + +func verifyMap(t *testing.T, tree *treemap.TreeMap[uintptr, uint64], ranges ...uintptr) { + t.Helper() + require.Equal(t, 0, len(ranges)%2, "ranges must be even number of values") + + iter := tree.Iterator() + for i := 0; i < len(ranges); i += 2 { + require.True(t, iter.Valid()) + entryEnd := iter.Key() + entryStart := entryEnd - uintptr(iter.Value()) + type pair struct { + start, end uintptr + } + expected := pair{ranges[i], ranges[i+1]} + actual := pair{entryStart, entryEnd} + require.Equal(t, expected, actual) + iter.Next() + } +} + +func TestNumBuckets(t *testing.T) { + t.Parallel() + + assert.Equal(t, 0, numBuckets(0)) + assert.Equal(t, 1, numBuckets(8)) + assert.Equal(t, 2, numBuckets(9)) + assert.Equal(t, 2, numBuckets(16)) + assert.Equal(t, 4, numBuckets(17)) + assert.Equal(t, 4, numBuckets(32)) + assert.Equal(t, 8, numBuckets(33)) + + check := func(sz int) { + b := numBuckets(sz) + // power of 2 + assert.Equal(t, 1, bits.OnesCount(uint(b))) + // that fits given size (each bucket holds 8 entries) + assert.True(t, b*4 < sz) + assert.True(t, b*8 >= sz) + } + check(7364) + check(1234567) + check(918373645623) +} diff --git a/linker/descriptors.go b/linker/descriptors.go index c61a6ada..361f3b6f 100644 --- a/linker/descriptors.go +++ b/linker/descriptors.go @@ -76,6 +76,11 @@ var _ protoreflect.FileDescriptor = (*result)(nil) var _ Result = (*result)(nil) var _ protoutil.DescriptorProtoWrapper = (*result)(nil) +func (r *result) RemoveAST() { + r.Result = parser.ResultWithoutAST(r.FileDescriptorProto()) + r.optionQualifiedNames = nil +} + func (r *result) AsProto() proto.Message { return r.FileDescriptorProto() } @@ -157,11 +162,11 @@ func (r *result) SourceLocations() protoreflect.SourceLocations { return &r.srcLocations } -func computeSourceLocIndex(locs []protoreflect.SourceLocation) map[interface{}]protoreflect.SourceLocation { - index := map[interface{}]protoreflect.SourceLocation{} - for _, loc := range locs { +func computeSourceLocIndex(locs []protoreflect.SourceLocation) map[interface{}]int { + index := map[interface{}]int{} + for i, loc := range locs { if loc.Next == 0 { - index[pathKey(loc.Path)] = loc + index[pathKey(loc.Path)] = i } } return index @@ -373,7 +378,7 @@ type srcLocs struct { protoreflect.SourceLocations file *result locs []protoreflect.SourceLocation - index map[interface{}]protoreflect.SourceLocation + index map[interface{}]int } func (s *srcLocs) Len() int { @@ -385,7 +390,11 @@ func (s *srcLocs) Get(i int) protoreflect.SourceLocation { } func (s *srcLocs) ByPath(p protoreflect.SourcePath) protoreflect.SourceLocation { - return s.index[pathKey(p)] + index, ok := s.index[pathKey(p)] + if !ok { + return protoreflect.SourceLocation{} + } + return s.locs[index] } func (s *srcLocs) ByDescriptor(d protoreflect.Descriptor) protoreflect.SourceLocation { diff --git a/linker/linker.go b/linker/linker.go index 5acce786..c9e6e84e 100644 --- a/linker/linker.go +++ b/linker/linker.go @@ -152,6 +152,9 @@ type Result interface { // is otherwise not useful since all option values are treated as // unknown. CanonicalProto() *descriptorpb.FileDescriptorProto + + // RemoveAST drops the AST information from this result. + RemoveAST() } // ErrorUnusedImport may be passed to a warning reporter when an unused diff --git a/linker/symbols.go b/linker/symbols.go index f7af5778..4bc93f10 100644 --- a/linker/symbols.go +++ b/linker/symbols.go @@ -43,7 +43,12 @@ type packageSymbols struct { children map[protoreflect.FullName]*packageSymbols files map[protoreflect.FileDescriptor]struct{} symbols map[protoreflect.FullName]symbolEntry - exts map[protoreflect.FullName]map[protoreflect.FieldNumber]ast.SourcePos + exts map[extNumber]ast.SourcePos +} + +type extNumber struct { + extendee protoreflect.FullName + tag protoreflect.FieldNumber } type symbolEntry struct { @@ -361,7 +366,7 @@ func (s *packageSymbols) commitFileLocked(f protoreflect.FileDescriptor) { s.symbols = map[protoreflect.FullName]symbolEntry{} } if s.exts == nil { - s.exts = map[protoreflect.FullName]map[protoreflect.FieldNumber]ast.SourcePos{} + s.exts = map[extNumber]ast.SourcePos{} } _ = walk.Descriptors(f, func(d protoreflect.Descriptor) error { pos := sourcePositionFor(d) @@ -503,7 +508,7 @@ func (s *packageSymbols) commitResultLocked(r *result) { s.symbols = map[protoreflect.FullName]symbolEntry{} } if s.exts == nil { - s.exts = map[protoreflect.FullName]map[protoreflect.FieldNumber]ast.SourcePos{} + s.exts = map[extNumber]ast.SourcePos{} } _ = walk.DescriptorProtos(r.FileDescriptorProto(), func(fqn protoreflect.FullName, d proto.Message) error { pos := nameStart(r.FileNode(), r.Node(d)) @@ -537,20 +542,16 @@ func (s *packageSymbols) addExtension(extendee protoreflect.FullName, tag protor defer s.mu.Unlock() if s.exts == nil { - s.exts = map[protoreflect.FullName]map[protoreflect.FieldNumber]ast.SourcePos{} + s.exts = map[extNumber]ast.SourcePos{} } - usedExtTags := s.exts[extendee] - if usedExtTags == nil { - usedExtTags = map[protoreflect.FieldNumber]ast.SourcePos{} - s.exts[extendee] = usedExtTags - } - if existing, ok := usedExtTags[tag]; ok { + extNum := extNumber{extendee: extendee, tag: tag} + if existing, ok := s.exts[extNum]; ok { if err := handler.HandleErrorf(pos, "extension with tag %d for message %s already defined at %v", tag, extendee, existing); err != nil { return err } } else { - usedExtTags[tag] = pos + s.exts[extNum] = pos } return nil } diff --git a/linker/symbols_test.go b/linker/symbols_test.go index 02a06aea..ad1667ae 100644 --- a/linker/symbols_test.go +++ b/linker/symbols_test.go @@ -124,18 +124,14 @@ func TestSymbolsImport(t *testing.T) { assert.Contains(t, syms, protoreflect.FullName("foo.bar.s")) assert.Contains(t, syms, protoreflect.FullName("foo.bar.xtra")) exts := pkg.exts - assert.Equal(t, 1, len(exts)) - extNums := exts["foo.bar.Foo"] - assert.Equal(t, 2, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(10)) - assert.Contains(t, extNums, protoreflect.FieldNumber(11)) + assert.Equal(t, 2, len(exts)) + assert.Contains(t, exts, extNumber{"foo.bar.Foo", 10}) + assert.Contains(t, exts, extNumber{"foo.bar.Foo", 11}) pkg = s.getPackage("google.protobuf") exts = pkg.exts assert.Equal(t, 1, len(exts)) - extNums = exts["google.protobuf.FieldOptions"] - assert.Equal(t, 1, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(20000)) + assert.Contains(t, exts, extNumber{"google.protobuf.FieldOptions", 20000}) }) } } @@ -184,27 +180,16 @@ func TestSymbolExtensions(t *testing.T) { pkg := s.getPackage("foo.bar") exts := pkg.exts - assert.Equal(t, 1, len(exts)) - extNums := exts["foo.bar.Foo"] - assert.Equal(t, 2, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(11)) - assert.Contains(t, extNums, protoreflect.FieldNumber(12)) + assert.Equal(t, 2, len(exts)) + assert.Contains(t, exts, extNumber{"foo.bar.Foo", 11}) + assert.Contains(t, exts, extNumber{"foo.bar.Foo", 12}) pkg = s.getPackage("google.protobuf") exts = pkg.exts assert.Equal(t, 3, len(exts)) - assert.Contains(t, exts, protoreflect.FullName("google.protobuf.FileOptions")) - assert.Contains(t, exts, protoreflect.FullName("google.protobuf.FieldOptions")) - assert.Contains(t, exts, protoreflect.FullName("google.protobuf.MessageOptions")) - extNums = exts["google.protobuf.FileOptions"] - assert.Equal(t, 1, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(10101)) - extNums = exts["google.protobuf.FieldOptions"] - assert.Equal(t, 1, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(10101)) - extNums = exts["google.protobuf.MessageOptions"] - assert.Equal(t, 1, len(extNums)) - assert.Contains(t, extNums, protoreflect.FieldNumber(10101)) + assert.Contains(t, exts, extNumber{"google.protobuf.FileOptions", 10101}) + assert.Contains(t, exts, extNumber{"google.protobuf.FieldOptions", 10101}) + assert.Contains(t, exts, extNumber{"google.protobuf.MessageOptions", 10101}) } func parseAndLink(t *testing.T, contents string) Result {