Merge #31554 #31610 #31689

31554: exec: initial commit of execgen tool r=solongordon a=solongordon

Execgen will be our tool for generating templated code necessary for
columnarized execution. So far it only generates the
EncDatumRowsToColVec function, which is used by the columnarizer to
convert a RowSource into a columnarized Operator.

Release note: None

31610: sql: fix pg_catalog.pg_constraint's confkey column r=BramGruneir a=BramGruneir

Prior to this patch, all columns in the index were included instead of only the
ones being used in the foreign key reference.

Fixes #31545.

Release note (bug fix): Fix pg_catalog.pg_constraint's confkey column from
including columns that were not involved in the foreign key reference.

31689: storage: pick up fix for Raft uncommitted entry size tracking r=benesch a=tschottdorf

Waiting for the upstream PR

etcd-io/etcd#10199

to merge, but this is going to be what the result will look like.

----

The tracking of the uncommitted portion of the log had a bug where
it wasn't releasing everything as it should've. As a result, over
time, all proposals would be dropped. We're hitting this way earlier
in our import tests, which propose large proposals. As an intentional
implementation detail, a proposal that itself exceeds the max
uncommitted log size is allowed only if the uncommitted log is empty.
Due to the leak, we weren't ever hitting this case and so AddSSTable
commands were often dropped indefinitely.

Fixes #31184.
Fixes #28693.
Fixes #31642.

Optimistically:
Fixes #31675.
Fixes #31654.
Fixes #31446.

Release note: None

Co-authored-by: Solon Gordon <solon@cockroachlabs.com>
Co-authored-by: Bram Gruneir <bram@cockroachlabs.com>
Co-authored-by: Tobias Schottdorf <tobias.schottdorf@gmail.com>

Makefile

@@ -694,6 +694,8 @@ PROTOBUF_TARGETS := bin/.go_protobuf_sources bin/.gw_protobuf_sources bin/.cpp_p
 
 DOCGEN_TARGETS := bin/.docgen_bnfs bin/.docgen_functions
 
+EXECGEN_TARGETS = pkg/sql/exec/rowstovec.og.go
+
 OPTGEN_TARGETS = \
 	pkg/sql/opt/memo/expr.og.go \
 	pkg/sql/opt/operator.og.go \
@@ -735,7 +737,7 @@ BUILDINFO = .buildinfo/tag .buildinfo/rev
 BUILD_TAGGED_RELEASE =
 
 $(go-targets): bin/.bootstrap $(BUILDINFO) $(CGO_FLAGS_FILES) $(PROTOBUF_TARGETS)
-$(go-targets): $(SQLPARSER_TARGETS) $(OPTGEN_TARGETS)
+$(go-targets): $(SQLPARSER_TARGETS) $(EXECGEN_TARGETS) $(OPTGEN_TARGETS)
 $(go-targets): override LINKFLAGS += \
 	-X "github.com/cockroachdb/cockroach/pkg/build.tag=$(shell cat .buildinfo/tag)" \
 	-X "github.com/cockroachdb/cockroach/pkg/build.rev=$(shell cat .buildinfo/rev)" \
@@ -914,7 +916,7 @@ dupl: bin/.bootstrap
 
 .PHONY: generate
 generate: ## Regenerate generated code.
-generate: protobuf $(DOCGEN_TARGETS) $(OPTGEN_TARGETS) $(SQLPARSER_TARGETS) $(SETTINGS_DOC_PAGE) bin/langgen
+generate: protobuf $(DOCGEN_TARGETS) $(EXECGEN_TARGETS) $(OPTGEN_TARGETS) $(SQLPARSER_TARGETS) $(SETTINGS_DOC_PAGE) bin/langgen
 	$(GO) generate $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LINKFLAGS)' $(PKG)
 
 .PHONY: lint
@@ -962,6 +964,7 @@ $(ARCHIVE): $(ARCHIVE).tmp
 ARCHIVE_EXTRAS = \
 	$(BUILDINFO) \
 	$(SQLPARSER_TARGETS) \
+	$(EXECGEN_TARGETS) \
 	$(OPTGEN_TARGETS) \
 	pkg/ui/distccl/bindata.go pkg/ui/distoss/bindata.go
 
@@ -1305,6 +1308,9 @@ settings-doc-gen := $(if $(filter buildshort,$(MAKECMDGOALS)),$(COCKROACHSHORT),
 $(SETTINGS_DOC_PAGE): $(settings-doc-gen)
 	@$(settings-doc-gen) gen settings-list --format=html > $@
 
+pkg/sql/exec/%.og.go: bin/execgen
+	execgen $@
+
 optgen-defs := pkg/sql/opt/ops/*.opt
 optgen-norm-rules := pkg/sql/opt/norm/rules/*.opt
 optgen-xform-rules := pkg/sql/opt/xform/rules/*.opt
@@ -1362,7 +1368,7 @@ clean: clean-c-deps
 .PHONY: maintainer-clean
 maintainer-clean: ## Like clean, but also remove some auto-generated source code.
 maintainer-clean: clean ui-maintainer-clean
-	rm -f $(SQLPARSER_TARGETS) $(OPTGEN_TARGETS) $(UI_PROTOS_OSS) $(UI_PROTOS_CCL)
+	rm -f $(SQLPARSER_TARGETS) $(EXECGEN_TARGETS) $(OPTGEN_TARGETS) $(UI_PROTOS_OSS) $(UI_PROTOS_CCL)
 
 .PHONY: unsafe-clean
 unsafe-clean: ## Like maintainer-clean, but also remove ALL untracked/ignored files.
@@ -1380,6 +1386,7 @@ bins = \
   bin/cockroach-oss \
   bin/cockroach-short \
   bin/docgen \
+  bin/execgen \
   bin/generate-binary \
   bin/github-post \
   bin/github-pull-request-make \
@@ -1403,6 +1410,7 @@ testbins = \
   bin/logictestccl
 
 # Mappings for binaries that don't live in pkg/cmd.
+execgen-package = ./pkg/sql/exec/execgen/cmd/execgen
 langgen-package = ./pkg/sql/opt/optgen/cmd/langgen
 optgen-package = ./pkg/sql/opt/optgen/cmd/optgen
 logictest-package = ./pkg/sql/logictest
@@ -1414,7 +1422,7 @@ logictest-bins := bin/logictest bin/logictestopt bin/logictestccl
 # Additional dependencies for binaries that depend on generated code.
 #
 # TODO(benesch): Derive this automatically. This is getting out of hand.
-bin/workload bin/docgen bin/roachtest $(logictest-bins): $(SQLPARSER_TARGETS) $(PROTOBUF_TARGETS)
+bin/workload bin/docgen bin/execgen bin/roachtest $(logictest-bins): $(SQLPARSER_TARGETS) $(PROTOBUF_TARGETS)
 bin/workload bin/roachtest $(logictest-bins): $(C_LIBS_CCL) $(CGO_FLAGS_FILES)
 bin/roachtest bin/logictestopt: $(OPTGEN_TARGETS)
 

build/variables.mk

@@ -44,6 +44,7 @@ define VALID_VARS
   C_LIBS_OSS
   DOCGEN_TARGETS
   DUPLFLAGS
+  EXECGEN_TARGETS
   EXTRA_XCMAKE_FLAGS
   EXTRA_XCONFIGURE_FLAGS
   FILES
@@ -156,6 +157,7 @@ define VALID_VARS
   cmake-flags
   configure-flags
   cyan
+  execgen-package
   go-targets
   go-targets-ccl
   have-defs

pkg/sql/distsqlrun/columnarizer.go

@@ -15,13 +15,9 @@
 package distsqlrun
 
 import (
-	"fmt"
-
 	"github.com/cockroachdb/cockroach/pkg/sql/exec"
 	"github.com/cockroachdb/cockroach/pkg/sql/exec/types"
-	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
-	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 )
 
 // columnarizer turns a RowSource input into an exec.Operator output, by reading
@@ -91,115 +87,9 @@ func (c *columnarizer) Next() exec.ColBatch {
 
 	// Write each column into the output batch.
 	for idx, ct := range columnTypes {
-		vec := c.batch.ColVec(idx)
-		switch ct.SemanticType {
-		// TODO(solon): these should be autogenerated from a template.
-		case sqlbase.ColumnType_BOOL:
-			col := vec.Bool()
-			for i := uint16(0); i < nRows; i++ {
-				ed := c.buffered[i][idx]
-				if err := ed.EnsureDecoded(&ct, &c.da); err != nil {
-					panic(err)
-				}
-				if ed.Datum == tree.DNull {
-					vec.SetNull(i)
-				}
-				col.Set(i, bool(*ed.Datum.(*tree.DBool)))
-			}
-		case sqlbase.ColumnType_INT:
-			switch ct.Width {
-			case 8:
-				col := vec.Int8()
-				for i := uint16(0); i < nRows; i++ {
-					ed := c.buffered[i][idx]
-					if err := ed.EnsureDecoded(&ct, &c.da); err != nil {
-						panic(err)
-					}
-					if ed.Datum == tree.DNull {
-						vec.SetNull(i)
-					}
-					col[i] = int8(*ed.Datum.(*tree.DInt))
-				}
-			case 16:
-				col := vec.Int16()
-				for i := uint16(0); i < nRows; i++ {
-					ed := c.buffered[i][idx]
-					if err := ed.EnsureDecoded(&ct, &c.da); err != nil {
-						panic(err)
-					}
-					if ed.Datum == tree.DNull {
-						vec.SetNull(i)
-					}
-					col[i] = int16(*ed.Datum.(*tree.DInt))
-				}
-			case 32:
-				col := vec.Int32()
-				for i := uint16(0); i < nRows; i++ {
-					ed := c.buffered[i][idx]
-					if err := ed.EnsureDecoded(&ct, &c.da); err != nil {
-						panic(err)
-					}
-					if ed.Datum == tree.DNull {
-						vec.SetNull(i)
-					}
-					col[i] = int32(*ed.Datum.(*tree.DInt))
-				}
-			case 0, 64:
-				col := vec.Int64()
-				for i := uint16(0); i < nRows; i++ {
-					if c.buffered[i][idx].Datum == nil {
-						if err := c.buffered[i][idx].EnsureDecoded(&ct, &c.da); err != nil {
-							panic(err)
-						}
-					}
-					if c.buffered[i][idx].Datum == tree.DNull {
-						vec.SetNull(i)
-					}
-					col[i] = int64(*c.buffered[i][idx].Datum.(*tree.DInt))
-				}
-			default:
-				panic(fmt.Sprintf("integer with unknown width %d", ct.Width))
-			}
-		case sqlbase.ColumnType_FLOAT:
-			col := vec.Float64()
-			for i := uint16(0); i < nRows; i++ {
-				ed := c.buffered[i][idx]
-				if err := ed.EnsureDecoded(&ct, &c.da); err != nil {
-					panic(err)
-				}
-				if ed.Datum == tree.DNull {
-					vec.SetNull(i)
-				}
-				col[i] = float64(*ed.Datum.(*tree.DFloat))
-			}
-		case sqlbase.ColumnType_BYTES:
-			col := vec.Bytes()
-			for i := uint16(0); i < nRows; i++ {
-				if c.buffered[i][idx].Datum == nil {
-					if err := c.buffered[i][idx].EnsureDecoded(&ct, &c.da); err != nil {
-						panic(err)
-					}
-				}
-				if c.buffered[i][idx].Datum == tree.DNull {
-					vec.SetNull(i)
-				}
-				col.Set(i, encoding.UnsafeConvertStringToBytes(string(*c.buffered[i][idx].Datum.(*tree.DBytes))))
-			}
-		case sqlbase.ColumnType_STRING:
-			col := vec.Bytes()
-			for i := uint16(0); i < nRows; i++ {
-				if c.buffered[i][idx].Datum == nil {
-					if err := c.buffered[i][idx].EnsureDecoded(&ct, &c.da); err != nil {
-						panic(err)
-					}
-				}
-				if c.buffered[i][idx].Datum == tree.DNull {
-					vec.SetNull(i)
-				}
-				col.Set(i, encoding.UnsafeConvertStringToBytes(string(*c.buffered[i][idx].Datum.(*tree.DString))))
-			}
-		default:
-			panic(fmt.Sprintf("Unsupported column type %s", ct.SQLString()))
+		err := exec.EncDatumRowsToColVec(c.buffered[:nRows], c.batch.ColVec(idx), idx, &ct, &c.da)
+		if err != nil {
+			panic(err)
 		}
 	}
 	return c.batch

pkg/sql/exec/.gitignore

@@ -0,0 +1 @@
+*.og.go