feat(values): create a Dictionary interface and implementation (#3341)

This adds a Dictionary interface and implementation to the `values`
package. It also updates all of the `values` and usages to acknowledge
that dictionaries exist.

compile.go

@@ -175,6 +175,9 @@ func (t *TableObject) Equal(rhs values.Value) bool {
 func (t *TableObject) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (t *TableObject) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (t *TableObject) Get(i int) values.Value {
 	panic("cannot index into stream")
@@ -280,6 +283,9 @@ func (f *function) Object() values.Object {
 func (f *function) Function() values.Function {
 	return f
 }
+func (f *function) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary))
+}
 func (f *function) Equal(rhs values.Value) bool {
 	if f.Type() != rhs.Type() {
 		return false

compiler/runtime.go

@@ -663,6 +663,9 @@ func (f *functionValue) Object() values.Object {
 func (f *functionValue) Function() values.Function {
 	return f
 }
+func (f *functionValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary))
+}
 func (f *functionValue) Equal(rhs values.Value) bool {
 	if f.Type() != rhs.Type() {
 		return false

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/SAP/go-hdb v0.14.1
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
 	github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01
+	github.com/benbjohnson/immutable v0.2.1
 	github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0
 	github.com/c-bata/go-prompt v0.2.2
 	github.com/cespare/xxhash v1.1.0

go.sum

@@ -68,6 +68,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 h1:FSqtT0UCk
 github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/aws/aws-sdk-go v1.29.16 h1:Gbtod7Y4W/Ai7wPtesdvgGVTkFN8JxAaGouRLlcQfQs=
 github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/benbjohnson/immutable v0.2.1 h1:EVv7H1ju7cDg/a8HUF4hAH4DBrMJh6RWWFwq9JfoO9I=
+github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 h1:MaVh0h9+KaMnJcoDvvIGp+O3fefdWm+8MBUX6ELTJTM=

internal/arrowutil/array_values.gen.go

@@ -80,6 +80,9 @@ func (v Int64ArrayValue) Object() values.Object {
 func (v Int64ArrayValue) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (v Int64ArrayValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (v Int64ArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {
@@ -161,6 +164,9 @@ func (v Uint64ArrayValue) Object() values.Object {
 func (v Uint64ArrayValue) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (v Uint64ArrayValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (v Uint64ArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {
@@ -246,6 +252,9 @@ func (v Float64ArrayValue) Object() values.Object {
 func (v Float64ArrayValue) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (v Float64ArrayValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (v Float64ArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {
@@ -331,6 +340,9 @@ func (v BooleanArrayValue) Object() values.Object {
 func (v BooleanArrayValue) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (v BooleanArrayValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (v BooleanArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {
@@ -414,6 +426,9 @@ func (v StringArrayValue) Object() values.Object {
 func (v StringArrayValue) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Array, semantic.Function))
 }
+func (v StringArrayValue) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary))
+}
 
 func (v StringArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {

internal/arrowutil/array_values.gen.go.tmpl

@@ -51,6 +51,7 @@ func (v {{.Name}}ArrayValue) Regexp() *regexp.Regexp { panic(values.UnexpectedKi
 func (v {{.Name}}ArrayValue) Array() values.Array { return v }
 func (v {{.Name}}ArrayValue) Object() values.Object { panic(values.UnexpectedKind(semantic.Array, semantic.Object)) }
 func (v {{.Name}}ArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) }
+func (v {{.Name}}ArrayValue) Dict() values.Dictionary { panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) }
 
 func (v {{.Name}}ArrayValue) Equal(other values.Value) bool {
 	if other.Type().Nature() != semantic.Array {

interpreter/interpreter.go

@@ -789,6 +789,9 @@ func (f function) Object() values.Object {
 func (f function) Function() values.Function {
 	return f
 }
+func (f function) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary))
+}
 func (f function) Equal(rhs values.Value) bool {
 	if f.Type() != rhs.Type() {
 		return false
@@ -1241,6 +1244,8 @@ func resolveValue(v values.Value) (semantic.Node, bool, error) {
 			return nil, false, err
 		}
 		return node, true, nil
+	case semantic.Dictionary:
+		return nil, false, errors.New(codes.Unimplemented, "cannot resolve dictionary value")
 	default:
 		return nil, false, errors.Newf(codes.Internal, "cannot resolve value of type %v", k)
 	}

interpreter/package.go

@@ -122,6 +122,9 @@ func (p *Package) Object() values.Object {
 func (p *Package) Function() values.Function {
 	panic(values.UnexpectedKind(semantic.Object, semantic.Function))
 }
+func (p *Package) Dict() values.Dictionary {
+	panic(values.UnexpectedKind(semantic.Object, semantic.Dictionary))
+}
 func (p *Package) Equal(rhs values.Value) bool {
 	if p.Type() != rhs.Type() {
 		return false

semantic/monotype.go

@@ -39,6 +39,8 @@ func NewMonoType(tbl flatbuffers.Table, t fbsemantic.MonoType) (MonoType, error)
 		tbler = new(fbsemantic.Record)
 	case fbsemantic.MonoTypeFun:
 		tbler = new(fbsemantic.Fun)
+	case fbsemantic.MonoTypeDict:
+		tbler = new(fbsemantic.Dict)
 	default:
 		return MonoType{}, errors.Newf(codes.Internal, "unknown type (%v)", t)
 	}
@@ -78,6 +80,8 @@ func (mt MonoType) Nature() Nature {
 		return Object
 	case fbsemantic.MonoTypeFun:
 		return Function
+	case fbsemantic.MonoTypeDict:
+		return Dictionary
 	case fbsemantic.MonoTypeNONE,
 		fbsemantic.MonoTypeVar:
 		fallthrough
@@ -96,6 +100,7 @@ const (
 	Arr     = Kind(fbsemantic.MonoTypeArr)
 	Record  = Kind(fbsemantic.MonoTypeRecord)
 	Fun     = Kind(fbsemantic.MonoTypeFun)
+	Dict    = Kind(fbsemantic.MonoTypeDict)
 )
 
 // Kind returns what kind of monotype the receiver is.
@@ -341,6 +346,40 @@ func (mt MonoType) Extends() (MonoType, bool, error) {
 	return monoTypeFromVar(v), true, nil
 }
 
+func getDict(tbl fbTabler) (*fbsemantic.Dict, error) {
+	dict, ok := tbl.(*fbsemantic.Dict)
+	if !ok {
+		return nil, errors.New(codes.Internal, "MonoType is not a dictionary")
+	}
+	return dict, nil
+}
+
+// KeyType returns the type for the key in a Dictionary.
+func (mt MonoType) KeyType() (MonoType, error) {
+	dict, err := getDict(mt.tbl)
+	if err != nil {
+		return MonoType{}, err
+	}
+	var tbl flatbuffers.Table
+	if !dict.K(&tbl) {
+		return MonoType{}, errors.New(codes.Internal, "missing dictionary key type")
+	}
+	return NewMonoType(tbl, dict.KType())
+}
+
+// ValueType returns the type for the value in a Dictionary.
+func (mt MonoType) ValueType() (MonoType, error) {
+	dict, err := getDict(mt.tbl)
+	if err != nil {
+		return MonoType{}, err
+	}
+	var tbl flatbuffers.Table
+	if !dict.V(&tbl) {
+		return MonoType{}, errors.New(codes.Internal, "missing dictionary value type")
+	}
+	return NewMonoType(tbl, dict.VType())
+}
+
 // Argument represents a function argument.
 type Argument struct {
 	*fbsemantic.Argument
@@ -562,6 +601,16 @@ func (mt MonoType) string(m map[uint64]uint64) string {
 		}
 		sb.WriteString(rt.string(m))
 		return sb.String()
+	case Dict:
+		kt, err := mt.KeyType()
+		if err != nil {
+			return "<" + err.Error() + ">"
+		}
+		vt, err := mt.ValueType()
+		if err != nil {
+			return "<" + err.Error() + ">"
+		}
+		return "[" + kt.string(m) + ": " + vt.string(m) + "]"
 	default:
 		return "<" + fmt.Sprintf("unknown monotype (%v)", tk) + ">"
 	}
@@ -653,6 +702,23 @@ func ExtendObjectType(properties []PropertyType, extends *uint64) MonoType {
 	return mt
 }
 
+// NewDictType will construct a new Dict MonoType
+// where the key element for the dict is keyType and
+// the value element for the dict is valueType.
+func NewDictType(keyType, valueType MonoType) MonoType {
+	builder := flatbuffers.NewBuilder(32)
+	offset := buildDictType(builder, keyType, valueType)
+	builder.Finish(offset)
+
+	buf := builder.FinishedBytes()
+	arr := fbsemantic.GetRootAsDict(buf, 0)
+	mt, err := NewMonoType(arr.Table(), fbsemantic.MonoTypeDict)
+	if err != nil {
+		panic(err)
+	}
+	return mt
+}
+
 // copyMonoType will reconstruct the type contained within the
 // MonoType for the new builder. When building a new buffer,
 // flatbuffers cannot reference data in another buffer and the
@@ -720,6 +786,13 @@ func copyMonoType(builder *flatbuffers.Builder, t MonoType) flatbuffers.UOffsetT
 		}
 		retn := monoTypeFromFunc(fun.Retn, fun.RetnType())
 		return buildFunctionType(builder, retn, args)
+	case fbsemantic.MonoTypeDict:
+		var dict fbsemantic.Dict
+		dict.Init(table.Bytes, table.Pos)
+
+		key := monoTypeFromFunc(dict.K, dict.KType())
+		value := monoTypeFromFunc(dict.V, dict.VType())
+		return buildDictType(builder, key, value)
 	default:
 		panic(fmt.Sprintf("unknown monotype (%v)", t.mt))
 	}
@@ -830,6 +903,19 @@ func buildObjectType(builder *flatbuffers.Builder, properties []PropertyType, ex
 	return fbsemantic.RecordEnd(builder)
 }
 
+// buildDictType will construct a dict type in the builder
+// and return the offset for the type.
+func buildDictType(builder *flatbuffers.Builder, keyType, valueType MonoType) flatbuffers.UOffsetT {
+	koffset := copyMonoType(builder, keyType)
+	voffset := copyMonoType(builder, valueType)
+	fbsemantic.DictStart(builder)
+	fbsemantic.DictAddKType(builder, keyType.mt)
+	fbsemantic.DictAddK(builder, koffset)
+	fbsemantic.DictAddVType(builder, valueType.mt)
+	fbsemantic.DictAddV(builder, voffset)
+	return fbsemantic.DictEnd(builder)
+}
+
 func updateTVarMap(counter *uint64, m map[uint64]uint64, tv uint64) {
 	if _, ok := m[tv]; ok {
 		return

semantic/semantictest/cmp.go

@@ -172,6 +172,15 @@ func TransformValue(v values.Value) map[string]interface{} {
 		return map[string]interface{}{
 			"type": v.Type().String(),
 		}
+	case semantic.Dictionary:
+		elements := make(map[interface{}]interface{})
+		v.Dict().Range(func(key, value values.Value) {
+			elements[TransformValue(key)] = TransformValue(value)
+		})
+		return map[string]interface{}{
+			"type":     semantic.Dictionary.String(),
+			"elements": elements,
+		}
 	default:
 		panic(fmt.Errorf("unexpected value type %v", v.Type()))
 	}

semantic/types.go

@@ -21,22 +21,24 @@ const (
 	Array
 	Object
 	Function
+	Dictionary
 )
 
 var natureNames = []string{
-	Invalid:  "invalid",
-	String:   "string",
-	Bytes:    "bytes",
-	Int:      "int",
-	UInt:     "uint",
-	Float:    "float",
-	Bool:     "bool",
-	Time:     "time",
-	Duration: "duration",
-	Regexp:   "regexp",
-	Array:    "array",
-	Object:   "object",
-	Function: "function",
+	Invalid:    "invalid",
+	String:     "string",
+	Bytes:      "bytes",
+	Int:        "int",
+	UInt:       "uint",
+	Float:      "float",
+	Bool:       "bool",
+	Time:       "time",
+	Duration:   "duration",
+	Regexp:     "regexp",
+	Array:      "array",
+	Object:     "object",
+	Function:   "function",
+	Dictionary: "dictionary",
 }
 
 func (n Nature) String() string {

stdlib/json/encode.go

@@ -64,7 +64,7 @@ func convertValue(v values.Value) (interface{}, error) {
 		var rangeErr error
 		arr.Range(func(i int, v values.Value) {
 			if rangeErr != nil {
-				return //short circuit if we already hit an error
+				return // short circuit if we already hit an error
 			}
 			val, err := convertValue(v)
 			if err != nil {
@@ -83,7 +83,7 @@ func convertValue(v values.Value) (interface{}, error) {
 		var rangeErr error
 		obj.Range(func(k string, v values.Value) {
 			if rangeErr != nil {
-				return //short circuit if we already hit an error
+				return // short circuit if we already hit an error
 			}
 			val, err := convertValue(v)
 			if err != nil {
@@ -98,6 +98,30 @@ func convertValue(v values.Value) (interface{}, error) {
 		return o, nil
 	case semantic.Function:
 		return nil, errors.New(codes.Invalid, "cannot encode a function value")
+	case semantic.Dictionary:
+		dict := v.Dict()
+		d := make(map[interface{}]interface{}, dict.Len())
+		var rangeErr error
+		dict.Range(func(k, v values.Value) {
+			if rangeErr != nil {
+				return // short circuit if we already hit an error
+			}
+			key, err := convertValue(k)
+			if err != nil {
+				rangeErr = err
+				return
+			}
+			val, err := convertValue(v)
+			if err != nil {
+				rangeErr = err
+				return
+			}
+			d[key] = val
+		})
+		if rangeErr != nil {
+			return nil, rangeErr
+		}
+		return d, nil
 	default:
 		return nil, errors.Newf(codes.Unknown, "unknown nature %v", n)
 	}