From 9c3cf772426aef125c979c81d93e0dda37ab95b8 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Mon, 23 Nov 2020 09:19:47 -0600 Subject: [PATCH] feat(values): create a Dictionary interface and implementation (#3341) This adds a Dictionary interface and implementation to the `values` package. It also updates all of the `values` and usages to acknowledge that dictionaries exist. --- compile.go | 6 + compiler/runtime.go | 3 + go.mod | 1 + go.sum | 2 + internal/arrowutil/array_values.gen.go | 15 + internal/arrowutil/array_values.gen.go.tmpl | 1 + interpreter/interpreter.go | 5 + interpreter/package.go | 3 + semantic/monotype.go | 86 ++++++ semantic/semantictest/cmp.go | 9 + semantic/types.go | 28 +- stdlib/json/encode.go | 28 +- stdlib/universe/histogram.go | 8 + stdlib/universe/typeconv.go | 21 ++ values/array.go | 3 + values/dict.go | 299 ++++++++++++++++++ values/dict_test.go | 326 ++++++++++++++++++++ values/function.go | 4 + values/object.go | 3 + values/objects/table.go | 4 + values/values.go | 20 +- 21 files changed, 858 insertions(+), 17 deletions(-) create mode 100644 values/dict.go create mode 100644 values/dict_test.go diff --git a/compile.go b/compile.go index 7d54dacd9a..83ec84ef2a 100644 --- a/compile.go +++ b/compile.go @@ -175,6 +175,9 @@ func (t *TableObject) Equal(rhs values.Value) bool { func (t *TableObject) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (t *TableObject) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (t *TableObject) Get(i int) values.Value { panic("cannot index into stream") @@ -280,6 +283,9 @@ func (f *function) Object() values.Object { func (f *function) Function() values.Function { return f } +func (f *function) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary)) +} func (f *function) Equal(rhs values.Value) bool { if f.Type() != rhs.Type() { return false diff --git a/compiler/runtime.go b/compiler/runtime.go index bc8df5e329..d8587b390f 100644 --- a/compiler/runtime.go +++ b/compiler/runtime.go @@ -663,6 +663,9 @@ func (f *functionValue) Object() values.Object { func (f *functionValue) Function() values.Function { return f } +func (f *functionValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary)) +} func (f *functionValue) Equal(rhs values.Value) bool { if f.Type() != rhs.Type() { return false diff --git a/go.mod b/go.mod index 0c1ac81de5..b468cb5049 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/SAP/go-hdb v0.14.1 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 + github.com/benbjohnson/immutable v0.2.1 github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 github.com/c-bata/go-prompt v0.2.2 github.com/cespare/xxhash v1.1.0 diff --git a/go.sum b/go.sum index 3f95eeccce..86bb5b25d2 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 h1:FSqtT0UCk github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/aws/aws-sdk-go v1.29.16 h1:Gbtod7Y4W/Ai7wPtesdvgGVTkFN8JxAaGouRLlcQfQs= github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/benbjohnson/immutable v0.2.1 h1:EVv7H1ju7cDg/a8HUF4hAH4DBrMJh6RWWFwq9JfoO9I= +github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 h1:MaVh0h9+KaMnJcoDvvIGp+O3fefdWm+8MBUX6ELTJTM= diff --git a/internal/arrowutil/array_values.gen.go b/internal/arrowutil/array_values.gen.go index 22a75705ff..164bb7bebc 100644 --- a/internal/arrowutil/array_values.gen.go +++ b/internal/arrowutil/array_values.gen.go @@ -80,6 +80,9 @@ func (v Int64ArrayValue) Object() values.Object { func (v Int64ArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v Int64ArrayValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (v Int64ArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { @@ -161,6 +164,9 @@ func (v Uint64ArrayValue) Object() values.Object { func (v Uint64ArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v Uint64ArrayValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (v Uint64ArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { @@ -246,6 +252,9 @@ func (v Float64ArrayValue) Object() values.Object { func (v Float64ArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v Float64ArrayValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (v Float64ArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { @@ -331,6 +340,9 @@ func (v BooleanArrayValue) Object() values.Object { func (v BooleanArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v BooleanArrayValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (v BooleanArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { @@ -414,6 +426,9 @@ func (v StringArrayValue) Object() values.Object { func (v StringArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v StringArrayValue) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (v StringArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { diff --git a/internal/arrowutil/array_values.gen.go.tmpl b/internal/arrowutil/array_values.gen.go.tmpl index 491e801d8f..24e58d108d 100644 --- a/internal/arrowutil/array_values.gen.go.tmpl +++ b/internal/arrowutil/array_values.gen.go.tmpl @@ -51,6 +51,7 @@ func (v {{.Name}}ArrayValue) Regexp() *regexp.Regexp { panic(values.UnexpectedKi func (v {{.Name}}ArrayValue) Array() values.Array { return v } func (v {{.Name}}ArrayValue) Object() values.Object { panic(values.UnexpectedKind(semantic.Array, semantic.Object)) } func (v {{.Name}}ArrayValue) Function() values.Function { panic(values.UnexpectedKind(semantic.Array, semantic.Function)) } +func (v {{.Name}}ArrayValue) Dict() values.Dictionary { panic(values.UnexpectedKind(semantic.Array, semantic.Dictionary)) } func (v {{.Name}}ArrayValue) Equal(other values.Value) bool { if other.Type().Nature() != semantic.Array { diff --git a/interpreter/interpreter.go b/interpreter/interpreter.go index dd6836c4f0..20f45723f1 100644 --- a/interpreter/interpreter.go +++ b/interpreter/interpreter.go @@ -789,6 +789,9 @@ func (f function) Object() values.Object { func (f function) Function() values.Function { return f } +func (f function) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Function, semantic.Dictionary)) +} func (f function) Equal(rhs values.Value) bool { if f.Type() != rhs.Type() { return false @@ -1241,6 +1244,8 @@ func resolveValue(v values.Value) (semantic.Node, bool, error) { return nil, false, err } return node, true, nil + case semantic.Dictionary: + return nil, false, errors.New(codes.Unimplemented, "cannot resolve dictionary value") default: return nil, false, errors.Newf(codes.Internal, "cannot resolve value of type %v", k) } diff --git a/interpreter/package.go b/interpreter/package.go index 44bf3b5f87..04701d3133 100644 --- a/interpreter/package.go +++ b/interpreter/package.go @@ -122,6 +122,9 @@ func (p *Package) Object() values.Object { func (p *Package) Function() values.Function { panic(values.UnexpectedKind(semantic.Object, semantic.Function)) } +func (p *Package) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Object, semantic.Dictionary)) +} func (p *Package) Equal(rhs values.Value) bool { if p.Type() != rhs.Type() { return false diff --git a/semantic/monotype.go b/semantic/monotype.go index 72acfb5511..5891fcb4a0 100644 --- a/semantic/monotype.go +++ b/semantic/monotype.go @@ -39,6 +39,8 @@ func NewMonoType(tbl flatbuffers.Table, t fbsemantic.MonoType) (MonoType, error) tbler = new(fbsemantic.Record) case fbsemantic.MonoTypeFun: tbler = new(fbsemantic.Fun) + case fbsemantic.MonoTypeDict: + tbler = new(fbsemantic.Dict) default: return MonoType{}, errors.Newf(codes.Internal, "unknown type (%v)", t) } @@ -78,6 +80,8 @@ func (mt MonoType) Nature() Nature { return Object case fbsemantic.MonoTypeFun: return Function + case fbsemantic.MonoTypeDict: + return Dictionary case fbsemantic.MonoTypeNONE, fbsemantic.MonoTypeVar: fallthrough @@ -96,6 +100,7 @@ const ( Arr = Kind(fbsemantic.MonoTypeArr) Record = Kind(fbsemantic.MonoTypeRecord) Fun = Kind(fbsemantic.MonoTypeFun) + Dict = Kind(fbsemantic.MonoTypeDict) ) // Kind returns what kind of monotype the receiver is. @@ -341,6 +346,40 @@ func (mt MonoType) Extends() (MonoType, bool, error) { return monoTypeFromVar(v), true, nil } +func getDict(tbl fbTabler) (*fbsemantic.Dict, error) { + dict, ok := tbl.(*fbsemantic.Dict) + if !ok { + return nil, errors.New(codes.Internal, "MonoType is not a dictionary") + } + return dict, nil +} + +// KeyType returns the type for the key in a Dictionary. +func (mt MonoType) KeyType() (MonoType, error) { + dict, err := getDict(mt.tbl) + if err != nil { + return MonoType{}, err + } + var tbl flatbuffers.Table + if !dict.K(&tbl) { + return MonoType{}, errors.New(codes.Internal, "missing dictionary key type") + } + return NewMonoType(tbl, dict.KType()) +} + +// ValueType returns the type for the value in a Dictionary. +func (mt MonoType) ValueType() (MonoType, error) { + dict, err := getDict(mt.tbl) + if err != nil { + return MonoType{}, err + } + var tbl flatbuffers.Table + if !dict.V(&tbl) { + return MonoType{}, errors.New(codes.Internal, "missing dictionary value type") + } + return NewMonoType(tbl, dict.VType()) +} + // Argument represents a function argument. type Argument struct { *fbsemantic.Argument @@ -562,6 +601,16 @@ func (mt MonoType) string(m map[uint64]uint64) string { } sb.WriteString(rt.string(m)) return sb.String() + case Dict: + kt, err := mt.KeyType() + if err != nil { + return "<" + err.Error() + ">" + } + vt, err := mt.ValueType() + if err != nil { + return "<" + err.Error() + ">" + } + return "[" + kt.string(m) + ": " + vt.string(m) + "]" default: return "<" + fmt.Sprintf("unknown monotype (%v)", tk) + ">" } @@ -653,6 +702,23 @@ func ExtendObjectType(properties []PropertyType, extends *uint64) MonoType { return mt } +// NewDictType will construct a new Dict MonoType +// where the key element for the dict is keyType and +// the value element for the dict is valueType. +func NewDictType(keyType, valueType MonoType) MonoType { + builder := flatbuffers.NewBuilder(32) + offset := buildDictType(builder, keyType, valueType) + builder.Finish(offset) + + buf := builder.FinishedBytes() + arr := fbsemantic.GetRootAsDict(buf, 0) + mt, err := NewMonoType(arr.Table(), fbsemantic.MonoTypeDict) + if err != nil { + panic(err) + } + return mt +} + // copyMonoType will reconstruct the type contained within the // MonoType for the new builder. When building a new buffer, // flatbuffers cannot reference data in another buffer and the @@ -720,6 +786,13 @@ func copyMonoType(builder *flatbuffers.Builder, t MonoType) flatbuffers.UOffsetT } retn := monoTypeFromFunc(fun.Retn, fun.RetnType()) return buildFunctionType(builder, retn, args) + case fbsemantic.MonoTypeDict: + var dict fbsemantic.Dict + dict.Init(table.Bytes, table.Pos) + + key := monoTypeFromFunc(dict.K, dict.KType()) + value := monoTypeFromFunc(dict.V, dict.VType()) + return buildDictType(builder, key, value) default: panic(fmt.Sprintf("unknown monotype (%v)", t.mt)) } @@ -830,6 +903,19 @@ func buildObjectType(builder *flatbuffers.Builder, properties []PropertyType, ex return fbsemantic.RecordEnd(builder) } +// buildDictType will construct a dict type in the builder +// and return the offset for the type. +func buildDictType(builder *flatbuffers.Builder, keyType, valueType MonoType) flatbuffers.UOffsetT { + koffset := copyMonoType(builder, keyType) + voffset := copyMonoType(builder, valueType) + fbsemantic.DictStart(builder) + fbsemantic.DictAddKType(builder, keyType.mt) + fbsemantic.DictAddK(builder, koffset) + fbsemantic.DictAddVType(builder, valueType.mt) + fbsemantic.DictAddV(builder, voffset) + return fbsemantic.DictEnd(builder) +} + func updateTVarMap(counter *uint64, m map[uint64]uint64, tv uint64) { if _, ok := m[tv]; ok { return diff --git a/semantic/semantictest/cmp.go b/semantic/semantictest/cmp.go index 85025a5095..f1307f2ba1 100644 --- a/semantic/semantictest/cmp.go +++ b/semantic/semantictest/cmp.go @@ -172,6 +172,15 @@ func TransformValue(v values.Value) map[string]interface{} { return map[string]interface{}{ "type": v.Type().String(), } + case semantic.Dictionary: + elements := make(map[interface{}]interface{}) + v.Dict().Range(func(key, value values.Value) { + elements[TransformValue(key)] = TransformValue(value) + }) + return map[string]interface{}{ + "type": semantic.Dictionary.String(), + "elements": elements, + } default: panic(fmt.Errorf("unexpected value type %v", v.Type())) } diff --git a/semantic/types.go b/semantic/types.go index b855205973..2333a429ef 100644 --- a/semantic/types.go +++ b/semantic/types.go @@ -21,22 +21,24 @@ const ( Array Object Function + Dictionary ) var natureNames = []string{ - Invalid: "invalid", - String: "string", - Bytes: "bytes", - Int: "int", - UInt: "uint", - Float: "float", - Bool: "bool", - Time: "time", - Duration: "duration", - Regexp: "regexp", - Array: "array", - Object: "object", - Function: "function", + Invalid: "invalid", + String: "string", + Bytes: "bytes", + Int: "int", + UInt: "uint", + Float: "float", + Bool: "bool", + Time: "time", + Duration: "duration", + Regexp: "regexp", + Array: "array", + Object: "object", + Function: "function", + Dictionary: "dictionary", } func (n Nature) String() string { diff --git a/stdlib/json/encode.go b/stdlib/json/encode.go index 1229c0bdc5..5f559797f6 100644 --- a/stdlib/json/encode.go +++ b/stdlib/json/encode.go @@ -64,7 +64,7 @@ func convertValue(v values.Value) (interface{}, error) { var rangeErr error arr.Range(func(i int, v values.Value) { if rangeErr != nil { - return //short circuit if we already hit an error + return // short circuit if we already hit an error } val, err := convertValue(v) if err != nil { @@ -83,7 +83,7 @@ func convertValue(v values.Value) (interface{}, error) { var rangeErr error obj.Range(func(k string, v values.Value) { if rangeErr != nil { - return //short circuit if we already hit an error + return // short circuit if we already hit an error } val, err := convertValue(v) if err != nil { @@ -98,6 +98,30 @@ func convertValue(v values.Value) (interface{}, error) { return o, nil case semantic.Function: return nil, errors.New(codes.Invalid, "cannot encode a function value") + case semantic.Dictionary: + dict := v.Dict() + d := make(map[interface{}]interface{}, dict.Len()) + var rangeErr error + dict.Range(func(k, v values.Value) { + if rangeErr != nil { + return // short circuit if we already hit an error + } + key, err := convertValue(k) + if err != nil { + rangeErr = err + return + } + val, err := convertValue(v) + if err != nil { + rangeErr = err + return + } + d[key] = val + }) + if rangeErr != nil { + return nil, rangeErr + } + return d, nil default: return nil, errors.Newf(codes.Unknown, "unknown nature %v", n) } diff --git a/stdlib/universe/histogram.go b/stdlib/universe/histogram.go index ad4006eedb..8dd1c65287 100644 --- a/stdlib/universe/histogram.go +++ b/stdlib/universe/histogram.go @@ -302,6 +302,10 @@ func (b linearBins) Function() values.Function { return b } +func (b linearBins) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Dictionary, semantic.Function)) +} + func (b linearBins) Equal(rhs values.Value) bool { if b.Type() != rhs.Type() { return false @@ -424,6 +428,10 @@ func (b logarithmicBins) Function() values.Function { return b } +func (b logarithmicBins) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Dictionary, semantic.Function)) +} + func (b logarithmicBins) Equal(rhs values.Value) bool { if b.Type() != rhs.Type() { return false diff --git a/stdlib/universe/typeconv.go b/stdlib/universe/typeconv.go index 10649d6464..69c34a3237 100644 --- a/stdlib/universe/typeconv.go +++ b/stdlib/universe/typeconv.go @@ -88,6 +88,9 @@ func (c *stringConv) Object() values.Object { func (c *stringConv) Function() values.Function { return c } +func (c *stringConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *stringConv) Equal(rhs values.Value) bool { f, ok := rhs.(*stringConv) return ok && (c == f) @@ -183,6 +186,9 @@ func (c *intConv) Object() values.Object { func (c *intConv) Function() values.Function { return c } +func (c *intConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *intConv) Equal(rhs values.Value) bool { f, ok := rhs.(*intConv) return ok && (c == f) @@ -273,6 +279,9 @@ func (c *uintConv) Object() values.Object { func (c *uintConv) Function() values.Function { return c } +func (c *uintConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *uintConv) Equal(rhs values.Value) bool { f, ok := rhs.(*uintConv) return ok && (c == f) @@ -363,6 +372,9 @@ func (c *floatConv) Object() values.Object { func (c *floatConv) Function() values.Function { return c } +func (c *floatConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *floatConv) Equal(rhs values.Value) bool { f, ok := rhs.(*floatConv) return ok && (c == f) @@ -449,6 +461,9 @@ func (c *boolConv) Object() values.Object { func (c *boolConv) Function() values.Function { return c } +func (c *boolConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *boolConv) Equal(rhs values.Value) bool { f, ok := rhs.(*boolConv) return ok && (c == f) @@ -555,6 +570,9 @@ func (c *timeConv) Object() values.Object { func (c *timeConv) Function() values.Function { return c } +func (c *timeConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *timeConv) Equal(rhs values.Value) bool { f, ok := rhs.(*timeConv) return ok && (c == f) @@ -635,6 +653,9 @@ func (c *durationConv) Object() values.Object { func (c *durationConv) Function() values.Function { return c } +func (c *durationConv) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Float, semantic.Dictionary)) +} func (c *durationConv) Equal(rhs values.Value) bool { f, ok := rhs.(*durationConv) return ok && (c == f) diff --git a/values/array.go b/values/array.go index 2e21f39e46..194d6a7536 100644 --- a/values/array.go +++ b/values/array.go @@ -131,6 +131,9 @@ func (a *array) Object() Object { func (a *array) Function() Function { panic(UnexpectedKind(semantic.Array, semantic.Function)) } +func (a *array) Dict() Dictionary { + panic(UnexpectedKind(semantic.Array, semantic.Dictionary)) +} func (a *array) Equal(rhs Value) bool { if !a.Type().Equal(rhs.Type()) { return false diff --git a/values/dict.go b/values/dict.go new file mode 100644 index 0000000000..f3fb715689 --- /dev/null +++ b/values/dict.go @@ -0,0 +1,299 @@ +package values + +import ( + "regexp" + + "github.com/benbjohnson/immutable" + "github.com/influxdata/flux/codes" + "github.com/influxdata/flux/internal/errors" + "github.com/influxdata/flux/semantic" +) + +// Dictionary defines the interface for a dictionary. +// +// A Dictionary is immutable. Changes to a Dictionary +// will return a new Dictionary. +type Dictionary interface { + Value + + // Get will retrieve a Value out of the Dictionary. + // If the key is not present, the def Value will + // be returned instead. + Get(key, def Value) Value + + // Insert will insert a Value into the Dictionary + // using the key and value. It will return a new + // Dictionary with the key/value inserted. If the + // key was already in the Dictionary, it will + // be replaced. + // + // Attempting to insert a null value for the key + // will return an error. + Insert(key, value Value) (Dictionary, error) + + // Remove will remove the key/value pair that + // matches with the key. It will return a new + // Dictionary with the key/value removed. + Remove(key Value) Dictionary + + // Range will iterate over each element in + // the Dictionary. + Range(func(key, value Value)) + + // Len returns the number of elements inside of + // Dictionary. + Len() int +} + +type dict struct { + t semantic.MonoType + data *immutable.SortedMap +} + +func (d dict) Get(key, def Value) Value { + if !key.IsNull() { + v, ok := d.data.Get(key) + if ok { + return v.(Value) + } + } + return def +} + +func (d dict) Insert(key, value Value) (Dictionary, error) { + if key.IsNull() { + return nil, errors.New(codes.Invalid, "null value cannot be used as a dictionary key") + } + data := d.data.Set(key, value) + return dict{t: d.t, data: data}, nil +} + +func (d dict) Remove(key Value) Dictionary { + if key.IsNull() { + return d + } + data := d.data.Delete(key) + return dict{t: d.t, data: data} +} + +func (d dict) Range(f func(key, value Value)) { + itr := d.data.Iterator() + for { + key, value := itr.Next() + if key == nil { + return + } + f(key.(Value), value.(Value)) + } +} + +func (d dict) Len() int { + return d.data.Len() +} + +func (d dict) Type() semantic.MonoType { return d.t } +func (d dict) IsNull() bool { return false } +func (d dict) Str() string { + panic(UnexpectedKind(semantic.Dictionary, semantic.String)) +} +func (d dict) Bytes() []byte { + panic(UnexpectedKind(semantic.Dictionary, semantic.Bytes)) +} +func (d dict) Int() int64 { + panic(UnexpectedKind(semantic.Dictionary, semantic.Int)) +} +func (d dict) UInt() uint64 { + panic(UnexpectedKind(semantic.Dictionary, semantic.UInt)) +} +func (d dict) Float() float64 { + panic(UnexpectedKind(semantic.Dictionary, semantic.Float)) +} +func (d dict) Bool() bool { + panic(UnexpectedKind(semantic.Dictionary, semantic.Bool)) +} +func (d dict) Time() Time { + panic(UnexpectedKind(semantic.Dictionary, semantic.Time)) +} +func (d dict) Duration() Duration { + panic(UnexpectedKind(semantic.Dictionary, semantic.Duration)) +} +func (d dict) Regexp() *regexp.Regexp { + panic(UnexpectedKind(semantic.Dictionary, semantic.Regexp)) +} +func (d dict) Array() Array { + panic(UnexpectedKind(semantic.Dictionary, semantic.Array)) +} +func (d dict) Object() Object { + panic(UnexpectedKind(semantic.Dictionary, semantic.Object)) +} +func (d dict) Function() Function { + panic(UnexpectedKind(semantic.Dictionary, semantic.Function)) +} +func (d dict) Dict() Dictionary { + return d +} + +func (d dict) Equal(v Value) bool { + if !d.t.Equal(v.Type()) { + return false + } + + other := v.Dict() + if d.data.Len() != other.Len() { + return false + } + + equal := true + other.Range(func(key, value Value) { + if !equal { + return + } + + v, ok := d.data.Get(key) + if !ok { + equal = false + return + } + equal = value.Equal(v.(Value)) + }) + return equal +} + +type ( + intComparer struct{} + uintComparer struct{} + floatComparer struct{} + stringComparer struct{} + timeComparer struct{} +) + +func (c intComparer) Compare(a, b interface{}) int { + if i, j := a.(Value).Int(), b.(Value).Int(); i < j { + return -1 + } else if i > j { + return 1 + } + return 0 +} + +func (c uintComparer) Compare(a, b interface{}) int { + if i, j := a.(Value).UInt(), b.(Value).UInt(); i < j { + return -1 + } else if i > j { + return 1 + } + return 0 +} + +func (c floatComparer) Compare(a, b interface{}) int { + if i, j := a.(Value).Float(), b.(Value).Float(); i < j { + return -1 + } else if i > j { + return 1 + } + return 0 +} + +func (c stringComparer) Compare(a, b interface{}) int { + if i, j := a.(Value).Str(), b.(Value).Str(); i < j { + return -1 + } else if i > j { + return 1 + } + return 0 +} + +func (c timeComparer) Compare(a, b interface{}) int { + if i, j := a.(Value).Time(), b.(Value).Time(); i < j { + return -1 + } else if i > j { + return 1 + } + return 0 +} + +func dictComparer(dictType semantic.MonoType) immutable.Comparer { + if dictType.Nature() != semantic.Dictionary { + panic(UnexpectedKind(dictType.Nature(), semantic.Dictionary)) + } + keyType, err := dictType.KeyType() + if err != nil { + panic(err) + } + switch n := keyType.Nature(); n { + case semantic.Int: + return intComparer{} + case semantic.UInt: + return uintComparer{} + case semantic.Float: + return floatComparer{} + case semantic.String: + return stringComparer{} + case semantic.Time: + return timeComparer{} + default: + panic(errors.Newf(codes.Internal, "invalid key nature: %s", n)) + } +} + +// NewDict will construct a new Dictionary with the given key type. +func NewDict(dictType semantic.MonoType) Dictionary { + return dict{ + t: dictType, + data: immutable.NewSortedMap( + dictComparer(dictType), + ), + } +} + +// DictionaryBuilder can be used to construct a Dictionary +// with in-place memory instead of successive Insert calls +// that create new Dictionary values. +type DictionaryBuilder struct { + t semantic.MonoType + b *immutable.SortedMapBuilder +} + +// NewDictBuilder will create a new DictionaryBuilder for the given +// key type. +func NewDictBuilder(dictType semantic.MonoType) DictionaryBuilder { + builder := immutable.NewSortedMapBuilder( + immutable.NewSortedMap( + dictComparer(dictType), + ), + ) + return DictionaryBuilder{t: dictType, b: builder} +} + +// Dict will construct a new Dictionary using the inserted values. +func (d *DictionaryBuilder) Dict() Dictionary { + return dict{ + t: d.t, + data: d.b.Map(), + } +} + +// Get will retrieve a Value if it is present. +func (d *DictionaryBuilder) Get(key Value) (Value, bool) { + v, ok := d.b.Get(key) + if !ok { + return nil, false + } + return v.(Value), true +} + +// Insert will insert a new key/value pair into the Dictionary. +func (d *DictionaryBuilder) Insert(key, value Value) error { + if key.IsNull() { + return errors.New(codes.Invalid, "null value cannot be used as a dictionary key") + } + d.b.Set(key, value) + return nil +} + +// Remove will remove a key/value pair from the Dictionary. +func (d *DictionaryBuilder) Remove(key Value) { + if !key.IsNull() { + d.b.Delete(key) + } +} diff --git a/values/dict_test.go b/values/dict_test.go new file mode 100644 index 0000000000..6ea401dca0 --- /dev/null +++ b/values/dict_test.go @@ -0,0 +1,326 @@ +package values_test + +import ( + "math/rand" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/influxdata/flux/semantic" + "github.com/influxdata/flux/values" +) + +func TestDict_Type(t *testing.T) { + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + dict := values.NewDict(dictType) + + // Dictionary should return a valid dictionary type. + if want, got := dictType, dict.Type(); !cmp.Equal(want, got) { + t.Errorf("unexpected type -want/+got:\n%s", cmp.Diff(want, got)) + } + + b := values.NewDictBuilder(dictType) + dict = b.Dict() + + // Should have a valid dictionary type when using the builder. + if want, got := dictType, dict.Type(); !cmp.Equal(want, got) { + t.Errorf("unexpected type -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Should continue to have a valid dictionary type after an insert. + dict, _ = dict.Insert(values.NewInt(2), values.NewInt(4)) + if want, got := dictType, dict.Type(); !cmp.Equal(want, got) { + t.Errorf("unexpected type -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Should continue to have a valid dictionary type after a removal. + dict = dict.Remove(values.NewInt(2)) + if want, got := dictType, dict.Type(); !cmp.Equal(want, got) { + t.Errorf("unexpected type -want/+got:\n%s", cmp.Diff(want, got)) + } +} + +func TestDict_Get(t *testing.T) { + // Insert values to ensure this works with multiple values. + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + b.Insert(values.NewInt(0), values.NewInt(0)) + b.Insert(values.NewInt(1), values.NewInt(1)) + b.Insert(values.NewInt(2), values.NewInt(1)) + b.Insert(values.NewInt(3), values.NewInt(2)) + b.Insert(values.NewInt(4), values.NewInt(3)) + b.Insert(values.NewInt(5), values.NewInt(5)) + b.Insert(values.NewInt(6), values.NewInt(8)) + b.Insert(values.NewInt(7), values.NewInt(13)) + dict := b.Dict() + + // Retrieve existing value. + if want, got := values.NewInt(8), dict.Get(values.NewInt(6), values.Null); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Retrieve default value when non-existant value is retrieved. + if want, got := values.NewInt(-1), dict.Get(values.NewInt(8), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } +} + +func TestDict_Insert(t *testing.T) { + // Insert a single value into the dictionary. + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + _ = b.Insert(values.NewInt(7), values.NewInt(13)) + dict := b.Dict() + + // Retrieve a non-existant value. + if want, got := values.NewInt(-1), dict.Get(values.NewInt(6), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Insert that value and check that it exists. + dict2, _ := dict.Insert(values.NewInt(6), values.NewInt(8)) + if want, got := values.NewInt(8), dict2.Get(values.NewInt(6), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // The previous dictionary should not be changed. + if want, got := values.NewInt(-1), dict.Get(values.NewInt(6), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Replace an existing value. + dict3, _ := dict2.Insert(values.NewInt(7), values.NewInt(20)) + if want, got := values.NewInt(20), dict3.Get(values.NewInt(7), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // The previous dictionary should not be changed. + if want, got := values.NewInt(13), dict2.Get(values.NewInt(7), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // Attempts to insert null return an error. + if _, err := dict3.Insert(values.Null, values.NewInt(0)); err == nil { + t.Error("expected error") + } +} + +func TestDict_Remove(t *testing.T) { + // Insert two values into the dictionary. + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + _ = b.Insert(values.NewInt(6), values.NewInt(8)) + _ = b.Insert(values.NewInt(7), values.NewInt(13)) + dict := b.Dict() + + // Remove one value. + dict2 := dict.Remove(values.NewInt(6)) + + // It should no longer be present in the dictionary. + if want, got := values.NewInt(-1), dict2.Get(values.NewInt(6), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // The other value should be present. + if want, got := values.NewInt(13), dict2.Get(values.NewInt(7), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } + + // If we check the original dictionary, the value is still present. + if want, got := values.NewInt(8), dict.Get(values.NewInt(6), values.NewInt(-1)); !want.Equal(got) { + t.Errorf("unexpected value -want/+got:\n%s", cmp.Diff(want, got)) + } +} + +func TestDict_Range(t *testing.T) { + dictType := semantic.NewDictType(semantic.BasicString, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + b.Insert(values.NewString("a"), values.NewInt(2)) + b.Insert(values.NewString("b"), values.NewInt(6)) + b.Insert(values.NewString("c"), values.NewInt(4)) + dict := b.Dict() + + want := map[string]int64{ + "a": 2, + "b": 6, + "c": 4, + } + dict.Range(func(key, value values.Value) { + if want, got := want[key.Str()], value.Int(); want != got { + t.Errorf("unexpected value -want/+got:\n\t- %d\n\t+ %d", want, got) + } + delete(want, key.Str()) + }) + + if len(want) > 0 { + t.Errorf("some values were not checked: %v", want) + } +} + +func TestDict_Len(t *testing.T) { + dictType := semantic.NewDictType(semantic.BasicString, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + b.Insert(values.NewString("a"), values.NewInt(2)) + b.Insert(values.NewString("b"), values.NewInt(6)) + b.Insert(values.NewString("c"), values.NewInt(4)) + dict := b.Dict() + + // The starting length should be 3. + if want, got := 3, dict.Len(); want != got { + t.Errorf("unexpected length -want/+got:\n\t- %d\n\t+ %d", want, got) + } + + // Insert a value and check the length. + dict2, _ := dict.Insert(values.NewString("d"), values.NewInt(3)) + if want, got := 4, dict2.Len(); want != got { + t.Errorf("unexpected length -want/+got:\n\t- %d\n\t+ %d", want, got) + } + + // The original length should not change. + if want, got := 3, dict.Len(); want != got { + t.Errorf("unexpected length -want/+got:\n\t- %d\n\t+ %d", want, got) + } + + // Removing an element changes the length. + dict3 := dict.Remove(values.NewString("c")) + if want, got := 2, dict3.Len(); want != got { + t.Errorf("unexpected length -want/+got:\n\t- %d\n\t+ %d", want, got) + } + + // The original length should not change. + if want, got := 3, dict.Len(); want != got { + t.Errorf("unexpected length -want/+got:\n\t- %d\n\t+ %d", want, got) + } +} + +func TestDict_Equal(t *testing.T) { + dictType := semantic.NewDictType(semantic.BasicString, semantic.BasicInt) + b := values.NewDictBuilder(dictType) + b.Insert(values.NewString("a"), values.NewInt(2)) + b.Insert(values.NewString("b"), values.NewInt(6)) + b.Insert(values.NewString("c"), values.NewInt(4)) + dict := b.Dict() + + // Should equal itself. + if !dict.Equal(dict) { + t.Error("expected values to be equal") + } + + // Should not equal other values. + if dict.Equal(values.NewString("a")) { + t.Error("expected values to be not equal") + } + + // Insert a value and they should not be equal. + dict2, _ := dict.Insert(values.NewString("d"), values.NewInt(5)) + if dict.Equal(dict2) { + t.Error("expected values to be not equal") + } + + // Remove the value and they should be equal again. + dict3 := dict2.Remove(values.NewString("d")) + if !dict.Equal(dict3) { + t.Error("expected values to be equal") + } + + // Overwrite an existing value and they should not be equal. + dict4, _ := dict.Insert(values.NewString("c"), values.NewInt(0)) + if dict.Equal(dict4) { + t.Error("expected values to be not equal") + } +} + +var benchmarkKeys []values.Value + +func init() { + benchmarkKeys = make([]values.Value, 0, 100) + + gen := rand.New(rand.NewSource(0)) + for i := 0; i < 100; i++ { + key := values.NewInt(gen.Int63()) + benchmarkKeys = append(benchmarkKeys, key) + } +} + +func BenchmarkDict_Get(b *testing.B) { + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + dict := values.NewDict(dictType) + gen := rand.New(rand.NewSource(time.Now().UnixNano())) + for _, key := range benchmarkKeys { + value := values.NewInt(gen.Int63()) + dict, _ = dict.Insert(key, value) + } + + b.ResetTimer() + b.ReportAllocs() + + def := values.NewInt(0) + for i := 0; i < b.N; i++ { + for _, key := range benchmarkKeys { + dict.Get(key, def) + } + } +} + +func BenchmarkDict_Insert(b *testing.B) { + // We're going to insert repeatedly to the dictionary + // with random pre-determined values to random locations. + gen := rand.New(rand.NewSource(time.Now().UnixNano())) + + dvalues := make([]values.Value, len(benchmarkKeys)) + for i := 0; i < len(benchmarkKeys); i++ { + dvalues[i] = values.NewInt(gen.Int63()) + } + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + dict := values.NewDict(dictType) + for i, key := range benchmarkKeys { + dict, _ = dict.Insert(key, dvalues[i]) + } + } +} + +func BenchmarkDict_Remove(b *testing.B) { + // We're going to insert values to each of the benchmark + // keys and then determine an order to remove them. + dictType := semantic.NewDictType(semantic.BasicInt, semantic.BasicInt) + dict := values.NewDict(dictType) + gen := rand.New(rand.NewSource(time.Now().UnixNano())) + + // Generate random values for each of the benchmark keys. + for _, key := range benchmarkKeys { + value := values.NewInt(gen.Int63()) + dict, _ = dict.Insert(key, value) + } + + // Determine an order to remove them. We use a pre-determined + // random order to give an idea of how the dictionary deals + // with random access while minimizing the affect the random + // number generator has on timing. + // The order is determined by doing a Fisher-Yates shuffle. + indices := make([]int, len(benchmarkKeys)) + for i := range indices { + indices[i] = i + } + for i := len(indices) - 1; i > 0; i-- { + j := gen.Intn(i + 1) + indices[i], indices[j] = indices[j], indices[i] + } + + keys := make([]values.Value, len(indices)) + for i, idx := range indices { + keys[i] = benchmarkKeys[idx] + } + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + d := dict + for _, key := range keys { + d = d.Remove(key) + } + } +} diff --git a/values/function.go b/values/function.go index 2184525e6d..9a02c511aa 100644 --- a/values/function.go +++ b/values/function.go @@ -96,6 +96,10 @@ func (f *function) Function() Function { return f } +func (f *function) Dict() Dictionary { + panic(UnexpectedKind(semantic.Function, semantic.Dictionary)) +} + func (f *function) Equal(rhs Value) bool { if f.t != rhs.Type() { return false diff --git a/values/object.go b/values/object.go index c73ba46c0f..1db03cd5cb 100644 --- a/values/object.go +++ b/values/object.go @@ -245,6 +245,9 @@ func (o *object) Object() Object { func (o *object) Function() Function { panic(UnexpectedKind(semantic.Object, semantic.Function)) } +func (o *object) Dict() Dictionary { + panic(UnexpectedKind(semantic.Object, semantic.Dictionary)) +} func (o *object) Equal(rhs Value) bool { if rhs.Type().Nature() != semantic.Object { return false diff --git a/values/objects/table.go b/values/objects/table.go index 1f9b2f0ccd..8d24c9e85d 100644 --- a/values/objects/table.go +++ b/values/objects/table.go @@ -140,6 +140,10 @@ func (t *Table) Function() values.Function { panic(values.UnexpectedKind(semantic.Object, semantic.Function)) } +func (t *Table) Dict() values.Dictionary { + panic(values.UnexpectedKind(semantic.Object, semantic.Dictionary)) +} + // Table returns a copy of the Table that can be called // with Do. Either Do or Done must be called on the // returned Table. diff --git a/values/values.go b/values/values.go index eec85dd9c0..b5b2a07953 100644 --- a/values/values.go +++ b/values/values.go @@ -31,6 +31,7 @@ type Value interface { Array() Array Object() Object Function() Function + Dict() Dictionary Equal(Value) bool } @@ -97,6 +98,10 @@ func (v value) Function() Function { CheckKind(v.t.Nature(), semantic.Function) return v.v.(Function) } +func (v value) Dict() Dictionary { + CheckKind(v.t.Nature(), semantic.Dictionary) + return v.v.(Dictionary) +} func (v value) Equal(r Value) bool { if v.Type().Nature() != r.Type().Nature() { return false @@ -131,6 +136,8 @@ func (v value) Equal(r Value) bool { return v.Array().Equal(r.Array()) case semantic.Function: return v.Function().Equal(r.Function()) + case semantic.Dictionary: + return v.Dict().Equal(r.Dict()) default: return false } @@ -148,7 +155,7 @@ var ( Null = null{} ) -// Extract the primitive value from the Value interface. +// Unwrap will extract the primitive value from the Value interface. func Unwrap(v Value) interface{} { if v.IsNull() { return nil @@ -188,6 +195,14 @@ func Unwrap(v Value) interface{} { o[k] = val }) return o + case semantic.Dictionary: + dict := v.Dict() + d := make(map[interface{}]interface{}, dict.Len()) + dict.Range(func(key, value Value) { + k := Unwrap(key) + d[k] = Unwrap(value) + }) + return d case semantic.Function: // there is no primitive value for a Function object, just return itself. return v @@ -294,7 +309,7 @@ func CheckKind(got, exp semantic.Nature) { } } -// isTimeable checks if value v is Timeable +// IsTimeable checks if value v is Timeable. func IsTimeable(v Value) bool { return v.Type().Nature() == semantic.Time || v.Type().Nature() == semantic.Duration } @@ -315,4 +330,5 @@ func (n null) Regexp() *regexp.Regexp { panic(UnexpectedKind(semantic.Invalid, func (n null) Array() Array { panic(UnexpectedKind(semantic.Invalid, semantic.Array)) } func (n null) Object() Object { panic(UnexpectedKind(semantic.Invalid, semantic.Object)) } func (n null) Function() Function { panic(UnexpectedKind(semantic.Invalid, semantic.Function)) } +func (n null) Dict() Dictionary { panic(UnexpectedKind(semantic.Invalid, semantic.Dictionary)) } func (n null) Equal(Value) bool { return false }