From d8d56d63fb0bcf825a0ef010cc96950b7687bd53 Mon Sep 17 00:00:00 2001 From: js00070 <909645105@qq.com> Date: Sat, 12 Oct 2019 21:13:43 +0800 Subject: [PATCH 1/5] fix vectorized builtinJSONObjectSig --- expression/builtin_json_vec.go | 44 ++++++++++++++++++++++-- expression/builtin_json_vec_test.go | 53 +++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go index 1e78a8800f903..b6b75c555f08b 100644 --- a/expression/builtin_json_vec.go +++ b/expression/builtin_json_vec.go @@ -15,6 +15,8 @@ package expression import ( "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" ) @@ -91,11 +93,49 @@ func (b *builtinJSONSetSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Column } func (b *builtinJSONObjectSig) vectorized() bool { - return false + return true } func (b *builtinJSONObjectSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + nr := input.NumRows() + if len(b.args)&1 == 1 { + err := ErrIncorrectParameterCount.GenWithStackByArgs(ast.JSONObject) + return err + } + + jsons := make([]map[string]interface{}, nr) + for i := 0; i < nr; i++ { + jsons[i] = make(map[string]interface{}, len(b.args)>>1) + } + + result.ReserveJSON(nr) + for i := 0; i < len(b.args); i++ { + if i&1 == 1 { + keyCol := input.Column(i - 1) + valueCol := input.Column(i) + + var key string + var value json.BinaryJSON + for j := 0; j < nr; j++ { + if keyCol.IsNull(j) { + err := errors.New("JSON documents may not contain NULL member names") + return err + } + key = keyCol.GetString(j) + if valueCol.IsNull(j) { + value = json.CreateBinary(nil) + } else { + value = valueCol.GetJSON(j) + } + jsons[j][key] = value + } + } + } + + for i := 0; i < nr; i++ { + result.AppendJSON(json.CreateBinary(jsons[i])) + } + return nil } func (b *builtinJSONArrayInsertSig) vectorized() bool { diff --git a/expression/builtin_json_vec_test.go b/expression/builtin_json_vec_test.go index c98631d9dd578..b4466c21b8ddd 100644 --- a/expression/builtin_json_vec_test.go +++ b/expression/builtin_json_vec_test.go @@ -18,6 +18,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/types" ) var vecBuiltinJSONCases = map[string][]vecExprBenchCase{ @@ -30,16 +31,48 @@ var vecBuiltinJSONCases = map[string][]vecExprBenchCase{ ast.JSONArray: {}, ast.JSONArrayInsert: {}, ast.JSONContains: {}, - ast.JSONObject: {}, - ast.JSONSet: {}, - ast.JSONSearch: {}, - ast.JSONReplace: {}, - ast.JSONDepth: {}, - ast.JSONUnquote: {}, - ast.JSONRemove: {}, - ast.JSONMerge: {}, - ast.JSONInsert: {}, - ast.JSONQuote: {}, + ast.JSONObject: { + { + retEvalType: types.ETJson, + childrenTypes: []types.EvalType{ + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + types.ETString, types.ETJson, + }, + geners: []dataGenerator{ + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + &randLenStrGener{10, 20}, nil, + }, + }, + }, + ast.JSONSet: {}, + ast.JSONSearch: {}, + ast.JSONReplace: {}, + ast.JSONDepth: {}, + ast.JSONUnquote: {}, + ast.JSONRemove: {}, + ast.JSONMerge: {}, + ast.JSONInsert: {}, + ast.JSONQuote: {}, } func (s *testEvaluatorSuite) TestVectorizedBuiltinJSONFunc(c *C) { From 714b892e9f81361923601e091579905a66513b03 Mon Sep 17 00:00:00 2001 From: js00070 <909645105@qq.com> Date: Mon, 14 Oct 2019 15:03:00 +0800 Subject: [PATCH 2/5] fix builtinJSONObjectSig --- expression/builtin_json_vec.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go index b6b75c555f08b..efe0b856a6e7e 100644 --- a/expression/builtin_json_vec.go +++ b/expression/builtin_json_vec.go @@ -16,6 +16,7 @@ package expression import ( "github.com/pingcap/errors" "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" ) @@ -108,11 +109,38 @@ func (b *builtinJSONObjectSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Col jsons[i] = make(map[string]interface{}, len(b.args)>>1) } + argBuffers := make([]*chunk.Column, len(b.args)) + var err error + for i := 0; i < len(b.args); i++ { + if i&1 == 0 { + argBuffers[i], err = b.bufAllocator.get(types.ETString, nr) + if err != nil { + return err + } + if err := b.args[i].VecEvalString(b.ctx, input, argBuffers[i]); err != nil { + return err + } + } else { + argBuffers[i], err = b.bufAllocator.get(types.ETJson, nr) + if err != nil { + return err + } + if err := b.args[i].VecEvalJSON(b.ctx, input, argBuffers[i]); err != nil { + return err + } + } + } + defer func() { + for i := 0; i < len(argBuffers); i++ { + b.bufAllocator.put(argBuffers[i]) + } + }() + result.ReserveJSON(nr) for i := 0; i < len(b.args); i++ { if i&1 == 1 { - keyCol := input.Column(i - 1) - valueCol := input.Column(i) + keyCol := argBuffers[i-1] + valueCol := argBuffers[i] var key string var value json.BinaryJSON From 39dce3559907a02da43807558e99e9e8095b0532 Mon Sep 17 00:00:00 2001 From: js00070 <909645105@qq.com> Date: Mon, 14 Oct 2019 15:58:07 +0800 Subject: [PATCH 3/5] code style fix --- expression/builtin_json_vec.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go index efe0b856a6e7e..52b73c3f56c08 100644 --- a/expression/builtin_json_vec.go +++ b/expression/builtin_json_vec.go @@ -113,19 +113,17 @@ func (b *builtinJSONObjectSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Col var err error for i := 0; i < len(b.args); i++ { if i&1 == 0 { - argBuffers[i], err = b.bufAllocator.get(types.ETString, nr) - if err != nil { + if argBuffers[i], err = b.bufAllocator.get(types.ETString, nr); err != nil { return err } - if err := b.args[i].VecEvalString(b.ctx, input, argBuffers[i]); err != nil { + if err = b.args[i].VecEvalString(b.ctx, input, argBuffers[i]); err != nil { return err } } else { - argBuffers[i], err = b.bufAllocator.get(types.ETJson, nr) - if err != nil { + if argBuffers[i], err = b.bufAllocator.get(types.ETJson, nr); err != nil { return err } - if err := b.args[i].VecEvalJSON(b.ctx, input, argBuffers[i]); err != nil { + if err = b.args[i].VecEvalJSON(b.ctx, input, argBuffers[i]); err != nil { return err } } From 0f17a2307858417305e13d5b6586ef9d812eb640 Mon Sep 17 00:00:00 2001 From: js00070 <909645105@qq.com> Date: Tue, 15 Oct 2019 11:16:03 +0800 Subject: [PATCH 4/5] fix buffer allocate problem in vectorized builtinJSONObjectSig --- expression/builtin_json_vec.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go index 52b73c3f56c08..e2bf72c63ee6f 100644 --- a/expression/builtin_json_vec.go +++ b/expression/builtin_json_vec.go @@ -116,6 +116,10 @@ func (b *builtinJSONObjectSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Col if argBuffers[i], err = b.bufAllocator.get(types.ETString, nr); err != nil { return err } + defer func(buf *chunk.Column) { + b.bufAllocator.put(buf) + }(argBuffers[i]) + if err = b.args[i].VecEvalString(b.ctx, input, argBuffers[i]); err != nil { return err } @@ -123,16 +127,15 @@ func (b *builtinJSONObjectSig) vecEvalJSON(input *chunk.Chunk, result *chunk.Col if argBuffers[i], err = b.bufAllocator.get(types.ETJson, nr); err != nil { return err } + defer func(buf *chunk.Column) { + b.bufAllocator.put(buf) + }(argBuffers[i]) + if err = b.args[i].VecEvalJSON(b.ctx, input, argBuffers[i]); err != nil { return err } } } - defer func() { - for i := 0; i < len(argBuffers); i++ { - b.bufAllocator.put(argBuffers[i]) - } - }() result.ReserveJSON(nr) for i := 0; i < len(b.args); i++ { From c8f5c98a4bb502c9f6ffb6ffed348a597c1813e6 Mon Sep 17 00:00:00 2001 From: js00070 <909645105@qq.com> Date: Wed, 23 Oct 2019 17:44:27 +0800 Subject: [PATCH 5/5] satisfy gofmt --- expression/builtin_json_vec_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/expression/builtin_json_vec_test.go b/expression/builtin_json_vec_test.go index cf21cbde2f149..db3a26d4a5d65 100644 --- a/expression/builtin_json_vec_test.go +++ b/expression/builtin_json_vec_test.go @@ -73,9 +73,9 @@ var vecBuiltinJSONCases = map[string][]vecExprBenchCase{ ast.JSONUnquote: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&jsonStringGener{}}}, }, - ast.JSONRemove: {}, - ast.JSONMerge: {}, - ast.JSONInsert: {}, + ast.JSONRemove: {}, + ast.JSONMerge: {}, + ast.JSONInsert: {}, ast.JSONQuote: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETJson}}, },