From e57561f226e4d0de2c4537aa37ec776fcd317e80 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 30 Nov 2022 13:32:44 +0800 Subject: [PATCH 1/4] util, executor: refine the memory usage of delete multiple tables --- executor/delete.go | 11 +++++++---- util/chunk/row.go | 22 +++++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/executor/delete.go b/executor/delete.go index 9e64a47669f9f..8b97d45a028fb 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -176,7 +176,9 @@ func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []p } // tblRowMap[info.TblID][handle] hold the row datas binding to this table and this handle. _, exist := tblRowMap[info.TblID].Get(handle) - memDelta := tblRowMap[info.TblID].Set(handle, joinedRow[info.Start:info.End]) + row := make([]types.Datum, info.End-info.Start) + copy(row, joinedRow[info.Start:info.End]) + memDelta := tblRowMap[info.TblID].Set(handle, row) if !exist { memDelta += types.EstimatedMemUsage(joinedRow, 1) memDelta += int64(handle.ExtraMemSize()) @@ -192,6 +194,7 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error { fields := retTypes(e.children[0]) chk := tryNewCacheChunk(e.children[0]) memUsageOfChk := int64(0) + joinedDatumRowBuffer := make([]types.Datum, len(fields)) for { e.memTracker.Consume(-memUsageOfChk) iter := chunk.NewIterator4Chunk(chk) @@ -206,13 +209,13 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error { e.memTracker.Consume(memUsageOfChk) for joinedChunkRow := iter.Begin(); joinedChunkRow != iter.End(); joinedChunkRow = iter.Next() { - joinedDatumRow := joinedChunkRow.GetDatumRow(fields) - err := e.composeTblRowMap(tblRowMap, colPosInfos, joinedDatumRow) + joinedDatumRowBuffer = joinedChunkRow.GetDatumRowWithBuffer(fields, joinedDatumRowBuffer) + err := e.composeTblRowMap(tblRowMap, colPosInfos, joinedDatumRowBuffer) if err != nil { return err } } - chk = chunk.Renew(chk, e.maxChunkSize) + chk = tryNewCacheChunk(e.children[0]) } return e.removeRowsInTblRowMap(tblRowMap) diff --git a/util/chunk/row.go b/util/chunk/row.go index 96aa92acb65fb..9613e552e634e 100644 --- a/util/chunk/row.go +++ b/util/chunk/row.go @@ -15,10 +15,9 @@ package chunk import ( - "strconv" - "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/types" + "strconv" ) // Row represents a row of data, can be used to access values. @@ -115,10 +114,14 @@ func (r Row) GetJSON(colIdx int) types.BinaryJSON { // Keep in mind that GetDatumRow has a reference to r.c, which is a chunk, // this function works only if the underlying chunk is valid or unchanged. func (r Row) GetDatumRow(fields []*types.FieldType) []types.Datum { - datumRow := make([]types.Datum, 0, r.c.NumCols()) - for colIdx := 0; colIdx < r.c.NumCols(); colIdx++ { - datum := r.GetDatum(colIdx, fields[colIdx]) - datumRow = append(datumRow, datum) + datumRow := make([]types.Datum, r.c.NumCols()) + return r.GetDatumRowWithBuffer(fields, datumRow) +} + +// GetDatumRowWithBuffer +func (r Row) GetDatumRowWithBuffer(fields []*types.FieldType, datumRow []types.Datum) []types.Datum { + for colIdx := 0; colIdx < len(datumRow); colIdx++ { + r.GetDatumWithBuffer(colIdx, fields[colIdx], &datumRow[colIdx]) } return datumRow } @@ -126,6 +129,12 @@ func (r Row) GetDatumRow(fields []*types.FieldType) []types.Datum { // GetDatum implements the chunk.Row interface. func (r Row) GetDatum(colIdx int, tp *types.FieldType) types.Datum { var d types.Datum + r.GetDatumWithBuffer(colIdx, tp, &d) + return d +} + +// GetDatumWithBuffer implements the chunk.Row interface. +func (r Row) GetDatumWithBuffer(colIdx int, tp *types.FieldType, d *types.Datum) { switch tp.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: if !r.IsNull(colIdx) { @@ -192,7 +201,6 @@ func (r Row) GetDatum(colIdx int, tp *types.FieldType) types.Datum { d.SetMysqlJSON(r.GetJSON(colIdx)) } } - return d } // GetRaw returns the underlying raw bytes with the colIdx. From 47dba1cf3f3821fb344b323d500524a4f7f61783 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 30 Nov 2022 13:37:52 +0800 Subject: [PATCH 2/4] tiny change --- util/chunk/row.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/chunk/row.go b/util/chunk/row.go index 9613e552e634e..b9cfc9f180de9 100644 --- a/util/chunk/row.go +++ b/util/chunk/row.go @@ -15,9 +15,10 @@ package chunk import ( + "strconv" + "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/types" - "strconv" ) // Row represents a row of data, can be used to access values. @@ -118,7 +119,7 @@ func (r Row) GetDatumRow(fields []*types.FieldType) []types.Datum { return r.GetDatumRowWithBuffer(fields, datumRow) } -// GetDatumRowWithBuffer +// GetDatumRowWithBuffer gets datum using the buffer datumRow. func (r Row) GetDatumRowWithBuffer(fields []*types.FieldType, datumRow []types.Datum) []types.Datum { for colIdx := 0; colIdx < len(datumRow); colIdx++ { r.GetDatumWithBuffer(colIdx, fields[colIdx], &datumRow[colIdx]) @@ -133,7 +134,7 @@ func (r Row) GetDatum(colIdx int, tp *types.FieldType) types.Datum { return d } -// GetDatumWithBuffer implements the chunk.Row interface. +// GetDatumWithBuffer gets datum using the buffer d. func (r Row) GetDatumWithBuffer(colIdx int, tp *types.FieldType, d *types.Datum) { switch tp.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: From 75574e8d2fc1b1cae9fb6a10e73ba67c4a5fe969 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 30 Nov 2022 15:17:54 +0800 Subject: [PATCH 3/4] refine --- executor/delete.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/executor/delete.go b/executor/delete.go index 8b97d45a028fb..3aa0932a07c22 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -175,9 +175,13 @@ func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []p return err } // tblRowMap[info.TblID][handle] hold the row datas binding to this table and this handle. - _, exist := tblRowMap[info.TblID].Get(handle) - row := make([]types.Datum, info.End-info.Start) - copy(row, joinedRow[info.Start:info.End]) + row, exist := tblRowMap[info.TblID].Get(handle) + if !exist { + row = make([]types.Datum, info.End-info.Start) + } + for i, d := range joinedRow[info.Start:info.End] { + d.Copy(&row[i]) + } memDelta := tblRowMap[info.TblID].Set(handle, row) if !exist { memDelta += types.EstimatedMemUsage(joinedRow, 1) From a1957d2f65ebfe75e11a54b94d743380c4539794 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 30 Nov 2022 17:55:24 +0800 Subject: [PATCH 4/4] lint --- util/chunk/row.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/chunk/row.go b/util/chunk/row.go index b9cfc9f180de9..04aba8f768bdd 100644 --- a/util/chunk/row.go +++ b/util/chunk/row.go @@ -135,7 +135,7 @@ func (r Row) GetDatum(colIdx int, tp *types.FieldType) types.Datum { } // GetDatumWithBuffer gets datum using the buffer d. -func (r Row) GetDatumWithBuffer(colIdx int, tp *types.FieldType, d *types.Datum) { +func (r Row) GetDatumWithBuffer(colIdx int, tp *types.FieldType, d *types.Datum) types.Datum { switch tp.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: if !r.IsNull(colIdx) { @@ -202,6 +202,7 @@ func (r Row) GetDatumWithBuffer(colIdx int, tp *types.FieldType, d *types.Datum) d.SetMysqlJSON(r.GetJSON(colIdx)) } } + return *d } // GetRaw returns the underlying raw bytes with the colIdx.