From cf349414012cafccb5069e08dd49235ec09d9cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Thu, 29 Dec 2022 16:30:21 +0800 Subject: [PATCH] ttl: make the TTL compatible with dumpling, lightning and BR (#40216) close pingcap/tidb#40215 --- br/pkg/lightning/restore/tidb.go | 2 +- br/pkg/restore/db.go | 8 +++ br/pkg/stream/BUILD.bazel | 3 ++ br/pkg/stream/rewrite_meta_rawkv.go | 5 ++ br/pkg/stream/rewrite_meta_rawkv_test.go | 49 +++++++++++++++++ br/tests/br_ttl/run.sh | 54 +++++++++++++++++++ br/tests/lightning_ttl/config.toml | 2 + .../data/ttldb-schema-create.sql | 1 + .../lightning_ttl/data/ttldb.t1-schema.sql | 4 ++ br/tests/lightning_ttl/run.sh | 26 +++++++++ parser/ast/ddl.go | 41 ++++++++++++-- parser/ast/ddl_test.go | 34 ++++++++++++ parser/format/format.go | 6 +++ 13 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 br/tests/br_ttl/run.sh create mode 100644 br/tests/lightning_ttl/config.toml create mode 100644 br/tests/lightning_ttl/data/ttldb-schema-create.sql create mode 100644 br/tests/lightning_ttl/data/ttldb.t1-schema.sql create mode 100644 br/tests/lightning_ttl/run.sh diff --git a/br/pkg/lightning/restore/tidb.go b/br/pkg/lightning/restore/tidb.go index 00d49af646e3c..98c780e65dc98 100644 --- a/br/pkg/lightning/restore/tidb.go +++ b/br/pkg/lightning/restore/tidb.go @@ -152,7 +152,7 @@ func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string } var res strings.Builder - ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment, &res) + ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment|format.RestoreWithTTLEnableOff, &res) retStmts := make([]string, 0, len(stmts)) for _, stmt := range stmts { diff --git a/br/pkg/restore/db.go b/br/pkg/restore/db.go index 132e3294f1617..1f3f5d949e26e 100644 --- a/br/pkg/restore/db.go +++ b/br/pkg/restore/db.go @@ -308,6 +308,10 @@ func (db *DB) CreateTables(ctx context.Context, tables []*metautil.Table, return errors.Trace(err) } } + + if ttlInfo := table.Info.TTLInfo; ttlInfo != nil { + ttlInfo.Enable = false + } } if err := batchSession.CreateTables(ctx, m, db.tableIDAllocFilter()); err != nil { return err @@ -336,6 +340,10 @@ func (db *DB) CreateTable(ctx context.Context, table *metautil.Table, } } + if ttlInfo := table.Info.TTLInfo; ttlInfo != nil { + ttlInfo.Enable = false + } + err := db.se.CreateTable(ctx, table.DB.Name, table.Info, db.tableIDAllocFilter()) if err != nil { log.Error("create table failed", diff --git a/br/pkg/stream/BUILD.bazel b/br/pkg/stream/BUILD.bazel index f75f9f37d81ea..e5fbc5c87b870 100644 --- a/br/pkg/stream/BUILD.bazel +++ b/br/pkg/stream/BUILD.bazel @@ -56,8 +56,11 @@ go_test( "//br/pkg/storage", "//br/pkg/streamhelper", "//meta", + "//parser/ast", "//parser/model", + "//parser/mysql", "//tablecodec", + "//types", "//util/codec", "//util/table-filter", "@com_github_pingcap_kvproto//pkg/brpb", diff --git a/br/pkg/stream/rewrite_meta_rawkv.go b/br/pkg/stream/rewrite_meta_rawkv.go index 7398abdbb2cb9..3c559ec124ad8 100644 --- a/br/pkg/stream/rewrite_meta_rawkv.go +++ b/br/pkg/stream/rewrite_meta_rawkv.go @@ -336,6 +336,11 @@ func (sr *SchemasReplace) rewriteTableInfo(value []byte, dbID int64) ([]byte, bo } } + // Force to disable TTL_ENABLE when restore + if newTableInfo.TTLInfo != nil { + newTableInfo.TTLInfo.Enable = false + } + if sr.AfterTableRewritten != nil { sr.AfterTableRewritten(false, newTableInfo) } diff --git a/br/pkg/stream/rewrite_meta_rawkv_test.go b/br/pkg/stream/rewrite_meta_rawkv_test.go index d2cbe24e8295d..cd3cf00d46305 100644 --- a/br/pkg/stream/rewrite_meta_rawkv_test.go +++ b/br/pkg/stream/rewrite_meta_rawkv_test.go @@ -7,7 +7,10 @@ import ( "encoding/json" "testing" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/types" filter "github.com/pingcap/tidb/util/table-filter" "github.com/stretchr/testify/require" ) @@ -312,6 +315,52 @@ func TestRewriteValueForExchangePartition(t *testing.T) { require.Equal(t, tableInfo.ID, pt1ID+100) } +func TestRewriteValueForTTLTable(t *testing.T) { + var ( + dbId int64 = 40 + tableID int64 = 100 + colID int64 = 1000 + colName = "t" + tableName = "t1" + tableInfo model.TableInfo + ) + + tbl := model.TableInfo{ + ID: tableID, + Name: model.NewCIStr(tableName), + Columns: []*model.ColumnInfo{ + { + ID: colID, + Name: model.NewCIStr(colName), + FieldType: *types.NewFieldType(mysql.TypeTimestamp), + }, + }, + TTLInfo: &model.TTLInfo{ + ColumnName: model.NewCIStr(colName), + IntervalExprStr: "1", + IntervalTimeUnit: int(ast.TimeUnitDay), + Enable: true, + }, + } + value, err := json.Marshal(&tbl) + require.Nil(t, err) + + sr := MockEmptySchemasReplace(nil) + newValue, needRewrite, err := sr.rewriteTableInfo(value, dbId) + require.Nil(t, err) + require.True(t, needRewrite) + + err = json.Unmarshal(newValue, &tableInfo) + require.Nil(t, err) + require.Equal(t, tableInfo.Name.String(), tableName) + require.Equal(t, tableInfo.ID, sr.DbMap[dbId].TableMap[tableID].NewTableID) + require.NotNil(t, tableInfo.TTLInfo) + require.Equal(t, colName, tableInfo.TTLInfo.ColumnName.O) + require.Equal(t, "1", tableInfo.TTLInfo.IntervalExprStr) + require.Equal(t, int(ast.TimeUnitDay), tableInfo.TTLInfo.IntervalTimeUnit) + require.False(t, tableInfo.TTLInfo.Enable) +} + // db:70->80 - // | - t0:71->81 - // | | - p0:72->82 diff --git a/br/tests/br_ttl/run.sh b/br/tests/br_ttl/run.sh new file mode 100644 index 0000000000000..cfb1a38c8281b --- /dev/null +++ b/br/tests/br_ttl/run.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Copyright 2022 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" + +PROGRESS_FILE="$TEST_DIR/progress_file" +BACKUPMETAV1_LOG="$TEST_DIR/backup.log" +BACKUPMETAV2_LOG="$TEST_DIR/backupv2.log" +RESTORE_LOG="$TEST_DIR/restore.log" +rm -rf $PROGRESS_FILE + +run_sql "create schema $DB;" +run_sql "create table $DB.ttl_test_tbl(id int primary key, t datetime) TTL=\`t\` + interval 1 day TTL_ENABLE='ON'" + +# backup db +echo "full backup meta v2 start..." +unset BR_LOG_TO_TERM +rm -f $BACKUPMETAV2_LOG +run_br backup full --log-file $BACKUPMETAV2_LOG -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR --use-backupmeta-v2 + +echo "full backup meta v1 start..." +rm -f $BACKUPMETAV1_LOG +run_br backup full --log-file $BACKUPMETAV1_LOG -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +TTL_MARK='![ttl]' +CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */" + +# restore v2 +run_sql "DROP DATABASE $DB;" +echo "restore ttl table start v2..." +run_br restore db --db $DB -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR +run_sql "show create table $DB.ttl_test_tbl;" +check_contains "$CREATE_SQL_CONTAINS" + +# restore v1 +run_sql "DROP DATABASE $DB;" +echo "restore ttl table start v1..." +run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR +run_sql "show create table $DB.ttl_test_tbl;" +check_contains "$CREATE_SQL_CONTAINS" diff --git a/br/tests/lightning_ttl/config.toml b/br/tests/lightning_ttl/config.toml new file mode 100644 index 0000000000000..d2152b47c922a --- /dev/null +++ b/br/tests/lightning_ttl/config.toml @@ -0,0 +1,2 @@ +[tikv-importer] +backend = 'local' diff --git a/br/tests/lightning_ttl/data/ttldb-schema-create.sql b/br/tests/lightning_ttl/data/ttldb-schema-create.sql new file mode 100644 index 0000000000000..46609f11e6635 --- /dev/null +++ b/br/tests/lightning_ttl/data/ttldb-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE `ttldb`; diff --git a/br/tests/lightning_ttl/data/ttldb.t1-schema.sql b/br/tests/lightning_ttl/data/ttldb.t1-schema.sql new file mode 100644 index 0000000000000..7531d7f18ae01 --- /dev/null +++ b/br/tests/lightning_ttl/data/ttldb.t1-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE `t1` ( + `id` int(11) PRIMARY KEY, + `t` datetime +) TTL = `t` + INTERVAL 1 DAY TTL_ENABLE = 'ON'; diff --git a/br/tests/lightning_ttl/run.sh b/br/tests/lightning_ttl/run.sh new file mode 100644 index 0000000000000..4a1d9ffc04d57 --- /dev/null +++ b/br/tests/lightning_ttl/run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# +# Copyright 2022 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +run_sql 'drop database if exists ttldb;' +run_lightning + +TTL_MARK='![ttl]' +CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */" + +run_sql 'show create table ttldb.t1' +check_contains "$CREATE_SQL_CONTAINS" diff --git a/parser/ast/ddl.go b/parser/ast/ddl.go index f6d60e16863cd..fa9507ea0e7b4 100644 --- a/parser/ast/ddl.go +++ b/parser/ast/ddl.go @@ -1096,7 +1096,8 @@ func (n *CreateTableStmt) Restore(ctx *format.RestoreCtx) error { ctx.WritePlain(")") } - for i, option := range n.Options { + options := tableOptionsWithRestoreTTLFlag(ctx.Flags, n.Options) + for i, option := range options { ctx.WritePlain(" ") if err := option.Restore(ctx); err != nil { return errors.Annotatef(err, "An error occurred while splicing CreateTableStmt TableOption: [%v]", i) @@ -3573,11 +3574,21 @@ func (n *AlterTableStmt) Restore(ctx *format.RestoreCtx) error { if err := n.Table.Restore(ctx); err != nil { return errors.Annotate(err, "An error occurred while restore AlterTableStmt.Table") } - var specs []*AlterTableSpec + specs := make([]*AlterTableSpec, 0, len(n.Specs)) for _, spec := range n.Specs { - if !(spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag()) { - specs = append(specs, spec) + if spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag() { + continue + } + if spec.Tp == AlterTableOption { + newOptions := tableOptionsWithRestoreTTLFlag(ctx.Flags, spec.Options) + if len(newOptions) == 0 { + continue + } + newSpec := *spec + newSpec.Options = newOptions + spec = &newSpec } + specs = append(specs, spec) } for i, spec := range specs { if i == 0 || spec.Tp == AlterTablePartition || spec.Tp == AlterTableRemovePartitioning || spec.Tp == AlterTableImportTablespace || spec.Tp == AlterTableDiscardTablespace { @@ -4510,3 +4521,25 @@ func restorePlacementStmtInSpecialComment(ctx *format.RestoreCtx, n DDLNode) err return n.Restore(ctx) }) } + +func tableOptionsWithRestoreTTLFlag(flags format.RestoreFlags, options []*TableOption) []*TableOption { + if !flags.HasRestoreWithTTLEnableOff() { + return options + } + + newOptions := make([]*TableOption, 0, len(options)) + for _, opt := range options { + if opt.Tp == TableOptionTTLEnable { + continue + } + + newOptions = append(newOptions, opt) + if opt.Tp == TableOptionTTL { + newOptions = append(newOptions, &TableOption{ + Tp: TableOptionTTLEnable, + BoolValue: false, + }) + } + } + return newOptions +} diff --git a/parser/ast/ddl_test.go b/parser/ast/ddl_test.go index e6107f34513ec..156a66398426f 100644 --- a/parser/ast/ddl_test.go +++ b/parser/ast/ddl_test.go @@ -869,3 +869,37 @@ func TestTableOptionTTLRestore(t *testing.T) { runNodeRestoreTestWithFlags(t, testCases, "%s", extractNodeFunc, ca.flags) } } + +func TestTableOptionTTLRestoreWithTTLEnableOffFlag(t *testing.T) { + sourceSQL1 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR" + sourceSQL2 := "alter table t ttl_enable = 'ON'" + sourceSQL3 := "alter table t remove ttl" + sourceSQL4 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR ttl_enable = 'ON'" + sourceSQL5 := "alter table t ttl_enable = 'ON' placement policy p1" + cases := []struct { + sourceSQL string + flags format.RestoreFlags + expectSQL string + }{ + {sourceSQL1, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"}, + {sourceSQL1, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"}, + {sourceSQL2, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"}, + {sourceSQL2, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"}, + {sourceSQL3, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` REMOVE TTL"}, + {sourceSQL3, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![ttl] REMOVE TTL */"}, + {sourceSQL4, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"}, + {sourceSQL4, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"}, + {sourceSQL5, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![placement] PLACEMENT POLICY = `p1` */"}, + } + + extractNodeFunc := func(node Node) Node { + return node + } + + for _, ca := range cases { + testCases := []NodeRestoreTestCase{ + {ca.sourceSQL, ca.expectSQL}, + } + runNodeRestoreTestWithFlagsStmtChange(t, testCases, "%s", extractNodeFunc, ca.flags) + } +} diff --git a/parser/format/format.go b/parser/format/format.go index adada122e255e..284d4dff4e9df 100644 --- a/parser/format/format.go +++ b/parser/format/format.go @@ -235,6 +235,7 @@ const ( RestoreTiDBSpecialComment SkipPlacementRuleForRestore + RestoreWithTTLEnableOff ) const ( @@ -321,6 +322,11 @@ func (rf RestoreFlags) HasSkipPlacementRuleForRestoreFlag() bool { return rf.has(SkipPlacementRuleForRestore) } +// HasRestoreWithTTLEnableOff returns a boolean indicating whether to force set TTL_ENABLE='OFF' when restoring a TTL table +func (rf RestoreFlags) HasRestoreWithTTLEnableOff() bool { + return rf.has(RestoreWithTTLEnableOff) +} + // RestoreCtx is `Restore` context to hold flags and writer. type RestoreCtx struct { Flags RestoreFlags