From 1eadca1126cedb4637a7e8bf9507038f43798e6e Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Tue, 7 Feb 2023 23:38:55 +0800 Subject: [PATCH 01/11] support is_ipv4 and is_ipv6 expr to tiflash support is_ipv4 and is_ipv6 expr to pushdown to tiflash --- expression/expression.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/expression/expression.go b/expression/expression.go index 6b6ce4c26e0a0..0c635d04c2693 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -1271,6 +1271,8 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { return true case ast.GetFormat: return true + case ast.IsIPv4, ast.IsIPv6: + return true } return false } From c615b3e91bcc3085bd83454ebf57afcc216c6f96 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Tue, 7 Feb 2023 23:41:40 +0800 Subject: [PATCH 02/11] add integration test for is_ipv4 and is_ipv6 --- planner/core/integration_test.go | 90 ++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 39e724ffc6c7a..39a563cf0967a 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8398,3 +8398,93 @@ func TestIssue40285(t *testing.T) { tk.MustExec("CREATE TABLE t(col1 enum('p5', '9a33x') NOT NULL DEFAULT 'p5',col2 tinyblob DEFAULT NULL) ENGINE = InnoDB DEFAULT CHARSET = latin1 COLLATE = latin1_bin;") tk.MustQuery("(select last_value(col1) over () as r0 from t) union all (select col2 as r0 from t);") } + +func TestIsIPv4ToTiFlash(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(v4 varchar(100), v6 varchar(100))") + tk.MustExec("insert into t values('123.123.123.123', 'F746:C349:48E3:22F2:81E0:0EA8:E7B6:8286')") + tk.MustExec("insert into t values('0.0.0.0', '0000:0000:0000:0000:0000:0000:0000:0000')") + tk.MustExec("insert into t values('127.0.0.1', '2001:0:2851:b9f0:6d:2326:9036:f37a')") + tk.MustExec("insert into t values('192.168.0.0/10', 'fe80::2dc3:25a5:49a1:6002%24')") + tk.MustExec("insert into t values('192.168.99.22.123', '4207:A33A:58D3:F2C3:8EDC:A548:3EC7:0D00:0D00')") + tk.MustExec("insert into t values('999.999.999.999', '4207:A33A:58D3:F2C3:8EDC:A548::0D00')") + tk.MustExec("insert into t values('3.2.1.', '4207::::8EDC:A548:3EC7:0D00')") + tk.MustExec("insert into t values('3..2.1', '4207:::::A548:3EC7:0D00')") + tk.MustExec("insert into t values('...', '::::::')") + tk.MustExec("insert into t values('4556456', '4556456')") + tk.MustExec("insert into t values('ajdjioa', 'ajdjioa')") + tk.MustExec("insert into t values('', '')") + tk.MustExec("insert into t values(null,null)") + + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + + // Create virtual tiflash replica info. + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + rows := [][]interface{}{ + {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, + {" └─Projection_4", "mpp[tiflash]", "is_ipv4(test.t.v4)->Column#4"}, + {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, + } + tk.MustQuery("explain select is_ipv4(v4) from t;").CheckAt([]int{0, 2, 4}, rows) +} + +func TestIsIPv6ToTiFlash(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(v4 varchar(100), v6 varchar(100))") + tk.MustExec("insert into t values('123.123.123.123', 'F746:C349:48E3:22F2:81E0:0EA8:E7B6:8286')") + tk.MustExec("insert into t values('0.0.0.0', '0000:0000:0000:0000:0000:0000:0000:0000')") + tk.MustExec("insert into t values('127.0.0.1', '2001:0:2851:b9f0:6d:2326:9036:f37a')") + tk.MustExec("insert into t values('192.168.0.0/10', 'fe80::2dc3:25a5:49a1:6002%24')") + tk.MustExec("insert into t values('192.168.99.22.123', '4207:A33A:58D3:F2C3:8EDC:A548:3EC7:0D00:0D00')") + tk.MustExec("insert into t values('999.999.999.999', '4207:A33A:58D3:F2C3:8EDC:A548::0D00')") + tk.MustExec("insert into t values('3.2.1.', '4207::::8EDC:A548:3EC7:0D00')") + tk.MustExec("insert into t values('3..2.1', '4207:::::A548:3EC7:0D00')") + tk.MustExec("insert into t values('...', '::::::')") + tk.MustExec("insert into t values('4556456', '4556456')") + tk.MustExec("insert into t values('ajdjioa', 'ajdjioa')") + tk.MustExec("insert into t values('', '')") + tk.MustExec("insert into t values(null,null)") + + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + + // Create virtual tiflash replica info. + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + rows := [][]interface{}{ + {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, + {" └─Projection_4", "mpp[tiflash]", "is_ipv6(test.t.v6)->Column#4"}, + {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, + } + tk.MustQuery("explain select is_ipv6(v6) from t;").CheckAt([]int{0, 2, 4}, rows) +} From aa359fe910e82fddb74273bc8bbee3069b8b5e20 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Mon, 13 Feb 2023 22:59:04 +0800 Subject: [PATCH 03/11] Update expr_to_pb_test.go --- expression/expr_to_pb_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 6756601be0e04..cf16fb78fc186 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1257,6 +1257,16 @@ func TestExprPushDownToFlash(t *testing.T) { function, err = NewFunction(mock.NewContext(), ast.Unhex, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) + + // IsIPv4 + function, err = NewFunction(mock.NewContext(), ast.IsIPv4, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) + + // IsIPv6 + function, err = NewFunction(mock.NewContext(), ast.IsIPv6, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) pushed, remained = PushDownExprs(sc, exprs, client, kv.TiFlash) require.Len(t, pushed, len(exprs)) From e3d52f465590aa35bca4ee376a22510450a60718 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Mon, 13 Feb 2023 23:00:40 +0800 Subject: [PATCH 04/11] Update integration_test.go --- planner/core/integration_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 27d8414d943d0..399a3d772d008 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8441,16 +8441,6 @@ func TestIsIPv4ToTiFlash(t *testing.T) { tk.MustExec("insert into t values('123.123.123.123', 'F746:C349:48E3:22F2:81E0:0EA8:E7B6:8286')") tk.MustExec("insert into t values('0.0.0.0', '0000:0000:0000:0000:0000:0000:0000:0000')") tk.MustExec("insert into t values('127.0.0.1', '2001:0:2851:b9f0:6d:2326:9036:f37a')") - tk.MustExec("insert into t values('192.168.0.0/10', 'fe80::2dc3:25a5:49a1:6002%24')") - tk.MustExec("insert into t values('192.168.99.22.123', '4207:A33A:58D3:F2C3:8EDC:A548:3EC7:0D00:0D00')") - tk.MustExec("insert into t values('999.999.999.999', '4207:A33A:58D3:F2C3:8EDC:A548::0D00')") - tk.MustExec("insert into t values('3.2.1.', '4207::::8EDC:A548:3EC7:0D00')") - tk.MustExec("insert into t values('3..2.1', '4207:::::A548:3EC7:0D00')") - tk.MustExec("insert into t values('...', '::::::')") - tk.MustExec("insert into t values('4556456', '4556456')") - tk.MustExec("insert into t values('ajdjioa', 'ajdjioa')") - tk.MustExec("insert into t values('', '')") - tk.MustExec("insert into t values(null,null)") tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") @@ -8486,16 +8476,6 @@ func TestIsIPv6ToTiFlash(t *testing.T) { tk.MustExec("insert into t values('123.123.123.123', 'F746:C349:48E3:22F2:81E0:0EA8:E7B6:8286')") tk.MustExec("insert into t values('0.0.0.0', '0000:0000:0000:0000:0000:0000:0000:0000')") tk.MustExec("insert into t values('127.0.0.1', '2001:0:2851:b9f0:6d:2326:9036:f37a')") - tk.MustExec("insert into t values('192.168.0.0/10', 'fe80::2dc3:25a5:49a1:6002%24')") - tk.MustExec("insert into t values('192.168.99.22.123', '4207:A33A:58D3:F2C3:8EDC:A548:3EC7:0D00:0D00')") - tk.MustExec("insert into t values('999.999.999.999', '4207:A33A:58D3:F2C3:8EDC:A548::0D00')") - tk.MustExec("insert into t values('3.2.1.', '4207::::8EDC:A548:3EC7:0D00')") - tk.MustExec("insert into t values('3..2.1', '4207:::::A548:3EC7:0D00')") - tk.MustExec("insert into t values('...', '::::::')") - tk.MustExec("insert into t values('4556456', '4556456')") - tk.MustExec("insert into t values('ajdjioa', 'ajdjioa')") - tk.MustExec("insert into t values('', '')") - tk.MustExec("insert into t values(null,null)") tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") From 18147aad4d37dba1fbf943c846cfd1789d4865ed Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Tue, 14 Feb 2023 23:00:14 +0800 Subject: [PATCH 05/11] Update integration_test.go --- planner/core/integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index be382ed3b38ab..f7440e0c325cc 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8431,6 +8431,7 @@ func TestIssue40285(t *testing.T) { tk.MustExec("CREATE TABLE t(col1 enum('p5', '9a33x') NOT NULL DEFAULT 'p5',col2 tinyblob DEFAULT NULL) ENGINE = InnoDB DEFAULT CHARSET = latin1 COLLATE = latin1_bin;") tk.MustQuery("(select last_value(col1) over () as r0 from t) union all (select col2 as r0 from t);") } + // https://github.com/pingcap/tidb/issues/41273 func TestIssue41273(t *testing.T) { store := testkit.CreateMockStore(t) From 278050340549a38414644e10af406e401974b171 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 08:32:36 +0800 Subject: [PATCH 06/11] Update expr_to_pb_test.go --- expression/expr_to_pb_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index cf16fb78fc186..f485ef7743fa8 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1257,12 +1257,12 @@ func TestExprPushDownToFlash(t *testing.T) { function, err = NewFunction(mock.NewContext(), ast.Unhex, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) - + // IsIPv4 function, err = NewFunction(mock.NewContext(), ast.IsIPv4, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) - + // IsIPv6 function, err = NewFunction(mock.NewContext(), ast.IsIPv6, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) From c17346b50c1ccc040f5226ce3f1bfbe78fa8b46b Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 16:39:40 +0800 Subject: [PATCH 07/11] Update integration_test.go --- planner/core/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index f7440e0c325cc..130e7f1bec404 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8480,7 +8480,7 @@ func TestIsIPv4ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv4(test.t.v4)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, @@ -8515,7 +8515,7 @@ func TestIsIPv6ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv6(test.t.v6)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, From 32a8952fbd498ef7e79822868c300113c8d15bba Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 17:13:57 +0800 Subject: [PATCH 08/11] Update integration_test.go --- planner/core/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 130e7f1bec404..cd725112c2022 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8480,7 +8480,7 @@ func TestIsIPv4ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, + {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv4(test.t.v4)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, From f617a277fd4b4c0733e6cf84b994cfc81e10dab5 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 17:40:14 +0800 Subject: [PATCH 09/11] Update integration_test.go --- planner/core/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index cd725112c2022..643af9ed9d8af 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8515,7 +8515,7 @@ func TestIsIPv6ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, + {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv6(test.t.v6)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, From 30158e21b717c9b2d5f8065526787bf4410052fd Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 17:57:53 +0800 Subject: [PATCH 10/11] Update integration_test.go --- planner/core/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 643af9ed9d8af..7dd7be744d42a 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8515,7 +8515,7 @@ func TestIsIPv6ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, + {"TableReader_9", "root", "MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv6(test.t.v6)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, From 957ebd99e8beb9be572dd7b7f79e7e81d382d752 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 15 Feb 2023 19:16:37 +0800 Subject: [PATCH 11/11] Update integration_test.go --- planner/core/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 7dd7be744d42a..bd210fbeb0297 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -8480,7 +8480,7 @@ func TestIsIPv4ToTiFlash(t *testing.T) { } rows := [][]interface{}{ - {"TableReader_9", "root", " MppVersion: 1, data:ExchangeSender_8"}, + {"TableReader_9", "root", "MppVersion: 1, data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, {" └─Projection_4", "mpp[tiflash]", "is_ipv4(test.t.v4)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"},