Improve compressed DML expression pushdown

Try to constify query constraints when filtering batches to cover a wider range of expressions that are safe to evaluate when we do the batch filtering. This will allow the following expressions to be usable with compressed DML batch filtering: - IMMUTABLE and STABLE functions - SQLValueFunction (CURRENT_TIME,CURRENT_USER,LOCAL_TIME,...) - Prepared Statement parameters
timescale · May 9, 2024 · f41cf0c · f41cf0c
1 parent ca125cf
commit f41cf0c
Show file tree

Hide file tree

Showing 4 changed files with 228 additions and 80 deletions.
diff --git a/.unreleased/pr_6895 b/.unreleased/pr_6895
@@ -0,0 +1 @@
+Implements: #6895 Improve compressed DML expression pushdown
diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c
@@ -24,6 +24,7 @@
 #include <nodes/execnodes.h>
 #include <nodes/pg_list.h>
 #include <nodes/print.h>
+#include <optimizer/optimizer.h>
 #include <parser/parsetree.h>
 #include <parser/parse_coerce.h>
 #include <storage/lmgr.h>
@@ -2531,18 +2532,27 @@ find_matching_index(Relation comp_chunk_rel, List **index_filters, List **heap_f
 
 /*
  * This method will evaluate the predicates, extract
- * left and right operands, check if one of the operands is
- * a simple Var type. If its Var type extract its corresponding
- * column name from hypertable_compression catalog table.
- * If extracted column is a SEGMENT BY column then save column
- * name, value specified in the predicate. This information will
- * be used to build scan keys later.
+ * left and right operands, check if any of the operands
+ * can be used for batch filtering and if so, it will
+ * create a BatchFilter object and add it to the corresponding
+ * list.
+ * Any segmentby filter is put into index_filters list other
+ * filters are put into heap_filters list.
  */
 static void
 process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates, List **heap_filters,
 				   List **index_filters, List **is_null)
 {
 	ListCell *lc;
+	/*
+	 * We dont want to forward boundParams from the execution state here
+	 * as we dont want to constify join params in the predicates.
+	 * Constifying JOIN params would not be safe as we don't redo
+	 * this part in rescan.
+	 */
+	PlannerGlobal glob = { .boundParams = NULL };
+	PlannerInfo root = { .glob = &glob };
+
 	foreach (lc, predicates)
 	{
 		Node *node = copyObject(lfirst(lc));
@@ -2565,7 +2575,11 @@ process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates, L
 					continue;
 
 				if (!IsA(expr, Const))
-					continue;
+				{
+					expr = (Expr *) estimate_expression_value(&root, (Node *) expr);
+					if (!IsA(expr, Const))
+						continue;
+				}
 
 				arg_value = castNode(Const, expr);
 

diff --git a/tsl/test/expected/compression_update_delete.out b/tsl/test/expected/compression_update_delete.out
@@ -1,6 +1,7 @@
 -- This file and its contents are licensed under the Timescale License.
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-TIMESCALE for a copy of the license.
+\set EXPLAIN 'EXPLAIN (costs off, timing off, summary off, analyze)'
 CREATE OR REPLACE VIEW compressed_chunk_info_view AS
 SELECT
    h.schema_name AS hypertable_schema,
@@ -2944,106 +2945,215 @@ EXPLAIN (analyze, timing off, costs off, summary off) DELETE FROM test_meta_filt
                Rows Removed by Filter: 10
 (8 rows)
 
--- test commutator handling in compressed dml constraints
-CREATE TABLE test_commutator(time timestamptz NOT NULL, device text);
-SELECT table_name FROM create_hypertable('test_commutator', 'time');
-   table_name    
------------------
- test_commutator
-(1 row)
-
-INSERT INTO test_commutator SELECT '2020-01-01', 'a';
-INSERT INTO test_commutator SELECT '2020-01-01', 'b';
-INSERT INTO test_commutator SELECT '2020-01-01', 'c';
-ALTER TABLE test_commutator SET (timescaledb.compress, timescaledb.compress_segmentby='device');
-NOTICE:  default order by for hypertable "test_commutator" is set to ""time" DESC"
-SELECT compress_chunk(show_chunks('test_commutator'));
+-- test expression pushdown in compressed dml constraints
+CREATE TABLE test_pushdown(time timestamptz NOT NULL, device text);
+SELECT table_name FROM create_hypertable('test_pushdown', 'time');
+  table_name   
+---------------
+ test_pushdown
+(1 row)
+
+INSERT INTO test_pushdown SELECT '2020-01-01', 'a';
+INSERT INTO test_pushdown SELECT '2020-01-01', 'b';
+INSERT INTO test_pushdown SELECT '2020-01-01 05:00', 'c';
+CREATE TABLE devices(device text);
+INSERT INTO devices VALUES ('a'), ('b'), ('c');
+ALTER TABLE test_pushdown SET (timescaledb.compress, timescaledb.compress_segmentby='device');
+NOTICE:  default order by for hypertable "test_pushdown" is set to ""time" DESC"
+SELECT compress_chunk(show_chunks('test_pushdown'));
               compress_chunk              
 ------------------------------------------
  _timescaledb_internal._hyper_39_79_chunk
 (1 row)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE 'a' = device; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE 'a' = device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 1
    Tuples decompressed: 1
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=1 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
                Filter: ('a'::text = device)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE device < 'c' ; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device < 'c' ; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 2
    Tuples decompressed: 2
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=2 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=2 loops=1)
                Filter: (device < 'c'::text)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE 'c' > device; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE 'c' > device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 2
    Tuples decompressed: 2
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=2 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=2 loops=1)
                Filter: ('c'::text > device)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE 'c' >= device; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE 'c' >= device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 3
    Tuples decompressed: 3
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=3 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=3 loops=1)
                Filter: ('c'::text >= device)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE device > 'b'; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device > 'b'; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 1
    Tuples decompressed: 1
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=1 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
                Filter: (device > 'b'::text)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE 'b' < device; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device = CURRENT_USER; ROLLBACK;
+                                        QUERY PLAN                                        
+------------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Custom Scan (ChunkAppend) on test_pushdown (actual rows=0 loops=1)
+               Chunks excluded during startup: 0
+               ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=0 loops=1)
+                     Filter: (device = CURRENT_USER)
+(7 rows)
+
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE 'b' < device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 1
    Tuples decompressed: 1
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=1 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
                Filter: ('b'::text < device)
 (7 rows)
 
-BEGIN; EXPLAIN (costs off, timing off, summary off, analyze) DELETE FROM test_commutator WHERE 'b' <= device; ROLLBACK;
-                                      QUERY PLAN                                      
---------------------------------------------------------------------------------------
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE 'b' <= device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
  Custom Scan (HypertableModify) (actual rows=0 loops=1)
    Batches decompressed: 2
    Tuples decompressed: 2
-   ->  Delete on test_commutator (actual rows=0 loops=1)
-         Delete on _hyper_39_79_chunk test_commutator_1
-         ->  Seq Scan on _hyper_39_79_chunk test_commutator_1 (actual rows=2 loops=1)
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=2 loops=1)
                Filter: ('b'::text <= device)
 (7 rows)
 
+-- cant pushdown OR atm
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device = 'a' OR device = 'b'; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 3
+   Tuples decompressed: 3
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=2 loops=1)
+               Filter: ((device = 'a'::text) OR (device = 'b'::text))
+               Rows Removed by Filter: 1
+(8 rows)
+
+-- test stable function
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE time = timestamptz('2020-01-01 05:00'); ROLLBACK;
+                                        QUERY PLAN                                         
+-------------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 1
+   Tuples decompressed: 1
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
+               Filter: ("time" = 'Wed Jan 01 05:00:00 2020 PST'::timestamp with time zone)
+(7 rows)
+
+-- test sqlvaluefunction
+BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device = substring(CURRENT_USER,length(CURRENT_USER)+1) || 'c'; ROLLBACK;
+                                                         QUERY PLAN                                                          
+-----------------------------------------------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 1
+   Tuples decompressed: 1
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Custom Scan (ChunkAppend) on test_pushdown (actual rows=1 loops=1)
+               Chunks excluded during startup: 0
+               ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
+                     Filter: (device = ("substring"((CURRENT_USER)::text, (length((CURRENT_USER)::text) + 1)) || 'c'::text))
+(9 rows)
+
+-- no filtering in decompression
+BEGIN; :EXPLAIN DELETE FROM test_pushdown p USING devices d WHERE p.device=d.device; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 3
+   Tuples decompressed: 3
+   ->  Delete on test_pushdown p (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk p_1
+         ->  Merge Join (actual rows=3 loops=1)
+               Merge Cond: (p_1.device = d.device)
+               ->  Sort (actual rows=3 loops=1)
+                     Sort Key: p_1.device
+                     Sort Method: quicksort 
+                     ->  Seq Scan on _hyper_39_79_chunk p_1 (actual rows=3 loops=1)
+               ->  Sort (actual rows=3 loops=1)
+                     Sort Key: d.device
+                     Sort Method: quicksort 
+                     ->  Seq Scan on devices d (actual rows=3 loops=1)
+(15 rows)
+
+-- can filter in decompression even before executing join
+BEGIN; :EXPLAIN DELETE FROM test_pushdown p USING devices d WHERE p.device=d.device AND d.device ='b' ; ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 1
+   Tuples decompressed: 1
+   ->  Delete on test_pushdown p (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk p_1
+         ->  Nested Loop (actual rows=1 loops=1)
+               ->  Seq Scan on devices d (actual rows=1 loops=1)
+                     Filter: (device = 'b'::text)
+                     Rows Removed by Filter: 2
+               ->  Materialize (actual rows=1 loops=1)
+                     ->  Seq Scan on _hyper_39_79_chunk p_1 (actual rows=1 loops=1)
+                           Filter: (device = 'b'::text)
+(12 rows)
+
+-- test prepared statement
+PREPARE q1(text) AS DELETE FROM test_pushdown WHERE device = $1;
+BEGIN; :EXPLAIN EXECUTE q1('a'); ROLLBACK;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Custom Scan (HypertableModify) (actual rows=0 loops=1)
+   Batches decompressed: 1
+   Tuples decompressed: 1
+   ->  Delete on test_pushdown (actual rows=0 loops=1)
+         Delete on _hyper_39_79_chunk test_pushdown_1
+         ->  Seq Scan on _hyper_39_79_chunk test_pushdown_1 (actual rows=1 loops=1)
+               Filter: (device = 'a'::text)
+(7 rows)
+