From 2fddf056fcf7724cd18eaa75b69affc1854547dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 26 Apr 2024 11:38:28 -0300 Subject: [PATCH] Update the watermark when truncating a CAgg In #5261 we cached the Continuous Aggregate watermark value in a metadata table to improve performance avoiding compute the watermark at planning time. Manually DML operations on a CAgg are not recommended and instead the user should use the `refresh_continuous_aggregate` procedure. But we handle `TRUNCATE` over CAggs generating the necessary invalidation logs so make sense to also update the watermark. --- .unreleased/feature_6865 | 1 + src/process_utility.c | 19 ++++++++++++++++++- tsl/test/expected/cagg_ddl-13.out | 29 +++++++++++++++++++++++++++++ tsl/test/expected/cagg_ddl-14.out | 29 +++++++++++++++++++++++++++++ tsl/test/expected/cagg_ddl-15.out | 29 +++++++++++++++++++++++++++++ tsl/test/expected/cagg_ddl-16.out | 29 +++++++++++++++++++++++++++++ tsl/test/sql/cagg_ddl.sql.in | 10 ++++++++++ 7 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 .unreleased/feature_6865 diff --git a/.unreleased/feature_6865 b/.unreleased/feature_6865 new file mode 100644 index 00000000000..8584a15f928 --- /dev/null +++ b/.unreleased/feature_6865 @@ -0,0 +1 @@ +Implements: #6865 Update the watermark when truncating a CAgg diff --git a/src/process_utility.c b/src/process_utility.c index b61d04a9d4d..5851c1ab371 100644 --- a/src/process_utility.c +++ b/src/process_utility.c @@ -69,6 +69,7 @@ #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" +#include "ts_catalog/continuous_aggs_watermark.h" #include "tss_callbacks.h" #include "utils.h" #include "with_clause_parser.h" @@ -889,7 +890,7 @@ process_truncate(ProcessUtilityArgs *args) TruncateStmt *stmt = (TruncateStmt *) args->parsetree; Cache *hcache = ts_hypertable_cache_pin(); ListCell *cell; - List *hypertables = NIL; + List *hypertables = NIL, *mat_hypertables = NIL; List *relations = NIL; bool list_changed = false; MemoryContext oldctx, parsetreectx = GetMemoryChunkContext(args->parsetree); @@ -966,6 +967,9 @@ process_truncate(ProcessUtilityArgs *args) /* mark list as changed because we'll add the materialization hypertable */ list_changed = true; + + /* list of materialization hypertables to reset the watermark */ + mat_hypertables = lappend(mat_hypertables, mat_ht); } list_append = true; @@ -1103,6 +1107,19 @@ process_truncate(ProcessUtilityArgs *args) } } + /* For all materialization hypertables, reset the watermark */ + foreach (cell, mat_hypertables) + { + Hypertable *mat_ht = lfirst(cell); + + Assert(mat_ht != NULL); + + /* Force update the watermark */ + bool isnull; + int64 watermark = ts_hypertable_get_open_dim_max_value(mat_ht, 0, &isnull); + ts_cagg_watermark_update(mat_ht, watermark, isnull, true); + } + ts_cache_release(hcache); return DDL_DONE; diff --git a/tsl/test/expected/cagg_ddl-13.out b/tsl/test/expected/cagg_ddl-13.out index 037ef716f25..5e52f744406 100644 --- a/tsl/test/expected/cagg_ddl-13.out +++ b/tsl/test/expected/cagg_ddl-13.out @@ -2025,3 +2025,32 @@ SELECT * FROM conditions_daily ORDER BY bucket, avg; NYC | Thu Nov 01 17:00:00 2018 PDT | 15 (6 rows) +-- Test TRUNCATE over a Realtime CAgg +ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_before; + watermak_before +------------------------------ + Fri Nov 02 17:00:00 2018 PDT +(1 row) + +TRUNCATE conditions_daily; +-- Watermark should be reseted +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_after; + watermak_after +--------------------------------- + Sun Nov 23 16:00:00 4714 PST BC +(1 row) + +-- Should return ROWS because the watermark was reseted by the TRUNCATE +SELECT * FROM conditions_daily ORDER BY bucket, avg; + location | bucket | avg +----------+------------------------------+----- + SFO | Sun Dec 31 16:00:00 2017 PST | 55 + SFO | Mon Jan 01 16:00:00 2018 PST | 65 + NYC | Mon Jan 01 16:00:00 2018 PST | 65 + por | Mon Jan 01 16:00:00 2018 PST | 100 + NYC | Wed Oct 31 17:00:00 2018 PDT | 65 + NYC | Thu Nov 01 17:00:00 2018 PDT | 15 +(6 rows) + diff --git a/tsl/test/expected/cagg_ddl-14.out b/tsl/test/expected/cagg_ddl-14.out index 037ef716f25..5e52f744406 100644 --- a/tsl/test/expected/cagg_ddl-14.out +++ b/tsl/test/expected/cagg_ddl-14.out @@ -2025,3 +2025,32 @@ SELECT * FROM conditions_daily ORDER BY bucket, avg; NYC | Thu Nov 01 17:00:00 2018 PDT | 15 (6 rows) +-- Test TRUNCATE over a Realtime CAgg +ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_before; + watermak_before +------------------------------ + Fri Nov 02 17:00:00 2018 PDT +(1 row) + +TRUNCATE conditions_daily; +-- Watermark should be reseted +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_after; + watermak_after +--------------------------------- + Sun Nov 23 16:00:00 4714 PST BC +(1 row) + +-- Should return ROWS because the watermark was reseted by the TRUNCATE +SELECT * FROM conditions_daily ORDER BY bucket, avg; + location | bucket | avg +----------+------------------------------+----- + SFO | Sun Dec 31 16:00:00 2017 PST | 55 + SFO | Mon Jan 01 16:00:00 2018 PST | 65 + NYC | Mon Jan 01 16:00:00 2018 PST | 65 + por | Mon Jan 01 16:00:00 2018 PST | 100 + NYC | Wed Oct 31 17:00:00 2018 PDT | 65 + NYC | Thu Nov 01 17:00:00 2018 PDT | 15 +(6 rows) + diff --git a/tsl/test/expected/cagg_ddl-15.out b/tsl/test/expected/cagg_ddl-15.out index 037ef716f25..5e52f744406 100644 --- a/tsl/test/expected/cagg_ddl-15.out +++ b/tsl/test/expected/cagg_ddl-15.out @@ -2025,3 +2025,32 @@ SELECT * FROM conditions_daily ORDER BY bucket, avg; NYC | Thu Nov 01 17:00:00 2018 PDT | 15 (6 rows) +-- Test TRUNCATE over a Realtime CAgg +ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_before; + watermak_before +------------------------------ + Fri Nov 02 17:00:00 2018 PDT +(1 row) + +TRUNCATE conditions_daily; +-- Watermark should be reseted +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_after; + watermak_after +--------------------------------- + Sun Nov 23 16:00:00 4714 PST BC +(1 row) + +-- Should return ROWS because the watermark was reseted by the TRUNCATE +SELECT * FROM conditions_daily ORDER BY bucket, avg; + location | bucket | avg +----------+------------------------------+----- + SFO | Sun Dec 31 16:00:00 2017 PST | 55 + SFO | Mon Jan 01 16:00:00 2018 PST | 65 + NYC | Mon Jan 01 16:00:00 2018 PST | 65 + por | Mon Jan 01 16:00:00 2018 PST | 100 + NYC | Wed Oct 31 17:00:00 2018 PDT | 65 + NYC | Thu Nov 01 17:00:00 2018 PDT | 15 +(6 rows) + diff --git a/tsl/test/expected/cagg_ddl-16.out b/tsl/test/expected/cagg_ddl-16.out index cd99cd50b38..0b4a26fdd63 100644 --- a/tsl/test/expected/cagg_ddl-16.out +++ b/tsl/test/expected/cagg_ddl-16.out @@ -2025,3 +2025,32 @@ SELECT * FROM conditions_daily ORDER BY bucket, avg; NYC | Thu Nov 01 17:00:00 2018 PDT | 15 (6 rows) +-- Test TRUNCATE over a Realtime CAgg +ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_before; + watermak_before +------------------------------ + Fri Nov 02 17:00:00 2018 PDT +(1 row) + +TRUNCATE conditions_daily; +-- Watermark should be reseted +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_after; + watermak_after +--------------------------------- + Sun Nov 23 16:00:00 4714 PST BC +(1 row) + +-- Should return ROWS because the watermark was reseted by the TRUNCATE +SELECT * FROM conditions_daily ORDER BY bucket, avg; + location | bucket | avg +----------+------------------------------+----- + SFO | Sun Dec 31 16:00:00 2017 PST | 55 + SFO | Mon Jan 01 16:00:00 2018 PST | 65 + NYC | Mon Jan 01 16:00:00 2018 PST | 65 + por | Mon Jan 01 16:00:00 2018 PST | 100 + NYC | Wed Oct 31 17:00:00 2018 PDT | 65 + NYC | Thu Nov 01 17:00:00 2018 PDT | 15 +(6 rows) + diff --git a/tsl/test/sql/cagg_ddl.sql.in b/tsl/test/sql/cagg_ddl.sql.in index bb7fd24f000..0bd39e0360a 100644 --- a/tsl/test/sql/cagg_ddl.sql.in +++ b/tsl/test/sql/cagg_ddl.sql.in @@ -1288,3 +1288,13 @@ ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=true \d+ conditions_daily CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT * FROM conditions_daily ORDER BY bucket, avg; + +-- Test TRUNCATE over a Realtime CAgg +ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_before; +TRUNCATE conditions_daily; +-- Watermark should be reseted +SELECT COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)), '-infinity'::timestamp with time zone) AS watermak_after; +-- Should return ROWS because the watermark was reseted by the TRUNCATE +SELECT * FROM conditions_daily ORDER BY bucket, avg;