Stats improvement for Uncompressed Chunks

.github/gh_matrix_builder.py

@@ -139,14 +139,35 @@ def macos_config(overrides):
 
 # common installcheck_args for all pg15 tests
 # partialize_finalize is ignored due to #4937
-pg15_installcheck_args = "IGNORES='partialize_finalize'"
+pg15_installcheck_args = "IGNORES='partialize_finalize transparent_decompress_chunk-15'"
+
+# transparent_decompress_chunk-[12,13,14] is ignored due to #5118
+pg14_installcheck_args = "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed partialize_finalize transparent_decompress_chunk-14'"
+
+pg13_installcheck_args = "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed partialize_finalize transparent_decompress_chunk-13'"
+
+pg12_installcheck_args = "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed partialize_finalize transparent_decompress_chunk-12'"
 
 # always test debug build on latest of all supported pg versions
-m["include"].append(build_debug_config({"pg": PG12_LATEST}))
 m["include"].append(
-    build_debug_config({"pg": PG13_LATEST, "cc": "clang-14", "cxx": "clang++-14"})
+    build_debug_config({"pg": PG12_LATEST, "installcheck_args": pg12_installcheck_args})
 )
-m["include"].append(build_debug_config({"pg": PG14_LATEST}))
+
+m["include"].append(
+    build_debug_config(
+        {
+            "pg": PG13_LATEST,
+            "cc": "clang-14",
+            "cxx": "clang++-14",
+            "installcheck_args": pg13_installcheck_args,
+        }
+    )
+)
+
+m["include"].append(
+    build_debug_config({"pg": PG14_LATEST, "installcheck_args": pg14_installcheck_args})
+)
+
 m["include"].append(
     build_debug_config({"pg": PG15_LATEST, "installcheck_args": pg15_installcheck_args})
 )
@@ -185,7 +206,7 @@ def macos_config(overrides):
         "pg": PG13_EARLIEST,
         # The early releases don't build with llvm 14.
         "pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
-        "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'",
+        "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13 transparent_decompress_chunk-13'",
         "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
     }
     m["include"].append(build_debug_config(pg13_debug_earliest))

.github/workflows/windows-build-and-test.yaml

@@ -53,7 +53,7 @@ jobs:
         os: [ windows-2022 ]
         build_type: ${{ fromJson(needs.config.outputs.build_type) }}
         ignores: ["chunk_adaptive metadata"]
-        tsl_ignores: ["compression_algos remote_connection"]
+        tsl_ignores: ["compression_algos remote_connection telemetry_stats-13 telemetry_stats-14"]
         tsl_skips: ["bgw_db_scheduler bgw_db_scheduler_fixed cagg_ddl_dist_ht
           data_fetcher dist_compression dist_remote_error remote_txn"]
         pg_config: ["-cfsync=off -cstatement_timeout=60s"]

CHANGELOG.md

@@ -80,6 +80,9 @@ Sooner to that time, we will announce the specific version of TimescaleDB in whi
 * #5262 Extend enabling compression on a continuous aggregrate with 'compress_segmentby' and 'compress_orderby' parameters
 
 **Bugfixes**
+* #4926 Fix corruption when inserting into compressed chunks
+* #5118 Enable auto vacuum for uncompressed chunks
+* #5218 Add role-level security to job error log
 * #5214 Fix use of prepared statement in async module
 * #5218 Add role-level security to job error log
 * #5239 Fix next_start calculation for fixed schedules

sql/size_utils.sql

@@ -394,6 +394,86 @@ $BODY$ SET search_path TO pg_catalog, pg_temp;
 -- Estimated number of rows according to catalog tables
 CREATE OR REPLACE FUNCTION @extschema@.approximate_row_count(relation REGCLASS)
 RETURNS BIGINT
+LANGUAGE PLPGSQL VOLATILE STRICT AS
+$BODY$
+DECLARE
+    local_table_name       NAME = NULL;
+    local_schema_name      NAME = NULL;
+    is_distributed   BOOL = FALSE;
+    is_compressed    BOOL = FALSE;
+    uncompressed_row_count BIGINT = 0;
+    compressed_row_count BIGINT = 0;
+    local_compressed_hypertable_id INTEGER = 0;
+    local_compressed_chunk_id INTEGER = 0;
+    compressed_hypertable_oid  OID;
+    local_compressed_chunk_oid  OID;
+    max_compressed_row_count BIGINT = 1000;
+    is_compressed_chunk INTEGER;
+BEGIN
+    SELECT relname, nspname FROM pg_class c
+    INNER JOIN pg_namespace n ON (n.OID = c.relnamespace)
+    INTO local_table_name, local_schema_name
+    WHERE c.OID = relation;
+
+    -- Check for input relation is Hypertable
+    IF EXISTS (SELECT 1
+               FROM _timescaledb_catalog.hypertable WHERE table_name = local_table_name AND schema_name = local_schema_name) THEN
+        SELECT compressed_hypertable_id FROM _timescaledb_catalog.hypertable INTO local_compressed_hypertable_id
+        WHERE table_name = local_table_name AND schema_name = local_schema_name;
+        IF local_compressed_hypertable_id IS NOT NULL THEN
+           uncompressed_row_count = _timescaledb_internal.get_approx_row_count(relation);
+
+           WITH compressed_hypertable AS (SELECT table_name, schema_name FROM _timescaledb_catalog.hypertable ht
+           WHERE ht.id = local_compressed_hypertable_id)
+           SELECT c.oid INTO compressed_hypertable_oid FROM pg_class c
+           INNER JOIN compressed_hypertable h ON (c.relname = h.table_name)
+           INNER JOIN pg_namespace n ON (n.nspname = h.schema_name);
+
+           compressed_row_count = _timescaledb_internal.get_approx_row_count(compressed_hypertable_oid);
+           RETURN (uncompressed_row_count + (compressed_row_count * max_compressed_row_count));
+        ELSE
+           uncompressed_row_count = _timescaledb_internal.get_approx_row_count(relation);
+           RETURN uncompressed_row_count;
+        END IF;
+    END IF;
+    -- Check for input relation is CHUNK
+    IF EXISTS (SELECT 1 FROM _timescaledb_catalog.chunk WHERE table_name = local_table_name AND schema_name = local_schema_name) THEN
+        with compressed_chunk as (select 1 as is_compressed_chunk from _timescaledb_catalog.chunk c
+        inner join _timescaledb_catalog.hypertable h on (c.hypertable_id = h.compressed_hypertable_id)
+        where c.table_name = local_table_name and c.schema_name = local_schema_name ),
+        chunk_temp as (select compressed_chunk_id from _timescaledb_catalog.chunk c where c.table_name = local_table_name and c.schema_name = local_schema_name)
+        select ct.compressed_chunk_id, cc.is_compressed_chunk from chunk_temp ct LEFT OUTER JOIN compressed_chunk cc ON 1 = 1
+        INTO local_compressed_chunk_id, is_compressed_chunk;
+        -- 'input is chunk #1';
+        IF is_compressed_chunk IS NULL AND local_compressed_chunk_id IS NOT NULL THEN
+        -- 'Include both uncompressed  and compressed chunk #2';
+            WITH compressed_ns_oid AS ( SELECT table_name, oid FROM _timescaledb_catalog.chunk ch INNER JOIN pg_namespace ns ON
+            (ch.id = local_compressed_chunk_id and ch.schema_name = ns.nspname))
+            SELECT c.oid FROM pg_class c INNER JOIN compressed_ns_oid
+            ON ( c.relnamespace = compressed_ns_oid.oid AND c.relname = compressed_ns_oid.table_name)
+            INTO local_compressed_chunk_oid;
+
+            uncompressed_row_count = _timescaledb_internal.get_approx_row_count(relation);
+            compressed_row_count = _timescaledb_internal.get_approx_row_count(local_compressed_chunk_oid);
+            RETURN uncompressed_row_count + (compressed_row_count * max_compressed_row_count);
+        ELSIF is_compressed_chunk IS NULL AND local_compressed_chunk_id IS NULL THEN
+        -- 'input relation is uncompressed chunk #3';
+            uncompressed_row_count = _timescaledb_internal.get_approx_row_count(relation);
+            RETURN uncompressed_row_count;
+        ELSE
+        -- 'compressed chunk only #4';
+            compressed_row_count = _timescaledb_internal.get_approx_row_count(relation) * max_compressed_row_count;
+            RETURN compressed_row_count;
+        END IF;
+    END IF;
+    -- Check for input relation is Plain RELATION
+    uncompressed_row_count = _timescaledb_internal.get_approx_row_count(relation);
+    RETURN uncompressed_row_count;
+END;
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+CREATE OR REPLACE FUNCTION _timescaledb_internal.get_approx_row_count(relation REGCLASS)
+RETURNS BIGINT
 LANGUAGE SQL VOLATILE STRICT AS
 $BODY$
   WITH RECURSIVE inherited_id(oid) AS

sql/updates/reverse-dev.sql

@@ -2,3 +2,5 @@ DROP FUNCTION _timescaledb_internal.ping_data_node(NAME, INTERVAL);
 
 CREATE OR REPLACE FUNCTION _timescaledb_internal.ping_data_node(node_name NAME) RETURNS BOOLEAN
 AS '@MODULE_PATHNAME@', 'ts_data_node_ping' LANGUAGE C VOLATILE;
+
+DROP FUNCTION IF EXISTS _timescaledb_internal.get_approx_row_count(REGCLASS);

src/cross_module_fn.c

@@ -381,12 +381,6 @@ func_call_on_data_nodes_default(FunctionCallInfo finfo, List *data_node_oids)
 	pg_unreachable();
 }
 
-static void
-update_compressed_chunk_relstats_default(Oid uncompressed_relid, Oid compressed_relid)
-{
-	error_no_default_fn_community();
-}
-
 static void
 dist_update_stale_chunk_metadata_default(Chunk *new_chunk, List *chunk_data_nodes)
 {
@@ -560,7 +554,6 @@ TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default = {
 	.chunk_create_empty_table = error_no_default_fn_pg_community,
 	.chunk_create_replica_table = error_no_default_fn_pg_community,
 	.hypertable_distributed_set_replication_factor = error_no_default_fn_pg_community,
-	.update_compressed_chunk_relstats = update_compressed_chunk_relstats_default,
 	.health_check = error_no_default_fn_pg_community,
 	.mn_get_foreign_join_paths = mn_get_foreign_join_path_default_fn_pg_community,
 };

src/cross_module_fn.h

@@ -204,7 +204,6 @@ typedef struct CrossModuleFunctions
 	PGFunction chunk_freeze_chunk;
 	PGFunction chunk_unfreeze_chunk;
 	PGFunction chunks_drop_stale;
-	void (*update_compressed_chunk_relstats)(Oid uncompressed_relid, Oid compressed_relid);
 	PGFunction health_check;
 	void (*mn_get_foreign_join_paths)(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel,
 									  RelOptInfo *innerrel, JoinType jointype,

src/planner/planner.c

@@ -1332,20 +1332,6 @@ timescaledb_get_relation_info_hook(PlannerInfo *root, Oid relation_objectid, boo
 					 * IndexPaths at all
 					 */
 					rel->indexlist = NIL;
-
-					/* Relation size estimates are messed up on compressed chunks due to there
-					 * being no actual pages for the table in the storage manager.
-					 */
-					rel->pages = (BlockNumber) uncompressed_chunk->rd_rel->relpages;
-					rel->tuples = (double) uncompressed_chunk->rd_rel->reltuples;
-					if (rel->pages == 0)
-						rel->allvisfrac = 0.0;
-					else if (uncompressed_chunk->rd_rel->relallvisible >= (int32) rel->pages)
-						rel->allvisfrac = 1.0;
-					else
-						rel->allvisfrac =
-							(double) uncompressed_chunk->rd_rel->relallvisible / rel->pages;
-
 					table_close(uncompressed_chunk, NoLock);
 				}
 			}

src/process_utility.c

@@ -768,30 +768,6 @@ typedef struct ChunkPair
 	Oid compressed_relid;
 } ChunkPair;
 
-static void
-add_compressed_chunk_to_vacuum(Hypertable *ht, Oid comp_chunk_relid, void *arg)
-{
-	VacuumCtx *ctx = (VacuumCtx *) arg;
-	Chunk *compressed_chunk = ts_chunk_get_by_relid(comp_chunk_relid, true);
-	VacuumRelation *chunk_vacuum_rel;
-
-	Chunk *chunk_parent;
-	/* chunk is from a compressed hypertable */
-	Assert(TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht));
-
-	/*chunks for internal compression table have a parent */
-	chunk_parent = ts_chunk_get_compressed_chunk_parent(compressed_chunk);
-	Assert(chunk_parent != NULL);
-
-	ChunkPair *cp = palloc(sizeof(ChunkPair));
-	cp->uncompressed_relid = chunk_parent->table_id;
-	cp->compressed_relid = comp_chunk_relid;
-	ctx->chunk_pairs = lappend(ctx->chunk_pairs, cp);
-	/* analyze/vacuum the compressed rel instead */
-	chunk_vacuum_rel = makeVacuumRelation(NULL, comp_chunk_relid, NIL);
-	ctx->chunk_rels = lappend(ctx->chunk_rels, chunk_vacuum_rel);
-}
-
 /* Adds a chunk to the list of tables to be vacuumed */
 static void
 add_chunk_to_vacuum(Hypertable *ht, Oid chunk_relid, void *arg)
@@ -801,30 +777,11 @@ add_chunk_to_vacuum(Hypertable *ht, Oid chunk_relid, void *arg)
 	VacuumRelation *chunk_vacuum_rel;
 	RangeVar *chunk_range_var;
 
-	/* If the chunk has an associated compressed chunk, analyze that instead
-	 * When we compress a chunk, we save stats for the raw chunk, do
-	 * not modify that. Data now lives in the compressed chunk, so
-	 * analyze it.
-	 */
-	if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID)
-	{
-		Chunk *comp_chunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, true);
-		ChunkPair *cp = palloc(sizeof(ChunkPair));
-		cp->uncompressed_relid = chunk_relid;
-		cp->compressed_relid = comp_chunk->table_id;
-		ctx->chunk_pairs = lappend(ctx->chunk_pairs, cp);
-		/* analyze/vacuum the compressed rel instead */
-		chunk_vacuum_rel = makeVacuumRelation(NULL, comp_chunk->table_id, NIL);
-		ctx->chunk_rels = lappend(ctx->chunk_rels, chunk_vacuum_rel);
-	}
-	else
-	{
-		chunk_range_var = copyObject(ctx->ht_vacuum_rel->relation);
-		chunk_range_var->relname = NameStr(chunk->fd.table_name);
-		chunk_range_var->schemaname = NameStr(chunk->fd.schema_name);
-		chunk_vacuum_rel =
-			makeVacuumRelation(chunk_range_var, chunk_relid, ctx->ht_vacuum_rel->va_cols);
-	}
+	chunk_range_var = copyObject(ctx->ht_vacuum_rel->relation);
+	chunk_range_var->relname = NameStr(chunk->fd.table_name);
+	chunk_range_var->schemaname = NameStr(chunk->fd.schema_name);
+	chunk_vacuum_rel =
+		makeVacuumRelation(chunk_range_var, chunk_relid, ctx->ht_vacuum_rel->va_cols);
 	ctx->chunk_rels = lappend(ctx->chunk_rels, chunk_vacuum_rel);
 }
 
@@ -851,7 +808,6 @@ ts_get_all_vacuum_rels(bool is_vacuumcmd)
 	{
 		Form_pg_class classform = (Form_pg_class) GETSTRUCT(tuple);
 		Hypertable *ht;
-		Chunk *chunk;
 		Oid relid;
 
 		relid = classform->oid;
@@ -873,16 +829,8 @@ ts_get_all_vacuum_rels(bool is_vacuumcmd)
 
 		ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK);
 		if (ht)
-		{
 			if (hypertable_is_distributed(ht))
 				continue;
-		}
-		else
-		{
-			chunk = ts_chunk_get_by_relid(relid, false);
-			if (chunk && chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID)
-				continue;
-		}
 
 		/*
 		 * Build VacuumRelation(s) specifying the table OIDs to be processed.
@@ -950,17 +898,8 @@ process_vacuum(ProcessUtilityArgs *args)
 					 */
 					if (hypertable_is_distributed(ht))
 						continue;
-
-					if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht))
-					{
-						ctx.ht_vacuum_rel = vacuum_rel;
-						foreach_chunk(ht, add_compressed_chunk_to_vacuum, &ctx);
-					}
-					else
-					{
-						ctx.ht_vacuum_rel = vacuum_rel;
-						foreach_chunk(ht, add_chunk_to_vacuum, &ctx);
-					}
+					ctx.ht_vacuum_rel = vacuum_rel;
+					foreach_chunk(ht, add_chunk_to_vacuum, &ctx);
 				}
 			}
 			vacuum_rels = lappend(vacuum_rels, vacuum_rel);
@@ -978,12 +917,6 @@ process_vacuum(ProcessUtilityArgs *args)
 
 		/* ACL permission checks inside vacuum_rel and analyze_rel called by this ExecVacuum */
 		ExecVacuum(args->parse_state, stmt, is_toplevel);
-		foreach (lc, ctx.chunk_pairs)
-		{
-			ChunkPair *cp = (ChunkPair *) lfirst(lc);
-			ts_cm_functions->update_compressed_chunk_relstats(cp->uncompressed_relid,
-															  cp->compressed_relid);
-		}
 	}
 	/*
 	Restore original list. stmt->rels which has references to