Refactor option to skip compressed in Hypercore TAM scans

Replace the scankey flag used to skip compressed data when starting a
Hypercore scan with a function that sets this option on the scan
descriptor. Internally, use the scan flags instead of scankey flags to
convey this setting.

Overriding scankey flags was not ideal since this is supposed to be
per-column settings and not overall scan settings.

It is possible to set the scan flags when calling table_beginscan(),
but current table scan functions do not expose flags and instead have
a separate function for each flag settings. Hypercore could define its
own beginscan function to do the same, but this is left for the
future.

tsl/src/compression/api.c

@@ -1138,13 +1138,8 @@ fetch_unmatched_uncompressed_chunk_into_tuplesort(Tuplesortstate *segment_tuples
 	TableScanDesc scan;
 	TupleTableSlot *slot = table_slot_create(uncompressed_chunk_rel, NULL);
 	Snapshot snapshot = GetLatestSnapshot();
-	ScanKeyData scankey = {
-		/* Let compression TAM know it should only return tuples from the
-		 * non-compressed relation. No actual scankey necessary */
-		.sk_flags = SK_NO_COMPRESSED,
-	};
-
-	scan = table_beginscan(uncompressed_chunk_rel, snapshot, 0, &scankey);
+	scan = table_beginscan(uncompressed_chunk_rel, snapshot, 0, NULL);
+	hypercore_scan_set_skip_compressed(scan);
 
 	while (table_scan_getnextslot(scan, ForwardScanDirection, slot))
 	{
@@ -1213,8 +1208,9 @@ fetch_matching_uncompressed_chunk_into_tuplesort(Tuplesortstate *segment_tupleso
 	snapshot = GetLatestSnapshot();
 	/* Let compression TAM know it should only return tuples from the
 	 * non-compressed relation. */
-	scankey->sk_flags = SK_NO_COMPRESSED;
+
 	scan = table_beginscan(uncompressed_chunk_rel, snapshot, nsegbycols_nonnull, scankey);
+	hypercore_scan_set_skip_compressed(scan);
 	TupleTableSlot *slot = table_slot_create(uncompressed_chunk_rel, NULL);
 
 	while (table_scan_getnextslot(scan, ForwardScanDirection, slot))

tsl/src/compression/compression.c

@@ -182,12 +182,8 @@ static void
 RelationDeleteAllRows(Relation rel, Snapshot snap)
 {
 	TupleTableSlot *slot = table_slot_create(rel, NULL);
-	ScanKeyData scankey = {
-		/* Let compression TAM know it should only return tuples from the
-		 * non-compressed relation. No actual scankey necessary */
-		.sk_flags = SK_NO_COMPRESSED,
-	};
-	TableScanDesc scan = table_beginscan(rel, snap, 0, &scankey);
+	TableScanDesc scan = table_beginscan(rel, snap, 0, NULL);
+	hypercore_scan_set_skip_compressed(scan);
 
 	while (table_scan_getnextslot(scan, ForwardScanDirection, slot))
 	{
@@ -576,13 +572,9 @@ compress_chunk_sort_relation(CompressionSettings *settings, Relation in_rel)
 	Tuplesortstate *tuplesortstate;
 	TableScanDesc scan;
 	TupleTableSlot *slot;
-	ScanKeyData scankey = {
-		/* Let compression TAM know it should only return tuples from the
-		 * non-compressed relation. No actual scankey necessary */
-		.sk_flags = SK_NO_COMPRESSED,
-	};
 	tuplesortstate = compression_create_tuplesort_state(settings, in_rel);
-	scan = table_beginscan(in_rel, GetLatestSnapshot(), 0, &scankey);
+	scan = table_beginscan(in_rel, GetLatestSnapshot(), 0, NULL);
+	hypercore_scan_set_skip_compressed(scan);
 	slot = table_slot_create(in_rel, NULL);
 
 	while (table_scan_getnextslot(scan, ForwardScanDirection, slot))

tsl/src/hypercore/hypercore_handler.c

@@ -367,6 +367,7 @@ typedef struct HypercoreScanDescData
 	int32 compressed_row_count;
 	HypercoreScanState hs_scan_state;
 	bool reset;
+	bool skip_compressed; /* Skip compressed data when scanning */
 #if PG17_GE
 	/* These fields are only used for ANALYZE */
 	ReadStream *canalyze_read_stream;
@@ -381,6 +382,19 @@ static bool hypercore_getnextslot_noncompressed(HypercoreScanDesc scan, ScanDire
 static bool hypercore_getnextslot_compressed(HypercoreScanDesc scan, ScanDirection direction,
 											 TupleTableSlot *slot);
 
+void
+hypercore_scan_set_skip_compressed(TableScanDesc scan)
+{
+	HypercoreScanDesc hscan;
+
+	if (!REL_IS_HYPERCORE(scan->rs_rd))
+		return;
+
+	hscan = (HypercoreScanDesc) scan;
+	scan->rs_flags |= SO_HYPERCORE_SKIP_COMPRESSED;
+	hscan->hs_scan_state = HYPERCORE_SCAN_NON_COMPRESSED;
+}
+
 #if PG17_GE
 static int
 compute_targrows(Relation rel)
@@ -514,7 +528,7 @@ hypercore_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key
 	scan->compressed_rel = table_open(hsinfo->compressed_relid, AccessShareLock);
 
 	if ((ts_guc_enable_transparent_decompression == 2) ||
-		(keys && keys->sk_flags & SK_NO_COMPRESSED))
+		(flags & SO_HYPERCORE_SKIP_COMPRESSED))
 	{
 		/*
 		 * Don't read compressed data if transparent decompression is enabled
@@ -570,7 +584,7 @@ hypercore_rescan(TableScanDesc sscan, ScanKey key, bool set_params, bool allow_s
 	scan->reset = true;
 	scan->hs_scan_state = HYPERCORE_SCAN_START;
 
-	if (key && key->sk_flags & SK_NO_COMPRESSED)
+	if (sscan->rs_flags & SO_HYPERCORE_SKIP_COMPRESSED)
 		scan->hs_scan_state = HYPERCORE_SCAN_NON_COMPRESSED;
 
 	if (scan->cscan_desc)

tsl/src/hypercore/hypercore_handler.h

@@ -13,10 +13,12 @@
 
 #include "hypertable.h"
 
-/* Scan key flag (skey.h) to indicate that a table scan should only return
- * tuples from the non-compressed relation. Bits 16-31 are reserved for
- * individual access methods, so use bit 16. */
-#define SK_NO_COMPRESSED 0x8000
+typedef enum HypercoreScanOptions
+{
+	/* Normal scan options stretch to 9th bit. Start at bit 15 out of 32 to be
+	 * safe. */
+	SO_HYPERCORE_SKIP_COMPRESSED = 1 << 15,
+} HypercoreScanOptions;
 
 extern void hypercore_set_analyze_relid(Oid relid);
 extern const TableAmRoutine *hypercore_routine(void);
@@ -26,6 +28,7 @@ extern void hypercore_alter_access_method_finish(Oid relid, bool to_other_am);
 extern Datum hypercore_handler(PG_FUNCTION_ARGS);
 extern void hypercore_xact_event(XactEvent event, void *arg);
 extern bool hypercore_set_truncate_compressed(bool onoff);
+extern void hypercore_scan_set_skip_compressed(TableScanDesc scan);
 
 typedef struct ColumnCompressionSettings
 {