[mono][sgen] Prevent concurrent sweep from blocking major collections (…

…dotnet#98154)

Sweeping is the final GC stage, that does a final iteration on the memory in order to prepare internal data structures for future allocation, free memory etc. For large objects this is done during GC while for small objects it can be done concurrently with the mutator, after we resume the world. We attempt to trigger the next major collectio when the heap grows with another third of its size at the moment of the last collection. This heap limit is the major trigger size and it can only be computed after the sweep job has finished. This is because sweep iterates each major block to determine if they have any marked objects, attempting to free the blocks if possible.

Because there is a dependency for setting the major trigger size on the sweep job completion, before this change we were just blocking any new major collections to happen if concurrent sweep wasn't finished. This means that if the sweep job takes longer than expected and the mutator does excessive allocation in short amount of time, the memory usage can increase aggresively. It is unclear how relevant this scenario is in practice, but it is easy to reproduce in a micro benchmark allocating large objects.

The fix relies on computing estimates for the current heap size and the soon to be computed trigger size. We estimate the current live number of major blocks by adding the number of major blocks allocated since the last collection to the number of blocks already traversed and determined to be live by sweep. This estimate will be a lower limit. We determine the number of major blocks during the previous collection by subtracting the number of major blocks before sweep starts by the number of blocks already traversed and determined to be free. This estimate will be an upper limit. If the lower limit of the current heap size exceeds the upper limit of the trigger size, then there is no point in waiting for sweep to finish. We know we will require a major collection.

src/mono/mono/sgen/sgen-gc.h

@@ -716,6 +716,8 @@ struct _SgenMajorCollector {
 	gboolean (*ptr_is_from_pinned_alloc) (char *ptr);
 	void (*report_pinned_memory_usage) (void);
 	size_t (*get_num_major_sections) (void);
+	size_t (*get_min_live_major_sections) (void);
+	size_t (*get_max_last_major_survived_sections) (void);
 	size_t (*get_num_empty_blocks) (void);
 	size_t (*get_bytes_survived_last_sweep) (void);
 	gboolean (*handle_gc_param) (const char *opt);

src/mono/mono/sgen/sgen-marksweep.c

@@ -1524,6 +1524,7 @@ static size_t *sweep_num_blocks;
 
 static volatile size_t num_major_sections_before_sweep;
 static volatile size_t num_major_sections_freed_in_sweep;
+static volatile size_t num_major_sections_survived_in_sweep;
 
 static void
 sgen_worker_clear_free_block_lists (WorkerData *worker)
@@ -1707,6 +1708,7 @@ ensure_block_is_checked_for_sweeping (guint32 block_index, gboolean wait, gboole
 
 		/* FIXME: Do we need the heap boundaries while we do nursery collections? */
 		update_heap_boundaries_for_block (block);
+		SGEN_ATOMIC_ADD_P (num_major_sections_survived_in_sweep, 1);
 	} else {
 		/*
 		 * Blocks without live objects are removed from the
@@ -1842,6 +1844,7 @@ major_sweep (void)
 
 	num_major_sections_before_sweep = num_major_sections;
 	num_major_sections_freed_in_sweep = 0;
+	num_major_sections_survived_in_sweep = 0;
 
 	SGEN_ASSERT (0, !sweep_job, "We haven't finished the last sweep?");
 	if (concurrent_sweep) {
@@ -2319,6 +2322,28 @@ get_num_major_sections (void)
 	return num_major_sections;
 }
 
+// Conservative values for computing trigger size, without needing concurrent sweep to finish
+// As concurrent sweep job advances in execution, these values get closer to the real value.
+// This contains at least the number of blocks determined to be live by sweep job (which increases
+// as sweep progresses) plus any new blocks allocated by the application.
+static size_t
+get_min_live_major_sections (void)
+{
+	// Note that num_major_sections gets decremented for each freed block, so to obtain the real block count
+	// we would need to add back num_major_sections_freed_in_sweep, but this is racy so we are being conservative.
+	if (num_major_sections > num_major_sections_before_sweep)
+		return num_major_sections_survived_in_sweep + (num_major_sections - num_major_sections_before_sweep);
+	else
+		return num_major_sections_survived_in_sweep;
+}
+
+static size_t
+get_max_last_major_survived_sections (void)
+{
+	// num_major_sections_freed_in_sweep increases as sweep progresses.
+	return num_major_sections_before_sweep - num_major_sections_freed_in_sweep;
+}
+
 static size_t
 get_num_empty_blocks (void)
 {
@@ -2886,6 +2911,8 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 	collector->ptr_is_from_pinned_alloc = ptr_is_from_pinned_alloc;
 	collector->report_pinned_memory_usage = major_report_pinned_memory_usage;
 	collector->get_num_major_sections = get_num_major_sections;
+	collector->get_min_live_major_sections = get_min_live_major_sections;
+	collector->get_max_last_major_survived_sections = get_max_last_major_survived_sections;
 	collector->get_num_empty_blocks = get_num_empty_blocks;
 	collector->get_bytes_survived_last_sweep = get_bytes_survived_last_sweep;
 	collector->handle_gc_param = major_handle_gc_param;

src/mono/mono/sgen/sgen-memory-governor.c

@@ -130,6 +130,20 @@ sgen_memgov_calculate_minor_collection_allowance (void)
 	}
 }
 
+// This can be called while sweep is running to determine earlier if there is so much memory growth
+// that we know we will require a GC once sweep finishes.
+static gboolean
+sgen_need_major_collection_conservative (void)
+{
+	size_t min_heap_size = sgen_los_memory_usage + sgen_major_collector.get_min_live_major_sections () * sgen_major_collector.section_size;
+
+	size_t max_last_collection_heap_size = last_collection_los_memory_usage + sgen_major_collector.get_max_last_major_survived_sections () * sgen_major_collector.section_size;
+	size_t max_allowance = GDOUBLE_TO_SIZE (max_last_collection_heap_size * SGEN_DEFAULT_ALLOWANCE_HEAP_SIZE_RATIO);
+	max_allowance = MAX (max_allowance, GDOUBLE_TO_SIZE (MIN_MINOR_COLLECTION_ALLOWANCE));
+
+	return min_heap_size > max_allowance;
+}
+
 static size_t
 get_heap_size (void)
 {
@@ -184,9 +198,11 @@ sgen_need_major_collection (mword space_needed, gboolean *forced)
 		return FALSE;
 	}
 
-	/* FIXME: This is a cop-out.  We should have some way of figuring this out. */
-	if (!sgen_major_collector.have_swept ())
+	if (!sgen_major_collector.have_swept ()) {
+		if (sgen_need_major_collection_conservative ())
+			return TRUE;
 		return FALSE;
+	}
 
 	if (space_needed > sgen_memgov_available_free_space ())
 		return TRUE;