Skip to content

Commit

Permalink
Avoid scaning the used region if there are no free units (dotnet#81525)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cshung
cshung committed Apr 5, 2023
1 parent 0f11212 commit 3309b0a
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 51 deletions.
128 changes: 84 additions & 44 deletions src/coreclr/gc/gc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3657,6 +3657,8 @@ bool region_allocator::init (uint8_t* start, uint8_t* end, size_t alignment, uin
global_region_end = (uint8_t*)align_region_down ((size_t)actual_end);
global_region_left_used = global_region_start;
global_region_right_used = global_region_end;
num_left_used_free_units = 0;
num_right_used_free_units = 0;

// Note: I am allocating a map that covers the whole reserved range.
// We can optimize it to only cover the current heap range.
Expand Down Expand Up @@ -3750,14 +3752,23 @@ void region_allocator::print_map (const char* msg)
}
current_index = region_map_right_start;
end_index = region_map_right_end;
if (i == 0)
{
assert (count_free_units == num_left_used_free_units);
}
else
{
assert (count_free_units == num_left_used_free_units + num_right_used_free_units);
}
}

count_free_units += (uint32_t)(region_map_right_start - region_map_left_end);
assert(count_free_units == total_free_units);

uint32_t total_regions = (uint32_t)((global_region_end - global_region_start) / region_alignment);

dprintf (REGIONS_LOG, ("[%s]-----end printing----[%d total, left used %zd, right used %zd]\n", heap_type, total_regions, (region_map_left_end - region_map_left_start), (region_map_right_end - region_map_right_start)));
dprintf (REGIONS_LOG, ("[%s]-----end printing----[%d total, left used %zd (free: %d), right used %zd (free: %d)]\n", heap_type, total_regions,
(region_map_left_end - region_map_left_start), num_left_used_free_units, (region_map_right_end - region_map_right_start), num_right_used_free_units));
#endif //_DEBUG
}

Expand Down Expand Up @@ -3841,59 +3852,75 @@ uint8_t* region_allocator::allocate (uint32_t num_units, allocate_direction dire

print_map ("before alloc");

while (((direction == allocate_forward) && (current_index < end_index)) ||
((direction == allocate_backward) && (current_index > end_index)))
if (((direction == allocate_forward) && (num_left_used_free_units >= num_units)) ||
((direction == allocate_backward) && (num_right_used_free_units >= num_units)))
{
uint32_t current_val = *(current_index - ((direction == -1) ? 1 : 0));
uint32_t current_num_units = get_num_units (current_val);
bool free_p = is_unit_memory_free (current_val);
dprintf (REGIONS_LOG, ("ALLOC[%s: %zd]%d->%d", (free_p ? "F" : "B"), (size_t)current_num_units,
(int)(current_index - region_map_left_start), (int)(current_index + current_num_units - region_map_left_start)));

if (free_p)
while (((direction == allocate_forward) && (current_index < end_index)) ||
((direction == allocate_backward) && (current_index > end_index)))
{
if (current_num_units >= num_units)
{
dprintf (REGIONS_LOG, ("found %zd contiguous free units(%d->%d), sufficient",
(size_t)current_num_units,
(int)(current_index - region_map_left_start),
(int)(current_index - region_map_left_start + current_num_units)));
uint32_t current_val = *(current_index - ((direction == allocate_backward) ? 1 : 0));
uint32_t current_num_units = get_num_units (current_val);
bool free_p = is_unit_memory_free (current_val);
dprintf (REGIONS_LOG, ("ALLOC[%s: %zd]%d->%d", (free_p ? "F" : "B"), (size_t)current_num_units,
(int)(current_index - region_map_left_start), (int)(current_index + current_num_units - region_map_left_start)));

uint32_t* busy_block;
uint32_t* free_block;
if (direction == 1)
{
busy_block = current_index;
free_block = current_index + num_units;
}
else
if (free_p)
{
if (current_num_units >= num_units)
{
busy_block = current_index - num_units;
free_block = current_index - current_num_units;
}
dprintf (REGIONS_LOG, ("found %zd contiguous free units(%d->%d), sufficient",
(size_t)current_num_units,
(int)(current_index - region_map_left_start),
(int)(current_index - region_map_left_start + current_num_units)));

make_busy_block (busy_block, num_units);
if ((current_num_units - num_units) > 0)
{
make_free_block (free_block, (current_num_units - num_units));
}
if (direction == allocate_forward)
{
assert (num_left_used_free_units >= num_units);
num_left_used_free_units -= num_units;
}
else
{
assert (direction == allocate_backward);
assert (num_right_used_free_units >= num_units);
num_right_used_free_units -= num_units;
}

total_free_units -= num_units;
print_map ("alloc: found in free");
uint32_t* busy_block;
uint32_t* free_block;
if (direction == 1)
{
busy_block = current_index;
free_block = current_index + num_units;
}
else
{
busy_block = current_index - num_units;
free_block = current_index - current_num_units;
}

make_busy_block (busy_block, num_units);
if ((current_num_units - num_units) > 0)
{
make_free_block (free_block, (current_num_units - num_units));
}

total_free_units -= num_units;
print_map ("alloc: found in free");

leave_spin_lock();
leave_spin_lock();

return region_address_of (busy_block);
return region_address_of (busy_block);
}
}
}

if (direction == allocate_forward)
{
current_index += current_num_units;
}
else
{
current_index -= current_num_units;
if (direction == allocate_forward)
{
current_index += current_num_units;
}
else
{
current_index -= current_num_units;
}
}
}

Expand Down Expand Up @@ -4009,6 +4036,17 @@ void region_allocator::delete_region_impl (uint8_t* region_start)

int free_block_size = current_val;
uint32_t* free_index = current_index;

if (free_index <= region_map_left_end)
{
num_left_used_free_units += free_block_size;
}
else
{
assert (free_index >= region_map_right_start);
num_right_used_free_units += free_block_size;
}

if ((current_index != region_map_left_start) && (current_index != region_map_right_start))
{
uint32_t previous_val = *(current_index - 1);
Expand All @@ -4031,13 +4069,15 @@ void region_allocator::delete_region_impl (uint8_t* region_start)
}
if (region_end == global_region_left_used)
{
num_left_used_free_units -= free_block_size;
region_map_left_end = free_index;
dprintf (REGIONS_LOG, ("adjust global left used from %p to %p",
global_region_left_used, region_address_of (free_index)));
global_region_left_used = region_address_of (free_index);
}
else if (region_start == global_region_right_used)
{
num_right_used_free_units -= free_block_size;
region_map_right_start = free_index + free_block_size;
dprintf (REGIONS_LOG, ("adjust global right used from %p to %p",
global_region_right_used, region_address_of (free_index + free_block_size)));
Expand Down
17 changes: 10 additions & 7 deletions src/coreclr/gc/gcpriv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5007,14 +5007,14 @@ typedef bool (*region_allocator_callback_fn)(uint8_t*);
//
// For each region we encode the info with a busy block in the map. This block has the
// same # of uints as the # of units this region occupies. And we store the # in
// the starting uint. These uints can be converted to bytes since we have multiple units
// for larger regions anyway. I haven't done that since this will need to be changed in
// the near future based on more optimal allocation strategies.
// the first and last uint. These uints can be converted to bytes since we have multiple
// units for larger regions anyway. I haven't done that since this will need to be changed
// in the near future based on more optimal allocation strategies.
//
// When we allocate, we search forward to find contiguous free units >= num_units
// We do take the opportunity to coalesce free blocks but we do not coalesce busy blocks.
// When we decommit a region, we simply mark its block free. Free blocks are coalesced
// opportunistically when we need to walk them.
// When we allocate, if we knew there could be free blocks that fits, we search forward to find
// contiguous free units >= num_units. Otherwise we simply allocate at the end. We coalesce
// free blocks but we do not coalesce busy blocks. When we delete a region, we mark the block
// free and coalesced them with its free neighbors if any.
//
// TODO: to accommodate 32-bit processes, we reserve in segment sizes and divide each seg
// into regions.
Expand All @@ -5040,6 +5040,9 @@ class region_allocator
uint32_t* region_map_right_start;
uint32_t* region_map_right_end;

uint32_t num_left_used_free_units;
uint32_t num_right_used_free_units;

uint8_t* region_address_of (uint32_t* map_index);
uint32_t* region_map_index_of (uint8_t* address);

Expand Down

0 comments on commit 3309b0a

Please sign in to comment.