reduce handle table lock contention (dotnet#94159)

when a handle is created it's using the home heap GC set for this thread but this home heap is for GC heap allocation. a thread could be allocating a lot of handles but hardly any allocation so multiple threads could contend on the same handle table which does not handle this situation well (it causes long suspension). also when there are way fewer heaps than number of procs, this means we are only allocating on way fewer handle tables.

this change moves the balancing of the handle allocations into the handle table code itself. I've repurposed the alloc_count field to include the handle table index and switch to a different table after the thread has allocated 15 handles. this uses all handle tables.

src/coreclr/gc/gc.cpp

@@ -19072,9 +19072,9 @@ allocation_state gc_heap::try_allocate_more_space (alloc_context* acontext, size
 #ifdef MULTIPLE_HEAPS
 void gc_heap::balance_heaps (alloc_context* acontext)
 {
-    if (acontext->alloc_count < 4)
+    if (acontext->get_alloc_count() < 4)
     {
-        if (acontext->alloc_count == 0)
+        if (acontext->get_alloc_count() == 0)
         {
             int home_hp_num = heap_select::select_heap (acontext);
             acontext->set_home_heap (GCHeap::GetHeap (home_hp_num));
@@ -19123,12 +19123,12 @@ void gc_heap::balance_heaps (alloc_context* acontext)
 #endif //HEAP_BALANCE_INSTRUMENTATION
                 set_home_heap = TRUE;
             }
-            else if ((acontext->alloc_count & 15) == 0)
+            else if ((acontext->get_alloc_count() & 15) == 0)
                 set_home_heap = TRUE;
         }
         else
         {
-            if ((acontext->alloc_count & 3) == 0)
+            if ((acontext->get_alloc_count() & 3) == 0)
                 set_home_heap = TRUE;
         }
 
@@ -19158,7 +19158,7 @@ void gc_heap::balance_heaps (alloc_context* acontext)
                 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMP[p%3d] ph h%3d, hh: %3d, ah: %3d (%dmb-%dmb), ac: %5d(%s)",
                     proc_no, proc_hp_num, home_hp->heap_number,
                     org_hp_num, (total_size / 1024 / 1024), (org_size / 1024 / 1024),
-                    acontext->alloc_count,
+                    acontext->get_alloc_count(),
                     ((proc_hp_num == home_hp->heap_number) ? "AC" : "H")));
 #endif //HEAP_BALANCE_INSTRUMENTATION
 
@@ -19172,7 +19172,7 @@ void gc_heap::balance_heaps (alloc_context* acontext)
 
                 if (((size_t)org_size + 2 * delta) >= (size_t)total_size)
                 {
-                    acontext->alloc_count++;
+                    acontext->inc_alloc_count();
                     return;
                 }
 
@@ -19256,10 +19256,10 @@ void gc_heap::balance_heaps (alloc_context* acontext)
                         // alloc_count often increases by multiples of 16 (due to logic at top of routine),
                         // and we want to advance the starting point by 4 between successive calls,
                         // therefore the shift right by 2 bits
-                        int heap_num = start + ((acontext->alloc_count >> 2) + new_home_hp_num) % count;
+                        int heap_num = start + ((acontext->get_alloc_count() >> 2) + new_home_hp_num) % count;
 
 #ifdef HEAP_BALANCE_INSTRUMENTATION
-                        dprintf(HEAP_BALANCE_TEMP_LOG, ("TEMP starting at h%d (home_heap_num = %d, alloc_count = %d)", heap_num, new_home_hp_num, acontext->alloc_count));
+                        dprintf(HEAP_BALANCE_TEMP_LOG, ("TEMP starting at h%d (home_heap_num = %d, alloc_count = %d)", heap_num, new_home_hp_num, acontext->get_alloc_count()));
 #endif //HEAP_BALANCE_INSTRUMENTATION
 
                         for (int tries = max_tries; --tries >= 0; heap_num++)
@@ -19375,7 +19375,7 @@ void gc_heap::balance_heaps (alloc_context* acontext)
             }
         }
     }
-    acontext->alloc_count++;
+    acontext->inc_alloc_count();
 }
 
 ptrdiff_t gc_heap::get_balance_heaps_uoh_effective_budget (int generation_num)
@@ -49366,7 +49366,7 @@ GCHeap::FixAllocContext (gc_alloc_context* context, void* arg, void *heap)
 #ifdef MULTIPLE_HEAPS
 
     if (arg != 0)
-        acontext->alloc_count = 0;
+        acontext->init_alloc_count();
 
     uint8_t * alloc_ptr = acontext->alloc_ptr;
 
@@ -50570,6 +50570,7 @@ void GCHeap::AssignHeap (alloc_context* acontext)
     // Assign heap based on processor
     acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext)));
     acontext->set_home_heap(acontext->get_alloc_heap());
+    acontext->init_handle_info();
 }
 
 GCHeap* GCHeap::GetHeap (int n)

src/coreclr/gc/gc.h

@@ -240,6 +240,49 @@ struct alloc_context : gc_alloc_context
     {
         gc_reserved_2 = heap;
     }
+
+    // How the alloc_count field is organized -
+    // 
+    // high 16-bits are for the handle info, out of which
+    // high 10 bits store the cpu index. 
+    // low 6 bits store the number of handles allocated so far (before the next reset).
+    // 
+    // low 16-bits are for the actual alloc_count used by balance_heaps
+    inline void init_alloc_count()
+    {
+        alloc_count &= 0xffff0000;
+    }
+
+    inline uint16_t get_alloc_count()
+    {
+        return (uint16_t)alloc_count;
+    }
+
+    inline void inc_alloc_count()
+    {
+        int high_16_bits = (uint32_t)alloc_count >> 16;
+        int low_16_bits = alloc_count & 0xffff;
+        // When we overflow we don't start from 0 because we would't want to go through the init logic again
+        // in balance_heaps.
+        low_16_bits = (low_16_bits == 0xffff) ? 16 : (low_16_bits + 1);
+
+        alloc_count = (high_16_bits << 16) | low_16_bits;
+    }
+
+    inline void init_handle_info()
+    {
+        // Start the handle table index based on the AC value to make it random. There may have been handles
+        // already allocated before this and that's fine.
+        int cpu_index = ((size_t)this >> 4) % g_num_processors;
+        int handle_info = cpu_index << 6;
+        alloc_count = handle_info << 16;
+    }
+
+    inline void set_handle_info (int handle_info)
+    {
+        int low_16_bits = alloc_count & 0xffff;
+        alloc_count = low_16_bits | (handle_info << 16);
+    }
 #endif // FEATURE_SVR_GC
 };
 

src/coreclr/gc/gchandletable.cpp

@@ -25,9 +25,55 @@ bool GCHandleStore::ContainsHandle(OBJECTHANDLE handle)
     return _underlyingBucket.Contains(handle);
 }
 
+// this is the number of handles we allocate in a handle table before we switch to the next table.
+#define HANDLE_THRESHOLD (15)
+static int s_numTableSlots = 0;
+
+HHANDLETABLE GCHandleStore::GetTable()
+{
+    if (!s_numTableSlots)
+    {
+        s_numTableSlots = getNumberOfSlots();
+    }
+
+    if (s_numTableSlots == 1)
+        return _underlyingBucket.pTable[0];
+
+    gc_alloc_context* ctx = GetCurrentThreadAllocContext();
+    if (!ctx)
+        return _underlyingBucket.pTable[0];
+
+    // high 16-bits are for the handle info.
+    int handleInfo = (uint32_t)(ctx->alloc_count) >> 16;
+
+    // high 10 bits store the cpu index. 
+    // low 6 bits store the # of handles allocated so far (before the next reset).
+    int numHandles = handleInfo & 0x3f;
+    int cpuIndex = handleInfo >> 6;
+    int savedCpuIndex = cpuIndex;
+    if (numHandles == HANDLE_THRESHOLD)
+    {
+        numHandles = 0;
+        cpuIndex++;
+        cpuIndex = (cpuIndex > (s_numTableSlots - 1)) ? 0 : cpuIndex;
+    }
+    else
+    {
+        numHandles++;
+    }
+
+    int newHandleInfo = numHandles | (cpuIndex << 6);
+
+    int low_16_bits = ctx->alloc_count & 0xffff;
+    ctx->alloc_count = low_16_bits | (newHandleInfo << 16);
+
+    HHANDLETABLE handletable = _underlyingBucket.pTable[cpuIndex];
+    return handletable;
+}
+
 OBJECTHANDLE GCHandleStore::CreateHandleOfType(Object* object, HandleType type)
 {
-    HHANDLETABLE handletable = _underlyingBucket.pTable[GetCurrentThreadHomeHeapNumber()];
+    HHANDLETABLE handletable = GetTable();
     return ::HndCreateHandle(handletable, type, ObjectToOBJECTREF(object));
 }
 

src/coreclr/gc/gchandletableimpl.h

@@ -25,6 +25,9 @@ class GCHandleStore : public IGCHandleStore
     virtual ~GCHandleStore();
 
     HandleTableBucket _underlyingBucket;
+
+private:
+    HHANDLETABLE GetTable();
 };
 
 extern GCHandleStore* g_gcGlobalHandleStore;

src/coreclr/gc/handletable.cpp

@@ -906,9 +906,6 @@ void HndNotifyGcCycleComplete(HHANDLETABLE hTable, uint32_t condemned, uint32_t
 #endif
 }
 
-extern int getNumberOfSlots();
-
-
 /*
  * HndCountHandles
  *

src/coreclr/gc/objecthandle.cpp

@@ -1854,6 +1854,11 @@ int GetCurrentThreadHomeHeapNumber()
     return g_theGCHeap->GetHomeHeapNumber();
 }
 
+gc_alloc_context* GetCurrentThreadAllocContext()
+{
+    return GCToEEInterface::GetAllocContext();
+}
+
 bool HandleTableBucket::Contains(OBJECTHANDLE handle)
 {
     LIMITED_METHOD_CONTRACT;

src/coreclr/gc/objecthandle.h

@@ -71,6 +71,10 @@ uint32_t     CompareExchangeVariableHandleType(OBJECTHANDLE handle, uint32_t old
  */
 
 int GetCurrentThreadHomeHeapNumber();
+gc_alloc_context* GetCurrentThreadAllocContext();
+
+// helper for getting the total number of handle table slots we can allocate in
+int getNumberOfSlots();
 
 /*
  * Table maintenance routines