Add config var to allow disabling automatic CPU group assignment for …

…threads (dotnet#38568)

- When CPU groups are enabled through config, some new threads (Thread.Start, thread pool threads) are automatically assigned to a CPU group, starting with the process' CPU group and once it is filled up, assigning new threads to a different CPU group. An app may have native components that also use threads, and may want to do its own thread-spreading (for instance in DllMain), this config allows such apps to disable the automatic CPU group assignment which would otherwise override an assignment made in DllMain. The new config var does not affect GC threads.
- This was requested for .NET Framework, porting to .NET Core as well

src/coreclr/src/inc/clrconfigvalues.h

@@ -553,6 +553,8 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadSuspendInjection, W("INTERNAL_ThreadSusp
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_DefaultStackSize, W("DefaultStackSize"), 0, "Stack size to use for new VM threads when thread is created with default stack size (dwStackSize == 0).")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_Thread_DeadThreadCountThresholdForGCTrigger, W("Thread_DeadThreadCountThresholdForGCTrigger"), 75, "In the heuristics to clean up dead threads, this threshold must be reached before triggering a GC will be considered. Set to 0 to disable triggering a GC based on dead threads.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_Thread_DeadThreadGCTriggerPeriodMilliseconds, W("Thread_DeadThreadGCTriggerPeriodMilliseconds"), 1000 * 60 * 30, "In the heuristics to clean up dead threads, this much time must have elapsed since the previous max-generation GC before triggering another GC will be considered")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_Thread_UseAllCpuGroups, W("Thread_UseAllCpuGroups"), 0, "Specifies whether to query and use CPU group information for determining the processor count.")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_Thread_AssignCpuGroups, W("Thread_AssignCpuGroups"), 1, "Specifies whether to automatically distribute threads created by the CLR across CPU Groups. Effective only when Thread_UseAllCpuGroups and GCCpuGroup are enabled.")
 
 ///
 /// Threadpool
@@ -568,7 +570,6 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("Thread
 #else // !TARGET_ARM64
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("ThreadPool_UnfairSemaphoreSpinLimit"), 0x46, "Maximum number of spins a thread pool worker thread performs before waiting for work")
 #endif // TARGET_ARM64
-RETAIL_CONFIG_DWORD_INFO(EXTERNAL_Thread_UseAllCpuGroups, W("Thread_UseAllCpuGroups"), 0, "Specifies if to automatically distribute thread across CPU Groups")
 
 CONFIG_DWORD_INFO(INTERNAL_ThreadpoolTickCountAdjustment, W("ThreadpoolTickCountAdjustment"), 0, "")
 

src/coreclr/src/inc/utilcode.h

@@ -1272,6 +1272,7 @@ class CPUGroupInfo
     static WORD m_nProcessors;
     static BOOL m_enableGCCPUGroups;
     static BOOL m_threadUseAllCpuGroups;
+    static BOOL m_threadAssignCpuGroups;
     static WORD m_initialGroup;
     static CPU_Group_Info *m_CPUGroupInfoArray;
     static bool s_hadSingleProcessorAtStartup;
@@ -1285,6 +1286,7 @@ class CPUGroupInfo
     static void EnsureInitialized();
     static BOOL CanEnableGCCPUGroups();
     static BOOL CanEnableThreadUseAllCpuGroups();
+    static BOOL CanAssignCpuGroupsToThreads();
     static WORD GetNumActiveProcessors();
     static void GetGroupForProcessor(WORD processor_number,
 		    WORD *group_number, WORD *group_processor_number);

src/coreclr/src/utilcode/util.cpp

@@ -850,6 +850,7 @@ BYTE * ClrVirtualAllocWithinRange(const BYTE *pMinAddr,
 
 /*static*/ BOOL CPUGroupInfo::m_enableGCCPUGroups = FALSE;
 /*static*/ BOOL CPUGroupInfo::m_threadUseAllCpuGroups = FALSE;
+/*static*/ BOOL CPUGroupInfo::m_threadAssignCpuGroups = FALSE;
 /*static*/ WORD CPUGroupInfo::m_nGroups = 0;
 /*static*/ WORD CPUGroupInfo::m_nProcessors = 0;
 /*static*/ WORD CPUGroupInfo::m_initialGroup = 0;
@@ -991,6 +992,7 @@ DWORD LCM(DWORD u, DWORD v)
 #if !defined(FEATURE_REDHAWK) && (defined(TARGET_AMD64) || defined(TARGET_ARM64))
     BOOL enableGCCPUGroups     = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCCpuGroup) != 0;
     BOOL threadUseAllCpuGroups = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_Thread_UseAllCpuGroups) != 0;
+    BOOL threadAssignCpuGroups = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_Thread_AssignCpuGroups) != 0;
 
     if (!enableGCCPUGroups)
         return;
@@ -1006,10 +1008,11 @@ DWORD LCM(DWORD u, DWORD v)
     CPUGroupInfo::GetThreadGroupAffinity(GetCurrentThread(), &groupAffinity);
     m_initialGroup = groupAffinity.Group;
 
-	// only enable CPU groups if more than one group exists
-	BOOL hasMultipleGroups = m_nGroups > 1;
-	m_enableGCCPUGroups = enableGCCPUGroups && hasMultipleGroups;
-	m_threadUseAllCpuGroups = threadUseAllCpuGroups && hasMultipleGroups;
+    // only enable CPU groups if more than one group exists
+    BOOL hasMultipleGroups = m_nGroups > 1;
+    m_enableGCCPUGroups = enableGCCPUGroups && hasMultipleGroups;
+    m_threadUseAllCpuGroups = threadUseAllCpuGroups && hasMultipleGroups;
+    m_threadAssignCpuGroups = threadAssignCpuGroups && hasMultipleGroups;
 #endif // TARGET_AMD64 || TARGET_ARM64
 
     // Determine if the process is affinitized to a single processor (or if the system has a single processor)
@@ -1164,8 +1167,8 @@ DWORD LCM(DWORD u, DWORD v)
     WORD i, minGroup = 0;
     DWORD minWeight = 0;
 
-    // m_enableGCCPUGroups and m_threadUseAllCpuGroups must be TRUE
-    _ASSERTE(m_enableGCCPUGroups && m_threadUseAllCpuGroups);
+    // m_enableGCCPUGroups, m_threadUseAllCpuGroups, and m_threadAssignCpuGroups must be TRUE
+    _ASSERTE(m_enableGCCPUGroups && m_threadUseAllCpuGroups && m_threadAssignCpuGroups);
 
     for (i = 0; i < m_nGroups; i++)
     {
@@ -1204,8 +1207,8 @@ DWORD LCM(DWORD u, DWORD v)
 {
     LIMITED_METHOD_CONTRACT;
 #if (defined(TARGET_AMD64) || defined(TARGET_ARM64))
-    // m_enableGCCPUGroups and m_threadUseAllCpuGroups must be TRUE
-    _ASSERTE(m_enableGCCPUGroups && m_threadUseAllCpuGroups);
+    // m_enableGCCPUGroups, m_threadUseAllCpuGroups, and m_threadAssignCpuGroups must be TRUE
+    _ASSERTE(m_enableGCCPUGroups && m_threadUseAllCpuGroups && m_threadAssignCpuGroups);
 
     WORD group = gf->Group;
     m_CPUGroupInfoArray[group].activeThreadWeight -= m_CPUGroupInfoArray[group].groupWeight;
@@ -1238,6 +1241,12 @@ BOOL CPUGroupInfo::GetCPUGroupRange(WORD group_number, WORD* group_begin, WORD*
     LIMITED_METHOD_CONTRACT;
     return m_threadUseAllCpuGroups;
 }
+
+/*static*/ BOOL CPUGroupInfo::CanAssignCpuGroupsToThreads()
+{
+    LIMITED_METHOD_CONTRACT;
+    return m_threadAssignCpuGroups;
+}
 #endif // HOST_WINDOWS
 
 //******************************************************************************

src/coreclr/src/vm/threads.cpp

@@ -510,10 +510,14 @@ void Thread::ChooseThreadCPUGroupAffinity()
         GC_TRIGGERS;
     }
     CONTRACTL_END;
-#ifndef TARGET_UNIX
-    if (!CPUGroupInfo::CanEnableGCCPUGroups() || !CPUGroupInfo::CanEnableThreadUseAllCpuGroups())
-         return;
 
+#ifndef TARGET_UNIX
+    if (!CPUGroupInfo::CanEnableGCCPUGroups() ||
+        !CPUGroupInfo::CanEnableThreadUseAllCpuGroups() ||
+        !CPUGroupInfo::CanAssignCpuGroupsToThreads())
+    {
+        return;
+    }
 
     //Borrow the ThreadStore Lock here: Lock ThreadStore before distributing threads
     ThreadStoreLockHolder TSLockHolder(TRUE);
@@ -541,10 +545,14 @@ void Thread::ClearThreadCPUGroupAffinity()
         GC_NOTRIGGER;
     }
     CONTRACTL_END;
-#ifndef TARGET_UNIX
-    if (!CPUGroupInfo::CanEnableGCCPUGroups() || !CPUGroupInfo::CanEnableThreadUseAllCpuGroups())
-         return;
 
+#ifndef TARGET_UNIX
+    if (!CPUGroupInfo::CanEnableGCCPUGroups() ||
+        !CPUGroupInfo::CanEnableThreadUseAllCpuGroups() ||
+        !CPUGroupInfo::CanAssignCpuGroupsToThreads())
+    {
+        return;
+    }
 
     ThreadStoreLockHolder TSLockHolder(TRUE);