Move tailcall dispatcher into corelib (dotnet#38938)

Fix dotnet#35559

src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs

@@ -292,6 +292,40 @@ public static IntPtr AllocateTypeAssociatedMemory(Type type, int size)
         [MethodImpl(MethodImplOptions.InternalCall)]
         private static unsafe extern TailCallTls* GetTailCallInfo(IntPtr retAddrSlot, IntPtr* retAddr);
 
+        private static unsafe void DispatchTailCalls(
+            IntPtr callersRetAddrSlot,
+            delegate*<IntPtr, IntPtr, IntPtr*, void> callTarget,
+            IntPtr retVal)
+        {
+            IntPtr callersRetAddr;
+            TailCallTls* tls = GetTailCallInfo(callersRetAddrSlot, &callersRetAddr);
+            PortableTailCallFrame* prevFrame = tls->Frame;
+            if (callersRetAddr == prevFrame->TailCallAwareReturnAddress)
+            {
+                prevFrame->NextCall = callTarget;
+                return;
+            }
+
+            PortableTailCallFrame newFrame;
+            newFrame.Prev = prevFrame;
+
+            try
+            {
+                tls->Frame = &newFrame;
+
+                do
+                {
+                    newFrame.NextCall = null;
+                    callTarget(tls->ArgBuffer, retVal, &newFrame.TailCallAwareReturnAddress);
+                    callTarget = newFrame.NextCall;
+                } while (callTarget != null);
+            }
+            finally
+            {
+                tls->Frame = prevFrame;
+            }
+        }
+
         [MethodImpl(MethodImplOptions.InternalCall)]
         internal static extern long GetILBytesJitted();
 
@@ -439,7 +473,7 @@ internal unsafe struct PortableTailCallFrame
     {
         public PortableTailCallFrame* Prev;
         public IntPtr TailCallAwareReturnAddress;
-        public IntPtr NextCall;
+        public delegate*<IntPtr, IntPtr, IntPtr*, void> NextCall;
     }
 
     [StructLayout(LayoutKind.Sequential)]

src/coreclr/src/vm/dllimport.h

@@ -191,7 +191,6 @@ enum ILStubTypes
     ILSTUB_WRAPPERDELEGATE_INVOKE        = 0x80000007,
     ILSTUB_TAILCALL_STOREARGS            = 0x80000008,
     ILSTUB_TAILCALL_CALLTARGET           = 0x80000009,
-    ILSTUB_TAILCALL_DISPATCH             = 0x8000000A,
 };
 
 #ifdef FEATURE_COMINTEROP
@@ -231,7 +230,6 @@ inline bool SF_IsInstantiatingStub      (DWORD dwStubFlags) { LIMITED_METHOD_CON
 #endif
 inline bool SF_IsTailCallStoreArgsStub  (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_STOREARGS); }
 inline bool SF_IsTailCallCallTargetStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_CALLTARGET); }
-inline bool SF_IsTailCallDispatcherStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_DISPATCH); }
 
 inline bool SF_IsCOMStub               (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COM)); }
 inline bool SF_IsCOMLateBoundStub      (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COMLATEBOUND)); }
@@ -243,8 +241,7 @@ inline bool SF_IsSharedStub(DWORD dwStubFlags)
 {
     WRAPPER_NO_CONTRACT;
 
-    if (SF_IsTailCallStoreArgsStub(dwStubFlags) || SF_IsTailCallCallTargetStub(dwStubFlags) ||
-        SF_IsTailCallDispatcherStub(dwStubFlags))
+    if (SF_IsTailCallStoreArgsStub(dwStubFlags) || SF_IsTailCallCallTargetStub(dwStubFlags))
     {
         return false;
     }

src/coreclr/src/vm/ilstubcache.cpp

@@ -256,11 +256,6 @@ MethodDesc* ILStubCache::CreateNewMethodDesc(LoaderHeap* pCreationHeap, MethodTa
         pMD->GetILStubResolver()->SetStubType(ILStubResolver::TailCallCallTargetStub);
     }
     else
-    if (SF_IsTailCallDispatcherStub(dwStubFlags))
-    {
-        pMD->GetILStubResolver()->SetStubType(ILStubResolver::TailCallDispatcherStub);
-    }
-    else
 #ifdef FEATURE_COMINTEROP
     if (SF_IsCOMStub(dwStubFlags))
     {

src/coreclr/src/vm/ilstubresolver.cpp

@@ -88,7 +88,6 @@ LPCUTF8 ILStubResolver::GetStubMethodName()
         case WrapperDelegateStub:    return "IL_STUB_WrapperDelegate_Invoke";
         case TailCallStoreArgsStub:  return "IL_STUB_StoreTailCallArgs";
         case TailCallCallTargetStub: return "IL_STUB_CallTailCallTarget";
-        case TailCallDispatcherStub: return "IL_STUB_DispatchTailCalls";
         default:
             UNREACHABLE_MSG("Unknown stub type");
     }

src/coreclr/src/vm/ilstubresolver.h

@@ -93,7 +93,6 @@ class ILStubResolver : DynamicResolver
 #endif
         TailCallStoreArgsStub,
         TailCallCallTargetStub,
-        TailCallDispatcherStub,
     };
 
     ILStubType GetStubType();

src/coreclr/src/vm/jitinterface.cpp

@@ -13939,7 +13939,7 @@ bool CEEInfo::getTailCallHelpersInternal(CORINFO_RESOLVED_TOKEN* callToken,
     pResult->flags = (CORINFO_TAILCALL_HELPERS_FLAGS)outFlags;
     pResult->hStoreArgs = (CORINFO_METHOD_HANDLE)pStoreArgsMD;
     pResult->hCallTarget = (CORINFO_METHOD_HANDLE)pCallTargetMD;
-    pResult->hDispatcher = (CORINFO_METHOD_HANDLE)TailCallHelp::GetOrCreateTailCallDispatcherMD();
+    pResult->hDispatcher = (CORINFO_METHOD_HANDLE)TailCallHelp::GetOrLoadTailCallDispatcherMD();
     return true;
 }
 

src/coreclr/src/vm/mscorlib.h

@@ -706,9 +706,10 @@ DEFINE_METHOD(RUNTIME_HELPERS,      GET_RAW_ARRAY_DATA,      GetRawArrayData, No
 DEFINE_METHOD(RUNTIME_HELPERS,      GET_UNINITIALIZED_OBJECT, GetUninitializedObject, NoSig)
 DEFINE_METHOD(RUNTIME_HELPERS,      ENUM_EQUALS,            EnumEquals, NoSig)
 DEFINE_METHOD(RUNTIME_HELPERS,      ENUM_COMPARE_TO,        EnumCompareTo, NoSig)
-DEFINE_METHOD(RUNTIME_HELPERS,      ALLOC_TAILCALL_ARG_BUFFER, AllocTailCallArgBuffer, SM_Int_IntPtr_RetIntPtr)
+DEFINE_METHOD(RUNTIME_HELPERS,      ALLOC_TAILCALL_ARG_BUFFER, AllocTailCallArgBuffer,  SM_Int_IntPtr_RetIntPtr)
 DEFINE_METHOD(RUNTIME_HELPERS,      GET_TAILCALL_INFO,      GetTailCallInfo, NoSig)
-DEFINE_METHOD(RUNTIME_HELPERS,      FREE_TAILCALL_ARG_BUFFER, FreeTailCallArgBuffer, SM_RetVoid)
+DEFINE_METHOD(RUNTIME_HELPERS,      FREE_TAILCALL_ARG_BUFFER, FreeTailCallArgBuffer,    SM_RetVoid)
+DEFINE_METHOD(RUNTIME_HELPERS,      DISPATCH_TAILCALLS,     DispatchTailCalls,          NoSig)
 
 DEFINE_CLASS(UNSAFE,                InternalCompilerServices,       Unsafe)
 DEFINE_METHOD(UNSAFE,               AS_POINTER,             AsPointer, NoSig)

src/coreclr/src/vm/tailcallhelp.cpp

@@ -108,226 +108,29 @@ struct TailCallInfo
 };
 
 static MethodDesc* s_tailCallDispatcherMD;
-MethodDesc* TailCallHelp::GetTailCallDispatcherMD()
+MethodDesc* TailCallHelp::GetOrLoadTailCallDispatcherMD()
 {
-    LIMITED_METHOD_CONTRACT;
-
+    CONTRACTL
+    {
+        THROWS;
+        GC_TRIGGERS;
+        INJECT_FAULT(ThrowOutOfMemory());
+    }
+    CONTRACTL_END;
+
+    if (s_tailCallDispatcherMD == NULL)
+        s_tailCallDispatcherMD = MscorlibBinder::GetMethod(METHOD__RUNTIME_HELPERS__DISPATCH_TAILCALLS);
+
     return s_tailCallDispatcherMD;
 }
 
-
-// This creates the dispatcher used to dispatch sequences of tailcalls. In C#
-// code it is the following function. Once C# gets function pointer support this
-// function can be put in System.Private.CoreLib.
-// private static unsafe void DispatchTailCalls(
-//     IntPtr callersRetAddrSlot, IntPtr callTarget, IntPtr retVal)
-// {
-//     IntPtr callersRetAddr;
-//     TailCallTls* tls = GetTailCallInfo(callersRetAddrSlot, &callersRetAddr);
-//     PortableTailCallFrame* prevFrame = tls->Frame;
-//     if (callersRetAddr == prevFrame->TailCallAwareReturnAddress)
-//     {
-//         prevFrame->NextCall = callTarget;
-//         return;
-//     }
-//
-//     PortableTailCallFrame newFrame;
-//     newFrame.Prev = prevFrame;
-//
-//     try
-//     {
-//         tls->Frame = &newFrame;
-//
-//         do
-//         {
-//             newFrame.NextCall = IntPtr.Zero;
-//             var fptr = (func* void(IntPtr, IntPtr, void*))callTarget;
-//             fptr(tls->ArgBuffer, retVal, &newFrame.TailCallAwareReturnAddress);
-//             callTarget = newFrame.NextCall;
-//         } while (callTarget != IntPtr.Zero);
-//     }
-//     finally
-//     {
-//         tls->Frame = prevFrame;
-//     }
-// }
-MethodDesc* TailCallHelp::GetOrCreateTailCallDispatcherMD()
+MethodDesc* TailCallHelp::GetTailCallDispatcherMD()
 {
-    STANDARD_VM_CONTRACT;
-
-    if (s_tailCallDispatcherMD != NULL)
-        return s_tailCallDispatcherMD;
-
-    SigBuilder sigBuilder;
-    sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT);
-
-    sigBuilder.AppendData(3);
-    sigBuilder.AppendElementType(ELEMENT_TYPE_VOID);
-
-    sigBuilder.AppendElementType(ELEMENT_TYPE_I);
-    sigBuilder.AppendElementType(ELEMENT_TYPE_I);
-    sigBuilder.AppendElementType(ELEMENT_TYPE_I);
-
-    const int ARG_CALLERS_RET_ADDR_SLOT = 0;
-    const int ARG_CALL_TARGET = 1;
-    const int ARG_RET_VAL = 2;
-
-    DWORD cbSig;
-    PCCOR_SIGNATURE pSig = AllocateSignature(
-        MscorlibBinder::GetModule()->GetLoaderAllocator(), sigBuilder, &cbSig);
-
-    SigTypeContext emptyCtx;
-
-    ILStubLinker sl(MscorlibBinder::GetModule(),
-                    Signature(pSig, cbSig),
-                    &emptyCtx,
-                    NULL,
-                    FALSE,
-                    FALSE);
-
-    ILCodeStream* pCode = sl.NewCodeStream(ILStubLinker::kDispatch);
-
-    DWORD retAddrLcl = pCode->NewLocal(ELEMENT_TYPE_I);
-    DWORD tlsLcl = pCode->NewLocal(ELEMENT_TYPE_I);
-    DWORD prevFrameLcl = pCode->NewLocal(ELEMENT_TYPE_I);
-    TypeHandle frameTyHnd = MscorlibBinder::GetClass(CLASS__PORTABLE_TAIL_CALL_FRAME);
-    DWORD newFrameEntryLcl = pCode->NewLocal(LocalDesc(frameTyHnd));
-    DWORD argsLcl = pCode->NewLocal(ELEMENT_TYPE_I);
-    ILCodeLabel* noUnwindLbl = pCode->NewCodeLabel();
-    ILCodeLabel* loopStart = pCode->NewCodeLabel();
-    ILCodeLabel* afterTryFinally = pCode->NewCodeLabel();
-
-    // tls = RuntimeHelpers.GetTailcallInfo(callersRetAddrSlot, &retAddr);
-    pCode->EmitLDARG(ARG_CALLERS_RET_ADDR_SLOT);
-    pCode->EmitLDLOCA(retAddrLcl);
-    pCode->EmitCALL(METHOD__RUNTIME_HELPERS__GET_TAILCALL_INFO, 2, 1);
-    pCode->EmitSTLOC(tlsLcl);
-
-    // prevFrame = tls.Frame;
-    pCode->EmitLDLOC(tlsLcl);
-    pCode->EmitLDFLD(FIELD__TAIL_CALL_TLS__FRAME);
-    pCode->EmitSTLOC(prevFrameLcl);
-
-    // if (retAddr != prevFrame.TailCallAwareReturnAddress) goto noUnwindLbl;
-    pCode->EmitLDLOC(retAddrLcl);
-    pCode->EmitLDLOC(prevFrameLcl);
-    pCode->EmitLDFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__TAILCALL_AWARE_RETURN_ADDRESS);
-    pCode->EmitBNE_UN(noUnwindLbl);
-
-    // prevFrame->NextCall = callTarget;
-    pCode->EmitLDLOC(prevFrameLcl);
-    pCode->EmitLDARG(ARG_CALL_TARGET);
-    pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL);
-
-    // return;
-    pCode->EmitRET();
-
-    // Ok, we are the "first" dispatcher.
-    pCode->EmitLabel(noUnwindLbl);
-
-    // newFrameEntry.Prev = prevFrame;
-    pCode->EmitLDLOCA(newFrameEntryLcl);
-    pCode->EmitLDLOC(prevFrameLcl);
-    pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__PREV);
-
-    // try {
-    pCode->BeginTryBlock();
-
-    // tls->Frame = &newFrameEntry;
-    pCode->EmitLDLOC(tlsLcl);
-    pCode->EmitLDLOCA(newFrameEntryLcl);
-    pCode->EmitSTFLD(FIELD__TAIL_CALL_TLS__FRAME);
-
-    // do {
-    pCode->EmitLabel(loopStart);
-
-    // newFrameEntry.NextCall = 0
-    pCode->EmitLDLOCA(newFrameEntryLcl);
-    pCode->EmitLDC(0);
-    pCode->EmitCONV_I();
-    pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL);
-
-    SigBuilder calliSig;
-    calliSig.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT);
-    calliSig.AppendData(3);
-    calliSig.AppendElementType(ELEMENT_TYPE_VOID);
-    calliSig.AppendElementType(ELEMENT_TYPE_I);
-    calliSig.AppendElementType(ELEMENT_TYPE_I);
-    calliSig.AppendElementType(ELEMENT_TYPE_I);
-
-    DWORD cbCalliSig;
-    PCCOR_SIGNATURE pCalliSig = (PCCOR_SIGNATURE)calliSig.GetSignature(&cbCalliSig);
-
-    // callTarget(tls->ArgBuffer, retVal, &newFrameEntry.TailCallAwareReturnAddress)
-    // arg buffer
-    pCode->EmitLDLOC(tlsLcl);
-    pCode->EmitLDFLD(FIELD__TAIL_CALL_TLS__ARG_BUFFER);
-
-    // ret val
-    pCode->EmitLDARG(ARG_RET_VAL);
-
-    // TailCallAwareReturnAddress
-    pCode->EmitLDLOCA(newFrameEntryLcl);
-    pCode->EmitLDFLDA(FIELD__PORTABLE_TAIL_CALL_FRAME__TAILCALL_AWARE_RETURN_ADDRESS);
-
-    // callTarget
-    pCode->EmitLDARG(ARG_CALL_TARGET);
-
-    pCode->EmitCALLI(pCode->GetSigToken(pCalliSig, cbCalliSig), 2, 0);
-
-    // callTarget = newFrameEntry.NextCall;
-    pCode->EmitLDLOC(newFrameEntryLcl);
-    pCode->EmitLDFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL);
-    pCode->EmitSTARG(ARG_CALL_TARGET);
-
-    // } while (callTarget != IntPtr.Zero);
-    pCode->EmitLDARG(ARG_CALL_TARGET);
-    pCode->EmitBRTRUE(loopStart);
-
-    // }
-    pCode->EmitLEAVE(afterTryFinally);
-    pCode->EndTryBlock();
-
-    // finally {
-    pCode->BeginFinallyBlock();
-
-    // tls->Frame = prevFrame;
-    pCode->EmitLDLOC(tlsLcl);
-    pCode->EmitLDLOC(prevFrameLcl);
-    pCode->EmitSTFLD(FIELD__TAIL_CALL_TLS__FRAME);
-
-    // }
-    pCode->EmitENDFINALLY();
-    pCode->EndFinallyBlock();
-
-    // afterTryFinally:
-    pCode->EmitLabel(afterTryFinally);
-
-    // return;
-    pCode->EmitRET();
-
-    Module* mscorlib = MscorlibBinder::GetModule();
-    MethodDesc* pDispatchTailCallsMD =
-        ILStubCache::CreateAndLinkNewILStubMethodDesc(
-            MscorlibBinder::GetModule()->GetLoaderAllocator(),
-            mscorlib->GetILStubCache()->GetOrCreateStubMethodTable(mscorlib),
-            ILSTUB_TAILCALL_DISPATCH,
-            mscorlib,
-            pSig, cbSig,
-            &emptyCtx,
-            &sl);
-
-#ifdef _DEBUG
-    LOG((LF_STUBS, LL_INFO1000, "TAILCALLHELP: DispatchTailCalls IL created\n"));
-    sl.LogILStub(CORJIT_FLAGS());
-#endif
-
-    // We might waste a MethodDesc here if we lose the race, but that is very
-    // unlikely and since this initialization only happens once not a big deal.
-    InterlockedCompareExchangeT(&s_tailCallDispatcherMD, pDispatchTailCallsMD, NULL);
+    LIMITED_METHOD_CONTRACT;
     return s_tailCallDispatcherMD;
 }
 
+
 void TailCallHelp::CreateTailCallHelperStubs(
     MethodDesc* pCallerMD, MethodDesc* pCalleeMD,
     MetaSig& callSiteSig, bool virt, bool thisArgByRef,

src/coreclr/src/vm/tailcallhelp.h

@@ -23,7 +23,7 @@ class TailCallHelp
         MethodDesc** storeArgsStub, bool* storeArgsNeedsTarget,
         MethodDesc** callTargetStub);
 
-    static MethodDesc* GetOrCreateTailCallDispatcherMD();
+    static MethodDesc* GetOrLoadTailCallDispatcherMD();
     static MethodDesc* GetTailCallDispatcherMD();
 private: