Large version bubble Support (dotnet/coreclr#21727)

* Preliminary Changes * Module Index Resolution * Change infoModule encoding * Change referencing module in R2R * Pre-condition Check * Virtual Method Module Resolution * Remove Workarounds and add conditional import loading * Add signature kind module override * Add ELEMENT_TYPE_MODULE_ZAPSIG * Add switch to enable large version bubble * Cleanup * Change Native header check * Add large version bubble test * Add Large Version Bubble Checks * Cleanup * Revert unnecessary check * Change EncodeMethod Version Bubble Condition * Add Large Version Bubble asserts * Cleanup * Add default argument to runtests.py * Change test PreCommands * Revert whitespace changes * Change breaking conditional check * Streamline Version Bubble test * Address PR Feedback * Address PR Feedback dotnet/coreclr#2 * Remove dead code * Add crossgen-time ifdef Commit migrated from dotnet/coreclr@9fe3286
picenka21 · Jan 15, 2019 · 8c26340 · 8c26340
1 parent e84f654
commit 8c26340
Show file tree

Hide file tree

Showing 28 changed files with 341 additions and 74 deletions.
diff --git a/src/coreclr/src/inc/corcompile.h b/src/coreclr/src/inc/corcompile.h
@@ -1812,6 +1812,7 @@ extern bool g_fNGenWinMDResilient;
 
 #ifdef FEATURE_READYTORUN_COMPILER
 extern bool g_fReadyToRunCompilation;
+extern bool g_fLargeVersionBubble;
 #endif
 
 inline bool IsReadyToRunCompilation()
@@ -1823,4 +1824,11 @@ inline bool IsReadyToRunCompilation()
 #endif
 }
 
+#ifdef FEATURE_READYTORUN_COMPILER
+inline bool IsLargeVersionBubbleEnabled()
+{
+    return g_fLargeVersionBubble;
+}
+#endif
+
 #endif /* COR_COMPILE_H_ */
diff --git a/src/coreclr/src/inc/coregen.h b/src/coreclr/src/inc/coregen.h
@@ -13,6 +13,7 @@
 #define NGENWORKER_FLAGS_TUNING                  0x0001
 
 #define NGENWORKER_FLAGS_MISSINGDEPENDENCIESOK   0x0004
+#define NGENWORKER_FLAGS_LARGEVERSIONBUBBLE      0x0008
 
 #define NGENWORKER_FLAGS_WINMD_RESILIENT         0x1000
 #define NGENWORKER_FLAGS_READYTORUN              0x2000

diff --git a/src/coreclr/src/inc/pedecoder.h b/src/coreclr/src/inc/pedecoder.h
@@ -49,6 +49,7 @@ typedef DPTR(struct CORCOMPILE_HEADER) PTR_CORCOMPILE_HEADER;
 
 #include "readytorun.h"
 typedef DPTR(struct READYTORUN_HEADER) PTR_READYTORUN_HEADER;
+typedef DPTR(struct READYTORUN_SECTION) PTR_READYTORUN_SECTION;
 
 typedef DPTR(IMAGE_COR20_HEADER)    PTR_IMAGE_COR20_HEADER;
 
@@ -400,7 +401,7 @@ class PEDecoder
     IMAGE_NT_HEADERS *FindNTHeaders() const;
     IMAGE_COR20_HEADER *FindCorHeader() const;
     CORCOMPILE_HEADER *FindNativeHeader() const;
-   READYTORUN_HEADER *FindReadyToRunHeader() const;
+    READYTORUN_HEADER *FindReadyToRunHeader() const;
 
     // Flat mapping utilities
     RVA InternalAddressToRva(SIZE_T address) const;

diff --git a/src/coreclr/src/inc/readytorun.h b/src/coreclr/src/inc/readytorun.h
@@ -60,7 +60,8 @@ enum ReadyToRunSectionType
     READYTORUN_SECTION_AVAILABLE_TYPES              = 108,
     READYTORUN_SECTION_INSTANCE_METHOD_ENTRYPOINTS  = 109,
     READYTORUN_SECTION_INLINING_INFO                = 110, // Added in V2.1
-    READYTORUN_SECTION_PROFILEDATA_INFO             = 111  // Added in V2.2
+    READYTORUN_SECTION_PROFILEDATA_INFO             = 111, // Added in V2.2
+    READYTORUN_SECTION_MANIFEST_METADATA            = 112  // Added in V2.3
 
 	// If you add a new section consider whether it is a breaking or non-breaking change.
 	// Usually it is non-breaking, but if it is preferable to have older runtimes fail

diff --git a/src/coreclr/src/tools/crossgen/crossgen.cpp b/src/coreclr/src/tools/crossgen/crossgen.cpp
@@ -152,6 +152,9 @@ void PrintUsageHelper()
 #ifdef FEATURE_READYTORUN_COMPILER
        W("    /ReadyToRun          - Generate images resilient to the runtime and\n")
        W("                           dependency versions\n")
+       W("    /LargeVersionBubble  - Generate image with a version bubble including all\n")
+       W("                           input assemblies\n")
+
 #endif
 #ifdef FEATURE_WINMD_RESILIENT
        W(" WinMD Parameters\n")
@@ -422,6 +425,7 @@ int _cdecl wmain(int argc, __in_ecount(argc) WCHAR **argv)
     LPWSTR pwzSearchPathForManagedPDB = NULL;
     LPCWSTR pwzOutputFilename = NULL;
     LPCWSTR pwzPublicKeys = nullptr;
+    bool fLargeVersionBubbleSwitch = false;
 
 #if !defined(FEATURE_MERGE_JIT_AND_ENGINE)
     LPCWSTR pwszCLRJITPath = nullptr;
@@ -518,6 +522,11 @@ int _cdecl wmain(int argc, __in_ecount(argc) WCHAR **argv)
         {
             dwFlags &= ~NGENWORKER_FLAGS_READYTORUN;
         }
+        else if (MatchParameter(*argv, W("LargeVersionBubble")))
+        {
+            dwFlags |= NGENWORKER_FLAGS_LARGEVERSIONBUBBLE;
+            fLargeVersionBubbleSwitch = true;
+        }
 #endif
         else if (MatchParameter(*argv, W("NoMetaData")))
         {

diff --git a/src/coreclr/src/utilcode/pedecoder.cpp b/src/coreclr/src/utilcode/pedecoder.cpp
@@ -2481,14 +2481,34 @@ PTR_CVOID PEDecoder::GetNativeManifestMetadata(COUNT_T *pSize) const
     CONTRACT(PTR_CVOID)
     {
         INSTANCE_CHECK;
-        PRECONDITION(CheckNativeHeader());
+        PRECONDITION(HasReadyToRunHeader() || CheckNativeHeader());
         POSTCONDITION(CheckPointer(RETVAL, NULL_OK)); // TBD - may not store metadata for IJW
         NOTHROW;
         GC_NOTRIGGER;
     }
     CONTRACT_END;
+
+    IMAGE_DATA_DIRECTORY *pDir;
+    if (HasReadyToRunHeader())
+    {
+        READYTORUN_HEADER * pHeader = GetReadyToRunHeader();
 
-    IMAGE_DATA_DIRECTORY *pDir = GetMetaDataHelper(METADATA_SECTION_MANIFEST);
+        PTR_READYTORUN_SECTION pSections = dac_cast<PTR_READYTORUN_SECTION>(dac_cast<TADDR>(pHeader) + sizeof(READYTORUN_HEADER));
+        for (DWORD i = 0; i < pHeader->NumberOfSections; i++)
+        {
+            // Verify that section types are sorted
+            _ASSERTE(i == 0 || (pSections[i - 1].Type < pSections[i].Type));
+
+            READYTORUN_SECTION * pSection = pSections + i;
+            if (pSection->Type == READYTORUN_SECTION_MANIFEST_METADATA)
+                // Set pDir to the address of the manifest metadata section
+                pDir = &pSection->Section;
+        }
+    }
+    else
+    {
+        pDir = GetMetaDataHelper(METADATA_SECTION_MANIFEST);
+    }
 
     if (pSize != NULL)
         *pSize = VAL32(pDir->Size);

diff --git a/src/coreclr/src/vm/ceeload.cpp b/src/coreclr/src/vm/ceeload.cpp
@@ -3537,7 +3537,7 @@ BOOL Module::IsInCurrentVersionBubble()
         return TRUE;
 
     if (IsReadyToRunCompilation())
-        return FALSE;
+        return IsLargeVersionBubbleEnabled();
 
 #ifdef FEATURE_COMINTEROP
     if (g_fNGenWinMDResilient)
@@ -9978,7 +9978,7 @@ Module *Module::GetModuleFromIndex(DWORD ix)
     }
     CONTRACT_END;
 
-    if (HasNativeImage())
+    if (HasNativeOrReadyToRunImage())
     {
         RETURN ZapSig::DecodeModuleFromIndex(this, ix);
     }
@@ -10011,7 +10011,7 @@ Module *Module::GetModuleFromIndexIfLoaded(DWORD ix)
         NOTHROW;
         GC_NOTRIGGER;
         MODE_ANY;
-        PRECONDITION(HasNativeImage());
+        PRECONDITION(HasNativeOrReadyToRunImage());
         POSTCONDITION(CheckPointer(RETVAL, NULL_OK));
     }
     CONTRACT_END;
@@ -10051,15 +10051,24 @@ IMDInternalImport *Module::GetNativeAssemblyImport(BOOL loadAllowed)
         if (loadAllowed) THROWS;                         else NOTHROW;
         if (loadAllowed) INJECT_FAULT(COMPlusThrowOM()); else FORBID_FAULT;
         MODE_ANY;
-        PRECONDITION(HasNativeImage());
-        POSTCONDITION(CheckPointer(RETVAL));
+        PRECONDITION(HasNativeOrReadyToRunImage());
+        POSTCONDITION(loadAllowed ?
+                      CheckPointer(RETVAL):
+                      CheckPointer(RETVAL, NULL_OK));
     }
     CONTRACT_END;
 
-    RETURN GetFile()->GetPersistentNativeImage()->GetNativeMDImport(loadAllowed);
+    // Check if image is R2R
+    if (GetFile()->IsILImageReadyToRun())
+    {
+        RETURN GetFile()->GetOpenedILimage()->GetNativeMDImport(loadAllowed);
+    }
+    else
+    {
+        RETURN GetFile()->GetPersistentNativeImage()->GetNativeMDImport(loadAllowed);
+    }
 }
 
-
 /*static*/
 void Module::RestoreMethodTablePointerRaw(MethodTable ** ppMT,
                                           Module *pContainingModule,

diff --git a/src/coreclr/src/vm/ceeload.h b/src/coreclr/src/vm/ceeload.h
@@ -2739,6 +2739,7 @@ class Module
     BYTE *GetNativeFixupBlobData(RVA fixup);
 
     IMDInternalImport *GetNativeAssemblyImport(BOOL loadAllowed = TRUE);
+    IMDInternalImport *GetNativeAssemblyImportIfLoaded();
 
     BOOL FixupNativeEntry(CORCOMPILE_IMPORT_SECTION * pSection, SIZE_T fixupIndex, SIZE_T *fixup);
 

diff --git a/src/coreclr/src/vm/genericdict.cpp b/src/coreclr/src/vm/genericdict.cpp
@@ -686,8 +686,20 @@ Dictionary::PopulateEntry(
             pBlob = p.GetPtr();
         }
 
-        CORCOMPILE_FIXUP_BLOB_KIND signatureKind = (CORCOMPILE_FIXUP_BLOB_KIND)CorSigUncompressData(pBlob);
-        switch (signatureKind)
+        BYTE signatureKind = *pBlob++;
+        if (signatureKind & ENCODE_MODULE_OVERRIDE)
+        {
+            DWORD moduleIndex = CorSigUncompressData(pBlob);
+            Module * pSignatureModule = pModule->GetModuleFromIndex(moduleIndex);
+            if (pInfoModule == pModule)
+            {
+                pInfoModule = pSignatureModule;
+            }
+            _ASSERTE(pInfoModule == pSignatureModule);
+            signatureKind &= ~ENCODE_MODULE_OVERRIDE;
+        }
+
+        switch ((CORCOMPILE_FIXUP_BLOB_KIND) signatureKind)
         {
             case ENCODE_DECLARINGTYPE_HANDLE:   kind = DeclaringTypeHandleSlot; break;
             case ENCODE_TYPE_HANDLE:            kind = TypeHandleSlot; break;

diff --git a/src/coreclr/src/vm/jitinterface.cpp b/src/coreclr/src/vm/jitinterface.cpp
@@ -686,7 +686,7 @@ bool IsInSameVersionBubble(Assembly * current, Assembly * target)
     if (current == target)
         return true;
 
-    return false;
+    return IsLargeVersionBubbleEnabled();
 }
 
 // Returns true if the assemblies defining current and target are in the same version bubble

diff --git a/src/coreclr/src/vm/peimage.cpp b/src/coreclr/src/vm/peimage.cpp
@@ -460,7 +460,7 @@ IMDInternalImport* PEImage::GetNativeMDImport(BOOL loadAllowed)
     CONTRACTL
     {
         INSTANCE_CHECK;
-        PRECONDITION(HasNativeHeader());
+        PRECONDITION(HasNativeHeader() || HasReadyToRunHeader());
         if (loadAllowed) GC_TRIGGERS;                    else GC_NOTRIGGER;
         if (loadAllowed) THROWS;                         else NOTHROW;
         if (loadAllowed) INJECT_FAULT(COMPlusThrowOM()); else FORBID_FAULT;
@@ -485,7 +485,7 @@ void PEImage::OpenNativeMDImport()
     CONTRACTL
     {
         INSTANCE_CHECK;
-        PRECONDITION(HasNativeHeader());
+        PRECONDITION(HasNativeHeader() || HasReadyToRunHeader());
         GC_TRIGGERS;
         THROWS;
         MODE_ANY;

diff --git a/src/coreclr/src/vm/readytoruninfo.cpp b/src/coreclr/src/vm/readytoruninfo.cpp
@@ -607,7 +607,8 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT
                                                     pamTracker, &m_pPersistentInlineTrackingMap);
         }
     }
-    // Fpr format version 2.2 and later, there is an optional profile-data section
+
+    // For format version 2.2 and later, there is an optional profile-data section
     if (IsImageVersionAtLeast(2, 2))
     {
         IMAGE_DATA_DIRECTORY * pProfileDataInfoDir = FindSection(READYTORUN_SECTION_PROFILEDATA_INFO);
@@ -619,6 +620,7 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT
             pModule->SetMethodProfileList(pMethodProfileList);  
         }
     }
+
 }
 
 static bool SigMatchesMethodDesc(MethodDesc* pMD, SigPointer &sig, Module * pModule)

diff --git a/src/coreclr/src/vm/readytoruninfo.h b/src/coreclr/src/vm/readytoruninfo.h
@@ -43,6 +43,7 @@ class ReadyToRunInfo
     NativeFormat::NativeArray       m_methodDefEntryPoints;
     NativeFormat::NativeHashtable   m_instMethodEntryPoints;
     NativeFormat::NativeHashtable   m_availableTypesHashtable;
+    NativeFormat::NativeHashtable   m_pMetaDataHashtable;
 
     Crst                            m_Crst;
     PtrHashMap                      m_entryPointToMethodDescMap;

diff --git a/src/coreclr/src/vm/vars.hpp b/src/coreclr/src/vm/vars.hpp
@@ -812,6 +812,7 @@ extern CEECompileInfo *g_pCEECompileInfo;
 
 #ifdef FEATURE_READYTORUN_COMPILER
 extern bool g_fReadyToRunCompilation;
+extern bool g_fLargeVersionBubble;
 #endif
 
 // Returns true if this is NGen compilation process.

diff --git a/src/coreclr/src/vm/zapsig.cpp b/src/coreclr/src/vm/zapsig.cpp
@@ -760,7 +760,7 @@ Module *ZapSig::DecodeModuleFromIndexIfLoaded(Module *fromModule,
                         fValidAssemblyRef = FALSE;
                     }
                 }
-                
+
                 if (fValidAssemblyRef)
                 {
                     pAssembly = fromModule->GetAssemblyIfLoaded(
@@ -803,11 +803,11 @@ TypeHandle ZapSig::DecodeType(Module *pEncodeModuleContext,
     TypeHandle th = p.GetTypeHandleThrowing(pInfoModule,
                                             &typeContext,
                                             ClassLoader::LoadTypes,
-                                            level,                                            
+                                            level,
                                             level < CLASS_LOADED, // For non-full loads, drop a level when loading generic arguments
                                             NULL,
                                             pZapSigContext);
-                                            
+
     return th;
 }
 
@@ -1127,7 +1127,11 @@ BOOL ZapSig::EncodeMethod(
     TypeHandle ownerType;
 
 #ifdef FEATURE_READYTORUN_COMPILER
-    if (IsReadyToRunCompilation())
+
+    // For methods encoded outside of the version bubble, we use pResolvedToken which describes the metadata token from which the method originated
+    // For tokens inside the version bubble we are not constrained by the contents of pResolvedToken and as such we skip this codepath
+    // FUTURE: This condition should likely be changed or reevaluated once support for smaller version bubbles is implemented.
+    if (IsReadyToRunCompilation() && (!IsLargeVersionBubbleEnabled() || !pMethod->GetModule()->IsInCurrentVersionBubble()))
     {
         if (pResolvedToken == NULL)
         {
@@ -1186,7 +1190,9 @@ BOOL ZapSig::EncodeMethod(
     }
 
 #ifdef FEATURE_READYTORUN_COMPILER
-    if (IsReadyToRunCompilation())
+
+    // FUTURE: This condition should likely be changed or reevaluated once support for smaller version bubbles is implemented.
+    if (IsReadyToRunCompilation() && (!IsLargeVersionBubbleEnabled() || !pMethod->GetModule()->IsInCurrentVersionBubble()))
     {
         if (pConstrainedResolvedToken != NULL)
         {
@@ -1202,7 +1208,6 @@ BOOL ZapSig::EncodeMethod(
             // GetSvcLogger()->Printf(W("ReadyToRun: Method reference outside of current version bubble cannot be encoded\n"));
             ThrowHR(E_FAIL);
         }
-        _ASSERTE(pReferencingModule == GetAppDomain()->ToCompilationDomain()->GetTargetModule());
 
         methodToken = pResolvedToken->token;
 
@@ -1312,6 +1317,14 @@ BOOL ZapSig::EncodeMethod(
         if (fEncodeUsingResolvedTokenSpecStreams && pResolvedToken != NULL && pResolvedToken->pTypeSpec != NULL)
         {
             _ASSERTE(pResolvedToken->cbTypeSpec > 0);
+
+            if (IsReadyToRunCompilation() && pMethod->GetModule()->IsInCurrentVersionBubble() && pInfoModule != (Module *) pResolvedToken->tokenScope)
+            {
+                pSigBuilder->AppendElementType((CorElementType)ELEMENT_TYPE_MODULE_ZAPSIG);
+                DWORD index = (*((EncodeModuleCallback)pfnEncodeModule))(pEncodeModuleContext, (Module *) pResolvedToken->tokenScope);
+                pSigBuilder->AppendData(index);
+            }
+
             pSigBuilder->AppendBlob((PVOID)pResolvedToken->pTypeSpec, pResolvedToken->cbTypeSpec);
         }
         else

diff --git a/src/coreclr/src/zap/zapimage.cpp b/src/coreclr/src/zap/zapimage.cpp
@@ -1082,7 +1082,7 @@ HANDLE ZapImage::GenerateFile(LPCWSTR wszOutputFileName, CORCOMPILE_NGEN_SIGNATU
 
 HANDLE ZapImage::SaveImage(LPCWSTR wszOutputFileName, LPCWSTR wszDllPath, CORCOMPILE_NGEN_SIGNATURE * pNativeImageSig)
 {
-    if (!IsReadyToRunCompilation())
+    if(!IsReadyToRunCompilation() || IsLargeVersionBubbleEnabled())
     {
         OutputManifestMetadata();
     }
@@ -1835,6 +1835,10 @@ void ZapImage::Compile()
         OutputTypesTableForReadyToRun(m_pMDImport);
         OutputInliningTableForReadyToRun();
         OutputProfileDataForReadyToRun();
+        if (IsLargeVersionBubbleEnabled())
+        {
+            OutputManifestMetadataForReadyToRun();
+        }
     }
     else
 #endif

diff --git a/src/coreclr/src/zap/zapimage.h b/src/coreclr/src/zap/zapimage.h
@@ -596,6 +596,7 @@ class ZapImage
     void OutputTypesTableForReadyToRun(IMDInternalImport * pMDImport);
     void OutputInliningTableForReadyToRun();
     void OutputProfileDataForReadyToRun();
+    void OutputManifestMetadataForReadyToRun();
 
     void CopyDebugDirEntry();
     void CopyWin32VersionResource();
@@ -691,6 +692,7 @@ class ZapImage
     }
 
     static void __stdcall TryCompileMethodStub(LPVOID pContext, CORINFO_METHOD_HANDLE hStub, CORJIT_FLAGS jitFlags);
+    static DWORD EncodeModuleHelper(LPVOID compileContext, CORINFO_MODULE_HANDLE referencedModule);
 
     BOOL IsVTableGapMethod(mdMethodDef md);