sfx+oob split, shim project refactoring, P2Ps

Split the build of shared framework and out-of-band projects so that it's possible to just build the shard framework projects, which was requested by the CLR teams. In the next infrastructure rollout, the current "libs.ref+libs.src" subsets should be removed. Generate the targeting pack with an incomplete (without shims) frameworklist as part of the "libs.sfx" subset. Build the shim projects after the shared framework and oob projects and re-generate the targeting pack's frameworklist to include the shims. Refactor the shims so that they aren't grouped anymore by generated & manual. Instead group them by "ref" and "src" same as other libraries in the tree which allows to reference the source project shims and read from the ReferenceAssembly metadata. Use ProjectReferences in traversal projects and shim projects instead of named references that point to binplace directories (see graph below) This allows to build parts of the product dynamically, i.e. the shim projects, apicompat.proj, sfx.proj and oob.proj.
hwoodiwiss · Feb 24, 2022 · 9ebe0ef · 9ebe0ef
1 parent fbf95fd
commit 9ebe0ef
Show file tree

Hide file tree

Showing 113 changed files with 843 additions and 963 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -304,7 +304,6 @@
 
   <PropertyGroup>
     <CustomBeforeNoTargets>$(RepositoryEngineeringDir)NoTargetsSdk.BeforeTargets.targets</CustomBeforeNoTargets>
-    <CustomAfterTraversalProps>$(RepositoryEngineeringDir)TraversalSdk.AfterProps.props</CustomAfterTraversalProps>
     <CustomAfterTraversalTargets>$(RepositoryEngineeringDir)TraversalSdk.AfterTargets.targets</CustomAfterTraversalTargets>
   </PropertyGroup>
 </Project>
diff --git a/docs/coding-guidelines/project-guidelines.md b/docs/coding-guidelines/project-guidelines.md
@@ -6,10 +6,13 @@ In order to work in the dotnet/runtime repo you must first run build.cmd/sh from
 - Restore tools
 - Restore external dependencies
  - CoreCLR - Copy to `bin\runtime\$(BuildTargetFramework)-$(TargetOS)-$(Configuration)-$(TargetArchitecture)`
-- Build targeting pack
- - Build src\libraries\ref.proj which builds all references assembly projects. For reference assembly project information see [ref](#ref)
-- Build product
- - Build src\libraries\src.proj which builds all the source library projects. For source library project information see [src](#src).
+- Build shared framework projects
+ - Build src\libraries\sfx.proj which builds all shared framework projects.
+- Build out of band projects
+ - Build src\libraries\oob.proj which builds all the out-of-band (OOB) projects.
+
+For reference assembly project information see [ref](#ref)
+For source library project information see [src](#src)
 
 # Build Pivots
 Below is a list of all the various options we pivot the project builds on:

diff --git a/eng/NoTargetsSdk.BeforeTargets.targets b/eng/NoTargetsSdk.BeforeTargets.targets
@@ -1,9 +1,8 @@
 <Project>
 
   <PropertyGroup>
-    <!-- NoTargets SDK needs a TFM set. Use the latest .NETCoreApp TFM that is supported by the SDK.
-         Only set a default if the project isn't multi-targeting. -->
-    <TargetFramework Condition="'$(TargetFramework)' == '' and '$(TargetFrameworks)' == ''">$(NetCoreAppToolCurrent)</TargetFramework>
+    <!-- NoTargets SDK needs a TFM set. Set a default if the project doesn't multi target. -->
+    <TargetFramework Condition="'$(TargetFramework)' == '' and '$(TargetFrameworks)' == ''">$(NetCoreAppCurrent)</TargetFramework>
   </PropertyGroup>
 
 </Project>
diff --git a/eng/Subsets.props b/eng/Subsets.props
@@ -71,7 +71,7 @@
     <DefaultLibrariesSubsets Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
                                         '$(BuildTargetFramework)' == '' or
                                         '$(BuildAllConfigurations)' == 'true'">libs.native+</DefaultLibrariesSubsets>
-    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.sfx+libs.oob</DefaultLibrariesSubsets>
     <DefaultLibrariesSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultLibrariesSubsets)+libs.pretest</DefaultLibrariesSubsets>
 
     <DefaultHostSubsets>host.native+host.tools</DefaultHostSubsets>
@@ -144,8 +144,10 @@
     <!-- Libs -->
     <SubsetName Include="Libs" Description="The libraries native part, refs and source assemblies, test infra and packages, but NOT the tests (use Libs.Tests to request those explicitly). Equivalent to: $(DefaultLibrariesSubsets)" />
     <SubsetName Include="Libs.Native" Description="The native libraries used in the shared framework." />
-    <SubsetName Include="Libs.Ref" Description="The managed reference libraries." />
-    <SubsetName Include="Libs.Src" Description="The managed implementation libraries." />
+    <SubsetName Include="Libs.Sfx" Description="The managed shared framework libraries." />
+    <SubsetName Include="Libs.Oob" Description="The managed out-of-band libraries." />
+    <SubsetName Include="Libs.Ref" OnDemand="true" Description="The managed reference libraries." />
+    <SubsetName Include="Libs.Src" OnDemand="true" Description="The managed implementation libraries." />
     <SubsetName Include="Libs.PreTest" Description="Test assets which are necessary to run tests." />
     <SubsetName Include="Libs.Packages" Description="The projects that produce NuGet packages from libraries." />
     <SubsetName Include="Libs.Tests" OnDemand="true" Description="The test projects. Note that building this doesn't execute tests: you must also pass the '-test' argument." />
@@ -322,12 +324,20 @@
     <ProjectToBuild Include="$(SharedNativeRoot)libs\build-native.proj" Category="libs" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+libs.ref+'))">
-    <ProjectToBuild Include="$(LibrariesProjectRoot)ref.proj" Category="libs" />
+  <ItemGroup Condition="$(_subset.Contains('+libs.ref+')) or $(_subset.Contains('+libs.src+')) or $(_subset.Contains('+libs.sfx+'))">
+    <ProjectToBuild Include="$(LibrariesProjectRoot)sfx.proj"
+                    Category="libs"
+                    Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
+                               '$(BuildTargetFramework)' == '' or
+                               '$(BuildAllConfigurations)' == 'true'">
+      <AdditionalProperties Condition="$(_subset.Contains('+libs.ref+'))">%(AdditionalProperties);RefOnly=true</AdditionalProperties>
+    </ProjectToBuild>
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+libs.src+'))">
-    <ProjectToBuild Include="$(LibrariesProjectRoot)src.proj" Category="libs" />
+  <ItemGroup Condition="$(_subset.Contains('+libs.ref+')) or $(_subset.Contains('+libs.src+')) or $(_subset.Contains('+libs.oob+'))">
+    <ProjectToBuild Include="$(LibrariesProjectRoot)oob.proj" Category="libs">
+      <AdditionalProperties Condition="$(_subset.Contains('+libs.ref+'))">%(AdditionalProperties);RefOnly=true</AdditionalProperties>
+    </ProjectToBuild>
   </ItemGroup>
 
   <ItemGroup Condition="$(_subset.Contains('+mono.wasmruntime+'))">

diff --git a/eng/TraversalSdk.AfterProps.props b/eng/TraversalSdk.AfterProps.props
diff --git a/eng/TraversalSdk.AfterTargets.targets b/eng/TraversalSdk.AfterTargets.targets
@@ -9,10 +9,17 @@
     <VSTestDependsOn>$(VSTestDependsOn);ResolveP2PReferences</VSTestDependsOn>
     <PackDependsOn>$(PackDependsOn);ResolveP2PReferences</PackDependsOn>
     <PublishDependsOn>$(PublishDependsOn);ResolveP2PReferences</PublishDependsOn>
+    <GetTargetPathDependsOn>ResolveP2PReferences</GetTargetPathDependsOn>
+    <!-- Filter out ProjectReferences which aren't compatible with the project's TargetFramework. -->
+    <OmitIncompatibleProjectReferences>true</OmitIncompatibleProjectReferences>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(FilterTraversalProjectReferences)' == 'true'">
-    <ProjectReference Update="@(ProjectReference)" SkipGetTargetFrameworkProperties="false" />
+    <!-- Override the Traversal SDK setting as filtering relies on the TargetFrameworkProperties being fetched
+         and don't flow the BuildTargetFramework property down. -->
+    <ProjectReference Update="@(ProjectReference)"
+                      SkipGetTargetFrameworkProperties="false"
+                      UndefineProperties="%(UndefineProperties);BuildTargetFramework"  />
   </ItemGroup>
 
-</Project>
+</Project>
diff --git a/eng/generators.targets b/eng/generators.targets
@@ -13,8 +13,7 @@
          - references System.Runtime.InteropServices -->
     <EnabledGenerators Include="DllImportGenerator"
                        Condition="'$(EnableDllImportGenerator)' == ''
-                        and '$(IsFrameworkSupportFacade)' != 'true'
-                        and '$(IsSourceProject)' == 'true'
+                        and '$(IsRuntimeAssembly)' == 'true'
                         and '$(MSBuildProjectExtension)' == '.csproj'
                         and (
                           ('@(Reference)' != ''
@@ -25,8 +24,7 @@
                             and '$(DisableImplicitAssemblyReferences)' == 'false'))" />
     <EnabledGenerators Include="DllImportGenerator"
                        Condition="'$(EnableDllImportGenerator)' == ''
-                        and '$(IsFrameworkSupportFacade)' != 'true'
-                        and '$(IsSourceProject)' == 'true'
+                        and '$(IsRuntimeAssembly)' == 'true'
                         and '$(MSBuildProjectExtension)' == '.csproj'
                         and ('$(TargetFrameworkIdentifier)' == '.NETStandard' or '$(TargetFrameworkIdentifier)' == '.NETFramework' or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' and $([MSBuild]::VersionLessThan($(TargetFrameworkVersion), '$(NetCoreAppCurrentVersion)'))))" />
   </ItemGroup>

diff --git a/eng/illink.targets b/eng/illink.targets
@@ -198,14 +198,22 @@
     </ItemGroup>
   </Target>
 
-  <!-- ILLink.Tasks arguments common to runs for both individual libraries and for the entire runtime pack -->
-  <Target Name="SetCommonILLinkArgs">
+  <Target Name="PrepareForAssembliesTrim">
+    <!-- ILLink.Tasks arguments common to runs for both individual libraries and for the entire runtime pack -->
     <PropertyGroup>
       <!-- don't remove attributes after build, our tooling is not ready for that -->
       <ILLinkArgs>$(ILLinkArgs) --ignore-link-attributes true</ILLinkArgs>
       <!-- ignore unresolved references -->
       <ILLinkArgs>$(ILLinkArgs) --skip-unresolved true</ILLinkArgs>
     </PropertyGroup>
+
+    <!-- When running from Desktop MSBuild, DOTNET_HOST_PATH is not set.
+         In this case, explicitly specify the path to the dotnet host. -->
+    <PropertyGroup Condition="'$(DOTNET_HOST_PATH)' == ''">
+      <!-- This is defined when building in Visual Studio. -->
+      <_DotNetHostDirectory>$(NetCoreRoot)</_DotNetHostDirectory>
+      <_DotNetHostFileName>$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetHostFileName>
+    </PropertyGroup>
   </Target>
 
   <!-- ILLinkTrimAssembly
@@ -215,7 +223,7 @@
   <UsingTask TaskName="ILLink" AssemblyFile="$(ILLinkTasksAssembly)" Condition="'$(ILLinkTasksAssembly)' != ''" />
   <Target Name="ILLinkTrimAssembly"
           Condition="'$(ILLinkTrimAssembly)' == 'true'"
-          DependsOnTargets="SetCommonILLinkArgs">
+          DependsOnTargets="PrepareForAssembliesTrim">
     <PropertyGroup>
       <!-- default action for assemblies with IsTrimmable attribute -->
       <ILLinkArgs>$(ILLinkArgs) --trim-mode skip</ILLinkArgs>
@@ -285,21 +293,12 @@
       <ILLinkArgs Condition="@(_DependencyDirectories->Count()) > 0">$(ILLinkArgs) -d @(_DependencyDirectories->'"%(Identity)"', ' -d ')</ILLinkArgs>
     </PropertyGroup>
 
-    <!-- When running from Desktop MSBuild, DOTNET_HOST_PATH is not set.
-         In this case, explicitly specify the path to the dotnet host. -->
-    <PropertyGroup Condition=" '$(DOTNET_HOST_PATH)' == '' ">
-      <!-- This is defined when building in Visual Studio. -->
-      <_DotNetHostDirectory>$(NetCoreRoot)</_DotNetHostDirectory>
-      <_DotNetHostFileName>$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetHostFileName>
-    </PropertyGroup>
-
     <ILLink AssemblyPaths=""
             RootAssemblyNames="@(ILLinkTrimInputAssembly)"
             OutputDirectory="$(ILLinkTrimOutputPath)"
             ExtraArgs="$(ILLinkArgs)"
             ToolExe="$(_DotNetHostFileName)"
             ToolPath="$(_DotNetHostDirectory)" />
-
   </Target>
 
   <!-- ILLink reporting.

diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml
@@ -74,7 +74,7 @@ jobs:
       - ${{ each variable in parameters.variables }}:
         - ${{ variable }}
       - name: liveRuntimeBuildParams
-        value: 'libs.ref -c Release -ci'
+        value: 'libs.sfx+libs.oob /p:RefOnly=true -c Release -ci'
       - name: compilerArg
         value: ''
       - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}:

diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml
@@ -57,7 +57,7 @@ stages:
   - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
     - template: /eng/common/templates/job/source-index-stage1.yml
       parameters:
-        sourceIndexBuildCommand: build.cmd -subset libs.ref+libs.src -binarylog -os Linux -ci
+        sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os Linux -ci
 
   #
   # Build CoreCLR

diff --git a/eng/pipelines/runtime-richnav.yml b/eng/pipelines/runtime-richnav.yml
@@ -37,7 +37,7 @@ jobs:
       richCodeNavigationEnvironment: "production"
       richCodeNavigationLanguage: "csharp"
       timeoutInMinutes: 240
-      buildArgs: -s libs.ref+libs.src -c debug -allConfigurations
+      buildArgs: -s libs.sfx+libs.oob -allConfigurations
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/referenceAssemblies.props b/eng/referenceAssemblies.props
@@ -1,14 +1,9 @@
 <Project>
-  <PropertyGroup Condition="'$(BuildAllProjects)' == 'true' and
-                            !$(BuildTargetFramework.StartsWith('netstandard')) and
-                            !$(BuildTargetFramework.StartsWith('net4'))">
-    <AdditionalBuildTargetFrameworks>$(AdditionalBuildTargetFrameworks);netstandard2.0;netstandard2.1</AdditionalBuildTargetFrameworks>
 
+  <PropertyGroup>
     <!-- Reference assemblies are special and don't initialize fields or have empty finalizers, etc. -->
     <RunAnalyzers>false</RunAnalyzers>
-  </PropertyGroup>
 
-  <PropertyGroup>
     <!-- disable warnings about unused fields -->
     <NoWarn>$(NoWarn);CS0169;CS0649;CS8618</NoWarn>
 
@@ -30,4 +25,5 @@
       <_Parameter1_IsLiteral>true</_Parameter1_IsLiteral>
     </AssemblyAttribute>
   </ItemGroup>
+
 </Project>
diff --git a/eng/references.targets b/eng/references.targets
@@ -16,19 +16,17 @@
 
   <ItemGroup Condition="'@(ProjectReference)' != ''">
     <_coreLibProjectReference Include="@(ProjectReference->WithMetadataValue('Identity', '$(CoreLibProject)'))" />
-    <ProjectReference Update="@(_coreLibProjectReference)">
-      <!-- Don't flow TargetFramework and Platform to use same inputs and outputs as the CoreLib's build as part of the runtime. -->
-      <UndefineProperties>$(UndefineProperties);TargetFramework;Platform</UndefineProperties>
+    <ProjectReference Update="@(_coreLibProjectReference)"
+                      Private="false">
       <SetConfiguration Condition="'$(RuntimeFlavor)' == 'CoreCLR' and
                                    '$(Configuration)' != '$(CoreCLRConfiguration)'">Configuration=$(CoreCLRConfiguration)</SetConfiguration>
       <SetConfiguration Condition="'$(RuntimeFlavor)' == 'Mono' and
                                    '$(Configuration)' != '$(MonoConfiguration)'">Configuration=$(MonoConfiguration)</SetConfiguration>
-      <Private>false</Private>
     </ProjectReference>
     <!-- If a CoreLib ProjectReference is present, make all P2P assets non transitive. -->
-    <ProjectReference Update="@(ProjectReference)"
+    <ProjectReference Update="@(ProjectReference->WithMetadataValue('PrivateAssets', ''))"
                       PrivateAssets="all"
-                      Condition="'@(_coreLibProjectReference)' != ''" />
+                      Condition="'$(IsSourceProject)' == 'true' and '@(_coreLibProjectReference)' != ''" />
   </ItemGroup>
 
   <!-- Disable TargetArchitectureMismatch warning when we reference CoreLib as it is platform specific. -->
@@ -78,8 +76,19 @@
 
   <Target Name="ValidateReferenceAssemblyProjectReferences"
           AfterTargets="ResolveReferences"
-          Condition="'$(IsReferenceAssembly)' == 'true'">
+          Condition="'$(IsReferenceAssembly)' == 'true' and
+                     '$(SkipValidateReferenceAssemblyProjectReferences)' != 'true'">
     <Error Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference' and '%(ReferencePath.IsReferenceAssembly)' != 'true' and '%(ReferencePath.ReferenceAssembly)' == ''"
            Text="Reference assemblies must only reference other reference assemblies and '%(ReferencePath.ProjectReferenceOriginalItemSpec)' is not a reference assembly project and does not set 'ProduceReferenceAssembly'." />
   </Target>
+
+  <!-- An opt-in target to trim out private assemblies from the ref assembly ReferencePath. -->
+  <Target Name="TrimOutPrivateAssembliesFromReferencePath"
+          Condition="'$(CompileUsingReferenceAssemblies)' == 'true' and '$(TrimOutPrivateAssembliesFromReferencePath)' == 'true'"
+          AfterTargets="FindReferenceAssembliesForReferences">
+    <ItemGroup>
+      <ReferencePathWithRefAssemblies Remove="@(ReferencePathWithRefAssemblies)"
+                                      Condition="$(NetCoreAppLibraryNoReference.Contains('%(Filename);'))" />
+    </ItemGroup>
+  </Target>
 </Project>
diff --git a/eng/resolveContract.targets b/eng/resolveContract.targets
@@ -28,7 +28,10 @@
           DependsOnTargets="ResolveProjectReferences"
           AfterTargets="GetTargetPathWithTargetPlatformMoniker">
     <ItemGroup>
-      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContract)" />
+      <!-- Allow to point to a different reference project than what GenFacades uses to generate the type forwards. -->
+      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContractOverride)" />
+      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContract)"
+                                           Condition="'@(ResolvedMatchingContractOverride)' == ''" />
     </ItemGroup>
   </Target>
 
@@ -38,7 +41,7 @@
           Condition="'@(ProjectReference)' != '' and '@(_ResolvedProjectReferencePaths)' != ''">
     <!-- If we have a ProjectReference to CoreLib, we need to compile against implementation assemblies. -->
     <PropertyGroup Condition="@(_ResolvedProjectReferencePaths->AnyHaveMetadataValue('MSBuildSourceProjectFile', '$(CoreLibProject)'))">
-      <CompileUsingReferenceAssemblies>false</CompileUsingReferenceAssemblies>
+      <CompileUsingReferenceAssemblies Condition="'$(CompileUsingReferenceAssemblies)' == ''">false</CompileUsingReferenceAssemblies>
     </PropertyGroup>
     <!-- Clear the ReferenceAssembly attribute on resolved P2Ps that set SkipUseReferenceAssembly to true. -->
     <ItemGroup>

diff --git a/eng/targetingpacks.targets b/eng/targetingpacks.targets
@@ -146,6 +146,13 @@
       <_targetingPackReferenceExclusion Include="@(DefaultReferenceExclusion)" />
     </ItemGroup>
 
+    <!-- Filter out shims from the targeting pack references as an opt-in. -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and
+                          '$(SkipTargetingPackShimReferences)' == 'true'">
+      <_targetingPackReferenceExclusion Include="@(NetFxReference)" />
+      <_targetingPackReferenceExclusion Include="netstandard" />
+    </ItemGroup>
+
     <ItemGroup>
       <_targetingPackReferenceWithProjectName Include="@(Reference->WithMetadataValue('ExternallyResolved', 'true')->Metadata('Filename'))"
                                               OriginalIdentity="%(Identity)" />

diff --git a/eng/versioning.targets b/eng/versioning.targets
@@ -107,7 +107,10 @@
   <Target Name="AddOSPlatformAttributes"
           BeforeTargets="GenerateAssemblyInfo"
           AfterTargets="PrepareForBuild"
-          Condition="'$(TargetPlatformIdentifier)' == '' and '$(IsTestProject)' != 'true' and '$(TargetFrameworkIdentifier)' != '.NETFramework'">
+          Condition="'$(TargetPlatformIdentifier)' == '' and
+                     '$(IsTestProject)' != 'true' and
+                     '$(TargetFrameworkIdentifier)' != '.NETFramework'
+                     and '$(AddOSPlatformAttributes)' != 'false'">
     <!-- Defensively de-dupe the values -->
     <ItemGroup>
       <_unsupportedOSPlatforms Include="$(UnsupportedOSPlatforms)" />
@@ -133,12 +136,15 @@
     </ItemGroup>
   </Target>
 
-  <!-- Removes assembly level attributes from test projects. -->
-  <Target Name="RemoveSupportedOSPlatformAttributeFromTestProjects"
+  <!-- Remove assembly level attributes from certain projects.
+       Use a target for that until https://github.com/dotnet/sdk/issues/14836 is implemented. -->
+  <Target Name="RemoveSupportedOSTargetPlatformAttributeFromProjects"
           AfterTargets="GetAssemblyAttributes"
-          Condition="'$(IsTestProject)' == 'true'">
+          BeforeTargets="CreateGeneratedAssemblyInfoInputsCacheFile">
     <ItemGroup>
-      <AssemblyAttribute Remove="System.Runtime.Versioning.SupportedOSPlatformAttribute" />
+      <AssemblyAttribute Remove="System.Runtime.Versioning.SupportedOSPlatformAttribute"
+                         Condition="'$(IsTestProject)' == 'true' or '$(AddOSPlatformAttributes)' == 'false'" />
+      <!-- Don't include target platform attributes, since we use the target platform to represent RIDs instead. -->
       <AssemblyAttribute Remove="System.Runtime.Versioning.TargetPlatformAttribute" />
     </ItemGroup>
   </Target>