Split unix test builds in slices (dotnet#17785)

* Split unix test builds in slices Ports dotnet#17161 to linux * Address review feedback
EgorBo · May 4, 2018 · ad0f22c · ad0f22c
1 parent 37e6436
commit ad0f22c
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 45 deletions.
diff --git a/build-test.sh b/build-test.sh
@@ -295,28 +295,75 @@ build_Tests_internal()
     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
-    __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
-    __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
-    __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 
-    # Generate build command
-    buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+    if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
+        # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
+        # See https://github.com/Microsoft/msbuild/issues/2993
 
-    echo "Building step '$stepName' via $buildCommand"
+        # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
+        export __SkipPackageRestore=false
+        export __SkipTargetingPackBuild=false
+        export __BuildLoopCount=2
+        export __TestGroupToBuild=1
+        __AppendToLog=false
 
-    # Invoke MSBuild
-    eval $buildCommand
+        if [ -n __priority1 ]; then
+            export __BuildLoopCount=16
+            export __TestGroupToBuild=2
+        fi
 
-    # Invoke MSBuild
-    # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+        for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
+        do
+            __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
+            __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
+            __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 
-    # Make sure everything is OK
-    if [ $? -ne 0 ]; then
-        echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
-        echo "    $__BuildLog"
-        echo "    $__BuildWrn"
-        echo "    $__BuildErr"
-        exit 1
+            export TestBuildSlice=$slice
+
+            # Generate build command
+            buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+            echo "Building step '$stepName' slice=$slice via $buildCommand"
+
+            # Invoke MSBuild
+            eval $buildCommand
+
+            # Make sure everything is OK
+            if [ $? -ne 0 ]; then
+                echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+                echo "    $__BuildLog"
+                echo "    $__BuildWrn"
+                echo "    $__BuildErr"
+                exit 1
+            fi
+            export __SkipPackageRestore=true
+            export __SkipTargetingPackBuild=true
+            __AppendToLog=true
+        done
+    else
+        __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
+        __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
+        __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
+
+        # Generate build command
+        buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+        echo "Building step '$stepName' via $buildCommand"
+
+        # Invoke MSBuild
+        eval $buildCommand
+
+        # Invoke MSBuild
+        # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+
+        # Make sure everything is OK
+        if [ $? -ne 0 ]; then
+            echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+            echo "    $__BuildLog"
+            echo "    $__BuildWrn"
+            echo "    $__BuildErr"
+            exit 1
+        fi
     fi
 }
 
@@ -342,6 +389,7 @@ usage()
     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
+    echo "priority1 - include priority=1 tests in the build"
     exit 1
 }
 
@@ -461,6 +509,7 @@ __RunTests=0
 __RebuildTests=0
 __BuildTestWrappers=0
 __GenerateLayoutOnly=
+__priority1=
 CORE_ROOT=
 
 while :; do
@@ -615,6 +664,10 @@ while :; do
         msbuildonunsupportedplatform)
             __msbuildonunsupportedplatform=1
             ;;
+        priority1)
+            __priority1=1
+            __UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1"
+            ;;
         *)
             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
             ;;

diff --git a/tests/src/dirs.proj b/tests/src/dirs.proj
@@ -33,26 +33,16 @@
       <DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
     </ItemGroup>
 
-    <!-- Unix builds do not support subgroups -->
-    <ItemGroup Condition="$(__BuildOS) != 'Windows_NT'" >
-      <Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
-        <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
-      </Project>
-      <Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
-        <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
-      </Project>
-    </ItemGroup>
-
     <!-- Test build is divided in slices which can be created within Test Group
          Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->
 
-    <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
+    <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
       <Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
+    <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
       <Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -61,7 +51,7 @@
     <!-- Test build is divided in slices which can be created within Test Group
          Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->    
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
       <Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -76,7 +66,7 @@
       </Project>      
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
       <Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -85,7 +75,7 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
       <Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -112,7 +102,7 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
       <Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -124,7 +114,7 @@
       </Project>      
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">    
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
       <Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -136,7 +126,7 @@
       </Project>
     </ItemGroup>    
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
       <Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -151,7 +141,7 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
       <Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -160,7 +150,7 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
       <Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -169,19 +159,19 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
       <Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
       <Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>    
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
       <Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
@@ -202,31 +192,31 @@
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
       <Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
       <Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>    
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
       <Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
       <Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>
     </ItemGroup>    
 
-    <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
+    <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
       <Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
         <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
       </Project>