Add a GC Reliability Framework job to the CI

Documentation/project-docs/ci-trigger-phrases.md

@@ -11,7 +11,10 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 
 - **Windows_NT x64 Release Priority 1 Build & Test:** "test Windows_NT pri1"
 - **Windows_NT x64 Release IL RoundTrip Build & Test:** "test Windows_NT ilrt"
-- **Windows_NT x64 Release Long-Running GC Build & Test:**: "test Windows_NT x64 Release longgc"
+- **Windows_NT x64 Release Long-Running GC Build & Test:**: "test Windows_NT Release longgc"
+- **Windows_NT x64 Release GC Simulator:**: "test Windows_NT Release gcsimulator"
+- **Windows_NT x64 Release Standalone GC:**: "test Windows_NT Release standalone_gc"
+- **Windows_NT x64 Release GC Reliability Framework:**: "test Windows_NT Release gc_reliability_framework"
 - **Windows_NT x64 Release Ready-To-Run Priority 0 Build & Test:** "test Windows_NT Release r2r"
 - **Windows_NT x64 Checked Ready-To-Run Priority 0 Build & Test:** "test Windows_NT Checked r2r"
 - **Windows_NT x64 Release Ready-To-Run Priority 1 Build & Test:** "test Windows_NT Release pri1r2r"
@@ -69,6 +72,8 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Windows_NT x64 Checked GCStress=0xc JitStress=2 Build & Test:** "test Windows_NT gcstress0xc_jitstress2"
 - **Windows_NT x64 Checked GCStress=0xc MinOpts Heap Verify 1 Build & Test:** "test Windows_NT gcstress0xc_minopts_heapverify1"
 - **Windows_NT x64 Checked Long-Running GC Build & Test:**: "test Windows_NT x64 Checked longgc"
+- **Windows_NT x64 Checked Standalone GC:**: "test Windows_NT Checked standalone_gc"
+- **Windows_NT x64 Checked GC Reliability Framework:**: "test Windows_NT Checked gc_reliability_framework"
 - **Windows_NT x64 Formatting:**: "test Windows_NT formatting"
 - **Windows_NT x64 Checked CoreFX Baseline Build & Test:** "test Windows_NT corefx_baseline"
 - **Windows_NT x64 Checked CoreFX MinOpts Build & Test:** "test Windows_NT corefx_minopts"
@@ -166,6 +171,11 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Ubuntu x64 Release Priority 1 Build & Test:** "test Ubuntu pri1"
 - **Ubuntu x64 Release IL RoundTrip Build & Test:** "test Ubuntu ilrt"
 - **Ubuntu x64 Release Long-Running GC Build & Test:**: "test Ubuntu Release longgc"
+- **Ubuntu x64 Release GC Simulator:**: "test Ubuntu Release gcsimulator"
+- **Ubuntu x64 Release Standalone GC:**: "test Ubuntu Release standalone_gc"
+- **Ubuntu x64 Checked Standalone GC:**: "test Ubuntu Checked standalone_gc"
+- **Ubuntu x64 Release GC Reliability Framework:**: "test Ubuntu Release gc_reliability_framework"
+- **Ubuntu x64 Checked GC Reliability Framework:**: "test Ubuntu Checked gc_reliability_framework"
 - **Ubuntu x64 Release Ready-To-Run Priority 0 Build & Test:** "test Ubuntu Release r2r"
 - **Ubuntu x64 Checked Ready-To-Run Priority 0 Build & Test:** "test Ubuntu Checked r2r"
 - **Ubuntu x64 Release Ready-To-Run Priority 1 Build & Test:** "test Ubuntu Release pri1r2r"
@@ -248,6 +258,11 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **OSX x64 Release Priority 1 Build & Test:** "test OSX pri1"
 - **OSX x64 Release IL RoundTrip Build & Test:** "test OSX ilrt"
 - **OSX x64 Release Long-Running GC Build & Test:**: "test OSX Release longgc"
+- **OSX x64 Release GC Simulator:**: "test OSX10.12 Release gcsimulator"
+- **OSX x64 Release Standalone GC:**: "test OSX10.12 Release standalone_gc"
+- **OSX x64 Checked Standalone GC:**: "test OSX10.12 Checked standalone_gc"
+- **OSX x64 Release GC Reliability Framework:**: "test OSX10.12 Release gc_reliability_framework"
+- **OSX x64 Checked GC Reliability Framework:**: "test OSX10.12 Checked gc_reliability_framework"
 - **OSX x64 Release Ready-To-Run Priority 0 Build & Test:** "test OSX Release r2r"
 - **OSX x64 Checked Ready-To-Run Priority 0 Build & Test:** "test OSX Checked r2r"
 - **OSX x64 Release Ready-To-Run Priority 1 Build & Test:** "test OSX Release pri1r2r"

Documentation/project-docs/garbage-collector-guidelines.md

@@ -26,12 +26,43 @@ Required Testing: Validation of the behavior of the affected APIs.
 ## Stress Testing ##
 Stress testing must run for at least **48 hours** against a debug build.
 
-Instructions for running stress are located in the repo at tests\src\GC\Stress\stress_run_readme.txt.
+Stress testing for checked and release builds can be done on pull requests with The .NET CI infrastructure.
+A stress run can be requested using the trigger phrase:
+
+```
+@dotnet_bot test <platform> <flavor> gc_reliability_framework
+```
+
+This will run the stress framework for the default amount of time (15 hours) on the given platform and build flavor.
 
 ## Functional Testing ##
 A functional test run executes the same code as a stress run, but only runs for 30 minutes.
 
 Instructions for running stress are located in the repo at tests\src\GC\Stress\stress_run_readme.txt.
 
+It is recommended that you run at least some of the below PR-triggered CI jobs:
+
+```
+@dotnet_bot test Windows_NT Checked longgc
+@dotnet_bot test OSX10.12 Checked longgc
+@dotnet_bot test Ubuntu Checked longgc
+@dotnet_bot test Windows_NT Checked standalone_gc
+@dotnet_bot test OSX10.12 Checked standalone_gc
+@dotnet_bot test Ubuntu Checked standalone_gc
+```
+
+The "Long GC" tests are a series of GC tests whose running time is too long or memory usage is too high to run with
+the rest of the Priority 0 unit tests. The "Standalone GC" build mode builds and runs the GC in a semi-standalone manner
+(see https://github.com/dotnet/coreclr/projects/3).
+
+You may also wish to run the GC Simulator tests. They may take up to 24 hours to complete and are known to sometimes fail on Ubuntu
+due to poor interactions with the Linux OOM killer. However, they have proven to be quite useful in finding bugs in the past:
+
+```
+@dotnet_bot test Windows_NT Release gcsimulator
+@dotnet_bot test Ubuntu Release gcsimulator
+@dotnet_bot test OSX10.12 Release gcsimulator
+```
+
 ## Performance Testing ##
 Coming soon.

netci.groovy

@@ -137,6 +137,7 @@ class Constants {
                'gcsimulator',
                'jitdiff',              
                'standalone_gc',
+               'gc_reliability_framework',
                'illink'] + r2rJitStressScenarios
 
     def static configurationList = ['Debug', 'Checked', 'Release']
@@ -205,6 +206,14 @@ def static isJitDiff(def scenario) {
     return (scenario == 'jitdiff')
 }
 
+def static isGcReliabilityFramework(def scenario) {
+    return (scenario == 'gc_reliability_framework')
+}
+
+def static scenarioNeedsPri1Build(def scenario) {
+    return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
+}
+
 def static setTestJobTimeOut(newJob, scenario) {
     if (isGCStressRelatedTesting(scenario)) {
         Utilities.setJobTimeout(newJob, 4320)
@@ -227,6 +236,9 @@ def static setTestJobTimeOut(newJob, scenario) {
     else if (isJitDiff(scenario)) {
         Utilities.setJobTimeout(newJob, 240)
     }
+    else if (isGcReliabilityFramework(scenario)) {
+        Utilities.setJobTimeout(newJob, 1440)
+    }
     // Non-test jobs use the default timeout value.
 }
 
@@ -497,6 +509,11 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
             Utilities.addPeriodicTrigger(job, '@weekly')
             break
+        case 'gc_reliability_framework':
+            assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
+            assert (configuration == 'Release' || configuration == 'Checked')
+            // Only triggered by phrase.
+            break
         case 'ilrt':
             assert !(os in bidailyCrossList)
             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
@@ -766,6 +783,16 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
                             }
                             break
+                        case 'standalone_gc':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
+                        case 'gc_reliability_framework':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
                         case 'minopts':
                         case 'forcerelocs':
                         case 'jitstress1':
@@ -939,6 +966,11 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
                             }
                             break
+                        case 'gc_reliability_framework':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
                         case 'minopts':
                         case 'forcerelocs':
                         case 'jitstress1':
@@ -1307,7 +1339,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     // binaries are sent to a default directory whose name is about
                     // 35 characters long.
 
-                    else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
+                    else if (scenarioNeedsPri1Build(scenario)) {
                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
                     }
                     else if (isLongGc(scenario)) {
@@ -1435,6 +1467,12 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
                             }
                         }
+                        else if (isGcReliabilityFramework(scenario)) {
+                            buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
+                            buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
+                            Utilities.addArchival(newJob, "stdout.txt")
+                            Utilities.addArchival(newJob, "Logs/**")
+                        }
                         else if (architecture == 'x64' || architecture == 'x86') {
                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
                         }
@@ -1870,6 +1908,7 @@ combinedScenarios.each { scenario ->
                                     return
                                 }
                                 break
+                            case 'gc_reliability_framework':
                             case 'standalone_gc':
                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
                                     return
@@ -2120,10 +2159,12 @@ combinedScenarios.each { scenario ->
                                     return
                                 }
                                 break
+                            case 'gc_reliability_framework':
                             case 'standalone_gc':
                                 if (configuration != 'Release' && configuration != 'Checked') {
                                     return
                                 }
+                                break
                             case 'coverage':
                                 //We only want Ubuntu Release for coverage
                                 if (os != 'Ubuntu') {
@@ -2164,7 +2205,7 @@ combinedScenarios.each { scenario ->
                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
                     // from the default scenario
                     def testBuildScenario = scenario
-                    if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
+                    if (scenarioNeedsPri1Build(scenario)) {
                         testBuildScenario = 'pri1'
                     }
                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
@@ -2210,6 +2251,7 @@ combinedScenarios.each { scenario ->
                     def runilasmroundtripStr = ''
                     def gcstressStr = ''
                     def illinkStr = ''
+                    def layoutOnlyStr =''
 
                     if (scenario == 'r2r' ||
                         scenario == 'pri1r2r' ||
@@ -2286,6 +2328,10 @@ combinedScenarios.each { scenario ->
                         }
                     }
 
+                    if (isGcReliabilityFramework(scenario)) {
+                        layoutOnlyStr = '--build-overlay-only'
+                    }
+
                     def folder = getJobFolder(scenario)
                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
                         // Add parameters for the inputs
@@ -2438,11 +2484,29 @@ combinedScenarios.each { scenario ->
                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
                 --limitedDumpGeneration \\
-                ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${illinkStr} ${sequentialString} ${playlistString}""")
+                ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
+                ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
+                ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
+
+                                if (isGcReliabilityFramework(scenario)) {
+                                    // runtest.sh doesn't actually execute the reliability framework - do it here.
+                                    if (serverGCString != '') {
+                                        shell("export COMPlus_gcServer=1")
+                                    }
+
+                                    shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
+                                }
                             }
                         }
                     }
 
+                    if (isGcReliabilityFramework(scenario))
+                    {
+                        // Both of these are emitted by the RF
+                        Utilities.addArchival(newJob, "stdout.txt")
+                        Utilities.addArchival(newJob, "Logs/**")
+                    }
+
                     if (scenario == 'coverage') {
                         // Publish coverage reports
                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')

tests/scripts/run-gc-reliability-framework.cmd

@@ -0,0 +1,10 @@
+@rem Licensed to the .NET Foundation under one or more agreements.
+@rem The .NET Foundation licenses this file to you under the MIT license.
+@rem See the LICENSE file in the project root for more information.
+
+@echo off
+
+set CORE_ROOT=%CD%\bin\tests\Windows_NT.%1.%2\Tests\Core_Root
+set FRAMEWORK_DIR=%CD%\bin\tests\Windows_NT.%1.%2\GC\Stress\Framework\ReliabilityFramework
+powershell "%CORE_ROOT%\CoreRun.exe %FRAMEWORK_DIR%\ReliabilityFramework.exe %FRAMEWORK_DIR%\testmix_gc.config | tee stdout.txt"
+

tests/scripts/run-gc-reliability-framework.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export CORE_ROOT=`pwd`/bin/tests/Windows_NT.$1.$2/Tests/coreoverlay
+FRAMEWORK_DIR=`pwd`/bin/tests/Windows_NT.$1.$2/GC/Stress/Framework/ReliabilityFramework
+$CORE_ROOT/corerun $FRAMEWORK_DIR/ReliabilityFramework.exe $FRAMEWORK_DIR/testmix_gc.config | tee stdout.txt
+

tests/src/GC/Stress/Framework/ReliabilityFramework.csproj

@@ -10,9 +10,9 @@
     <OutputType>Exe</OutputType>
     <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
-    <CLRTestKind>BuildAndRun</CLRTestKind>
+    <CLRTestKind>BuildOnly</CLRTestKind>
+    <GenerateRunScript>false</GenerateRunScript>
     <CLRTestPriority>1</CLRTestPriority>
-    <CLRTestExecutionArguments>testmix_gc.config /maximumExecutionTime:-1</CLRTestExecutionArguments>
     <DefineConstants>$(DefineConstants);STATIC;PROJECTK_BUILD</DefineConstants>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->