Add portable linux source build leg (#75546)

Add portable linux source build leg Adds a portable linux source build leg to the official build. The idea is that these packages produced by this build should be relied upon in downstream PR validation, rather than the RID specific assets. This should allow for cleaner SB logic in downstream repos. While doing this, I cleaned up a couple parameters to make it clearer what they were doing Now we will run the following jobs: - Run only linux-x64 portable officially - Run centos.7 and banana.24 in rolling CI - Run linux-x64 in PR (dev-innerloop)
dotnet · Oct 6, 2022 · b1f5113 · b1f5113
1 parent 11b4ef9
commit b1f5113
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 22 deletions.
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -21,7 +21,8 @@ parameters:
   shouldContinueOnError: false
   dependOnEvaluatePaths: false
   isOfficialBuild: false
-  buildingOnSourceBuildImage: false
+  isSourceBuild: false
+  isNonPortableSourceBuild: false
   runtimeFlavor: 'coreclr'
   runtimeVariant: ''
   helixQueues: ''
@@ -113,9 +114,9 @@ jobs:
           value: ''
 
       - name: _sclEnableCommand
-        ${{ if eq(parameters.buildingOnSourceBuildImage, true) }}:
+        ${{ if eq(parameters.isSourceBuild, true) }}:
           value: scl enable llvm-toolset-7.0 --
-        ${{ if ne(parameters.buildingOnSourceBuildImage, true) }}:
+        ${{ if ne(parameters.isSourceBuild, true) }}:
           value: ''
 
       - ${{ each variable in parameters.variables }}:
@@ -136,15 +137,15 @@ jobs:
           artifact: Mono_Offsets_${{monoCrossAOTTargetOS}}
           path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'
 
-    - ${{ if eq(parameters.buildingOnSourceBuildImage, true) }}:
+    - ${{ if eq(parameters.isSourceBuild, true) }}:
       - template: /eng/common/templates/steps/source-build.yml
         parameters:
           platform:
             buildScript: $(_sclEnableCommand) $(Build.SourcesDirectory)$(dir)build$(scriptExt)
-            nonPortable: true
-            # Use a custom RID that isn't in the RID graph here to validate we don't break the usage of custom rids that aren't in the graph.
-            targetRID: banana.24-x64
+            nonPortable: ${{ parameters.isNonPortableSourceBuild }}
+            targetRID: ${{ parameters.targetRid }}
             runtimeOS: linux
+            name: ${{ parameters.platform }}
 
     - ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS', 'MacCatalyst') }}:
       - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} ${{ parameters.archType }} azDO
@@ -163,7 +164,7 @@ jobs:
           filePath: $(Build.SourcesDirectory)/eng/pipelines/mono/update-machine-certs.ps1
 
     # Build
-    - ${{ if eq(parameters.buildingOnSourceBuildImage, false) }}:
+    - ${{ if eq(parameters.isSourceBuild, false) }}:
       - script: $(_sclEnableCommand) $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_crossBuildPropertyArg) $(_cxx11Parameter) $(_richCodeNavigationParam) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
         displayName: Build product
         ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}:

diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml
@@ -279,7 +279,57 @@ jobs:
         disableClrTest: true
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# Linux x64 Source Build
+# Centos 7 x64 Source Build
+
+- ${{ if containsValue(parameters.platforms, 'SourceBuild_Centos7_x64') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: Linux
+      archType: x64
+      targetRid: centos.7-x64
+      platform: Centos7_Linux_x64
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container:
+        image: centos-7-source-build-20210714125450-5d87b80
+        registry: mcr
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        stagedBuild: ${{ parameters.stagedBuild }}
+        buildConfig: ${{ parameters.buildConfig }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        isSourceBuild: true
+        isNonPortableSourceBuild: true
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Non-existent RID Source Build
+
+- ${{ if containsValue(parameters.platforms, 'SourceBuild_Banana24_x64') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: Linux
+      archType: x64
+      targetRid: banana.24-x64
+      platform: Banana24_Linux_x64
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container:
+        image: centos-7-source-build-20210714125450-5d87b80
+        registry: mcr
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        stagedBuild: ${{ parameters.stagedBuild }}
+        buildConfig: ${{ parameters.buildConfig }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        isSourceBuild: true
+        isNonPortableSourceBuild: true
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Portable Linux x64 Source Build
 
 - ${{ if containsValue(parameters.platforms, 'SourceBuild_Linux_x64') }}:
   - template: xplat-setup.yml
@@ -300,8 +350,9 @@ jobs:
         stagedBuild: ${{ parameters.stagedBuild }}
         buildConfig: ${{ parameters.buildConfig }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        isSourceBuild: true
+        isNonPortableSourceBuild: false
         ${{ insert }}: ${{ parameters.jobParameters }}
-        buildingOnSourceBuildImage: true
 
 # Linux s390x
 

diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml
@@ -163,13 +163,11 @@ jobs:
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/common/global-build-job.yml
-    buildConfig: release
+    buildConfig: Release
     platforms:
     - SourceBuild_Linux_x64
     jobParameters:
-      nameSuffix: SourceBuild
-      timeoutInMinutes: 120
-      condition:
-        or(
-          eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
-          eq(variables['isRollingBuild'], true))
+      nameSuffix: PortableSourceBuild
+      extraStepsParameters:
+        name: SourceBuildPackages
+      timeoutInMinutes: 95
diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml
@@ -364,9 +364,6 @@ stages:
         runTests: false
         useHelix: false
 
-  #
-  # Build Sourcebuild leg
-  #
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
       jobTemplate: /eng/pipelines/common/global-build-job.yml
@@ -375,13 +372,12 @@ stages:
       platforms:
       - SourceBuild_Linux_x64
       jobParameters:
-        nameSuffix: SourceBuild
+        nameSuffix: PortableSourceBuild
         extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
         extraStepsParameters:
           name: SourceBuildPackages
         timeoutInMinutes: 95
 
-
   #
   # Installer Build
   #

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -1296,3 +1296,41 @@ jobs:
           eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true),
           eq(variables['isRollingBuild'], true))
+
+#
+# Sourcebuild legs
+# We have 3 important legs for source-build:
+# - Centos.7 (ensures that RH keeps working)
+# - Linux-x64 portable (used for dependency flow and downstream PR verification)
+# - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about.
+#
+# Running all of these everywhere is wasteful. Run Banana.24 and CentOS.7 in rolling CI,
+# Run Linux-x64 in PR.
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    buildConfig: Release
+    helixQueueGroup: pr
+    platforms:
+    - SourceBuild_Centos7_x64
+    jobParameters:
+      nameSuffix: Centos7SourceBuild
+      extraStepsParameters:
+        name: SourceBuildPackages
+      timeoutInMinutes: 95
+      condition: eq(variables['isRollingBuild'], true)
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    buildConfig: Debug
+    helixQueueGroup: pr
+    platforms:
+    - SourceBuild_Banana24_x64
+    jobParameters:
+      nameSuffix: Banana24SourceBuild
+      extraStepsParameters:
+        name: SourceBuildPackages
+      timeoutInMinutes: 95
+      condition: eq(variables['isRollingBuild'], true)