From 5e51b8725aeb4c445d1a2b031ff9358919bbb8f5 Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:38:18 -0700 Subject: [PATCH] Update common Docker engineering infrastructure with latest --- eng/common/templates/1es-official.yml | 4 + eng/common/templates/1es-unofficial.yml | 6 +- eng/common/templates/jobs/build-images.yml | 76 ++----- eng/common/templates/jobs/generate-matrix.yml | 40 +++- eng/common/templates/jobs/post-build.yml | 10 + .../stages/build-test-publish-repo.yml | 204 ++++++++++++------ .../stages/dotnet/build-test-publish-repo.yml | 11 + .../common-init-for-matrix-and-build.yml | 62 ++++++ eng/common/templates/variables/common.yml | 6 + .../templates/variables/docker-images.yml | 2 +- .../templates/variables/dotnet/common.yml | 7 + 11 files changed, 289 insertions(+), 139 deletions(-) create mode 100644 eng/common/templates/steps/common-init-for-matrix-and-build.yml diff --git a/eng/common/templates/1es-official.yml b/eng/common/templates/1es-official.yml index 60091b72fd..69d1d4f353 100644 --- a/eng/common/templates/1es-official.yml +++ b/eng/common/templates/1es-official.yml @@ -46,9 +46,13 @@ extends: ignoreDirectories: $(Build.SourcesDirectory)/versions whatIf: ${{ parameters.cgDryRun }} showAlertLink: true + policheck: + enabled: true sourceRepositoriesToScan: exclude: - repository: InternalVersionsRepo - repository: PublicVersionsRepo sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true stages: ${{ parameters.stages }} diff --git a/eng/common/templates/1es-unofficial.yml b/eng/common/templates/1es-unofficial.yml index 30faab0427..24ddeaff3d 100644 --- a/eng/common/templates/1es-unofficial.yml +++ b/eng/common/templates/1es-unofficial.yml @@ -45,11 +45,13 @@ extends: parameters: pool: ${{ parameters.pool }} sdl: - enableAllTools: ${{ not(parameters.disableSDL) }} componentgovernance: ignoreDirectories: $(Build.SourcesDirectory)/versions whatIf: true showAlertLink: true + enableAllTools: ${{ not(parameters.disableSDL) }} + policheck: + enabled: true sbom: enabled: true sourceRepositoriesToScan: @@ -57,4 +59,6 @@ extends: - repository: InternalVersionsRepo - repository: PublicVersionsRepo sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true stages: ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 2b6ba00735..46c469334b 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -4,18 +4,19 @@ parameters: matrix: {} dockerClientOS: null buildJobTimeout: 60 + commonInitStepsForMatrixAndBuild: [] customInitSteps: [] noCache: false internalProjectName: null publicProjectName: null - internalVersionsRepoRef: null - publicVersionsRepoRef: null + isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} - condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true'))) dependsOn: - - PreBuildValidation + - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - PreBuildValidation - CopyBaseImages - GenerateBuildMatrix pool: ${{ parameters.pool }} @@ -24,59 +25,11 @@ jobs: timeoutInMinutes: ${{ parameters.buildJobTimeout }} variables: imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions) - versionsRepoPath: versions sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo imageInfoContainerDir: $(artifactsPath)/imageInfo - ${{ if eq(parameters.noCache, false) }}: - versionsBasePath: $(versionsRepoPath)/ - pipelineDisabledCache: false - ${{ if eq(parameters.noCache, true) }}: - versionsBasePath: "" - pipelineDisabledCache: true steps: - - checkout: self - - ${{ if and(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.publicVersionsRepoRef }} - path: s/$(versionsRepoPath) - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.internalVersionsRepoRef }} - path: s/$(versionsRepoPath) - - ${{ if eq(parameters.noCache, false) }}: - - powershell: | - $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") - if ($pathSeparatorIndex -ge 0) { - $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) - } - else { - $buildRepoName = "$(Build.Repository.Name)" - } - - $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" - $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" - $manifest = "$buildRepoName/$(manifest)" - $testResultsDirectory = "$buildRepoName/$testResultsDirectory" - - if ("$(testScriptPath)") { - $testScriptPath = "$buildRepoName/$(testScriptPath)" - } - - echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" - echo "##vso[task.setvariable variable=manifest]$manifest" - echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" - echo "##vso[task.setvariable variable=engPath]$engPath" - echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" - echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" - displayName: Override Common Paths - - powershell: | - if ("${{ parameters.noCache }}" -eq "false") { - $baseContainerRepoPath = "/repo/$(buildRepoName)" - } - else { - $baseContainerRepoPath = "/repo" - } - echo "##vso[task.setvariable variable=baseContainerRepoPath]$baseContainerRepoPath" - displayName: Set Base Container Repo Path + - ${{ parameters.commonInitStepsForMatrixAndBuild }} - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self parameters: cleanupDocker: true @@ -96,13 +49,9 @@ jobs: # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates # the environment variable for us. - $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json" - if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push" - } - - if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.publicProjectName }}" -and ${env:PUBLIC-MIRROR_SERVER} -ne "") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push" } # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache @@ -128,7 +77,6 @@ jobs: --os-type $(osType) --architecture $(architecture) --retry - --source-repo $(publicGitRepoUri) --digests-out-var 'builtImages' --acr-subscription '$(acr-staging.subscription)' --acr-resource-group '$(acr-staging.resourceGroup)' @@ -141,7 +89,7 @@ jobs: displayName: Publish Image Info File Artifact internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: # The following task depends on the SBOM Manifest Generator task installed on the agent. # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves. - powershell: | @@ -193,11 +141,11 @@ jobs: } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}: - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self parameters: condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(sbomDirectory) diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml index fb5e8332ed..4fc4ae759c 100644 --- a/eng/common/templates/jobs/generate-matrix.yml +++ b/eng/common/templates/jobs/generate-matrix.yml @@ -5,28 +5,43 @@ parameters: customBuildLegGroupArgs: "" isTestStage: false internalProjectName: null + noCache: false + commonInitStepsForMatrixAndBuild: [] jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: + - ${{ parameters.commonInitStepsForMatrixAndBuild }} - template: /eng/common/templates/steps/retain-build.yml@self - template: /eng/common/templates/steps/init-docker-linux.yml@self - template: /eng/common/templates/steps/validate-branch.yml@self parameters: internalProjectName: ${{ parameters.internalProjectName }} + - template: /eng/common/templates/steps/set-image-info-path-var.yml + parameters: + publicSourceBranch: $(publicSourceBranch) - ${{ if eq(parameters.isTestStage, true) }}: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) artifactName: image-info - - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]--image-info $(artifactsPath)/image-info.json" - displayName: Set GenerateBuildMatrix Variables - - ${{ if eq(parameters.isTestStage, false) }}: - - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]" - displayName: Set GenerateBuildMatrix Variables + - powershell: | + $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)" + + if ("${{ parameters.isTestStage}}" -eq "true") { + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" + } + elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" + } + + echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions" + displayName: Set GenerateBuildMatrix Variables - script: > - $(runImageBuilderCmd) generateBuildMatrix + echo "##vso[task.setvariable variable=generateBuildMatrixCommand] + generateBuildMatrix --manifest $(manifest) --type ${{ parameters.matrixType }} --os-type '*' @@ -35,6 +50,13 @@ jobs: ${{ parameters.customBuildLegGroupArgs }} $(imageBuilder.pathArgs) $(manifestVariables) - $(additionalGenerateBuildMatrixOptions) - displayName: Generate ${{ parameters.matrixType }} Matrix - name: matrix + $(commonMatrixAndBuildOptions) + $(additionalGenerateBuildMatrixOptions)" + displayName: Set GenerateBuildMatrix Command + - template: /eng/common/templates/steps/run-imagebuilder.yml@self + parameters: + name: matrix + displayName: Generate ${{ parameters.matrixType }} Matrix + serviceConnection: $(build.serviceConnectionName) + internalProjectName: internal + args: $(generateBuildMatrixCommand) diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml index 707b6c1d75..d470a64100 100644 --- a/eng/common/templates/jobs/post-build.yml +++ b/eng/common/templates/jobs/post-build.yml @@ -52,15 +52,24 @@ jobs: } displayName: Prune Publish Artifacts - powershell: | + $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)" + if ($imageInfoFiles.Count -eq 0) { + echo "No image info files found." + echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true" + exit 0 + } + New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force $(runImageBuilderCmd) mergeImageInfo ` --manifest $(manifest) ` $(imageInfosContainerDir) ` $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` $(manifestVariables) + name: MergeImageInfoFiles displayName: Merge Image Info Files - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) path: $(sbomOutputDir) artifactName: sboms displayName: Publish SBOM Artifact @@ -68,6 +77,7 @@ jobs: publicProjectName: ${{ parameters.publicProjectName }} - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) path: $(imageInfosHostDir)$(imageInfosOutputSubDir) artifactName: image-info displayName: Publish Image Info File Artifact diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index 29a72a48fc..8ec26e70fe 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -25,6 +25,8 @@ parameters: internalVersionsRepoRef: null publicVersionsRepoRef: null + isInternalServicingValidation: false + linuxAmd64Pool: vmImage: $(defaultLinuxAmd64PoolImage) linuxArm32Pool: @@ -37,6 +39,8 @@ parameters: vmImage: $(defaultWindows1809PoolImage) windows2022Pool: vmImage: $(defaultWindows2022PoolImage) + windows2025Pool: + vmImage: $(defaultWindows2025PoolImage) stages: @@ -46,24 +50,25 @@ stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) jobs: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - template: /eng/common/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages @@ -77,8 +82,13 @@ stages: pool: ${{ parameters.linuxAmd64Pool }} customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} internalProjectName: ${{ parameters.internalProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + noCache: ${{ parameters.noCache }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_amd64 @@ -86,12 +96,17 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm64 @@ -99,12 +114,17 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm32 @@ -112,12 +132,17 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows1809_amd64 @@ -125,12 +150,17 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows2022_amd64 @@ -138,6 +168,30 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + - template: /eng/common/templates/jobs/build-images.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -151,12 +205,17 @@ stages: matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} ################################################################################ # Post-Build @@ -174,11 +233,12 @@ stages: ################################################################################ # Test Images ################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - stage: Test dependsOn: Post_Build condition: " and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), ne(variables['testScriptPath'], ''), and( contains(variables['stages'], 'test'), @@ -197,6 +257,12 @@ stages: isTestStage: true internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + commonInitStepsForMatrixAndBuild: + - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + parameters: + noCache: ${{ parameters.noCache }} + internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} + publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_amd64 @@ -237,6 +303,14 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + - template: /eng/common/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + customInitSteps: ${{ parameters.customTestInitSteps }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: WindowsLtsc2016_amd64 @@ -249,44 +323,46 @@ stages: ################################################################################ # Publish Images ################################################################################ -- stage: Publish - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " - and( - not(canceled()), +- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - stage: Publish + ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Test + ${{ else }}: + dependsOn: Post_Build + condition: " and( - contains(variables['stages'], 'publish'), - or( + not(canceled()), + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + and( + contains(variables['stages'], 'publish'), or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), or( and( - not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build')), and( contains(variables['stages'], 'test'), in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), + or( and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} + not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + and( + not(contains(variables['stages'], 'test')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build'))))), + not( + or( + contains(variables['stages'], 'build'), + contains(variables['stages'], 'test'))))))" + jobs: + - template: /eng/common/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customPublishInitSteps }} diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index 53f9e8a637..471b01fc37 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -18,6 +18,7 @@ parameters: linuxAmd64Pool: "" buildMatrixType: platformDependencyGraph testMatrixType: platformVersionedOs + isInternalServicingValidation: false stages: - template: /eng/common/templates/stages/build-test-publish-repo.yml@self @@ -25,6 +26,7 @@ stages: noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} @@ -120,3 +122,12 @@ stages: image: $(windowsServer2022PublicPoolImage) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: image: $(windowsServer2022InternalPoolImage) + + # Windows Server 2025 + windows2025Pool: + os: windows + name: $(windowsServer2025PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2025PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2025InternalPoolImage) diff --git a/eng/common/templates/steps/common-init-for-matrix-and-build.yml b/eng/common/templates/steps/common-init-for-matrix-and-build.yml new file mode 100644 index 0000000000..2f12a54d7c --- /dev/null +++ b/eng/common/templates/steps/common-init-for-matrix-and-build.yml @@ -0,0 +1,62 @@ +parameters: + noCache: false + internalVersionsRepoRef: null + publicVersionsRepoRef: null + versionsRepoPath: versions + +steps: +- checkout: self +- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(parameters.noCache, false)) }}: + - checkout: ${{ parameters.publicVersionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} +- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.noCache, false)) }}: + - checkout: ${{ parameters.internalVersionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} +- powershell: | + $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" + if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" + } + + if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + } + + if ("${{ parameters.noCache }}" -eq "false") { + $versionsBasePath = "${{ parameters.versionsRepoPath }}/" + $pipelineDisabledCache = "false" + + $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") + if ($pathSeparatorIndex -ge 0) { + $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) + } + else { + $buildRepoName = "$(Build.Repository.Name)" + } + + $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" + + $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" + $manifest = "$buildRepoName/$(manifest)" + $testResultsDirectory = "$buildRepoName/$testResultsDirectory" + + if ("$(testScriptPath)") { + $testScriptPath = "$buildRepoName/$(testScriptPath)" + } + + echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" + echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" + echo "##vso[task.setvariable variable=manifest]$manifest" + echo "##vso[task.setvariable variable=engPath]$engPath" + echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" + echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" + } + else { + $versionsBasePath = "" + $pipelineDisabledCache = "true" + } + + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" + echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" + displayName: Set Common Variables for Matrix and Build diff --git a/eng/common/templates/variables/common.yml b/eng/common/templates/variables/common.yml index 7b0aafe450..38a62317ac 100644 --- a/eng/common/templates/variables/common.yml +++ b/eng/common/templates/variables/common.yml @@ -44,6 +44,10 @@ variables: value: "" - name: customCopyBaseImagesArgs value: "" +- name: additionalGenerateBuildMatrixOptions + value: "" +- name: trimCachedImagesForMatrix + value: false - name: defaultLinuxAmd64PoolImage value: ubuntu-latest @@ -57,6 +61,8 @@ variables: value: windows-2019 - name: defaultWindows2022PoolImage value: windows-2022 +- name: defaultWindows2025PoolImage + value: windows-2025 - name: default1ESInternalPoolName value: NetCore1ESPool-Internal diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 949c53d75c..69fd91ec7c 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2543116 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2555712 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner diff --git a/eng/common/templates/variables/dotnet/common.yml b/eng/common/templates/variables/dotnet/common.yml index 70bbdc0033..afbe264bd7 100644 --- a/eng/common/templates/variables/dotnet/common.yml +++ b/eng/common/templates/variables/dotnet/common.yml @@ -47,6 +47,13 @@ variables: - name: windowsServer2022PoolName value: Docker-2022-${{ variables['System.TeamProject'] }} +- name: windowsServer2025PublicPoolImage + value: Server2025-NESDockerBuilds +- name: windowsServer2025InternalPoolImage + value: Server2025-NESDockerBuilds-1ESPT +- name: windowsServer2025PoolName + value: Docker-2025-${{ variables['System.TeamProject'] }} + - group: DotNet-Docker-Common - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-Docker-Secrets-WIF