Add Linux VSTS CI (PowerShell#7527)

Update Start-PSPester and Start-PSxUnit to upload test results when running in VSTS
Disable tests which require non-root using when running as root
remove verifications in Test-Connection tests which verify buggy behavior
Skip tests which don't behave correctly in VSTS (due to running in a container?)
- Test-Connection - PowerShell#7528
- Set-Date - PowerShell#7529

.vsts-ci/linux.yml

@@ -0,0 +1,78 @@
+name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr)
+variables:
+  DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  POWERSHELL_TELEMETRY_OPTOUT: 1
+  # Avoid expensive initialization of dotnet cli, see: http://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+  # Enable VSTS debug mode until stabilitized
+  system.debug: 'true'
+
+resources:
+- repo: self
+  clean: true
+phases:
+- phase: Linux_CI
+
+  queue:
+    name: Hosted Linux Preview
+  steps:
+  - powershell:  Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))"
+    displayName: Set Build Name for Non-PR
+    condition: ne(variables['Build.Reason'], 'PullRequest')
+
+# Several scripts require TMPDIR to exist
+# Set it to AGENT_TEMPDIRECTORY so that we know it exists
+  - powershell: |
+      Write-Host "##vso[task.setvariable variable=TEMP]$env:AGENT_TEMPDIRECTORY"
+    displayName: Set TEMP to AGENT_TEMPDIRECTORY
+    condition: succeededOrFailed()
+
+  - powershell: |
+      git submodule update --init
+    displayName: SubModule Init
+    condition: succeededOrFailed()
+
+  - powershell: |
+      apt-get update
+      apt-get install -y --no-install-recommends less
+    displayName: Install less
+    condition: succeededOrFailed()
+
+  - powershell: |
+      tools/travis.ps1 -Stage Bootstrap
+    displayName: Bootstrap
+    condition: succeeded()
+
+  - powershell: |
+      $ErrorActionPreference = 'continue'
+      tools/travis.ps1
+    displayName: Build and test
+    condition: succeeded()
+
+  - powershell: |
+      tools/travis.ps1 -Stage Failure
+    displayName: After Failure
+    condition: failed()
+
+  - powershell: |
+      tools/travis.ps1 -Stage Success
+    displayName: After Success
+    condition: succeeded()
+
+  # Uploads any packages as an artifact
+  - powershell: |
+      Get-ChildItem -Path *.rpm, *.deb, *.tar.gz -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
+        Write-Host "##vso[artifact.upload containerfolder=artifacts;artifactname=artifacts]$_"
+      }
+    displayName: Publish Artifacts
+    condition: succeededOrFailed()
+    continueOnError: true
+
+  # Uploads any Test results as an artifact
+  - powershell: |
+      Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
+        Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_"
+      }
+    displayName: Publish Test Results
+    condition: succeededOrFailed()
+    continueOnError: true

.vsts-ci/mac.yml

@@ -32,13 +32,6 @@ phases:
     displayName: SubModule Init
     condition: succeededOrFailed()
 
-  - powershell: |
-      rvm install ruby-2.3.3;
-      rvm --default use 2.3.3;
-      displayName: Install Ruby
-    condition: succeededOrFailed()
-    enabled: false
-
   - powershell: |
       tools/travis.ps1 -Stage Bootstrap
     displayName: Bootstrap
@@ -68,74 +61,3 @@ phases:
     displayName: Publish Artifacts
     condition: succeededOrFailed()
     continueOnError: true
-
-  # Uploads any Test results as an artifact
-  - powershell: |
-      Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
-        Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_"
-      }
-    displayName: Publish Test Results
-    condition: succeededOrFailed()
-    continueOnError: true
-
-  # Publish the various Test results as Test results to VSTS
-  # Separate tasks are used because a failure will stop the task
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results TestResultsSudo.xml
-    continueOnError: true
-    inputs:
-      testRunner: NUnit
-      testResultsFiles: '**\TestResultsSudo.xml'
-      testRunTitle: Sudo
-      mergeTestResults: true
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results TestResultsNoSudo.xml
-    continueOnError: true
-    inputs:
-      testRunner: NUnit
-      testResultsFiles: '**\TestResultsNoSudo.xml'
-      testRunTitle: NoSudo
-      mergeTestResults: true
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results TestResultsSudo.ExpTest.*.xml
-    continueOnError: true
-    inputs:
-      testRunner: NUnit
-      testResultsFiles: '**\TestResultsSudo.ExpTest.*.xml'
-      testRunTitle: Sudo Experimental Features
-      mergeTestResults: true
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results TestResultsNoSudo.ExpTest.*.xml
-    continueOnError: true
-    inputs:
-      testRunner: NUnit
-      testResultsFiles: '**\TestResultsNoSudo.ExpTest.*.xml'
-      testRunTitle: Sudo Experimental Features
-      mergeTestResults: true
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results ParallelXUnitTestResults.xml
-    continueOnError: true
-    inputs:
-      testRunner: XUnit
-      testResultsFiles: '**\ParallelXUnitTestResults.xml'
-      testRunTitle: Parallel XUnit
-      mergeTestResults: true
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    displayName: Publish Test Results SequentialXUnitTestResults.xml
-    continueOnError: true
-    inputs:
-      testRunner: XUnit
-      testResultsFiles: '**\SequentialXUnitTestResults.xml'
-      testRunTitle: Sequential XUnit
-      mergeTestResults: true

build.psm1

@@ -1022,7 +1022,9 @@ function Start-PSPester {
         [Parameter(ParameterSetName='PassThru',HelpMessage='Run commands on Linux with sudo.')]
         [switch]$Sudo,
         [switch]$IncludeFailingTest,
-        [string]$ExperimentalFeatureName
+        [string]$ExperimentalFeatureName,
+        [Parameter(HelpMessage='Title to publish the results as.')]
+        [string]$Title = 'PowerShell Core Tests'
     )
 
     if (-not (Get-Module -ListAvailable -Name $Pester -ErrorAction SilentlyContinue | Where-Object { $_.Version -ge "4.2" } ))
@@ -1260,7 +1262,8 @@ function Start-PSPester {
 
                     $passThruCommand = { & $powershell $PSFlags -c $command }
                     if ($Sudo.IsPresent) {
-                        $passThruCommand =  { & sudo $powershell $PSFlags -c $command }
+                        # -E says to preserve the environment
+                        $passThruCommand =  { & sudo -E $powershell $PSFlags -c $command }
                     }
 
                     $writeCommand = { Write-Host $_ }
@@ -1298,12 +1301,40 @@ function Start-PSPester {
         }
     }
 
+    Publish-TestResults -Path $OutputFile -Title $Title
+
     if($ThrowOnFailure)
     {
         Test-PSPesterResults -TestResultsFile $OutputFile
     }
 }
 
+function Publish-TestResults
+{
+    param(
+        [Parameter(Mandatory)]
+        [string]
+        $Title,
+
+        [Parameter(Mandatory)]
+        [ValidateScript({Test-Path -Path $_})]
+        [string]
+        $Path,
+
+        [ValidateSet('NUnit','XUnit')]
+        [string]
+        $Type='NUnit'
+    )
+
+    # In VSTS publish Test Results
+    if($env:TF_BUILD)
+    {
+        $resolvedPath = (Resolve-Path -Path $Path).ProviderPath
+        Write-Host "##vso[results.publish type=$Type;mergeResults=true;runTitle=$Title;publishRunAttachments=true;resultFiles=$resolvedPath;]"
+        Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$resolvedPath"
+    }
+}
+
 function script:Start-UnelevatedProcess
 {
     param(
@@ -1528,6 +1559,9 @@ function Start-PSxUnit {
 
         # Run sequential tests first, and then run the tests that can execute in parallel
         dotnet xunit -configuration $Options.configuration -xml $SequentialTestResultsFile -namespace "PSTests.Sequential" -parallel none
+
+        Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential'
+
         $extraParams = @()
 
         # we are having intermittent issues on macOS with these tests failing.
@@ -1542,6 +1576,7 @@ function Start-PSxUnit {
         }
 
         dotnet xunit -configuration $Options.configuration -xml $ParallelTestResultsFile -namespace "PSTests.Parallel" -nobuild @extraParams
+        Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel'
     }
     finally {
         Pop-Location

test/powershell/Modules/Microsoft.PowerShell.Management/New-Item.Tests.ps1

@@ -1,5 +1,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+
+Import-Module HelpersCommon
+
 function Clean-State
 {
     if (Test-Path $FullyQualifiedLink)
@@ -263,7 +266,7 @@ Describe "New-Item with links fails for non elevated user." -Tags "CI" {
         $FullyQualifiedFolder = Join-Path -Path $tmpDirectory -ChildPath $testfolder
     }
 
-    It "Should error correctly when failing to create a symbolic link" {
+    It "Should error correctly when failing to create a symbolic link" -Skip:(Test-IsRoot) {
         # This test expects that /sbin exists but is not writable by the user
         { New-Item -ItemType SymbolicLink -Path "/sbin/powershell-test" -Target $FullyQualifiedFolder -ErrorAction Stop } |
         Should -Throw -ErrorId "NewItemSymbolicLinkElevationRequired,Microsoft.PowerShell.Commands.NewItemCommand"

test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+Import-Module HelpersCommon
+
 Describe "Test-Connection" -tags "CI" {
     BeforeAll {
         $oldInformationPreference = $InformationPreference
@@ -42,9 +44,7 @@ Describe "Test-Connection" -tags "CI" {
             $replies[0].Address      | Should -BeExactly $targetAddressIPv6
             $replies[0].Status       | Should -BeExactly "Success"
             # TODO: Here and below we skip the check on Unix because .Net Core issue
-            if (!$isWindows) {
-                $replies[0].Buffer.Count | Should -Be 0
-            } else {
+            if ($isWindows) {
                 $replies[0].Buffer.Count | Should -Be 32
             }
         }
@@ -78,19 +78,19 @@ Describe "Test-Connection" -tags "CI" {
             }
         }
 
-        It "Force IPv4 with implicit PingOptions" {
+        # In VSTS, address is 0.0.0.0
+        It "Force IPv4 with implicit PingOptions" -Skip:(Test-IsVstsLinux) {
             $result = Test-Connection $realName -Count 1 -IPv4
 
             $result.Replies[0].Address              | Should -BeExactly $realAddress
             $result.Replies[0].Options.Ttl          | Should -BeLessThan 128
-            if (!$isWindows) {
-                $result.Replies[0].Options.DontFragment | Should -BeNullOrEmpty
-            } else {
+            if ($isWindows) {
                 $result.Replies[0].Options.DontFragment | Should -BeFalse
             }
         }
 
-        It "Force IPv4 with explicit PingOptions" {
+        # In VSTS, address is 0.0.0.0
+        It "Force IPv4 with explicit PingOptions" -Skip:(Test-IsVstsLinux) {
             $result1 = Test-Connection $realName -Count 1 -IPv4 -MaxHops 10 -DontFragment
 
             $result2 = Test-Connection $realName -Count 1 -IPv4 -MaxHops 1 -DontFragment
@@ -147,12 +147,10 @@ Describe "Test-Connection" -tags "CI" {
             { Test-Connection $targetName -BufferSize 65501 } | Should -Throw -ErrorId "ParameterArgumentValidationError,Microsoft.PowerShell.Commands.TestConnectionCommand"
         }
 
-        It "BufferSize works" {
+        It "BufferSize works" -Pending:(!$IsWindows) {
             $result = Test-Connection $targetName -Count 1 -BufferSize 2
 
-            if (!$isWindows) {
-                $result.Replies[0].Buffer.Count | Should -Be 0
-            } else {
+            if ($isWindows) {
                 $result.Replies[0].Buffer.Count | Should -Be 2
             }
         }
@@ -191,9 +189,7 @@ Describe "Test-Connection" -tags "CI" {
             $result.Count           | Should -BeGreaterThan 4
             $result[0].Address      | Should -BeExactly $targetAddress
             $result[0].Status       | Should -BeExactly "Success"
-            if (!$isWindows) {
-                $result[0].Buffer.Count | Should -Be 0
-            } else {
+            if ($isWindows) {
                 $result[0].Buffer.Count | Should -Be 32
             }
         }
@@ -219,7 +215,8 @@ Describe "Test-Connection" -tags "CI" {
     }
 
     Context "TraceRoute" {
-        It "TraceRoute works" {
+        # Hangs in VSTS Linux
+        It "TraceRoute works" -skip:(Test-IsVstsLinux) {
             $result = Test-Connection $realName -TraceRoute
             $replies = $result.Replies
             # Check target host reply.
@@ -245,19 +242,20 @@ Describe "Test-Connection" -tags "CI" {
             }
         }
 
-        It "Quiet works" {
+        # Hangs in VSTS Linux
+        It "Quiet works" -skip:(Test-IsVstsLinux) {
             $result = Test-Connection $realName -TraceRoute -Quiet
 
             $result | Should -BeTrue
         }
     }
-    
+
     Context "Connection" {
         BeforeAll {
             # Ensure the local host listen on port 80
             $WebListener = Start-WebListener
         }
-        
+
         It "Test connection to local host port 80" {
             Test-Connection '127.0.0.1' -TCPPort $WebListener.HttpPort | Should -BeTrue
         }
@@ -266,4 +264,4 @@ Describe "Test-Connection" -tags "CI" {
             Test-Connection $UnreachableAddress -TCPPort 80 -TimeOut 1 | Should -BeFalse
         }
     }
-} 
+}

test/powershell/Modules/Microsoft.PowerShell.Utility/Export-Alias.Tests.ps1

@@ -1,5 +1,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+
+Import-Module HelpersCommon
+
 Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
 
 	BeforeAll {
@@ -94,7 +97,7 @@ Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
 		$fulltestpath| Should -FileContentMatchExactly '"abcd02","efgh02","","None"'
     }
 
-	It "Export-Alias for Force ReadOnly Test" {
+	It "Export-Alias for Force ReadOnly Test" -Skip:(Test-IsRoot) {
 		Export-Alias $fulltestpath abcd01
 		if ( $IsWindows )
 		{