Use arcade's version of dotnet to build (dotnet#22755)

* Use arcade dotnet * Add cmake_msbuild.cmd Move msbuild.cmd to cmake_msbuild.bat Document intent that this file is only used to resolve Windows cmake dependency on desktop msbuild.exe Remove one instance of msbuild.cmd * Fix inittools.cmd * Remove spurious setup_vs_tools.cmd calls
EgorBo · Mar 2, 2019 · 42bc2de · 42bc2de
1 parent c1801e8
commit 42bc2de
Show file tree

Hide file tree

Showing 31 changed files with 147 additions and 224 deletions.
diff --git a/Documentation/building/unix-test-instructions.md b/Documentation/building/unix-test-instructions.md
@@ -24,17 +24,13 @@ Please note that this builds the Priority 0 tests. To build priority 1:
 
 During development there are many instances where building an individual test is fast and necessary. All of the necessary tools to build are under `coreclr/Tools`. It is possible to use `coreclr/Tools/MSBuild.dll` as you would normally use MSBuild with a few caveats.
 
-Note that `coreclr/Tools/msbuild.sh` exists as well to make the call shorter.
+Note that `coreclr/dotnet.sh` exists as well to make the call shorter.
 
 **!! Note !! -- Passing /p:__BuildOs=[OSX|Linux] is required.** 
 
 ## Building an Individual Test Example
 
->`coreclr/Tools/msbuild.sh /maxcpucount  coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
-
-Or
-
->`coreclr/Tools/dotnetcli/dotnet coreclr/Tools/MSBuild.dll /maxcpucount coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
+>`coreclr/dotnet.sh msbuild /maxcpucount  coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
 
 ## Aarch64/armhf multi-arch
 

diff --git a/Documentation/workflow/RunningTests.md b/Documentation/workflow/RunningTests.md
@@ -31,12 +31,10 @@ Note: The CoreCLR must be built prior to building an individual test. See first
 
 ### Examples
 
-* Using the `msbuild.sh` script
-  * `coreclr/Tools/msbuild.sh /maxcpucount coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
-* Calling `MSBuild.dll` directly
-  * `coreclr/Tools/dotnetcli/dotnet coreclr/Tools/MSBuild.dll /maxcpucount coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
+* Using the `dotnet.sh` script
+  * `coreclr/dotnet.sh msbuild /maxcpucount coreclr/tests/src/JIT/CodeGenBringUpTests/Array1.csproj /p:__BuildType=Release /p:__BuildOS=OSX`
 
 ## Additional Documents
 
 * [Windows](https://github.com/dotnet/coreclr/blob/master/Documentation/building/windows-test-instructions.md)
-* [Non-Windows](https://github.com/dotnet/coreclr/blob/master/Documentation/building/unix-test-instructions.md)
+* [Non-Windows](https://github.com/dotnet/coreclr/blob/master/Documentation/building/unix-test-instructions.md)
diff --git a/build-packages.cmd b/build-packages.cmd
@@ -33,6 +33,11 @@ if [!processedArgs!]==[] (
 
 :ArgsDone
 
+call "%__ProjectDir%"\setup_vs_tools.cmd
+
+REM setup_vs_tools.cmd will correctly echo error message.
+if NOT '%ERRORLEVEL%' == '0' exit /b 1
+
 call %__ProjectDir%/dotnet.cmd msbuild /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
   /p:__BuildOS=Windows_NT /flp:v=detailed;Append;LogFile=build-packages.log^
   /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^

diff --git a/build-test.cmd b/build-test.cmd
@@ -224,7 +224,7 @@ set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%"
 set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"
 set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr!
 
-call "%__ProjectDir%\msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
+call "%__ProjectDir%\cmake_msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
   /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^
   /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^
   /p:UsePartialNGENOptimization=false /maxcpucount^

diff --git a/build-test.sh b/build-test.sh
@@ -64,7 +64,6 @@ build_test_wrappers()
         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 
         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
-        echo "$nextCommand"
         eval $nextCommand
 
         if [ $? -ne 0 ]; then
@@ -587,7 +586,7 @@ __SourceDir="$__ProjectDir/src"
 __PackagesDir="$__ProjectDir/packages"
 __RootBinDir="$__ProjectDir/bin"
 __BuildToolsDir="$__ProjectDir/Tools"
-__DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
+__DotNetCli="$__ProjectDir/dotnet.sh"
 __UnprocessedBuildArgs=
 __CommonMSBuildArgs=
 __MSBCleanBuildArgs=
@@ -879,9 +878,6 @@ initTargetDistroRid
 
 # Override tool directory
 
-__CoreClrVersion=1.1.0
-__sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
-
 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
     build_Tests
 elif [ ! -z "$__BuildTestWrappersOnly" ]; then

diff --git a/build.cmd b/build.cmd
@@ -523,7 +523,7 @@ if %__BuildNative% EQU 1 (
     set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr!
     set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr!
 
-    call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
+    call %__ProjectDir%\cmake_msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
       /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^
       /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^
       /p:UsePartialNGENOptimization=false /maxcpucount %__IntermediatesDir%\install.vcxproj^
@@ -590,7 +590,7 @@ if %__BuildCrossArchNative% EQU 1 (
     set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr!
     set __Logging=!_MsbuildLog! !__MsbuildWrn! !__MsbuildErr!
 
-    call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
+    call %__ProjectDir%\cmake_msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
       /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^
       /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^
       /p:UsePartialNGENOptimization=false /maxcpucount^
@@ -849,8 +849,8 @@ if %__BuildPackages% EQU 1 (
     set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr!
 
     REM The conditions as to what to build are captured in the builds file.
-    call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
-      /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^
+    call %__ProjectDir%\dotnet.cmd msbuild /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^
+      /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^
       /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^
       /p:UsePartialNGENOptimization=false /maxcpucount^
       %__SourceDir%\.nuget\packages.builds^

diff --git a/build.sh b/build.sh
@@ -150,7 +150,7 @@ restore_optdata()
 
     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
-        local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
+        local DotNetCli="$__ProjectRoot/.dotnet/dotnet"
         if [ ! -f $DotNetCli ]; then
             source "$__ProjectRoot/init-tools.sh"
             if [ $? != 0 ]; then

diff --git a/msbuild.cmd → cmake_msbuild.cmd b/msbuild.cmd → cmake_msbuild.cmd
@@ -1,3 +1,7 @@
+:: Windows CMake has a dependency on desktop msbuild.exe because it generates
+:: Visual Studio *.vcxproj solutions.
+:: This file has cmake in its name to avoid accidentally introducing
+:: another dependency on desktop msbuild.exe
 @if not defined _echo @echo off
 setlocal
 
@@ -13,12 +17,6 @@ if NOT '%ERRORLEVEL%' == '0' exit /b 1
 set Platform=
 set __ProjectDir=
 
-:: Restore the Tools directory
-call %~dp0init-tools.cmd
-if NOT [%ERRORLEVEL%]==[0] (
-  exit /b 1
-)
-
 pushd %~dp0
 echo Running: msbuild.exe %*
 call msbuild.exe %*

diff --git a/dir.props b/dir.props
@@ -73,7 +73,7 @@
     <PackagesBinDir Condition="'$(__PackagesBinDir)'==''">$(BinDir).nuget\</PackagesBinDir>
 
     <ToolsDir Condition="'$(ToolsDir)'==''">$(ProjectDir)Tools\</ToolsDir>
-    <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(ToolsDir)dotnetcli\</DotnetCliPath>
+    <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(ProjectDir).dotnet\</DotnetCliPath>
     <OverrideToolHost>$(DotnetCliPath)dotnet</OverrideToolHost>
     <BuildToolsSemaphore Condition="'$(BuildToolsSemaphore)' == ''">$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll</BuildToolsSemaphore>
 

diff --git a/dotnet.cmd b/dotnet.cmd
@@ -3,11 +3,6 @@ setlocal
 
 set "__ProjectDir=%~dp0"
 
-call "%__ProjectDir%"\setup_vs_tools.cmd
-
-REM setup_vs_tools.cmd will correctly echo error message.
-if NOT '%ERRORLEVEL%' == '0' exit /b 1
-
 :: Clear the 'Platform' env variable for this session, as it's a per-project setting within the build, and
 :: misleading value (such as 'MCD' in HP PCs) may lead to build breakage (issue: #69).
 set Platform=
@@ -21,7 +16,7 @@ if NOT [%ERRORLEVEL%]==[0] (
 
 pushd %~dp0
 echo Running: dotnet %*
-call "%~dp0\Tools\dotnetcli\dotnet.exe" %*
+call "%~dp0\.dotnet\dotnet.exe" %*
 popd
 if NOT [%ERRORLEVEL%]==[0] (
   exit /b 1

diff --git a/dotnet.sh b/dotnet.sh
@@ -5,8 +5,7 @@ working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 echo "Running init-tools.sh"
 source $working_tree_root/init-tools.sh
 
-toolRuntime=$working_tree_root/Tools
-dotnet=$toolRuntime/dotnetcli/dotnet
+dotnet=$working_tree_root/.dotnet/dotnet
 
 echo "Running: $dotnet $@"
 $dotnet "$@"

diff --git a/eng/build-job.yml b/eng/build-job.yml
@@ -106,11 +106,6 @@ jobs:
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests -skipnuget $(clangArg) $(stripSymbolsArg) $(officialBuildIdArg)
         displayName: Build product
-        ${{ if eq(parameters.osGroup, 'FreeBSD') }}:
-          env:
-            # init-tools.sh extracts an SDK from here to the Tools/dotnetcli folder instead of trying to restore it.
-            # Once we are using Arcade, use DotNetCoreSdkDir instead, as we do below.
-            DotNetBootstrapCliTarPath: /dotnet-sdk-freebsd-x64.tar
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages $(officialBuildIdArg) $(ibcOptimizeArg) $(enforcePgoArg)
         displayName: Build product

diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -321,10 +321,12 @@ function LocateVisualStudio([object]$vsRequirements = $null){
 
 function InitializeBuildTool() {
   if (Test-Path variable:global:_BuildTool) {
+    Write-Host "variable:global:_BuildTool initialized." -ForegroundColor Red
     return $global:_BuildTool
   }
 
   if (-not $msbuildEngine) {
+    Write-Host "-not $msbuildEngine" -ForegroundColor Red
     $msbuildEngine = GetDefaultMSBuildEngine
   }
 

diff --git a/eng/configure-toolset.ps1 b/eng/configure-toolset.ps1
@@ -0,0 +1,5 @@
+# We depend on a local cli for a number of our buildtool
+# commands like init-tools so for now we need to disable
+# using the globally installed dotnet
+
+$script:useInstalledDotNetCli = $false
diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh
@@ -0,0 +1,5 @@
+# We depend on a local cli for a number of our buildtool
+# commands like init-tools so for now we need to disable
+# using the globally installed dotnet
+
+use_installed_dotnet_cli=false
diff --git a/eng/update-dependencies.cmd b/eng/update-dependencies.cmd
@@ -1,17 +1,7 @@
 @if not defined _echo @echo off
 setlocal
 
-:: Clear the 'Platform' env variable for this session, as it's a per-project setting within the build, and
-:: misleading value (such as 'MCD' in HP PCs) may lead to build breakage (issue: #69).
-set Platform=
-
-:: Restore the Tools directory
-call "%~dp0..\init-tools.cmd"
-if NOT [%ERRORLEVEL%]==[0] (
-  exit /b 1
-)
-
-set "DotNetCli=%~dp0..\Tools\dotnetcli\dotnet.exe"
+set "DotNetCli=%~dp0..\dotnet.cmd"
 set "DependenciesBuildProj=%~dp0..\tests\build.proj"
 
 echo Running: "%DotNetCli%" msbuild "%DependenciesBuildProj%" %*

diff --git a/init-tools.cmd b/init-tools.cmd
@@ -4,7 +4,7 @@ setlocal
 set INIT_TOOLS_LOG=%~dp0init-tools.log
 if [%PACKAGES_DIR%]==[] set PACKAGES_DIR=%~dp0packages
 if [%TOOLRUNTIME_DIR%]==[] set TOOLRUNTIME_DIR=%~dp0Tools
-set DOTNET_PATH=%TOOLRUNTIME_DIR%\dotnetcli\
+set DOTNET_PATH=%~dp0.dotnet\
 if [%DOTNET_CMD%]==[] set DOTNET_CMD=%DOTNET_PATH%dotnet.exe
 if [%BUILDTOOLS_SOURCE%]==[] set BUILDTOOLS_SOURCE=https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json
 set /P BUILDTOOLS_VERSION=< "%~dp0BuildToolsVersion.txt"
@@ -72,15 +72,14 @@ echo "init-tools.cmd: Setting arch to %_Arch% for build tools"
 :ArchSet
 
 echo Installing dotnet cli...
-if NOT exist "%DOTNET_PATH%" mkdir "%DOTNET_PATH%"
-set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-%_Arch%.zip
-set DOTNET_REMOTE_PATH=https://dotnetcli.azureedge.net/dotnet/Sdk/%DOTNET_VERSION%/%DOTNET_ZIP_NAME%
-set DOTNET_LOCAL_PATH=%DOTNET_PATH%%DOTNET_ZIP_NAME%
-echo Installing '%DOTNET_REMOTE_PATH%' to '%DOTNET_LOCAL_PATH%' >> "%INIT_TOOLS_LOG%"
-powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; $proxyCredentialsRequired = $false; do { try { $wc = New-Object Net.WebClient; if ($proxyCredentialsRequired) { [Net.WebRequest]::DefaultWebProxy.Credentials = [Net.CredentialCache]::DefaultNetworkCredentials; } $wc.DownloadFile('%DOTNET_REMOTE_PATH%', '%DOTNET_LOCAL_PATH%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $we = $_.Exception.InnerException -as [Net.WebException]; $proxyCredentialsRequired = ($we -ne $null -and ([Net.HttpWebResponse]$we.Response).StatusCode -eq [Net.HttpStatusCode]::ProxyAuthenticationRequired); Start-Sleep -Seconds (5 * $retryCount); $retryCount++; } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%DOTNET_LOCAL_PATH%', '%DOTNET_PATH%') } else { (New-Object -com shell.application).namespace('%DOTNET_PATH%').CopyHere((new-object -com shell.application).namespace('%DOTNET_LOCAL_PATH%').Items(),16) }" >> "%INIT_TOOLS_LOG%"
-if NOT exist "%DOTNET_LOCAL_PATH%" (
-  echo ERROR: Could not install dotnet cli correctly. 1>&2
-  goto :error
+set PS_DOTNET_INSTALL_SCRIPT=". %~dp0eng\configure-toolset.ps1; . %~dp0eng\common\tools.ps1; InitializeBuildTool"
+if NOT exist "%DOTNET_CMD%" (
+  echo running: powershell -NoProfile -ExecutionPolicy unrestricted -Command %PS_DOTNET_INSTALL_SCRIPT%
+  powershell -NoProfile -ExecutionPolicy unrestricted -Command %PS_DOTNET_INSTALL_SCRIPT%
+  if NOT exist "%DOTNET_CMD%" (
+    echo ERROR: Could not install dotnet cli correctly. 1>&2
+    goto :error
+  )
 )
 
 :afterdotnetrestore