Make functional tests behave like libraries tests

eng/testing/AndroidRunnerTemplate.sh

@@ -9,6 +9,10 @@ TARGET_OS=$3
 TEST_NAME=$4
 XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
 
+if [ -n "$5" ]; then
+    EXPECTED_EXIT_CODE="--expected-exit-code $5"
+fi
+
 cd $EXECUTION_DIR
 
 # it doesn't support parallel execution yet, so, here is a hand-made semaphore:
@@ -35,7 +39,8 @@ $HARNESS_RUNNER android test                \
     --instrumentation="net.dot.MonoRunner"  \
     --package-name="net.dot.$ASSEMBLY_NAME" \
     --app="$EXECUTION_DIR/bin/$TEST_NAME.apk" \
-    --output-directory="$XHARNESS_OUT"
+    --output-directory="$XHARNESS_OUT" \
+    $EXPECTED_EXIT_CODE
 
 _exitCode=$?
 

eng/testing/AppleRunnerTemplate.sh

@@ -7,9 +7,15 @@ ASSEMBLY_NAME=$1
 TARGET_ARCH=$2
 TARGET_OS=$3
 TEST_NAME=$4
+XHARNESS_CMD="test"
 XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
 XCODE_PATH=$(xcode-select -p)/../..
 
+if [ -n "$5" ]; then
+    XHARNESS_CMD="run"
+    EXPECTED_EXIT_CODE="--expected-exit-code $5"
+fi
+
 if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "x86" ]]; then TARGET=ios-simulator-32; fi
 if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "x64" ]]; then TARGET=ios-simulator-64; fi
 if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "arm" ]]; then TARGET=ios-device; fi
@@ -46,11 +52,12 @@ else
     HARNESS_RUNNER="dotnet xharness"
 fi
 
-$HARNESS_RUNNER ios test    \
+$HARNESS_RUNNER ios $XHARNESS_CMD    \
     --app="$EXECUTION_DIR/$TEST_NAME/$SCHEME_SDK/$TEST_NAME.app" \
     --targets="$TARGET" \
     --xcode="$XCODE_PATH"   \
-    --output-directory="$XHARNESS_OUT"
+    --output-directory="$XHARNESS_OUT" \
+    $EXPECTED_EXIT_CODE
 
 _exitCode=$?
 

eng/testing/tests.mobile.targets

@@ -38,6 +38,8 @@
       <AndroidAbi Condition="'$(TargetArchitecture)' == 'arm'">armeabi-v7a</AndroidAbi>
       <AndroidAbi Condition="'$(TargetArchitecture)' == 'x64'">x86_64</AndroidAbi>
       <AndroidAbi Condition="'$(TargetArchitecture)' == 'x86'">x86</AndroidAbi>
+
+      <MainLibraryFileName Condition="'$(MainLibraryFileName)' == ''">AndroidTestRunner.dll</MainLibraryFileName>
     </PropertyGroup>
 
     <Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
@@ -54,7 +56,7 @@
         RuntimeIdentifier="$(RuntimeIdentifier)"
         ProjectName="$(AssemblyName)"
         MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackNativeDir)include\mono-2.0"
-        MainLibraryFileName="AndroidTestRunner.dll"
+        MainLibraryFileName="$(MainLibraryFileName)"
         ForceInterpreter="$(MonoForceInterpreter)"
         StripDebugSymbols="False"
         OutputDir="$(BundleDir)"
@@ -92,6 +94,7 @@
 
     <PropertyGroup>
       <Optimized>true</Optimized>
+      <MainLibraryFileName Condition="'$(MainLibraryFileName)' == ''">AppleTestRunner.dll</MainLibraryFileName>
     </PropertyGroup>
     <ItemGroup>
       <AotInputAssemblies Condition="'$(RunAOTCompilation)' == 'true'" Include="$(PublishDir)*.dll" Exclude="$(PublishDir)System.Runtime.WindowsRuntime.dll">
@@ -122,7 +125,8 @@
         ProjectName="$(AssemblyName)"
         MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackNativeDir)include\mono-2.0"
         Assemblies="@(BundleAssemblies)"
-        MainLibraryFileName="AppleTestRunner.dll"
+        MainLibraryFileName="$(MainLibraryFileName)"
+        ForceAOT="$(RunAOTCompilation)"
         ForceInterpreter="$(MonoForceInterpreter)"
         UseConsoleUITemplate="True"
         GenerateXcodeProject="True"
@@ -184,7 +188,7 @@
     <PropertyGroup>
       <WasmAppDir>$(BundleDir)</WasmAppDir>
       <WasmBuildDir>$(PublishDir)</WasmBuildDir>
-      <WasmMainAssemblyPath>$(PublishDir)WasmTestRunner.dll</WasmMainAssemblyPath>
+      <WasmMainAssemblyPath Condition="'$(WasmMainAssemblyPath)' == ''">$(PublishDir)WasmTestRunner.dll</WasmMainAssemblyPath>
       <WasmMainJSPath>$(MonoProjectRoot)\wasm\runtime-test.js</WasmMainJSPath>
       <WasmInvariantGlobalization>$(InvariantGlobalization)</WasmInvariantGlobalization>
       <WasmGenerateRunV8Script>true</WasmGenerateRunV8Script>

eng/testing/tests.targets

@@ -96,7 +96,7 @@
       <RunTestsCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and
                                   $([MSBuild]::VersionGreaterThanOrEquals($(TargetFrameworkVersion), '$(NETCoreAppCurrentVersion)'))">$(RunTestsCommand) --runtime-path "$(TestHostRootPath.TrimEnd('\/'))"</RunTestsCommand>
       <RunTestsCommand Condition="'$(TestRspFile)' != '' and '$(RuntimeFlavor)' != 'Mono'">$(RunTestsCommand) --rsp-file "$(TestRspFile)"</RunTestsCommand>
-      <RunTestsCommand Condition="'$(TargetsMobile)' == 'true'">"$(RunScriptOutputPath)" $(AssemblyName) $(TargetArchitecture) $(TargetOS) $(TestProjectName)</RunTestsCommand>
+      <RunTestsCommand Condition="'$(TargetsMobile)' == 'true'">"$(RunScriptOutputPath)" $(AssemblyName) $(TargetArchitecture) $(TargetOS) $(TestProjectName) $(ExpectedExitCode)</RunTestsCommand>
       <RunTestsCommand Condition="'$(TargetOS)' == 'Browser'">"$(RunScriptOutputPath)" $(JSEngine) $(AssemblyName).dll $(_withoutCategories.Replace(';', ' -notrait category='))</RunTestsCommand>
     </PropertyGroup>
 

src/libraries/Directory.Build.props

@@ -35,7 +35,7 @@
 
   <!-- Define test projects and companions -->
   <PropertyGroup>
-    <IsTestProject>false</IsTestProject>
+    <IsTestProject Condition="'$(IsTestProject)' == ''">false</IsTestProject>
     <IsTestProject Condition="$(MSBuildProjectName.EndsWith('.UnitTests')) or $(MSBuildProjectName.EndsWith('.Tests'))">true</IsTestProject>
 
     <IsTestSupportProject>false</IsTestSupportProject>

src/tests/FunctionalTests/Android/Emulator/Interpreter/Android.Emulator.Interpreter.Test.csproj

@@ -1,81 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
-  <Import Project="..\..\common.props" />
-
-  <!-- Redirect 'dotnet publish' to in-tree runtime pack -->
-  <Target Name="TrickRuntimePackLocation" AfterTargets="ProcessFrameworkReferences">
-    <ItemGroup>
-      <RuntimePack>
-        <PackageDirectory>$(ArtifactsBinDir)microsoft.netcore.app.runtime.$(RuntimeIdentifier)\$(Configuration)</PackageDirectory>
-      </RuntimePack>
-    </ItemGroup>
-    <Message Text="Packaged ID: %(RuntimePack.PackageDirectory)" Importance="high" />
-  </Target>
-
-  <UsingTask TaskName="AndroidAppBuilderTask" AssemblyFile="$(AndroidAppBuilderTasksAssemblyPath)"/>
-
-  <Target Name="BuildApp" AfterTargets="CopyFilesToPublishDirectory">
-    <PropertyGroup>
-      <StripDebugSymbols>False</StripDebugSymbols>
-      <StripDebugSymbols Condition="'$(Configuration)' == 'Release'">True</StripDebugSymbols>
-      <AdbTool>$(ANDROID_SDK_ROOT)\platform-tools\adb</AdbTool>
-      <ApkDir>$(PublishDir)apk\</ApkDir>
-    </PropertyGroup>
-
-    <RemoveDir Directories="$(ApkDir)" />
-
-    <PropertyGroup>
-      <AndroidAbi Condition="'$(TargetArchitecture)' == 'arm64'">arm64-v8a</AndroidAbi>
-      <AndroidAbi Condition="'$(TargetArchitecture)' == 'arm'">armeabi-v7a</AndroidAbi>
-      <AndroidAbi Condition="'$(TargetArchitecture)' == 'x64'">x86_64</AndroidAbi>
-      <AndroidAbi Condition="'$(TargetArchitecture)' == 'x86'">x86</AndroidAbi>
-    </PropertyGroup>
-
-    <!-- TODO: delete once we switch to Android Crypto -->
-    <Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
-          SourceFiles="$(ANDROID_OPENSSL_AAR)\prefab\modules\crypto\libs\android.$(AndroidAbi)\libcrypto.so"
-          DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true"/>
-    <Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
-          SourceFiles="$(ANDROID_OPENSSL_AAR)\prefab\modules\ssl\libs\android.$(AndroidAbi)\libssl.so"
-          DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true"/>
-
-    <AndroidAppBuilderTask
-        RuntimeIdentifier="$(RuntimeIdentifier)"
-        SourceDir="$(PublishDir)"
-        ProjectName="Android.Emulator.Interpreter.Test"
-        ForceInterpreter="True"
-        MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackDir)\native\include\mono-2.0"
-        MainLibraryFileName="Android.Emulator.Interpreter.Test.dll"
-        StripDebugSymbols="$(StripDebugSymbols)"
-        OutputDir="$(ApkDir)">
-        <Output TaskParameter="ApkBundlePath" PropertyName="ApkBundlePath" />
-        <Output TaskParameter="ApkPackageId" PropertyName="ApkPackageId" />
-    </AndroidAppBuilderTask>
-
-    <Message Importance="High" Text="Apk:       $(ApkBundlePath)"/>
-    <Message Importance="High" Text="PackageId: $(ApkPackageId)"/>
-  </Target>
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <MonoForceInterpreter>true</MonoForceInterpreter>
+    <RunAOTCompilation>false</RunAOTCompilation>
+    <TestRuntime>true</TestRuntime>
+    <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
+    <MainLibraryFileName>Android.Emulator.Interpreter.Test.dll</MainLibraryFileName>
+    <ExpectedExitCode>42</ExpectedExitCode>
+  </PropertyGroup>
 
   <ItemGroup>
     <Compile Include="Program.cs" />
   </ItemGroup>
-
-  <Target Name="CopySampleAppToHelixTestDir" 
-          AfterTargets="Build"
-          DependsOnTargets="Publish;BuildApp" >
-    <PropertyGroup>
-      <OSPlatformConfig>$(TargetOS).AnyCPU.$(Configuration)</OSPlatformConfig>
-      <HelixArchiveRoot>$(ArtifactsDir)helix/</HelixArchiveRoot>
-      <HelixArchiveRunOnlyRoot>$(HelixArchiveRoot)runonly/</HelixArchiveRunOnlyRoot>
-      <HelixArchiveRunOnlyAppsDir>$(HelixArchiveRunOnlyRoot)$(OSPlatformConfig)/</HelixArchiveRunOnlyAppsDir>
-    </PropertyGroup>
-    <ItemGroup>
-      <_apkFiles Include="$(ApkBundlePath)" />
-    </ItemGroup>
-    <Copy SourceFiles="@(_apkFiles)"
-          DestinationFolder="$(HelixArchiveRunOnlyAppsDir)/%(RecursiveDir)" />
-
-    <Message Importance="High" Text="ApkBundlePath: $(ApkBundlePath)"/>
-    <Message Importance="High" Text="TestArchiveTestsDir: $(TestArchiveTestsDir)"/>
-  </Target>
 </Project>

src/tests/FunctionalTests/Android/Emulator/JIT/Android.Emulator.JIT.Test.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <RunAOTCompilation>false</RunAOTCompilation>
+    <TestRuntime>true</TestRuntime>
+    <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
+    <MainLibraryFileName>Android.Emulator.JIT.Test.dll</MainLibraryFileName>
+    <ExpectedExitCode>42</ExpectedExitCode>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+</Project>

src/tests/FunctionalTests/Android/Emulator/JIT/Program.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+public class Program
+{
+    public static int Main(string[] args)
+    {
+        Console.WriteLine("Hello, Android!"); // logcat
+        return 42;
+    }
+}

src/tests/FunctionalTests/Directory.Build.props

@@ -3,7 +3,9 @@
     <RunAnalyzers>false</RunAnalyzers>
     <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsTestProject>true</IsTestProject>
+    <IsFunctionalTest>true</IsFunctionalTest>
   </PropertyGroup>
 
-  <Import Project="..\..\mono\Directory.Build.props" />
+  <Import Project="..\..\libraries\Directory.Build.props" />
 </Project>

src/tests/FunctionalTests/Directory.Build.targets

@@ -1,3 +1,3 @@
 <Project>
-  <Import Project="..\..\mono\Directory.Build.targets" />
+  <Import Project="..\..\libraries\Directory.Build.targets" />
 </Project>

src/tests/FunctionalTests/iOS/Simulator/AOT/iOS.Simulator.Aot.Test.csproj

@@ -1,66 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <Import Project="..\..\common.props" />
-
-  <Import Project="$(RepoTasksDir)AotCompilerTask\MonoAOTCompiler.props" />
-  <UsingTask TaskName="AppleAppBuilderTask"
-             AssemblyFile="$(AppleAppBuilderTasksAssemblyPath)" />
-
-  <UsingTask TaskName="MonoAOTCompiler"
-             AssemblyFile="$(MonoAOTCompilerTasksAssemblyPath)" />
-
-  <Target Name="BuildAppBundle" AfterTargets="CopyFilesToPublishDirectory">
-    <PropertyGroup>
-      <AppDir>$(PublishDir)\app</AppDir>
-      <IosSimulator Condition="'$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'x86'">iPhone 11</IosSimulator>
-      <Optimized Condition="'$(Configuration)' == 'Release'">True</Optimized>
-    </PropertyGroup>
-
-    <RemoveDir Directories="$(AppDir)" />
-
-    <ItemGroup>
-      <AotInputAssemblies Include="$(PublishDir)\*.dll">
-        <AotArguments>@(MonoAOTCompilerDefaultAotArguments, ';')</AotArguments>
-        <ProcessArguments>@(MonoAOTCompilerDefaultProcessArguments, ';')</ProcessArguments>
-      </AotInputAssemblies>
-    </ItemGroup>
-
-    <Message Importance="High" Text="Output publish dir: $(PublishDir)"/>
-
-    <MonoAOTCompiler
-        CompilerBinaryPath="$(MicrosoftNetCoreAppRuntimePackDir)native\cross\mono-aot-cross"
-        Mode="Full"
-        OutputType="AsmOnly"
-        Assemblies="@(AotInputAssemblies)"
-        AotModulesTablePath="$(AppDir)\modules.m"
-        AotModulesTableLanguage="ObjC"
-        UseLLVM="True"
-        LLVMPath="$(MicrosoftNetCoreAppRuntimePackDir)native\cross">
-        <Output TaskParameter="CompiledAssemblies" ItemName="BundleAssemblies" />
-    </MonoAOTCompiler>
-
-    <AppleAppBuilderTask
-        TargetOS="$(TargetOS)"
-        Arch="$(TargetArchitecture)"
-        ProjectName="iOS.Simulator.Aot.Test"
-        MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackDir)native\include\mono-2.0"
-        Assemblies="@(BundleAssemblies)"
-        MainLibraryFileName="iOS.Simulator.Aot.Test.dll"
-        GenerateXcodeProject="True"
-        BuildAppBundle="True"
-        DevTeamProvisioning="$(DevTeamProvisioning)"
-        OutputDirectory="$(AppDir)"
-        Optimized="$(Optimized)"
-        ForceAOT="True"
-        ForceInterpreter="$(MonoForceInterpreter)"
-        AppDir="$(PublishDir)">
-        <Output TaskParameter="AppBundlePath" PropertyName="AppBundlePath" />
-        <Output TaskParameter="XcodeProjectPath" PropertyName="XcodeProjectPath" />
-    </AppleAppBuilderTask>
-
-    <Message Importance="High" Text="Xcode: $(XcodeProjectPath)"/>
-    <Message Importance="High" Text="App:   $(AppBundlePath)"/>
-  </Target>
-
-  <Import Project="..\..\common.targets" />
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <MonoForceInterpreter>true</MonoForceInterpreter>
+    <RunAOTCompilation>true</RunAOTCompilation>
+    <TestRuntime>true</TestRuntime>
+    <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
+    <MainLibraryFileName>iOS.Simulator.AOT.Test.dll</MainLibraryFileName>
+    <IncludesTestRunner>false</IncludesTestRunner>
+    <ExpectedExitCode>42</ExpectedExitCode>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+  </ItemGroup>
 </Project>