[mono] Add LLVM support for iOS test runner; use runtime packs (dotne…

…t#35106)

eng/Version.Details.xml

@@ -174,9 +174,9 @@
       <Uri>https://github.com/mono/linker</Uri>
       <Sha>8caef57d1f3bc7a188e5dd26d43a2d34151223f9</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XHarness.Tests.Runners" Version="1.0.0-prerelease.20214.2">
+    <Dependency Name="Microsoft.DotNet.XHarness.Tests.Runners" Version="1.0.0-prerelease.20220.2">
       <Uri>https://github.com/dotnet/xharness</Uri>
-      <Sha>b5bfc4fcd431ab1fd560bdea7642c7998e30f24d</Sha>
+      <Sha>e9223bfb694b252b44a4e5a0c72afa016f676c5e</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -115,7 +115,7 @@
     <SystemDataSqlClientVersion>4.8.0</SystemDataSqlClientVersion>
     <!-- Testing -->
     <MicrosoftNETTestSdkVersion>16.7.0-preview-20200416-02</MicrosoftNETTestSdkVersion>
-    <MicrosoftDotNetXHarnessTestsRunnersVersion>1.0.0-prerelease.20214.2</MicrosoftDotNetXHarnessTestsRunnersVersion>
+    <MicrosoftDotNetXHarnessTestsRunnersVersion>1.0.0-prerelease.20220.2</MicrosoftDotNetXHarnessTestsRunnersVersion>
     <XUnitVersion>2.4.1</XUnitVersion>
     <TraceEventVersion>2.0.5</TraceEventVersion>
     <NewtonsoftJsonVersion>12.0.3</NewtonsoftJsonVersion>

src/mono/mono.proj

@@ -943,19 +943,20 @@
     <PropertyGroup>
       <TestDir>$(ArtifactsDir)bin\$(TestName)\$(NetCoreAppCurrent)-$(Configuration)</TestDir>
       <TestDir Condition="!Exists('$(TestDir)')">$(ArtifactsDir)bin\$(TestName)\$(NetCoreAppCurrent)-Unix-$(Configuration)</TestDir>
-      <BclDir>$(ArtifactsDir)bin\runtime\$(NetCoreAppCurrent)-$(TargetOS)-$(Configuration)-$(TargetArchitecture)</BclDir>
+      <RuntimePackDir>$(ArtifactsDir)bin\lib-runtime-packs\runtimes\ios-$(TargetArchitecture)</RuntimePackDir>
       <BundleDir>$(TestDir)\Bundle</BundleDir>
     </PropertyGroup>
     <ItemGroup>
       <TestBinaries Include="$(TestDir)\*.*"/>
       <AppleTestRunnerBinaries Include="msbuild\AppleTestRunner\bin\*.*" />
-      <BclBinaries Include="$(BclDir)\*.*" Exclude="$(BclDir)\System.Runtime.WindowsRuntime.dll" />
+      <BclBinaries Include="$(RuntimePackDir)\lib\$(NetCoreAppCurrent)\*.*" Exclude="$(RuntimePackDir)\lib\$(NetCoreAppCurrent)\System.Runtime.WindowsRuntime.dll" />
+      <BclBinaries Include="$(RuntimePackDir)\native\*.*" />
       <MonoRuntimeBinaries Include="$(BinDir)\*.*" />
     </ItemGroup>
 
     <Error Condition="'$(TestName)' == ''" Text="TestName is not set" />
     <Error Condition="!Exists('$(TestDir)')" Text="TestDir=$(TestDir) doesn't exist" />
-    <Error Condition="!Exists('$(BclDir)')" Text="BclDir=$(BclDir) doesn't exist" />
+    <Error Condition="!Exists('$(RuntimePackDir)')" Text="RuntimePackDir=$(RuntimePackDir) doesn't exist" />
 
     <RemoveDir Directories="$(BundleDir)" />
 
@@ -964,11 +965,25 @@
     <Copy SourceFiles="@(BclBinaries)" DestinationFolder="$(BundleDir)\%(RecursiveDir)" SkipUnchangedFiles="true"/>
     <Copy SourceFiles="@(MonoRuntimeBinaries)" DestinationFolder="$(BundleDir)\%(RecursiveDir)" SkipUnchangedFiles="true"/>
 
+    <!-- xharness expects 'xunit-excludes.txt' with xunit categories to ignore in the bundle -->
+    <ItemGroup>
+      <XunitIgnoreCategories Include="-notrait category=OuterLoop"/>
+      <XunitIgnoreCategories Include="-notrait category=failing"/>
+      <XunitIgnoreCategories Include="-notrait category=nonnetcoreapptests"/>
+      <XunitIgnoreCategories Include="-notrait category=nonosxtests"/>
+      <XunitIgnoreCategories Include="-notrait category=nonmonotests"/>
+    </ItemGroup>
+    <WriteLinesToFile
+        File="$(BundleDir)\xunit-excludes.txt"
+        Lines="@(XunitIgnoreCategories)"
+        Overwrite="true"
+        Encoding="Unicode"/>
+
    <AppleAppBuilderTask 
         Arch="$(TargetArchitecture)"
         ProjectName="$(TestName)"
-        MonoRuntimeHeaders="$(BinDir)include\mono-2.0"
-        CrossCompiler="$(BinDir)cross\mono-aot-cross"
+        MonoRuntimeHeaders="$(RuntimePackDir)\native\include\mono-2.0"
+        CrossCompiler="$(RuntimePackDir)\native\cross\mono-aot-cross"
         MainLibraryFileName="AppleTestRunner.dll"
         UseConsoleUITemplate="True"
         GenerateXcodeProject="True"
@@ -991,7 +1006,7 @@
     <Exec Condition="'$(TargetArchitecture)' == 'x64'" Command="xcrun simctl boot &quot;iPhone 11&quot;" />
     <Exec Condition="'$(TargetArchitecture)' == 'x64' and '$(ShowIosSimulator)' != 'false'" Command="open -a Simulator" />
     <Exec Condition="'$(TargetArchitecture)' == 'x64'" Command="xcrun simctl install &quot;iPhone 11&quot; $(AppBundlePath)" />
-    <Exec Condition="'$(TargetArchitecture)' == 'x64'" Command="xcrun simctl launch --console booted net.dot.$(TestName) testlib:$(TestName).dll" />
+    <Exec Condition="'$(TargetArchitecture)' == 'x64'" Command="xcrun simctl launch --console booted net.dot.$(TestName) testlib:$(TestName).dll --auto-exit" />
   </Target>
 
   <!-- Ordering matters! Overwriting the Build target. -->

src/mono/msbuild/AppleAppBuilder/AotCompiler.cs

@@ -20,11 +20,13 @@ public static void PrecompileLibraries(
         string binDir,
         string[] libsToPrecompile,
         IDictionary<string, string> envVariables,
-        bool optimize)
+        bool optimize,
+        bool useLlvm,
+        string? llvmPath)
     {
         Parallel.ForEach(libsToPrecompile,
             new ParallelOptions { MaxDegreeOfParallelism = parallel ? Environment.ProcessorCount : 1 },
-            lib => PrecompileLibrary(crossCompiler, arch, binDir, lib, envVariables, optimize));
+            lib => PrecompileLibrary(crossCompiler, arch, binDir, lib, envVariables, optimize, useLlvm, llvmPath));
     }
 
     private static void PrecompileLibrary(
@@ -33,14 +35,17 @@ private static void PrecompileLibrary(
         string binDir,
         string libToPrecompile,
         IDictionary<string, string> envVariables,
-        bool optimize)
+        bool optimize,
+        bool useLlvm,
+        string? llvmPath)
     {
         Utils.LogInfo($"[AOT] {libToPrecompile}");
 
         var crossArgs = new StringBuilder();
         crossArgs
-            .Append(" -O=gsharedvt,float32")
-            .Append(" --nollvm")
+            .Append(" -O=gsharedvt")
+            .Append(" -O=-float32")
+            .Append(useLlvm ? " --llvm" : " --nollvm")
             .Append(" --debug");
 
         string libName = Path.GetFileNameWithoutExtension(libToPrecompile);
@@ -54,16 +59,23 @@ private static void PrecompileLibrary(
             .Append("dwarfdebug,")
             .Append("outfile=").Append(Path.Combine(binDir, libName + ".dll.s,"))
             .Append("msym-dir=").Append(Path.Combine(binDir, "Msym,"))
-            // TODO: enable aotdata
-            //.Append("data-outfile=").Append(Path.Combine(binDir, libName + ".aotdata,"))
+            .Append("data-outfile=").Append(Path.Combine(binDir, libName + ".aotdata,"))
             //  TODO: enable direct-pinvokes (to get rid of -force_loads)
             //.Append("direct-pinvoke,")
-            .Append("full,");
+            .Append("full,")
+            .Append("mattr=+crc,")   // enable System.Runtime.Intrinsics.Arm
+            .Append("mattr=+base,"); // (Crc32 and ArmBase for now)
+
+        if (useLlvm)
+        {
+            aotArgs
+                .Append("llvm-path=").Append(llvmPath).Append(',')
+                .Append("llvm-outfile=").Append(Path.Combine(binDir, libName + ".dll-llvm.o,"));
+            // it has -llvm.o suffix because we need both LLVM and non-LLVM bits for every assembly
+            // most of the stuff from non-llvm .o will be linked out.
+        }
 
         // TODO: enable Interpreter
-        // TODO: enable LLVM
-        // TODO: enable System.Runtime.Intrinsics.Arm (LLVM-only for now)
-        // e.g. .Append("mattr=+crc,")
 
         crossArgs
             .Append(" --aot=").Append(aotArgs).Append(" ")

src/mono/msbuild/AppleAppBuilder/AppleAppBuilder.cs

@@ -96,6 +96,18 @@ public class AppleAppBuilderTask : Task
     /// </summary>
     public bool UseConsoleUITemplate { get; set; }
 
+    /// <summary>
+    /// Use LLVM for FullAOT
+    /// The cross-compiler must be built with LLVM support
+    /// </summary>
+    public bool UseLlvm { get; set; }
+
+    /// <summary>
+    /// Path to LLVM binaries (opt and llc)
+    /// It's required if UseLlvm is set
+    /// </summary>
+    public string? LlvmPath { get; set; }
+
     /// <summary>
     /// Path to *.app bundle
     /// </summary>
@@ -127,6 +139,12 @@ public override bool Execute()
             throw new ArgumentException($"ProjectName='{ProjectName}' should not contain spaces");
         }
 
+        if (UseLlvm && !string.IsNullOrEmpty(LlvmPath))
+        {
+            // otherwise we might accidentally use some random llc/opt from PATH (installed with clang)
+            throw new ArgumentException($"LlvmPath shoun't be empty when UseLlvm is set");
+        }
+
         string[] excludes = new string[0];
         if (ExcludeFromAppDir != null)
         {
@@ -154,7 +172,7 @@ public override bool Execute()
 
             AotCompiler.PrecompileLibraries(CrossCompiler, Arch, !DisableParallelAot, binDir, libsToAot,
                 new Dictionary<string, string> { {"MONO_PATH", AppDir} },
-                Optimized);
+                Optimized, UseLlvm, LlvmPath);
         }
 
         // generate modules.m

src/mono/msbuild/AppleAppBuilder/Templates/main-console.m

@@ -45,7 +45,7 @@ - (void)viewDidLoad {
 
     summaryLabel = [[UILabel alloc] initWithFrame: CGRectMake(10.0, 0.0, applicationFrame.size.width - 10.0, 50)];
     summaryLabel.textColor = [UIColor whiteColor];
-    summaryLabel.font = [UIFont boldSystemFontOfSize: 14];
+    summaryLabel.font = [UIFont boldSystemFontOfSize: 12];
     summaryLabel.numberOfLines = 2;
     summaryLabel.textAlignment = NSTextAlignmentLeft;
 #ifdef TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR
@@ -67,7 +67,7 @@ - (void)viewDidLoad {
 void
 mono_ios_set_summary (const char* value)
 {
-    NSString* nsstr = [NSString stringWithUTF8String:strdup(value)];
+    NSString* nsstr = [NSString stringWithUTF8String:value];
     dispatch_async(dispatch_get_main_queue(), ^{
         summaryLabel.text = nsstr;
     });
@@ -77,7 +77,7 @@ - (void)viewDidLoad {
 void
 mono_ios_append_output (const char* value)
 {
-    NSString* nsstr = [NSString stringWithUTF8String:strdup(value)];
+    NSString* nsstr = [NSString stringWithUTF8String:value];
     dispatch_async(dispatch_get_main_queue(), ^{
         logLabel.text = [logLabel.text stringByAppendingString:nsstr];
         CGRect caretRect = [logLabel caretRectForPosition:logLabel.endOfDocument];

src/mono/msbuild/AppleAppBuilder/Xcode.cs

@@ -22,7 +22,7 @@ public static string GenerateXCode(
         string? nativeMainSource = null)
     {
         // bundle everything as resources excluding native files
-        string[] excludes = {".dll.o", ".dll.s", ".dwarf", ".m", ".h", ".a"};
+        string[] excludes = {".dll.o", ".dll.s", ".dwarf", ".m", ".h", ".a", ".bc"};
 
         string[] resources = Directory.GetFiles(workspace)
             .Where(f => !excludes.Any(e => f.EndsWith(e, StringComparison.InvariantCultureIgnoreCase)))

src/mono/msbuild/AppleTestRunner/AppleTestRunner.cs

@@ -11,14 +11,20 @@
 using Microsoft.DotNet.XHarness.Tests.Runners;
 using Microsoft.DotNet.XHarness.Tests.Runners.Core;
 
-public class SimpleTestRunner : iOSApplicatonEntryPoint, IDevice
+public class SimpleTestRunner : iOSApplicationEntryPoint, IDevice
 {
-    private static List<string> testLibs = new List<string>();
+    // to be wired once https://github.com/dotnet/xharness/pull/46 is merged
+    [DllImport("__Internal")]
+    public extern static void mono_ios_append_output (string value);
+
+    [DllImport("__Internal")]
+    public extern static void mono_ios_set_summary (string value);
+
+    private static List<string> s_testLibs = new List<string>();
+    private static string? s_MainTestName;
 
     public static async Task<int> Main(string[] args)
     {
-        // Redirect all Console.WriteLines to iOS UI
-        Console.SetOut(new AppleConsole());
         Console.WriteLine($"ProcessorCount = {Environment.ProcessorCount}");
 
         Console.Write("Args: ");
@@ -30,39 +36,78 @@ public static async Task<int> Main(string[] args)
 
         foreach (string arg in args.Where(a => a.StartsWith("testlib:")))
         {
-            testLibs.Add(arg.Remove(0, "testlib:".Length));
+            s_testLibs.Add(arg.Remove(0, "testlib:".Length));
         }
+        bool verbose = args.Contains("--verbose");
 
-        if (testLibs.Count < 1)
+        if (s_testLibs.Count < 1)
         {
             // Look for *.Tests.dll files if target test suites are not set via "testlib:" arguments
-            testLibs = Directory.GetFiles(Environment.CurrentDirectory, "*.Tests.dll").ToList();
+            s_testLibs = Directory.GetFiles(Environment.CurrentDirectory, "*.Tests.dll").ToList();
         }
 
-        if (testLibs.Count < 1)
+        if (s_testLibs.Count < 1)
         {
             Console.WriteLine($"Test libs were not found (*.Tests.dll was not found in {Environment.CurrentDirectory})");
             return -1;
         }
 
         Console.Write("Test libs: ");
-        foreach (string testLib in testLibs)
+        foreach (string testLib in s_testLibs)
         {
             Console.WriteLine(testLib);
         }
         Console.WriteLine(".");
+        s_MainTestName = Path.GetFileNameWithoutExtension(s_testLibs[0]);
+
+        mono_ios_set_summary($"Starting tests...");
+        var simpleTestRunner = new SimpleTestRunner(verbose);
+        simpleTestRunner.TestStarted += (target, e) =>
+        {
+            mono_ios_append_output($"[STARTING] {e}\n");
+        };
+
+        int failed = 0, passed = 0, skipped = 0;
+        simpleTestRunner.TestCompleted += (target, e) =>
+        {
+            if (e.Item2 == TestResult.Passed)
+            {
+                passed++;
+            }
+            else if (e.Item2 == TestResult.Failed)
+            {
+                failed++;
+            }
+            else if (e.Item2 == TestResult.Skipped)
+            {
+                skipped++;
+            }
+            mono_ios_set_summary($"{s_MainTestName}\nPassed:{passed}, Failed: {failed}, Skipped:{skipped}");
+        };
 
-        AppleConsole.mono_ios_set_summary($"Running\n{Path.GetFileName(testLibs[0])} tests...");
-        var simpleTestRunner = new SimpleTestRunner();
         await simpleTestRunner.RunAsync();
-        AppleConsole.mono_ios_set_summary("Done.");
+        mono_ios_append_output($"\nDone.\n");
         Console.WriteLine("----- Done -----");
         return 0;
     }
 
+    public SimpleTestRunner(bool verbose)
+    {
+        if (verbose)
+        {
+            MinimumLogLevel = MinimumLogLevel.Verbose;
+            _maxParallelThreads = 1;
+        }
+        else
+        {
+            MinimumLogLevel = MinimumLogLevel.Info;
+            _maxParallelThreads = Environment.ProcessorCount;
+        }
+    }
+
     protected override IEnumerable<TestAssemblyInfo> GetTestAssemblies()
     {
-        foreach (string file in testLibs!)
+        foreach (string file in s_testLibs)
         {
             yield return new TestAssemblyInfo(Assembly.LoadFrom(file), file);
         }
@@ -73,13 +118,15 @@ protected override void TerminateWithSuccess()
         Console.WriteLine("[TerminateWithSuccess]");
     }
 
-    protected override int? MaxParallelThreads => Environment.ProcessorCount;
+    private int? _maxParallelThreads;
+
+    protected override int? MaxParallelThreads => _maxParallelThreads;
 
     protected override IDevice Device => this;
 
     protected override TestRunnerType TestRunner => TestRunnerType.Xunit;
 
-    protected override string? IgnoreFilesDirectory { get; }
+    protected override string? IgnoreFilesDirectory => Environment.CurrentDirectory;
 
     public string BundleIdentifier => "net.dot.test-runner";
 
@@ -95,22 +142,3 @@ protected override void TerminateWithSuccess()
 
     public string? Locale { get; }
 }
-
-internal class AppleConsole : TextWriter
-{
-    public override Encoding Encoding => Encoding.Default;
-
-    [DllImport("__Internal")]
-    public extern static void mono_ios_append_output (string value);
-
-    [DllImport("__Internal")]
-    public extern static void mono_ios_set_summary (string value);
-
-    public override void Write(string? value)
-    {
-        if (value != null)
-        {
-            mono_ios_append_output(value);
-        }
-    }
-}