diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs new file mode 100644 index 0000000000000..119f70f00bffb --- /dev/null +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Xml; + +public class XUnitLogChecker +{ + private static class Patterns + { + public const string OpenTag = @"(\B<\w+)|(\B)|(\]\]>)"; + } + + private readonly struct TagResult + { + public TagResult(string matchValue, TagCategory matchCategory) + { + Value = matchValue; + Category = matchCategory; + } + + public string Value { get; init; } + public TagCategory Category { get; init; } + } + + private enum TagCategory { OPENING, CLOSING } + + private const int SUCCESS = 0; + private const int MISSING_ARGS = -1; + private const int SOMETHING_VERY_WRONG = -2; + + static int Main(string[] args) + { + // Maybe add a '--help' flag that also gets triggered in this case. + if (args.Length < 2) + { + Console.WriteLine("[XUnitLogChecker]: The path to the log file and" + + " the name of the wrapper are required for an" + + " accurate check and fixing."); + return MISSING_ARGS; + } + + // Creating variables for code clarity and ease of understanding. + + string resultsDir = args[0]; + string wrapperName = args[1]; + + // Browser-Wasm tests follow a different test files layout in Helix. + // Everything is located in a root folder, rather than the usual path + // with the wrapper name other platforms follow. + + string tempLogName = string.IsNullOrEmpty(wrapperName) + ? "tempLog.xml" + : $"{wrapperName}.tempLog.xml"; + + string finalLogName = string.IsNullOrEmpty(wrapperName) + ? "testResults.xml" + : $"{wrapperName}.testResults.xml"; + + string statsCsvName = string.IsNullOrEmpty(wrapperName) + ? "testStats.csv" + : $"{wrapperName}.testStats.csv"; + + string tempLogPath = Path.Combine(resultsDir, tempLogName); + string finalLogPath = Path.Combine(resultsDir, finalLogName); + string statsCsvPath = Path.Combine(resultsDir, statsCsvName); + + // If there is not even the temp log, then something went very badly with + // the work item in question. It will need a developer/engineer to look + // at it urgently. + + if (!File.Exists(tempLogPath)) + { + Console.WriteLine("[XUnitLogChecker]: No logs were found. Something" + + " went very wrong with this item..."); + Console.WriteLine($"[XUnitLogChecker]: Expected log name: '{tempLogName}'"); + + return SOMETHING_VERY_WRONG; + } + + // Read the stats csv file. + IEnumerable workItemStats = File.ReadLines(statsCsvPath); + + // The first value at the top of the csv represents the amount of tests + // that were expected to be run. + // + // NOTE: We know for certain the csv only includes numbers. Therefore, + // we're fine in using Int32.Parse() directly. + + int numExpectedTests = Int32.Parse(workItemStats.First().Split(',').First()); + + // The last line of the csv represents the status when the work item + // finished, successfully or not. It has the following format: + // (Tests Run, Tests Passed, Tests Failed, Tests Skipped) + + int[] workItemEndStatus = workItemStats.Last() + .Split(',') + .Select(x => Int32.Parse(x)) + .ToArray(); + + // If the final results log file is present, then we can assume everything + // went fine, and it's ready to go without any further processing. We just + // check the stats csv file to know how many tests were run, and display a + // brief summary of the work item. + + if (File.Exists(finalLogPath)) + { + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" + + " complete successfully!"); + + PrintWorkItemSummary(numExpectedTests, workItemEndStatus); + return SUCCESS; + } + + // Here goes the main core of the XUnit Log Checker :) + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did not" + + " finish running. Checking and fixing the log..."); + + FixTheXml(tempLogPath); + PrintWorkItemSummary(numExpectedTests, workItemEndStatus); + + // Rename the temp log to the final log, so that Helix can use it without + // knowing what transpired here. + File.Move(tempLogPath, finalLogPath); + return SUCCESS; + } + + static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus) + { + Console.WriteLine($"\n{workItemEndStatus[0]}/{numExpectedTests} tests run."); + Console.WriteLine($"* {workItemEndStatus[1]} tests passed."); + Console.WriteLine($"* {workItemEndStatus[2]} tests failed."); + Console.WriteLine($"* {workItemEndStatus[3]} tests skipped.\n"); + } + + static void FixTheXml(string xFile) + { + var tags = new Stack(); + string tagText = string.Empty; + + // Flag to ensure we don't process tag-like-looking things while reading through + // a test's output. + bool inOutput = false; + bool inCData = false; + + foreach (string line in File.ReadLines(xFile)) + { + // Get all XML tags found in the current line and sort them in order + // of appearance. + Match[] opens = Regex.Matches(line, Patterns.OpenTag).ToArray(); + Match[] closes = Regex.Matches(line, Patterns.CloseTag).ToArray(); + TagResult[] allTags = GetOrderedTagMatches(opens, closes); + + foreach (TagResult tr in allTags) + { + // Found an opening tag. Push into the stack and move on to the next one. + if (tr.Category == TagCategory.OPENING) + { + // Get the name of the next tag. We need solely the text, so we + // ask LINQ to lend us a hand in removing the symbols from the string. + tagText = new String(tr.Value.Where(c => char.IsLetter(c)).ToArray()); + + // We are beginning to process a test's output. Set the flag to + // treat everything as such, until we get the closing output tag. + if (tagText.Equals("output") && !inOutput && !inCData) + inOutput = true; + else if (tagText.Equals("CDATA") && !inCData) + inCData = true; + + tags.Push(tagText); + continue; + } + + // Found a closing tag. If we're currently in an output state, then + // check whether it's the output closing tag. Otherwise, ignore it. + // This is because in that case, it's just part of the output's text, + // rather than an actual XML log tag. + if (tr.Category == TagCategory.CLOSING) + { + // As opposed to the usual XML tags we can find in the logs, + // the CDATA closing one doesn't have letters, so we treat it + // as a special case. + tagText = tr.Value.Equals("]]>") + ? "CDATA" + : new String(tr.Value + .Where(c => char.IsLetter(c)) + .ToArray()); + + if (inCData) + { + if (tagText.Equals("CDATA")) + { + tags.Pop(); + inCData = false; + } + else continue; + } + + if (inOutput) + { + if (tagText.Equals("output")) + { + tags.Pop(); + inOutput = false; + } + else continue; + } + + if (tagText.Equals(tags.Peek())) + { + tags.Pop(); + } + } + } + } + + if (tags.Count == 0) + { + Console.WriteLine($"[XUnitLogChecker]: XUnit log file '{xFile}' was A-OK!"); + } + + // Write the missing closings for all the opened tags we found. + using (StreamWriter xsw = File.AppendText(xFile)) + while (tags.Count > 0) + { + string tag = tags.Pop(); + if (tag.Equals("CDATA")) + xsw.WriteLine("]]>"); + else + xsw.WriteLine($""); + } + + Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); + } + + static TagResult[] GetOrderedTagMatches(Match[] openingTags, Match[] closingTags) + { + var result = new TagResult[openingTags.Length + closingTags.Length]; + int resIndex = 0; + int opIndex = 0; + int clIndex = 0; + + // Fill up the result array with the tags found, in order of appearance + // in the original log file line. + // + // As long as the result array hasn't been filled, then we know for certain + // that there's at least one unprocessed tag. + + while (resIndex < result.Length) + { + if (opIndex < openingTags.Length) + { + // We still have pending tags on both lists, opening and closing. + // So, we add the one that appears first in the given log line. + if (clIndex < closingTags.Length) + { + if (openingTags[opIndex].Index < closingTags[clIndex].Index) + { + result[resIndex++] = new TagResult(openingTags[opIndex].Value, + TagCategory.OPENING); + opIndex++; + } + else + { + result[resIndex++] = new TagResult(closingTags[clIndex].Value, + TagCategory.CLOSING); + clIndex++; + } + } + + // Only opening tags remaining, so just add them. + else + { + result[resIndex++] = new TagResult(openingTags[opIndex].Value, + TagCategory.OPENING); + opIndex++; + } + } + + // Only closing tags remaining, so just add them. + else + { + result[resIndex++] = new TagResult(closingTags[clIndex].Value, + TagCategory.CLOSING); + clIndex++; + } + } + return result; + } +} + diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj new file mode 100644 index 0000000000000..d339f492d63d3 --- /dev/null +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj @@ -0,0 +1,13 @@ + + + + Exe + $(NetCoreAppToolCurrent) + true + + + + + + + diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 213fc3d36d1bb..81e07deb6041e 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -333,7 +333,8 @@ public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string public string WrapTestExecutionWithReporting(string testExecutionExpression, ITestInfo test) { StringBuilder builder = new(); - builder.AppendLine($"if ({_filterLocalIdentifier} is null || {_filterLocalIdentifier}.ShouldRunTest(@\"{test.ContainingType}.{test.Method}\", {test.TestNameExpression}))"); + builder.AppendLine($"if ({_filterLocalIdentifier} is null || {_filterLocalIdentifier}.ShouldRunTest(@\"{test.ContainingType}.{test.Method}\"," + + $" {test.TestNameExpression}))"); builder.AppendLine("{"); builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;"); @@ -341,11 +342,17 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();"); builder.AppendLine(testExecutionExpression); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());"); + + builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\"," + + $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("catch (System.Exception ex) {"); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());"); + + builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\"," + + $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); @@ -359,6 +366,7 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe public string GenerateSkippedTestReporting(ITestInfo skippedTest) { - return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty);"; + return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\"," + + $" System.TimeSpan.Zero, string.Empty, tempLogSw, statsCsvSw);"; } } diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index a76025db7d727..816b567c2e653 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -154,12 +154,25 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};"))); builder.AppendLine("System.Collections.Generic.HashSet testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();"); + builder.Append("\n"); // Make the FullRunner.g.cs file a bit more readable. + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.xml"");"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.testStats.csv""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.testStats.csv"");"); + builder.Append("\n"); + builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);"); + // builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new(TestCount.Count);"); builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();"); builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();"); builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); + builder.Append("\n"); + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.tempLog.xml""))"); + builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); + builder.AppendLine("statsCsvSw.WriteLine($\"{TestCount.Count},0,0,0\");"); + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); StringBuilder testExecutorBuilder = new(); @@ -177,19 +190,27 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos if (testsLeftInCurrentTestExecutor == 0) { if (currentTestExecutor != 0) + { testExecutorBuilder.AppendLine("}"); + } + currentTestExecutor++; - testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{"); - builder.AppendLine($"TestExecutor{currentTestExecutor}();"); - testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well + testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(System.IO.StreamWriter tempLogSw, System.IO.StreamWriter statsCsvSw) {{"); + builder.AppendLine($"TestExecutor{currentTestExecutor}(tempLogSw, statsCsvSw);"); + testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well } + testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); totalTestsEmitted++; testsLeftInCurrentTestExecutor--; } + testExecutorBuilder.AppendLine("}"); } + // Closing the 'using' statements that stream the temporary files. + builder.AppendLine("}\n"); + builder.AppendLine($@"string testResults = summary.GetTestResultOutput(""{assemblyName}"");"); builder.AppendLine($@"string workitemUploadRoot = System.Environment.GetEnvironmentVariable(""HELIX_WORKITEM_UPLOAD_ROOT"");"); builder.AppendLine($@"if (workitemUploadRoot != null) System.IO.File.WriteAllText(System.IO.Path.Combine(workitemUploadRoot, ""{assemblyName}.testResults.xml.txt""), testResults);"); @@ -197,7 +218,6 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("return 100;"); builder.Append(testExecutorBuilder); - builder.AppendLine("public static class TestCount { public const int Count = " + totalTestsEmitted.ToString() + "; }"); return builder.ToString(); } @@ -220,12 +240,24 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); + builder.Append("\n"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.xml"");"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.testStats.csv""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.testStats.csv"");"); + builder.Append("\n"); + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); StringBuilder testExecutorBuilder = new(); int testsLeftInCurrentTestExecutor = 0; int currentTestExecutor = 0; + // Open the stream writer for the temp log. + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml""))"); + builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); + builder.AppendLine($"statsCsvSw.WriteLine(\"{testInfos.Length},0,0,0\");"); + if (testInfos.Length > 0) { // Break tests into groups of 50 so that we don't create an unreasonably large main method @@ -237,15 +269,26 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI { if (currentTestExecutor != 0) testExecutorBuilder.AppendLine("}"); + currentTestExecutor++; - testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(XUnitWrapperLibrary.TestSummary summary, XUnitWrapperLibrary.TestFilter filter, XUnitWrapperLibrary.TestOutputRecorder outputRecorder, System.Diagnostics.Stopwatch stopwatch){{"); - builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch);"); + testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(" + + "XUnitWrapperLibrary.TestSummary summary, " + + "XUnitWrapperLibrary.TestFilter filter, " + + "XUnitWrapperLibrary.TestOutputRecorder outputRecorder, " + + "System.Diagnostics.Stopwatch stopwatch, " + + "System.IO.StreamWriter tempLogSw, " + + "System.IO.StreamWriter statsCsvSw){"); + + builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch, tempLogSw, statsCsvSw);"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well } + testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); testsLeftInCurrentTestExecutor--; } + testExecutorBuilder.AppendLine("}"); + builder.AppendLine("}"); } builder.AppendLine("return summary;"); diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index b230fa485a055..c81a82d99423d 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -3,40 +3,167 @@ // using System; +using System.IO; using System.Collections.Generic; using System.Text; namespace XUnitWrapperLibrary; public class TestSummary { - readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output); + readonly record struct TestResult + { + readonly string Name; + readonly string ContainingTypeName; + readonly string MethodName; + readonly TimeSpan Duration; + readonly Exception? Exception; + readonly string? SkipReason; + readonly string? Output; + + public TestResult(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + Exception? exception, + string? skipReason, + string? output) + { + Name = name; + ContainingTypeName = containingTypeName; + MethodName = methodName; + Duration = duration; + Exception = exception; + SkipReason = skipReason; + Output = output; + } + + public string ToXmlString() + { + var testResultSb = new StringBuilder(); + testResultSb.Append($@"" + : string.Empty; + + if (Exception is not null) + { + string? message = Exception.Message; + + if (Exception is System.Reflection.TargetInvocationException tie) + { + if (tie.InnerException is not null) + { + message = $"{message}\n INNER EXCEPTION--\n" + + $"{tie.InnerException.GetType()}--\n" + + $"{tie.InnerException.Message}--\n" + + $"{tie.InnerException.StackTrace}"; + } + } + + if (string.IsNullOrWhiteSpace(message)) + { + message = "NoExceptionMessage"; + } + + testResultSb.Append($@" result=""Fail"">" + + $@"" + + $"" + + "{outputElement}"); + } + else if (SkipReason is not null) + { + testResultSb.Append($@" result=""Skip"">"); + } + else + { + testResultSb.AppendLine($@" result=""Pass"">{outputElement}"); + } + + return testResultSb.ToString(); + } + } public int PassedTests { get; private set; } = 0; public int FailedTests { get; private set; } = 0; public int SkippedTests { get; private set; } = 0; + public int TotalTests { get; private set; } = 0; private readonly List _testResults = new(); - private DateTime _testRunStart = DateTime.Now; - public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output) + public void ReportPassedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + string output, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { PassedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null, output)); + TotalTests++; + var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output); + _testResults.Add(result); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } - public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output) + public void ReportFailedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + Exception ex, + string output, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { FailedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null, output)); + TotalTests++; + var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output); + _testResults.Add(result); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } - public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason) + public void ReportSkippedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + string reason, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { SkippedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason, null)); + TotalTests++; + var result = new TestResult(name, containingTypeName, methodName, duration, null, reason, null); + _testResults.Add(result); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } + // NOTE: This will likely change or be removed altogether with the existence of the temp log. public string GetTestResultOutput(string assemblyName) { double totalRunSeconds = (DateTime.Now - _testRunStart).TotalSeconds; @@ -69,38 +196,7 @@ public string GetTestResultOutput(string assemblyName) foreach (var test in _testResults) { - resultsFile.Append($@"" : string.Empty; - if (test.Exception is not null) - { - string exceptionMessage = test.Exception.Message; - if (test.Exception is System.Reflection.TargetInvocationException tie) - { - if (tie.InnerException != null) - { - exceptionMessage = $"{exceptionMessage} \n INNER EXCEPTION--\n {tie.InnerException.GetType()}--\n{tie.InnerException.Message}--\n{tie.InnerException.StackTrace}"; - } - } - if (string.IsNullOrWhiteSpace(exceptionMessage)) - { - exceptionMessage = "NoExceptionMessage"; - } - - string? stackTrace = test.Exception.StackTrace; - if (string.IsNullOrWhiteSpace(stackTrace)) - { - stackTrace = "NoStackTrace"; - } - resultsFile.AppendLine($@"result=""Fail"">{outputElement}"); - } - else if (test.SkipReason is not null) - { - resultsFile.AppendLine($@"result=""Skip"">"); - } - else - { - resultsFile.AppendLine($@" result=""Pass"">{outputElement}"); - } + resultsFile.AppendLine(test.ToXmlString()); } resultsFile.AppendLine(""); diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 522d147daff80..4880291225401 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -65,6 +65,9 @@ $(TestBinDir)Tests\Core_Root\ + $([MSBuild]::NormalizeDirectory($(CoreRootDirectory))) + $(TestBinDir)Common\XUnitLogChecker\ + $([MSBuild]::NormalizeDirectory($(XUnitLogCheckerDirectory))) $(TestBinDir)LegacyPayloads\ $([MSBuild]::NormalizeDirectory($(LegacyPayloadsRootDirectory))) $(TestBinDir)MergedPayloads\ @@ -358,14 +361,19 @@ <_MergedWrapperRunScript Include="$([System.IO.Path]::ChangeExtension('%(_MergedWrapperMarker.Identity)', '.$(TestScriptExtension)'))" /> + <_MergedWrapperDirectory>$([System.IO.Path]::GetDirectoryName('%(_MergedWrapperRunScript.Identity)')) <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)')) <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) + <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) + <_MergedWrapperRunScriptDirectoryRelative Condition="'$(_MergedWrapperRunScriptRelative)' != ''">$([System.IO.Path]::GetDirectoryName($(_MergedWrapperRunScriptRelative))) + <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'">call + <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> <_MergedWrapperOutOfProcessTestFiles @@ -387,6 +395,12 @@ + + dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/ + dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/ + $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) + + @@ -395,6 +409,7 @@ + @@ -445,8 +460,10 @@ <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) <_MergedWrapperName>%(_MergedWrapperMarker.FileName) + <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle', %(_MergedWrapperRunScript.FullPath))) <_MergedWrapperRunScriptRelative Condition="'$(TestWrapperTargetsWindows)' != 'true'">./$(_MergedWrapperRunScriptRelative) + <_MergedWrapperRunScriptDirectoryRelative Condition="'$(_MergedWrapperRunScriptRelative)' != ''">$([System.IO.Path]::GetDirectoryName($(_MergedWrapperRunScriptRelative))) @@ -697,6 +714,7 @@ + diff --git a/src/tests/build.proj b/src/tests/build.proj index 094b00ca0790b..56a58aaca9f80 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -25,6 +25,7 @@ + @@ -508,6 +509,10 @@ Projects="$(MSBuildProjectFile)" Targets="EmitTestExclusionList" Properties="XunitTestBinBase=$(XunitTestBinBase)" /> + +