Fix terminal logger encoding

src/Microsoft.TestPlatform.Build/PublicAPI/PublicAPI.Unshipped.txt

@@ -116,6 +116,8 @@ override Microsoft.TestPlatform.Build.Tasks.VSTestTask.Execute() -> bool
 override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.GenerateCommandLineCommands() -> string?
 override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.GenerateFullPathToTool() -> string?
 override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.LogEventsFromTextOutput(string! singleLine, Microsoft.Build.Framework.MessageImportance messageImportance) -> void
+override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.StandardErrorEncoding.get -> System.Text.Encoding!
+override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.StandardOutputEncoding.get -> System.Text.Encoding!
 override Microsoft.TestPlatform.Build.Tasks.VSTestTask2.ToolName.get -> string?
 static Microsoft.TestPlatform.Build.Trace.Tracing.Trace(string! message) -> void
 static Microsoft.TestPlatform.Build.Trace.Tracing.traceEnabled -> bool

src/Microsoft.TestPlatform.Build/Tasks/VSTestTask2.cs

@@ -2,8 +2,10 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
+using System.Text;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
@@ -40,6 +42,9 @@ public class VSTestTask2 : ToolTask, ITestTask
     public string? VSTestArtifactsProcessingMode { get; set; }
     public string? VSTestSessionCorrelationId { get; set; }
 
+    protected override Encoding StandardErrorEncoding => !_disableUtf8ConsoleEncoding ? Encoding.UTF8 : base.StandardErrorEncoding;
+    protected override Encoding StandardOutputEncoding => !_disableUtf8ConsoleEncoding ? Encoding.UTF8 : base.StandardOutputEncoding;
+
     private readonly string _testResultSplitter = "++++";
     private readonly string[] _testResultSplitterArray = new[] { "++++" };
 
@@ -50,6 +55,7 @@ public class VSTestTask2 : ToolTask, ITestTask
     private readonly string[] _fullErrorSplitterArray = new[] { "~~~~" };
 
     private readonly string _fullErrorNewlineSplitter = "!!!!";
+    private readonly bool _disableUtf8ConsoleEncoding;
 
     protected override string? ToolName
     {
@@ -64,12 +70,15 @@ public class VSTestTask2 : ToolTask, ITestTask
 
     public VSTestTask2()
     {
+        // Unless user opted out, use UTF encoding, which we force in vstest.console.
+        _disableUtf8ConsoleEncoding = Environment.GetEnvironmentVariable("VSTEST_DISABLE_UTF8_CONSOLE_ENCODING") == "1";
         LogStandardErrorAsError = false;
         StandardOutputImportance = "Normal";
     }
 
     protected override void LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance)
     {
+        Debug.WriteLine($"vstestTask2: received output {singleLine}, importance {messageImportance}");
         if (singleLine.StartsWith(_errorSplitter))
         {
             var parts = singleLine.Split(_errorSplitterArray, StringSplitOptions.None);

src/vstest.console/Internal/MSBuildLogger.cs

@@ -112,8 +112,9 @@ private void TestResultHandler(object? sender, TestResultEventArgs e)
                 Output.Information(false, info);
                 break;
             case TestOutcome.Failed:
-
                 var result = e.Result;
+                Debug.WriteLine(">>>>ERR:" + result.ErrorMessage);
+                Debug.WriteLine(">>>>STK:" + result.ErrorStackTrace);
                 if (!StringUtils.IsNullOrWhiteSpace(result.ErrorStackTrace))
                 {
                     var maxLength = 1000;

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using Microsoft.TestPlatform.TestUtilities;
@@ -20,7 +20,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj");
         InvokeDotnetTest($@"{projectPath} /p:VsTestUseMSBuildOutput=true /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}");
 
         // The output:
@@ -31,7 +31,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo)
         // C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\UnitTest1.cs(41): error VSTEST1: (FailingTest) SampleUnitTestProject.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<2>. Actual:<3>.  [C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\SimpleTestProject.csproj::TargetFramework=net462]
         // C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\UnitTest1.cs(41): error VSTEST1: (FailingTest) SampleUnitTestProject.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<2>. Actual:<3>.  [C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\SimpleTestProject.csproj::TargetFramework=netcoreapp3.1]
 
-        StdOutputContains("error VSTEST1: (FailingTest) SampleUnitTestProject.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<2>. Actual:<3>.");
+        StdOutputContains("TerminalLoggerUnitTests.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<ğğğ𦮙我們剛才從𓋴𓅓𓏏𓇏𓇌𓀀 (System.String)>. Actual:<3 (System.Int32)>.");
         // We are sending those as low prio messages, they won't show up on screen but will be in binlog.
         //StdOutputContains("passed PassingTest");
         //StdOutputContains("skipped SkippingTest");

test/TestAssets/TerminalLoggerTestProject/TerminalLoggerTestProject.csproj

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <AssemblyName>SimpleTestProject</AssemblyName>
+    <TargetFrameworks>$(NetFrameworkMinimum);$(NetCoreAppMinimum)</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MSTest.TestFramework">
+      <Version>$(MSTestTestFrameworkVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="MSTest.TestAdapter">
+      <Version>$(MSTestTestAdapterVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk">
+      <Version>$(PackageVersion)</Version>
+    </PackageReference>
+  </ItemGroup>
+  <ItemGroup Condition=" '$(TargetFramework)' == '$(NetFrameworkMinimum)' ">
+    <Reference Include="System.Runtime" />
+    <Reference Include="System" />
+    <Reference Include="Microsoft.CSharp" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+</Project>

test/TestAssets/TerminalLoggerTestProject/UnitTest1.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#if NETFRAMEWORK
+using System;
+using System.IO;
+#endif
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace TerminalLoggerUnitTests
+{
+    /// <summary>
+    /// The unit test 1.
+    /// </summary>
+    [TestClass]
+    public class UnitTest1
+    {
+        /// <summary>
+        /// The passing test.
+        /// </summary>
+        [TestMethod]
+        public void PassingTest()
+        {
+            Assert.AreEqual(2, 2);
+        }
+
+        /// <summary>
+        /// The failing test.
+        /// </summary>
+        [TestMethod]
+        public void FailingTest()
+        {
+            // test characters taken from https://pages.ucsd.edu/~dkjordan/chin/unitestuni.html
+            Assert.AreEqual("ğğğ𦮙我們剛才從𓋴𓅓𓏏𓇏𓇌𓀀", 3);
+        }
+
+        /// <summary>
+        /// The skipping test.
+        /// </summary>
+        [Ignore]
+        [TestMethod]
+        public void SkippingTest()
+        {
+        }
+    }
+}

test/TestAssets/TestAssets.sln

@@ -132,7 +132,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MultiHostTestExecutionProje
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NonDll.TestAdapter", "NonDll.TestAdapter\NonDll.TestAdapter.csproj", "{429552A4-4C18-4355-94C5-80DC88C48405}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetStandard2Library", "NetStandard2Library\NetStandard2Library.csproj", "{080F0AD2-E7AE-42C8-B867-56D78576735D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NetStandard2Library", "NetStandard2Library\NetStandard2Library.csproj", "{080F0AD2-E7AE-42C8-B867-56D78576735D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TerminalLoggerTestProject", "TerminalLoggerTestProject\TerminalLoggerTestProject.csproj", "{C69BEEA4-BE37-4155-8DD0-130C62D8BF6D}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -384,6 +386,10 @@ Global
 		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C69BEEA4-BE37-4155-8DD0-130C62D8BF6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C69BEEA4-BE37-4155-8DD0-130C62D8BF6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C69BEEA4-BE37-4155-8DD0-130C62D8BF6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C69BEEA4-BE37-4155-8DD0-130C62D8BF6D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE