[HTTP/3] Stress hack for msquic dropping connections (#84793)

* Implement the same hack as for functional tests to prevent msquic from dropping connections

* Feedback: removed code sharing and used reflaction

* Try to fix missing build dependency

* Feedback - removed test only function and replaced with shared code + some reflection

* fix argument handling in build-local.ps1

* do not use MsQuicLibraryVersion

* copy msquic interop utils to the SDK base image

* use LinkBase in Functional.Tests.csproj for wildcard include

* Use MsQuicLibraryVersion in out internal logging and as a side-effect avoid the property being trimmed

* Comment

* also copy files to the Linux container

---------

Co-authored-by: antonfirsov <antonfir@gmail.com>

eng/docker/build-docker-sdk.ps1

@@ -36,6 +36,7 @@ if ($buildWindowsContainers)
   # 2. Runtime pack (microsoft.netcore.app.runtime.win-x64)
   # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
   # 4. testhost
+  # 5. msquic interop sources (needed for HttpStress)
   $binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
   $dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context"
 
@@ -51,6 +52,8 @@ if ($buildWindowsContainers)
                      -Destination $dockerContext\testhost
   Copy-Item -Recurse -Path $REPO_ROOT_DIR\eng\targetingpacks.targets `
                      -Destination $dockerContext\targetingpacks.targets
+  Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop `
+                     -Destination $dockerContext\msquic-interop
 
   # In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
   $testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"

eng/docker/libraries-sdk.linux.Dockerfile

@@ -26,6 +26,7 @@ RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily
 # 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
 # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
 # 4. testhost
+# 5. msquic interop sources (needed for HttpStress)
 
 COPY --from=corefxbuild \
     /repo/artifacts/bin/microsoft.netcore.app.ref \
@@ -43,6 +44,10 @@ COPY --from=corefxbuild \
     /repo/artifacts/bin/testhost \
     /live-runtime-artifacts/testhost
 
+COPY --from=corefxbuild \
+    /repo/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop \
+    /live-runtime-artifacts/msquic-interop
+
 # Add AspNetCore bits to testhost:
 ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
 ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props

@@ -5,7 +5,8 @@
 
         <!-- Stress projects have their own global.json, the directory above that also has it is the repository root. -->
         <RepositoryRoot>$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)../, global.json))/</RepositoryRoot>
-
+
+        <MsQuicInteropIncludes Condition="'$(MsQuicInteropIncludes)' == ''">$(RepositoryRoot)src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/*.cs</MsQuicInteropIncludes>
         <TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">$(RepositoryRoot)eng/targetingpacks.targets</TargetingPacksTargetsLocation>
         <ProductVersion>8.0.0</ProductVersion>
         <NetCoreAppCurrent>net8.0</NetCoreAppCurrent>
@@ -14,4 +15,4 @@
         <MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
         <MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.runtime.$(OutputRID)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
     </PropertyGroup>
-</Project>
+</Project>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile

@@ -28,6 +28,7 @@ WORKDIR /app
 COPY . .
 
 RUN dotnet build -c $CONFIGURATION \
+    -p:MsQuicInteropIncludes="/live-runtime-artifacts/msquic-interop/*.cs" \
     -p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \
     -p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \
     -p:MicrosoftNetCoreAppRuntimePackDir=/live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64/$CONFIGURATION/

src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
     <Nullable>enable</Nullable>
     <EnablePreviewFeatures>True</EnablePreviewFeatures>
+    <NoWarn>CA2252</NoWarn>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,6 +17,11 @@
     <PackageReference Include="System.Net.Http.WinHttpHandler" Version="4.5.4" />
   </ItemGroup>
 
+  <!-- Shared production code -->
+  <ItemGroup>
+    <Compile Include="$(MsQuicInteropIncludes)" LinkBase="MsQuicInterop" />
+  </ItemGroup>
+
   <PropertyGroup>
     <!-- These may lead to duplicate generated classes with local (non-docker) Linux builds. -->
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs

@@ -13,6 +13,8 @@
 using System.Threading.Tasks;
 using System.Net;
 using HttpStress;
+using System.Net.Quic;
+using Microsoft.Quic;
 
 [assembly:SupportedOSPlatform("windows")]
 [assembly:SupportedOSPlatform("linux")]
@@ -26,6 +28,8 @@ public static class Program
     {
         public enum ExitCode { Success = 0, StressError = 1, CliError = 2 };
 
+        public static readonly bool IsQuicSupported = QuicListener.IsSupported && QuicConnection.IsSupported;
+
         public static async Task<int> Main(string[] args)
         {
             if (!TryParseCli(args, out Configuration? config))
@@ -158,6 +162,9 @@ private static async Task<ExitCode> Run(Configuration config)
 
             string GetAssemblyInfo(Assembly assembly) => $"{assembly.Location}, modified {new FileInfo(assembly.Location).LastWriteTime}";
 
+            Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");
+            string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
+
             Console.WriteLine("       .NET Core: " + GetAssemblyInfo(typeof(object).Assembly));
             Console.WriteLine("    ASP.NET Core: " + GetAssemblyInfo(typeof(WebHost).Assembly));
             Console.WriteLine(" System.Net.Http: " + GetAssemblyInfo(typeof(System.Net.Http.HttpClient).Assembly));
@@ -169,6 +176,8 @@ private static async Task<ExitCode> Run(Configuration config)
             Console.WriteLine("     Concurrency: " + config.ConcurrentRequests);
             Console.WriteLine("  Content Length: " + config.MaxContentLength);
             Console.WriteLine("    HTTP Version: " + config.HttpVersion);
+            Console.WriteLine("  QUIC supported: " + (IsQuicSupported ? "yes" : "no"));
+            Console.WriteLine("  MsQuic Version: " + msQuicLibraryVersion);
             Console.WriteLine("        Lifetime: " + (config.ConnectionLifetime.HasValue ? $"{config.ConnectionLifetime.Value.TotalMilliseconds}ms" : "(infinite)"));
             Console.WriteLine("      Operations: " + string.Join(", ", usedClientOperations.Select(o => o.name)));
             Console.WriteLine("     Random Seed: " + config.RandomSeed);
@@ -177,6 +186,23 @@ private static async Task<ExitCode> Run(Configuration config)
             Console.WriteLine("Query Parameters: " + config.MaxParameters);
             Console.WriteLine();
 
+            if (config.HttpVersion == HttpVersion.Version30 && IsQuicSupported)
+            {
+                unsafe
+                {
+                    // If the system gets overloaded, MsQuic has a tendency to drop incoming connections, see https://github.com/dotnet/runtime/issues/55979.
+                    // So in case we're running H/3 stress test, we're using the same hack as for System.Net.Quic tests, which increases the time limit for pending operations in MsQuic thread pool.
+                    object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
+                    QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
+                    QUIC_SETTINGS settings = default(QUIC_SETTINGS);
+                    settings.IsSet.MaxWorkerQueueDelayUs = 1;
+                    settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
+                    if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
+                    {
+                        Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
+                    }
+                }
+            }
 
             StressServer? server = null;
             if (config.RunMode.HasFlag(RunMode.server))

src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1

@@ -14,7 +14,7 @@ if (-not ([string]::IsNullOrEmpty($args[0]))) {
 
 $LibrariesConfiguration = "Release"
 if (-not ([string]::IsNullOrEmpty($args[1]))) {
-    $LibrariesConfiguration = $args[0]
+    $LibrariesConfiguration = $args[1]
 }
 
 $TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"
@@ -53,11 +53,11 @@ if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
 Write-Host "Building solution."
 dotnet build -c $StressConfiguration
 
-$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1"
+$Runscript=".\run-stress-$StressConfiguration-$LibrariesConfiguration.ps1"
 if (-not (Test-Path $Runscript)) {
     Write-Host "Generating Runscript."
     Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec --roll-forward Major ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args"
 }
 
 Write-Host "To run tests type:"
-Write-Host "$Runscript [stress test args]"
+Write-Host "$Runscript [stress test args]"

src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile

@@ -12,6 +12,7 @@ ARG VERSION=8.0
 ARG CONFIGURATION=Release
 
 RUN dotnet build -c $env:CONFIGURATION `
+    -p:MsQuicInteropIncludes="C:/live-runtime-artifacts/msquic-interop/*.cs" `
     -p:TargetingPacksTargetsLocation=C:/live-runtime-artifacts/targetingpacks.targets `
     -p:MicrosoftNetCoreAppRefPackDir=C:/live-runtime-artifacts/microsoft.netcore.app.ref/ `
     -p:MicrosoftNetCoreAppRuntimePackDir=C:/live-runtime-artifacts/microsoft.netcore.app.runtime.win-x64/$env:CONFIGURATION/

src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs

@@ -130,7 +130,7 @@ static MsQuicApi()
             }
             string? gitHash = Marshal.PtrToStringUTF8((IntPtr)libGitHash);
 
-            MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} version={version} commit={gitHash}";
+            MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {version} ({gitHash})";
 
             if (version < s_minMsQuicVersion)
             {
@@ -143,7 +143,7 @@ static MsQuicApi()
 
             if (NetEventSource.Log.IsEnabled())
             {
-                NetEventSource.Info(null, $"Loaded MsQuic library version '{version}', commit '{gitHash}'.");
+                NetEventSource.Info(null, $"Loaded MsQuic library '{MsQuicLibraryVersion}'.");
             }
 
             // Assume SChannel is being used on windows and query for the actual provider from the library if querying is supported

src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs

@@ -335,7 +335,7 @@ public async Task ConnectWithServerCertificateCallback()
             // TODO: the exception may change if we implement https://github.com/dotnet/runtime/issues/73152 to make server close
             // connections with CONNECTION_REFUSED in such cases
             var authEx = await Assert.ThrowsAsync<AuthenticationException>(() => clientTask);
-            Assert.Contains("UserCanceled", authEx.Message);
+            Assert.Contains(TlsAlertMessage.UserCanceled.ToString(), authEx.Message);
             Assert.Equal(clientOptions.ClientAuthenticationOptions.TargetHost, receivedHostName);
             await Assert.ThrowsAsync<ArgumentException>(async () => await listener.AcceptConnectionAsync());
 

src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs

@@ -13,8 +13,8 @@
 using Xunit.Abstractions;
 using System.Diagnostics.Tracing;
 using System.Net.Sockets;
+using System.Reflection;
 using Microsoft.Quic;
-using static Microsoft.Quic.MsQuic;
 
 namespace System.Net.Quic.Tests
 {
@@ -44,14 +44,20 @@ public abstract class QuicTestBase : IDisposable
 
         static unsafe QuicTestBase()
         {
-            Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{MsQuicApi.MsQuicLibraryVersion}'.");
+            // If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack.
+            Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");
+
+            string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
+            Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'.");
 
             if (IsSupported)
             {
+                object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
+                QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
                 QUIC_SETTINGS settings = default(QUIC_SETTINGS);
                 settings.IsSet.MaxWorkerQueueDelayUs = 1;
                 settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
-                if (StatusFailed(MsQuicApi.Api.ApiTable->SetParam(null, QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
+                if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
                 {
                     Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
                 }

src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj

@@ -16,6 +16,7 @@
     <Compile Include="$(CommonPath)System\Net\ArrayBuffer.cs" Link="ProductionCode\Common\System\Net\ArrayBuffer.cs" />
     <Compile Include="$(CommonPath)System\Net\MultiArrayBuffer.cs" Link="ProductionCode\Common\System\Net\MultiArrayBuffer.cs" />
     <Compile Include="$(CommonPath)System\Net\StreamBuffer.cs" Link="ProductionCode\Common\System\Net\StreamBuffer.cs" />
+    <Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
     <Compile Include="$(CommonTestPath)System\IO\ConnectedStreams.cs" Link="Common\System\IO\ConnectedStreams.cs" />
     <Compile Include="$(CommonTestPath)System\Net\Capability.Security.cs" Link="Common\System\Net\Capability.Security.cs" />
     <Compile Include="$(CommonTestPath)System\Net\Configuration.cs" Link="Common\System\Net\Configuration.cs" />
@@ -31,18 +32,8 @@
   </ItemGroup>
   <!-- Shared production code -->
   <ItemGroup>
-    <Compile Include="$(CommonPath)System\Net\Logging\NetEventSource.Common.cs" Link="Common\System\Net\Logging\NetEventSource.Common.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicSafeHandle.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicSafeHandle.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\Internal\ThrowHelper.cs" Link="ProductionCode\System\Net\Quic\Internal\ThrowHelper.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" Link="ProductionCode\System\Net\Quic\Interop\*.cs" />
-    <Compile Include="..\..\src\System\Net\Quic\NetEventSource.Quic.cs" Link="ProductionCode\System\Net\Quic\NetEventSource.Quic.cs" />
     <Compile Include="..\..\src\System\Net\Quic\QuicDefaults.cs" Link="ProductionCode\System\Net\Quic\QuicDefaults.cs" />
-    <Compile Include="$(CommonPath)Interop\Windows\Interop.Libraries.cs" Link="Common\Interop\Windows\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'windows'" />
-    <Compile Include="$(CommonPath)Interop\Linux\Interop.Libraries.cs" Link="Common\Interop\Linux\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'linux'" />
-    <Compile Include="$(CommonPath)Interop\OSX\Interop.Libraries.cs" Link="Common\Interop\OSX\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'osx'" />
-    <Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
+    <Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" LinkBase="ProductionCode\System\Net\Quic\Interop" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="$(CommonTestPath)StreamConformanceTests\StreamConformanceTests.csproj" />