From 2deebf04c5a238682163c08aed4e2c57aa7f3efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20P=C3=ADchov=C3=A1?= <11718369+ManickaP@users.noreply.github.com> Date: Fri, 28 Apr 2023 10:37:08 +0200 Subject: [PATCH] [HTTP/3] Stress hack for msquic dropping connections (#84793) * Implement the same hack as for functional tests to prevent msquic from dropping connections * Feedback: removed code sharing and used reflaction * Try to fix missing build dependency * Feedback - removed test only function and replaced with shared code + some reflection * fix argument handling in build-local.ps1 * do not use MsQuicLibraryVersion * copy msquic interop utils to the SDK base image * use LinkBase in Functional.Tests.csproj for wildcard include * Use MsQuicLibraryVersion in out internal logging and as a side-effect avoid the property being trimmed * Comment * also copy files to the Linux container --------- Co-authored-by: antonfirsov --- eng/docker/build-docker-sdk.ps1 | 3 +++ eng/docker/libraries-sdk.linux.Dockerfile | 5 ++++ .../HttpStress/Directory.Build.props | 5 ++-- .../tests/StressTests/HttpStress/Dockerfile | 1 + .../StressTests/HttpStress/HttpStress.csproj | 7 +++++ .../tests/StressTests/HttpStress/Program.cs | 26 +++++++++++++++++++ .../StressTests/HttpStress/build-local.ps1 | 6 ++--- .../StressTests/HttpStress/windows.Dockerfile | 1 + .../src/System/Net/Quic/Internal/MsQuicApi.cs | 4 +-- .../tests/FunctionalTests/MsQuicTests.cs | 2 +- .../tests/FunctionalTests/QuicTestBase.cs | 12 ++++++--- .../System.Net.Quic.Functional.Tests.csproj | 13 ++-------- 12 files changed, 63 insertions(+), 22 deletions(-) diff --git a/eng/docker/build-docker-sdk.ps1 b/eng/docker/build-docker-sdk.ps1 index bc73312efcb6c..1a09343defbf6 100755 --- a/eng/docker/build-docker-sdk.ps1 +++ b/eng/docker/build-docker-sdk.ps1 @@ -36,6 +36,7 @@ if ($buildWindowsContainers) # 2. Runtime pack (microsoft.netcore.app.runtime.win-x64) # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK # 4. testhost + # 5. msquic interop sources (needed for HttpStress) $binArtifacts = "$REPO_ROOT_DIR\artifacts\bin" $dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context" @@ -51,6 +52,8 @@ if ($buildWindowsContainers) -Destination $dockerContext\testhost Copy-Item -Recurse -Path $REPO_ROOT_DIR\eng\targetingpacks.targets ` -Destination $dockerContext\targetingpacks.targets + Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop ` + -Destination $dockerContext\msquic-interop # In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress): $testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App" diff --git a/eng/docker/libraries-sdk.linux.Dockerfile b/eng/docker/libraries-sdk.linux.Dockerfile index 8d7d6169a9fe9..401ce9f1ffd43 100644 --- a/eng/docker/libraries-sdk.linux.Dockerfile +++ b/eng/docker/libraries-sdk.linux.Dockerfile @@ -26,6 +26,7 @@ RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily # 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64) # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK # 4. testhost +# 5. msquic interop sources (needed for HttpStress) COPY --from=corefxbuild \ /repo/artifacts/bin/microsoft.netcore.app.ref \ @@ -43,6 +44,10 @@ COPY --from=corefxbuild \ /repo/artifacts/bin/testhost \ /live-runtime-artifacts/testhost +COPY --from=corefxbuild \ + /repo/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop \ + /live-runtime-artifacts/msquic-interop + # Add AspNetCore bits to testhost: ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*" ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App" diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props index 509bca64fdc3d..3ee8cbfa7ae05 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props @@ -5,7 +5,8 @@ $([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)../, global.json))/ - + + $(RepositoryRoot)src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/*.cs $(RepositoryRoot)eng/targetingpacks.targets 8.0.0 net8.0 @@ -14,4 +15,4 @@ $(RepositoryRoot)artifacts/bin/microsoft.netcore.app.ref/ $(RepositoryRoot)artifacts/bin/microsoft.netcore.app.runtime.$(OutputRID)/$(Configuration)/ - \ No newline at end of file + diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile index e56f8c80c3212..6dc83bce1895e 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile @@ -28,6 +28,7 @@ WORKDIR /app COPY . . RUN dotnet build -c $CONFIGURATION \ + -p:MsQuicInteropIncludes="/live-runtime-artifacts/msquic-interop/*.cs" \ -p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \ -p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \ -p:MicrosoftNetCoreAppRuntimePackDir=/live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64/$CONFIGURATION/ diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj index bb71d66d214c2..b98a4bd529c76 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj @@ -4,6 +4,8 @@ $(NetCoreAppCurrent) enable True + CA2252 + true @@ -15,6 +17,11 @@ + + + + + false diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs index a1a2769273591..5a60db6364aad 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs @@ -13,6 +13,8 @@ using System.Threading.Tasks; using System.Net; using HttpStress; +using System.Net.Quic; +using Microsoft.Quic; [assembly:SupportedOSPlatform("windows")] [assembly:SupportedOSPlatform("linux")] @@ -26,6 +28,8 @@ public static class Program { public enum ExitCode { Success = 0, StressError = 1, CliError = 2 }; + public static readonly bool IsQuicSupported = QuicListener.IsSupported && QuicConnection.IsSupported; + public static async Task Main(string[] args) { if (!TryParseCli(args, out Configuration? config)) @@ -158,6 +162,9 @@ private static async Task Run(Configuration config) string GetAssemblyInfo(Assembly assembly) => $"{assembly.Location}, modified {new FileInfo(assembly.Location).LastWriteTime}"; + Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi"); + string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + Console.WriteLine(" .NET Core: " + GetAssemblyInfo(typeof(object).Assembly)); Console.WriteLine(" ASP.NET Core: " + GetAssemblyInfo(typeof(WebHost).Assembly)); Console.WriteLine(" System.Net.Http: " + GetAssemblyInfo(typeof(System.Net.Http.HttpClient).Assembly)); @@ -169,6 +176,8 @@ private static async Task Run(Configuration config) Console.WriteLine(" Concurrency: " + config.ConcurrentRequests); Console.WriteLine(" Content Length: " + config.MaxContentLength); Console.WriteLine(" HTTP Version: " + config.HttpVersion); + Console.WriteLine(" QUIC supported: " + (IsQuicSupported ? "yes" : "no")); + Console.WriteLine(" MsQuic Version: " + msQuicLibraryVersion); Console.WriteLine(" Lifetime: " + (config.ConnectionLifetime.HasValue ? $"{config.ConnectionLifetime.Value.TotalMilliseconds}ms" : "(infinite)")); Console.WriteLine(" Operations: " + string.Join(", ", usedClientOperations.Select(o => o.name))); Console.WriteLine(" Random Seed: " + config.RandomSeed); @@ -177,6 +186,23 @@ private static async Task Run(Configuration config) Console.WriteLine("Query Parameters: " + config.MaxParameters); Console.WriteLine(); + if (config.HttpVersion == HttpVersion.Version30 && IsQuicSupported) + { + unsafe + { + // If the system gets overloaded, MsQuic has a tendency to drop incoming connections, see https://github.com/dotnet/runtime/issues/55979. + // So in case we're running H/3 stress test, we're using the same hack as for System.Net.Quic tests, which increases the time limit for pending operations in MsQuic thread pool. + object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty()))); + QUIC_SETTINGS settings = default(QUIC_SETTINGS); + settings.IsSet.MaxWorkerQueueDelayUs = 1; + settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default + if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings))) + { + Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs."); + } + } + } StressServer? server = null; if (config.RunMode.HasFlag(RunMode.server)) diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1 b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1 index b0509e14879d9..dbdd2e696c634 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1 +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1 @@ -14,7 +14,7 @@ if (-not ([string]::IsNullOrEmpty($args[0]))) { $LibrariesConfiguration = "Release" if (-not ([string]::IsNullOrEmpty($args[1]))) { - $LibrariesConfiguration = $args[0] + $LibrariesConfiguration = $args[1] } $TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64" @@ -53,11 +53,11 @@ if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) { Write-Host "Building solution." dotnet build -c $StressConfiguration -$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1" +$Runscript=".\run-stress-$StressConfiguration-$LibrariesConfiguration.ps1" if (-not (Test-Path $Runscript)) { Write-Host "Generating Runscript." Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec --roll-forward Major ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args" } Write-Host "To run tests type:" -Write-Host "$Runscript [stress test args]" \ No newline at end of file +Write-Host "$Runscript [stress test args]" diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile index 4c4539b43fd93..b090d4eb24636 100644 --- a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile +++ b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile @@ -12,6 +12,7 @@ ARG VERSION=8.0 ARG CONFIGURATION=Release RUN dotnet build -c $env:CONFIGURATION ` + -p:MsQuicInteropIncludes="C:/live-runtime-artifacts/msquic-interop/*.cs" ` -p:TargetingPacksTargetsLocation=C:/live-runtime-artifacts/targetingpacks.targets ` -p:MicrosoftNetCoreAppRefPackDir=C:/live-runtime-artifacts/microsoft.netcore.app.ref/ ` -p:MicrosoftNetCoreAppRuntimePackDir=C:/live-runtime-artifacts/microsoft.netcore.app.runtime.win-x64/$env:CONFIGURATION/ diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs index 53842f8c43ac7..778b2d72d45cb 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs @@ -130,7 +130,7 @@ static MsQuicApi() } string? gitHash = Marshal.PtrToStringUTF8((IntPtr)libGitHash); - MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} version={version} commit={gitHash}"; + MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {version} ({gitHash})"; if (version < s_minMsQuicVersion) { @@ -143,7 +143,7 @@ static MsQuicApi() if (NetEventSource.Log.IsEnabled()) { - NetEventSource.Info(null, $"Loaded MsQuic library version '{version}', commit '{gitHash}'."); + NetEventSource.Info(null, $"Loaded MsQuic library '{MsQuicLibraryVersion}'."); } // Assume SChannel is being used on windows and query for the actual provider from the library if querying is supported diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs index 345aa6d236a58..ae10560992461 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs @@ -335,7 +335,7 @@ public async Task ConnectWithServerCertificateCallback() // TODO: the exception may change if we implement https://github.com/dotnet/runtime/issues/73152 to make server close // connections with CONNECTION_REFUSED in such cases var authEx = await Assert.ThrowsAsync(() => clientTask); - Assert.Contains("UserCanceled", authEx.Message); + Assert.Contains(TlsAlertMessage.UserCanceled.ToString(), authEx.Message); Assert.Equal(clientOptions.ClientAuthenticationOptions.TargetHost, receivedHostName); await Assert.ThrowsAsync(async () => await listener.AcceptConnectionAsync()); diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs index 34872a740e1fb..c5f7fb5c751e4 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs @@ -13,8 +13,8 @@ using Xunit.Abstractions; using System.Diagnostics.Tracing; using System.Net.Sockets; +using System.Reflection; using Microsoft.Quic; -using static Microsoft.Quic.MsQuic; namespace System.Net.Quic.Tests { @@ -44,14 +44,20 @@ public abstract class QuicTestBase : IDisposable static unsafe QuicTestBase() { - Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{MsQuicApi.MsQuicLibraryVersion}'."); + // If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack. + Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi"); + + string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'."); if (IsSupported) { + object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty()))); QUIC_SETTINGS settings = default(QUIC_SETTINGS); settings.IsSet.MaxWorkerQueueDelayUs = 1; settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default - if (StatusFailed(MsQuicApi.Api.ApiTable->SetParam(null, QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings))) + if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings))) { Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs."); } diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj b/src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj index 2b31dee92406b..55ddd30f86abb 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj @@ -16,6 +16,7 @@ + @@ -31,18 +32,8 @@ - - - - - - - - - - - +