Enabling performance tests #1796
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,242 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Xml.Linq; | ||
| using FluentAssertions; | ||
| using Microsoft.NET.TestFramework; | ||
| using Microsoft.NET.TestFramework.Assertions; | ||
| using Microsoft.NET.TestFramework.Commands; | ||
| using Microsoft.NET.TestFramework.ProjectConstruction; | ||
| using Microsoft.Xunit.Performance.Api; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Microsoft.NET.Perf.Tests | ||
| { | ||
| public class BuildPerf : SdkTest | ||
| { | ||
| public BuildPerf(ITestOutputHelper log) : base(log) | ||
| { | ||
| } | ||
|
|
||
| // These tests are currently disabled for full framework MSBuild because the CI machines don't | ||
| // have an MSBuild that supports the /restore command-line argument | ||
| // Also, Microsoft.Xunit.Performance.Api.ProcessExitInfo doesn't handle non-Windows | ||
| // information gathering - disabling for non-Windows | ||
|
There was a problem hiding this comment. Please file a bug for this in https://github.com/microsoft/xunit-performance/issues and link to that issue in this comment. There was a problem hiding this comment. File a bug in https://github.com/Microsoft/xunit-performance for this, and link to it from this comment. |
||
| [CoreMSBuildAndWindowsOnlyTheory] | ||
| [InlineData(ProjectPerfOperation.CleanBuild)] | ||
| [InlineData(ProjectPerfOperation.BuildWithNoChanges)] | ||
| public void BuildNetCore2App(ProjectPerfOperation operation) | ||
| { | ||
| var testProject = new TestProject() | ||
| { | ||
| Name = "NetCoreApp", | ||
| TargetFrameworks = "netcoreapp2.0", | ||
| IsSdkProject = true, | ||
| IsExe = true | ||
| }; | ||
|
|
||
| var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: operation.ToString()); | ||
|
|
||
| TestProject(testAsset.Path, ".NET Core 2 Console App", operation); | ||
|
There was a problem hiding this comment. This name, TestProject, does it mean go run a test on the project or is it a more of a name to create TestProject, which happens to do things like build. The name seems strange here. Does not read like an action. At least, I didn't read it as such at first. |
||
| } | ||
|
|
||
| [CoreMSBuildAndWindowsOnlyTheory] | ||
| [InlineData(ProjectPerfOperation.CleanBuild)] | ||
| [InlineData(ProjectPerfOperation.BuildWithNoChanges)] | ||
| public void BuildNetStandard2Library(ProjectPerfOperation operation) | ||
| { | ||
| var testProject = new TestProject() | ||
| { | ||
| Name = "NetCoreApp", | ||
| TargetFrameworks = "netstandard2.0", | ||
| IsSdkProject = true | ||
| }; | ||
|
|
||
| var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: operation.ToString()); | ||
|
|
||
| TestProject(testAsset.Path, ".NET Standard 2.0 Library", operation); | ||
| } | ||
|
|
||
| [CoreMSBuildAndWindowsOnlyTheory] | ||
| [InlineData(ProjectPerfOperation.CleanBuild)] | ||
| [InlineData(ProjectPerfOperation.BuildWithNoChanges)] | ||
| public void BuildMVCApp(ProjectPerfOperation operation) | ||
| { | ||
| var testDir = _testAssetsManager.CreateTestDirectory(identifier: operation.ToString()); | ||
| var newCommand = new DotnetCommand(Log); | ||
| newCommand.WorkingDirectory = testDir.Path; | ||
|
|
||
| newCommand.Execute("new", "mvc", "--no-restore").Should().Pass(); | ||
|
|
||
| TestProject(testDir.Path, "ASP.NET Core MVC app", operation); | ||
| } | ||
|
|
||
| [CoreMSBuildOnlyTheory(Skip ="The code for these scenarios needs to be acquired during the test run (instead of relying on hard-coded local path)")] | ||
|
|
||
|
There was a problem hiding this comment. NIT: empty line here seems weird. |
||
| [InlineData("SmallP2POldCsproj", ProjectPerfOperation.CleanBuild)] | ||
| [InlineData("SmallP2POldCsproj", ProjectPerfOperation.BuildWithNoChanges)] | ||
| [InlineData("SmallP2PNewCsproj", ProjectPerfOperation.CleanBuild)] | ||
| [InlineData("SmallP2PNewCsproj", ProjectPerfOperation.BuildWithNoChanges)] | ||
| [InlineData("LargeP2POldCsproj", ProjectPerfOperation.CleanBuild)] | ||
| [InlineData("LargeP2POldCsproj", ProjectPerfOperation.BuildWithNoChanges)] | ||
| public void BuildProjectFromPerfSuite(string name, ProjectPerfOperation operation) | ||
| { | ||
| string sourceProject = Path.Combine(@"C:\MSBPerf\3", name); | ||
| var testDir = _testAssetsManager.CreateTestDirectory("Perf_" + name, identifier: operation.ToString()); | ||
| FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); | ||
|
|
||
| // The generated projects target .NET Core 2.1, retarget them to .NET Core 2.0 | ||
| foreach (var projFile in Directory.GetFiles(testDir.Path, "*.csproj", SearchOption.AllDirectories)) | ||
| { | ||
| var project = XDocument.Load(projFile); | ||
| var ns = project.Root.Name.Namespace; | ||
|
|
||
| // Find both TargetFramework and TargetFrameworks elements | ||
| var targetFrameworkElements = project.Root.Elements(ns + "PropertyGroup").Elements("TargetFramework"); | ||
| targetFrameworkElements = targetFrameworkElements.Concat(project.Root.Elements(ns + "PropertyGroup").Elements("TargetFrameworks")); | ||
|
|
||
| foreach (var tfElement in targetFrameworkElements) | ||
| { | ||
| tfElement.Value = tfElement.Value.Replace("netcoreapp2.1", "netcoreapp2.0"); | ||
| } | ||
|
|
||
| project.Save(projFile); | ||
| } | ||
|
|
||
| TestProject(testDir.Path, name, operation); | ||
| } | ||
|
|
||
| [CoreMSBuildOnlyTheory(Skip ="This test needs to clone the Roslyn repo and checkout a given commit instead of relying on a local copy of the repo")] | ||
| [InlineData(ProjectPerfOperation.CleanBuild)] | ||
| [InlineData(ProjectPerfOperation.BuildWithNoChanges)] | ||
| public void BuildRoslynCompilers(ProjectPerfOperation operation) | ||
| { | ||
|
|
||
| string sourceProject = @"C:\git\roslyn"; | ||
| var testDir = _testAssetsManager.CreateTestDirectory("Perf_Roslyn", identifier: operation.ToString()); | ||
| Console.WriteLine($"Mirroring {sourceProject} to {testDir.Path}..."); | ||
| FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); | ||
| Console.WriteLine("Done"); | ||
|
|
||
| // Override global.json from repo | ||
| File.Delete(Path.Combine(testDir.Path, "global.json")); | ||
|
|
||
| // Run Roslyn's restore script | ||
| var restoreCmd = new SdkCommandSpec() | ||
| { | ||
| FileName = Path.Combine(testDir.Path, "Restore.cmd"), | ||
| WorkingDirectory = testDir.Path | ||
| }; | ||
| TestContext.Current.AddTestEnvironmentVariables(restoreCmd); | ||
| restoreCmd.ToCommand().Execute().Should().Pass(); | ||
|
|
||
| TestProject(Path.Combine(testDir.Path, "Compilers.sln"), "Roslyn", operation); | ||
| } | ||
|
|
||
| public enum ProjectPerfOperation | ||
| { | ||
| CleanBuild, | ||
| BuildWithNoChanges, | ||
| NoOpRestore | ||
| } | ||
|
|
||
| private void TestProject(string projectFolderOrFile, string testName, ProjectPerfOperation perfOperation) | ||
| { | ||
| string testProjectPath; | ||
| string testProjectDirectory; | ||
| bool projectFileSpecified; | ||
|
|
||
| if (File.Exists(projectFolderOrFile)) | ||
| { | ||
| projectFileSpecified = true; | ||
| testProjectPath = projectFolderOrFile; | ||
| testProjectDirectory = Path.GetDirectoryName(projectFolderOrFile); | ||
| } | ||
| else | ||
| { | ||
| projectFileSpecified = false; | ||
| testProjectPath = Directory.GetFiles(projectFolderOrFile, "*.sln", SearchOption.AllDirectories).SingleOrDefault(); | ||
| if (testProjectPath == null) | ||
| { | ||
| testProjectPath = Directory.GetFiles(projectFolderOrFile, "*.csproj", SearchOption.AllDirectories).SingleOrDefault(); | ||
| if (testProjectPath == null) | ||
| { | ||
| throw new ArgumentException("Could not find project file to test in folder: " + projectFolderOrFile); | ||
| } | ||
| } | ||
| testProjectDirectory = Path.GetDirectoryName(testProjectPath); | ||
| } | ||
|
|
||
| TestCommand commandToTest; | ||
| var perfTest = new PerfTest(); | ||
| perfTest.ScenarioName = testName; | ||
|
|
||
| if (perfOperation == ProjectPerfOperation.NoOpRestore) | ||
| { | ||
| TestCommand restoreCommand; | ||
|
|
||
| if (TestContext.Current.ToolsetUnderTest.ShouldUseFullFrameworkMSBuild) | ||
| { | ||
| restoreCommand = new RestoreCommand(Log, testProjectPath); | ||
| } | ||
| else | ||
| { | ||
| restoreCommand = new RestoreCommand(Log, testProjectPath); | ||
| restoreCommand = new DotnetCommand(Log, "restore"); | ||
| if (projectFileSpecified) | ||
| { | ||
| restoreCommand.Arguments.Add(testProjectPath); | ||
| } | ||
| } | ||
| restoreCommand.WorkingDirectory = testProjectDirectory; | ||
|
|
||
| restoreCommand.Execute().Should().Pass(); | ||
|
|
||
| commandToTest = restoreCommand; | ||
| perfTest.TestName = "Restore (No-op)"; | ||
| } | ||
| else | ||
| { | ||
| if (TestContext.Current.ToolsetUnderTest.ShouldUseFullFrameworkMSBuild) | ||
| { | ||
| commandToTest = new BuildCommand(Log, projectFileSpecified ? testProjectPath : testProjectDirectory); | ||
| commandToTest.Arguments.Add("/restore"); | ||
| } | ||
| else | ||
| { | ||
| commandToTest = new DotnetCommand(Log, "build"); | ||
| if (projectFileSpecified) | ||
| { | ||
| commandToTest.Arguments.Add(testProjectPath); | ||
| } | ||
| } | ||
| commandToTest.WorkingDirectory = testProjectDirectory; | ||
|
|
||
| if (perfOperation == ProjectPerfOperation.CleanBuild) | ||
| { | ||
| perfTest.TestName = "Build"; | ||
| } | ||
| else if (perfOperation == ProjectPerfOperation.BuildWithNoChanges) | ||
| { | ||
| // Build once before taking folder snaspshot | ||
| commandToTest.Execute().Should().Pass(); | ||
|
|
||
| perfTest.TestName = "Build (no changes)"; | ||
| } | ||
| else | ||
| { | ||
| throw new ArgumentException("Unexpected perf operation: " + perfOperation); | ||
| } | ||
| } | ||
|
|
||
| perfTest.ProcessToMeasure = commandToTest.GetProcessStartInfo(); | ||
| perfTest.TestFolder = testProjectDirectory; | ||
|
|
||
| perfTest.Run(); | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Text; | ||
|
|
||
| namespace Microsoft.NET.Perf.Tests | ||
| { | ||
| class FolderSnapshot : IDisposable | ||
| { | ||
| string OriginalPath { get; set; } | ||
| string BackupPath { get; set; } | ||
|
|
||
|
There was a problem hiding this comment. NIT: there is an extra empty line here. |
||
|
|
||
| public static FolderSnapshot Create(string path) | ||
| { | ||
| FolderSnapshot ret = new FolderSnapshot(); | ||
|
|
||
|
|
||
| ret.OriginalPath = path; | ||
| ret.BackupPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); | ||
|
|
||
| MirrorFiles(ret.OriginalPath, ret.BackupPath); | ||
|
|
||
| return ret; | ||
| } | ||
|
|
||
| public void Restore() | ||
| { | ||
| MirrorFiles(BackupPath, OriginalPath); | ||
| } | ||
|
|
||
| public static void MirrorFiles(string source, string dest) | ||
| { | ||
| if (Directory.Exists(dest)) | ||
| { | ||
| Directory.Delete(dest, true); | ||
| } | ||
| CopyDirectory(source, dest); | ||
|
There was a problem hiding this comment. NIT: empty line after closing brackets. Check this in the rest of the files. We should add this to our editorconfig. |
||
| } | ||
|
|
||
| static void CopyDirectory(string source, string dest) | ||
| { | ||
| Directory.CreateDirectory(dest); | ||
|
|
||
| DirectoryInfo sourceInfo = new DirectoryInfo(source); | ||
| foreach (var fileInfo in sourceInfo.GetFiles()) | ||
| { | ||
| string destFile = Path.Combine(dest, fileInfo.Name); | ||
| fileInfo.CopyTo(destFile); | ||
| } | ||
| foreach (var subdirInfo in sourceInfo.GetDirectories()) | ||
| { | ||
| // Don't mirror .git folder | ||
| if (subdirInfo.Name == ".git") | ||
| { | ||
| continue; | ||
| } | ||
|
|
||
| string destDir = Path.Combine(dest, subdirInfo.Name); | ||
| CopyDirectory(subdirInfo.FullName, destDir); | ||
| } | ||
| } | ||
|
|
||
| public void Dispose() | ||
| { | ||
| Directory.Delete(BackupPath, true); | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we want the perf tests to run as part of the unit test switch, or do we want them to run separately (which, I believe, is what most of the other repos are doing)?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We want the perf tests to run with a single iteration during normal tests. That will ensure that they don't get functionally broken. When running in the perf lab, they should run with more iterations, but this will only happen automatically after a PR is merged (though it will be possible to request a perf run for an unmerged PR).