Added the Project to Facilitate Notebook Based Testing (#4448)

src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API.sln

@@ -13,6 +13,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GC.Infrastructure", "GC.Inf
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GC.Analysis.API.UnitTests", "GC.Analysis.API.UnitTests\GC.Analysis.API.UnitTests.csproj", "{A7D40245-CF5C-437E-B7E7-278AEA02954D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GC.Infrastructure.NotebookTests", "GC.Infrastructure.NotebookTests\GC.Infrastructure.NotebookTests.csproj", "{2A7F5822-08D8-4D13-8FE1-8CB1428C166E}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F95454D6-F187-455D-8DA4-3D79C092DC55}"
+	ProjectSection(SolutionItems) = preProject
+		.editorconfig = .editorconfig
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -39,6 +46,10 @@ Global
 		{A7D40245-CF5C-437E-B7E7-278AEA02954D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A7D40245-CF5C-437E-B7E7-278AEA02954D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A7D40245-CF5C-437E-B7E7-278AEA02954D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2A7F5822-08D8-4D13-8FE1-8CB1428C166E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A7F5822-08D8-4D13-8FE1-8CB1428C166E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A7F5822-08D8-4D13-8FE1-8CB1428C166E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A7F5822-08D8-4D13-8FE1-8CB1428C166E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/Exceptions/NotebookException.cs

@@ -0,0 +1,8 @@
+namespace GC.Infrastructure.NotebookTests.Exceptions
+{
+    public sealed class NotebookExecutionException : Exception
+    {
+        public NotebookExecutionException(string exceptionMessage, string notebookName)
+            : base($"{notebookName} failed with: {exceptionMessage}") { }
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/Exceptions/NotebookOutputDetectionException.cs

@@ -0,0 +1,9 @@
+namespace GC.Infrastructure.NotebookTests.Exceptions
+{
+    public sealed class NotebookOutputDetectionException : Exception
+    {
+        public NotebookOutputDetectionException(string notebooks)
+            : base($"The following notebooks contained outputs that should be cleared before checking in the notebooks: {notebooks}") 
+        {}
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/FunctionalTests.cs

@@ -0,0 +1,55 @@
+using GC.Infrastructure.NotebookTests.Exceptions;
+using System.Reflection;
+
+namespace GC.Infrastructure.NotebookTests
+{
+    public class FunctionalTests
+    {
+        [SetUp]
+        public void Setup()
+        {
+            if (!Utils.CheckDotnetReplInstalled())
+            {
+                Utils.InstallDotnetRepl();
+            }
+        }
+
+        public static IEnumerable<string> GetAllNotebooks()
+        {
+            string notebookPath = Utils.GetNotebookDirectoryPath();
+            return Directory.EnumerateFiles(notebookPath, "*.ipynb", SearchOption.AllDirectories);
+        }
+
+        [Test]
+        [TestCaseSource(nameof(GetAllNotebooks))]
+        public void FunctionalTest_RunAllNotebooksToCheckForOutputs_NoOutputsExpected(string notebook)
+        {
+            bool outputDetected = Utils.CheckIfNotebookHasOutputs(notebook);
+            Assert.False(outputDetected, $"Notebook {notebook} has outputs. No outputs are expected.");
+        }
+
+        [Test]
+        [TestCase("GCAnalysisExamples.ipynb")]
+        [TestCase("CustomDynamicEvents.ipynb")]
+        [TestCase("CPUAnalysisExamples.ipynb")]
+        public void FunctionalTest_RunExamples_ExpectsSuccess(string notebookName)
+            => Utils.RunNotebookThatsExpectedToPass(Path.Combine(Utils.GetNotebookDirectoryPath(), "Examples", notebookName));
+
+        [Test]
+        [TestCase("BenchmarkAnalysis.dib")]
+        public void FunctionalTest_RunAnalysisNotebooks_ExpectedSuccess(string notebookName)
+            => Utils.RunNotebookThatsExpectedToPass(Path.Combine(Utils.GetNotebookDirectoryPath(), notebookName));
+
+        [Test]
+        [TestCase("CompilationFailure.dib")]
+        [TestCase("CompilationFailure.ipynb")]
+        [TestCase("ExceptionFailure.dib")]
+        [TestCase("ExceptionFailure.ipynb")]
+        public void FunctionalTest_IntentionallyFailedNotebook_Failure(string notebookName)
+        {
+            string? executionDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+            string? notebookPath = Path.Combine(executionDirectory!, "TestNotebooks", notebookName);
+            Utils.RunNotebookThatsExpectedToFail(notebookPath);
+        }
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.csproj

@@ -0,0 +1,44 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="NUnit" Version="3.14.0" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.9.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="NUnit.Framework" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="TestNotebooks\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="TestNotebooks\CompilationFailure.dib">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestNotebooks\CompilationFailure.ipynb">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestNotebooks\ExceptionFailure.dib">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestNotebooks\ExceptionFailure.ipynb">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GC.Infrastructure.NotebookTests", "GC.Infrastructure.NotebookTests.csproj", "{947744DF-0F5D-4640-AC4C-7F5D76E3319F}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {B7770BE1-0EEE-474E-98C7-6B8FF7EC4ED0}
+	EndGlobalSection
+EndGlobal

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/NotebookParser/NotebookDataModel.cs

@@ -0,0 +1,152 @@
+using Newtonsoft.Json;
+
+namespace GC.Infrastructure.NotebookTests.NotebookParser
+{
+    // NotebookRoot myDeserializedClass = JsonConvert.DeserializeObject<NotebookRoot>(myJsonResponse);
+    public class Cell
+    {
+        [JsonProperty("cell_type")]
+        public required string CellType { get; set; }
+
+        [JsonProperty("execution_count")]
+        public required object ExecutionCount { get; set; }
+
+        [JsonProperty("metadata")]
+        public required Metadata Metadata { get; set; }
+
+        [JsonProperty("outputs")]
+        public List<Output> Outputs { get; } = new List<Output>();
+
+        [JsonProperty("source")]
+        public List<string> Source { get; } = new List<string>();
+    }
+
+    public class Data
+    {
+        [JsonProperty("text/html")]
+        public List<string> TextHtml { get; } = new List<string>();
+    }
+
+    public class DotnetInteractive
+    {
+        [JsonProperty("language")]
+        public required string Language { get; set; }
+
+        [JsonProperty("defaultKernelName")]
+        public required string DefaultKernelName { get; set; }
+
+        [JsonProperty("items")]
+        public List<Item> Items { get; } = new List<Item>();
+    }
+
+    public class Item
+    {
+        [JsonProperty("name")]
+        public required string Name { get; set; }
+    }
+
+    public class Kernelspec
+    {
+        [JsonProperty("display_name")]
+        public required string DisplayName { get; set; }
+
+        [JsonProperty("language")]
+        public required string Language { get; set; }
+
+        [JsonProperty("name")]
+        public required string Name { get; set; }
+    }
+
+    public class LanguageInfo
+    {
+        [JsonProperty("file_extension")]
+        public required string FileExtension { get; set; }
+
+        [JsonProperty("mimetype")]
+        public required string Mimetype { get; set; }
+
+        [JsonProperty("name")]
+        public required string Name { get; set; }
+
+        [JsonProperty("pygments_lexer")]
+        public required string PygmentsLexer { get; set; }
+
+        [JsonProperty("version")]
+        public required string Version { get; set; }
+    }
+
+    public class Metadata
+    {
+        [JsonProperty("dotnet_repl_cellExecutionStartTime")]
+        public DateTime DotnetReplCellExecutionStartTime { get; set; }
+
+        [JsonProperty("dotnet_repl_cellExecutionEndTime")]
+        public DateTime DotnetReplCellExecutionEndTime { get; set; }
+
+        [JsonProperty("dotnet_interactive")]
+        public required DotnetInteractive DotnetInteractive { get; set; }
+
+        [JsonProperty("polyglot_notebook")]
+        public required PolyglotNotebook PolyglotNotebook { get; set; }
+
+        [JsonProperty("kernelspec")]
+        public required Kernelspec Kernelspec { get; set; }
+
+        [JsonProperty("language_info")]
+        public required LanguageInfo LanguageInfo { get; set; }
+    }
+
+    public class Output
+    {
+        [JsonProperty("name")]
+        public required string Name { get; set; }
+
+        [JsonProperty("output_type")]
+        public required string OutputType { get; set; }
+
+        [JsonProperty("text")]
+        public List<string> Text { get; } = new List<string>();
+
+        [JsonProperty("data")]
+        public required Data Data { get; set; }
+
+        [JsonProperty("metadata")]
+        public required Metadata Metadata { get; set; }
+
+        [JsonProperty("ename")]
+        public required string Ename { get; set; }
+
+        [JsonProperty("evalue")]
+        public required string Evalue { get; set; }
+
+        [JsonProperty("traceback")]
+        public List<string> Traceback { get; } = new List<string>();
+    }
+
+    public class PolyglotNotebook
+    {
+        [JsonProperty("kernelName")]
+        public required string KernelName { get; set; }
+
+        [JsonProperty("defaultKernelName")]
+        public required string DefaultKernelName { get; set; }
+
+        [JsonProperty("items")]
+        public List<Item> Items { get; } = new List<Item>();
+    }
+
+    public class NotebookRoot
+    {
+        [JsonProperty("cells")]
+        public List<Cell> Cells { get; } = new List<Cell>();
+
+        [JsonProperty("metadata")]
+        public required Metadata Metadata { get; set; }
+
+        [JsonProperty("nbformat")]
+        public int Nbformat { get; set; }
+
+        [JsonProperty("nbformat_minor")]
+        public int NbformatMinor { get; set; }
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/NotebookRunner.cs

@@ -0,0 +1,64 @@
+using GC.Infrastructure.NotebookTests.NotebookParser;
+using Newtonsoft.Json;
+using System.Diagnostics;
+
+namespace GC.Infrastructure.NotebookTests
+{
+    public sealed class NotebookRunner : IDisposable
+    {
+        internal const int TIMEOUT = 120_000;
+        private bool disposedValue;
+        private Process? _dotnetReplProcess;
+        private readonly string _tempNotebookPath;
+
+        public NotebookRunner(string notebookPath)
+        {
+            NotebookPath = notebookPath;
+            _dotnetReplProcess = new Process();
+            _dotnetReplProcess.StartInfo.FileName = "dotnet-repl";
+            _dotnetReplProcess.StartInfo.UseShellExecute = false;
+            _dotnetReplProcess.StartInfo.Arguments = $"--run {notebookPath} --exit-after-run";
+            _tempNotebookPath = Path.GetTempFileName();
+            _dotnetReplProcess.StartInfo.Arguments += $" --output-path {_tempNotebookPath}";
+            _dotnetReplProcess.StartInfo.WorkingDirectory = Path.GetDirectoryName(notebookPath);
+            _dotnetReplProcess!.Start();
+            _dotnetReplProcess.WaitForExit(TIMEOUT);
+
+            // Assumption: dotnet-repl _always_ creates an output notebook.
+            NotebookRoot = JsonConvert.DeserializeObject<NotebookRoot>(File.ReadAllText(_tempNotebookPath));
+        }
+
+        public string NotebookPath { get; }
+        public NotebookRoot? NotebookRoot { get; private set; }
+        public bool? HasExited => _dotnetReplProcess?.HasExited;
+        public int? ExitCode => _dotnetReplProcess?.ExitCode;
+
+        private void Dispose(bool disposing)
+        {
+            if (!disposedValue)
+            {
+                if (disposing)
+                {
+                    NotebookRoot = null;
+                }
+
+                File.Delete(_tempNotebookPath);
+                _dotnetReplProcess?.Dispose();
+                _dotnetReplProcess = null;
+                disposedValue = true;
+            }
+        }
+
+        ~NotebookRunner()
+        {
+            Dispose(disposing: false);
+        }
+
+        public void Dispose()
+        {
+            // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+            Dispose(disposing: true);
+            System.GC.SuppressFinalize(this);
+        }
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/TestNotebooks/CompilationFailure.dib

@@ -0,0 +1,8 @@
+#!meta
+
+{"kernelInfo":{"defaultKernelName":"csharp","items":[{"aliases":[],"name":"csharp"}]}}
+
+#!csharp
+
+int x = 
+int y =