Skip to content

Commit

Permalink
[GH-35] - Making benchmarks usable for compilation analyzers
Browse files Browse the repository at this point in the history
  • Loading branch information
@tpodolak
tpodolak committed Sep 15, 2019
1 parent 1fee36c commit f5d0c95
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 6 deletions.
9 changes: 7 additions & 2 deletions benchmarks/NSubstitute.Analyzers.Benchmarks/CSharp/CSharpDiagnosticAnalyzersBenchmarks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public class CSharpDiagnosticAnalyzersBenchmarks : AbstractDiagnosticAnalyzersBe

protected override AnalyzerBenchmark UnusedReceivedAnalyzerBenchmark { get; }

protected override AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; }
protected override AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; set; }

protected override AbstractSolutionLoader SolutionLoader { get; }

Expand All @@ -45,7 +45,12 @@ public CSharpDiagnosticAnalyzersBenchmarks()
ReEntrantSetupAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new ReEntrantSetupAnalyzer());
SubstituteAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new SubstituteAnalyzer());
UnusedReceivedAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new UnusedReceivedAnalyzer());
ArgumentMatcherAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new ArgumentMatcherAnalyzer());
ArgumentMatcherAnalyzerBenchmark = CreateArgumentMatcherAnalyzerBenchmark();
}

protected override AnalyzerBenchmark CreateArgumentMatcherAnalyzerBenchmark()
{
return AnalyzerBenchmark.CreateBenchmark(Solution, new ArgumentMatcherAnalyzer());
}
}
}
13 changes: 12 additions & 1 deletion benchmarks/NSubstitute.Analyzers.Benchmarks/Shared/AbstractDiagnosticAnalyzersBenchmarks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Reflection;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Engines;
using Microsoft.CodeAnalysis;

namespace NSubstitute.Analyzers.Benchmarks.Shared
Expand All @@ -30,7 +31,7 @@ public abstract class AbstractDiagnosticAnalyzersBenchmarks

protected abstract AnalyzerBenchmark UnusedReceivedAnalyzerBenchmark { get; }

protected abstract AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; }
protected abstract AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; set; }

protected abstract AbstractSolutionLoader SolutionLoader { get; }

Expand Down Expand Up @@ -99,6 +100,16 @@ public void ArgumentMatcherAnalyzer()
ArgumentMatcherAnalyzerBenchmark.Run();
}

[IterationCleanup(Target = nameof(ArgumentMatcherAnalyzer))]
public void CleanUp()
{
// recreating argument matcher analyzer as it is a solution-wide stateful analyzer
// and there is no way to clear the state
ArgumentMatcherAnalyzerBenchmark = CreateArgumentMatcherAnalyzerBenchmark();
}

protected abstract AnalyzerBenchmark CreateArgumentMatcherAnalyzerBenchmark();

private string GetBenchmarkSourceProjectPath()
{
var rootDirectory = FindRootDirectory();
Expand Down
71 changes: 70 additions & 1 deletion benchmarks/NSubstitute.Analyzers.Benchmarks/Shared/AnalyzerBenchmark.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ private AnalyzerBenchmark(
DiagnosticAnalyzer analyzer,
IReadOnlyList<ContextAndAction<SyntaxNodeAnalysisContext>> syntaxNodeActions,
IReadOnlyList<ContextAndAction<CompilationStartAnalysisContext>> compilationStartActions,
IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> compilationEndActions,
IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> compilationActions,
IReadOnlyList<ContextAndAction<SemanticModelAnalysisContext>> semanticModelActions,
IReadOnlyList<ContextAndAction<SymbolAnalysisContext>> symbolActions,
Expand All @@ -30,6 +31,7 @@ private AnalyzerBenchmark(
Analyzer = analyzer;
SyntaxNodeActions = syntaxNodeActions;
CompilationStartActions = compilationStartActions;
CompilationEndActions = compilationEndActions;
CompilationActions = compilationActions;
SemanticModelActions = semanticModelActions;
SymbolActions = symbolActions;
Expand All @@ -52,6 +54,8 @@ public interface IContextAndAction

public IReadOnlyList<ContextAndAction<CompilationStartAnalysisContext>> CompilationStartActions { get; }

public IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> CompilationEndActions { get; }

public IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> CompilationActions { get; }

public IReadOnlyList<ContextAndAction<SemanticModelAnalysisContext>> SemanticModelActions { get; }
Expand Down Expand Up @@ -80,6 +84,7 @@ public static async Task<AnalyzerBenchmark> CreateBenchmarkAsync(Solution soluti
analyzer,
benchmarkAnalyzer.SyntaxNodeActions,
benchmarkAnalyzer.CompilationStartActions,
benchmarkAnalyzer.CompilationEndActions,
benchmarkAnalyzer.CompilationActions,
benchmarkAnalyzer.SemanticModelActions,
benchmarkAnalyzer.SymbolActions,
Expand Down Expand Up @@ -147,6 +152,11 @@ public void Run()
{
contextAndAction.Run();
}

foreach (var contextAndAction in CompilationEndActions)
{
contextAndAction.Run();
}
}

public static async Task<IReadOnlyList<ImmutableArray<Diagnostic>>> GetDiagnosticsAsync(Solution solution, DiagnosticAnalyzer analyzer)
Expand Down Expand Up @@ -192,6 +202,8 @@ private class BenchmarkAnalyzer : DiagnosticAnalyzer

internal List<ContextAndAction<CompilationStartAnalysisContext>> CompilationStartActions { get; } = new List<ContextAndAction<CompilationStartAnalysisContext>>();

internal List<ContextAndAction<CompilationAnalysisContext>> CompilationEndActions { get; } = new List<ContextAndAction<CompilationAnalysisContext>>();

internal List<ContextAndAction<CompilationAnalysisContext>> CompilationActions { get; } = new List<ContextAndAction<CompilationAnalysisContext>>();

internal List<ContextAndAction<SemanticModelAnalysisContext>> SemanticModelActions { get; } = new List<ContextAndAction<SemanticModelAnalysisContext>>();
Expand Down Expand Up @@ -254,7 +266,11 @@ public override void RegisterSyntaxNodeAction<TLanguageKindEnum>(Action<SyntaxNo
public override void RegisterCompilationStartAction(Action<CompilationStartAnalysisContext> action)
{
_context.RegisterCompilationStartAction(
x => _analyzer.CompilationStartActions.Add(new ContextAndAction<CompilationStartAnalysisContext>(x, action)));
x =>
{
_analyzer.CompilationStartActions.Add(new ContextAndAction<CompilationStartAnalysisContext>(x, action));
action(new BenchmarkCompilationStartAnalysisContext(_analyzer, x));
});
}

public override void RegisterCompilationAction(Action<CompilationAnalysisContext> action)
Expand Down Expand Up @@ -313,6 +329,59 @@ public override void RegisterOperationBlockStartAction(Action<OperationBlockStar
x => _analyzer.OperationBlockStartActions.Add(new ContextAndAction<OperationBlockStartAnalysisContext>(x, action)));
}
}

private class BenchmarkCompilationStartAnalysisContext : CompilationStartAnalysisContext
{
private readonly BenchmarkAnalyzer _analyzer;
private readonly CompilationStartAnalysisContext _inner;

#pragma warning disable RS1012 // Start action has no registered actions
public BenchmarkCompilationStartAnalysisContext(BenchmarkAnalyzer analyzer, CompilationStartAnalysisContext inner)
: base(inner.Compilation, inner.Options, inner.CancellationToken)
{
_analyzer = analyzer;
_inner = inner;
}
#pragma warning restore RS1012

public override void RegisterCompilationEndAction(Action<CompilationAnalysisContext> action)
{
_inner.RegisterCompilationEndAction(x => _analyzer.CompilationEndActions.Add(new ContextAndAction<CompilationAnalysisContext>(x, action)));
}

public override void RegisterSemanticModelAction(Action<SemanticModelAnalysisContext> action)
{
_inner.RegisterSemanticModelAction(x => _analyzer.SemanticModelActions.Add(new ContextAndAction<SemanticModelAnalysisContext>(x, action)));
}

public override void RegisterSymbolAction(Action<SymbolAnalysisContext> action, ImmutableArray<SymbolKind> symbolKinds)
{
_inner.RegisterSymbolAction(
x => _analyzer.SymbolActions.Add(new ContextAndAction<SymbolAnalysisContext>(x, action)), symbolKinds);
}

public override void RegisterCodeBlockStartAction<TLanguageKindEnum>(Action<CodeBlockStartAnalysisContext<TLanguageKindEnum>> action)
{
_inner.RegisterCodeBlockStartAction<TLanguageKindEnum>(x =>
_analyzer.CodeBlockStartActions.Add(
new ContextAndAction<CodeBlockStartAnalysisContext<TLanguageKindEnum>>(x, action)));
}

public override void RegisterCodeBlockAction(Action<CodeBlockAnalysisContext> action)
{
_inner.RegisterCodeBlockAction(x => _analyzer.CodeBlockActions.Add(new ContextAndAction<CodeBlockAnalysisContext>(x, action)));
}

public override void RegisterSyntaxTreeAction(Action<SyntaxTreeAnalysisContext> action)
{
_inner.RegisterSyntaxTreeAction(x => _analyzer.SyntaxTreeActions.Add(new ContextAndAction<SyntaxTreeAnalysisContext>(x, action)));
}

public override void RegisterSyntaxNodeAction<TLanguageKindEnum>(Action<SyntaxNodeAnalysisContext> action, ImmutableArray<TLanguageKindEnum> syntaxKinds)
{
_inner.RegisterSyntaxNodeAction(x => _analyzer.SyntaxNodeActions.Add(new ContextAndAction<SyntaxNodeAnalysisContext>(x, action)), syntaxKinds);
}
}
}
}
}
9 changes: 7 additions & 2 deletions .../NSubstitute.Analyzers.Benchmarks/VisualBasic/VisualBasicDiagnosticAnalyzersBenchmarks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public class VisualBasicDiagnosticAnalyzersBenchmarks : AbstractDiagnosticAnalyz

protected override AnalyzerBenchmark UnusedReceivedAnalyzerBenchmark { get; }

protected override AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; }
protected override AnalyzerBenchmark ArgumentMatcherAnalyzerBenchmark { get; set; }

protected override AbstractSolutionLoader SolutionLoader { get; }

Expand All @@ -45,7 +45,12 @@ public VisualBasicDiagnosticAnalyzersBenchmarks()
ReEntrantSetupAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new ReEntrantSetupAnalyzer());
SubstituteAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new SubstituteAnalyzer());
UnusedReceivedAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new UnusedReceivedAnalyzer());
ArgumentMatcherAnalyzerBenchmark = AnalyzerBenchmark.CreateBenchmark(Solution, new ArgumentMatcherAnalyzer());
ArgumentMatcherAnalyzerBenchmark = CreateArgumentMatcherAnalyzerBenchmark();
}

protected override AnalyzerBenchmark CreateArgumentMatcherAnalyzerBenchmark()
{
return AnalyzerBenchmark.CreateBenchmark(Solution, new ArgumentMatcherAnalyzer());
}
}
}

0 comments on commit f5d0c95

Please sign in to comment.