Skip to content

Commit

Permalink
[GH-35] - Making benchmarks usable for compilation analyzers
Browse files Browse the repository at this point in the history
  • Loading branch information
@tpodolak
tpodolak committed Sep 15, 2019
1 parent 1fee36c commit 117dcc4
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 65 deletions.
8 changes: 8 additions & 0 deletions benchmarks/NSubstitute.Analyzers.Benchmarks/Shared/AbstractDiagnosticAnalyzersBenchmarks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Reflection;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Engines;
using Microsoft.CodeAnalysis;

namespace NSubstitute.Analyzers.Benchmarks.Shared
Expand Down Expand Up @@ -99,6 +100,13 @@ public void ArgumentMatcherAnalyzer()
ArgumentMatcherAnalyzerBenchmark.Run();
}

[IterationCleanup(Target = nameof(ArgumentMatcherAnalyzer))]
public void CleanUp()
{
// clearing previously captured actions, as solution-wide analyzer keeps state
ArgumentMatcherAnalyzerBenchmark.RefreshActions();
}

private string GetBenchmarkSourceProjectPath()
{
var rootDirectory = FindRootDirectory();
Expand Down
205 changes: 140 additions & 65 deletions benchmarks/NSubstitute.Analyzers.Benchmarks/Shared/AnalyzerBenchmark.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,55 @@ namespace NSubstitute.Analyzers.Benchmarks.Shared
{
public class AnalyzerBenchmark
{
private AnalyzerBenchmark(
DiagnosticAnalyzer analyzer,
IReadOnlyList<ContextAndAction<SyntaxNodeAnalysisContext>> syntaxNodeActions,
IReadOnlyList<ContextAndAction<CompilationStartAnalysisContext>> compilationStartActions,
IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> compilationActions,
IReadOnlyList<ContextAndAction<SemanticModelAnalysisContext>> semanticModelActions,
IReadOnlyList<ContextAndAction<SymbolAnalysisContext>> symbolActions,
IReadOnlyList<IContextAndAction> codeBlockStartActions,
IReadOnlyList<ContextAndAction<CodeBlockAnalysisContext>> codeBlockActions,
IReadOnlyList<ContextAndAction<SyntaxTreeAnalysisContext>> syntaxTreeActions,
IReadOnlyList<ContextAndAction<OperationAnalysisContext>> operationActions,
IReadOnlyList<ContextAndAction<OperationBlockAnalysisContext>> operationBlockActions,
IReadOnlyList<ContextAndAction<OperationBlockStartAnalysisContext>> operationBlockStartActions)
private readonly BenchmarkAnalyzer _benchmarkAnalyzer;
private readonly Solution _solution;

private AnalyzerBenchmark(DiagnosticAnalyzer analyzer, BenchmarkAnalyzer benchmarkAnalyzer, Solution solution)
{
_benchmarkAnalyzer = benchmarkAnalyzer;
_solution = solution;
Analyzer = analyzer;
SyntaxNodeActions = syntaxNodeActions;
CompilationStartActions = compilationStartActions;
CompilationActions = compilationActions;
SemanticModelActions = semanticModelActions;
SymbolActions = symbolActions;
CodeBlockStartActions = codeBlockStartActions;
CodeBlockActions = codeBlockActions;
SyntaxTreeActions = syntaxTreeActions;
OperationActions = operationActions;
OperationBlockActions = operationBlockActions;
OperationBlockStartActions = operationBlockStartActions;
RefreshActions();
}

public void RefreshActions()
{
_benchmarkAnalyzer.ClearActions();

GetDiagnosticsAsync(_solution, _benchmarkAnalyzer)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();

SyntaxNodeActions = _benchmarkAnalyzer.SyntaxNodeActions;
CompilationStartActions = _benchmarkAnalyzer.CompilationStartActions;
CompilationEndActions = _benchmarkAnalyzer.CompilationEndActions;
CompilationActions = _benchmarkAnalyzer.CompilationActions;
SemanticModelActions = _benchmarkAnalyzer.SemanticModelActions;
SymbolActions = _benchmarkAnalyzer.SymbolActions;
CodeBlockStartActions = _benchmarkAnalyzer.CodeBlockStartActions;
CodeBlockActions = _benchmarkAnalyzer.CodeBlockActions;
SyntaxTreeActions = _benchmarkAnalyzer.SyntaxTreeActions;
OperationActions = _benchmarkAnalyzer.OperationActions;
OperationBlockActions = _benchmarkAnalyzer.OperationBlockActions;
OperationBlockStartActions = _benchmarkAnalyzer.OperationBlockStartActions;
}

private static async Task<IReadOnlyList<ImmutableArray<Diagnostic>>> GetDiagnosticsAsync(Solution solution, DiagnosticAnalyzer analyzer)
{
var results = new List<ImmutableArray<Diagnostic>>();
foreach (var project in solution.Projects)
{
project.TryGetCompilation(out var compilation);

var withAnalyzers = compilation.WithAnalyzers(
ImmutableArray.Create(analyzer),
project.AnalyzerOptions);
results.Add(await withAnalyzers.GetAnalyzerDiagnosticsAsync(CancellationToken.None)
.ConfigureAwait(false));
}

return results;
}

public interface IContextAndAction
Expand All @@ -48,47 +71,37 @@ public interface IContextAndAction

public DiagnosticAnalyzer Analyzer { get; }

public IReadOnlyList<ContextAndAction<SyntaxNodeAnalysisContext>> SyntaxNodeActions { get; }
public IReadOnlyList<ContextAndAction<SyntaxNodeAnalysisContext>> SyntaxNodeActions { get; private set; }

public IReadOnlyList<ContextAndAction<CompilationStartAnalysisContext>> CompilationStartActions { get; }
public IReadOnlyList<ContextAndAction<CompilationStartAnalysisContext>> CompilationStartActions { get; private set; }

public IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> CompilationActions { get; }
public IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> CompilationEndActions { get; private set; }

public IReadOnlyList<ContextAndAction<SemanticModelAnalysisContext>> SemanticModelActions { get; }
public IReadOnlyList<ContextAndAction<CompilationAnalysisContext>> CompilationActions { get; private set; }

public IReadOnlyList<ContextAndAction<SymbolAnalysisContext>> SymbolActions { get; }
public IReadOnlyList<ContextAndAction<SemanticModelAnalysisContext>> SemanticModelActions { get; private set; }

public IReadOnlyList<IContextAndAction> CodeBlockStartActions { get; }
public IReadOnlyList<ContextAndAction<SymbolAnalysisContext>> SymbolActions { get; private set; }

public IReadOnlyList<ContextAndAction<CodeBlockAnalysisContext>> CodeBlockActions { get; }
public IReadOnlyList<IContextAndAction> CodeBlockStartActions { get; private set; }

public IReadOnlyList<ContextAndAction<SyntaxTreeAnalysisContext>> SyntaxTreeActions { get; }
public IReadOnlyList<ContextAndAction<CodeBlockAnalysisContext>> CodeBlockActions { get; private set; }

public IReadOnlyList<ContextAndAction<OperationAnalysisContext>> OperationActions { get; }
public IReadOnlyList<ContextAndAction<SyntaxTreeAnalysisContext>> SyntaxTreeActions { get; private set; }

public IReadOnlyList<ContextAndAction<OperationBlockAnalysisContext>> OperationBlockActions { get; }
public IReadOnlyList<ContextAndAction<OperationAnalysisContext>> OperationActions { get; private set; }

public IReadOnlyList<ContextAndAction<OperationBlockStartAnalysisContext>> OperationBlockStartActions { get; }
public IReadOnlyList<ContextAndAction<OperationBlockAnalysisContext>> OperationBlockActions { get; private set; }

public static AnalyzerBenchmark CreateBenchmark(Solution solution, DiagnosticAnalyzer analyzer) => CreateBenchmarkAsync(solution, analyzer).GetAwaiter().GetResult();
public IReadOnlyList<ContextAndAction<OperationBlockStartAnalysisContext>> OperationBlockStartActions { get; private set; }

public static async Task<AnalyzerBenchmark> CreateBenchmarkAsync(Solution solution, DiagnosticAnalyzer analyzer)
public static AnalyzerBenchmark CreateBenchmark(Solution solution, DiagnosticAnalyzer analyzer)
{
var benchmarkAnalyzer = new BenchmarkAnalyzer(analyzer);
await GetDiagnosticsAsync(solution, benchmarkAnalyzer).ConfigureAwait(false);
return new AnalyzerBenchmark(
analyzer,
benchmarkAnalyzer.SyntaxNodeActions,
benchmarkAnalyzer.CompilationStartActions,
benchmarkAnalyzer.CompilationActions,
benchmarkAnalyzer.SemanticModelActions,
benchmarkAnalyzer.SymbolActions,
benchmarkAnalyzer.CodeBlockStartActions,
benchmarkAnalyzer.CodeBlockActions,
benchmarkAnalyzer.SyntaxTreeActions,
benchmarkAnalyzer.OperationActions,
benchmarkAnalyzer.OperationBlockActions,
benchmarkAnalyzer.OperationBlockStartActions);
benchmarkAnalyzer,
solution);
}

public void Run()
Expand Down Expand Up @@ -147,24 +160,11 @@ public void Run()
{
contextAndAction.Run();
}
}

public static async Task<IReadOnlyList<ImmutableArray<Diagnostic>>> GetDiagnosticsAsync(Solution solution, DiagnosticAnalyzer analyzer)
{
var results = new List<ImmutableArray<Diagnostic>>();
foreach (var project in solution.Projects)
foreach (var contextAndAction in CompilationEndActions)
{
var compilation = await project.GetCompilationAsync(CancellationToken.None)
.ConfigureAwait(false);

var withAnalyzers = compilation.WithAnalyzers(
ImmutableArray.Create(analyzer),
project.AnalyzerOptions);
results.Add(await withAnalyzers.GetAnalyzerDiagnosticsAsync(CancellationToken.None)
.ConfigureAwait(false));
contextAndAction.Run();
}

return results;
}

public class ContextAndAction<TContext> : IContextAndAction
Expand Down Expand Up @@ -192,6 +192,8 @@ private class BenchmarkAnalyzer : DiagnosticAnalyzer

internal List<ContextAndAction<CompilationStartAnalysisContext>> CompilationStartActions { get; } = new List<ContextAndAction<CompilationStartAnalysisContext>>();

internal List<ContextAndAction<CompilationAnalysisContext>> CompilationEndActions { get; } = new List<ContextAndAction<CompilationAnalysisContext>>();

internal List<ContextAndAction<CompilationAnalysisContext>> CompilationActions { get; } = new List<ContextAndAction<CompilationAnalysisContext>>();

internal List<ContextAndAction<SemanticModelAnalysisContext>> SemanticModelActions { get; } = new List<ContextAndAction<SemanticModelAnalysisContext>>();
Expand Down Expand Up @@ -225,6 +227,22 @@ public override void Initialize(AnalysisContext context)
_inner.Initialize(benchmarkContext);
}

public void ClearActions()
{
SyntaxNodeActions.Clear();
CompilationStartActions.Clear();
CompilationEndActions.Clear();
CompilationActions.Clear();
SemanticModelActions.Clear();
SymbolActions.Clear();
CodeBlockStartActions.Clear();
CodeBlockActions.Clear();
SyntaxTreeActions.Clear();
OperationActions.Clear();
OperationBlockActions.Clear();
OperationBlockStartActions.Clear();
}

private class BenchmarkAnalysisContext : AnalysisContext
{
private readonly BenchmarkAnalyzer _analyzer;
Expand Down Expand Up @@ -254,7 +272,11 @@ public override void RegisterSyntaxNodeAction<TLanguageKindEnum>(Action<SyntaxNo
public override void RegisterCompilationStartAction(Action<CompilationStartAnalysisContext> action)
{
_context.RegisterCompilationStartAction(
x => _analyzer.CompilationStartActions.Add(new ContextAndAction<CompilationStartAnalysisContext>(x, action)));
x =>
{
_analyzer.CompilationStartActions.Add(new ContextAndAction<CompilationStartAnalysisContext>(x, action));
action(new BenchmarkCompilationStartAnalysisContext(_analyzer, x));
});
}

public override void RegisterCompilationAction(Action<CompilationAnalysisContext> action)
Expand Down Expand Up @@ -313,6 +335,59 @@ public override void RegisterOperationBlockStartAction(Action<OperationBlockStar
x => _analyzer.OperationBlockStartActions.Add(new ContextAndAction<OperationBlockStartAnalysisContext>(x, action)));
}
}

private class BenchmarkCompilationStartAnalysisContext : CompilationStartAnalysisContext
{
private readonly BenchmarkAnalyzer _analyzer;
private readonly CompilationStartAnalysisContext _inner;

#pragma warning disable RS1012 // Start action has no registered actions
public BenchmarkCompilationStartAnalysisContext(BenchmarkAnalyzer analyzer, CompilationStartAnalysisContext inner)
: base(inner.Compilation, inner.Options, inner.CancellationToken)
{
_analyzer = analyzer;
_inner = inner;
}
#pragma warning restore RS1012

public override void RegisterCompilationEndAction(Action<CompilationAnalysisContext> action)
{
_inner.RegisterCompilationEndAction(x => _analyzer.CompilationEndActions.Add(new ContextAndAction<CompilationAnalysisContext>(x, action)));
}

public override void RegisterSemanticModelAction(Action<SemanticModelAnalysisContext> action)
{
_inner.RegisterSemanticModelAction(x => _analyzer.SemanticModelActions.Add(new ContextAndAction<SemanticModelAnalysisContext>(x, action)));
}

public override void RegisterSymbolAction(Action<SymbolAnalysisContext> action, ImmutableArray<SymbolKind> symbolKinds)
{
_inner.RegisterSymbolAction(
x => _analyzer.SymbolActions.Add(new ContextAndAction<SymbolAnalysisContext>(x, action)), symbolKinds);
}

public override void RegisterCodeBlockStartAction<TLanguageKindEnum>(Action<CodeBlockStartAnalysisContext<TLanguageKindEnum>> action)
{
_inner.RegisterCodeBlockStartAction<TLanguageKindEnum>(x =>
_analyzer.CodeBlockStartActions.Add(
new ContextAndAction<CodeBlockStartAnalysisContext<TLanguageKindEnum>>(x, action)));
}

public override void RegisterCodeBlockAction(Action<CodeBlockAnalysisContext> action)
{
_inner.RegisterCodeBlockAction(x => _analyzer.CodeBlockActions.Add(new ContextAndAction<CodeBlockAnalysisContext>(x, action)));
}

public override void RegisterSyntaxTreeAction(Action<SyntaxTreeAnalysisContext> action)
{
_inner.RegisterSyntaxTreeAction(x => _analyzer.SyntaxTreeActions.Add(new ContextAndAction<SyntaxTreeAnalysisContext>(x, action)));
}

public override void RegisterSyntaxNodeAction<TLanguageKindEnum>(Action<SyntaxNodeAnalysisContext> action, ImmutableArray<TLanguageKindEnum> syntaxKinds)
{
_inner.RegisterSyntaxNodeAction(x => _analyzer.SyntaxNodeActions.Add(new ContextAndAction<SyntaxNodeAnalysisContext>(x, action)), syntaxKinds);
}
}
}
}
}

0 comments on commit 117dcc4

Please sign in to comment.