Changed from using [assembly: CustomTestRunner...] to [assembly: Benc…

…hmarkTestRunner...].

Allow ConfigType to be specified using [assembly: BenchmarkTestRunner(typeof(MyConfig))].

samples/BenchmarkDotNet.Samples/Properties/AssemblyInfo.cs

@@ -2,7 +2,6 @@
 using System.Runtime.InteropServices;
 using BenchmarkDotNet.Properties;
 using BenchmarkDotNet.TestDriven;
-using TestDriven.Framework;
 
 [assembly: AssemblyTitle(BenchmarkDotNetInfo.Title + ".Samples")]
 [assembly: AssemblyProduct(BenchmarkDotNetInfo.Title + ".Samples")]
@@ -19,4 +18,5 @@
 [assembly: ComVisible(false)]
 [assembly: Guid("6f2232a9-0d0c-46cf-b08c-f6e28ab612e3")]
 
-[assembly: CustomTestRunner(typeof(FastAndDirtyBenchmarkTestRunner))]
+[assembly: BenchmarkTestRunner(typeof(FastAndDirtyConfig))]
+//[assembly: BenchmarkTestRunner(typeof(FastAndDirtyClrAndCoreConfig))]

src/BenchmarkDotNet.TestDriven/BenchmarkTestRunner.cs

@@ -1,4 +1,5 @@
-using System.Reflection;
+using System;
+using System.Reflection;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Reports;
@@ -7,30 +8,46 @@
 
 namespace BenchmarkDotNet.TestDriven
 {
-    public abstract class BenchmarkTestRunner : ITestRunner
+    public class BenchmarkTestRunner : ITestRunner
     {
-        public abstract IConfig GetConfig();
+        Type configType;
+
+        public BenchmarkTestRunner()
+        {
+        }
+
+        public BenchmarkTestRunner(Type configType)
+        {
+            this.configType = configType;
+        }
 
         public TestRunState RunMember(ITestListener testListener, Assembly assembly, MemberInfo member)
         {
-            var summary = run(member);
-            if (summary == null)
+            try
             {
-                return TestRunState.NoTests;
-            }
+                var summary = run(assembly, member);
+                if (summary == null)
+                {
+                    return TestRunState.NoTests;
+                }
 
-            foreach (var benchmark in summary.Benchmarks)
-            {
-                var testResult = getTestResult(benchmark);
-                testListener.TestFinished(testResult);
-            }
+                foreach (var benchmark in summary.Benchmarks)
+                {
+                    var testResult = getTestResult(benchmark);
+                    testListener.TestFinished(testResult);
+                }
 
-            testListener.TestResultsUrl(summary.ResultsDirectoryPath);
+                testListener.TestResultsUrl(summary.ResultsDirectoryPath);
 
-            return TestRunState.Success;
+                return TestRunState.Success;
+            }
+            catch(TestRunnerException e)
+            {
+                testListener.WriteLine(e.Message, Category.Warning);
+                return TestRunState.Error;
+            }
         }
 
-
         static TestResult getTestResult(Benchmark benchmark)
         {
             var testResult = new TestResult();
@@ -51,14 +68,15 @@ static string getTestRunnerName()
             return string.Format("BenchmarkDotNet {0}.{1}.{2}", version.Major, version.Minor, version.Build);
         }
 
-        Summary run(MemberInfo member)
+        Summary run(Assembly assembly, MemberInfo member)
         {
             if (member is TypeInfo)
             {
                 var type = (TypeInfo)member;
                 if (isBenchmarkType(type))
                 {
-                    return BenchmarkRunner.Run(type.AsType(), GetConfig());
+                    var config = getConfig(assembly);
+                    return BenchmarkRunner.Run(type.AsType(), config);
                 }
             }
             else if (member is MethodInfo)
@@ -67,13 +85,40 @@ Summary run(MemberInfo member)
                 if (isBenchmarkMethod(method))
                 {
                     var type = method.DeclaringType;
-                    return BenchmarkRunner.Run(type, new[] { method }, GetConfig());
+                    var config = getConfig(assembly);
+                    return BenchmarkRunner.Run(type, new[] { method }, config);
                 }
             }
 
             return null;
         }
 
+        IConfig getConfig(Assembly assembly)
+        {
+            var configType = getConfigType(assembly);
+            try
+            {
+                var config = (IConfig)Activator.CreateInstance(configType);
+                return config;
+            }
+            catch(InvalidCastException)
+            {
+                throw new TestRunnerException(string.Format("ConfigType `{0}` must implement `{1}`.",
+                    configType.FullName, typeof(IConfig).FullName));
+            }
+        }
+
+        Type getConfigType(Assembly assembly)
+        {
+            if (configType != null)
+            {
+                return configType;
+            }
+
+            var attribute = assembly.GetCustomAttribute<BenchmarkTestRunnerAttribute>();
+            return attribute.ConfigType;
+        }
+
         static bool isBenchmarkType(TypeInfo type)
         {
             foreach (var method in type.GetMethods())
@@ -101,5 +146,12 @@ public TestRunState RunAssembly(ITestListener testListener, Assembly assembly)
         {
             return TestRunState.NoTests;
         }
+
+        class TestRunnerException : Exception
+        {
+            internal TestRunnerException(string message) : base(message)
+            {
+            }
+        }
     }
 }

src/BenchmarkDotNet.TestDriven/BenchmarkTestRunnerAttribute.cs

@@ -0,0 +1,18 @@
+using System;
+using TestDriven.Framework;
+
+namespace BenchmarkDotNet.TestDriven
+{
+    public class BenchmarkTestRunnerAttribute : CustomTestRunnerAttribute
+    {
+        public BenchmarkTestRunnerAttribute(Type configType) : base(typeof(BenchmarkTestRunner))
+        {
+            ConfigType = configType;
+        }
+
+        public Type ConfigType
+        {
+            get;
+        }
+    }
+}

src/BenchmarkDotNet.TestDriven/FastAndDirtyClrAndCoreConfig.cs

@@ -0,0 +1,28 @@
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Configs;
+
+namespace BenchmarkDotNet.TestDriven
+{
+    // See https://perfdotnet.github.io/BenchmarkDotNet/faq.htm
+    public class FastAndDirtyClrAndCoreConfig : ManualConfig
+    {
+        public FastAndDirtyClrAndCoreConfig()
+        {
+            Add(DefaultConfig.Instance);
+
+            Add(Job.Clr
+                .WithLaunchCount(1)     // benchmark process will be launched only once
+                .WithIterationTime(100) // 100ms per iteration
+                .WithWarmupCount(3)     // 3 warmup iteration
+                .WithTargetCount(3)     // 3 target iteration
+            );
+
+            Add(Job.Core
+                .WithLaunchCount(1)     // benchmark process will be launched only once
+                .WithIterationTime(100) // 100ms per iteration
+                .WithWarmupCount(3)     // 3 warmup iteration
+                .WithTargetCount(3)     // 3 target iteration
+            );
+        }
+    }
+}

src/BenchmarkDotNet.TestDriven/FastAndDirtyBenchmarkTestRunner.cs → src/BenchmarkDotNet.TestDriven/FastAndDirtyConfig.cs

@@ -3,20 +3,19 @@
 
 namespace BenchmarkDotNet.TestDriven
 {
-    public class FastAndDirtyBenchmarkTestRunner : BenchmarkTestRunner
+    // See https://perfdotnet.github.io/BenchmarkDotNet/faq.htm
+    public class FastAndDirtyConfig : ManualConfig
     {
-        // See https://perfdotnet.github.io/BenchmarkDotNet/faq.htm
-        public override IConfig GetConfig()
+        public FastAndDirtyConfig()
         {
-            var config = new ManualConfig();
-            config.Add(DefaultConfig.Instance);
-            config.Add(Job.Default
+            Add(DefaultConfig.Instance);
+
+            Add(Job.Default
                 .WithLaunchCount(1)     // benchmark process will be launched only once
                 .WithIterationTime(100) // 100ms per iteration
                 .WithWarmupCount(3)     // 3 warmup iteration
                 .WithTargetCount(3)     // 3 target iteration
             );
-            return config;
         }
     }
 }

src/BenchmarkDotNet.TestDriven/Properties/AssemblyInfo.cs

@@ -1,9 +1,5 @@
 using System.Reflection;
 using System.Runtime.InteropServices;
-using TestDriven.Framework;
-using BenchmarkDotNet.TestDriven;
-
-[assembly: CustomTestRunner(typeof(FastAndDirtyBenchmarkTestRunner))]
 
 // General Information about an assembly is controlled through the following
 // set of attributes. Change these attribute values to modify the information

tests/BenchmarkDotNet.IntegrationTests.TestDriven/BenchmarkTestRunnerTests.cs

@@ -16,8 +16,7 @@ public class BenckmarkTestRunnerTests
         [Test]
         public void RunMember_NoBenchmarks_NoTests()
         {
-            var benchmarkTestRunner = Substitute.For<BenchmarkTestRunner>();
-            benchmarkTestRunner.GetConfig().Returns(new SuperQuickConfig());
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(QuickConfig));
             var testListener = Substitute.For<ITestListener>();
             var targetType = typeof(NoBenchmarks).GetTypeInfo();
 
@@ -29,8 +28,7 @@ public void RunMember_NoBenchmarks_NoTests()
         [Test]
         public void RunMember_SomeBenchmarks_Success2Tests()
         {
-            var benchmarkTestRunner = Substitute.For<BenchmarkTestRunner>();
-            benchmarkTestRunner.GetConfig().Returns(new SuperQuickConfig());
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(QuickConfig));
             var testListener = Substitute.For<ITestListener>();
             var targetType = typeof(SomeBenchmarks).GetTypeInfo();
 
@@ -43,8 +41,7 @@ public void RunMember_SomeBenchmarks_Success2Tests()
         [Test]
         public void RunMember_SomeBenchmarks_TestRunnerName()
         {
-            var benchmarkTestRunner = Substitute.For<BenchmarkTestRunner>();
-            benchmarkTestRunner.GetConfig().Returns(new SuperQuickConfig());
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(QuickConfig));
             var testListener = Substitute.For<ITestListener>();
             var targetType = typeof(SomeBenchmarks).GetTypeInfo();
             var version = typeof(BenchmarkAttribute).GetTypeInfo().Assembly.GetName().Version;
@@ -58,8 +55,7 @@ public void RunMember_SomeBenchmarks_TestRunnerName()
         [Test]
         public void RunMember_SomeBenchmarks_TestResultName()
         {
-            var benchmarkTestRunner = Substitute.For<BenchmarkTestRunner>();
-            benchmarkTestRunner.GetConfig().Returns(new SuperQuickConfig());
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(QuickConfig));
             var testListener = Substitute.For<ITestListener>();
             var targetType = typeof(SomeBenchmarks).GetTypeInfo();
             var method = new Action(new SomeBenchmarks().Benchmark1).GetMethodInfo();
@@ -73,8 +69,7 @@ public void RunMember_SomeBenchmarks_TestResultName()
         [Test]
         public void RunMember_SomeBenchmarksBenchmark1_TestResultName()
         {
-            var benchmarkTestRunner = Substitute.For<BenchmarkTestRunner>();
-            benchmarkTestRunner.GetConfig().Returns(new SuperQuickConfig());
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(QuickConfig));
             var testListener = Substitute.For<ITestListener>();
             var targetType = typeof(SomeBenchmarks).GetTypeInfo();
             var method = new Action(new SomeBenchmarks().Benchmark1).GetMethodInfo();
@@ -85,11 +80,24 @@ public void RunMember_SomeBenchmarksBenchmark1_TestResultName()
             testListener.Received(1).TestFinished(Arg.Any<TestResult>());
             testListener.Received().TestFinished(Arg.Is<TestResult>(r => r.Name == testName));
         }
+
+        [Test]
+        public void RunMember_ConfigTypeNotIConfig_ErrorWithWarning()
+        {
+            var benchmarkTestRunner = new BenchmarkTestRunner(typeof(object));
+            var testListener = Substitute.For<ITestListener>();
+            var targetType = typeof(SomeBenchmarks).GetTypeInfo();
+
+            var testRunState = benchmarkTestRunner.RunMember(testListener, targetType.Assembly, targetType);
+
+            Assert.That(testRunState, Is.EqualTo(TestRunState.Error));
+            testListener.Received(1).WriteLine(Arg.Any<string>(), Category.Warning);
+        }
     }
 
-    class SuperQuickConfig : ManualConfig
+    class QuickConfig : ManualConfig
     {
-        internal SuperQuickConfig()
+        public QuickConfig()
         {
             Add(Job.Default
                 .WithLaunchCount(1)     // benchmark process will be launched only once