From d8e9d87d3a523c01afb07d3b723b0202b57486c7 Mon Sep 17 00:00:00 2001
From: Nate Bosch <nbosch@google.com>
Date: Mon, 18 Sep 2023 12:37:33 -0700
Subject: [PATCH] Add a silent reporter (#2093)

Resolve a flaky test by eliminating the reporter output as a
potential race condition.

Add a `hidden` flag to `ReporterDetails` since the reporter is not worth
showing in the CLI usage help. Request the silent reporter for a test
which is flaky due to interleaving of forwarded `stdout` with the
reporter's output. During flakes the `hello` and the newline can come as
separate events in the stream, so sometimes the newline is written
following unexpected reporter content.
---
 .../test/test/runner/compiler_runtime_matrix_test.dart |  2 +-
 pkgs/test_core/CHANGELOG.md                            |  3 +++
 pkgs/test_core/lib/src/runner/configuration/args.dart  | 10 +++++-----
 .../lib/src/runner/configuration/reporters.dart        |  7 ++++++-
 pkgs/test_core/lib/src/runner/reporter.dart            |  9 +++++++++
 5 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
index d8f2615a4..98fbb89d3 100644
--- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
+++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
@@ -102,7 +102,7 @@ void main() {
         if (runtime.isDartVM) {
           test('forwards stdout/stderr', () async {
             await d.file('test.dart', _testWithStdOutAndErr).create();
-            var test = await runTest(testArgs);
+            var test = await runTest(testArgs, reporter: 'silent');
 
             expect(test.stdout, emitsThrough('hello'));
             expect(test.stderr, emits('world'));
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index dba17edd4..85cd8152e 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 * Pass --disable-program-split to dart2js to fix tests which use deferred
   loading.
+* Add a 'silent' reporter option. Keep it hidden in the CLI args help since it
+  is not useful in the general case, but can be useful for tests of the test
+  runner.
 
 ## 0.5.6
 
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index 5faa52da7..7489d5eb3 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -137,17 +137,17 @@ final ArgParser _parser = (() {
   parser.addFlag('fail-fast',
       help: 'Stop running tests after the first failure.\n');
 
-  var reporterDescriptions = <String, String>{};
-  for (var reporter in allReporters.keys) {
-    reporterDescriptions[reporter] = allReporters[reporter]!.description;
-  }
+  var reporterDescriptions = <String, String>{
+    for (final MapEntry(:key, :value) in allReporters.entries)
+      if (!value.hidden) key: value.description
+  };
 
   parser.addSeparator('Output:');
   parser.addOption('reporter',
       abbr: 'r',
       help: 'Set how to print test results.',
       defaultsTo: defaultReporter,
-      allowed: reporterDescriptions.keys.toList(),
+      allowed: allReporters.keys,
       allowedHelp: reporterDescriptions,
       valueHelp: 'option');
   parser.addOption('file-reporter',
diff --git a/pkgs/test_core/lib/src/runner/configuration/reporters.dart b/pkgs/test_core/lib/src/runner/configuration/reporters.dart
index d52f16328..271597e2e 100644
--- a/pkgs/test_core/lib/src/runner/configuration/reporters.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/reporters.dart
@@ -22,7 +22,8 @@ typedef ReporterFactory = Reporter Function(Configuration, Engine, StringSink);
 class ReporterDetails {
   final String description;
   final ReporterFactory factory;
-  ReporterDetails(this.description, this.factory);
+  final bool hidden;
+  ReporterDetails(this.description, this.factory, {this.hidden = false});
 }
 
 /// All reporters and their corresponding details.
@@ -58,6 +59,10 @@ final _allReporters = <String, ReporterDetails>{
       'https://dart.dev/go/test-docs/json_reporter.md).',
       (config, engine, sink) =>
           JsonReporter.watch(engine, sink, isDebugRun: config.debug)),
+  'silent': ReporterDetails(
+      hidden: true,
+      'A reporter with no output.',
+      (config, engine, sink) => SilentReporter()),
 };
 
 final defaultReporter = inTestTests
diff --git a/pkgs/test_core/lib/src/runner/reporter.dart b/pkgs/test_core/lib/src/runner/reporter.dart
index 7caa944a1..b1793370d 100644
--- a/pkgs/test_core/lib/src/runner/reporter.dart
+++ b/pkgs/test_core/lib/src/runner/reporter.dart
@@ -20,3 +20,12 @@ abstract class Reporter {
   /// paused.
   void resume();
 }
+
+/// A reporter that prints nothing.
+class SilentReporter implements Reporter {
+  @override
+  void pause() {}
+
+  @override
+  void resume() {}
+}