[FEATURE] Make suppressed error levels for ErrorHandler configurable

.php-cs-fixer.dist.php

@@ -15,6 +15,7 @@
     ->in(__DIR__ . '/tests/end-to-end')
     ->in(__DIR__ . '/tests/unit')
     ->notName('DeprecatedPhpFeatureTest.php')
+    ->notName('SuppressedDeprecatedPhpFeatureTest.php')
     ->notName('InterfaceWithMethodReturningDisjunctiveNormalFormType.php')
     ->notName('ReadonlyClass.php')
     ->notName('*.phpt');

ChangeLog-10.1.md

@@ -6,6 +6,7 @@ All notable changes of the PHPUnit 10.1 release series are documented in this fi
 
 ### Added
 
+* [#5302](https://github.com/sebastianbergmann/phpunit/issues/5302): Make suppressed error levels for ErrorHandler configurable
 * [#5168](https://github.com/sebastianbergmann/phpunit/issues/5168): Allow test runner extensions to disable default progress and result printing
 * [#5196](https://github.com/sebastianbergmann/phpunit/issues/5196): Optionally (fail|stop) on (notice|deprecation) events
 * [#5201](https://github.com/sebastianbergmann/phpunit/issues/5201): Add `TestCase::registerFailureType()` to register interface that marks exceptions to be treated as failures, not errors

phpunit.xsd

@@ -219,6 +219,12 @@
         <xs:attribute name="displayDetailsOnTestsThatTriggerErrors" type="xs:boolean" default="false"/>
         <xs:attribute name="displayDetailsOnTestsThatTriggerNotices" type="xs:boolean" default="false"/>
         <xs:attribute name="displayDetailsOnTestsThatTriggerWarnings" type="xs:boolean" default="false"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedPhpDeprecations" type="xs:boolean" default="true"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedPhpNotices" type="xs:boolean" default="true"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedPhpWarnings" type="xs:boolean" default="true"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedUserDeprecations" type="xs:boolean" default="false"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedUserNotices" type="xs:boolean" default="false"/>
+        <xs:attribute name="errorHandlerIgnoreSuppressedUserWarnings" type="xs:boolean" default="false"/>
     </xs:attributeGroup>
     <xs:group name="configGroup">
         <xs:all>

src/Runner/ErrorHandler.php

@@ -23,6 +23,8 @@
 use function set_error_handler;
 use PHPUnit\Event;
 use PHPUnit\Framework\TestCase;
+use PHPUnit\TextUI\Configuration\Configuration;
+use PHPUnit\TextUI\Configuration\Registry;
 
 /**
  * @internal This class is not covered by the backward compatibility promise for PHPUnit
@@ -31,10 +33,16 @@ final class ErrorHandler
 {
     private static ?self $instance = null;
     private bool $enabled          = false;
+    private array $ignoreSuppressedErrorLevels;
 
     public static function instance(): self
     {
-        return self::$instance ?? self::$instance = new self;
+        return self::$instance ?? self::$instance = new self(Registry::get());
+    }
+
+    public function __construct(Configuration $configuration)
+    {
+        $this->ignoreSuppressedErrorLevels = $this->buildSuppressibleErrorLevels($configuration);
     }
 
     /**
@@ -45,7 +53,7 @@ public function __invoke(int $errorNumber, string $errorString, string $errorFil
         $suppressed = !($errorNumber & error_reporting());
 
         if ($suppressed &&
-            in_array($errorNumber, [E_DEPRECATED, E_NOTICE, E_STRICT, E_WARNING], true)) {
+            in_array($errorNumber, $this->ignoreSuppressedErrorLevels, true)) {
             return false;
         }
 
@@ -169,4 +177,36 @@ private function testValueObjectForEvents(): Event\Code\Test
 
         throw new NoTestCaseObjectOnCallStackException;
     }
+
+    private function buildSuppressibleErrorLevels(Configuration $configuration): array
+    {
+        $levels = [];
+
+        if ($configuration->errorHandlerIgnoreSuppressedPhpNotices()) {
+            $levels[] = E_NOTICE;
+            $levels[] = E_STRICT;
+        }
+
+        if ($configuration->errorHandlerIgnoreSuppressedPhpWarnings()) {
+            $levels[] = E_WARNING;
+        }
+
+        if ($configuration->errorHandlerIgnoreSuppressedPhpDeprecations()) {
+            $levels[] = E_DEPRECATED;
+        }
+        // user land error levels
+        if ($configuration->errorHandlerIgnoreSuppressedUserNotices()) {
+            $levels[] = E_USER_NOTICE;
+        }
+
+        if ($configuration->errorHandlerIgnoreSuppressedUserWarnings()) {
+            $levels[] = E_USER_WARNING;
+        }
+
+        if ($configuration->errorHandlerIgnoreSuppressedUserDeprecations()) {
+            $levels[] = E_USER_DEPRECATED;
+        }
+
+        return $levels;
+    }
 }

src/TextUI/Configuration/Configuration.php

@@ -96,6 +96,12 @@ final class Configuration
     private readonly bool $displayDetailsOnTestsThatTriggerErrors;
     private readonly bool $displayDetailsOnTestsThatTriggerNotices;
     private readonly bool $displayDetailsOnTestsThatTriggerWarnings;
+    private readonly bool $errorHandlerIgnoreSuppressedPhpDeprecations;
+    private readonly bool $errorHandlerIgnoreSuppressedPhpNotices;
+    private readonly bool $errorHandlerIgnoreSuppressedPhpWarnings;
+    private readonly bool $errorHandlerIgnoreSuppressedUserDeprecations;
+    private readonly bool $errorHandlerIgnoreSuppressedUserNotices;
+    private readonly bool $errorHandlerIgnoreSuppressedUserWarnings;
     private readonly bool $reverseDefectList;
     private readonly bool $requireCoverageMetadata;
     private readonly bool $registerMockObjectsFromTestArgumentsRecursively;
@@ -135,7 +141,7 @@ final class Configuration
      * @psalm-param non-empty-list<string> $testSuffixes
      * @psalm-param list<array{className: class-string, parameters: array<string, string>}> $extensionBootstrappers
      */
-    public function __construct(?string $cliArgument, ?string $configurationFile, ?string $bootstrap, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, bool $filterDeprecations, bool $filterNotices, bool $filterWarnings, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnDeprecation, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int|string $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $registerMockObjectsFromTestArgumentsRecursively, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, ?array $testsCovering, ?array $testsUsing, ?string $filter, ?array $groups, ?array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, array $testSuffixes, Php $php)
+    public function __construct(?string $cliArgument, ?string $configurationFile, ?string $bootstrap, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, bool $filterDeprecations, bool $filterNotices, bool $filterWarnings, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnDeprecation, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int|string $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $errorHandlerIgnoreSuppressedPhpDeprecations, bool $errorHandlerIgnoreSuppressedPhpNotices, bool $errorHandlerIgnoreSuppressedPhpWarnings, bool $errorHandlerIgnoreSuppressedUserDeprecations, bool $errorHandlerIgnoreSuppressedUserNotices, bool $errorHandlerIgnoreSuppressedUserWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $registerMockObjectsFromTestArgumentsRecursively, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, ?array $testsCovering, ?array $testsUsing, ?string $filter, ?array $groups, ?array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, array $testSuffixes, Php $php)
     {
         $this->cliArgument                                     = $cliArgument;
         $this->configurationFile                               = $configurationFile;
@@ -209,6 +215,12 @@ public function __construct(?string $cliArgument, ?string $configurationFile, ?s
         $this->displayDetailsOnTestsThatTriggerErrors          = $displayDetailsOnTestsThatTriggerErrors;
         $this->displayDetailsOnTestsThatTriggerNotices         = $displayDetailsOnTestsThatTriggerNotices;
         $this->displayDetailsOnTestsThatTriggerWarnings        = $displayDetailsOnTestsThatTriggerWarnings;
+        $this->errorHandlerIgnoreSuppressedPhpDeprecations     = $errorHandlerIgnoreSuppressedPhpDeprecations;
+        $this->errorHandlerIgnoreSuppressedPhpNotices          = $errorHandlerIgnoreSuppressedPhpNotices;
+        $this->errorHandlerIgnoreSuppressedPhpWarnings         = $errorHandlerIgnoreSuppressedPhpWarnings;
+        $this->errorHandlerIgnoreSuppressedUserDeprecations    = $errorHandlerIgnoreSuppressedUserDeprecations;
+        $this->errorHandlerIgnoreSuppressedUserNotices         = $errorHandlerIgnoreSuppressedUserNotices;
+        $this->errorHandlerIgnoreSuppressedUserWarnings        = $errorHandlerIgnoreSuppressedUserWarnings;
         $this->reverseDefectList                               = $reverseDefectList;
         $this->requireCoverageMetadata                         = $requireCoverageMetadata;
         $this->registerMockObjectsFromTestArgumentsRecursively = $registerMockObjectsFromTestArgumentsRecursively;
@@ -873,6 +885,36 @@ public function displayDetailsOnTestsThatTriggerWarnings(): bool
         return $this->displayDetailsOnTestsThatTriggerWarnings;
     }
 
+    public function errorHandlerIgnoreSuppressedPhpDeprecations(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedPhpDeprecations;
+    }
+
+    public function errorHandlerIgnoreSuppressedPhpNotices(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedPhpNotices;
+    }
+
+    public function errorHandlerIgnoreSuppressedPhpWarnings(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedPhpWarnings;
+    }
+
+    public function errorHandlerIgnoreSuppressedUserDeprecations(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedUserDeprecations;
+    }
+
+    public function errorHandlerIgnoreSuppressedUserNotices(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedUserNotices;
+    }
+
+    public function errorHandlerIgnoreSuppressedUserWarnings(): bool
+    {
+        return $this->errorHandlerIgnoreSuppressedUserWarnings;
+    }
+
     public function reverseDefectList(): bool
     {
         return $this->reverseDefectList;

src/TextUI/Configuration/Merger.php

@@ -711,6 +711,13 @@ public function merge(CliConfiguration $cliConfiguration, XmlConfiguration $xmlC
             $sourceExcludeFiles       = $xmlConfiguration->source()->excludeFiles();
         }
 
+        $errorHandlerIgnoreSuppressedPhpDeprecations  = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedPhpDeprecations();
+        $errorHandlerIgnoreSuppressedPhpNotices       = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedPhpNotices();
+        $errorHandlerIgnoreSuppressedPhpWarnings      = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedPhpWarnings();
+        $errorHandlerIgnoreSuppressedUserDeprecations = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedUserDeprecations();
+        $errorHandlerIgnoreSuppressedUserNotices      = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedUserNotices();
+        $errorHandlerIgnoreSuppressedUserWarnings     = $xmlConfiguration->phpUnit()->errorHandlerIgnoreSuppressedUserWarnings();
+
         return new Configuration(
             $cliArgument,
             $configurationFile,
@@ -789,6 +796,12 @@ public function merge(CliConfiguration $cliConfiguration, XmlConfiguration $xmlC
             $displayDetailsOnTestsThatTriggerErrors,
             $displayDetailsOnTestsThatTriggerNotices,
             $displayDetailsOnTestsThatTriggerWarnings,
+            $errorHandlerIgnoreSuppressedPhpDeprecations,
+            $errorHandlerIgnoreSuppressedPhpNotices,
+            $errorHandlerIgnoreSuppressedPhpWarnings,
+            $errorHandlerIgnoreSuppressedUserDeprecations,
+            $errorHandlerIgnoreSuppressedUserNotices,
+            $errorHandlerIgnoreSuppressedUserWarnings,
             $reverseDefectList,
             $requireCoverageMetadata,
             $registerMockObjectsFromTestArgumentsRecursively,

src/TextUI/Configuration/Xml/DefaultConfiguration.php

@@ -97,6 +97,12 @@ public static function create(): self
                 false,
                 false,
                 false,
+                true,
+                true,
+                true,
+                false,
+                false,
+                false,
                 false,
                 false,
                 null,

src/TextUI/Configuration/Xml/Loader.php

@@ -770,6 +770,12 @@ private function phpunit(string $filename, DOMDocument $document): PHPUnit
             $this->getBooleanAttribute($document->documentElement, 'displayDetailsOnTestsThatTriggerErrors', false),
             $this->getBooleanAttribute($document->documentElement, 'displayDetailsOnTestsThatTriggerNotices', false),
             $this->getBooleanAttribute($document->documentElement, 'displayDetailsOnTestsThatTriggerWarnings', false),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedPhpDeprecations', true),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedPhpNotices', true),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedPhpWarnings', true),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedUserDeprecations', false),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedUserNotices', false),
+            $this->getBooleanAttribute($document->documentElement, 'errorHandlerIgnoreSuppressedUserWarnings', false),
             $this->getBooleanAttribute($document->documentElement, 'reverseDefectList', false),
             $requireCoverageMetadata,
             $bootstrap,