feature #37733 [PhpUnitBridge] Add ability to set a baseline for depr…

…ecation testing (alexpott)

This PR was submitted for the master branch but it was squashed and merged into the 5.x branch instead.

Discussion
----------

[PhpUnitBridge] Add ability to set a baseline for deprecation testing

| Q             | A
| ------------- | ---
| Branch?       | master
| Bug fix?      | no
| New feature?  | yes <!-- please update src/**/CHANGELOG.md files -->
| Deprecations? | no <!-- please update UPGRADE-*.md and src/**/CHANGELOG.md files -->
| Tickets       | Fix #37715, #34496
| License       | MIT
| Doc PR        | symfony/symfony-docs#... <!-- required for new features -->

This PR allows you set new options for `SYMFONY_DEPRECATIONS_HELPER` env var:
* `generateBaseline` - if this is set to TRUE any deprecations that occur will be written to the file defined in the `baselineFile` option
* `baselineFile` a path to a file that contains a json encoded array of deprecations that will be skipped.

### Questions
* If you set `generateBaseline` without also setting `baselineFile` an exception is thrown. We could use a default filename if one is not provided (like PHPStan).
* How much error checking should we do around the `baselineFile` variable - should we check if it is readable or should we rely on `file_get_contents`()?

### Still @todo
Add proper end-to-end testing using a .phpt test

Commits
-------

483236f [PhpUnitBridge] Add ability to set a baseline for deprecation testing

src/Symfony/Bridge/PhpUnit/DeprecationErrorHandler.php

@@ -129,6 +129,9 @@ public function handleError($type, $msg, $file, $line, $context = [])
         if ($deprecation->isMuted()) {
             return null;
         }
+        if ($this->getConfiguration()->isBaselineDeprecation($deprecation)) {
+            return null;
+        }
         $group = 'other';
 
         if ($deprecation->originatesFromAnObject()) {
@@ -207,6 +210,10 @@ public function shutdown()
             $isFailingAtShutdown = !$configuration->tolerates($this->deprecationGroups);
             $this->displayDeprecations($groups, $configuration, $isFailingAtShutdown);
 
+            if ($configuration->isGeneratingBaseline()) {
+                $configuration->writeBaseline();
+            }
+
             if ($isFailing || $isFailingAtShutdown) {
                 exit(1);
             }

src/Symfony/Bridge/PhpUnit/DeprecationErrorHandler/Configuration.php

@@ -37,12 +37,28 @@ class Configuration
     private $verboseOutput;
 
     /**
-     * @param int[]  $thresholds    A hash associating groups to thresholds
-     * @param string $regex         Will be matched against messages, to decide
-     *                              whether to display a stack trace
-     * @param bool[] $verboseOutput Keyed by groups
+     * @var bool
+     */
+    private $generateBaseline = false;
+
+    /**
+     * @var string
+     */
+    private $baselineFile = '';
+
+    /**
+     * @var array
+     */
+    private $baselineDeprecations = [];
+
+    /**
+     * @param int[]  $thresholds       A hash associating groups to thresholds
+     * @param string $regex            Will be matched against messages, to decide whether to display a stack trace
+     * @param bool[] $verboseOutput    Keyed by groups
+     * @param bool   $generateBaseline Whether to generate or update the baseline file
+     * @param string $baselineFile     The path to the baseline file
      */
-    private function __construct(array $thresholds = [], $regex = '', $verboseOutput = [])
+    private function __construct(array $thresholds = [], $regex = '', $verboseOutput = [], $generateBaseline = false, $baselineFile = '')
     {
         $groups = ['total', 'indirect', 'direct', 'self'];
 
@@ -87,6 +103,22 @@ private function __construct(array $thresholds = [], $regex = '', $verboseOutput
             }
             $this->verboseOutput[$group] = (bool) $status;
         }
+
+        if ($generateBaseline && !$baselineFile) {
+            throw new \InvalidArgumentException('You cannot use the "generateBaseline" configuration option without providing a "baselineFile" configuration option.');
+        }
+        $this->generateBaseline = $generateBaseline;
+        $this->baselineFile = $baselineFile;
+        if ($this->baselineFile && !$this->generateBaseline) {
+            if (is_file($this->baselineFile)) {
+                $map = json_decode(file_get_contents($this->baselineFile));
+                foreach ($map as $baseline_deprecation) {
+                    $this->baselineDeprecations[$baseline_deprecation->location][$baseline_deprecation->message] = $baseline_deprecation->count;
+                }
+            } else {
+                throw new \InvalidArgumentException(sprintf('The baselineFile "%s" does not exist.', $this->baselineFile));
+            }
+        }
     }
 
     /**
@@ -125,6 +157,61 @@ public function tolerates(array $deprecationGroups)
         return true;
     }
 
+    /**
+     * @return bool
+     */
+    public function isBaselineDeprecation(Deprecation $deprecation)
+    {
+        if ($deprecation->originatesFromAnObject()) {
+            $location = $deprecation->originatingClass().'::'.$deprecation->originatingMethod();
+        } else {
+            $location = 'procedural code';
+        }
+
+        $message = $deprecation->getMessage();
+        $result = isset($this->baselineDeprecations[$location][$message]) && $this->baselineDeprecations[$location][$message] > 0;
+        if ($this->generateBaseline) {
+            if ($result) {
+                ++$this->baselineDeprecations[$location][$message];
+            } else {
+                $this->baselineDeprecations[$location][$message] = 1;
+                $result = true;
+            }
+        } elseif ($result) {
+            --$this->baselineDeprecations[$location][$message];
+        }
+
+        return $result;
+    }
+
+    /**
+     * @return bool
+     */
+    public function isGeneratingBaseline()
+    {
+        return $this->generateBaseline;
+    }
+
+    public function getBaselineFile()
+    {
+        return $this->baselineFile;
+    }
+
+    public function writeBaseline()
+    {
+        $map = [];
+        foreach ($this->baselineDeprecations as $location => $messages) {
+            foreach ($messages as $message => $count) {
+                $map[] = [
+                    'location' => $location,
+                    'message' => $message,
+                    'count' => $count,
+                ];
+            }
+        }
+        file_put_contents($this->baselineFile, json_encode($map, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES));
+    }
+
     /**
      * @param string $message
      *
@@ -161,7 +248,7 @@ public static function fromUrlEncodedString($serializedConfiguration)
     {
         parse_str($serializedConfiguration, $normalizedConfiguration);
         foreach (array_keys($normalizedConfiguration) as $key) {
-            if (!\in_array($key, ['max', 'disabled', 'verbose', 'quiet'], true)) {
+            if (!\in_array($key, ['max', 'disabled', 'verbose', 'quiet', 'generateBaseline', 'baselineFile'], true)) {
                 throw new \InvalidArgumentException(sprintf('Unknown configuration option "%s".', $key));
             }
         }
@@ -171,6 +258,8 @@ public static function fromUrlEncodedString($serializedConfiguration)
             'disabled' => false,
             'verbose' => true,
             'quiet' => [],
+            'generateBaseline' => false,
+            'baselineFile' => '',
         ];
 
         if ('' === $normalizedConfiguration['disabled'] || filter_var($normalizedConfiguration['disabled'], \FILTER_VALIDATE_BOOLEAN)) {
@@ -188,7 +277,13 @@ public static function fromUrlEncodedString($serializedConfiguration)
             }
         }
 
-        return new self($normalizedConfiguration['max'], '', $verboseOutput);
+        return new self(
+            isset($normalizedConfiguration['max']) ? $normalizedConfiguration['max'] : [],
+            '',
+            $verboseOutput,
+            filter_var($normalizedConfiguration['generateBaseline'], \FILTER_VALIDATE_BOOLEAN),
+            $normalizedConfiguration['baselineFile']
+        );
     }
 
     /**

src/Symfony/Bridge/PhpUnit/Tests/DeprecationErrorHandler/ConfigurationTest.php

@@ -13,10 +13,13 @@
 
 use PHPUnit\Framework\TestCase;
 use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\Configuration;
+use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\Deprecation;
 use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\DeprecationGroup;
 
 class ConfigurationTest extends TestCase
 {
+    private $files;
+
     public function testItThrowsOnStringishValue()
     {
         $this->expectException(\InvalidArgumentException::class);
@@ -244,4 +247,169 @@ private function buildGroups($counts)
 
         return $groups;
     }
+
+    public function testBaselineGenerationEmptyFile()
+    {
+        $filename = $this->createFile();
+        $configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
+        $this->assertTrue($configuration->isGeneratingBaseline());
+        $trace = debug_backtrace();
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
+        $configuration->writeBaseline();
+        $this->assertEquals($filename, $configuration->getBaselineFile());
+        $expected_baseline = [
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 1',
+                'count' => 2,
+            ],
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 2',
+                'count' => 1,
+            ],
+        ];
+        $this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
+    }
+
+    public function testBaselineGenerationNoFile()
+    {
+        $filename = $this->createFile();
+        $configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
+        $this->assertTrue($configuration->isGeneratingBaseline());
+        $trace = debug_backtrace();
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
+        $configuration->writeBaseline();
+        $this->assertEquals($filename, $configuration->getBaselineFile());
+        $expected_baseline = [
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 1',
+                'count' => 2,
+            ],
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 2',
+                'count' => 2,
+            ],
+        ];
+        $this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
+
+    }
+
+    public function testExistingBaseline()
+    {
+        $filename = $this->createFile();
+        $baseline = [
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 1',
+                'count' => 1,
+            ],
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 2',
+                'count' => 1,
+            ],
+        ];
+        file_put_contents($filename, json_encode($baseline));
+
+        $configuration = Configuration::fromUrlEncodedString('baselineFile=' . urlencode($filename));
+        $this->assertFalse($configuration->isGeneratingBaseline());
+        $trace = debug_backtrace();
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
+        $this->assertFalse($configuration->isBaselineDeprecation(new Deprecation('Test message 3', $trace, '')));
+        $this->assertEquals($filename, $configuration->getBaselineFile());
+    }
+
+    public function testExistingBaselineAndGeneration()
+    {
+        $filename = $this->createFile();
+        $baseline = [
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 1',
+                'count' => 1,
+            ],
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 2',
+                'count' => 1,
+            ],
+        ];
+        file_put_contents($filename, json_encode($baseline));
+        $configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
+        $this->assertTrue($configuration->isGeneratingBaseline());
+        $trace = debug_backtrace();
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
+        $this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 3', $trace, '')));
+        $configuration->writeBaseline();
+        $this->assertEquals($filename, $configuration->getBaselineFile());
+        $expected_baseline = [
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 2',
+                'count' => 1,
+            ],
+            [
+                'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
+                'message' => 'Test message 3',
+                'count' => 1,
+            ],
+        ];
+        $this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
+    }
+
+    public function testBaselineArgumentException()
+    {
+        $this->expectException(\InvalidArgumentException::class);
+        $this->expectExceptionMessage('You cannot use the "generateBaseline" configuration option without providing a "baselineFile" configuration option.');
+        Configuration::fromUrlEncodedString('generateBaseline=true');
+    }
+
+    public function testBaselineFileException()
+    {
+        $filename = $this->createFile();
+        unlink($filename);
+        $this->expectException(\InvalidArgumentException::class);
+        $this->expectExceptionMessage(sprintf('The baselineFile "%s" does not exist.', $filename));
+        Configuration::fromUrlEncodedString('baselineFile=' . urlencode($filename));
+    }
+
+    public function testBaselineFileWriteError()
+    {
+        $filename = $this->createFile();
+        chmod($filename, 0444);
+        $this->expectError();
+        $this->expectErrorMessageMatches('/failed to open stream: Permission denied/');
+        $configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
+        $configuration->writeBaseline();
+    }
+
+    protected function setUp(): void
+    {
+        $this->files = [];
+    }
+
+    protected function tearDown(): void
+    {
+        foreach ($this->files as $file) {
+            if (file_exists($file)) {
+                unlink($file);
+            }
+        }
+    }
+
+    private function createFile() {
+        $filename = tempnam(sys_get_temp_dir(), 'sf-');
+        $this->files[] = $filename;
+        return $filename;
+    }
+
 }