[test] Notify about flacky tests

.github/workflows/tests.yml

@@ -1,6 +1,9 @@
 name: Tests
 
-on: [ 'push', 'pull_request' ]
+on: [ 'pull_request' ]
+
+permissions:
+  pull-requests: write
 
 jobs:
   ci:
@@ -9,8 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        php: [ 8.2, 8.3, 8.4, 8.5 ]
-        dependencies: [ lowest , highest ]
+        php: [ "8.4",  "8.5" ]
+        dependencies: [ highest ]
       max-parallel: 2
 
     name: Tests PHP${{ matrix.php }}
@@ -33,3 +36,5 @@ jobs:
 
       - name: Run Tests
         run: composer test:ci
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

src/FlakyPRCommentPlugin.php

@@ -0,0 +1,93 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo;
+
+use Internal\Container\Container;
+use Testo\Common\Environment;
+use Testo\Common\EventListenerCollector;
+use Testo\Common\PluginConfigurator;
+use Testo\Core\Value\Status;
+use Testo\Event\Framework\SessionFinished;
+use Testo\Event\Test\TestPipelineFinished;
+
+final class FlakyPRCommentPlugin implements PluginConfigurator
+{
+    #[\Override]
+    public function configure(Container $container): void
+    {
+        // Check that we're in GitHub Actions and this is a PR
+        $token = \getenv('GITHUB_TOKEN');
+        $repo = \getenv('GITHUB_REPOSITORY'); // owner/repo
+        $ref = (string) \getenv('GITHUB_REF'); // refs/pull/123/merge
+        if (!$token || !$repo || !\preg_match('#^refs/pull/(\d+)/#', $ref, $m)) {
+            return; // not in CI or not a PR — do nothing
+        }
+
+        $prNumber = $m[1];
+
+        $listeners = $container->get(EventListenerCollector::class);
+        $listeners->addListener(TestPipelineFinished::class, $this->onTestFinished(...));
+        $listeners->addListener(
+            SessionFinished::class,
+            fn(SessionFinished $e) => $this->postComment($token, $repo, $prNumber),
+        );
+    }
+
+    /** @var list<string> */
+    private array $flakyTests = [];
+
+    private function onTestFinished(TestPipelineFinished $event): void
+    {
+        if ($event->testResult->status !== Status::Flaky) {
+            return;
+        }
+
+        $case = $event->testInfo->caseInfo->definition->reflection?->getShortName();
+        $test = $event->testInfo->testDefinition->reflection->getName();
+        $this->flakyTests[] = $case === null ? "{$test}()" : "{$case}::{$test}()";
+    }
+
+    private function postComment(string $token, string $repo, string $prNumber): void
+    {
+        if ($this->flakyTests === []) {
+            return;
+        }
+
+        $list = \implode("\n", \array_map(
+            static fn(string $name) => "- `{$name}`",
+            $this->flakyTests,
+        ));
+
+        $env = \sprintf(
+            "**Environment:** PHP %s (%s, %s, %s)%s%s%s",
+            Environment::getPhpVersion(),
+            Environment::getThread(),
+            Environment::getOs(),
+            Environment::getCpu(),
+            Environment::hasXDebug()
+                ? \sprintf(', Xdebug %s [%s]', Environment::getXDebugVersion(), \implode(', ', Environment::getXDebugMode()))
+                : '',
+            Environment::isOpCacheEnabled() ? ', OPcache' : '',
+            Environment::isJitEnabled() ? ', JIT' : '',
+        );
+
+        @\file_get_contents(
+            "https://api.github.com/repos/{$repo}/issues/{$prNumber}/comments",
+            context: \stream_context_create([
+                'http' => [
+                    'method' => 'POST',
+                    'header' => \implode("\r\n", [
+                        "Authorization: Bearer {$token}",
+                        'Content-Type: application/json',
+                        'User-Agent: Testo',
+                    ]),
+                    'content' => \json_encode([
+                        'body' => "⚠️ **Flaky tests detected**\n\n{$list}\n\n{$env}",
+                    ]),
+                ],
+            ]),
+        );
+    }
+}

testo.php

@@ -4,41 +4,18 @@
 
 use Testo\Application\Config\ApplicationConfig;
 use Testo\Application\Config\FinderConfig;
-use Testo\Application\Config\Plugin\SuitePlugins;
 use Testo\Application\Config\SuiteConfig;
-use Testo\Bench\BenchmarkPlugin;
-use Testo\Inline\InlineTestPlugin;
 
 return new ApplicationConfig(
-    suites: \array_merge(
-        [
-            new SuiteConfig(
-                name: 'SRC',
-                location: new FinderConfig(
-                    include: ['src'],
-                ),
-                plugins: SuitePlugins::only(
-                    new InlineTestPlugin(),
-                    new BenchmarkPlugin(),
-                ),
+    suites: [
+        new SuiteConfig(
+            name: 'sandbox',
+            location: new FinderConfig(
+                include: ['tests/Testo'],
             ),
-        ],
-        # If running in CI, skip the sandbox
-        \filter_var(dump(\getenv('TESTO_CI')), FILTER_VALIDATE_BOOLEAN) ? [] : [
-            new SuiteConfig(
-                name: 'sandbox',
-                location: new FinderConfig(
-                    include: ['tests/Testo'],
-                ),
-            ),
-        ],
-        require 'tests/Assert/suites.php',
-        require 'tests/Common/suites.php',
-        require 'tests/Lifecycle/suites.php',
-        require 'tests/Data/suites.php',
-        require 'tests/Bench/suites.php',
-        require 'tests/Application/suites.php',
-        require 'tests/Output/suites.php',
-        require 'tests/Test/suites.php',
-    ),
+        ),
+    ],
+    plugins: [
+        new \Testo\FlakyPRCommentPlugin(),
+    ],
 );