Capture stderr of test via temp file and output it in test results [C…

…loses #420] (#438)

src/Runner/CliTester.php

@@ -50,6 +50,7 @@ public function run(): ?int
 
 		if ($this->options['--info']) {
 			$job = new Job(new Test(__DIR__ . '/info.php'), $this->interpreter);
+			$job->setTempDirectory($this->options['--temp']);
 			$job->run();
 			echo $job->getTest()->stdout;
 			return null;

src/Runner/Job.php

@@ -28,8 +28,7 @@ class Job
 	public const RUN_USLEEP = 10000;
 
 	public const
-		RUN_ASYNC = 1,
-		RUN_COLLECT_ERRORS = 2;
+		RUN_ASYNC = 1;
 
 	private Test $test;
 	private PhpInterpreter $interpreter;
@@ -42,9 +41,7 @@ class Job
 
 	/** @var resource|null */
 	private $stdout;
-
-	/** @var resource|null */
-	private $stderr;
+	private ?string $stderrFile;
 	private int $exitCode = self::CODE_NONE;
 
 	/** @var string[]  output headers */
@@ -67,6 +64,14 @@ public function __construct(Test $test, PhpInterpreter $interpreter, ?array $env
 	}
 
 
+	public function setTempDirectory(?string $path): void
+	{
+		$this->stderrFile = $path === null
+			? null
+			: $path . DIRECTORY_SEPARATOR . 'Job.pid-' . getmypid() . '.' . uniqid() . '.stderr';
+	}
+
+
 	public function setEnvironmentVariable(string $name, string $value): void
 	{
 		$this->envVars[$name] = $value;
@@ -81,7 +86,7 @@ public function getEnvironmentVariable(string $name): string
 
 	/**
 	 * Runs single test.
-	 * @param  int  $flags  self::RUN_ASYNC | self::RUN_COLLECT_ERRORS
+	 * @param  int  $flags  self::RUN_ASYNC
 	 */
 	public function run(int $flags = 0): void
 	{
@@ -103,7 +108,7 @@ public function run(int $flags = 0): void
 			[
 				['pipe', 'r'],
 				['pipe', 'w'],
-				['pipe', 'w'],
+				$this->stderrFile ? ['file', $this->stderrFile, 'w'] : ['pipe', 'w'],
 			],
 			$pipes,
 			dirname($this->test->getFile()),
@@ -115,19 +120,15 @@ public function run(int $flags = 0): void
 			putenv($name);
 		}
 
-		[$stdin, $this->stdout, $stderr] = $pipes;
+		[$stdin, $this->stdout] = $pipes;
 		fclose($stdin);
-		if ($flags & self::RUN_COLLECT_ERRORS) {
-			$this->stderr = $stderr;
-		} else {
-			fclose($stderr);
+
+		if (isset($pipes[2])) {
+			fclose($pipes[2]);
 		}
 
 		if ($flags & self::RUN_ASYNC) {
 			stream_set_blocking($this->stdout, false); // on Windows does not work with proc_open()
-			if ($this->stderr) {
-				stream_set_blocking($this->stderr, false);
-			}
 		} else {
 			while ($this->isRunning()) {
 				usleep(self::RUN_USLEEP); // stream_select() doesn't work with proc_open()
@@ -146,9 +147,6 @@ public function isRunning(): bool
 		}
 
 		$this->test->stdout .= stream_get_contents($this->stdout);
-		if ($this->stderr) {
-			$this->test->stderr .= stream_get_contents($this->stderr);
-		}
 
 		$status = proc_get_status($this->proc);
 		if ($status['running']) {
@@ -158,8 +156,9 @@ public function isRunning(): bool
 		$this->duration += microtime(true);
 
 		fclose($this->stdout);
-		if ($this->stderr) {
-			fclose($this->stderr);
+		if ($this->stderrFile) {
+			$this->test->stderr .= file_get_contents($this->stderrFile);
+			unlink($this->stderrFile);
 		}
 
 		$code = proc_close($this->proc);

src/Runner/Output/ConsolePrinter.php

@@ -98,6 +98,7 @@ public function finish(Test $test): void
 
 		$title = ($test->title ? "$test->title | " : '') . substr($test->getSignature(), strlen($this->baseDir));
 		$message = '   ' . str_replace("\n", "\n   ", trim((string) $test->message)) . "\n\n";
+		$message = preg_replace('/^   $/m', '', $message);
 		if ($test->getResult() === Test::FAILED) {
 			$this->buffer .= Dumper::color('red', "-- FAILED: $title") . "\n$message";
 		} elseif ($test->getResult() === Test::SKIPPED && $this->displaySkipped) {

src/Runner/Test.php

@@ -84,6 +84,15 @@ public function getDuration(): ?float
 	}
 
 
+	/**
+	 * Full output (stdout + stderr)
+	 */
+	public function getOutput(): string
+	{
+		return $this->stdout . ($this->stderr ? "\nSTDERR:\n" . $this->stderr : '');
+	}
+
+
 	/**
 	 * @return static
 	 */

src/Runner/TestHandler.php

@@ -74,7 +74,9 @@ public function initiate(string $file): void
 
 		foreach ($tests as $test) {
 			$this->runner->prepareTest($test);
-			$this->runner->addJob(new Job($test, $php, $this->runner->getEnvironmentVariables()));
+			$job = new Job($test, $php, $this->runner->getEnvironmentVariables());
+			$job->setTempDirectory($this->tempDir);
+			$this->runner->addJob($job);
 		}
 	}
 
@@ -194,10 +196,11 @@ private function initiateTestCase(Test $test, $foo, PhpInterpreter $interpreter)
 
 		if ($methods === null) {
 			$job = new Job($test->withArguments(['method' => TestCase::ListMethods]), $interpreter, $this->runner->getEnvironmentVariables());
+			$job->setTempDirectory($this->tempDir);
 			$job->run();
 
 			if (in_array($job->getExitCode(), [Job::CODE_ERROR, Job::CODE_FAIL, Job::CODE_SKIP], true)) {
-				return $test->withResult($job->getExitCode() === Job::CODE_SKIP ? Test::SKIPPED : Test::FAILED, $job->getTest()->stdout);
+				return $test->withResult($job->getExitCode() === Job::CODE_SKIP ? Test::SKIPPED : Test::FAILED, $job->getTest()->getOutput());
 			}
 
 			$stdout = $job->getTest()->stdout;
@@ -244,7 +247,7 @@ private function assessExitCode(Job $job, string|int $code): ?Test
 			$message = $job->getExitCode() !== Job::CODE_FAIL
 				? "Exited with error code {$job->getExitCode()} (expected $code)"
 				: '';
-			return $job->getTest()->withResult(Test::FAILED, trim($message . "\n" . $job->getTest()->stdout));
+			return $job->getTest()->withResult(Test::FAILED, trim($message . "\n" . $job->getTest()->getOutput()));
 		}
 
 		return null;

tests/Runner/Job.phpt

@@ -13,7 +13,8 @@ require __DIR__ . '/../bootstrap.php';
 test(function () {
 	$test = (new Test('Job.test.phptx'))->withArguments(['one', 'two' => 1])->withArguments(['three', 'two' => 2]);
 	$job = new Job($test, createInterpreter());
-	$job->run($job::RUN_COLLECT_ERRORS);
+	$job->setTempDirectory(Tester\Helpers::prepareTempDir(sys_get_temp_dir()));
+	$job->run();
 
 	Assert::false($job->isRunning());
 	Assert::same($test, $job->getTest());

tests/Runner/Runner.multiple-fails.phpt

@@ -47,6 +47,7 @@ $interpreter = createInterpreter()
 $runner = new Runner($interpreter);
 $runner->paths[] = __DIR__ . '/multiple-fails/*.phptx';
 $runner->outputHandlers[] = $logger = new Logger;
+$runner->setTempDirectory(Tester\Helpers::prepareTempDir(sys_get_temp_dir()));
 $runner->run();
 
 Assert::match(
@@ -86,8 +87,8 @@ Assert::same(Test::FAILED, $logger->results['testcase-pre-fail.phptx'][0]);
 
 Assert::match(
 	defined('PHPDBG_VERSION')
-		? '%A%Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%'
-		: 'Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%',
+		? '%A%Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%%A?%'
+		: 'Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%%A?%',
 	trim($logger->results['testcase-syntax-error.phptx'][1])
 );
 Assert::same(Test::FAILED, $logger->results['testcase-syntax-error.phptx'][0]);

tests/Runner/Test.phpt

@@ -20,6 +20,7 @@ test(function () {
 	Assert::null($test->message);
 	Assert::same('', $test->stdout);
 	Assert::same('', $test->stderr);
+	Assert::same('', $test->getOutput());
 	Assert::same('some/Test.phpt', $test->getFile());
 	Assert::same([], $test->getArguments());
 	Assert::same('some/Test.phpt', $test->getSignature());

tests/RunnerOutput/OutputHandlers.expect.console.txt

@@ -7,14 +7,24 @@ F.sF.sFsF.s
    line
    stdout.Failed:
 
-   in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');
+   in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
 -- FAILED: Title for output handlers | 02-title.fail.phptx
    Multi
    line
    stdout.Failed:
 
-   in %a%02-title.fail.phptx(11) Tester\Assert::fail('');
+   in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
 -- FAILED: 03-message.fail.phptx
    Multi
@@ -23,14 +33,24 @@ F.sF.sFsF.s
    line
    message.
 
-   in %a%03-message.fail.phptx(7) Tester\Assert::fail("Multi\nline\nmessage.");
+   in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
 -- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
    Multi
    line
    stdout.Failed:
 
-   in %a%04-args.fail.phptx(11) Tester\Assert::fail('');
+   in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
 
 FAILURES! (11 tests, 4 failures, 4 skipped, %a% seconds)

tests/RunnerOutput/OutputHandlers.expect.consoleWithSkip.txt

@@ -7,43 +7,63 @@ F.sF.sFsF.s
    line
    stdout.Failed:
 
-   in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');
+   in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');
 
--- Skipped: %a?%01-basic.skip.phptx
+   STDERR:
+   Multi
+   line
+   stderr.
+
+-- Skipped: 01-basic.skip.phptx
 
 
--- FAILED: Title for output handlers | 02-title.fail.phptx
+-- FAILED: Title for output handlers | %a?%02-title.fail.phptx
    Multi
    line
    stdout.Failed:
 
-   in %a%02-title.fail.phptx(11) Tester\Assert::fail('');
+   in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
--- Skipped: Title for output handlers | 02-title.skip.phptx
+-- Skipped: Title for output handlers | %a?%02-title.skip.phptx
 
 
--- FAILED: 03-message.fail.phptx
+-- FAILED: %a?%03-message.fail.phptx
    Multi
    line
    stdout.Failed: Multi
    line
    message.
 
-   in %a%03-message.fail.phptx(7) Tester\Assert::fail("Multi\nline\nmessage.");
+   in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");
 
--- Skipped: 03-message.skip.phptx
+   STDERR:
+   Multi
+   line
+   stderr.
+
+-- Skipped: %a?%03-message.skip.phptx
    Multi
    line
    message.
 
--- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
+-- FAILED: %a?%04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
    Multi
    line
    stdout.Failed:
 
-   in %a%04-args.fail.phptx(11) Tester\Assert::fail('');
+   in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
 
--- Skipped: 04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
+-- Skipped: %a?%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
    Multi
    line
    message.

tests/RunnerOutput/OutputHandlers.expect.jUnit.xml

@@ -6,7 +6,12 @@
 line
 stdout.Failed:
 
-in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');"/>
+in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');
+
+STDERR:
+Multi
+line
+stderr."/>
 		</testcase>
 		<testcase classname="%a%01-basic.pass.phptx" name="%a%01-basic.pass.phptx"/>
 		<testcase classname="%a%01-basic.skip.phptx" name="%a%01-basic.skip.phptx">
@@ -17,7 +22,12 @@ in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');"/>
 line
 stdout.Failed:
 
-in %a%02-title.fail.phptx(11) Tester\Assert::fail('');"/>
+in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');
+
+STDERR:
+Multi
+line
+stderr."/>
 		</testcase>
 		<testcase classname="%a%02-title.pass.phptx" name="%a%02-title.pass.phptx"/>
 		<testcase classname="%a%02-title.skip.phptx" name="%a%02-title.skip.phptx">
@@ -30,7 +40,12 @@ stdout.Failed: Multi
 line
 message.
 
-in %a%03-message.fail.phptx(7) Tester\Assert::fail(&quot;Multi\nline\nmessage.&quot;);"/>
+in %a%03-message.fail.phptx(%d%) Tester\Assert::fail(&quot;Multi\nline\nmessage.&quot;);
+
+STDERR:
+Multi
+line
+stderr."/>
 		</testcase>
 		<testcase classname="%a%03-message.skip.phptx" name="%a%03-message.skip.phptx">
 			<skipped/>
@@ -40,7 +55,12 @@ in %a%03-message.fail.phptx(7) Tester\Assert::fail(&quot;Multi\nline\nmessage.&q
 line
 stdout.Failed:
 
-in %a%04-args.fail.phptx(11) Tester\Assert::fail('');"/>
+in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');
+
+STDERR:
+Multi
+line
+stderr."/>
 		</testcase>
 		<testcase classname="%a%04-args.pass.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini" name="%a%04-args.pass.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini"/>
 		<testcase classname="%a%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini" name="%a%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini">