Permalink
Fetching contributors…
Cannot retrieve contributors at this time
444 lines (414 sloc) 13.3 KB
#!/usr/bin/env php
<?php
/**
* @package PHPT Tests runner
* @author Nazar Mokrynskyi <nazar@mokrynskyi.com>
* @license 0BSD
*/
/**
* @param string $text
*
* @return string
*/
function colorize ($text) {
if (has_color_support()) {
$callback = function ($matches) {
$color_codes = [
'g' => 32, // Green
'y' => 33, // Yellow
'r' => 31, // Red
];
$color = $color_codes[$matches[1]];
$text = $matches[2];
return "\033[{$color}m$text\033[0m";
};
} else {
$callback = function ($matches) {
return $matches[2];
};
}
return preg_replace_callback(
'#<([gyr])>(.*)</\1>#Us',
$callback,
$text
);
}
/**
* Returns true if ANSI escape sequences are supported.
*
* Based on code of Fabien Potencier:
* \Symfony\Component\Console\Output\StreamOutput::hasColorSupport()
*
* @param bool|null $set_to
*
* @return bool true if ANSI escape sequences are supported.
*/
function has_color_support ($set_to = null) {
static $result;
if ($set_to !== null) {
$result = $set_to;
}
if ($result !== null) {
return $result;
}
if (strpos(PHP_OS, 'WIN') === false) {
$result = function_exists('posix_isatty') && @posix_isatty(STDOUT);
} else {
$result =
PHP_WINDOWS_VERSION_MAJOR.'.'.PHP_WINDOWS_VERSION_MINOR.'.'.PHP_WINDOWS_VERSION_BUILD === '10.0.10586' ||
getenv('ANSICON') !== false ||
getenv('ConEmuANSI') === 'ON' ||
getenv('TERM') === 'xterm';
}
return $result;
}
/**
* Output something to console
*
* Will colorize stuff in process
*
* @param bool $clean Clean current line before output
* @param string $text
*/
function out ($text, $clean = false) {
if ($clean) {
echo "\r";
}
echo colorize($text);
}
/**
* Output something to console and add new line at the end
*
* Will colorize stuff in process
*
* @param bool $clean Clean current line before output
* @param string $text
*/
function line ($text = '', $clean = false) {
out($text, $clean);
echo "\n";
}
/**
* @param string $binary
* @param string $test_file
* @param string $base_text
*
* @return string `skipped`, `success` or `error`
*/
function run_test ($binary, $test_file, $base_text) {
out("<y>$base_text ...</y>");
$test_file = realpath($test_file);
@unlink("$test_file.exp");
@unlink("$test_file.out");
@unlink("$test_file.diff");
$parsed_test = parse_test($test_file);
/**
* Check required sections
*/
if (!isset($parsed_test['FILE'])) {
line("<r>$base_text ERROR</r>", true);
line('--FILE-- section MUST be present');
return 'error';
}
$output_sections = ['EXPECT', 'EXPECTF', 'EXPECTREGEX'];
if (!array_intersect(array_keys($parsed_test), $output_sections)) {
line("<r>$base_text ERROR</r>", true);
line('One of the following sections MUST be present: '.implode(',', $output_sections));
return 'error';
}
$php_arguments = [
'-d variables_order=EGPCS',
'-d error_reporting='.E_ALL,
'-d display_errors=1',
'-d xdebug.default_enable=0'
];
if (isset($parsed_test['INI'])) {
foreach (explode("\n", trim($parsed_test['INI'])) as $line) {
list($key, $value) = array_map('trim', explode('=', $line, 2));
$php_arguments[] = "-d $key=$value";
}
}
$script_arguments = $parsed_test['ARGS'] ?? '';
$working_dir = dirname($test_file);
if (isset($parsed_test['SKIPIF'])) {
$result = execute_code($binary, $working_dir, $parsed_test['SKIPIF'], $php_arguments, $script_arguments, $test_file);
if (stripos($result, 'skip') === 0) {
line("<y>$base_text SKIPPED</y>", true);
line(ltrim(substr($result, 4)));
return 'skipped';
}
}
$started_time = microtime(true);
$output = rtrim(execute_code($binary, $working_dir, $parsed_test['FILE'], $php_arguments, $script_arguments, $test_file, true));
$result = compare_output(PHP_BINARY, $output, $base_text, $test_file, $php_arguments, $script_arguments, $parsed_test, $started_time);
isset($parsed_test['CLEAN']) && execute_code($binary, $working_dir, $parsed_test['CLEAN'], $php_arguments, $script_arguments, $test_file);
if ($result === 'success') {
unlink("$test_file.test_code.php");
}
return $result;
}
/**
* @param string $test_file
*
* @return string[]
*/
function parse_test ($test_file) {
$result = [];
$current_key = null;
foreach (file($test_file) as $line) {
if (preg_match("/^--(SKIPIF|INI|ARGS|FILE|EXPECT|EXPECTF|EXPECTREGEX|CLEAN)--\n$/", $line, $match)) {
$current_key = $match[1];
$result[$current_key] = '';
} elseif ($current_key) {
if (!isset($result[$current_key])) {
$result[$current_key] = '';
}
$result[$current_key] .= $line;
}
}
return $result;
}
/**
* @param string $binary
* @param string $working_dir
* @param string $code
* @param string[] $php_arguments
* @param string $script_arguments
* @param string $test_file
* @param bool $keep_file
*
* @return string
*/
function execute_code ($binary, $working_dir, $code, $php_arguments, $script_arguments, $test_file, $keep_file = false) {
if ($keep_file) {
$tmp_code_file = "$test_file.test_code.php";
} else {
$tmp_code_file = tempnam($working_dir, basename($test_file));
}
file_put_contents($tmp_code_file, $code);
putenv("TEST_FILE=\"$test_file\"");
$arguments = implode(' ', $php_arguments);
$prepared_file = escapeshellarg($tmp_code_file);
$output = shell_exec("$binary $arguments -f $prepared_file -- $script_arguments 2>&1");
if (!$keep_file) {
unlink($tmp_code_file);
}
return $output;
}
/**
* @param float $started In seconds
*
* @return string
*/
function time_since_started ($started) {
return round((microtime(true) - $started) * 1000, 2).' ms';
}
/**
* @param string $binary
* @param string $output
* @param string $base_text
* @param string $test_file
* @param array $php_arguments
* @param string $script_arguments
* @param string[] $parsed_test
* @param float $started_time
*
* @return string `success` or `error`
*/
function compare_output ($binary, $output, $base_text, $test_file, $php_arguments, $script_arguments, $parsed_test, $started_time) {
$working_dir = dirname($test_file);
$execution_time = time_since_started($started_time);
if (isset($parsed_test['EXPECT'])) {
$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECT'], $php_arguments, $script_arguments, $test_file));
if ($expect === $output) {
line("<g>$base_text SUCCESS</g> $execution_time", true);
return 'success';
}
} elseif (isset($parsed_test['EXPECTF'])) {
$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECTF'], $php_arguments, $script_arguments, $test_file));
$regex = str_replace(
[
'%s',
'%S',
'%a',
'%A',
'%w',
'%i',
'%d',
'%x',
'%f',
'%c'
],
[
'[^\r\n]+',
'[^\r\n]*',
'.+',
'.*',
'\s*',
'[+-]?\d+',
'\d+',
'[0-9a-fA-F]+',
'[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?',
'.'
],
preg_quote($expect, '/')
);
if (preg_match("/^$regex\$/s", $output)) {
line("<g>$base_text SUCCESS</g> $execution_time", true);
return 'success';
}
} else {
$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECREGEX'], $php_arguments, $script_arguments, $test_file));
$regex = preg_quote($expect, '/');
if (preg_match("/^$regex\$/s", $output)) {
line("<g>$base_text SUCCESS</g> $execution_time", true);
return 'success';
}
}
line("<r>$base_text ERROR:</r> $execution_time", true);
$diff = preg_replace_callback(
'/^([-+]).*$/m',
function ($match) {
return $match[1] === '-' ? "<r>$match[0]</r>" : "<g>$match[0]</g>";
},
compute_diff($test_file, $expect, $output)
);
line($diff);
return 'error';
}
function compute_diff ($test_file, $expect, $output) {
$exp_file = "$test_file.exp";
$out_file = "$test_file.out";
file_put_contents($exp_file, $expect."\n");
file_put_contents($out_file, $output."\n");
$diff = rtrim(
shell_exec(
"diff --old-line-format='-%3dn %L' --new-line-format='+%3dn %L' --from-file=".escapeshellarg($exp_file).' '.escapeshellarg($out_file)
)
);
file_put_contents("$test_file.diff", $diff);
return $diff;
}
/**
* @param string|string[] $target
*
* @return string[]
*/
function find_tests ($target) {
if (is_array($target)) {
return array_merge(...array_map('find_tests', $target));
}
if (is_dir($target)) {
$iterator = new RegexIterator(
new RecursiveIteratorIterator(
new RecursiveDirectoryIterator($target)
),
'/.*\.phpt$/',
RecursiveRegexIterator::GET_MATCH
);
return array_merge(...array_values(iterator_to_array($iterator)));
}
return file_exists($target) ? [$target] : [];
}
$options = getopt('hb:cC', []);
$targets = array_filter(
array_slice($argv, 1),
function ($path) {
return is_dir($path) || (is_file($path) && substr($path, -5) === '.phpt');
}
);
line("<g>PHPT Tests runner</g>\n");
if (isset($options['c'])) {
has_color_support(true);
} elseif (isset($options['C'])) {
has_color_support(false);
}
if (!$targets || isset($options['h'])) {
line(
<<<HTML
<y>Usage:</y>
phpt-tests-runner [-h] [-c] [-C] [-b binary] [files] [directories]
<y>Arguments:</y>
<g>-h</g> Print this help message
<g>-b</g> Specify custom PHP binary to be used (current is used by default)
<g>-c</g> Force colored output
<g>-C</g> Force no colored output
<y>Examples:</y>
Execute tests from tests directory:
<g>phpt-tests-runner tests</g>
Execute tests single test:
<g>phpt-tests-runner tests/sample.phpt</g>
Execute tests from tests directory, but skip slow tests using environment variable:
<g>SKIP_SLOW_TESTS=1 phpt-tests-runner tests</g>
<y>PHPT Format:</y>
This runner uses modification of PHPT format used by PHP itself, so that it can run many original PHPT tests without any changes.
PHPT test if text file with *.phpt extension.
Each file contains sections followed by section contents, everything before first section is ignored, you can use it for storing test description.
Required sections are <g>--FILE--</g> and one of [<g>--EXPECT--</g>, <g>--EXPECTF--</g>, <g>--EXPECTREGEX--</g>].
<y>PHPT sections supported:</y>
<g>--FILE--</g> The test source code
<g>--EXPECT--</g> The expected output from the test script (will be executed as PHP script, so it might be code as well as plain text)
<g>--EXPECTF--</g> Similar to <g>--EXPECT--</g>, but it uses substitution tags for strings, spaces, digits, which may vary between test runs
The following is a list of all tags and what they are used to represent:
<g>%s</g> One or more of anything (character or white space) except the end of line character
<g>%S</g> Zero or more of anything (character or white space) except the end of line character
<g>%a</g> One or more of anything (character or white space) including the end of line character
<g>%A</g> Zero or more of anything (character or white space) including the end of line character
<g>%w</g> Zero or more white space characters
<g>%i</g> A signed integer value, for example +3142, -3142
<g>%d</g> An unsigned integer value, for example 123456
<g>%x</g> One or more hexadecimal character. That is, characters in the range 0-9, a-f, A-F
<g>%f</g> A floating point number, for example: 3.142, -3.142, 3.142E-10, 3.142e+10
<g>%c</g> A single character of any sort (.)
<g>--EXPECTREGEX--</g> Similar to <g>--EXPECT--</g>, but is treated as regular expression
<g>--SKIPIF--</g> If output of execution starts with `skip` then test will be skipped
<g>--INI--</g> Specific php.ini setting for the test, one per line
<g>--ARGS--</g> A single line defining the arguments passed to php
<g>--CLEAN--</g> Code that is executed after a test completes
<y>PHPT tests examples:</y>
Examples can be found at <<g>https://qa.php.net/phpt_details.php</g>> (taking into account differences here)
<y>Main differences from original PHPT tests files:</y>
1. <g>--TEST--</g> is not required and not even used (files names are used instead)
2. Only sub-set of sections supported and only sub-set of <g>--EXPECTF--</g> tags
3. <g>--EXPECT*--</g> sections are interpreted as code and its output is used as expected result
HTML
);
exit;
}
$tests = find_tests($targets);
sort($tests, SORT_NATURAL);
$tests_count = count($tests);
if (!$tests_count) {
line('<r>No tests found, there is nothing to do here</r>');
exit(1);
}
line("<y>$tests_count tests found</y>, running them:");
$max_length = 0;
foreach ($tests as $test_file) {
$max_length = max($max_length, strlen($test_file));
}
$binary = $options['b'] ?? PHP_BINARY;
$results = [
'skipped' => 0,
'success' => 0,
'error' => 0
];
$started_time = microtime(true);
foreach ($tests as $index => $test_file) {
$base_text = sprintf("%' 3d/$tests_count %s", $index + 1, str_pad($test_file, $max_length));
$results[run_test($binary, $test_file, $base_text)]++;
}
line("\nResults:");
if ($results['skipped']) {
line(sprintf("<y>%' 3d/$tests_count tests (%' 6.2f%%) skipped</y>", $results['skipped'], $results['skipped'] / $tests_count * 100));
}
if ($results['success']) {
line(sprintf("<g>%' 3d/$tests_count tests (%' 6.2f%%) succeed</g>", $results['success'], $results['success'] / $tests_count * 100));
}
if ($results['error']) {
line(sprintf("<r>%' 3d/$tests_count tests (%' 6.2f%%) failed</r>", $results['error'], $results['error'] / $tests_count * 100));
}
line(sprintf('Time: %s', time_since_started($started_time)));
if ($results['error']) {
exit(1);
}