Skip to content

Commit

Permalink
Improve diff mechanism to hit prevent unnecessary file_get_contents
Browse files Browse the repository at this point in the history
  • Loading branch information
@muglug
muglug committed Nov 6, 2016
1 parent 721e0d0 commit e9b4eb6
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 23 deletions.
128 changes: 112 additions & 16 deletions src/Psalm/Checker/FileChecker.php
View file
Expand Up @@ -9,6 +9,7 @@

class FileChecker implements StatementsSource
{
const PARSER_CACHE_DIRECTORY = 'php-parser';
const REFERENCE_CACHE_NAME = 'references';
const GOOD_RUN_NAME = 'good_run';

Expand Down Expand Up @@ -316,8 +317,6 @@ protected function getStatements()
*/
public static function getStatementsForFile($file_name)
{
$contents = (string) file_get_contents($file_name);

$stmts = [];

$from_cache = false;
Expand All @@ -327,29 +326,29 @@ public static function getStatementsForFile($file_name)
$cache_directory = Config::getInstance()->getCacheDirectory();

if ($cache_directory) {
$key = md5($contents);
$cache_directory .= '/' . self::PARSER_CACHE_DIRECTORY;

$cache_location = $cache_directory . '/' . $key;
$cache_location = $cache_directory . '/' . self::getParserCacheKey($file_name);

if (is_readable($cache_location)) {
if (is_readable($cache_location) && filemtime($cache_location) >= filemtime($file_name)) {
/** @var array<int, \PhpParser\Node> */
$stmts = unserialize((string) file_get_contents($cache_location));
$stmts = unserialize((string)file_get_contents($cache_location));
$from_cache = true;
}
}

if (!$stmts && $contents) {
if (!$stmts) {
$parser = (new ParserFactory())->create(ParserFactory::PREFER_PHP7);

$stmts = $parser->parse($contents);
$stmts = $parser->parse((string)file_get_contents($file_name));
}

if ($cache_directory && $cache_location) {
if ($from_cache) {
touch($cache_location);
} else {
if (!file_exists($cache_directory)) {
mkdir($cache_directory);
if (!is_dir($cache_directory)) {
mkdir($cache_directory, 0777, true);
}

file_put_contents($cache_location, serialize($stmts));
Expand Down Expand Up @@ -675,18 +674,26 @@ public static function canDiffFiles()
}

/**
* @param string $file
* @return boolean
* @return int
*/
public static function hasFileChanged($file)
public static function getLastGoodRun()
{
if (self::$last_good_run === null) {
$cache_directory = Config::getInstance()->getCacheDirectory();

self::$last_good_run = filemtime($cache_directory . '/' . self::GOOD_RUN_NAME) ?: 0;
}

return filemtime($file) > self::$last_good_run;
return self::$last_good_run;
}

/**
* @param string $file
* @return boolean
*/
public static function hasFileChanged($file)
{
return filemtime($file) > self::getLastGoodRun();
}

/**
Expand All @@ -707,16 +714,17 @@ function ($file_name) {
}

/**
* @param int $start_time
* @return void
*/
public static function goodRun()
public static function goodRun($start_time)
{
$cache_directory = Config::getInstance()->getCacheDirectory();

if ($cache_directory) {
$run_cache_location = $cache_directory . '/' . self::GOOD_RUN_NAME;

touch($run_cache_location);
touch($run_cache_location, $start_time);

$deleted_files = self::getDeletedReferencedFiles();

Expand All @@ -730,6 +738,22 @@ public static function goodRun()
serialize(self::$file_references)
);
}

$cache_directory .= '/' . self::PARSER_CACHE_DIRECTORY;

if (is_dir($cache_directory)) {
$directory_files = scandir($cache_directory);

foreach ($directory_files as $directory_file) {
$full_path = $cache_directory . '/' . $directory_file;

if ($directory_file[0] === '.') {
continue;
}

touch($full_path);
}
}
}
}

Expand All @@ -748,4 +772,76 @@ public static function clearCache()
FunctionChecker::clearCache();
StatementsChecker::clearCache();
}

/**
* @param float $time_before
* @return int
*/
public static function deleteOldParserCaches($time_before)
{
$cache_directory = Config::getInstance()->getCacheDirectory();

$removed_count = 0;

if ($cache_directory) {
$cache_directory .= '/' . self::PARSER_CACHE_DIRECTORY;

if (is_dir($cache_directory)) {
$directory_files = scandir($cache_directory);

foreach ($directory_files as $directory_file) {
$full_path = $cache_directory . '/' . $directory_file;

if ($directory_file[0] === '.') {
continue;
}

if (filemtime($full_path) < $time_before && is_writable($full_path)) {
unlink($full_path);
$removed_count++;
}
}
}
}

return $removed_count;
}

/**
* @param array<string> $file_names
* @param int $min_time
* @return void
*/
public static function touchParserCaches(array $file_names, $min_time)
{
$cache_directory = Config::getInstance()->getCacheDirectory();

if ($cache_directory) {
$cache_directory .= '/' . self::PARSER_CACHE_DIRECTORY;

if (is_dir($cache_directory)) {
foreach ($file_names as $file_name) {
$hash_file_name = $cache_directory . '/' . self::getParserCacheKey($file_name);

if (file_exists($hash_file_name)) {
if (filemtime($hash_file_name) < $min_time) {
touch($hash_file_name, $min_time);
}
}
else {
throw new \InvalidArgumentException('Cannot touch file ' . $hash_file_name);
}
}
}
}
}

/**
* @param string $file_name
* @return string
*/
protected static function getParserCacheKey($file_name)
{
return md5($file_name);
}
}
45 changes: 42 additions & 3 deletions src/Psalm/Checker/ProjectChecker.php
View file
Expand Up @@ -39,6 +39,8 @@ public static function check($debug = false, $is_diff = false)
{
$cwd = getcwd();

$start_checks = (int)microtime(true);

if (!$cwd) {
throw new \InvalidArgumentException('Cannot work with empty cwd');
}
Expand All @@ -59,6 +61,8 @@ public static function check($debug = false, $is_diff = false)
}
}

$files_checked = [];

if ($diff_files === null || $deleted_files === null || count($diff_files) > 200) {
foreach (self::$config->getIncludeDirs() as $dir_name) {
self::checkDirWithConfig($dir_name, self::$config, $debug);
Expand All @@ -75,7 +79,19 @@ public static function check($debug = false, $is_diff = false)
self::checkDiffFilesWithConfig(self::$config, $debug, $file_list);
}

IssueBuffer::finish(true);
$removed_parser_files = FileChecker::deleteOldParserCaches(
$is_diff ? FileChecker::getLastGoodRun() : $start_checks
);

if ($debug && $removed_parser_files) {
echo 'Removed ' . $removed_parser_files . ' old parser caches' . PHP_EOL;
}

if ($is_diff) {
FileChecker::touchParserCaches(self::getAllFiles(self::$config), $start_checks);
}

IssueBuffer::finish(true, (int)$start_checks);
}

/**
Expand Down Expand Up @@ -114,8 +130,6 @@ protected static function checkDirWithConfig($dir_name, Config $config, $debug)
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir_name));
$iterator->rewind();

$files = [];

while ($iterator->valid()) {
if (!$iterator->isDot()) {
$extension = $iterator->getExtension();
Expand All @@ -141,6 +155,31 @@ protected static function checkDirWithConfig($dir_name, Config $config, $debug)
}
}

protected static function getAllFiles(Config $config)
{
$file_extensions = $config->getFileExtensions();
$file_names = [];

foreach ($config->getIncludeDirs() as $dir_name) {
/** @var RecursiveDirectoryIterator */
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir_name));
$iterator->rewind();

while ($iterator->valid()) {
if (!$iterator->isDot()) {
$extension = $iterator->getExtension();
if (in_array($extension, $file_extensions)) {
$file_names[] = (string)$iterator->getRealPath();
}
}

$iterator->next();
}
}

return $file_names;
}

/**
* @param string $dir_name
* @param Config $config
Expand Down
9 changes: 5 additions & 4 deletions src/Psalm/IssueBuffer.php
View file
Expand Up @@ -84,19 +84,20 @@ public static function add(Issue\CodeIssue $e)
}

/**
* @param bool $is_full
* @param bool $is_full
* @param int|null $start_time
* @return void
*/
public static function finish($is_full = false)
public static function finish($is_full = false, $start_time = null)
{
Checker\FileChecker::updateReferenceCache();

if (count(self::$errors)) {
exit(1);
}

if ($is_full) {
Checker\FileChecker::goodRun();
if ($is_full && $start_time) {
Checker\FileChecker::goodRun($start_time);
}
}

Expand Down

0 comments on commit e9b4eb6

Please sign in to comment.