From 88184da084304ebfe0a1eb10513fdb3db135457e Mon Sep 17 00:00:00 2001 From: Max Jacobson Date: Tue, 11 Oct 2016 13:02:59 -0400 Subject: [PATCH] Upgrade PhpParser to support php 7 This library has a handy upgrade guide here: https://github.com/nikic/PHP-Parser/blob/master/UPGRADE-2.0.md Seems there's just one backwards-incompatible change for our wrapper code. I'd like to get this into the next release before we release the OOM fixes. Some php repos are using newer syntax and each file fails to parse, and then it hits an OOM memory that users aren't seeing. Example: https://codeclimate.com/github/etsy/phan/builds/277 Would like to verify on our phan fork on the beta channel that this will work. --- spec/cc/engine/analyzers/php/main_spec.rb | 11 + spec/fixtures/from_phan_php7.php | 273 ++++++++++++++++++++++ vendor/php-parser/composer.json | 2 +- vendor/php-parser/composer.lock | 34 ++- vendor/php-parser/parser.php | 2 +- 5 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 spec/fixtures/from_phan_php7.php diff --git a/spec/cc/engine/analyzers/php/main_spec.rb b/spec/cc/engine/analyzers/php/main_spec.rb index 3da556f8..873ab725 100644 --- a/spec/cc/engine/analyzers/php/main_spec.rb +++ b/spec/cc/engine/analyzers/php/main_spec.rb @@ -90,6 +90,17 @@ expect(run_engine(engine_conf)).to eq("") }.to output(/Skipping file/).to_stderr end + + it "can parse php 7 code" do + create_source_file("foo.php", File.read(fixture_path("from_phan_php7.php"))) + issues = run_engine(engine_conf).strip.split("\0") + result = issues.first.strip + json = JSON.parse(result) + expect(json["location"]).to eq({ + "path" => "foo.php", + "lines" => { "begin" => 117, "end" => 118 }, + }) + end end def engine_conf diff --git a/spec/fixtures/from_phan_php7.php b/spec/fixtures/from_phan_php7.php new file mode 100644 index 00000000..fbbd1b38 --- /dev/null +++ b/spec/fixtures/from_phan_php7.php @@ -0,0 +1,273 @@ +kind); + } + + /** + * @param string|Node|null $node + * An AST node + * + * @param int $indent + * The indentation level for the string + * + * @return string + * A string representation of an AST node + */ + public static function nodeToString( + $node, + $name = null, + int $indent = 0 + ) : string { + $string = str_repeat("\t", $indent); + + if ($name !== null) { + $string .= "$name => "; + } + + if (is_string($node)) { + return $string . $node . "\n"; + } + + if (!$node) { + return $string . 'null' . "\n"; + } + + if (!is_object($node)) { + return $string . $node . "\n"; + } + + $string .= \ast\get_kind_name($node->kind); + + $string .= ' [' + . self::astFlagDescription($node->flags ?? 0) + . ']'; + + if (isset($node->lineno)) { + $string .= ' #' . $node->lineno; + } + + if ($node instanceof Decl) { + if (isset($node->endLineno)) { + $string .= ':' . $node->endLineno; + } + } + + if (isset($node->name)) { + $string .= ' name:' . $node->name; + } + + $string .= "\n"; + + foreach ($node->children ?? [] as $name => $child_node) { + $string .= self::nodeToString( + $child_node, + $name, + $indent + 1 + ); + } + + return $string; + } + + /** + * @return string + * Get a string representation of AST node flags such as + * 'ASSIGN_DIV|TYPE_ARRAY' + */ + public static function astFlagDescription(int $flag) : string + { + $flag_names = []; + foreach (self::$AST_FLAG_ID_NAME_MAP as $id => $name) { + if ($flag == $id) { + $flag_names[] = $name; + } + } + + return implode('|', $flag_names); + } + + /** + * @return string + * Pretty-printer for debug_backtrace + * + * @suppress PhanUnreferencedMethod + */ + public static function backtrace(int $levels = 0) + { + $bt = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, $levels+1); + foreach ($bt as $level => $context) { + if (!$level) { + continue; + } + echo "#".($level-1)." {$context['file']}:{$context['line']} {$context['class']} "; + if (!empty($context['type'])) { + echo $context['class'].$context['type']; + } + echo $context['function']; + echo "\n"; + } + } + + /** + * Note that flag IDs are not unique. You're likely going to get + * an incorrect name back from this. So sorry. + * + * @suppress PhanUnreferencedProperty + */ + private static $AST_FLAG_ID_NAME_MAP = [ + \ast\flags\ASSIGN_ADD => 'ASSIGN_ADD', + \ast\flags\ASSIGN_BITWISE_AND => 'ASSIGN_BITWISE_AND', + \ast\flags\ASSIGN_BITWISE_OR => 'ASSIGN_BITWISE_OR', + \ast\flags\ASSIGN_BITWISE_XOR => 'ASSIGN_BITWISE_XOR', + \ast\flags\ASSIGN_CONCAT => 'ASSIGN_CONCAT', + \ast\flags\ASSIGN_DIV => 'ASSIGN_DIV', + \ast\flags\ASSIGN_MOD => 'ASSIGN_MOD', + \ast\flags\ASSIGN_MUL => 'ASSIGN_MUL', + \ast\flags\ASSIGN_POW => 'ASSIGN_POW', + \ast\flags\ASSIGN_SHIFT_LEFT => 'ASSIGN_SHIFT_LEFT', + \ast\flags\ASSIGN_SHIFT_RIGHT => 'ASSIGN_SHIFT_RIGHT', + \ast\flags\ASSIGN_SUB => 'ASSIGN_SUB', + \ast\flags\BINARY_ADD => 'BINARY_ADD', + \ast\flags\BINARY_BITWISE_AND => 'BINARY_BITWISE_AND', + \ast\flags\BINARY_BITWISE_OR => 'BINARY_BITWISE_OR', + \ast\flags\BINARY_BITWISE_XOR => 'BINARY_BITWISE_XOR', + \ast\flags\BINARY_BOOL_XOR => 'BINARY_BOOL_XOR', + \ast\flags\BINARY_CONCAT => 'BINARY_CONCAT', + \ast\flags\BINARY_DIV => 'BINARY_DIV', + \ast\flags\BINARY_IS_EQUAL => 'BINARY_IS_EQUAL', + \ast\flags\BINARY_IS_IDENTICAL => 'BINARY_IS_IDENTICAL', + \ast\flags\BINARY_IS_NOT_EQUAL => 'BINARY_IS_NOT_EQUAL', + \ast\flags\BINARY_IS_NOT_IDENTICAL => 'BINARY_IS_NOT_IDENTICAL', + \ast\flags\BINARY_IS_SMALLER => 'BINARY_IS_SMALLER', + \ast\flags\BINARY_IS_SMALLER_OR_EQUAL => 'BINARY_IS_SMALLER_OR_EQUAL', + \ast\flags\BINARY_MOD => 'BINARY_MOD', + \ast\flags\BINARY_MUL => 'BINARY_MUL', + \ast\flags\BINARY_POW => 'BINARY_POW', + \ast\flags\BINARY_SHIFT_LEFT => 'BINARY_SHIFT_LEFT', + \ast\flags\BINARY_SHIFT_RIGHT => 'BINARY_SHIFT_RIGHT', + \ast\flags\BINARY_SPACESHIP => 'BINARY_SPACESHIP', + \ast\flags\BINARY_SUB => 'BINARY_SUB', + \ast\flags\CLASS_ABSTRACT => 'CLASS_ABSTRACT', + \ast\flags\CLASS_FINAL => 'CLASS_FINAL', + \ast\flags\CLASS_INTERFACE => 'CLASS_INTERFACE', + \ast\flags\CLASS_TRAIT => 'CLASS_TRAIT', + \ast\flags\MODIFIER_ABSTRACT => 'MODIFIER_ABSTRACT', + \ast\flags\MODIFIER_FINAL => 'MODIFIER_FINAL', + \ast\flags\MODIFIER_PRIVATE => 'MODIFIER_PRIVATE', + \ast\flags\MODIFIER_PROTECTED => 'MODIFIER_PROTECTED', + \ast\flags\MODIFIER_PUBLIC => 'MODIFIER_PUBLIC', + \ast\flags\MODIFIER_STATIC => 'MODIFIER_STATIC', + \ast\flags\NAME_FQ => 'NAME_FQ', + \ast\flags\NAME_NOT_FQ => 'NAME_NOT_FQ', + \ast\flags\NAME_RELATIVE => 'NAME_RELATIVE', + \ast\flags\PARAM_REF => 'PARAM_REF', + \ast\flags\PARAM_VARIADIC => 'PARAM_VARIADIC', + \ast\flags\RETURNS_REF => 'RETURNS_REF', + \ast\flags\TYPE_ARRAY => 'TYPE_ARRAY', + \ast\flags\TYPE_BOOL => 'TYPE_BOOL', + \ast\flags\TYPE_CALLABLE => 'TYPE_CALLABLE', + \ast\flags\TYPE_DOUBLE => 'TYPE_DOUBLE', + \ast\flags\TYPE_LONG => 'TYPE_LONG', + \ast\flags\TYPE_NULL => 'TYPE_NULL', + \ast\flags\TYPE_OBJECT => 'TYPE_OBJECT', + \ast\flags\TYPE_STRING => 'TYPE_STRING', + \ast\flags\UNARY_BITWISE_NOT => 'UNARY_BITWISE_NOT', + \ast\flags\UNARY_BOOL_NOT => 'UNARY_BOOL_NOT', + \ast\flags\BINARY_BOOL_AND => 'BINARY_BOOL_AND', + \ast\flags\BINARY_BOOL_OR => 'BINARY_BOOL_OR', + \ast\flags\BINARY_IS_GREATER => 'BINARY_IS_GREATER', + \ast\flags\BINARY_IS_GREATER_OR_EQUAL => 'BINARY_IS_GREATER_OR_EQUAL', + \ast\flags\CLASS_ANONYMOUS => 'CLASS_ANONYMOUS', + \ast\flags\EXEC_EVAL => 'EXEC_EVAL', + \ast\flags\EXEC_INCLUDE => 'EXEC_INCLUDE', + \ast\flags\EXEC_INCLUDE_ONCE => 'EXEC_INCLUDE_ONCE', + \ast\flags\EXEC_REQUIRE => 'EXEC_REQUIRE', + \ast\flags\EXEC_REQUIRE_ONCE => 'EXEC_REQUIRE_ONCE', + \ast\flags\MAGIC_CLASS => 'MAGIC_CLASS', + \ast\flags\MAGIC_DIR => 'MAGIC_DIR', + \ast\flags\MAGIC_FILE => 'MAGIC_FILE', + \ast\flags\MAGIC_FUNCTION => 'MAGIC_FUNCTION', + \ast\flags\MAGIC_LINE => 'MAGIC_LINE', + \ast\flags\MAGIC_METHOD => 'MAGIC_METHOD', + \ast\flags\MAGIC_NAMESPACE => 'MAGIC_NAMESPACE', + \ast\flags\MAGIC_TRAIT => 'MAGIC_TRAIT', + \ast\flags\UNARY_MINUS => 'UNARY_MINUS', + \ast\flags\UNARY_PLUS => 'UNARY_PLUS', + \ast\flags\UNARY_SILENCE => 'UNARY_SILENCE', + \ast\flags\USE_CONST => 'USE_CONST', + \ast\flags\USE_FUNCTION => 'USE_FUNCTION', + \ast\flags\USE_NORMAL => 'USE_NORMAL', + ]; +} diff --git a/vendor/php-parser/composer.json b/vendor/php-parser/composer.json index 03d7b56b..3af5e357 100644 --- a/vendor/php-parser/composer.json +++ b/vendor/php-parser/composer.json @@ -1,5 +1,5 @@ { "require": { - "nikic/php-parser": "1.0.1" + "nikic/php-parser": "2.1.1" } } diff --git a/vendor/php-parser/composer.lock b/vendor/php-parser/composer.lock index d32de746..7f955d5c 100644 --- a/vendor/php-parser/composer.lock +++ b/vendor/php-parser/composer.lock @@ -1,38 +1,46 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file" + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "This file is @generated automatically" ], - "hash": "72b997b4115652c1f9828b9dc482ec6b", + "hash": "b194f5642b2d727bd49dc06023d53f58", + "content-hash": "2c41b7a3b4f5ef7fa2f79a868b71e518", "packages": [ { "name": "nikic/php-parser", - "version": "v1.0.1", + "version": "v2.1.1", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "b9a60372f26356feb85b4b9ca50a395a5f0d7f34" + "reference": "4dd659edadffdc2143e4753df655d866dbfeedf0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/b9a60372f26356feb85b4b9ca50a395a5f0d7f34", - "reference": "b9a60372f26356feb85b4b9ca50a395a5f0d7f34", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/4dd659edadffdc2143e4753df655d866dbfeedf0", + "reference": "4dd659edadffdc2143e4753df655d866dbfeedf0", "shasum": "" }, "require": { "ext-tokenizer": "*", - "php": ">=5.3" + "php": ">=5.4" }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, + "bin": [ + "bin/php-parse" + ], "type": "library", "extra": { "branch-alias": { - "dev-master": "1.0-dev" + "dev-master": "2.1-dev" } }, "autoload": { - "files": [ - "lib/bootstrap.php" - ] + "psr-4": { + "PhpParser\\": "lib/PhpParser" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -48,13 +56,15 @@ "parser", "php" ], - "time": "2014-10-14 19:40:07" + "time": "2016-09-16 12:04:44" } ], "packages-dev": [], "aliases": [], "minimum-stability": "stable", "stability-flags": [], + "prefer-stable": false, + "prefer-lowest": false, "platform": [], "platform-dev": [] } diff --git a/vendor/php-parser/parser.php b/vendor/php-parser/parser.php index 097d19f5..e3432f99 100644 --- a/vendor/php-parser/parser.php +++ b/vendor/php-parser/parser.php @@ -7,7 +7,7 @@ try { - $parser = new PhpParser\Parser(new PhpParser\Lexer\Emulative); + $parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7); $code = file_get_contents("php://stdin"); $stmts = $parser->parse($code);