From 9733ab7a863f515215fadbf7236d1f96f5a974df Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 18 Feb 2016 11:08:13 +0100 Subject: [PATCH 1/6] Test extraction of variables and concatenated strings --- tests/PhpCodeExtractorTest.php | 5 ++++- tests/files/special-chars.php | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/PhpCodeExtractorTest.php b/tests/PhpCodeExtractorTest.php index 3c9c12a3..3a8a7dfa 100644 --- a/tests/PhpCodeExtractorTest.php +++ b/tests/PhpCodeExtractorTest.php @@ -68,6 +68,9 @@ public function testSpecialChars() $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'plain')); $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'DATE \\a\\t TIME')); $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "FIELD\tFIELD")); - $this->assertCount(3, $translations); + $this->assertFalse($translations->find(null, "text ")); + $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "text concatenated with 'comments'")); + $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "Stop at the variable")); + $this->assertCount(5, $translations); } } diff --git a/tests/files/special-chars.php b/tests/files/special-chars.php index 5dd008ca..e9e021f5 100644 --- a/tests/files/special-chars.php +++ b/tests/files/special-chars.php @@ -1,7 +1,15 @@
-

+

+

+

+

From 242c93fd6160ba2f27e62df0322ece1fee601eac Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 18 Feb 2016 11:25:23 +0100 Subject: [PATCH 2/6] Add space after function name --- tests/files/special-chars.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/files/special-chars.php b/tests/files/special-chars.php index e9e021f5..ba0e06e8 100644 --- a/tests/files/special-chars.php +++ b/tests/files/special-chars.php @@ -1,5 +1,5 @@
-

+

From ab91bb73c00709f13e93152ce7c6bea5433c38b6 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 18 Feb 2016 11:33:11 +0100 Subject: [PATCH 3/6] Improve code quality --- src/Utils/PhpFunctionsScanner.php | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Utils/PhpFunctionsScanner.php b/src/Utils/PhpFunctionsScanner.php index fb28bdda..dd55d921 100644 --- a/src/Utils/PhpFunctionsScanner.php +++ b/src/Utils/PhpFunctionsScanner.php @@ -37,18 +37,20 @@ public function getFunctions() continue; } - //add an argument to the current function - if (isset($bufferFunctions[0]) && ($value[0] === T_CONSTANT_ENCAPSED_STRING)) { - $bufferFunctions[0][2][] = \Gettext\Extractors\PhpCode::convertString($value[1]); - continue; - } - - //new function found - if (($value[0] === T_STRING) && is_string($this->tokens[$k + 1]) && ($this->tokens[$k + 1] === '(')) { - array_unshift($bufferFunctions, array($value[1], $value[2], array())); - ++$k; - - continue; + switch ($value[0]) { + case T_CONSTANT_ENCAPSED_STRING: + //add an argument to the current function + if (isset($bufferFunctions[0])) { + $bufferFunctions[0][2][] = \Gettext\Extractors\PhpCode::convertString($value[1]); + } + break; + case T_STRING: + //new function found + if (is_string($this->tokens[$k + 1]) && ($this->tokens[$k + 1] === '(')) { + array_unshift($bufferFunctions, array($value[1], $value[2], array())); + ++$k; + } + break; } } From 52400278e634d8f53fd586a00cbd2284596f7f70 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 18 Feb 2016 12:00:47 +0100 Subject: [PATCH 4/6] Skip comments and white spaces when looking for opening parenthesis after function name --- src/Utils/PhpFunctionsScanner.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Utils/PhpFunctionsScanner.php b/src/Utils/PhpFunctionsScanner.php index dd55d921..cad8b095 100644 --- a/src/Utils/PhpFunctionsScanner.php +++ b/src/Utils/PhpFunctionsScanner.php @@ -46,9 +46,16 @@ public function getFunctions() break; case T_STRING: //new function found - if (is_string($this->tokens[$k + 1]) && ($this->tokens[$k + 1] === '(')) { - array_unshift($bufferFunctions, array($value[1], $value[2], array())); - ++$k; + for ($j = $k + 1; $j < $count; $j++) { + $nextToken = $this->tokens[$j]; + if (is_array($nextToken) && ($nextToken[0] === T_COMMENT || $nextToken[0] === T_WHITESPACE)) { + continue; + } + if ($nextToken === '(') { + array_unshift($bufferFunctions, array($value[1], $value[2], array())); + $k = $j; + } + break; } break; } From f7708f2a12aa35fc7c8d0f6adf77045e6ad1c510 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 18 Feb 2016 12:30:27 +0100 Subject: [PATCH 5/6] Allow strings concatenation --- src/Utils/PhpFunctionsScanner.php | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Utils/PhpFunctionsScanner.php b/src/Utils/PhpFunctionsScanner.php index cad8b095..e463b4c6 100644 --- a/src/Utils/PhpFunctionsScanner.php +++ b/src/Utils/PhpFunctionsScanner.php @@ -24,16 +24,22 @@ public function getFunctions() $count = count($this->tokens); $bufferFunctions = array(); $functions = array(); + $concatenating = false; for ($k = 0; $k < $count; ++$k) { $value = $this->tokens[$k]; - //close the current function if (is_string($value)) { + if ($value === '.') { + //concatenating strings + $concatenating = true; + continue; + } + $concatenating = false; if ($value === ')' && isset($bufferFunctions[0])) { + //close the current function $functions[] = array_shift($bufferFunctions); } - continue; } @@ -41,7 +47,12 @@ public function getFunctions() case T_CONSTANT_ENCAPSED_STRING: //add an argument to the current function if (isset($bufferFunctions[0])) { - $bufferFunctions[0][2][] = \Gettext\Extractors\PhpCode::convertString($value[1]); + $string = \Gettext\Extractors\PhpCode::convertString($value[1]); + if ($concatenating && !empty($bufferFunctions[0][2])) { + $bufferFunctions[0][2][count($bufferFunctions[0][2]) - 1] .= $string; + } else { + $bufferFunctions[0][2][] = $string; + } } break; case T_STRING: @@ -59,6 +70,9 @@ public function getFunctions() } break; } + if ($concatenating && $value[0] !== T_COMMENT && $value[0] !== T_WHITESPACE) { + $concatenating = false; + } } return $functions; From 9d2f004468ffc0ca6582a478574140c510ed63cd Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Fri, 19 Feb 2016 09:16:06 +0100 Subject: [PATCH 6/6] Stop parsing function arguments when we find something that's not a string --- src/Utils/ParsedFunction.php | 127 ++++++++++++++++++++++++++++++ src/Utils/PhpFunctionsScanner.php | 59 +++++++++----- 2 files changed, 165 insertions(+), 21 deletions(-) create mode 100644 src/Utils/ParsedFunction.php diff --git a/src/Utils/ParsedFunction.php b/src/Utils/ParsedFunction.php new file mode 100644 index 00000000..1f6fd3c0 --- /dev/null +++ b/src/Utils/ParsedFunction.php @@ -0,0 +1,127 @@ +name = $name; + $this->line = $line; + $this->arguments = array(); + $this->argumentIndex = -1; + $this->argumentStopped = false; + } + + /** + * Stop extracting strings from the current argument (because we found something that's not a string). + */ + public function stopArgument() + { + if ($this->argumentIndex === -1) { + $this->argumentIndex = 0; + } + $this->argumentStopped = true; + } + + /** + * Go to the next argument because we a comma was found. + */ + public function nextArgument() + { + if ($this->argumentIndex === -1) { + // This should neve occur, but let's stay safe - During test/development an Exception should be thrown. + $this->argumentIndex = 1; + } else { + ++$this->argumentIndex; + } + $this->argumentStopped = false; + } + + /** + * Add a string to the current argument. + * + * @param string $chunk + */ + public function addArgumentChunk($chunk) + { + if ($this->argumentStopped === false) { + if ($this->argumentIndex === -1) { + $this->argumentIndex = 0; + } + if (isset($this->arguments[$this->argumentIndex])) { + $this->arguments[$this->argumentIndex] .= $chunk; + } else { + $this->arguments[$this->argumentIndex] = $chunk; + } + } + } + + /** + * A closing parenthesis was found: return the final data. + * + * @return array{ + * + * @var string The function name. + * @var int The line where the function starts. + * @var string[] the strings extracted from the function arguments. + * } + */ + public function close() + { + $arguments = array(); + for ($i = 0; $i <= $this->argumentIndex; ++$i) { + $arguments[$i] = isset($this->arguments[$i]) ? $this->arguments[$i] : ''; + } + + return array( + $this->name, + $this->line, + $arguments, + ); + } +} diff --git a/src/Utils/PhpFunctionsScanner.php b/src/Utils/PhpFunctionsScanner.php index e463b4c6..b487535c 100644 --- a/src/Utils/PhpFunctionsScanner.php +++ b/src/Utils/PhpFunctionsScanner.php @@ -2,6 +2,8 @@ namespace Gettext\Utils; +use Gettext\Extractors\PhpCode; + class PhpFunctionsScanner extends FunctionsScanner { protected $tokens; @@ -23,22 +25,34 @@ public function getFunctions() { $count = count($this->tokens); $bufferFunctions = array(); + /* @var ParsedFunction[] $bufferFunctions */ $functions = array(); - $concatenating = false; + /* @var ParsedFunction[] $functions */ for ($k = 0; $k < $count; ++$k) { $value = $this->tokens[$k]; if (is_string($value)) { - if ($value === '.') { - //concatenating strings - $concatenating = true; - continue; - } - $concatenating = false; - if ($value === ')' && isset($bufferFunctions[0])) { - //close the current function - $functions[] = array_shift($bufferFunctions); + $s = $value; + } else { + $s = token_name($value[0]).' >'.$value[1].'<'; + } + + if (is_string($value)) { + if (isset($bufferFunctions[0])) { + switch ($value) { + case ',': + $bufferFunctions[0]->nextArgument(); + break; + case ')': + $functions[] = array_shift($bufferFunctions)->close(); + break; + case '.': + break; + default: + $bufferFunctions[0]->stopArgument(); + break; + } } continue; } @@ -47,31 +61,34 @@ public function getFunctions() case T_CONSTANT_ENCAPSED_STRING: //add an argument to the current function if (isset($bufferFunctions[0])) { - $string = \Gettext\Extractors\PhpCode::convertString($value[1]); - if ($concatenating && !empty($bufferFunctions[0][2])) { - $bufferFunctions[0][2][count($bufferFunctions[0][2]) - 1] .= $string; - } else { - $bufferFunctions[0][2][] = $string; - } + $bufferFunctions[0]->addArgumentChunk(PhpCode::convertString($value[1])); } break; case T_STRING: + if (isset($bufferFunctions[0])) { + $bufferFunctions[0]->stopArgument(); + } //new function found - for ($j = $k + 1; $j < $count; $j++) { + for ($j = $k + 1; $j < $count; ++$j) { $nextToken = $this->tokens[$j]; if (is_array($nextToken) && ($nextToken[0] === T_COMMENT || $nextToken[0] === T_WHITESPACE)) { continue; } if ($nextToken === '(') { - array_unshift($bufferFunctions, array($value[1], $value[2], array())); + array_unshift($bufferFunctions, new ParsedFunction($value[1], $value[2])); $k = $j; } break; } break; - } - if ($concatenating && $value[0] !== T_COMMENT && $value[0] !== T_WHITESPACE) { - $concatenating = false; + case T_WHITESPACE: + case T_COMMENT: + break; + default: + if (isset($bufferFunctions[0])) { + $bufferFunctions[0]->stopArgument(); + } + break; } }