Merge pull request #22 from jbrooksuk/tokenizer-speed

Change _identifyTokens to write to an internal array of "new tokens"
PHPCheckstyle · Aug 28, 2014 · 6b755ae · 6b755ae
2 parents 1946f4e + 55aa6e3
commit 6b755ae
Showing 1 changed file with 21 additions and 19 deletions.
diff --git a/src/PHPCheckstyle/Tokenizer.php b/src/PHPCheckstyle/Tokenizer.php
@@ -38,6 +38,13 @@ class Tokenizer {
 	 */
 	private $tokens;
 
+	/**
+	 * The array of "new" tokens.
+	 * 
+	 * @var Array
+	 */
+	private $newTokens = array();
+
 	/**
 	 * Position of the index in the current file.
 	 * 
@@ -84,7 +91,6 @@ public function __construct() {
 	public function tokenize($filename) {
 
 		// Read the file
-		$contents = "";
 		if (filesize($filename)) {
 			$fp = fopen($filename, "rb");
 			$this->content = fread($fp, filesize($filename));
@@ -275,6 +281,7 @@ public function peekPrvsValidToken() {
 	public function reset() {
 		$this->index = 0;
 		$this->tokens = array();
+		$this->newTokens = array();
 		$this->tokenNumber = 0;
 		$this->lineNumber = 1;
 	}
@@ -294,11 +301,7 @@ public function checkToken($token, $id, $text = false) {
 		$result = false;
 		if ($token->id == $id) {
 			if ($text) {
-				if ($token->text == $text) {
-					$result = true;
-				} else {
-					$result = false;
-				}
+				$result = $token->text == $text;
 			} else {
 				$result = true;
 			}
@@ -423,9 +426,7 @@ public function checkNextValidToken($id, $text = false, $startPos = null) {
 	 *        	The token text
 	 * @return an array of tokens
 	 */
-	private function _identifyTokens($tokenText, $tokenID) {
-		$newTokens = array();
-
+	private function _identifyTokens($tokenText, $tokenID) {		
 		// Split the data up by newlines
 		// To correctly handle T_NEW_LINE inside comments and HTML
 		$splitData = preg_split('#(\r\n|\n|\r)#', $tokenText, null, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
@@ -525,11 +526,11 @@ private function _identifyTokens($tokenText, $tokenID) {
 					default:
 				}
 			}
-			
-			$newTokens[] = $tokenInfo;
+
+			$this->newTokens[] = $tokenInfo;
 		}
 
-		return $newTokens;
+		return $this->newTokens;
 	}
 
 	/**
@@ -539,7 +540,7 @@ private function _identifyTokens($tokenText, $tokenID) {
 	 *
 	 * @param String $source
 	 *        	The source code to analyse
-	 * @return an array of tokens
+	 * @return array
 	 */
 	private function _getAllTokens($source) {
 		$newTokens = array();
@@ -577,13 +578,13 @@ private function _getAllTokens($source) {
 
 					// Parse the beginning of the text
 					$beforeText = substr($tokenText, 0, $startPos);
-					$newTokens = array_merge($newTokens, $this->_identifyTokens($beforeText, $tokenID));
+					$this->_identifyTokens($beforeText, $tokenID);
 
 					// The open tag
 					$openTag = new TokenInfo();
 					$openTag->id = T_OPEN_TAG;
 					$openTag->text = SHORT_OPEN_TAG;
-					$this->tokenNumber + 1;
+					$this->tokenNumber++;
 					$openTag->position = $this->tokenNumber;
 					$openTag->line = $this->lineNumber;
 					$newTokens[] = $openTag;
@@ -601,7 +602,7 @@ private function _getAllTokens($source) {
 					$closeTag = new TokenInfo();
 					$closeTag->id = T_CLOSE_TAG;
 					$closeTag->text = CLOSE_TAG;
-					$this->tokenNumber + 1;
+					$this->tokenNumber++;
 					$closeTag->position = $this->tokenNumber;
 					$closeTag->line = $this->lineNumber;
 					$newTokens[] = $closeTag;
@@ -615,10 +616,10 @@ private function _getAllTokens($source) {
 			}
 
 			// Identify the tokens
-			$newTokens = array_merge($newTokens, $this->_identifyTokens($tokenText, $tokenID));
+			$this->_identifyTokens($tokenText, $tokenID);
 		}
-		
-		return $newTokens;
+
+		return $this->newTokens;
 	}
 
 	/**
@@ -671,4 +672,5 @@ public function isTokenInList($tokenToCheck, $tokenList) {
 		}
 		return false;
 	}
+
 }