Merge pull request #492 from PHPCSStandards/feature/tests-tokenizer-r…

…eorganize

Feature/tests tokenizer reorganize

phpcs.xml.dist

@@ -139,7 +139,7 @@
 
     <!-- This test file specifically *needs* Windows line endings for testing purposes. -->
     <rule ref="Generic.Files.LineEndings.InvalidEOLChar">
-        <exclude-pattern>tests/Core/Tokenizer/StableCommentWhitespaceWinTest\.php</exclude-pattern>
+        <exclude-pattern>tests/Core/Tokenizer/PHP/StableCommentWhitespaceWinTest\.php</exclude-pattern>
     </rule>
 
     <!-- Avoid false positive with this sniff detecting itself -->

tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php → tests/Core/Tokenizer/PHP/AnonClassParenthesisOwnerTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class AnonClassParenthesisOwnerTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/ArrayKeywordTest.php → tests/Core/Tokenizer/PHP/ArrayKeywordTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class ArrayKeywordTest extends AbstractTokenizerTestCase
 {
@@ -21,7 +23,6 @@ final class ArrayKeywordTest extends AbstractTokenizerTestCase
      *
      * @dataProvider dataArrayKeyword
      * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
-     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
      *
      * @return void
      */
@@ -35,10 +36,6 @@ public function testArrayKeyword($testMarker, $testContent='array')
         $this->assertSame(T_ARRAY, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_ARRAY (code)');
         $this->assertSame('T_ARRAY', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_ARRAY (type)');
 
-        $this->assertArrayHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is not set');
-        $this->assertArrayHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is not set');
-        $this->assertArrayHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is not set');
-
     }//end testArrayKeyword()
 
 
@@ -84,7 +81,6 @@ public static function dataArrayKeyword()
      *
      * @dataProvider dataArrayType
      * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
-     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
      *
      * @return void
      */
@@ -98,10 +94,6 @@ public function testArrayType($testMarker, $testContent='array')
         $this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
         $this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
 
-        $this->assertArrayNotHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is set');
-        $this->assertArrayNotHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is set');
-        $this->assertArrayNotHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is set');
-
     }//end testArrayType()
 
 
@@ -163,7 +155,6 @@ public static function dataArrayType()
      *
      * @dataProvider dataNotArrayKeyword
      * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
-     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
      *
      * @return void
      */
@@ -177,10 +168,6 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
         $this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
         $this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
 
-        $this->assertArrayNotHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is set');
-        $this->assertArrayNotHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is set');
-        $this->assertArrayNotHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is set');
-
     }//end testNotArrayKeyword()
 
 

tests/Core/Tokenizer/AttributesTest.php → tests/Core/Tokenizer/PHP/AttributesTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class AttributesTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/BackfillEnumTest.php → tests/Core/Tokenizer/PHP/BackfillEnumTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class BackfillEnumTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.php → tests/Core/Tokenizer/PHP/BackfillExplicitOctalNotationTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class BackfillExplicitOctalNotationTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/BackfillFnTokenTest.php → tests/Core/Tokenizer/PHP/BackfillFnTokenTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class BackfillFnTokenTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/BackfillMatchTokenTest.php → tests/Core/Tokenizer/PHP/BackfillMatchTokenTest.php

@@ -8,8 +8,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
 
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 use PHP_CodeSniffer\Util\Tokens;
 
 final class BackfillMatchTokenTest extends AbstractTokenizerTestCase

tests/Core/Tokenizer/BackfillNumericSeparatorTest.php → tests/Core/Tokenizer/PHP/BackfillNumericSeparatorTest.php

@@ -7,8 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
 
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 use PHP_CodeSniffer\Util\Tokens;
 
 final class BackfillNumericSeparatorTest extends AbstractTokenizerTestCase

tests/Core/Tokenizer/BackfillReadonlyTest.php → tests/Core/Tokenizer/PHP/BackfillReadonlyTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class BackfillReadonlyTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/BitwiseOrTest.php → tests/Core/Tokenizer/PHP/BitwiseOrTest.php

@@ -7,7 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 
 final class BitwiseOrTest extends AbstractTokenizerTestCase
 {

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php → tests/Core/Tokenizer/PHP/ContextSensitiveKeywordsTest.php

@@ -7,8 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
 
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 use PHP_CodeSniffer\Util\Tokens;
 
 final class ContextSensitiveKeywordsTest extends AbstractTokenizerTestCase

tests/Core/Tokenizer/DNFTypesTest.php → tests/Core/Tokenizer/PHP/DNFTypesTest.php

@@ -7,8 +7,9 @@
  * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
  */
 
-namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\PHP;
 
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
 use PHP_CodeSniffer\Util\Tokens;
 
 final class DNFTypesTest extends AbstractTokenizerTestCase
@@ -18,17 +19,16 @@ final class DNFTypesTest extends AbstractTokenizerTestCase
     /**
      * Test that parentheses when **not** used in a type declaration are correctly tokenized.
      *
-     * @param string    $testMarker      The comment prefacing the target token.
-     * @param int|false $owner           Optional. The parentheses owner or false when no parentheses owner is expected.
-     * @param bool      $skipCheckInside Optional. Skip checking correct token type inside the parentheses.
-     *                                   Use judiciously for combined normal + DNF tests only.
+     * @param string $testMarker      The comment prefacing the target token.
+     * @param bool   $skipCheckInside Optional. Skip checking correct token type inside the parentheses.
+     *                                Use judiciously for combined normal + DNF tests only.
      *
      * @dataProvider dataNormalParentheses
-     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createParenthesisNestingMap
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::processAdditional
      *
      * @return void
      */
-    public function testNormalParentheses($testMarker, $owner=false, $skipCheckInside=false)
+    public function testNormalParentheses($testMarker, $skipCheckInside=false)
     {
         $tokens = $this->phpcsFile->getTokens();
 
@@ -39,40 +39,14 @@ public function testNormalParentheses($testMarker, $owner=false, $skipCheckInsid
         $this->assertSame(T_OPEN_PARENTHESIS, $opener['code'], 'Token tokenized as '.$opener['type'].', not T_OPEN_PARENTHESIS (code)');
         $this->assertSame('T_OPEN_PARENTHESIS', $opener['type'], 'Token tokenized as '.$opener['type'].', not T_OPEN_PARENTHESIS (type)');
 
-        if ($owner !== false) {
-            $this->assertArrayHasKey('parenthesis_owner', $opener, 'Parenthesis owner is not set');
-            $this->assertSame(($openPtr + $owner), $opener['parenthesis_owner'], 'Opener parenthesis owner is not the expected token');
-        } else {
-            $this->assertArrayNotHasKey('parenthesis_owner', $opener, 'Parenthesis owner is set');
-        }
-
-        $this->assertArrayHasKey('parenthesis_opener', $opener, 'Parenthesis opener is not set');
-        $this->assertArrayHasKey('parenthesis_closer', $opener, 'Parenthesis closer is not set');
-        $this->assertSame($openPtr, $opener['parenthesis_opener'], 'Parenthesis opener is not the expected token');
-
         $closePtr = $opener['parenthesis_closer'];
         $closer   = $tokens[$closePtr];
 
         $this->assertSame(')', $closer['content'], 'Content of type close parenthesis is not ")"');
         $this->assertSame(T_CLOSE_PARENTHESIS, $closer['code'], 'Token tokenized as '.$closer['type'].', not T_CLOSE_PARENTHESIS (code)');
         $this->assertSame('T_CLOSE_PARENTHESIS', $closer['type'], 'Token tokenized as '.$closer['type'].', not T_CLOSE_PARENTHESIS (type)');
 
-        if ($owner !== false) {
-            $this->assertArrayHasKey('parenthesis_owner', $closer, 'Parenthesis owner is not set');
-            $this->assertSame(($openPtr + $owner), $closer['parenthesis_owner'], 'Closer parenthesis owner is not the expected token');
-        } else {
-            $this->assertArrayNotHasKey('parenthesis_owner', $closer, 'Parenthesis owner is set');
-        }
-
-        $this->assertArrayHasKey('parenthesis_opener', $closer, 'Parenthesis opener is not set');
-        $this->assertArrayHasKey('parenthesis_closer', $closer, 'Parenthesis closer is not set');
-        $this->assertSame($closePtr, $closer['parenthesis_closer'], 'Parenthesis closer is not the expected token');
-
         for ($i = ($openPtr + 1); $i < $closePtr; $i++) {
-            $this->assertArrayHasKey('nested_parenthesis', $tokens[$i], "Nested parenthesis key not set on token $i ({$tokens[$i]['type']})");
-            $this->assertArrayHasKey($openPtr, $tokens[$i]['nested_parenthesis'], 'Nested parenthesis is missing target parentheses set');
-            $this->assertSame($closePtr, $tokens[$i]['nested_parenthesis'][$openPtr], 'Nested parenthesis closer not set correctly');
-
             // If there are ampersands, make sure these are tokenized as bitwise and.
             if ($skipCheckInside === false && $tokens[$i]['content'] === '&') {
                 $this->assertSame(T_BITWISE_AND, $tokens[$i]['code'], 'Token tokenized as '.$tokens[$i]['type'].', not T_BITWISE_AND (code)');
@@ -133,44 +107,36 @@ public static function dataNormalParentheses()
             ],
             'parens with owner: function; & in default value'                 => [
                 'testMarker' => '/* testParensOwnerFunctionAmpersandInDefaultValue */',
-                'owner'      => -3,
             ],
             'parens with owner: closure; param declared by & ref'             => [
                 'testMarker' => '/* testParensOwnerClosureAmpersandParamRef */',
-                'owner'      => -1,
             ],
             'parens with owner: if'                                           => [
                 'testMarker' => '/* testParensOwnerIf */',
-                'owner'      => -2,
             ],
             'parens without owner in if condition'                            => [
                 'testMarker' => '/* testParensNoOwnerInIfCondition */',
             ],
             'parens with owner: for'                                          => [
                 'testMarker' => '/* testParensOwnerFor */',
-                'owner'      => -2,
             ],
             'parens without owner in for condition'                           => [
                 'testMarker' => '/* testParensNoOwnerInForCondition */',
             ],
             'parens with owner: match'                                        => [
                 'testMarker' => '/* testParensOwnerMatch */',
-                'owner'      => -1,
             ],
             'parens with owner: array'                                        => [
                 'testMarker' => '/* testParensOwnerArray */',
-                'owner'      => -2,
             ],
             'parens without owner in array; function call with & in callable' => [
                 'testMarker' => '/* testParensNoOwnerFunctionCallWithAmpersandInCallable */',
             ],
             'parens with owner: fn; & in return value'                        => [
                 'testMarker' => '/* testParensOwnerArrowFn */',
-                'owner'      => -1,
             ],
             'parens with owner: list with reference vars'                     => [
                 'testMarker' => '/* testParensOwnerListWithRefVars */',
-                'owner'      => -1,
             ],
             'parens without owner, function call with DNF look-a-like param'  => [
                 'testMarker' => '/* testParensNoOwnerFunctionCallwithDNFLookALikeParam */',
@@ -199,11 +165,9 @@ public static function dataNormalParentheses()
             ],
             'parens with owner: closure; & in default value'                  => [
                 'testMarker' => '/* testParensOwnerClosureAmpersandInDefaultValue */',
-                'owner'      => -2,
             ],
             'parens with owner: fn; dnf used within'                          => [
                 'testMarker'      => '/* testParensOwnerArrowDNFUsedWithin */',
-                'owner'           => -2,
                 'skipCheckInside' => true,
             ],
             'parens without owner: default value for param in arrow function' => [
@@ -228,7 +192,6 @@ public static function dataNormalParentheses()
      *
      * @dataProvider dataDNFTypeParentheses
      * @covers       PHP_CodeSniffer\Tokenizers\PHP::processAdditional
-     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createParenthesisNestingMap
      *
      * @return void
      */
@@ -243,29 +206,15 @@ public function testDNFTypeParentheses($testMarker)
         $this->assertSame(T_TYPE_OPEN_PARENTHESIS, $opener['code'], 'Token tokenized as '.$opener['type'].', not T_TYPE_OPEN_PARENTHESIS (code)');
         $this->assertSame('T_TYPE_OPEN_PARENTHESIS', $opener['type'], 'Token tokenized as '.$opener['type'].', not T_TYPE_OPEN_PARENTHESIS (type)');
 
-        $this->assertArrayNotHasKey('parenthesis_owner', $opener, 'Parenthesis owner is set');
-        $this->assertArrayHasKey('parenthesis_opener', $opener, 'Parenthesis opener is not set');
-        $this->assertArrayHasKey('parenthesis_closer', $opener, 'Parenthesis closer is not set');
-        $this->assertSame($openPtr, $opener['parenthesis_opener'], 'Parenthesis opener is not the expected token');
-
         $closePtr = $opener['parenthesis_closer'];
         $closer   = $tokens[$closePtr];
 
         $this->assertSame(')', $closer['content'], 'Content of type close parenthesis is not ")"');
         $this->assertSame(T_TYPE_CLOSE_PARENTHESIS, $closer['code'], 'Token tokenized as '.$closer['type'].', not T_TYPE_CLOSE_PARENTHESIS (code)');
         $this->assertSame('T_TYPE_CLOSE_PARENTHESIS', $closer['type'], 'Token tokenized as '.$closer['type'].', not T_TYPE_CLOSE_PARENTHESIS (type)');
 
-        $this->assertArrayNotHasKey('parenthesis_owner', $closer, 'Parenthesis owner is set');
-        $this->assertArrayHasKey('parenthesis_opener', $closer, 'Parenthesis opener is not set');
-        $this->assertArrayHasKey('parenthesis_closer', $closer, 'Parenthesis closer is not set');
-        $this->assertSame($closePtr, $closer['parenthesis_closer'], 'Parenthesis closer is not the expected token');
-
         $intersectionCount = 0;
         for ($i = ($openPtr + 1); $i < $closePtr; $i++) {
-            $this->assertArrayHasKey('nested_parenthesis', $tokens[$i], "Nested parenthesis key not set on token $i ({$tokens[$i]['type']})");
-            $this->assertArrayHasKey($openPtr, $tokens[$i]['nested_parenthesis'], 'Nested parenthesis is missing target parentheses set');
-            $this->assertSame($closePtr, $tokens[$i]['nested_parenthesis'][$openPtr], 'Nested parenthesis closer not set correctly');
-
             if ($tokens[$i]['content'] === '&') {
                 $this->assertSame(
                     T_TYPE_INTERSECTION,