diff --git a/components/Blueprints/class-runner.php b/components/Blueprints/class-runner.php index e7f563de..4d7ab908 100644 --- a/components/Blueprints/class-runner.php +++ b/components/Blueprints/class-runner.php @@ -55,7 +55,7 @@ use WordPress\HttpClient\Client; use WordPress\Zip\ZipFilesystem; -use function WordPress\Encoding\utf8_is_valid_byte_stream; +use function WordPress\Encoding\_wp_has_noncharacters_fallback; use function WordPress\Filesystem\wp_unix_sys_get_temp_dir; use function WordPress\Zip\is_zip_file_stream; @@ -383,7 +383,7 @@ private function load_blueprint() { if ( function_exists( 'mb_check_encoding' ) ) { $is_valid_utf8 = mb_check_encoding( $blueprint_string, 'UTF-8' ); } else { - $is_valid_utf8 = utf8_is_valid_byte_stream( $blueprint_string ); + $is_valid_utf8 = ! _wp_has_noncharacters_fallback( $blueprint_string ); } if ( ! $is_valid_utf8 ) { diff --git a/components/DataLiberation/Tests/CSSTokenizerTest.php b/components/DataLiberation/Tests/CSSTokenizerTest.php new file mode 100644 index 00000000..1c568c2a --- /dev/null +++ b/components/DataLiberation/Tests/CSSTokenizerTest.php @@ -0,0 +1,1330 @@ +collect_tokens( $processor ); + $this->assertSame( $expected_tokens, $actual_tokens ); + } + + /** + * Provides the test cases from the @rmenke/css-tokenizer-test test corpus. + * + * @see https://github.com/romainmenke/css-tokenizer-tests/ + * @return array + */ + static public function corpus_provider(): array { + return json_decode(file_get_contents(__DIR__ . '/css-test-cases.json'), true); + } + + /** + * Collects all tokens from a CSS processor into an array. + * + * @param CSSTokenizer $processor The CSS processor. + * @return array Array of tokens with type, raw, startIndex, endIndex, structured. + */ + static public function collect_tokens( CSSTokenizer $processor, $keys = null ): array { + $tokens = array(); + + while ( $processor->next_token() ) { + $type = $processor->get_token_type(); + + $byte_start = $processor->get_token_start(); + $byte_end = $byte_start + $processor->get_token_length(); + + $token = array( + 'type' => $type, + 'raw' => $processor->get_unnormalized_token(), + 'startIndex' => $byte_start, + 'endIndex' => $byte_end, + 'normalized' => $processor->get_normalized_token(), + 'value' => $processor->get_token_value(), + ); + if ( null !== $processor->get_token_unit() ) { + $token['unit'] = $processor->get_token_unit(); + } + + if ( null !== $keys ) { + $token = array_intersect_key( $token, array_flip( $keys ) ); + } + + $tokens[] = $token; + } + + return $tokens; + } + + /** + * Tests handling of non-UTF-8 byte sequences in identifiers. + * + * Invalid UTF-8 sequences should be replaced with U+FFFD replacement characters + * during tokenization, allowing the CSS to continue processing. + */ + public function test_non_utf8_sequences_in_identifiers(): void { + // Invalid UTF-8 sequence 0xC0 0x80 (overlong encoding). + $css = ".class\xF1name"; + + $expected = array( + // .class�name (0xF1 replaced with U+FFFD). + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => "class\xF1name", + 'normalized' => 'class�name', + 'value' => 'class�name', + ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value', 'unit'] ); + $this->assertSame( $expected, $actual_tokens ); + } + + public function test_invalid_utf8_with_valid_prefix_in_identifiers(): void { + // Invalid 2-byte prefix is replaced with a single U+FFFD. + $css = ".test\xE2\x80name"; + + $expected = array( + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => "test\xE2\x80name", + 'normalized' => 'test�name', + 'value' => 'test�name', + ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value'] ); + $this->assertSame( $expected, $actual_tokens ); + } + + public function test_invalid_utf8_with_two_single_byte_invalid_sequences(): void { + // Two distinct single byte invalid sequences are replaced with + // two separate U+FFFD replacement characters. + $css = ".test\xE2\xE2name"; + + $expected = array( + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => "test\xE2\xE2name", + 'normalized' => 'test��name', + 'value' => 'test��name', + ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value'] ); + $this->assertSame( $expected, $actual_tokens ); + } + + /** + * Legacy test to ensure basic tokenization still works. + */ + public function test_tokenize_labels_core_tokens(): void { + $css = << CSSTokenizer::TOKEN_AT_KEYWORD, 'raw' => '@media' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'screen' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'and' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_PAREN, 'raw' => '(' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'min-width' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '10px' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => "\n\t" ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'background' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'url(' ), + array( 'type' => CSSTokenizer::TOKEN_STRING, 'raw' => '"/images/a.png"' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => "\n" ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of complex selectors with pseudo-classes. + */ + public function test_complex_selector_with_pseudo_classes(): void { + $css = 'a:hover::before, div.class#id:not(.disabled)'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'a' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'hover' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'before' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'div' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '.' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'class' ), + array( 'type' => CSSTokenizer::TOKEN_HASH, 'raw' => '#id' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'not(' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '.' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'disabled' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of CSS comments. + */ + public function test_css_comments(): void { + $css = '/* This is a comment */ .class { color: red; /* Another comment */ }'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_COMMENT, 'raw' => '/* This is a comment */' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '.' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'class' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'red' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_COMMENT, 'raw' => '/* Another comment */' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of media queries. + */ + public function test_media_query(): void { + $css = '@media screen and (min-width: 768px) and (max-width: 1024px)'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_AT_KEYWORD, 'raw' => '@media' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'screen' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'and' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_PAREN, 'raw' => '(' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'min-width' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '768px' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'and' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_PAREN, 'raw' => '(' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'max-width' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '1024px' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of keyframes animation. + */ + public function test_keyframes_animation(): void { + $css = '@keyframes slide-in { 0% { opacity: 0; } 100% { opacity: 1; } }'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_AT_KEYWORD, 'raw' => '@keyframes' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'slide-in' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_PERCENTAGE, 'raw' => '0%' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'opacity' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_PERCENTAGE, 'raw' => '100%' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'opacity' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '1' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of vendor-prefixed properties. + */ + public function test_vendor_prefixed_properties(): void { + $css = '-webkit-transform: rotate(45deg); -moz-border-radius: 5px;'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => '-webkit-transform' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'rotate(' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '45deg' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => '-moz-border-radius' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '5px' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of attribute selectors. + */ + public function test_attribute_selectors(): void { + $css = 'input[type="text"][required], a[href^="https://"]'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'input' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACKET, 'raw' => '[' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'type' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '=' ), + array( 'type' => CSSTokenizer::TOKEN_STRING, 'raw' => '"text"' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACKET, 'raw' => ']' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACKET, 'raw' => '[' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'required' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACKET, 'raw' => ']' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'a' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACKET, 'raw' => '[' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'href' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '^' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '=' ), + array( 'type' => CSSTokenizer::TOKEN_STRING, 'raw' => '"https://"' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACKET, 'raw' => ']' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of calc() function with complex expressions. + */ + public function test_calc_function(): void { + $css = 'width: calc(100% - 20px * 2 + 5em);'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'width' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'calc(' ), + array( 'type' => CSSTokenizer::TOKEN_PERCENTAGE, 'raw' => '100%' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '-' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '20px' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '*' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '2' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '+' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '5em' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of RGB/RGBA color functions. + */ + public function test_color_functions(): void { + $css = 'color: rgb(255, 128, 0); background: rgba(0, 0, 0, 0.5);'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'rgb(' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '255' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '128' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'background' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'rgba(' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '0.5' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of CSS custom properties (variables). + */ + public function test_css_variables(): void { + $css = '--main-color: #ff0000; color: var(--main-color);'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => '--main-color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_HASH, 'raw' => '#ff0000' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'var(' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => '--main-color' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of gradient functions. + */ + public function test_gradient_functions(): void { + $css = 'background: linear-gradient(to right, red 0%, blue 100%);'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'background' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'linear-gradient(' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'to' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'right' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'red' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_PERCENTAGE, 'raw' => '0%' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'blue' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_PERCENTAGE, 'raw' => '100%' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of grid layout properties. + */ + public function test_grid_layout(): void { + $css = 'grid-template-columns: repeat(3, 1fr); gap: 10px 20px;'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'grid-template-columns' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'repeat(' ), + array( 'type' => CSSTokenizer::TOKEN_NUMBER, 'raw' => '3' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '1fr' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'gap' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '10px' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '20px' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of URL functions with various formats. + */ + public function test_url_formats(): void { + $css = 'background: url("image.png"), url(\'font.woff\'), url(https://example.com/bg.jpg);'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'background' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'url(' ), + array( 'type' => CSSTokenizer::TOKEN_STRING, 'raw' => '"image.png"' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_FUNCTION, 'raw' => 'url(' ), + array( 'type' => CSSTokenizer::TOKEN_STRING, 'raw' => "'font.woff'" ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, 'raw' => ')' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_URL, 'raw' => 'url(https://example.com/bg.jpg)' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of !important declarations. + */ + public function test_important_declarations(): void { + $css = 'color: red !important; margin: 0px !important;'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'red' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '!' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'important' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'margin' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DIMENSION, 'raw' => '0px' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '!' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'important' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of multiple selectors with combinators. + */ + public function test_complex_combinators(): void { + $css = 'div > p + span ~ a.link'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'div' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '>' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'p' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '+' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'span' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '~' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'a' ), + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '.' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'link' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests tokenization of escaped characters in identifiers. + */ + public function test_escaped_identifiers(): void { + $css = '.class\\:name, #id\\@special { color: blue; }'; + + $expected = array( + array( 'type' => CSSTokenizer::TOKEN_DELIM, 'raw' => '.' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'class\\:name' ), + array( 'type' => CSSTokenizer::TOKEN_COMMA, 'raw' => ',' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_HASH, 'raw' => '#id\\@special' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, 'raw' => '{' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'color' ), + array( 'type' => CSSTokenizer::TOKEN_COLON, 'raw' => ':' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_IDENT, 'raw' => 'blue' ), + array( 'type' => CSSTokenizer::TOKEN_SEMICOLON, 'raw' => ';' ), + array( 'type' => CSSTokenizer::TOKEN_WHITESPACE, 'raw' => ' ' ), + array( 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, 'raw' => '}' ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw'] ); + $this->assertSame( $actual_tokens, $expected ); + } + + /** + * Tests that get_normalized_token() applies CSS normalization. + * + * Uses a comprehensive CSS selector with rules that includes: + * - CSS escapes in class names and IDs + * - URLs with escape sequences + * - String values with escapes and line endings + * - Comments with various line ending characters + * - Null bytes in identifiers + * - Mixed line endings (\r\n, \r, \f) that need normalization + */ + public function test_get_normalized_token_applies_normalization(): void { + // Comprehensive CSS with normalization requirements. + $css = "/* Comment\r\nwith\flines */\r\n" . + ".c\\6c ass.n\\61 me\r#id\\@value\r\n{\r\n" . + "\tbackground:\furl(path\\2f to\\2f image.png);\r\n" . + "\tcontent:\r\"text\\A string\";\r\n" . + "}"; + + $expected = array( + // Comment with \r\n and \f. + array( + 'type' => CSSTokenizer::TOKEN_COMMENT, + 'raw' => "/* Comment\r\nwith\flines */", + 'normalized' => "/* Comment\nwith\nlines */", + 'value' => null, + ), + // Whitespace with \r\n. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r\n", + 'normalized' => "\n", + 'value' => null, + ), + // Class selector delimiter. + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + // Class name with escape (\6c = 'l'), space gets consumed by escape. + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'c\\6c ass', + 'normalized' => 'class', // Escapes decoded. + 'value' => 'class', // Decoded: \6c → l, space consumed. + ), + // Delimiter. + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + // Identifier with escape (\61 = 'a'), space gets consumed. + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'n\\61 me', + 'normalized' => 'name', // Escapes decoded. + 'value' => 'name', // Decoded: \61 → a, space consumed. + ), + // Whitespace with \r. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r", + 'normalized' => "\n", + 'value' => null, + ), + // ID selector with escape. + array( + 'type' => CSSTokenizer::TOKEN_HASH, + 'raw' => '#id\\@value', + 'normalized' => '#id@value', // Escapes decoded. + 'value' => 'id@value', // Decoded value. + ), + // Whitespace with \r\n. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r\n", + 'normalized' => "\n", + 'value' => null, + ), + // Opening brace. + array( + 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, + 'raw' => '{', + 'normalized' => '{', + 'value' => null, + ), + // Whitespace with \r\n and tab (consumed together). + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r\n\t", + 'normalized' => "\n\t", + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'background', + 'normalized' => 'background', + 'value' => 'background', + ), + // Colon. + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + // Whitespace with \f. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\f", + 'normalized' => "\n", + 'value' => null, + ), + // URL token with escapes (entire url(...) is one token). + array( + 'type' => CSSTokenizer::TOKEN_URL, + 'raw' => 'url(path\\2f to\\2f image.png)', + 'normalized' => 'url(path/to/image.png)', // Escapes decoded. + 'value' => 'path/to/image.png', // Decoded: \2f → /, spaces consumed. + ), + // Semicolon. + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + // Whitespace with \r\n and tab (consumed together). + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r\n\t", + 'normalized' => "\n\t", + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'content', + 'normalized' => 'content', + 'value' => 'content', + ), + // Colon. + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + // Whitespace with \r. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r", + 'normalized' => "\n", + 'value' => null, + ), + // String with escape (\A = newline, space consumed). + array( + 'type' => CSSTokenizer::TOKEN_STRING, + 'raw' => '"text\\A string"', + 'normalized' => "\"text\nstring\"", // Escapes decoded, quotes preserved. + 'value' => "text\nstring", // \A → \n, space consumed. + ), + // Semicolon. + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + // Whitespace with \r\n. + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => "\r\n", + 'normalized' => "\n", + 'value' => null, + ), + // Closing brace. + array( + 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, + 'raw' => '}', + 'normalized' => '}', + 'value' => null, + ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value'] ); + $this->assertSame( $expected, $actual_tokens ); + } + + public function test_dimension_token_value(): void { + $css = '10px;15em;20%;30pt;40pc;50vw;'; + $expected = array( + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '10px', + 'normalized' => '10px', + 'value' => '10', + 'unit' => 'px', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '15em', + 'normalized' => '15em', + 'value' => '15', + 'unit' => 'em', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_PERCENTAGE, + 'raw' => '20%', + 'normalized' => '20%', + 'value' => '20', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '30pt', + 'normalized' => '30pt', + 'value' => '30', + 'unit' => 'pt', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '40pc', + 'normalized' => '40pc', + 'value' => '40', + 'unit' => 'pc', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '50vw', + 'normalized' => '50vw', + 'value' => '50', + 'unit' => 'vw', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + ); + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value', 'unit'] ); + $this->assertSame( $expected, $actual_tokens ); + } + + /** + * Tests that create() validates encoding and only accepts UTF-8. + */ + public function test_create_validates_encoding(): void { + // UTF-8 encoding should work (default). + $tokenizer = CSSTokenizer::create( '.class { color: red; }' ); + $this->assertInstanceOf( CSSTokenizer::class, $tokenizer ); + + // UTF-8 encoding should work (explicit). + $tokenizer = CSSTokenizer::create( '.class { color: red; }', 'UTF-8' ); + $this->assertInstanceOf( CSSTokenizer::class, $tokenizer ); + + // Other encodings should return null. + $tokenizer = CSSTokenizer::create( '.class { color: red; }', 'ISO-8859-1' ); + $this->assertNull( $tokenizer ); + + $tokenizer = CSSTokenizer::create( '.class { color: red; }', 'Windows-1252' ); + $this->assertNull( $tokenizer ); + } + + /** + * Tests escape sequences in unusual and edge-case positions. + * + * Covers: + * - Multiple consecutive escapes + * - Escapes in function names + * - Escapes in at-keywords + * - Escapes in dimension units + * - Null byte escapes (\0) + * - Escaped special characters (@, #, !, etc.) + * - Escaped whitespace that gets consumed by the escape + * - Unicode escapes for various characters + */ + public function test_escape_sequences_in_unusual_places() { + // Complex CSS with escapes in many unusual but valid positions + $css = '@\\6D edia ' . // @media with \6D (m) and space consumed + '\\73 creen ' . // screen with \73 (s) and space consumed + '{' . + ' .\\63 l\\61 ss\\5F name ' . // .class_name with escapes and spaces consumed + "#\\69 d\\5C 0test\x00 " . // #id\0test with null byte escape (should be preserved) + // AND an actual null byte (should be replaced with a U+FFFD REPLACEMENT CHARACTER) + '{' . + ' c\\6F lor: ' . // color: with \6F (o) and space consumed + 'r\\65 d ' . // red with escape + '\\21 important;' . // !important with escaped ! + ' w\\69 dth: ' . // width: + '10\\70 x;' . // 10px (dimension with escaped unit) + ' background: ' . + '\\75 rl(' . // url( with escaped u + '"p\\61 th\\2F img\\2E png"' . // "path/img.png" with escapes + ');' . + ' content: "\\5C \\5C ";' . // "\\ \\" - escaped backslashes + ' font-family: \\22 Arial\\22 ;' . // "Arial" with escaped quotes + ' }' . + '}'; + + $expected = array( + // @\6D edia -> @media + array( + 'type' => CSSTokenizer::TOKEN_AT_KEYWORD, + 'raw' => '@\\6D edia', + 'normalized' => '@media', + 'value' => 'media', + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // \73 creen -> screen + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => '\\73 creen', + 'normalized' => 'screen', + 'value' => 'screen', + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, + 'raw' => '{', + 'normalized' => '{', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // Delimiter . + array( + 'type' => CSSTokenizer::TOKEN_DELIM, + 'raw' => '.', + 'normalized' => '.', + 'value' => '.', + ), + // \63 l\61 ss\5F name -> class_name + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => '\\63 l\\61 ss\\5F name', + 'normalized' => 'class_name', + 'value' => 'class_name', + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // #\69 d\5C 0test -> #id\0test (with encoded null byte) + array( + 'type' => CSSTokenizer::TOKEN_HASH, + 'raw' => "#\\69 d\\5C 0test\x00", + 'normalized' => "#id\\0test�", + // Ensure the value is normalized. + 'value' => "id\\0test�", + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_LEFT_BRACE, + 'raw' => '{', + 'normalized' => '{', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // c\6F lor -> color + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'c\\6F lor', + 'normalized' => 'color', + 'value' => 'color', + ), + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // r\65 d -> red + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'r\\65 d', + 'normalized' => 'red', + 'value' => 'red', + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // \21 important -> !important (single identifier with escaped !) + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => '\\21 important', + 'normalized' => '!important', + 'value' => '!important', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // w\69 dth -> width + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'w\\69 dth', + 'normalized' => 'width', + 'value' => 'width', + ), + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // 10\70 x -> 10px (dimension with escaped unit) + array( + 'type' => CSSTokenizer::TOKEN_DIMENSION, + 'raw' => '10\\70 x', + 'normalized' => '10px', + 'value' => '10', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'background', + 'normalized' => 'background', + 'value' => 'background', + ), + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // \75 rl( -> url( (escaped function name) + array( + 'type' => CSSTokenizer::TOKEN_FUNCTION, + 'raw' => '\\75 rl(', + 'normalized' => 'url(', + 'value' => 'url', + ), + // String with escapes: "p\61 th\2F img\2E png" + array( + 'type' => CSSTokenizer::TOKEN_STRING, + 'raw' => '"p\\61 th\\2F img\\2E png"', + 'normalized' => '"path/img.png"', + 'value' => 'path/img.png', + ), + array( + 'type' => CSSTokenizer::TOKEN_RIGHT_PAREN, + 'raw' => ')', + 'normalized' => ')', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'content', + 'normalized' => 'content', + 'value' => 'content', + ), + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // String with escaped backslashes: "\5C \5C " -> "\\" + // Each \5C sequence (with trailing space consumed) becomes one backslash + array( + 'type' => CSSTokenizer::TOKEN_STRING, + 'raw' => '"\\5C \\5C "', + 'normalized' => '"\\\\"', + 'value' => '\\\\', // Two backslashes total + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => 'font-family', + 'normalized' => 'font-family', + 'value' => 'font-family', + ), + array( + 'type' => CSSTokenizer::TOKEN_COLON, + 'raw' => ':', + 'normalized' => ':', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + // \22 Arial\22 -> "Arial" (escaped quotes make it an ident) + array( + 'type' => CSSTokenizer::TOKEN_IDENT, + 'raw' => '\\22 Arial\\22 ', + 'normalized' => '"Arial"', + 'value' => '"Arial"', + ), + array( + 'type' => CSSTokenizer::TOKEN_SEMICOLON, + 'raw' => ';', + 'normalized' => ';', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_WHITESPACE, + 'raw' => ' ', + 'normalized' => ' ', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, + 'raw' => '}', + 'normalized' => '}', + 'value' => null, + ), + array( + 'type' => CSSTokenizer::TOKEN_RIGHT_BRACE, + 'raw' => '}', + 'normalized' => '}', + 'value' => null, + ), + ); + + $processor = CSSTokenizer::create( $css ); + $actual_tokens = $this->collect_tokens( $processor, ['type', 'raw', 'normalized', 'value'] ); + $this->assertSame( $expected, $actual_tokens ); + } +} diff --git a/components/DataLiberation/Tests/css-test-cases.json b/components/DataLiberation/Tests/css-test-cases.json new file mode 100644 index 00000000..a1c10081 --- /dev/null +++ b/components/DataLiberation/Tests/css-test-cases.json @@ -0,0 +1,4923 @@ +{ + "tests/at-keyword/0001": { + "css": "@foo\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@foo", + "startIndex": 0, + "endIndex": 4, + "normalized": "@foo", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0002": { + "css": "@--\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@--", + "startIndex": 0, + "endIndex": 3, + "normalized": "@--", + "value": "--" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0003": { + "css": "@-1\n", + "tokens": [ + { + "type": "delim-token", + "raw": "@", + "startIndex": 0, + "endIndex": 1, + "normalized": "@", + "value": "@" + }, + { + "type": "number-token", + "raw": "-1", + "startIndex": 1, + "endIndex": 3, + "normalized": "-1", + "value": "-1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0004": { + "css": "@--1\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@--1", + "startIndex": 0, + "endIndex": 4, + "normalized": "@--1", + "value": "--1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0005": { + "css": "@\\@\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@\\@", + "startIndex": 0, + "endIndex": 3, + "normalized": "@@", + "value": "@" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0006": { + "css": "@_\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@_", + "startIndex": 0, + "endIndex": 2, + "normalized": "@_", + "value": "_" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0007": { + "css": "@\n", + "tokens": [ + { + "type": "delim-token", + "raw": "@", + "startIndex": 0, + "endIndex": 1, + "normalized": "@", + "value": "@" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0008": { + "css": "pvA3@\\\neBnP\n", + "tokens": [ + { + "type": "ident-token", + "raw": "pvA3", + "startIndex": 0, + "endIndex": 4, + "normalized": "pvA3", + "value": "pvA3" + }, + { + "type": "delim-token", + "raw": "@", + "startIndex": 4, + "endIndex": 5, + "normalized": "@", + "value": "@" + }, + { + "type": "delim-token", + "raw": "\\", + "startIndex": 5, + "endIndex": 6, + "normalized": "\\", + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + }, + { + "type": "ident-token", + "raw": "eBnP", + "startIndex": 7, + "endIndex": 11, + "normalized": "eBnP", + "value": "eBnP" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/at-keyword/0009": { + "css": "@aa𐀀\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@aa𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "@aa𐀀", + "value": "aa𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-string/0001": { + "css": "\"foo\n\"\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"foo", + "startIndex": 0, + "endIndex": 4, + "normalized": "\"foo", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + }, + { + "type": "bad-string-token", + "raw": "\"", + "startIndex": 5, + "endIndex": 6, + "normalized": "\"", + "value": "" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-string/0002": { + "css": "\"foo\\\n\"\n", + "tokens": [ + { + "type": "string-token", + "raw": "\"foo\\\n\"", + "startIndex": 0, + "endIndex": 7, + "normalized": "\"foo\\\n\"", + "value": "foo\\\n" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-string/0003": { + "css": "\"foo\r\n\"\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"foo", + "startIndex": 0, + "endIndex": 4, + "normalized": "\"foo", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\r\n", + "startIndex": 4, + "endIndex": 6, + "normalized": "\n", + "value": null + }, + { + "type": "bad-string-token", + "raw": "\"", + "startIndex": 6, + "endIndex": 7, + "normalized": "\"", + "value": "" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-string/0004": { + "css": "\"foo\\\r\n\"\n", + "tokens": [ + { + "type": "string-token", + "raw": "\"foo\\\r\n\"", + "startIndex": 0, + "endIndex": 8, + "normalized": "\"foo\\\n\"", + "value": "foo\\\n" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-string/0005": { + "css": "\"aa𐀀\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"aa𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "\"aa𐀀", + "value": "aa𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-url/0001": { + "css": "url(\n", + "tokens": [ + { + "type": "url-token", + "raw": "url(\n", + "startIndex": 0, + "endIndex": 5, + "normalized": "url(\n", + "value": "" + } + ] + }, + "tests/bad-url/0002": { + "css": "url( a\n", + "tokens": [ + { + "type": "url-token", + "raw": "url( a\n", + "startIndex": 0, + "endIndex": 7, + "normalized": "url( a\n", + "value": "a" + } + ] + }, + "tests/bad-url/0003": { + "css": "url( a a\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a\n", + "startIndex": 0, + "endIndex": 9, + "normalized": "url( a a\n", + "value": null + } + ] + }, + "tests/bad-url/0004": { + "css": "url( a a)\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a)", + "startIndex": 0, + "endIndex": 9, + "normalized": "url( a a)", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-url/0005": { + "css": "url( a a\\)\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a\\)\n", + "startIndex": 0, + "endIndex": 11, + "normalized": "url( a a)\n", + "value": null + } + ] + }, + "tests/bad-url/0006": { + "css": "url( \\\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( \\\n", + "startIndex": 0, + "endIndex": 7, + "normalized": "url( \\\n", + "value": null + } + ] + }, + "tests/bad-url/0007": { + "css": "url(a'')\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url(a'')", + "startIndex": 0, + "endIndex": 8, + "normalized": "url(a'')", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "normalized": "\n", + "value": null + } + ] + }, + "tests/bad-url/0008": { + "css": "url(a\")\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url(a\")", + "startIndex": 0, + "endIndex": 7, + "normalized": "url(a\")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/colon/0001": { + "css": ":\n", + "tokens": [ + { + "type": "colon-token", + "raw": ":", + "startIndex": 0, + "endIndex": 1, + "normalized": ":", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/comma/0001": { + "css": ",\n", + "tokens": [ + { + "type": "comma-token", + "raw": ",", + "startIndex": 0, + "endIndex": 1, + "normalized": ",", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/comment/0001": { + "css": "/* a comment */\n", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment */", + "startIndex": 0, + "endIndex": 15, + "normalized": "/* a comment */", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 15, + "endIndex": 16, + "normalized": "\n", + "value": null + } + ] + }, + "tests/comment/0002": { + "css": "/* a comment ", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment ", + "startIndex": 0, + "endIndex": 13, + "normalized": "/* a comment ", + "value": null + } + ] + }, + "tests/comment/0003": { + "css": "a/**/b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "a", + "startIndex": 0, + "endIndex": 1, + "normalized": "a", + "value": "a" + }, + { + "type": "comment", + "raw": "/**/", + "startIndex": 1, + "endIndex": 5, + "normalized": "/**/", + "value": null + }, + { + "type": "ident-token", + "raw": "b", + "startIndex": 5, + "endIndex": 6, + "normalized": "b", + "value": "b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/comment/0004": { + "css": "/*\\*/*/\n", + "tokens": [ + { + "type": "comment", + "raw": "/*\\*/", + "startIndex": 0, + "endIndex": 5, + "normalized": "/**/", + "value": null + }, + { + "type": "delim-token", + "raw": "*", + "startIndex": 5, + "endIndex": 6, + "normalized": "*", + "value": "*" + }, + { + "type": "delim-token", + "raw": "/", + "startIndex": 6, + "endIndex": 7, + "normalized": "/", + "value": "/" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/comment/0005": { + "css": "/* a comment *", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment *", + "startIndex": 0, + "endIndex": 14, + "normalized": "/* a comment *", + "value": null + } + ] + }, + "tests/comment/0006": { + "css": "/*a𐀀*/\n", + "tokens": [ + { + "type": "comment", + "raw": "/*a𐀀*/", + "startIndex": 0, + "endIndex": 9, + "normalized": "/*a𐀀*/", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "normalized": "\n", + "value": null + } + ] + }, + "tests/digit/0001": { + "css": "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n", + "tokens": [ + { + "type": "number-token", + "raw": "0", + "startIndex": 0, + "endIndex": 1, + "normalized": "0", + "value": "0" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "1", + "startIndex": 2, + "endIndex": 3, + "normalized": "1", + "value": "1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "2", + "startIndex": 4, + "endIndex": 5, + "normalized": "2", + "value": "2" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "3", + "startIndex": 6, + "endIndex": 7, + "normalized": "3", + "value": "3" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "4", + "startIndex": 8, + "endIndex": 9, + "normalized": "4", + "value": "4" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "5", + "startIndex": 10, + "endIndex": 11, + "normalized": "5", + "value": "5" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "6", + "startIndex": 12, + "endIndex": 13, + "normalized": "6", + "value": "6" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "7", + "startIndex": 14, + "endIndex": 15, + "normalized": "7", + "value": "7" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 15, + "endIndex": 16, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "8", + "startIndex": 16, + "endIndex": 17, + "normalized": "8", + "value": "8" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 17, + "endIndex": 18, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "9", + "startIndex": 18, + "endIndex": 19, + "normalized": "9", + "value": "9" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 19, + "endIndex": 20, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0001": { + "css": "10px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10px", + "startIndex": 0, + "endIndex": 4, + "normalized": "10px", + "value": "10", + "unit": "px" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0002": { + "css": "10\\70 x\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10\\70 x", + "startIndex": 0, + "endIndex": 7, + "normalized": "10px", + "value": "10", + "unit": "px" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0003": { + "css": "10--custom-px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10--custom-px", + "startIndex": 0, + "endIndex": 13, + "normalized": "10--custom-px", + "value": "10", + "unit": "--custom-px" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0004": { + "css": "10e2px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10e2px", + "startIndex": 0, + "endIndex": 6, + "normalized": "10e2px", + "value": "10e2", + "unit": "px" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0005": { + "css": "10E2PX\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10E2PX", + "startIndex": 0, + "endIndex": 6, + "normalized": "10E2PX", + "value": "10E2", + "unit": "PX" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0006": { + "css": "10\\0\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10\\0\n", + "startIndex": 0, + "endIndex": 5, + "normalized": "10�", + "value": "10", + "unit": "�" + } + ] + }, + "tests/dimension/0007": { + "css": "10a𐀀\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10a𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "10a𐀀", + "value": "10", + "unit": "a𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/dimension/0008": { + "css": "10a\u0000", + "tokens": [ + { + "type": "dimension-token", + "raw": "10a\u0000", + "startIndex": 0, + "endIndex": 4, + "normalized": "10a�", + "value": "10", + "unit": "a�" + } + ] + }, + "tests/escaped-code-point/0001": { + "css": "\\", + "tokens": [ + { + "type": "ident-token", + "raw": "\\", + "startIndex": 0, + "endIndex": 1, + "normalized": "�", + "value": "�" + } + ] + }, + "tests/escaped-code-point/0002": { + "css": "\\0", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0", + "startIndex": 0, + "endIndex": 2, + "normalized": "�", + "value": "�" + } + ] + }, + "tests/escaped-code-point/0003": { + "css": "\\\\", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\\", + "startIndex": 0, + "endIndex": 2, + "normalized": "\\", + "value": "\\" + } + ] + }, + "tests/escaped-code-point/0004": { + "css": "\\0a b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0a b", + "startIndex": 0, + "endIndex": 5, + "normalized": "\nb", + "value": "\nb" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0005": { + "css": "\\0ab \n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0ab ", + "startIndex": 0, + "endIndex": 5, + "normalized": "«", + "value": "«" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0006": { + "css": "\\0ab (foo)\n", + "tokens": [ + { + "type": "function-token", + "raw": "\\0ab (", + "startIndex": 0, + "endIndex": 6, + "normalized": "«(", + "value": "«" + }, + { + "type": "ident-token", + "raw": "foo", + "startIndex": 6, + "endIndex": 9, + "normalized": "foo", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 9, + "endIndex": 10, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 10, + "endIndex": 11, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0007": { + "css": "\\0ab (foo)\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0ab ", + "startIndex": 0, + "endIndex": 5, + "normalized": "«", + "value": "«" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 5, + "endIndex": 6, + "normalized": " ", + "value": null + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 6, + "endIndex": 7, + "normalized": "(", + "value": null + }, + { + "type": "ident-token", + "raw": "foo", + "startIndex": 7, + "endIndex": 10, + "normalized": "foo", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 10, + "endIndex": 11, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0008": { + "css": "\\0000ab\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0000ab\n", + "startIndex": 0, + "endIndex": 8, + "normalized": "«", + "value": "«" + } + ] + }, + "tests/escaped-code-point/0009": { + "css": "\\00000ab\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00000ab", + "startIndex": 0, + "endIndex": 8, + "normalized": "\nb", + "value": "\nb" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0010": { + "css": "\\110000\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\110000\n", + "startIndex": 0, + "endIndex": 8, + "normalized": "�", + "value": "�" + } + ] + }, + "tests/escaped-code-point/0011": { + "css": "\\00D800\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00D800\n", + "startIndex": 0, + "endIndex": 8, + "normalized": "�", + "value": "�" + } + ] + }, + "tests/escaped-code-point/0012": { + "css": "\\00DFFF\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00DFFF\n", + "startIndex": 0, + "endIndex": 8, + "normalized": "�", + "value": "�" + } + ] + }, + "tests/escaped-code-point/0013": { + "css": "\\\n", + "tokens": [ + { + "type": "delim-token", + "raw": "\\", + "startIndex": 0, + "endIndex": 1, + "normalized": "\\", + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0014": { + "css": "\\\u0000\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\u0000", + "startIndex": 0, + "endIndex": 2, + "normalized": "�", + "value": "�" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0015": { + "css": "\\\u0000\b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\u0000", + "startIndex": 0, + "endIndex": 2, + "normalized": "�", + "value": "�" + }, + { + "type": "delim-token", + "raw": "\b", + "startIndex": 2, + "endIndex": 3, + "normalized": "\b", + "value": "\b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/escaped-code-point/0016": { + "css": "\"a\\12\r\nb\"", + "tokens": [ + { + "type": "string-token", + "raw": "\"a\\12\r\nb\"", + "startIndex": 0, + "endIndex": 9, + "normalized": "\"a\u0012b\"", + "value": "a\u0012b" + } + ] + }, + "tests/full-stop/0001": { + "css": ".\n", + "tokens": [ + { + "type": "delim-token", + "raw": ".", + "startIndex": 0, + "endIndex": 1, + "normalized": ".", + "value": "." + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/full-stop/0002": { + "css": ".a\n", + "tokens": [ + { + "type": "delim-token", + "raw": ".", + "startIndex": 0, + "endIndex": 1, + "normalized": ".", + "value": "." + }, + { + "type": "ident-token", + "raw": "a", + "startIndex": 1, + "endIndex": 2, + "normalized": "a", + "value": "a" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/full-stop/0003": { + "css": ".1\n", + "tokens": [ + { + "type": "number-token", + "raw": ".1", + "startIndex": 0, + "endIndex": 2, + "normalized": ".1", + "value": ".1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/fuzz/01a166c0-ca20-43a5-9ab0-0984e4a5362b": { + "css": "4waPtwEEGH\\\u0000jV3zM6hh6w30N0PC 7m8KM0HcWGOPw28Gt(r19", + "tokens": [ + { + "type": "dimension-token", + "raw": "4waPtwEEGH\\\u0000jV3zM6hh6w30N0PC", + "startIndex": 0, + "endIndex": 28, + "normalized": "4waPtwEEGH�jV3zM6hh6w30N0PC", + "value": "4", + "unit": "waPtwEEGH�jV3zM6hh6w30N0PC" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 28, + "endIndex": 29, + "normalized": " ", + "value": null + }, + { + "type": "dimension-token", + "raw": "7m8KM0HcWGOPw28Gt", + "startIndex": 29, + "endIndex": 46, + "normalized": "7m8KM0HcWGOPw28Gt", + "value": "7", + "unit": "m8KM0HcWGOPw28Gt" + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 46, + "endIndex": 47, + "normalized": "(", + "value": null + }, + { + "type": "ident-token", + "raw": "r19", + "startIndex": 47, + "endIndex": 50, + "normalized": "r19", + "value": "r19" + } + ] + }, + "tests/fuzz/2abe9406-c063-4e9a-85ac-b13660671553": { + "css": "ak]P0A}808G\"lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\r\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9\"\"c34Q\"iiC0e45Da5f\"F5X3\"o(", + "tokens": [ + { + "type": "ident-token", + "raw": "ak", + "startIndex": 0, + "endIndex": 2, + "normalized": "ak", + "value": "ak" + }, + { + "type": "]-token", + "raw": "]", + "startIndex": 2, + "endIndex": 3, + "normalized": "]", + "value": null + }, + { + "type": "ident-token", + "raw": "P0A", + "startIndex": 3, + "endIndex": 6, + "normalized": "P0A", + "value": "P0A" + }, + { + "type": "}-token", + "raw": "}", + "startIndex": 6, + "endIndex": 7, + "normalized": "}", + "value": null + }, + { + "type": "dimension-token", + "raw": "808G", + "startIndex": 7, + "endIndex": 11, + "normalized": "808G", + "value": "808", + "unit": "G" + }, + { + "type": "string-token", + "raw": "\"lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\r\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9\"", + "startIndex": 11, + "endIndex": 102, + "normalized": "\"lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9\"", + "value": "lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9" + }, + { + "type": "string-token", + "raw": "\"c34Q\"", + "startIndex": 102, + "endIndex": 108, + "normalized": "\"c34Q\"", + "value": "c34Q" + }, + { + "type": "ident-token", + "raw": "iiC0e45Da5f", + "startIndex": 108, + "endIndex": 119, + "normalized": "iiC0e45Da5f", + "value": "iiC0e45Da5f" + }, + { + "type": "string-token", + "raw": "\"F5X3\"", + "startIndex": 119, + "endIndex": 125, + "normalized": "\"F5X3\"", + "value": "F5X3" + }, + { + "type": "function-token", + "raw": "o(", + "startIndex": 125, + "endIndex": 127, + "normalized": "o(", + "value": "o" + } + ] + }, + "tests/fuzz/4e630a47-507b-4b79-b00f-57f7dc1cc79d": { + "css": "\u000e7rSD6I5L1lglVRlL2X7BbEk\\3HCd\r94 \\\u0000skoW25d4%l64UUskN\"pHun\"!", + "tokens": [ + { + "type": "delim-token", + "raw": "\u000e", + "startIndex": 0, + "endIndex": 1, + "normalized": "\u000e", + "value": "\u000e" + }, + { + "type": "dimension-token", + "raw": "7rSD6I5L1lglVRlL2X7BbEk\\3HCd", + "startIndex": 1, + "endIndex": 29, + "normalized": "7rSD6I5L1lglVRlL2X7BbEk\u0003HCd", + "value": "7", + "unit": "rSD6I5L1lglVRlL2X7BbEk\u0003HCd" + }, + { + "type": "whitespace-token", + "raw": "\r", + "startIndex": 29, + "endIndex": 30, + "normalized": "\n", + "value": null + }, + { + "type": "number-token", + "raw": "94", + "startIndex": 30, + "endIndex": 32, + "normalized": "94", + "value": "94" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 32, + "endIndex": 33, + "normalized": " ", + "value": null + }, + { + "type": "ident-token", + "raw": "\\\u0000skoW25d4", + "startIndex": 33, + "endIndex": 43, + "normalized": "�skoW25d4", + "value": "�skoW25d4" + }, + { + "type": "delim-token", + "raw": "%", + "startIndex": 43, + "endIndex": 44, + "normalized": "%", + "value": "%" + }, + { + "type": "ident-token", + "raw": "l64UUskN", + "startIndex": 44, + "endIndex": 52, + "normalized": "l64UUskN", + "value": "l64UUskN" + }, + { + "type": "string-token", + "raw": "\"pHun\"", + "startIndex": 52, + "endIndex": 58, + "normalized": "\"pHun\"", + "value": "pHun" + }, + { + "type": "delim-token", + "raw": "!", + "startIndex": 58, + "endIndex": 59, + "normalized": "!", + "value": "!" + } + ] + }, + "tests/fuzz/4f865903-e4dd-4a0b-83ed-e630cfa9dcca": { + "css": "gzO0{(p{DzQ7\u0000(a1;r1iN7w)", + "tokens": [ + { + "type": "ident-token", + "raw": "gzO0", + "startIndex": 0, + "endIndex": 4, + "normalized": "gzO0", + "value": "gzO0" + }, + { + "type": "{-token", + "raw": "{", + "startIndex": 4, + "endIndex": 5, + "normalized": "{", + "value": null + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 5, + "endIndex": 6, + "normalized": "(", + "value": null + }, + { + "type": "ident-token", + "raw": "p", + "startIndex": 6, + "endIndex": 7, + "normalized": "p", + "value": "p" + }, + { + "type": "{-token", + "raw": "{", + "startIndex": 7, + "endIndex": 8, + "normalized": "{", + "value": null + }, + { + "type": "function-token", + "raw": "DzQ7\u0000(", + "startIndex": 8, + "endIndex": 14, + "normalized": "DzQ7�(", + "value": "DzQ7�" + }, + { + "type": "ident-token", + "raw": "a1", + "startIndex": 14, + "endIndex": 16, + "normalized": "a1", + "value": "a1" + }, + { + "type": "semicolon-token", + "raw": ";", + "startIndex": 16, + "endIndex": 17, + "normalized": ";", + "value": null + }, + { + "type": "ident-token", + "raw": "r1iN7w", + "startIndex": 17, + "endIndex": 23, + "normalized": "r1iN7w", + "value": "r1iN7w" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 23, + "endIndex": 24, + "normalized": ")", + "value": null + } + ] + }, + "tests/fuzz/5181013c-60ab-483b-9c06-fb32c7e1e7e8": { + "css": "565'E{z\u0000U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\r\nrey\thg7[5%rBK8RUC64Lu␌17O{E\\90873u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G'vK\ndhCEs3tValX ", + "tokens": [ + { + "type": "number-token", + "raw": "565", + "startIndex": 0, + "endIndex": 3, + "normalized": "565", + "value": "565" + }, + { + "type": "string-token", + "raw": "'E{z\u0000U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\r\nrey\thg7[5%rBK8RUC64Lu␌17O{E\\90873u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G'", + "startIndex": 3, + "endIndex": 263, + "normalized": "'E{z�U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\nrey\thg7[5%rBK8RUC64Lu␌17O{E򐡳u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G'", + "value": "E{z�U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\nrey\thg7[5%rBK8RUC64Lu␌17O{E򐡳u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G" + }, + { + "type": "ident-token", + "raw": "vK", + "startIndex": 263, + "endIndex": 265, + "normalized": "vK", + "value": "vK" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 265, + "endIndex": 266, + "normalized": "\n", + "value": null + }, + { + "type": "ident-token", + "raw": "dhCEs3tValX", + "startIndex": 266, + "endIndex": 277, + "normalized": "dhCEs3tValX", + "value": "dhCEs3tValX" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 277, + "endIndex": 278, + "normalized": " ", + "value": null + } + ] + }, + "tests/fuzz/6d07fc79-586f-4efa-a0a2-37d4dd3beb09": { + "css": "FWUNqr7uv8300nz\b,8lU0j6B186kh \u00009 \u000eGZafxf2GIhL9%", + "tokens": [ + { + "type": "ident-token", + "raw": "FWUNqr7uv8300nz", + "startIndex": 0, + "endIndex": 15, + "normalized": "FWUNqr7uv8300nz", + "value": "FWUNqr7uv8300nz" + }, + { + "type": "delim-token", + "raw": "\b", + "startIndex": 15, + "endIndex": 16, + "normalized": "\b", + "value": "\b" + }, + { + "type": "comma-token", + "raw": ",", + "startIndex": 16, + "endIndex": 17, + "normalized": ",", + "value": null + }, + { + "type": "dimension-token", + "raw": "8lU0j6B186kh", + "startIndex": 17, + "endIndex": 29, + "normalized": "8lU0j6B186kh", + "value": "8", + "unit": "lU0j6B186kh" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 29, + "endIndex": 30, + "normalized": " ", + "value": null + }, + { + "type": "ident-token", + "raw": "\u00009", + "startIndex": 30, + "endIndex": 32, + "normalized": "�9", + "value": "�9" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 32, + "endIndex": 33, + "normalized": " ", + "value": null + }, + { + "type": "delim-token", + "raw": "\u000e", + "startIndex": 33, + "endIndex": 34, + "normalized": "\u000e", + "value": "\u000e" + }, + { + "type": "ident-token", + "raw": "GZafxf2GIhL9", + "startIndex": 34, + "endIndex": 46, + "normalized": "GZafxf2GIhL9", + "value": "GZafxf2GIhL9" + }, + { + "type": "delim-token", + "raw": "%", + "startIndex": 46, + "endIndex": 47, + "normalized": "%", + "value": "%" + } + ] + }, + "tests/fuzz/7f49c8fc-8292-4a3e-828b-b5d028a80d5f": { + "css": "FZ 0B120h5QUbNbmTD2K8mAD傿i+Yv9V0KS14Ng18ag'\\\r\n{Xu)k2a76}y4\\6fb9ONI\\", + "tokens": [ + { + "type": "delim-token", + "raw": ">", + "startIndex": 0, + "endIndex": 1, + "normalized": ">", + "value": ">" + }, + { + "type": "ident-token", + "raw": "u", + "startIndex": 1, + "endIndex": 2, + "normalized": "u", + "value": "u" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 2, + "endIndex": 3, + "normalized": ")", + "value": null + }, + { + "type": "ident-token", + "raw": "k2a76", + "startIndex": 3, + "endIndex": 8, + "normalized": "k2a76", + "value": "k2a76" + }, + { + "type": "}-token", + "raw": "}", + "startIndex": 8, + "endIndex": 9, + "normalized": "}", + "value": null + }, + { + "type": "ident-token", + "raw": "y4\\6fb9ONI\\", + "startIndex": 9, + "endIndex": 20, + "normalized": "y4澹ONI�", + "value": "y4澹ONI�" + } + ] + }, + "tests/hash/0001": { + "css": "#1\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#1", + "startIndex": 0, + "endIndex": 2, + "normalized": "#1", + "value": "1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0002": { + "css": "#-2\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-2", + "startIndex": 0, + "endIndex": 3, + "normalized": "#-2", + "value": "-2" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0003": { + "css": "#--3\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#--3", + "startIndex": 0, + "endIndex": 4, + "normalized": "#--3", + "value": "--3" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0004": { + "css": "#---4\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#---4", + "startIndex": 0, + "endIndex": 5, + "normalized": "#---4", + "value": "---4" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0005": { + "css": "#a\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#a", + "startIndex": 0, + "endIndex": 2, + "normalized": "#a", + "value": "a" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0006": { + "css": "#-b\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-b", + "startIndex": 0, + "endIndex": 3, + "normalized": "#-b", + "value": "-b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0007": { + "css": "#--c\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#--c", + "startIndex": 0, + "endIndex": 4, + "normalized": "#--c", + "value": "--c" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0008": { + "css": "#---d\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#---d", + "startIndex": 0, + "endIndex": 5, + "normalized": "#---d", + "value": "---d" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0009": { + "css": "#_\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#_", + "startIndex": 0, + "endIndex": 2, + "normalized": "#_", + "value": "_" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0010": { + "css": "#_1\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#_1", + "startIndex": 0, + "endIndex": 3, + "normalized": "#_1", + "value": "_1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0011": { + "css": "#-\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-", + "startIndex": 0, + "endIndex": 2, + "normalized": "#-", + "value": "-" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0012": { + "css": "#+\n", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "normalized": "#", + "value": "#" + }, + { + "type": "delim-token", + "raw": "+", + "startIndex": 1, + "endIndex": 2, + "normalized": "+", + "value": "+" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0013": { + "css": "##\n", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "normalized": "#", + "value": "#" + }, + { + "type": "delim-token", + "raw": "#", + "startIndex": 1, + "endIndex": 2, + "normalized": "#", + "value": "#" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hash/0014": { + "css": "#", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "normalized": "#", + "value": "#" + } + ] + }, + "tests/hash/0015": { + "css": "#aa𐀀\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#aa𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "#aa𐀀", + "value": "aa𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0001": { + "css": "-\n", + "tokens": [ + { + "type": "delim-token", + "raw": "-", + "startIndex": 0, + "endIndex": 1, + "normalized": "-", + "value": "-" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0002": { + "css": "-1\n", + "tokens": [ + { + "type": "number-token", + "raw": "-1", + "startIndex": 0, + "endIndex": 2, + "normalized": "-1", + "value": "-1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0003": { + "css": "-.1\n", + "tokens": [ + { + "type": "number-token", + "raw": "-.1", + "startIndex": 0, + "endIndex": 3, + "normalized": "-.1", + "value": "-.1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0004": { + "css": "--1\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--1", + "startIndex": 0, + "endIndex": 3, + "normalized": "--1", + "value": "--1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0005": { + "css": "-0\n", + "tokens": [ + { + "type": "number-token", + "raw": "-0", + "startIndex": 0, + "endIndex": 2, + "normalized": "-0", + "value": "-0" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/hyphen-minus/0006": { + "css": "-->\n", + "tokens": [ + { + "type": "CDC-token", + "raw": "-->", + "startIndex": 0, + "endIndex": 3, + "normalized": "-->", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0001": { + "css": "foo\n", + "tokens": [ + { + "type": "ident-token", + "raw": "foo", + "startIndex": 0, + "endIndex": 3, + "normalized": "foo", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0002": { + "css": "--\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--", + "startIndex": 0, + "endIndex": 2, + "normalized": "--", + "value": "--" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0003": { + "css": "--0\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--0", + "startIndex": 0, + "endIndex": 3, + "normalized": "--0", + "value": "--0" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0004": { + "css": "-\\\n", + "tokens": [ + { + "type": "delim-token", + "raw": "-", + "startIndex": 0, + "endIndex": 1, + "normalized": "-", + "value": "-" + }, + { + "type": "delim-token", + "raw": "\\", + "startIndex": 1, + "endIndex": 2, + "normalized": "\\", + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0005": { + "css": "-\\ \n", + "tokens": [ + { + "type": "ident-token", + "raw": "-\\ ", + "startIndex": 0, + "endIndex": 3, + "normalized": "- ", + "value": "- " + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0006": { + "css": "--💅\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--💅", + "startIndex": 0, + "endIndex": 6, + "normalized": "--💅", + "value": "--💅" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0007": { + "css": "-§\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-§", + "startIndex": 0, + "endIndex": 3, + "normalized": "-§", + "value": "-§" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0008": { + "css": "-×\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-×", + "startIndex": 0, + "endIndex": 3, + "normalized": "-×", + "value": "-×" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident/0009": { + "css": "--a𐀀\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--a𐀀", + "startIndex": 0, + "endIndex": 7, + "normalized": "--a𐀀", + "value": "--a𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0001": { + "css": "url(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url(foo)", + "startIndex": 0, + "endIndex": 8, + "normalized": "url(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0002": { + "css": "\\75 Rl(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "\\75 Rl(foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "uRl(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0003": { + "css": "uR\\6c (foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "uR\\6c (foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "uRl(foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0004": { + "css": "url('foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 4, + "endIndex": 9, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 9, + "endIndex": 10, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 10, + "endIndex": 11, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0005": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 5, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 5, + "endIndex": 10, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 10, + "endIndex": 11, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0006": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 6, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 6, + "endIndex": 11, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 11, + "endIndex": 12, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 12, + "endIndex": 13, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0007": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "normalized": "url(", + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 7, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 7, + "endIndex": 12, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 12, + "endIndex": 13, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0008": { + "css": "not-url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "not-url(", + "startIndex": 0, + "endIndex": 8, + "normalized": "not-url(", + "value": "not-url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 8, + "endIndex": 11, + "normalized": " ", + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 11, + "endIndex": 16, + "normalized": "'foo'", + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 16, + "endIndex": 17, + "normalized": ")", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 17, + "endIndex": 18, + "normalized": "\n", + "value": null + } + ] + }, + "tests/ident-like/0009": { + "css": "url( foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url( foo)", + "startIndex": 0, + "endIndex": 11, + "normalized": "url( foo)", + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-curly-bracket/0001": { + "css": "{\n", + "tokens": [ + { + "type": "{-token", + "raw": "{", + "startIndex": 0, + "endIndex": 1, + "normalized": "{", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-parenthesis/0001": { + "css": "(\n", + "tokens": [ + { + "type": "(-token", + "raw": "(", + "startIndex": 0, + "endIndex": 1, + "normalized": "(", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/left-square-bracket/0001": { + "css": "[\n", + "tokens": [ + { + "type": "[-token", + "raw": "[", + "startIndex": 0, + "endIndex": 1, + "normalized": "[", + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/less-than/0001": { + "css": "<\n", + "tokens": [ + { + "type": "delim-token", + "raw": "<", + "startIndex": 0, + "endIndex": 1, + "normalized": "<", + "value": "<" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "normalized": "\n", + "value": null + } + ] + }, + "tests/less-than/0002": { + "css": " + * + * Legacy token from when CSS was embedded in HTML + * + * Modern CSS no longer needs these, but they're preserved for compatibility. + * In stylesheets, they're typically treated like whitespace. + * + * @see https://www.w3.org/TR/css-syntax-3/#typedef-CDC-token + */ + public const TOKEN_CDC = 'CDC-token'; + + /** + * CDO (Comment Delimiter Open) token: ) + * + * Comment Delimiter Close - legacy HTML comment syntax in CSS. + * + * @see https://www.w3.org/TR/css-syntax-3/#CDC-token-diagram + */ + if ( + $this->at + 2 < $this->length && + '-' === $this->css[ $this->at + 1 ] && + '>' === $this->css[ $this->at + 2 ] + ) { + // Consume them and return a . + $this->at += 3; + $this->token_type = self::TOKEN_CDC; + $this->token_length = 3; + return true; + } + + // Otherwise, if the input stream starts with an ident sequence, + // reconsume the current input code point, consume an ident-like + // token, and return it. + if ( $this->check_if_3_code_points_start_an_ident_sequence( $this->at ) ) { + return $this->consume_ident_like(); + } + + // Otherwise, return a with its value set to the current input code point. + ++$this->at; + $this->token_type = self::TOKEN_DELIM; + $this->token_length = 1; + return true; + } + + /* + * U+003C LESS-THAN SIGN (<) + * If followed by !--, this is a CDO token (