fix: ident parse (#292)

wixplosives · Jun 8, 2023 · c2867d4 · c2867d4
1 parent 0f6aaea
commit c2867d4
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 1 deletion.
diff --git a/packages/core/src/core.ts b/packages/core/src/core.ts
@@ -15,12 +15,14 @@ export interface TokyOptions<T extends Token<unknown>> {
     ): boolean;
     getUnclosedComment(inComment: string): string;
     createToken(value: string, type: T['type'], start: number, end: number): T;
+    shouldClose?(ch: string, previousChar: string): boolean;
     offset?: number;
 }
 
 export function tokenize<T extends Token<unknown>>(
     source: string,
     {
+        shouldClose,
         isDelimiter,
         isStringDelimiter,
         isWhitespace,
@@ -70,6 +72,10 @@ export function tokenize<T extends Token<unknown>>(
         } else if (!isWhitespace(ch) && isWhitespace(previousChar)) {
             pushBuffer();
             buffer += ch;
+        } else if(shouldClose?.(ch, previousChar)) {
+            pushBuffer();
+            buffer += ch;
+            pushBuffer(ch);
         } else {
             buffer += ch;
         }

diff --git a/packages/css-value-parser/src/tokenizer.ts b/packages/css-value-parser/src/tokenizer.ts
@@ -10,7 +10,7 @@ import {
 } from '@tokey/core';
 import type { Token, Descriptors } from '@tokey/core';
 
-type Delimiters = '(' | ')' | ',' | '/' | '+' | '-' | '*' | '#' | '.';
+type Delimiters = '(' | ')' | ',' | '/' | '+' | '-' | '*' | '#' | '.' | '%';
 
 export type CSSValueToken = Token<Descriptors | Delimiters>;
 
@@ -29,6 +29,20 @@ export function tokenizeValue(source: string, options: { offset?: number } = {})
             : getMultilineCommentStartType,
         isCommentEnd,
         getUnclosedComment,
+        shouldClose(ch, previousChar) {
+            if (isWhitespace(ch) && isWhitespace(previousChar)) {
+                return false;
+            }
+            if (previousChar === '\\') {
+                return false;
+            }
+            const isAllowedChars = /[-_a-zA-Z0-9]/.test(ch);
+            if (isAllowedChars) {
+                return false;
+            }
+            // match css identifier char don't allow non-ascii chars
+            return ch.charCodeAt(0) <= 127;
+        },
         offset: options.offset,
     });
 }
@@ -55,4 +69,5 @@ const isDelimiter = (char: string, previousChar: string) =>
         char === '-' ||
         char === '*' ||
         char === '#' ||
+        char === '%' ||
         char === '.');
diff --git a/packages/css-value-parser/src/value-parser.ts b/packages/css-value-parser/src/value-parser.ts
@@ -271,6 +271,7 @@ function parseNumber(
             nextToken.type === `-` ||
             nextToken.type === `+` ||
             nextToken.type === `.` ||
+            nextToken.type === `%` ||
             nextToken.type === `text`
         ) {
             const nextValue = numberValue + nextToken.value;

diff --git a/packages/css-value-parser/test/value-parser.spec.ts b/packages/css-value-parser/test/value-parser.spec.ts
@@ -244,6 +244,25 @@ describe(`value-parser`, () => {
                         literal({ value: `]`, start: 4, end: 5 }),
                     ],
                 },
+                {
+                    type: `<custom-ident>`,
+                    desc: `split custom ident`,
+                    source: `abc==efg`,
+                    expected: [
+                        customIdent({
+                            value: `abc`,
+                            start: 0,
+                            end: 3,
+                        }),
+                        literal({ value: `=`, start: 3, end: 4 }),
+                        literal({ value: `=`, start: 4, end: 5 }),
+                        customIdent({
+                            value: `efg`,
+                            start: 5,
+                            end: 8,
+                        }),
+                    ],
+                },
             ].forEach(createTest);
         });
         describe(`dashed-ident`, () => {