Skip to content

Commit 3e1bdf5

Browse files
ish1416antfu
ish1416
and
antfu
authored
fix(transformers): handle multi-token comments in rose-pine theme (#1118)
Co-authored-by: Anthony Fu <github@antfu.me>
1 parent e4dec23 commit 3e1bdf5

File tree

4 files changed

+347
-1
lines changed

4 files changed

+347
-1
lines changed

‎packages/transformers/src/shared/notation-transformer.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,35 @@ export function createCommentNotationTransformer(
8282
comment.line.children.splice(comment.line.children.indexOf(comment.token) - 1, 3)
8383
}
8484
else if (isEmpty) {
85+
// Handle multi-token comments
86+
if (comment.additionalTokens) {
87+
// Remove additional tokens first (in reverse order to maintain indices)
88+
for (let j = comment.additionalTokens.length - 1; j >= 0; j--) {
89+
const additionalToken = comment.additionalTokens[j]
90+
const tokenIndex = comment.line.children.indexOf(additionalToken)
91+
if (tokenIndex !== -1) {
92+
comment.line.children.splice(tokenIndex, 1)
93+
}
94+
}
95+
}
96+
// Remove the main token
8597
comment.line.children.splice(comment.line.children.indexOf(comment.token), 1)
8698
}
8799
else {
88100
const head = comment.token.children[0]
89101

90102
if (head.type === 'text') {
91103
head.value = comment.info.join('')
104+
105+
// For multi-token comments, clear the additional tokens
106+
if (comment.additionalTokens) {
107+
for (const additionalToken of comment.additionalTokens) {
108+
const additionalHead = additionalToken.children[0]
109+
if (additionalHead?.type === 'text') {
110+
additionalHead.value = ''
111+
}
112+
}
113+
}
92114
}
93115
}
94116
}

‎packages/transformers/src/shared/parse-comments.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export type ParsedComments = {
77
info: [prefix: string, content: string, suffix?: string]
88
isLineCommentOnly: boolean
99
isJsxStyle: boolean
10+
// For multi-token comments, store the additional tokens that need to be processed
11+
additionalTokens?: Element[]
1012
}[]
1113

1214
/**
@@ -96,7 +98,38 @@ export function parseComments(
9698
continue
9799

98100
const isLast = i === elements.length - 1
99-
const match = matchToken(head.value, isLast)
101+
let match = matchToken(head.value, isLast)
102+
let additionalTokens: Element[] | undefined
103+
104+
// Handle multi-token comments (e.g., rose-pine theme splits "//" and " [!code --]")
105+
// Check if current token might be the second part of a split comment
106+
if (!match && i > 0 && head.value.trim().startsWith('[!code')) {
107+
// Look back to see if the previous token contains the comment prefix
108+
const prevToken = elements[i - 1]
109+
if (prevToken?.type === 'element') {
110+
const prevHead = prevToken.children.at(0)
111+
if (prevHead?.type === 'text' && prevHead.value.includes('//')) {
112+
const combinedValue = prevHead.value + head.value
113+
const combinedMatch = matchToken(combinedValue, isLast)
114+
if (combinedMatch) {
115+
match = combinedMatch
116+
// We need to use the previous token as the main token and this as additional
117+
// But we need to adjust our approach since we're processing the second token
118+
// Let's create a special case for this
119+
out.push({
120+
info: combinedMatch,
121+
line,
122+
token: prevToken, // Use the previous token as the main token
123+
isLineCommentOnly: elements.length === 2 && prevToken.children.length === 1 && token.children.length === 1,
124+
isJsxStyle: false,
125+
additionalTokens: [token], // Current token is the additional one
126+
})
127+
continue // Skip normal processing for this token
128+
}
129+
}
130+
}
131+
}
132+
100133
if (!match)
101134
continue
102135

@@ -108,6 +141,7 @@ export function parseComments(
108141
token,
109142
isLineCommentOnly: elements.length === 3 && token.children.length === 1,
110143
isJsxStyle,
144+
additionalTokens,
111145
})
112146
}
113147
else {
@@ -117,6 +151,7 @@ export function parseComments(
117151
token,
118152
isLineCommentOnly: elements.length === 1 && token.children.length === 1,
119153
isJsxStyle: false,
154+
additionalTokens,
120155
})
121156
}
122157
}

‎packages/transformers/test/notation-diff-rose-pine.test.ts‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
import { createHighlighter } from 'shiki'
2+
import { describe, expect, it } from 'vitest'
3+
import { transformerNotationDiff, transformerNotationFocus, transformerNotationHighlight } from '../src'
4+
5+
describe('multi-token comment support', () => {
6+
it('transformerNotationDiff works with rose-pine theme (multi-token comments)', async () => {
7+
const highlighter = await createHighlighter({
8+
themes: ['rose-pine'],
9+
langs: ['javascript'],
10+
})
11+
12+
const code = 'const a = 1 // [!code --]'
13+
14+
const html = highlighter.codeToHtml(code, {
15+
lang: 'javascript',
16+
theme: 'rose-pine',
17+
transformers: [transformerNotationDiff()],
18+
})
19+
20+
// Should have the diff classes applied
21+
expect(html).toContain('has-diff')
22+
expect(html).toContain('diff remove')
23+
// Should not contain the comment notation in the output
24+
expect(html).not.toContain('[!code --]')
25+
})
26+
27+
it('transformerNotationDiff still works with dracula theme (single-token comments)', async () => {
28+
const highlighter = await createHighlighter({
29+
themes: ['dracula'],
30+
langs: ['javascript'],
31+
})
32+
33+
const code = 'const a = 1 // [!code --]'
34+
35+
const html = highlighter.codeToHtml(code, {
36+
lang: 'javascript',
37+
theme: 'dracula',
38+
transformers: [transformerNotationDiff()],
39+
})
40+
41+
// Should have the diff classes applied
42+
expect(html).toContain('has-diff')
43+
expect(html).toContain('diff remove')
44+
// Should not contain the comment notation in the output
45+
expect(html).not.toContain('[!code --]')
46+
})
47+
48+
it('transformerNotationDiff works with rose-pine theme for add notation', async () => {
49+
const highlighter = await createHighlighter({
50+
themes: ['rose-pine'],
51+
langs: ['javascript'],
52+
})
53+
54+
const code = 'const b = 2 // [!code ++]'
55+
56+
const html = highlighter.codeToHtml(code, {
57+
lang: 'javascript',
58+
theme: 'rose-pine',
59+
transformers: [transformerNotationDiff()],
60+
})
61+
62+
expect(html).toContain('has-diff')
63+
expect(html).toContain('diff add')
64+
expect(html).not.toContain('[!code ++]')
65+
})
66+
67+
it('transformerNotationHighlight works with rose-pine theme', async () => {
68+
const highlighter = await createHighlighter({
69+
themes: ['rose-pine'],
70+
langs: ['javascript'],
71+
})
72+
73+
const code = 'const c = 3 // [!code highlight]'
74+
75+
const html = highlighter.codeToHtml(code, {
76+
lang: 'javascript',
77+
theme: 'rose-pine',
78+
transformers: [transformerNotationHighlight()],
79+
})
80+
81+
expect(html).toContain('highlighted')
82+
expect(html).not.toContain('[!code highlight]')
83+
})
84+
85+
it('transformerNotationFocus works with rose-pine theme', async () => {
86+
const highlighter = await createHighlighter({
87+
themes: ['rose-pine'],
88+
langs: ['javascript'],
89+
})
90+
91+
const code = 'const d = 4 // [!code focus]'
92+
93+
const html = highlighter.codeToHtml(code, {
94+
lang: 'javascript',
95+
theme: 'rose-pine',
96+
transformers: [transformerNotationFocus()],
97+
})
98+
99+
expect(html).toContain('focused')
100+
expect(html).not.toContain('[!code focus]')
101+
})
102+
103+
it('handles multi-line code with mixed single and multi-token comments', async () => {
104+
const highlighter = await createHighlighter({
105+
themes: ['rose-pine'],
106+
langs: ['javascript'],
107+
})
108+
109+
const code = `const a = 1 // [!code --]
110+
const b = 2 // [!code ++]
111+
const c = 3 // [!code highlight]`
112+
113+
const html = highlighter.codeToHtml(code, {
114+
lang: 'javascript',
115+
theme: 'rose-pine',
116+
transformers: [transformerNotationDiff(), transformerNotationHighlight()],
117+
})
118+
119+
expect(html).toContain('has-diff')
120+
expect(html).toContain('diff remove')
121+
expect(html).toContain('diff add')
122+
expect(html).toContain('highlighted')
123+
expect(html).not.toContain('[!code')
124+
})
125+
126+
it('handles edge case where comment does not match pattern', async () => {
127+
const highlighter = await createHighlighter({
128+
themes: ['rose-pine'],
129+
langs: ['javascript'],
130+
})
131+
132+
const code = 'const e = 5 // [!invalid notation]'
133+
134+
const html = highlighter.codeToHtml(code, {
135+
lang: 'javascript',
136+
theme: 'rose-pine',
137+
transformers: [transformerNotationDiff()],
138+
})
139+
140+
// Should not have diff classes since notation is invalid
141+
expect(html).not.toContain('has-diff')
142+
expect(html).not.toContain('diff remove')
143+
// Should still contain the invalid notation
144+
expect(html).toContain('[!invalid notation]')
145+
})
146+
147+
it('handles single token without multi-token fallback', async () => {
148+
const highlighter = await createHighlighter({
149+
themes: ['rose-pine'],
150+
langs: ['javascript'],
151+
})
152+
153+
const code = 'const f = 6 // regular comment'
154+
155+
const html = highlighter.codeToHtml(code, {
156+
lang: 'javascript',
157+
theme: 'rose-pine',
158+
transformers: [transformerNotationDiff()],
159+
})
160+
161+
expect(html).not.toContain('has-diff')
162+
expect(html).toContain('regular comment')
163+
})
164+
165+
it('handles JSX parsing without notation', async () => {
166+
const highlighter = await createHighlighter({
167+
themes: ['rose-pine'],
168+
langs: ['jsx'],
169+
})
170+
171+
const code = 'const g = 7 {/* regular comment */}'
172+
173+
const html = highlighter.codeToHtml(code, {
174+
lang: 'jsx',
175+
theme: 'rose-pine',
176+
transformers: [transformerNotationDiff()],
177+
})
178+
179+
expect(html).not.toContain('has-diff')
180+
expect(html).toContain('regular comment')
181+
})
182+
183+
it('handles v1 match algorithm', async () => {
184+
const highlighter = await createHighlighter({
185+
themes: ['rose-pine'],
186+
langs: ['javascript'],
187+
})
188+
189+
const code = 'const h = 8 // [!code ++]'
190+
191+
const html = highlighter.codeToHtml(code, {
192+
lang: 'javascript',
193+
theme: 'rose-pine',
194+
transformers: [transformerNotationDiff({ matchAlgorithm: 'v1' })],
195+
})
196+
197+
expect(html).toContain('has-diff')
198+
expect(html).toContain('diff add')
199+
expect(html).not.toContain('[!code ++]')
200+
})
201+
202+
it('handles actual multi-token comment scenario from rose-pine theme', async () => {
203+
const highlighter = await createHighlighter({
204+
themes: ['rose-pine'],
205+
langs: ['javascript'],
206+
})
207+
208+
// This specifically tests the case where rose-pine theme splits the comment
209+
// into separate tokens: "//" and " [!code --]"
210+
const code = 'const i = 9 // [!code --]'
211+
212+
const html = highlighter.codeToHtml(code, {
213+
lang: 'javascript',
214+
theme: 'rose-pine',
215+
transformers: [transformerNotationDiff()],
216+
})
217+
218+
// Verify the multi-token handling works
219+
expect(html).toContain('has-diff')
220+
expect(html).toContain('diff remove')
221+
expect(html).not.toContain('[!code --]')
222+
})
223+
})

‎packages/transformers/test/parse-comments-multi-token.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import type { Element } from 'hast'
2+
import { describe, expect, it } from 'vitest'
3+
import { parseComments } from '../src/shared/parse-comments'
4+
5+
describe('parseComments multi-token handling', () => {
6+
it('handles multi-token comments where comment is split across tokens', () => {
7+
// Simulate how rose-pine theme splits "// [!code --]" into separate tokens
8+
const lines: Element[] = [{
9+
type: 'element',
10+
tagName: 'span',
11+
properties: { class: 'line' },
12+
children: [
13+
{
14+
type: 'element',
15+
tagName: 'span',
16+
properties: {},
17+
children: [{ type: 'text', value: 'const a = 1 ' }],
18+
},
19+
{
20+
type: 'element',
21+
tagName: 'span',
22+
properties: {},
23+
children: [{ type: 'text', value: '//' }],
24+
},
25+
{
26+
type: 'element',
27+
tagName: 'span',
28+
properties: {},
29+
children: [{ type: 'text', value: ' [!code --]' }],
30+
},
31+
],
32+
}]
33+
34+
const result = parseComments(lines, false, 'v3')
35+
36+
expect(result).toHaveLength(1)
37+
expect(result[0].info[1]).toBe(' [!code --]')
38+
expect(result[0].additionalTokens).toHaveLength(1)
39+
})
40+
41+
it('handles case where previous token does not contain comment prefix', () => {
42+
const lines: Element[] = [{
43+
type: 'element',
44+
tagName: 'span',
45+
properties: { class: 'line' },
46+
children: [
47+
{
48+
type: 'element',
49+
tagName: 'span',
50+
properties: {},
51+
children: [{ type: 'text', value: 'const a = 1' }],
52+
},
53+
{
54+
type: 'element',
55+
tagName: 'span',
56+
properties: {},
57+
children: [{ type: 'text', value: ' [!code --]' }],
58+
},
59+
],
60+
}]
61+
62+
const result = parseComments(lines, false, 'v3')
63+
64+
expect(result).toHaveLength(0)
65+
})
66+
})

0 commit comments

Comments
 (0)