Skip to content

perf: avoid repeated raw string copies in nested lists #4082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
unshorn wants to merge 1 commit into
base: master
Choose a base branch
from
+12 −8
Open

perf: avoid repeated raw string copies in nested lists #4082

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions src/Tokenizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,10 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
list(src: string): Tokens.List | undefined {
let cap = this.rules.block.list.exec(src);
if (cap) {
// src is only consumed from the front, so every raw below is a slice of
// listSrc rather than a string built up line by line
const listSrc = src;
const taken = () => listSrc.length - src.length;
let bull = cap[1].trim();
const isordered = bull.length > 1;

Expand All @@ -282,7 +286,6 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
// Check if current bullet point can start a new List Item
while (src) {
let endEarly = false;
let raw = '';
let itemContents = '';
if (!(cap = itemRegex.exec(src))) {
break;
Expand All @@ -292,8 +295,8 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
break;
}

raw = cap[0];
src = src.substring(raw.length);
const itemStart = taken();
src = src.substring(cap[0].length);

let line = expandTabs(cap[2].split('\n', 1)[0], cap[1].length);
let nextLine = src.split('\n', 1)[0];
Expand All @@ -313,7 +316,6 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
}

if (blankLine && this.rules.other.blankLine.test(nextLine)) { // Items begin with at most one blank line
raw += nextLine + '\n';
src = src.substring(nextLine.length + 1);
endEarly = true;
}
Expand Down Expand Up @@ -397,12 +399,16 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {

blankLine = !nextLine.trim();

raw += rawLine + '\n';
src = src.substring(rawLine.length + 1);
line = nextLineWithoutTabs.slice(indent);
}
}

// The loops above appended a '\n' the source may not have had. It could
// only land on the item that exhausts src, whose raw is trimmed below,
// and after which nothing reads endsWithBlankLine
const raw = listSrc.slice(itemStart, taken());

if (!list.loose) {
// If the previous item ended with a blank line, the list is loose
if (endsWithBlankLine) {
Expand All @@ -420,8 +426,6 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
text: itemContents,
tokens: [],
});

list.raw += raw;
}

// Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic
Expand All @@ -433,7 +437,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
// not a list since there were no items
return;
}
list.raw = list.raw.trimEnd();
list.raw = listSrc.slice(0, taken()).trimEnd();

// Item child tokens handled here at end because we needed to have the final item to trim it first
// First pass: tokenize items and finalize list.loose from spacers before placing checkboxes
Expand Down