Skip to content

feat: add greedy newline tokenizer #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bcoe merged 7 commits into from
Jan 7, 2021
Merged

feat: add greedy newline tokenizer #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6b4864f
feat: add greedy newline tokenizer
byCedric Dec 31, 2020
09a500c
feat: implement newline tokenizer in message
byCedric Dec 31, 2020
31531e7
feat: implement newline tokenizer in other nodes
byCedric Dec 31, 2020
239dffc
fix: do not consume newline on body end
byCedric Dec 31, 2020
9dcc412
chore: update snapshots to include newline
byCedric Dec 31, 2020
65f13e1
chore: remove debugging statement
byCedric Dec 31, 2020
8047450
refactor: move newline outside footer as sibling (#33)
byCedric Jan 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ in sync with the written specification on conventionalcommits.org.
/* Any non-newline whitespace: */
<whitespace> ::= <ZWNBSP> | <TAB> | <VT> | <FF> | <SP> | <NBSP> | <USP>

<message> ::= <summary>, <newline>+, <body>, <newline>*, <footer>+
| <summary>, <newline>*, <footer>+
<message> ::= <summary>, <newline>+, <body>, (<newline>+, <footer>)*
| <summary>, (<newline>+, <footer>)*
| <summary>, <newline>*

/* "!" should be added to the AST as a <breaking-change> node with the value "!" */
Expand All @@ -78,9 +78,9 @@ in sync with the written specification on conventionalcommits.org.
* <text> tokens of <body-text> should be appended as children to <body> */
<body-text> ::= [<breaking-change>, ":", <whitespace>*], text
/* Note: <pre-footer> is used during parsing, but not returned in the AST. */
<pre-footer> ::= <newline>*, <footer>+
<pre-footer> ::= <newline>+, <footer>

<footer> ::= <token>, <separator>, <whitespace>*, <value>, [<newline>]
<footer> ::= <token>, <separator>, <whitespace>*, <value>
/* "!" should be added to the AST as a <breaking-change> node with the value "!" */
<token> ::= <breaking-change>
| <type>, "(" <scope> ")", ["!"]
Expand Down
119 changes: 81 additions & 38 deletions lib/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ const Scanner = require('./scanner')
const { isWhitespace, isNewline, isParens } = require('./type-checks')

/*
* <message> ::= <summary>, <newline>*, <body>, <newline>*, <footer>+
* | <summary>, <newline>*, <footer>+
* <message> ::= <summary>, <newline>+, <body>, (<newline>+, <footer>)*
* | <summary>, (<newline>+, <footer>)*
* | <summary>, <newline>*
*
*/
function message (commitText) {
const scanner = new Scanner(commitText.trim())
const node = scanner.enter('message', [])

// <summary>
// <summary> ...
const s = summary(scanner)
if (s instanceof Error) {
throw s
Expand All @@ -22,27 +22,49 @@ function message (commitText) {
return scanner.exit(node)
}

// <summary> <newline>* <body>
if (isNewline(scanner.peek())) {
// TODO(@byCedric): include <newline>* in AST.
while (isNewline(scanner.peek())) {
scanner.next()
}
let nl
let b
// ... <newline>* <body> ...
nl = newline(scanner)
if (nl instanceof Error) {
throw nl
} else {
throw scanner.abort(node)
node.children.push(nl)
b = body(scanner)
if (b instanceof Error) {
b = null
} else {
node.children.push(b)
}
}
const b = body(scanner)
if (!(b instanceof Error)) {
node.children.push(b)
if (scanner.eof()) {
return scanner.exit(node)
}
// TODO(@byCedric): include <newline>* in AST.
while (isNewline(scanner.peek())) {
scanner.next()

// ... <newline>* <footer>+
if (b) {
nl = newline(scanner)
if (nl instanceof Error) {
throw nl
} else {
node.children.push(nl)
}
}
while (!scanner.eof()) {
const f = footer(scanner)
if (!(f instanceof Error)) node.children.push(f)
if (f instanceof Error) {
break
} else {
node.children.push(f)
}
nl = newline(scanner)
if (nl instanceof Error) {
break
} else {
node.children.push(nl)
}
}

return scanner.exit(node)
}

Expand Down Expand Up @@ -189,10 +211,13 @@ function body (scanner) {
const t = text(scanner)
node.children.push(t)
// <newline>, <body>*
if (isNewline(scanner.peek())) {
scanner.next()
const nl = newline(scanner)
if (!(nl instanceof Error)) {
const b = body(scanner)
if (!(b instanceof Error)) {
if (b instanceof Error) {
scanner.abort(nl)
} else {
node.children.push(nl)
Array.prototype.push.apply(node.children, b.children)
}
}
Expand All @@ -204,19 +229,17 @@ function body (scanner) {
*/
function preFooter (scanner) {
const node = scanner.enter('pre-footer', [])
while (isNewline(scanner.peek())) {
scanner.next()
}
let f
while (!scanner.eof()) {
newline(scanner)
f = footer(scanner)
if (f instanceof Error) return scanner.abort(node)
}
return scanner.exit(node)
}

/*
* <footer> ::= <token> <separator> <whitespace>* <value> <newline>?
* <footer> ::= <token> <separator> <whitespace>* <value>
*/
function footer (scanner) {
const node = scanner.enter('footer', [])
Expand All @@ -231,6 +254,7 @@ function footer (scanner) {
// <separator>
const s = separator(scanner)
if (s instanceof Error) {
scanner.abort(node)
return s
} else {
node.children.push(s)
Expand All @@ -242,16 +266,15 @@ function footer (scanner) {
node.children.push(ws)
}

// <value> <newline>?
// <value>
const v = value(scanner)
if (v instanceof Error) {
scanner.abort(node)
return v
} else {
node.children.push(v)
}
if (isNewline(scanner.peek())) {
scanner.next()
}

return scanner.exit(node)
}

Expand Down Expand Up @@ -330,18 +353,24 @@ function value (scanner) {
*/
function continuation (scanner) {
const node = scanner.enter('continuation', [])
if (isNewline(scanner.peek())) {
scanner.next()
const ws = whitespace(scanner)
if (ws instanceof Error) {
return ws
} else {
node.children.push(ws)
node.children.push(text(scanner))
}
// <newline>
const nl = newline(scanner)
if (nl instanceof Error) {
return nl
} else {
return scanner.abort(node)
node.children.push(nl)
}

// <whitespace> <text>
const ws = whitespace(scanner)
if (ws instanceof Error) {
scanner.abort(node)
return ws
} else {
node.children.push(ws)
node.children.push(text(scanner))
}

return scanner.exit(node)
}

Expand Down Expand Up @@ -385,4 +414,18 @@ function whitespace (scanner) {
return scanner.exit(node)
}

/*
* <newline>+ ::= [<CR>], <LF>
*/
function newline (scanner) {
const node = scanner.enter('newline', '')
while (isNewline(scanner.peek())) {
node.value += scanner.next()
}
if (node.value === '') {
return scanner.abort(node, ['<CR><LF>', '<LF>'])
}
return scanner.exit(node)
}

module.exports = message
Loading