Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] AST should include reference to markup #611

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[WIP] AST should include reference to markup #611

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
961c187
Started work on ast to markup mapping
RakeshUP Jan 16, 2020
f2ce85f
Still work in progress, output extremely unstable
RakeshUP Jan 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/idyll-ast/src/converters/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ function inverseConvertHelper(arrayElement, id) {
elementJson.value = arrayElement;
} else if (['var', 'derived', 'data', 'meta'].indexOf(arrayElement[0]) > -1) {
elementJson.type = arrayElement[0];
elementJson.raw = arrayElement[3];
elementJson.properties = {};
arrayElement[1].forEach(property => {
elementJson.properties[property[0]] = {
Expand All @@ -113,6 +114,7 @@ function inverseConvertHelper(arrayElement, id) {
} else {
elementJson.type = 'component';
elementJson.name = arrayElement[0];
elementJson.raw = arrayElement[3];
if (arrayElement[1].length !== 0) {
elementJson.properties = {};
arrayElement[1].forEach(property => {
Expand Down
6 changes: 5 additions & 1 deletion packages/idyll-compiler/src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ const {
cleanResults,
makeFullWidth,
wrapText,
autoLinkify
autoLinkify,
linkMarkupToNode
} = require('./processors/post');
const { convertV1ToV2 } = require('idyll-ast').converters;
const matter = require('gray-matter');
Expand Down Expand Up @@ -44,7 +45,10 @@ module.exports = function(input, options, alias, callback) {
return new Promise((resolve, reject) => reject(err));
}

const referenceMarkup = linkMarkupToNode(lexResults.positions, content);

let astTransform = Processor(output, options)
.pipe(referenceMarkup)
.pipe(hoistVariables)
.pipe(flattenChildren)
.pipe(makeFullWidth)
Expand Down
64 changes: 32 additions & 32 deletions packages/idyll-compiler/src/lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const shouldBreak = text => {
let currentInput = null;

const lex = function(options, alias = {}) {
const positions = [];
let { row, column, outer, skipLists, inComponent, gotName } = Object.assign(
{},
{
Expand Down Expand Up @@ -66,13 +67,19 @@ const lex = function(options, alias = {}) {
)
].join('|');
};
var updatePosition = function(lexeme) {
var updatePosition = function(lexeme, shouldNotPush, isStart, isEnd) {
if (!shouldNotPush && !isEnd) {
positions.push([row, column]);
}
var lines = lexeme.split('\n');
row += lines.length - 1;
if (lines.length > 1) {
column = 0;
column = 1;
}
column += lines[lines.length - 1].length;
if (!shouldNotPush && !isStart) {
positions.push([row, column]);
}
};

// Rules at the front are pre-processed,
Expand Down Expand Up @@ -163,22 +170,18 @@ const lex = function(options, alias = {}) {
return ['INLINE_CODE'].concat(formatToken(text.trim()));
});

lexer.addRule(/[\s\n]*(#{1,6})\s*([^\n\[]+)[\n\s]*/gm, function(
lexeme,
hashes,
text
) {
lexer.addRule(/(#{1,6})\s*([^\n\[]+)/gm, function(lexeme, hashes, text) {
if (this.reject) return;
updatePosition(lexeme);
return ['BREAK', 'HEADER_' + hashes.length]
return ['HEADER_' + hashes.length]
.concat(recurse(text, { skipLists: true }))
.concat(['HEADER_END']);
});

lexer.addRule(/[\s\n]*>\s*([^\n\[]+)[\n\s]*/gm, function(lexeme, text) {
lexer.addRule(/>\s*([^\n\[]+)/gm, function(lexeme, text) {
if (this.reject) return;
updatePosition(lexeme);
return ['BREAK', 'QUOTE_START']
return ['QUOTE_START']
.concat(recurse(text, { skipLists: true }))
.concat(['QUOTE_END']);
});
Expand Down Expand Up @@ -309,28 +312,28 @@ const lex = function(options, alias = {}) {
});

lexer.addRule(/(\n\s*\/\/[^\n]*|\/\/\s+[^\n]*)/, function(lexeme) {
updatePosition(lexeme);
updatePosition(lexeme, true);
if (lexeme.startsWith('\n')) {
return ['BREAK'];
}
});

lexer.addRule(/\/(\n?[^`\*\[\/\n\]!\\\d_])*/gm, function(lexeme) {
lexer.addRule(/\/([^`\*\[\/\n\]!\\\d_])*/gm, function(lexeme) {
this.reject = inComponent || lexeme.trim() === '';
if (this.reject) return;
updatePosition(lexeme);
return ['WORDS'].concat(formatToken(lexeme));
});

lexer.addRule(/(\n?[^`\*\[\/\n\]!\\\d_])+/, function(lexeme) {
lexer.addRule(/([^`\*\[\/\n\]!\\\d_])+/, function(lexeme) {
this.reject = inComponent || lexeme.trim() === '';
if (this.reject) return;
updatePosition(lexeme);
return ['WORDS'].concat(formatToken(lexeme));
});
// Match on separately so we can greedily match the
// other tags.
lexer.addRule(/[!\d\*_`] */, function(lexeme) {
lexer.addRule(/[!\d\*_`]\s*/, function(lexeme) {
this.reject = inComponent || lexeme.trim() === '';
if (this.reject) return;
updatePosition(lexeme);
Expand All @@ -343,29 +346,29 @@ const lex = function(options, alias = {}) {
return ['WORDS'].concat(formatToken(lexeme));
});

lexer.addRule(/\s*\n{2,}\s*/, function(lexeme) {
lexer.addRule(/\s*\n+?\s*/, function(lexeme) {
this.reject = inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['BREAK'];
});

lexer.addRule(/[ \t\n]+/, function(lexeme) {
updatePosition(lexeme);
updatePosition(lexeme, true);
});

lexer.addRule(/\[/, function(lexeme) {
inComponent = true;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, false, true);
return ['OPEN_BRACKET'];
});

lexer.addRule(/\]([ ]*)/, function(lexeme, trailingSpace) {
inComponent = false;
gotName = false;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, false, false, true);
var ret = ['CLOSE_BRACKET'];
if (trailingSpace) {
ret = ret.concat(['WORDS']).concat(formatToken(trailingSpace));
Expand All @@ -376,14 +379,14 @@ const lex = function(options, alias = {}) {
lexer.addRule(/\//, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['FORWARD_SLASH'];
});

lexer.addRule(/true|false/, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['BOOLEAN'].concat(formatToken(lexeme));
});

Expand All @@ -393,21 +396,21 @@ const lex = function(options, alias = {}) {
this.reject = !inComponent || gotName;
if (this.reject) return;
gotName = true;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['COMPONENT_NAME'].concat(formatToken(lexeme));
}
);
lexer.addRule(/[^+\-0-9:\s\/\]"'`\.][^:\s\/\]"'`\.]*/, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['COMPONENT_WORD'].concat(formatToken(lexeme));
});

lexer.addRule(/`[^`]*`/, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['EXPRESSION'].concat(formatToken(lexeme));
});

Expand All @@ -416,48 +419,45 @@ const lex = function(options, alias = {}) {
(lexeme.match(new RegExp(/\./, 'g')) || []).length >= 2;
this.reject = !inComponent || multiplePeriods;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['NUMBER'].concat(formatToken(lexeme));
});

lexer.addRule(/"[^"]*"/, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['STRING'].concat(formatToken(lexeme));
});
lexer.addRule(/'([^']*)'/, function(lexeme, str) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['STRING'].concat(formatToken('"' + str + '"'));
});

lexer.addRule(/:/, function(lexeme) {
this.reject = !inComponent;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['PARAM_SEPARATOR'];
});

lexer.addRule(/\s*$/, function(lexeme) {
this.reject = !outer;
if (this.reject) return;
updatePosition(lexeme);
updatePosition(lexeme, true);
return ['EOF'];
});

return function(str) {
currentInput = str;
var vals = [];
var output = [];
var positions = [];

lexer.input = str.trim();
var token = lexer.lex();
while (token) {
output.push(token);
positions.push([row, column]);
token = lexer.lex();
}
return {
Expand Down
33 changes: 32 additions & 1 deletion packages/idyll-compiler/src/processors/post.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ const cleanResults = (ast, options) => {
if (rawNodes.indexOf(name) > -1) {
return node;
}
if (node[3]) {
return [node[0], node[1], cleanResults(node[2], options), node[3]];
}
return [node[0], node[1], cleanResults(node[2], options)];
});
};
Expand Down Expand Up @@ -256,6 +259,33 @@ function getHyperLinksFromText(textNode) {
return textNode.match(regexURL);
}

const linkMarkupToNode = (lexPositions, content) => {
return ast => {
const positions = [...lexPositions];
const lines = content.split('\n');
return ast.map(node => {
let [startRow, startColumn] = positions.shift();
let [endRow, endColumn] = positions.shift();
let markup;
if (startRow !== endRow) {
markup = lines[startRow - 1].substring(startColumn - 1);
} else {
markup = lines[startRow - 1].substring(startColumn - 1, endColumn - 1);
}
while (startRow < endRow) {
startRow++;
if (startRow === endRow) {
markup += lines[startRow - 1].substring(0, endColumn - 1);
} else {
markup += lines[startRow - 1];
}
}
node[3] = markup;
return node;
});
};
};

module.exports = {
cleanResults,
flattenChildren,
Expand All @@ -266,5 +296,6 @@ module.exports = {
autoLinkifyHelper,
hyperLinkifiedVersion,
seperateTextAndHyperLink,
getHyperLinksFromText
getHyperLinksFromText,
linkMarkupToNode
};