Skip to content

Commit

Permalink
Syntax Color and Debug JavaScript/Python in Enso Source (#9440)
Browse files Browse the repository at this point in the history
  • Loading branch information
JaroslavTulach committed Mar 15, 2024
1 parent 4faae88 commit 8d9c25c
Show file tree
Hide file tree
Showing 6 changed files with 163 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,40 @@ static ForeignEvalNode parse(EpbLanguage epb, Source langAndCode, List<String> a

private String truffleId(Source langAndCode) {
var seq = langAndCode.getCharacters();
return seq.subSequence(0, splitAt(seq)).toString().toLowerCase();
var langAndLine = seq.subSequence(0, splitAt(seq, '#'));
var lang = langAndLine.subSequence(0, splitAt(langAndLine, ':'));
return lang.toString().toLowerCase();
}

private int splitAt(CharSequence seq) {
private int startLine(Source langAndCode) {
var seq = langAndCode.getCharacters();
var langAndLine = seq.subSequence(0, splitAt(seq, '#')).toString();
var at = splitAt(langAndLine, ':') + 1;
var line = langAndLine.substring(at);
return Integer.parseInt(line);
}

private String foreignSource(Source langAndCode) {
var seq = langAndCode.getCharacters();
var code = new StringBuilder();
var line = startLine(langAndCode);
while (line-- > 0) {
code.append("\n");
}
var realCode = seq.toString().substring(splitAt(seq, '#') + 1);
code.append(realCode);
return code.toString();
}

private int splitAt(CharSequence seq, char ch) {
var at = 0;
while (at < seq.length()) {
if (seq.charAt(at) == '#') {
if (seq.charAt(at) == ch) {
return at;
}
at++;
}
throw new ForeignParsingException("No # found", this);
}

private String foreignSource(Source langAndCode) {
var seq = langAndCode.getCharacters();
return seq.toString().substring(splitAt(seq) + 1);
throw new ForeignParsingException("No " + ch + " found", this);
}

@Override
Expand Down Expand Up @@ -91,18 +108,23 @@ private ForeignFunctionCallNode parseJs() {
var inner = context.getInnerContext();
var code = foreignSource(langAndCode);
var args = Arrays.stream(argNames).skip(1).collect(Collectors.joining(","));
var wrappedSrc = "var poly_enso_eval=function(" + args + "){\n" + code + "\n};poly_enso_eval";
Source source = Source.newBuilder("js", wrappedSrc, "").build();
var wrappedSrc = "var poly_enso_eval=function(" + args + "){" + code + "\n};poly_enso_eval";
Source source = newSource("js", wrappedSrc);
var fn = inner.evalPublic(this, source);
return JsForeignNode.build(fn);
}

private ForeignFunctionCallNode parseGeneric(
String language, Function<CallTarget, ForeignFunctionCallNode> nodeFactory) {
var ctx = EpbContext.get(this);
Source source =
Source.newBuilder(language, foreignSource(langAndCode), langAndCode.getName()).build();
Source source = newSource(language, foreignSource(langAndCode));
CallTarget ct = ctx.getEnv().parsePublic(source, argNames);
return nodeFactory.apply(ct);
}

private Source newSource(String language, String code) {
return Source.newBuilder(language, code, langAndCode.getName())
.uri(langAndCode.getURI())
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1916,6 +1916,7 @@ class IrToTruffle(
case Foreign.Definition(lang, code, _, _, _) =>
buildForeignBody(
lang,
body.location,
code,
arguments.map(_.name.name),
argSlotIdxs
Expand Down Expand Up @@ -1977,12 +1978,18 @@ class IrToTruffle(

private def buildForeignBody(
language: String,
location: Option[IdentifiedLocation],
code: String,
argumentNames: List[String],
argumentSlotIdxs: List[Int]
): RuntimeExpression = {
val src =
Source.newBuilder("epb", language + "#" + code, scopeName).build()
val line = location
.map(l => source.createSection(l.start, l.length).getStartLine())
.getOrElse(0)
val name = scopeName.replace('.', '_') + "." + language
val b = Source.newBuilder("epb", language + ":" + line + "#" + code, name)
b.uri(source.getURI())
val src = b.build()
val foreignCt = context.parseInternal(src, argumentNames: _*)
val argumentReaders = argumentSlotIdxs
.map(slotIdx =>
Expand Down
Empty file.
7 changes: 7 additions & 0 deletions tools/enso4igv/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,13 @@
<scope>test</scope>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.netbeans.modules</groupId>
<artifactId>org-netbeans-modules-lexer-nbbridge</artifactId>
<version>${netbeans.version}</version>
<scope>test</scope>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.netbeans.api</groupId>
<artifactId>org-netbeans-modules-extexecution-base</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,30 @@
"matches": "\\\""
}]
},
{
"name": "entity.name.function",
"contentName": "foreign.js",
"begin": "^(\\s*)(foreign)\\s+js\\s*([^\\s]+).*=(.*$)",
"end": "^(?!\\1\\s+)(?!\\s*$)",
"beginCaptures": {
"2" : { "name": "keyword.other" },
"3" : { "name": "entity.name.function"},
"4" : { "name": "header"}
},
"patterns": [ { "include": "source.js" }]
},
{
"name": "entity.name.function",
"contentName": "foreign.python",
"begin": "^(\\s*)(foreign)\\s+python\\s*([^\\s]+).*=(.*$)",
"end": "^(?!\\1\\s+)(?!\\s*$)",
"beginCaptures": {
"2" : { "name": "keyword.other" },
"3" : { "name": "entity.name.function"},
"4" : { "name": "header"}
},
"patterns": [ { "include": "source.python" }]
},
{
"contentName": "string.quoted.triple.begin",
"begin": "^(\\s*)(\"\"\"|''')",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package org.enso.tools.enso4igv;

import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;

public class EnsoTextMateTest {

private static Language<? extends TokenId> ensoLanguage;

@BeforeClass
public static void findEnsoLanguage() {
ensoLanguage = org.netbeans.api.lexer.Language.find("application/x-enso");
assertNotNull("Needs org-netbeans-modules-lexer-nbbridge dependency", ensoLanguage);
}

@Test
public void testSimpleMain() {
var code = """
main = 42
""";
var hier = org.netbeans.api.lexer.TokenHierarchy.create(code, ensoLanguage);
assertNotNull(hier);
var seq = hier.tokenSequence();
assertEquals(6, seq.tokenCount());
assertNextToken(seq, "main", "entity.name.function");
assertNextToken(seq, " ", null);
assertNextToken(seq, "=", "keyword.operator");
assertNextToken(seq, " ", null);
assertNextToken(seq, "42", "constant.character.numeric");
}

@Test
public void testForeignJavaScriptFunction() {
var code = """
foreign js dbg = '''
debugger;
""";
var hier = org.netbeans.api.lexer.TokenHierarchy.create(code, ensoLanguage);
assertNotNull(hier);
var seq = hier.tokenSequence();
assertEquals(4, seq.tokenCount());
assertNextToken(seq, "foreign js dbg = ", null);
assertNextToken(seq, "'''", "string.quoted.triple.begin");
assertNextToken(seq, "\n", null);
assertNextToken(seq, " debugger;\n", null);
assertFalse("EOF", seq.moveNext());
}

private static void assertNextToken(TokenSequence<?> seq, String expectedText, String expectedCategory) {
try {
assertNextTokenImpl(seq, expectedText, expectedCategory);
} catch (AssertionError err) {
var sb = new StringBuilder();
sb.append(err.getMessage()).append("\n");
sb.append("Remaining tokens:\n");
seq.movePrevious();
while (seq.moveNext()) {
final Object categories = seq.token().getProperty("categories");
sb.append(seq.token().text()).append(" categories: ").append(categories).append("\n");
}
throw new AssertionError(sb.toString(), err);
}
}

private static void assertNextTokenImpl(TokenSequence<?> seq, String expectedText, String expectedCategory) {
assertTrue("Can move next", seq.moveNext());
var tok = seq.token();
assertNotNull("Token found", tok);
if (expectedText != null) {
assertEquals(expectedText, tok.text().toString());
}
if (expectedCategory != null) {
var categories = (List<String>) tok.getProperty("categories");
assertNotNull("Categories found", categories);
var at = categories.indexOf(expectedCategory);
assertNotEquals("Category " + expectedCategory + " found in " + categories, -1, at);
}
}
}

0 comments on commit 8d9c25c

Please sign in to comment.