Skip to content

Commit

Permalink
Fix for #PyDev-1055: Pydev does not recognize named unicode character…
Browse files Browse the repository at this point in the history 
… escape in f-string (#258)
  • Loading branch information
@oluiscabral
oluiscabral committed May 15, 2020
1 parent b6cdb91 commit c2a03ef
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 41 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Expand Up @@ -53,4 +53,4 @@ plugins/org.python.pydev.core/pysrc/.project
plugins/org.python.pydev.core/pysrc/.settings
**/v2_indexcache/**
plugins/com.python.pydev.runalltests/fake_homedir*
.cache
.cache
5 changes: 2 additions & 3 deletions .../com.python.pydev.analysis/src/com/python/pydev/analysis/visitors/OccurrencesVisitor.java
View file
Expand Up @@ -35,7 +35,7 @@
import org.python.pydev.parser.fastparser.grammar_fstrings_common.FStringsAST;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.FStringsAST.FStringExpressionContent;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammar;
import org.python.pydev.parser.jython.FastCharStream;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammarFactory;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Assert;
import org.python.pydev.parser.jython.ast.Assign;
Expand Down Expand Up @@ -245,8 +245,7 @@ public Object visitStr(Str node) throws Exception {
FStringsAST ast = null;
if (s.trim().length() > 0) {
try {
FastCharStream in = new FastCharStream(s.toCharArray());
FStringsGrammar fStringsGrammar = new FStringsGrammar(in);
FStringsGrammar fStringsGrammar = FStringsGrammarFactory.createGrammar(s);
ast = fStringsGrammar.f_string();
//Note: we always try to generate a valid AST and get any errors in getParseErrors().
parseErrors = fStringsGrammar.getParseErrors();
Expand Down
45 changes: 17 additions & 28 deletions ...org.python.pydev.parser/src/org/python/pydev/parser/grammar_fstrings/FStringsGrammar.java
View file
Expand Up @@ -673,18 +673,7 @@ final public SimpleNode inner_balanced_expression_text_with_exclamation_and_colo
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TEXT:
label_13:
while (true) {
jj_consume_token(TEXT);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TEXT:
;
break;
default:
jj_la1[21] = jj_gen;
break label_13;
}
}
jj_consume_token(TEXT);
break;
case EXCLAMATION:
jj_consume_token(EXCLAMATION);
Expand All @@ -693,7 +682,7 @@ final public SimpleNode inner_balanced_expression_text_with_exclamation_and_colo
jj_consume_token(COLON);
break;
default:
jj_la1[22] = jj_gen;
jj_la1[21] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
Expand All @@ -704,7 +693,7 @@ final public SimpleNode inner_balanced_expression_text_with_exclamation_and_colo
;
break;
default:
jj_la1[23] = jj_gen;
jj_la1[22] = jj_gen;
break label_12;
}
}
Expand All @@ -714,7 +703,7 @@ final public SimpleNode inner_balanced_expression_text_with_exclamation_and_colo
errorBackSlashInvalidInFStrings(t);
break;
default:
jj_la1[24] = jj_gen;
jj_la1[23] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
Expand Down Expand Up @@ -756,7 +745,7 @@ final public void string() throws ParseException {
try {
start = getToken(1);
jj_consume_token(QUOTE);
label_14:
label_13:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case LPAREN:
Expand All @@ -773,8 +762,8 @@ final public void string() throws ParseException {
;
break;
default:
jj_la1[25] = jj_gen;
break label_14;
jj_la1[24] = jj_gen;
break label_13;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TEXT:
Expand Down Expand Up @@ -812,7 +801,7 @@ final public void string() throws ParseException {
errorBackSlashInvalidInFStrings(t);
break;
default:
jj_la1[26] = jj_gen;
jj_la1[25] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
Expand Down Expand Up @@ -844,7 +833,7 @@ final public void string2() throws ParseException {
try {
start = getToken(1);
jj_consume_token(QUOTE2);
label_15:
label_14:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case LPAREN:
Expand All @@ -861,8 +850,8 @@ final public void string2() throws ParseException {
;
break;
default:
jj_la1[27] = jj_gen;
break label_15;
jj_la1[26] = jj_gen;
break label_14;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TEXT:
Expand Down Expand Up @@ -900,7 +889,7 @@ final public void string2() throws ParseException {
errorBackSlashInvalidInFStrings(t);
break;
default:
jj_la1[28] = jj_gen;
jj_la1[27] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
Expand Down Expand Up @@ -931,13 +920,13 @@ final public void string2() throws ParseException {
public Token jj_nt;
private int jj_ntk;
private int jj_gen;
final private int[] jj_la1 = new int[29];
final private int[] jj_la1 = new int[28];
static private int[] jj_la1_0;
static {
jj_la1_init_0();
}
private static void jj_la1_init_0() {
jj_la1_0 = new int[] {0x1ffe0,0x100,0x1ffe0,0x1e2a0,0x800,0x1000,0x10000,0x1e080,0x10000,0x1e2a0,0x1e080,0x10000,0x1e2a0,0x1e2a0,0x1faa0,0x1faa0,0x1faa0,0x2a0,0x1faa0,0x1faa0,0x1faa0,0x10000,0x11800,0x11800,0x1faa0,0x1dfe0,0x1dfe0,0x1bfe0,0x1bfe0,};
jj_la1_0 = new int[] {0x1ffe0,0x100,0x1ffe0,0x1e2a0,0x800,0x1000,0x10000,0x1e080,0x10000,0x1e2a0,0x1e080,0x10000,0x1e2a0,0x1e2a0,0x1faa0,0x1faa0,0x1faa0,0x2a0,0x1faa0,0x1faa0,0x1faa0,0x11800,0x11800,0x1faa0,0x1dfe0,0x1dfe0,0x1bfe0,0x1bfe0,};
}

/** Constructor with user supplied FastCharStream. */
Expand All @@ -946,7 +935,7 @@ public FStringsGrammar(FastCharStream stream) {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 29; i++) jj_la1[i] = -1;
for (int i = 0; i < 28; i++) jj_la1[i] = -1;
}

/** Reinitialise. */
Expand All @@ -958,7 +947,7 @@ public FStringsGrammar(FStringsGrammarTokenManager tm) {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 29; i++) jj_la1[i] = -1;
for (int i = 0; i < 28; i++) jj_la1[i] = -1;
}

/** Reinitialise. */
Expand Down Expand Up @@ -1017,7 +1006,7 @@ public ParseException generateParseException() {
la1tokens[jj_kind] = true;
jj_kind = -1;
}
for (int i = 0; i < 29; i++) {
for (int i = 0; i < 28; i++) {
if (jj_la1[i] == jj_gen) {
for (int j = 0; j < 32; j++) {
if ((jj_la1_0[i] & (1<<j)) != 0) {
Expand Down
30 changes: 30 additions & 0 deletions ...hon.pydev.parser/src/org/python/pydev/parser/grammar_fstrings/FStringsGrammarFactory.java
View file
@@ -0,0 +1,30 @@
package org.python.pydev.parser.grammar_fstrings;

import org.python.pydev.parser.jython.FastCharStream;

public class FStringsGrammarFactory {

public static FStringsGrammar createGrammar(String s) {

char[] c = s.toCharArray();
for (int i = 0, z = 0; i < c.length; i++) {
if (c[i] == '\\' && i + 2 < c.length && c[i + 1] == 'N' && c[i + 2] == '{') {
c[i + 2] = '_';
i += 2;
z = i + 1;
for (; z < c.length; z++) {
if (c[z] == '}') {
c[z] = '_';
break;
}
}
}
}

FastCharStream in = new FastCharStream(c);
FStringsGrammar fStringsGrammar = new FStringsGrammar(in);

return fStringsGrammar;
}

}
7 changes: 5 additions & 2 deletions plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar_fstrings/build.xml
View file
@@ -1,7 +1,10 @@
<project name="jython-parser" default="parser">



<property name="javaccHome" value="c:\bin\javacc-5.0" />
<property name="python" value="c:\bin\Miniconda\envs\py27_tests\python.exe" />
<property environment="env"/>
<property name="javaccHome" value="${env.GRAMMAR_JAVACC_DIR}"/>
<property name="python" value="${env.GRAMMAR_PYTHON_EXE}" />
<property name="basedir" value="." />
<property name="parser.dir" value="${basedir}" />

Expand Down
20 changes: 17 additions & 3 deletions plugins/org.python.pydev.parser/tests/org/python/pydev/parser/FStringsParserTest.java
View file
Expand Up @@ -10,7 +10,7 @@
import org.python.pydev.parser.fastparser.grammar_fstrings_common.FStringsAST;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.SimpleNode;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammar;
import org.python.pydev.parser.jython.FastCharStream;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammarFactory;
import org.python.pydev.parser.jython.ParseException;
import org.python.pydev.shared_core.string.StringUtils;
import org.python.pydev.shared_core.structure.Tuple;
Expand All @@ -21,8 +21,7 @@
public class FStringsParserTest extends TestCase {

private Tuple<FStringsAST, List> check(String str) throws ParseException {
FastCharStream in = new FastCharStream(str.toCharArray());
FStringsGrammar fStringsGrammar = new FStringsGrammar(in);
FStringsGrammar fStringsGrammar = FStringsGrammarFactory.createGrammar(str);
FStringsAST ast = fStringsGrammar.f_string();
//Note: we always try to generate a valid AST and get any errors in getParseErrors().
List<ParseException> parseErrors = fStringsGrammar.getParseErrors();
Expand Down Expand Up @@ -77,6 +76,21 @@ private void checkError(String str, String... expected) throws ParseException {
}

public void testFStringParsing() throws ParseException, BadLocationException {
checkExprs("\\N{foo}", ArrayUtils.asSet());
checkExprs("\\N{\\N{foo}}", ArrayUtils.asSet());
checkExprs("\\N{\\N{}}", ArrayUtils.asSet());
checkExprs("\\N{\\N{\\N{foo}}}", ArrayUtils.asSet());
checkExprs("\\N{", ArrayUtils.asSet());
checkExprs("\\N{foo} {foo}", ArrayUtils.asSet("foo"));
checkExprs("\\N{foo}{foo}", ArrayUtils.asSet("foo"));
checkExprs("\\N{foo}\\N{foo}", ArrayUtils.asSet());
checkExprs("\\N{\\N{foo}}\\N{foo}", ArrayUtils.asSet());
checkExprs("\\N{\\N{}} \\N{foo}", ArrayUtils.asSet());
checkExprs("\\N{\\N{\\N{foo}}} \\N{foo}", ArrayUtils.asSet());
checkExprs("\\N{foo", ArrayUtils.asSet());
checkExprs("\\N{foo} \\N{\\N{\\N{foo}}}", ArrayUtils.asSet());
checkExprs("\\N{foo} \\N{foo} {foo}", ArrayUtils.asSet("foo"));

checkExprs("{val:{width}.{precision}f}", ArrayUtils.asSet("val", "width", "precision"));
checkExprs("{a:>{width}}", ArrayUtils.asSet("a", "width"));
checkExprs("{a:>{{width}}}", ArrayUtils.asSet("a"));
Expand Down
6 changes: 2 additions & 4 deletions plugins/org.python.pydev/src/org/python/pydev/editor/PyFStringScanner.java
View file
Expand Up @@ -11,7 +11,7 @@
import org.python.pydev.parser.fastparser.grammar_fstrings_common.FStringsAST;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.SimpleNode;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammar;
import org.python.pydev.parser.jython.FastCharStream;
import org.python.pydev.parser.grammar_fstrings.FStringsGrammarFactory;
import org.python.pydev.shared_core.partitioner.IToken;
import org.python.pydev.shared_core.partitioner.ITokenScanner;
import org.python.pydev.shared_core.partitioner.SubRuleToken;
Expand Down Expand Up @@ -64,9 +64,7 @@ public void setRange(final IDocument document, final int offset, final int lengt
if (buf.length() < 0) {
return;
}

FastCharStream in = new FastCharStream(buf.toCharArray());
FStringsGrammar fStringsGrammar = new FStringsGrammar(in);
FStringsGrammar fStringsGrammar = FStringsGrammarFactory.createGrammar(buf);
FStringsAST ast = null;
try {
ast = fStringsGrammar.f_string();
Expand Down

0 comments on commit c2a03ef

Please sign in to comment.