Release 2.9.0

CHANGELOG

@@ -1,3 +1,10 @@
+2.9.0
+Feb 07, 2020
+
+NEW: customizable escape character #159
+NEW: value separator setting moved from 'Code Style' to 'General'
++ lots of cleanup & rework
+
 2.8.2
 Jan 22, 2020
 

README.md

@@ -12,6 +12,8 @@ Compatible with _IntelliJ IDEA  PhpStorm  WebStorm  PyCharm  RubyMine  AppCode
 This plugin introduces CSV (_Comma-Separated Values_) as a language to Jetbrains IDE with a syntax definition, structured language elements and associated file types (.csv/.tsv/.psv).
 This enables default editor features like syntax validation, highlighting and inspections for CSV-alike files.
 
+![CSV Plugin Example](./docs/example.png)
+
 ## Features
 
 - CSV/TSV/PSV file detection
@@ -299,15 +301,15 @@ Annasusanna,Amsterdam,         1
 
 ### Actions
 
-#### File specific value separator
+#### File specific value separator & escape character
 
 ![Context menu](./docs/contextmenu.png)
 
-The action to switch the value separator used for CSV syntax validation of a specific file is part of its editors context menu.
+The action to switch the value separator (or escape character) - *which is used for CSV syntax validation of a specific file* - is part of its editors context menu.
 
 
 This action defines how the parser/validator/highlighter/etc. behaves. It does intentionally not change the file content.
-To be more precise: It **does not replace** previous separator characters by new ones or adjust the escaped texts.
+To be more precise: It **does not replace** previous separator/escape characters by new ones or adjust the escaped texts.
 
 #### Adjust column widths (table editor only)
 

build.gradle

@@ -24,7 +24,7 @@ jacocoTestReport {
 }
 
 group 'net.seesharpsoft.intellij.plugins'
-version '2.8.2'
+version '2.9.0'
 
 apply plugin: 'java'
 sourceCompatibility = javaVersion

gradle.properties

@@ -3,5 +3,5 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=1.8
-javaTargetVersion=1.8
+javaVersion=8
+javaTargetVersion=8

src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf

@@ -1,8 +1,6 @@
 {
   parserClass="net.seesharpsoft.intellij.plugins.csv.parser.CsvParser"
 
-  parserImports=["static net.seesharpsoft.intellij.plugins.csv.CsvParserUtil.*"]
-
   extends="com.intellij.extapi.psi.ASTWrapperPsiElement"
 
   psiClassPrefix="Csv"
@@ -15,8 +13,8 @@
   tokenTypeClass="net.seesharpsoft.intellij.plugins.csv.psi.CsvTokenType"
 
   tokens=[
-    TEXT='regexp:[^ ,;|\t\r\n"]+'
-    ESCAPED_TEXT='regexp:([,;|\t\r\n]|"")+'
+    TEXT='regexp:[^ ,;|\t\r\n"\\]+'
+    ESCAPED_TEXT='regexp:[,;|\t\r\n\\]|""|\\"'
     COMMA='regexp:[,;|\t]'
     QUOTE='"'
     CRLF='regexp:\n'
@@ -25,10 +23,10 @@
 
 csvFile ::= record (CRLF record)* [CRLF]
 
-record ::= field ( << separator >> COMMA field)*
+record ::= field (COMMA field)*
 
 field ::= (escaped | nonEscaped)
 
-private escaped ::= QUOTE ( TEXT | ESCAPED_TEXT)* QUOTE
+private escaped ::= QUOTE (TEXT | ESCAPED_TEXT)* QUOTE
 
-private nonEscaped ::= TEXT*
+private nonEscaped ::= TEXT*

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvEscapeCharacter.java

@@ -0,0 +1,30 @@
+package net.seesharpsoft.intellij.plugins.csv;
+
+import java.util.regex.Pattern;
+
+public enum CsvEscapeCharacter {
+    QUOTE("\"", "Double Quote (\")"),
+    BACKSLASH("\\", "Backslash (\\)");
+
+    private final String myCharacter;
+    private final String myDisplay;
+    private final Pattern myPattern;
+
+    CsvEscapeCharacter(String character, String display) {
+        myCharacter = character;
+        myDisplay = display;
+        myPattern = Pattern.compile(Pattern.quote(myCharacter + "\""));
+    }
+
+    public String getCharacter() {
+        return myCharacter;
+    }
+
+    public String getDisplay() {
+        return myDisplay;
+    }
+
+    public boolean isEscapedQuote(String text) {
+        return myPattern.matcher(text).matches();
+    }
+}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java

@@ -17,6 +17,7 @@
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.util.PsiTreeUtil;
 import net.seesharpsoft.intellij.lang.FileParserDefinition;
+import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvField;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvRecord;
@@ -26,6 +27,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
 public final class CsvHelper {
 
@@ -149,14 +151,55 @@ public static int getFieldEndOffset(PsiElement field) {
         return separator == null ? field.getContainingFile().getTextLength() : separator.getTextOffset();
     }
 
+    public static VirtualFile getVirtualFile(PsiFile psiFile) {
+        return psiFile == null ? null : psiFile.getOriginalFile().getVirtualFile();
+    }
+
+    public static Project getProject(PsiFile psiFile) {
+        return psiFile == null ? null : psiFile.getProject();
+    }
+
+    public static CsvValueSeparator getValueSeparator(CsvFile csvFile) {
+        return getValueSeparator(csvFile.getContainingFile());
+    }
+
+    public static CsvValueSeparator getValueSeparator(PsiFile psiFile) {
+        return getValueSeparator(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
+        return CsvFileAttributes.getInstance(project).getValueSeparator(project, virtualFile);
+    }
+
+    public static boolean hasValueSeparatorAttribute(@NotNull PsiFile psiFile) {
+        return CsvFileAttributes.getInstance(getProject(psiFile)).hasValueSeparatorAttribute(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(CsvFile csvFile) {
+        return getEscapeCharacter(csvFile.getContainingFile());
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(PsiFile psiFile) {
+        return getEscapeCharacter(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(Project project, VirtualFile virtualFile) {
+        return CsvFileAttributes.getInstance(project).getEscapeCharacter(project, virtualFile);
+    }
+
+    public static boolean hasEscapeCharacterAttribute(@NotNull PsiFile psiFile) {
+        return CsvFileAttributes.getInstance(getProject(psiFile)).hasEscapeCharacterAttribute(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
     public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile) {
+        CsvEscapeCharacter escapeCharacter = getEscapeCharacter(csvFile);
         Map<Integer, CsvColumnInfo<PsiElement>> columnInfoMap = new HashMap<>();
         CsvRecord[] records = PsiTreeUtil.getChildrenOfType(csvFile, CsvRecord.class);
         int row = 0;
         for (CsvRecord record : records) {
             int column = 0;
             for (CsvField field : record.getFieldList()) {
-                Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText()));
+                Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText(), escapeCharacter));
                 if (!columnInfoMap.containsKey(column)) {
                     columnInfoMap.put(column, new CsvColumnInfo(column, length, row));
                 } else if (columnInfoMap.get(column).getMaxLength() < length) {
@@ -170,28 +213,32 @@ public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile)
         return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile));
     }
 
-    public static String unquoteCsvValue(String content) {
+    public static String unquoteCsvValue(String content, CsvEscapeCharacter escapeCharacter) {
         if (content == null) {
             return "";
         }
         String result = content.trim();
         if (result.length() > 1 && result.startsWith("\"") && result.endsWith("\"")) {
             result = result.substring(1, result.length() - 1);
         }
-        result = result.replaceAll("(?:\")\"", "\"");
+        result = result.replaceAll("(?:" + Pattern.quote(escapeCharacter.getCharacter()) + ")\"", "\"");
         return result;
     }
 
-    private static boolean isQuotingRequired(String content, String separator) {
-        return content != null && (content.contains(separator) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
+    private static boolean isQuotingRequired(String content, CsvValueSeparator valueSeparator) {
+        return content != null &&
+                (content.contains(valueSeparator.getCharacter()) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
     }
 
-    public static String quoteCsvField(String content, String separator, boolean quotingEnforced) {
+    public static String quoteCsvField(String content,
+                                       CsvEscapeCharacter escapeCharacter,
+                                       CsvValueSeparator valueSeparator,
+                                       boolean quotingEnforced) {
         if (content == null) {
             return "";
         }
-        if (quotingEnforced || isQuotingRequired(content, separator)) {
-            String result = content.replaceAll("\"", "\"\"");
+        if (quotingEnforced || isQuotingRequired(content, valueSeparator)) {
+            String result = content.replaceAll("\"", escapeCharacter.getCharacter() + "\"");
             return "\"" + result + "\"";
         }
         return content;

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex

@@ -1,10 +1,11 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
-import com.intellij.lexer.FlexLexer;
 import com.intellij.psi.tree.IElementType;
-import com.intellij.openapi.project.Project;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
 import com.intellij.psi.TokenType;
+import com.intellij.lexer.FlexLexer;
+
+import java.util.regex.Pattern;
 
 %%
 
@@ -14,21 +15,25 @@ import com.intellij.psi.TokenType;
 %function advance
 %type IElementType
 %{
-  private String currentSeparator;
-
-  /**
-   * Provide constructor that supports a Project as parameter.
-   */
-  CsvLexer(java.io.Reader in, String separator) {
-    this(in);
-    this.currentSeparator = separator;
-  }
+    private CsvValueSeparator myValueSeparator;
+    private CsvEscapeCharacter myEscapeCharacter;
+
+    private static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
+
+    /**
+     * Provide constructor that supports a Project as parameter.
+     */
+    CsvLexer(java.io.Reader in, CsvValueSeparator valueSeparator, CsvEscapeCharacter escapeCharacter) {
+      this(in);
+      myValueSeparator = valueSeparator;
+      myEscapeCharacter = escapeCharacter;
+    }
 %}
 %eof{  return;
 %eof}
 
-TEXT=[^ ,;|\t\r\n\"]+
-ESCAPED_TEXT=([,;|\t\r\n]|\"\")+
+TEXT=[^ ,;|\t\r\n\"\\]+
+ESCAPED_TEXT=[,;|\t\r\n\\]|\"\"|\\\"
 QUOTE=\"
 COMMA=[,;|\t]
 EOL=\n
@@ -65,12 +70,18 @@ WHITE_SPACE=[ \f]+
 
 <ESCAPED_TEXT> {ESCAPED_TEXT}
 {
-    return CsvTypes.ESCAPED_TEXT;
+    String text = yytext().toString();
+    if (myEscapeCharacter.isEscapedQuote(text)
+        || ESCAPE_TEXT_PATTERN.matcher(text).matches()
+     ) {
+        return CsvTypes.ESCAPED_TEXT;
+    }
+    return TokenType.BAD_CHARACTER;
 }
 
 <YYINITIAL, AFTER_TEXT, UNESCAPED_TEXT> {COMMA}
 {
-    if (currentSeparator.equals(yytext().toString())) {
+    if (myValueSeparator.isValueSeparator(yytext().toString())) {
         yybegin(YYINITIAL);
         return CsvTypes.COMMA;
     }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerAdapter.java

@@ -3,7 +3,7 @@
 import com.intellij.lexer.FlexAdapter;
 
 public class CsvLexerAdapter extends FlexAdapter {
-    public CsvLexerAdapter(String separator) {
-        super(new CsvLexer(null, separator));
+    public CsvLexerAdapter(CsvValueSeparator separator, CsvEscapeCharacter escapeCharacter) {
+        super(new CsvLexer(null, separator, escapeCharacter));
     }
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserDefinition.java

@@ -15,7 +15,6 @@
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFileElementType;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
-import net.seesharpsoft.intellij.plugins.csv.settings.CsvCodeStyleSettings;
 import org.jetbrains.annotations.NotNull;
 
 public class CsvParserDefinition implements FileParserDefinition {
@@ -76,11 +75,11 @@ public PsiElement createElement(ASTNode node) {
 
     @Override
     public Lexer createLexer(@NotNull PsiFile file) {
-        return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file));
+        return new CsvLexerAdapter(CsvHelper.getValueSeparator(file), CsvHelper.getEscapeCharacter(file));
     }
 
     @Override
     public PsiParser createParser(@NotNull PsiFile file) {
         return createParser(file.getProject());
     }
-}
+}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSeparatorHolder.java

@@ -1,5 +1,5 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
 public interface CsvSeparatorHolder {
-    String getSeparator();
+    CsvValueSeparator getSeparator();
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvStorageHelper.java

@@ -17,7 +17,7 @@ public static String getRelativeFileUrl(Project project, VirtualFile virtualFile
             return null;
         }
         String url = virtualFile.getUserData(RELATIVE_FILE_URL);
-        if (url == null) {
+        if (url == null && project.getBasePath() != null) {
             String projectDir = PathUtil.getLocalPath(project.getBasePath());
             url = PathUtil.getLocalPath(virtualFile.getPath())
                     .replaceFirst("^" + Pattern.quote(projectDir), "");