diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf b/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf index f976a35a..1d6fc185 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf @@ -19,12 +19,13 @@ COMMA='regexp:[,:;|\t]' QUOTE='regexp:"' CRLF='regexp:\n' + COMMENT='regexp:#.*(\n|$)' ] } csvFile ::= record (CRLF record)* [CRLF] -record ::= field (COMMA field)* +record ::= (COMMENT | (field (COMMA field)*)) field ::= (escaped | nonEscaped) diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvColumnInfoMap.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvColumnInfoMap.java index 85953b28..3479b182 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvColumnInfoMap.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvColumnInfoMap.java @@ -10,16 +10,18 @@ public class CsvColumnInfoMap { private final Map> myReverseInfoColumnMap; private boolean hasErrors = false; + private boolean hasComments = false; - public CsvColumnInfoMap(Map> infoColumnMap, boolean hasErrorsArg) { + public CsvColumnInfoMap(Map> infoColumnMap, boolean hasErrorsArg, boolean hasCommentsArg) { this.myInfoColumnMap = infoColumnMap; this.myReverseInfoColumnMap = new HashMap<>(); buildReverseMap(); setHasErrors(hasErrorsArg); + setHasComments(hasCommentsArg); } public CsvColumnInfoMap(Map> infoColumnMap) { - this(infoColumnMap, false); + this(infoColumnMap, false, false); } private void buildReverseMap() { @@ -55,6 +57,14 @@ public void setHasErrors(boolean hasErrorsArg) { hasErrors = hasErrorsArg; } + public boolean hasComments() { + return hasComments; + } + + public void setHasComments(boolean hasCommentsArg) { + hasComments = hasCommentsArg; + } + public boolean hasEmptyLastLine() { CsvColumnInfo columnInfo = myInfoColumnMap.get(0); int size = columnInfo.getSize(); diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java index 4c472afa..48e0ce67 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java @@ -201,6 +201,7 @@ public static CsvColumnInfoMap createColumnInfoMap(CsvFile csvFile) Map> columnInfoMap = new HashMap<>(); CsvRecord[] records = PsiTreeUtil.getChildrenOfType(csvFile, CsvRecord.class); int row = 0; + boolean hasComments = false; for (CsvRecord record : records) { int column = 0; for (CsvField field : record.getFieldList()) { @@ -213,9 +214,12 @@ public static CsvColumnInfoMap createColumnInfoMap(CsvFile csvFile) columnInfoMap.get(column).addElement(field, row, getFieldStartOffset(field), getFieldEndOffset(field)); ++column; } + if (record.getComment() != null) { + hasComments = true; + } ++row; } - return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile)); + return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile), hasComments); } public static String unquoteCsvValue(String content, CsvEscapeCharacter escapeCharacter) { diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java index 3e6dfdd3..cf2d5841 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java @@ -4,6 +4,7 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiFile; +import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings; import org.jetbrains.annotations.NotNull; public class CsvLexerFactory { @@ -14,12 +15,13 @@ public static CsvLexerFactory getInstance() { } protected Lexer createLexer(@NotNull CsvValueSeparator separator, @NotNull CsvEscapeCharacter escapeCharacter) { - if (separator.isCustom()) { + if (separator.isCustom() || !CsvEditorSettings.getInstance().getCommentIndicator().isEmpty()) { return new CsvSharpLexer(new CsvSharpLexer.Configuration( separator.getCharacter(), "\n", escapeCharacter.getCharacter(), - "\"")); + "\"", + CsvEditorSettings.getInstance().getCommentIndicator())); } return new CsvLexerAdapter(separator, escapeCharacter); } diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSharpLexer.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSharpLexer.java index 0737c744..4fe85e89 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSharpLexer.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSharpLexer.java @@ -5,6 +5,7 @@ import net.seesharpsoft.UnhandledSwitchCaseException; import net.seesharpsoft.commons.util.Tokenizer; import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes; +import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -18,6 +19,7 @@ public class CsvSharpLexer extends LexerBase { private final Tokenizer tokenizer; + private final List> initialNextStateTokens; private final List> unquotedNextStateTokens; private final List> quotedNextStateTokens; @@ -29,23 +31,34 @@ public class CsvSharpLexer extends LexerBase { private IElementType currentTokenType; private boolean failed; + private static final Map INITIAL_NEXT_STATES = new HashMap<>(); private static final Map UNQUOTED_NEXT_STATES = new HashMap<>(); private static final Map QUOTED_NEXT_STATES = new HashMap<>(); static { + INITIAL_NEXT_STATES.put(TokenType.WHITESPACE, LexerState.Initial); + INITIAL_NEXT_STATES.put(TokenType.TEXT, LexerState.Unquoted); + INITIAL_NEXT_STATES.put(TokenType.VALUE_SEPARATOR, LexerState.Unquoted); + INITIAL_NEXT_STATES.put(TokenType.BEGIN_QUOTE, LexerState.Quoted); + INITIAL_NEXT_STATES.put(TokenType.RECORD_SEPARATOR, LexerState.Initial); + INITIAL_NEXT_STATES.put(TokenType.COMMENT, LexerState.Initial); + UNQUOTED_NEXT_STATES.put(TokenType.WHITESPACE, LexerState.Unquoted); UNQUOTED_NEXT_STATES.put(TokenType.TEXT, LexerState.Unquoted); + UNQUOTED_NEXT_STATES.put(TokenType.COMMENT_CHARACTER, LexerState.Unquoted); UNQUOTED_NEXT_STATES.put(TokenType.VALUE_SEPARATOR, LexerState.Unquoted); - UNQUOTED_NEXT_STATES.put(TokenType.RECORD_SEPARATOR, LexerState.Unquoted); UNQUOTED_NEXT_STATES.put(TokenType.BEGIN_QUOTE, LexerState.Quoted); + UNQUOTED_NEXT_STATES.put(TokenType.RECORD_SEPARATOR, LexerState.Initial); QUOTED_NEXT_STATES.put(TokenType.WHITESPACE, LexerState.Quoted); QUOTED_NEXT_STATES.put(TokenType.TEXT, LexerState.Quoted); + QUOTED_NEXT_STATES.put(TokenType.COMMENT_CHARACTER, LexerState.Quoted); QUOTED_NEXT_STATES.put(TokenType.ESCAPED_CHARACTER, LexerState.Quoted); QUOTED_NEXT_STATES.put(TokenType.END_QUOTE, LexerState.Unquoted); } enum LexerState { + Initial(INITIAL_NEXT_STATES), Unquoted(UNQUOTED_NEXT_STATES), Quoted(QUOTED_NEXT_STATES); @@ -71,22 +84,26 @@ enum TokenType { ESCAPED_CHARACTER, VALUE_SEPARATOR, RECORD_SEPARATOR, - WHITESPACE + WHITESPACE, + COMMENT, + COMMENT_CHARACTER } public static class Configuration { - public static final Configuration DEFAULT = new Configuration(",", "\n", "\"", "\""); + public static final Configuration DEFAULT = new Configuration(",", "\n", "\"", "\"", "#"); public String valueSeparator; public String recordSeparator; public String escapeCharacter; public String quoteCharacter; + public String commentCharacter; - public Configuration(String valueSeparator, String recordSeparator, String escapeCharacter, String quoteCharacter) { + public Configuration(String valueSeparator, String recordSeparator, String escapeCharacter, String quoteCharacter, String commentCharacter) { this.valueSeparator = Pattern.quote(valueSeparator); this.recordSeparator = Pattern.quote(recordSeparator); this.escapeCharacter = Pattern.quote(escapeCharacter); this.quoteCharacter = Pattern.quote(quoteCharacter); + this.commentCharacter = Pattern.quote(commentCharacter); } } @@ -102,17 +119,32 @@ public CsvSharpLexer(Configuration configuration) { tokenizer.add(TokenType.BEGIN_QUOTE, String.format("%s", configuration.quoteCharacter)); tokenizer.add(TokenType.VALUE_SEPARATOR, configuration.valueSeparator); tokenizer.add(TokenType.RECORD_SEPARATOR, configuration.recordSeparator); + if (!configuration.commentCharacter.isEmpty()) { + tokenizer.add(TokenType.COMMENT_CHARACTER, configuration.commentCharacter); + tokenizer.add(TokenType.COMMENT, configuration.commentCharacter + ".*(?=(\n|$))"); + } if (configuration.escapeCharacter.equals(configuration.quoteCharacter)) { tokenizer.add(TokenType.END_QUOTE, String.format("%s(?!%s)", configuration.quoteCharacter, configuration.quoteCharacter)); tokenizer.add(TokenType.ESCAPED_CHARACTER, String.format("(%s%s|%s|%s)+", configuration.quoteCharacter, configuration.quoteCharacter, configuration.valueSeparator, configuration.recordSeparator)); - tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s])+", configuration.valueSeparator, configuration.quoteCharacter, configuration.recordSeparator)); + if (!configuration.commentCharacter.isEmpty()) { + tokenizer.add(TokenType.TEXT, String.format("((?!(%s|%s))[^ \f%s%s])+", configuration.commentCharacter, configuration.valueSeparator, configuration.quoteCharacter, configuration.recordSeparator)); + } else { + tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s])+", configuration.valueSeparator, configuration.quoteCharacter, configuration.recordSeparator)); + } } else { tokenizer.add(TokenType.END_QUOTE, String.format("%s", configuration.quoteCharacter)); tokenizer.add(TokenType.ESCAPED_CHARACTER, String.format("(%s%s|%s%s|%s|%s)+", configuration.escapeCharacter, configuration.quoteCharacter, configuration.escapeCharacter, configuration.escapeCharacter, configuration.valueSeparator, configuration.recordSeparator)); - tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s%s])+", configuration.valueSeparator, configuration.escapeCharacter, configuration.quoteCharacter, configuration.recordSeparator)); + if (!configuration.commentCharacter.isEmpty()) { + tokenizer.add(TokenType.TEXT, String.format("((?!(%s|%s))[^ \f%s%s%s])+", configuration.commentCharacter, configuration.valueSeparator, configuration.escapeCharacter, configuration.quoteCharacter, configuration.recordSeparator)); + } else { + tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s%s])+", configuration.valueSeparator, configuration.escapeCharacter, configuration.quoteCharacter, configuration.recordSeparator)); + } } + initialNextStateTokens = LexerState.Initial.getPossibleTokens().stream() + .map(tokenizer::getToken) + .collect(Collectors.toList()); unquotedNextStateTokens = LexerState.Unquoted.getPossibleTokens().stream() .map(tokenizer::getToken) .collect(Collectors.toList()); @@ -126,14 +158,14 @@ public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, this.buffer = buffer; this.tokenStart = this.tokenEnd = startOffset; this.bufferEnd = endOffset; - this.currentState = initialState == 0 ? LexerState.Unquoted : LexerState.Quoted; + this.currentState = LexerState.values()[initialState]; this.currentTokenType = null; } @Override public int getState() { locateToken(); - return currentState == LexerState.Unquoted ? 0 : 1; + return currentState.ordinal(); } @Nullable @@ -178,6 +210,19 @@ protected void raiseFailure() { tokenEnd = bufferEnd; } + protected Collection> getCurrentTokenCollection() { + switch(this.currentState) { + case Initial: + return initialNextStateTokens; + case Unquoted: + return unquotedNextStateTokens; + case Quoted: + return quotedNextStateTokens; + default: + throw new UnhandledSwitchCaseException(this.currentState); + } + } + protected synchronized void locateToken() { if (currentTokenType != null) { return; @@ -193,7 +238,7 @@ protected synchronized void locateToken() { tokenizer.findToken(buffer, tokenStart, bufferEnd, - currentState == LexerState.Unquoted ? unquotedNextStateTokens : quotedNextStateTokens, + getCurrentTokenCollection(), null ); @@ -222,8 +267,12 @@ protected synchronized void locateToken() { currentTokenType = CsvTypes.COMMA; break; case TEXT: + case COMMENT_CHARACTER: currentTokenType = CsvTypes.TEXT; break; + case COMMENT: + currentTokenType = CsvTypes.COMMENT; + break; case WHITESPACE: currentTokenType = com.intellij.psi.TokenType.WHITE_SPACE; break; diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java index 67b982fb..0922dbf4 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java @@ -102,7 +102,7 @@ public void setEditable(boolean editable) { } public boolean isEditable() { - return this.tableIsEditable && !this.hasErrors(); + return this.tableIsEditable && !this.hasErrors() && !hasComments(); } public CsvColumnInfoMap getColumnInfoMap() { @@ -118,6 +118,14 @@ public boolean hasErrors() { return (columnInfoMap != null && columnInfoMap.hasErrors()); } + public boolean hasComments() { + if (!isValid()) { + return false; + } + CsvColumnInfoMap columnInfoMap = getColumnInfoMap(); + return (columnInfoMap != null && columnInfoMap.hasComments()); + } + protected Object[][] storeStateChange(Object[][] data) { Object[][] result = this.dataManagement.addState(data); saveChanges(); diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/highlighter/CsvSyntaxHighlighter.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/highlighter/CsvSyntaxHighlighter.java index 53270180..7da3ce9c 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/highlighter/CsvSyntaxHighlighter.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/highlighter/CsvSyntaxHighlighter.java @@ -24,6 +24,8 @@ public class CsvSyntaxHighlighter extends SyntaxHighlighterBase { createTextAttributesKey("CSV_DEFAULT_STRING", DefaultLanguageHighlighterColors.STRING); public static final TextAttributesKey ESCAPED_TEXT = createTextAttributesKey("CSV_ESCAPED_STRING", DefaultLanguageHighlighterColors.VALID_STRING_ESCAPE); + public static final TextAttributesKey COMMENT = + createTextAttributesKey("CSV_DEFAULT_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT); public static final TextAttributesKey BAD_CHARACTER = createTextAttributesKey("CSV_BAD_CHARACTER", HighlighterColors.BAD_CHARACTER); @@ -32,6 +34,7 @@ public class CsvSyntaxHighlighter extends SyntaxHighlighterBase { private static final TextAttributesKey[] QUOTE_KEYS = new TextAttributesKey[] {QUOTE}; private static final TextAttributesKey[] TEXT_KEYS = new TextAttributesKey[] {TEXT}; private static final TextAttributesKey[] ESCAPED_TEXT_KEYS = new TextAttributesKey[] {ESCAPED_TEXT}; + private static final TextAttributesKey[] COMMENT_KEYS = new TextAttributesKey[] {COMMENT}; private static final TextAttributesKey[] EMPTY_KEYS = new TextAttributesKey[0]; private final Project myProject; @@ -57,6 +60,8 @@ public TextAttributesKey[] getTokenHighlights(IElementType tokenType) { return QUOTE_KEYS; } else if (tokenType.equals(CsvTypes.TEXT)) { return TEXT_KEYS; + } else if (tokenType.equals(CsvTypes.COMMENT)) { + return COMMENT_KEYS; } else if (tokenType.equals(CsvTypes.ESCAPED_TEXT)) { return ESCAPED_TEXT_KEYS; } else if (tokenType.equals(TokenType.BAD_CHARACTER)) { diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/psi/CsvFile.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/psi/CsvFile.java index 6d681e2f..247122fa 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/psi/CsvFile.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/psi/CsvFile.java @@ -27,6 +27,7 @@ public void propertyChange(PropertyChangeEvent evt) { switch (evt.getPropertyName()) { case "defaultEscapeCharacter": case "defaultValueSeparator": + case "commentIndicator": FileContentUtilCore.reparseFiles(CsvFile.this.getVirtualFile()); break; default: diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvColorSettings.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvColorSettings.java index 0871860b..ff30707e 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvColorSettings.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvColorSettings.java @@ -36,6 +36,7 @@ public class CsvColorSettings implements ColorSettingsPage { attributesDescriptors.add(new AttributesDescriptor("Quote", CsvSyntaxHighlighter.QUOTE)); attributesDescriptors.add(new AttributesDescriptor("Text", CsvSyntaxHighlighter.TEXT)); attributesDescriptors.add(new AttributesDescriptor("Escaped Text", CsvSyntaxHighlighter.ESCAPED_TEXT)); + attributesDescriptors.add(new AttributesDescriptor("Comment", CsvSyntaxHighlighter.COMMENT)); COLUMN_HIGHLIGHT_ATTRIBUTES = new ArrayList<>(); for (int i = 0; i < MAX_COLUMN_HIGHLIGHT_COLORS; ++i) { diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java index 140e912a..56cc38d0 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java @@ -32,6 +32,8 @@ public class CsvEditorSettings implements PersistentStateComponent
- + @@ -16,7 +16,7 @@ - + @@ -97,7 +97,7 @@ - + @@ -251,7 +251,7 @@ - + @@ -292,7 +292,7 @@ - + @@ -382,6 +382,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java index f4133e2a..4f06fcb2 100644 --- a/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java +++ b/src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java @@ -42,6 +42,7 @@ public class CsvEditorSettingsProvider implements EditorOptionsProvider { private JComboBox comboEscapeCharacter; private JComboBox comboValueSeparator; private JCheckBox cbKeepTrailingWhitespaces; + private JTextField tfCommentIndicator; @NotNull @Override @@ -92,7 +93,8 @@ public boolean isModified() { isModified(cbAdjustColumnWidthOnOpen, csvEditorSettings.isTableAutoColumnWidthOnOpen()) || !Objects.equals(comboEscapeCharacter.getSelectedItem(), csvEditorSettings.getDefaultEscapeCharacter()) || !Objects.equals(comboValueSeparator.getSelectedItem(), csvEditorSettings.getDefaultValueSeparator()) || - isModified(cbKeepTrailingWhitespaces, csvEditorSettings.getKeepTrailingSpaces()); + isModified(cbKeepTrailingWhitespaces, csvEditorSettings.getKeepTrailingSpaces()) || + isModified(tfCommentIndicator, csvEditorSettings.getCommentIndicator()); } @Override @@ -117,6 +119,7 @@ public void reset() { comboEscapeCharacter.setSelectedItem(csvEditorSettings.getDefaultEscapeCharacter()); comboValueSeparator.setSelectedItem(csvEditorSettings.getDefaultValueSeparator()); cbKeepTrailingWhitespaces.setSelected(csvEditorSettings.getKeepTrailingSpaces()); + tfCommentIndicator.setText(csvEditorSettings.getCommentIndicator()); } @Override @@ -141,6 +144,7 @@ public void apply() throws ConfigurationException { csvEditorSettings.setDefaultEscapeCharacter((CsvEscapeCharacter)comboEscapeCharacter.getSelectedItem()); csvEditorSettings.setDefaultValueSeparator((CsvValueSeparator)comboValueSeparator.getSelectedItem()); csvEditorSettings.setKeepTrailingSpaces(cbKeepTrailingWhitespaces.isSelected()); + csvEditorSettings.setCommentIndicator(tfCommentIndicator.getText()); } protected void createUIComponents() { diff --git a/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java b/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java index cf1900e1..314cf851 100644 --- a/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java +++ b/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java @@ -5,6 +5,8 @@ import net.seesharpsoft.intellij.plugins.csv.CsvValueSeparator; import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings; +import static net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings.COMMENT_INDICATOR_DEFAULT; + public class CsvParsingTest extends ParsingTestCase { public CsvParsingTest() { @@ -12,7 +14,10 @@ public CsvParsingTest() { } public void testParsingTestData() { + // without comment support, default lexer is used + CsvEditorSettings.getInstance().setCommentIndicator(""); doTest(true); + CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT); } public void testParsingTestDataWithCustomParser() { @@ -29,12 +34,25 @@ public void testCustomMultiSymbolSeparator() { } public void testColonSeparator() { + // without comment support, default lexer is used + CsvEditorSettings.getInstance().setCommentIndicator(""); CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COLON); doTest(true); CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT); + CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT); } public void testAllSeparators() { + // without comment support, default lexer is used + CsvEditorSettings.getInstance().setCommentIndicator(""); + CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COMMA); + doTest(true); + CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT); + CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT); + } + + public void testCsvWithComments() { + // comment support by default (custom lexer is used) CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COMMA); doTest(true); CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT); diff --git a/src/test/resources/parser/CsvWithComments.csv b/src/test/resources/parser/CsvWithComments.csv new file mode 100644 index 00000000..16867781 --- /dev/null +++ b/src/test/resources/parser/CsvWithComments.csv @@ -0,0 +1,4 @@ +header#1,header#2 + # just a comment with whitespace at the start, "yes" - still a comment +"# not a comment",#and this also not +#but here again:,| a comment \ No newline at end of file diff --git a/src/test/resources/parser/CsvWithComments.txt b/src/test/resources/parser/CsvWithComments.txt new file mode 100644 index 00000000..b65ab1a4 --- /dev/null +++ b/src/test/resources/parser/CsvWithComments.txt @@ -0,0 +1,40 @@ +CSV File(0,156) + CsvRecordImpl(RECORD)(0,17) + CsvFieldImpl(FIELD)(0,8) + PsiElement(CsvTokenType.TEXT)('header')(0,6) + PsiElement(CsvTokenType.TEXT)('#')(6,7) + PsiElement(CsvTokenType.TEXT)('1')(7,8) + PsiElement(CsvTokenType.COMMA)(',')(8,9) + CsvFieldImpl(FIELD)(9,17) + PsiElement(CsvTokenType.TEXT)('header')(9,15) + PsiElement(CsvTokenType.TEXT)('#')(15,16) + PsiElement(CsvTokenType.TEXT)('2')(16,17) + PsiElement(CsvTokenType.CRLF)('\n')(17,18) + PsiWhiteSpace(' ')(18,20) + CsvRecordImpl(RECORD)(20,90) + PsiElement(CsvTokenType.COMMENT)('# just a comment with whitespace at the start, "yes" - still a comment')(20,90) + PsiElement(CsvTokenType.CRLF)('\n')(90,91) + CsvRecordImpl(RECORD)(91,127) + CsvFieldImpl(FIELD)(91,108) + PsiElement(CsvTokenType.QUOTE)('"')(91,92) + PsiElement(CsvTokenType.TEXT)('#')(92,93) + PsiWhiteSpace(' ')(93,94) + PsiElement(CsvTokenType.TEXT)('not')(94,97) + PsiWhiteSpace(' ')(97,98) + PsiElement(CsvTokenType.TEXT)('a')(98,99) + PsiWhiteSpace(' ')(99,100) + PsiElement(CsvTokenType.TEXT)('comment')(100,107) + PsiElement(CsvTokenType.QUOTE)('"')(107,108) + PsiElement(CsvTokenType.COMMA)(',')(108,109) + CsvFieldImpl(FIELD)(109,127) + PsiElement(CsvTokenType.TEXT)('#')(109,110) + PsiElement(CsvTokenType.TEXT)('and')(110,113) + PsiWhiteSpace(' ')(113,114) + PsiElement(CsvTokenType.TEXT)('this')(114,118) + PsiWhiteSpace(' ')(118,119) + PsiElement(CsvTokenType.TEXT)('also')(119,123) + PsiWhiteSpace(' ')(123,124) + PsiElement(CsvTokenType.TEXT)('not')(124,127) + PsiElement(CsvTokenType.CRLF)('\n')(127,128) + CsvRecordImpl(RECORD)(128,156) + PsiElement(CsvTokenType.COMMENT)('#but here again:,|\ta comment')(128,156) \ No newline at end of file