parser: ignore #_-commented forms in maps and tagged literals

grammars/clojure.bnf

@@ -76,6 +76,7 @@
   name("metadata|constructor|reader_.*")=form
 
   methods("reader_macro|metadata")=[toString]
+  consumeTokenMethod("ignored|symbol_nsq")="fast"
 }
 
 root ::= <<parseTree (root_entry)>>
@@ -115,7 +116,7 @@ literal ::= number | hexnum | rdxnum | ratio | bool | nil | string | char
 regexp ::= '#' <<nospace>> string {extends=literal}
 access ::= ('.' | '.-') symbol
 left access_left ::= !<<space>> '.'
-constructor ::= '#' symbol form
+constructor ::= '#' ignored symbol ignored form
 metadata ::= ("^" | "#^") (string | symbol | keyword | map)
 reader_macro ::= "'" | "~" | "~@" | "@" | "`" |  "#_" | "#'" | "#=" | symbolic_value | reader_cond | map_ns_prefix
 private symbolic_value ::= '##' &sym {pin=1}
@@ -131,5 +132,7 @@ private meta items_entry ::= (not_eof <<recover>>) <<param>> {pin=1 recoverWhile
 private list_body ::= <<items !')' form>>
 private set_body ::= <<items !'}' form>>
 private vec_body ::= <<items !']' form>>
-private map_body ::= <<items !'}' map_entry>>
-private map_entry ::= form form {pin=1}
+private map_body ::= ignored <<items !'}' map_entry>>
+private map_entry ::=  form ignored form ignored {pin=2}
+
+private ignored ::= (&'#_' form) * {pin(".*")=1}

testData/highlighting/ClojureFixes.clj

@@ -75,4 +75,8 @@
 @<warning>not-to-resolve</warning>
 #:some-ns {:some-key <warning>not-to-resolve</warning>}
 (#'clojure.uuid/default-uuid-reader)
-(clojure.uuid/<warning>default-uuid-reader</warning>)
+(clojure.uuid/<warning>default-uuid-reader</warning>)
+
+(def #_comment named-zero 0)
+{#_0 #_1 :a #_'(xxx)  'a :b 'b #_2 #_3}
+# #_comment dbg 10

testData/parser/CommentedForms.txt

@@ -0,0 +1,84 @@
+CFileImpl:a.clj
+  CListImpl(C_LIST)
+    PsiElement(()('(')
+    CSymbolImpl(C_SYMBOL)
+      PsiElement(sym)('def')
+    PsiWhiteSpace(' ')
+    CSymbolImpl(C_SYMBOL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(sym)('cc')
+    PsiWhiteSpace(' ')
+    CSymbolImpl(C_SYMBOL)
+      PsiElement(sym)('n')
+    PsiWhiteSpace(' ')
+    CLiteralImpl(C_LITERAL)
+      PsiElement(number)('0')
+    PsiElement())(')')
+  PsiWhiteSpace(' ')
+  CMapImpl(C_MAP)
+    PsiElement({)('{')
+    CLiteralImpl(C_LITERAL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(number)('0')
+    PsiWhiteSpace(' ')
+    CLiteralImpl(C_LITERAL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(number)('1')
+    PsiWhiteSpace(' ')
+    CKeywordImpl(C_KEYWORD)
+      PsiElement(:)(':')
+      CSymbolImpl(C_SYMBOL)
+        PsiElement(sym)('a')
+    PsiWhiteSpace(' ')
+    CListImpl(C_LIST)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(')(''')
+      PsiElement(()('(')
+      CSymbolImpl(C_SYMBOL)
+        PsiElement(sym)('xxx')
+      PsiElement())(')')
+    PsiWhiteSpace('  ')
+    CSymbolImpl(C_SYMBOL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(')(''')
+      PsiElement(sym)('a')
+    PsiWhiteSpace(' ')
+    CKeywordImpl(C_KEYWORD)
+      PsiElement(:)(':')
+      CSymbolImpl(C_SYMBOL)
+        PsiElement(sym)('b')
+    PsiWhiteSpace(' ')
+    CSymbolImpl(C_SYMBOL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(')(''')
+      PsiElement(sym)('b')
+    PsiWhiteSpace(' ')
+    CLiteralImpl(C_LITERAL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(number)('2')
+    PsiWhiteSpace(' ')
+    CLiteralImpl(C_LITERAL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(number)('3')
+    PsiElement(})('}')
+  PsiWhiteSpace(' ')
+  CConstructorImpl(C_CONSTRUCTOR)
+    PsiElement(#)('#')
+    PsiWhiteSpace(' ')
+    CSymbolImpl(C_SYMBOL)
+      CReaderMacroImpl(C_READER_MACRO)
+        PsiElement(#_)('#_')
+      PsiElement(sym)('asd')
+    PsiWhiteSpace(' ')
+    CSymbolImpl(C_SYMBOL)
+      PsiElement(sym)('dbg')
+    PsiWhiteSpace(' ')
+    CLiteralImpl(C_LITERAL)
+      PsiElement(number)('10')

testData/parser/SimpleRecover.txt

@@ -48,5 +48,5 @@ CFileImpl:a.clj
   PsiWhiteSpace(' ')
   CSymbolImpl(C_SYMBOL)
     PsiElement(sym)('sym')
-  PsiErrorElement:'.' or '/' expected, got ')'
+  PsiErrorElement:'.' expected, got ')'
     PsiElement())(')')

tests/lang/completion-tests.kt

@@ -2,6 +2,7 @@ package org.intellij.clojure.lang
 
 import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.codeInsight.lookup.Lookup.REPLACE_SELECT_CHAR
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.testFramework.fixtures.LightPlatformCodeInsightFixtureTestCase
 
 /**
@@ -14,6 +15,11 @@ class ClojureCompletionTest : LightPlatformCodeInsightFixtureTestCase() {
     val STR_ALIAS = "(alias 'str clojure.string)"
   }
 
+  override fun setUp() {
+    super.setUp()
+    Registry.get("ide.completion.variant.limit").setValue(10000)
+  }
+
   fun testCoreNs1() = doTest("<caret>", "clojure.core", "clojure.core")
   fun testCoreNs2() = doTest(":<caret>", "clojure.core", ":clojure.core")
   fun testCoreDefn1() = doTest("(<caret>)", "defn", "(defn)")

tests/lang/language-tests.kt

@@ -74,6 +74,7 @@ class ClojureParsingTest : ClojureParsingTestCase(ClojureParserDefinition()) {
   fun testSimpleRecover2() = doCodeTest("(a))(b)")
   fun testSimpleFixes() = doCodeTest(".1 x .-;comment\n1;unclosed eof\n\"x")
   fun testMapPrefix() = doCodeTest("#:asd{:a 1 :b #::{:c 2}  #::as {} :s1 #:: {} :s2 #:a {} :s3 #: a{} :s4 #:: a{} ")
+  fun testCommentedForms() = doCodeTest("(def #_cc n 0) {#_0 #_1 :a #_'(xxx)  'a :b 'b #_2 #_3} # #_asd dbg 10")
 
   fun testParseClojureLang() = walkAndParse(::walkClojureLang)
 //  fun testParseWellKnownLibs() = walkAndParse(::walkKnownLibs)
@@ -92,7 +93,7 @@ abstract class ClojureParsingTestCase(o: ClojureParserDefinitionBase) : ParsingT
     addExplicitExtension(LanguageBraceMatching.INSTANCE, ClojureLanguage, ClojureBraceMatcher())
   }
 
-  fun walkAndParse(walker: ((Path, String) -> Unit) -> Unit): Unit {
+  fun walkAndParse(walker: ((Path, String) -> Unit) -> Unit) {
     val stat = object {
       var duration = System.currentTimeMillis()
       var files: Int = 0
@@ -107,9 +108,11 @@ abstract class ClojureParsingTestCase(o: ClojureParserDefinitionBase) : ParsingT
       for (o in SyntaxTraverser.psiTraverser(psiFile)) {
         stat.nodes++
         val elementType = o.elementType
-        val message =
-            if (elementType == TokenType.BAD_CHARACTER) "bad char '${o.text}'"
-            else if (o is PsiErrorElement) "error: ${o.errorDescription}" else null
+        val message = when {
+          elementType == TokenType.BAD_CHARACTER -> "bad char '${o.text}'"
+          o is PsiErrorElement -> "error: ${o.errorDescription}"
+          else -> null
+        }
         if (message != null) {
           stat.errors++
           println("${psiFile.name} [${o.textRange.startOffset}]: $message")
@@ -126,7 +129,7 @@ abstract class ClojureParsingTestCase(o: ClojureParserDefinitionBase) : ParsingT
 }
 
 class ClojureHighlightingTest : LightPlatformCodeInsightFixtureTestCase() {
-  override fun getTestDataPath() = TEST_DATA_PATH + "/highlighting"
+  override fun getTestDataPath() = "$TEST_DATA_PATH/highlighting"
   override fun setUp() {
     super.setUp()
     FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, myFixture.project, null)
@@ -138,15 +141,15 @@ class ClojureHighlightingTest : LightPlatformCodeInsightFixtureTestCase() {
   fun testClojureLang() = walkAndHighlight(::walkClojureLang)
   fun testClojureScript() = walkAndHighlight(::walkClojureScriptLang)
 
-  fun doTest(extension: String): Unit {
+  private fun doTest(extension: String) {
     myFixture.configureByFile(getTestName(false) + ".$extension")
     myFixture.checkHighlighting()
   }
 
 //  fun testClojureFile() = "/clojure/core.clj".let { file ->
 //    walkAndHighlight { block -> walkFs(CLJ_LIB_FS, file, block) } }
 
-  fun walkAndHighlight(walker: ((Path, String) -> Unit) -> Unit) {
+  private fun walkAndHighlight(walker: ((Path, String) -> Unit) -> Unit) {
     val stat = object {
       var duration = System.currentTimeMillis()
       var files: Int = 0