SyntaxHighlighter: add parser to tokenize string literals and comments,

fixes #746

demo/src/main/scripts/args.nanorc

@@ -1,12 +1,13 @@
 syntax "ARGS"
 
-NUMBER:    "\<[-]?[0-9]*([Ee][+-]?[0-9]+)?\>"  "\<[-]?[0](\.[0-9]+)?\>"
-STRING:    ""(\\.|[^"])*"|'(\\.|[^'])*'|[a-zA-Z]+[a-zA-Z0-9]*"
-FUNCTION:  "\<(console|grab|inspect)\>"
-NULL:      "\<null\>"
-BOOLEAN:   "\<(true|false)\>"
-VARIABLE:  "\"(\\"|[^"])*\"\s*:"  "'(\'|[^'])*'\s*:" "(\[|,)\s*[a-zA-Z0-9]*\s*:"
-PLAIN:     "(:|\[|,|\])"
-ESCAPE:    "\\u[0-9a-fA-F]{4}|\\[bfnrt'"/\\]"
-COMMENT:   start="/\*" end="\*/"
-COMMENT:   "(//.*)"
+NUMBER:         "\<[-]?[0-9]*([Ee][+-]?[0-9]+)?\>"  "\<[-]?[0](\.[0-9]+)?\>"
+STRING:         "[a-zA-Z]+[a-zA-Z0-9]*"
+FUNCTION:       "\<(console|grab|inspect)\>"
+NULL:           "\<null\>"
+BOOLEAN:        "\<(true|false)\>"
+VARIABLE:       "(\[|,)\s*[a-zA-Z0-9]*\s*:"
+PLAIN:          "(:|\[|,|\])"
+ESCAPE:         "\\u[0-9a-fA-F]{4}|\\[bfnrt'"/\\]"
+$BLOCK_COMMENT: "/*, */"
+$LINE_COMMENT:  "//"
+$BALANCED_DELIMITERS  "", '"

demo/src/main/scripts/command.nanorc

@@ -1,7 +1,7 @@
 syntax "COMMAND"
 
-FUNCTION:  "[a-zA-Z]+[a-zA-Z0-9]*"
-VARIABLE: ".*="
-PLAIN:    "(\"|'|\.|=|:|\[|,|\])"
-COMMENT:  start="/\*" end="\*/"
-COMMENT:  "(^|[[:space:]])#.*$"
+FUNCTION:       "[a-zA-Z]+[a-zA-Z0-9]*"
+VARIABLE:       ".*="
+PLAIN:          "(\"|'|\.|=|:|\[|,|\])"
+COMMENT:        "(^|\\s+)#.*$"
+$BLOCK_COMMENT: "/*, */"

demo/src/main/scripts/default.nanorctheme

@@ -60,4 +60,10 @@ WHITESPACE  ,green
 +FUNCTION           FUNCTION: "[A-Za-z_][A-Za-z0-9_]*[[:space:]]*[(]" \n PLAIN:    "[(]"
 +TODO               color brightwhite,cyan "FIXME|TODO|XXX"
 +LINT               color ,green "[[:space:]]+$" \n color ,red "\t*"
-+LONG_LINE_WARNING  color ,red "^.{81,}$"
++LONG_LINE_WARNING  color ,red "^.{81,}$"
+#
+# parser
+#
+$LINE_COMMENT           COMMENT \n color brightwhite,cyan "FIXME|TODO|XXX"
+$BLOCK_COMMENT          COMMENT \n DOC_COMMENT: startWith=/** \n color brightwhite,cyan "FIXME|TODO|XXX"
+$BALANCED_DELIMITERS    STRING  \n VARIABLE: continueAs=\s*:

demo/src/main/scripts/gron.nanorc

@@ -2,9 +2,10 @@ syntax "GRON" "\.gron$"
 header "^\[$"
 
 NUMBER:     "\<[-]?[0-9]*([Ee][+-]?[0-9]+)?\>"  "\<[-]?[0](\.[0-9]+)?\>"
-STRING:     ""(\\.|[^"])*"|'(\\.|[^'])*'|[a-zA-Z]+[a-zA-Z0-9]*"
+STRING:     "[a-zA-Z]+[a-zA-Z0-9]*"
 NULL:       "\<null\>"
 BOOLEAN:    "\<(true|false)\>"
-VARIABLE:   "\"(\\"|[^"])*\"\s*:"  "'(\'|[^'])*'\s*:" "(\[|,)\s*[a-zA-Z0-9]*\s*:"
+VARIABLE:   "(\[|,)\s*[a-zA-Z0-9]*\s*:"
 PLAIN:      "(:|\[|,|\])"
 ESCAPE:     "\\u[0-9a-fA-F]{4}|\\[bfnrt'"/\\]"
+$BALANCED_DELIMITERS  "", '"

demo/src/main/scripts/groovy.nanorc

@@ -9,20 +9,11 @@ CLASS:       "\<[A-Z]{0,2}([A-Z]{1}[a-z]+){1,}\>"
 FUNCTION:    "\<(print|println|sleep)\>"
 KEYWORD:     "\<(abstract|class|extends|final|implements|import|instanceof|interface|native|package|private|protected|public|static|strictfp|super|synchronized|throws|volatile)\>"
 JUMP:        "\<(break|continue)\>"
-# Mono-quoted strings.
-STRING:      "'([^'\\]|\\.)*'|'''"
-STRING:      ""([^"\\]|\\.)*"|""""
-color normal "'''|""""
-# Triple-quoted strings.
-STRING:      start="'''([^'),]|$)" end="(^|[^(\\])'''"
-STRING:      start=""""([^"),]|$)" end="(^|[^(\\])""""
-#
 NULL:        "\<(null)\>"
 BOOLEAN:     "\<(true|false)\>"
 CONSTANT:    "\<[A-Z]+([_]{1}[A-Z]+){0,}\>"
 OPERATOR:    "[-+/*=<>?:!~%&|]"
 ~NUMBER:      "\b(([1-9][0-9]+)|0+)\.[0-9]+\b" "\b[1-9][0-9]*\b" "\b0[0-7]*\b" "\b0x[1-9a-f][0-9a-f]*\b"
-COMMENT:     "//.*"
-COMMENT:     start="/\*" end="\*/"
-DOC_COMMENT: start="/\*\*" end="\*/"
-+TODO
+$BALANCED_DELIMITERS:  "", ', ''', """"
+$LINE_COMMENT:         "//"
+$BLOCK_COMMENT:        "/*, */"

demo/src/main/scripts/java.nanorc

@@ -15,14 +15,12 @@ PACKAGE:     "(([a-z]{2,}[.]{1}){2,10}([a-z]{2,}){0,1})"
 CLASS:       "\<[A-Z]{0,2}([A-Z]{1}[a-z]+){1,}\>"
 KEYWORD:     "\<(abstract|class|extends|final|implements|import|instanceof|interface|native|package|private|protected|public|static|strictfp|super|synchronized|throws|volatile)\>"
 JUMP:        "\<(break|continue)\>"
-STRING:      ""[^"]*""
 NULL:        "\<(null)\>"
 BOOLEAN:     "\<(true|false)\>"
 CONSTANT:    "\<[A-Z]+([_]{1}[A-Z]+){0,}\>"
 OPERATOR:    "[-+/*=<>?:!~%&|]"
 ~NUMBER:     "\b(([1-9][0-9]+)|0+)\.[0-9]+\b" "\b[1-9][0-9]*\b" "\b0[0-7]*\b" "\b0x[1-9a-f][0-9a-f]*\b"
-COMMENT:     "//.*"
-COMMENT:     start="/\*" end="\*/"
-DOC_COMMENT: start="/\*\*" end="\*/"
-+TODO
+$BALANCED_DELIMITERS:  """
+$LINE_COMMENT:         "//"
+$BLOCK_COMMENT:        "/*, */"
 +LINT

demo/src/main/scripts/json.nanorc

@@ -4,6 +4,5 @@ header "^\{$"
 NUMBER:   "\<[-]?[0-9]*([Ee][+-]?[0-9]+)?\>"  "\<[-]?[0](\.[0-9]+)?\>"
 NULL:     "\<null\>"
 BOOLEAN:  "\<(true|false)\>"
-STRING:   ""(\\.|[^"])*"|'(\\.|[^'])*'"
-VARIABLE: "\"(\\"|[^"])*\"[[:space:]]*:"  "'(\'|[^'])*'[[:space:]]*:"
 ESCAPE:   "\\u[0-9a-fA-F]{4}|\\[bfnrt'"/\\]"
+$BALANCED_DELIMITERS  "", '"

demo/src/main/scripts/sh-repl.nanorc

@@ -13,6 +13,6 @@ FUNCTION: "\<(alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|com
 FUNCTION: "\<(cat|chmod|chown|cp|env|grep|install|ln|make|mkdir|mv|rm|sed|tar|touch|ls)\>"
 ~FUNCTION: "^\s+[0-9A-Z_]+\s+\(\)"
 ~VARIABLE: "\$\{?[0-9A-Z_!@#$*?-]+\}?"
-STRING:   ""(\\.|[^"])*"" "'(\\.|[^'])*'"
 COMMENT:  "^([[:space:]])*#.*$"
-COMMENT:  start="/\*" end="\*/"
+$BLOCK_COMMENT:       "/*, */"
+$BALANCED_DELIMITERS: "", '"