Punctuation control tags support in tokenizing and hinting.

MeltwaterArchive · Jul 31, 2013 · 4e09970 · 4e09970
1 parent 6fbe063
commit 4e09970
Show file tree

Hide file tree

Showing 10 changed files with 431 additions and 368 deletions.
diff --git a/minified/csdleditor.config.js b/minified/csdleditor.config.js
@@ -66,6 +66,12 @@ var CSDLEditorConfig = CSDLEditorConfig || {
         'return'
     ],
 
+    punctuationControl : [
+        '[keep(default)]',
+        '[keep(classic)]',
+        '[keep(extended)]'
+    ],
+
     targets : [
        "interaction.link",
        "interaction.sample",

diff --git a/minified/csdleditor.min.css b/minified/csdleditor.min.css
diff --git a/minified/csdleditor.min.js b/minified/csdleditor.min.js
diff --git a/src/csdleditor.codemirror.hint.csdl.js b/src/csdleditor.codemirror.hint.csdl.js
@@ -34,6 +34,13 @@
      */
     keywords = CSDLEditorConfig.keywords,
 
+    /**
+     * List of all available punctuation controls.
+     * 
+     * @type {Array}
+     */
+    punctuationControl = CSDLEditorConfig.punctuationControl,
+
     /**
      * String value of the current token that is being hinted.
      *
@@ -103,6 +110,17 @@
             candidates = candidates.concat(keywords);
         }
 
+        // punctuation control can only occur after "contains", "contains_any" and "contains_near" operators
+        if (previous !== undefined && previous.type === 'operator' && (
+            previous.string.toLowerCase() === 'contains'
+            || previous.string.toLowerCase() === 'contains_any'
+            || previous.string.toLowerCase() === 'continas_near'
+        )) {
+            candidates = candidates.concat(punctuationControl);
+        }
+
+        console.log(candidates);
+
         // and now filter the candidates
         for (i = 0; i < candidates.length; i++) {
             // if a substr of the candidate matches the string then autocomplete
@@ -134,7 +152,7 @@
         currentToken = token.string;
 
         // only match words and outside of comments
-        if (token.state.type !== 'comment' && token.state.type !== 'string' && /^[0-9a-zA-Z_<>=!\.]+$/.test(token.string)) {
+        if (token.state.type !== 'comment' && token.state.type !== 'string' && /^[0-9a-zA-Z_<>=!\.\(\)\[\]]+$/.test(token.string)) {
             var previousToken = CodeMirror.getPreviousToken(cm, cursor, token);
             list = getHintList(token, previousToken);
             list.sort();

diff --git a/src/csdleditor.codemirror.mode.csdl.js b/src/csdleditor.codemirror.mode.csdl.js
@@ -34,6 +34,13 @@
      */
     keywords = {},
 
+    /**
+     * Dictionary of all available CSDL punctuation controls.
+     * 
+     * @type {Object}
+     */
+    punctuationControl = {},
+
     /**
      * Default tokenizer that will try to recognize tokens.
      * 
@@ -42,8 +49,7 @@
      * @return {String|null}
      */
     tokenize = function(stream, state) {
-        var ch = stream.next(),
-            next = stream.peek();
+        var ch = stream.next();
 
         state.type = null;
 
@@ -125,6 +131,17 @@
             return 'keyword';
         }
 
+        // is it a start of punctuation control tag?
+        if (word === '[keep') {
+            stream.eatWhile(/^[^\s]/);
+            word = stream.current().toLowerCase();
+        }
+
+        // is it a punctuation control?
+        if (punctuationControl.hasOwnProperty(word)) {
+            return 'punctuation';
+        }
+
         // didn't match any token so return null
         return null;
     },
@@ -219,6 +236,7 @@
         operators = arrayToDictionary(parserConfig.operators);
         logicals = arrayToDictionary(parserConfig.logical);
         keywords = arrayToDictionary(parserConfig.keywords);
+        punctuationControl = arrayToDictionary(parserConfig.punctuationControl);
 
         function pushContext(stream, state, type) {
             state.context = {

diff --git a/src/csdleditor.config.js b/src/csdleditor.config.js
@@ -66,6 +66,12 @@ var CSDLEditorConfig = CSDLEditorConfig || {
         'return'
     ],
 
+    punctuationControl : [
+        '[keep(default)]',
+        '[keep(classic)]',
+        '[keep(extended)]'
+    ],
+
     targets : [
        "interaction.link",
        "interaction.sample",

diff --git a/src/csdleditor.css b/src/csdleditor.css
@@ -858,6 +858,9 @@ body > .csdl-popup .csdl-popup-content {
 .csdl-container.csdl-theme-light .CodeMirror .cm-operator {
   color: red;
 }
+.csdl-container.csdl-theme-light .CodeMirror .cm-punctuation {
+  color: #666;
+}
 .csdl-container.csdl-theme-light .CodeMirror .cm-bracket {
   font-weight: bold;
 }
@@ -906,6 +909,9 @@ body > .csdl-popup .csdl-popup-content {
 .csdl-container.csdl-theme-dark .CodeMirror .cm-operator {
   color: #F8F8F8;
 }
+.csdl-container.csdl-theme-dark .CodeMirror .cm-punctuation {
+  color: #bbb;
+}
 .csdl-container.csdl-theme-dark .CodeMirror .cm-bracket {
   color: #f8f8f4;
 }

diff --git a/src/csdleditor.js b/src/csdleditor.js
@@ -233,7 +233,8 @@ CSDLEditor.Loader.addComponent(function($) {
                         targets : this.config.targets,
                         operators : this.config.operators,
                         logical : this.config.logical,
-                        keywords : this.config.keywords
+                        keywords : this.config.keywords,
+                        punctuationControl : this.config.punctuationControl
                     },
                     extraKeys : {
                         'Esc' : function() {

diff --git a/src/csdleditor.less b/src/csdleditor.less
@@ -851,6 +851,10 @@ body > .csdl-popup {
             color: red;
         }
 
+        .cm-punctuation {
+            color: #666;
+        }
+
         .cm-bracket {
             font-weight: bold;
         }
@@ -922,6 +926,9 @@ body > .csdl-popup {
         .cm-operator {
             color: #F8F8F8;
         }
+        .cm-punctuation {
+            color: #bbb;
+        }
 
         .cm-bracket {
             color: #f8f8f4;

diff --git a/src/lib/codemirror.addon.closebrackets.js b/src/lib/codemirror.addon.closebrackets.js
@@ -1,5 +1,5 @@
 (function() {
-  var DEFAULT_BRACKETS = "()[]{}''\"\"";
+  var DEFAULT_BRACKETS = "(){}''\"\"";
   var SPACE_CHAR_REGEX = /\s/;
 
   CodeMirror.defineOption("autoCloseBrackets", false, function(cm, val, old) {