Added more tests for token based structured edits

src/ide-tools/src/Example/ClojureParser.java

@@ -1,7 +1,7 @@
 
 //----------------------------------------------------
 // The following code was generated by CUP v0.11a beta 20060608
-// Mon Nov 09 13:58:51 EST 2009
+// Mon Nov 09 18:24:07 EST 2009
 //----------------------------------------------------
 
 package Example;
@@ -13,7 +13,7 @@
 import clojure.lang.*;
 
 /** CUP v0.11a beta 20060608 generated parser.
-  * @version Mon Nov 09 13:58:51 EST 2009
+  * @version Mon Nov 09 18:24:07 EST 2009
   */
 public class ClojureParser extends java_cup.runtime.lr_parser {
 

src/ide-tools/src/Example/ClojureSym.java

@@ -1,7 +1,7 @@
 
 //----------------------------------------------------
 // The following code was generated by CUP v0.11a beta 20060608
-// Mon Nov 09 13:58:51 EST 2009
+// Mon Nov 09 18:24:07 EST 2009
 //----------------------------------------------------
 
 package Example;

src/ide-tools/src/org/enclojure/idetools/matchers.clj

@@ -100,7 +100,7 @@ to attempt to check/repair the stream by adding tokens where needed."
                       (if ((end-map token-key) (keyfn s)) ;...see if it matches
                         [(pop stack) [token]];...keep it and pop the stack
                         (let [i (first (end-map token-key))]
-                            [(conj stack i) [i]])) ;...else, insert the start token
+                            [stack [i token]])) ;...else, insert the start token
                                                    ; in place and push it onto the stack
                       (let [i (first (end-map token-key))]
                         [stack [i token]]));Nothing on the stack,
@@ -116,6 +116,7 @@ to attempt to check/repair the stream by adding tokens where needed."
   ([token-stream pairs]
     (fix-pairs token-stream identity pairs)))
 
+
 ;----- Helper functions to allow me to treat strings and tokens uniformly. -----
 (defmulti get-token-stream class)
 
@@ -134,9 +135,8 @@ to attempt to check/repair the stream by adding tokens where needed."
     (let [t (.next_token lexer)]
       (if (not= (.sym t) ClojureSym/EOF)
         (recur (conj tokens 
-                 (struct token-data t
-                   (.getPosition lexer)
-                   (-> lexer .yytext .length))))
+                 (assoc (.state t)
+                   :pos (.getPosition lexer))))
       tokens))))
 
 (defn
@@ -168,11 +168,12 @@ element in the token-stream and is used to equality testing."
                       (if ((end-map token-key) (keyfn s)) ;...see if it matches
                         [(pop stack) [] 0];...keep it and pop the stack
                         (let [i (first (end-map token-key))]
-                            [stack [{:insert i 
+                            [stack [{:token i
                                      :pos (+ pos insert-offset)}] (:len i)])) ;...else, insert the start token
                                                    ; in place and push it onto the stack
                       (let [i (first (end-map token-key))]
-                        [stack [{:insert i :pos pos}](:len i)]));Nothing on the stack,
+                        [stack [{:token i 
+                                 :pos (+ pos insert-offset)}] (:len i)]));Nothing on the stack,
                                                     ;so put the beginnning token
                                                     ;before the current token
                   :else [stack [] 0])] ; Not a match-pair.
@@ -182,10 +183,14 @@ element in the token-stream and is used to equality testing."
                         nstack (concat edit-ops t)))
              (if (pos? (count stack))
                 (concat edit-ops
-                  (reduce #(conj %1 {:insert (pairs %2)}) [] stack))
+                  (reduce
+                    #(conj %1 {:token (pairs %2)}) [] stack))
                edit-ops)))))
   ([token-stream pairs]
-    (get-fix-pairs-fns token-stream pairs identity)))
+    (get-fix-pairs-fns token-stream pairs identity))
+  ([token-stream]
+    (get-fix-pairs-fns token-stream *matched-pairs* identity)))
+
 
 ;(defn apply-edits
 ;  "Given a source (could be a string, document, etc.) apply each edit sequentally:

src/ide-tools/test/org/enclojure/idetools/matchers_test.clj

@@ -28,12 +28,22 @@
 
 ; For testing basic brace matching/corretion.
 (def -char-pairs-
+  {
+   \{ \} \( \) \[ \] \" \"
+  })
+
+; For testing basic brace matching/corretion.
+(def -char-token-pairs-
   {
    (make-char-token \{) (make-char-token \})
    (make-char-token \() (make-char-token \))
    (make-char-token \[) (make-char-token \])
    })
 
+(defn lex-string
+  [s]
+  (_Lexer. (StringReader. s)))
+
 (deftest matchers-test
   (testing "string patterns"
     (is (= "()" (apply str (matchers/fix-pairs "(" -char-pairs-))))
@@ -44,10 +54,13 @@
     (is (= "[]" (apply str (matchers/fix-pairs "]" -char-pairs-))))
     (is (= "{}" (apply str (matchers/fix-pairs "}" -char-pairs-))))
     (is (= "()([])" (apply str (matchers/fix-pairs ")(]" -char-pairs-))))
+    (is (= "(let [x 4] { []()})"
+          (apply str (matchers/fix-pairs "(let [x 4] { ])" -char-pairs-))))
     )
-  (testing "tokens"
-    (is (= [tokens/LEFT_PAREN tokens/RIGHT_PAREN tokens/EOF]
-            (matchers/fix-pairs "()" matchers/*matched-pairs*)))
+  (testing "lexer/token patterns"
+    (is (= (list tokens/RIGHT_PAREN)
+            (map :token (matchers/get-fix-pairs-fns
+                          (lex-string "(") matchers/*matched-pairs*))))
     )
   )