add test for call tokenize() then untokenize()

dhowe · Sep 14, 2020 · eb09256 · eb09256
1 parent 040dbc5
commit eb09256
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 1 deletion.
diff --git a/src/tokenizer.js b/src/tokenizer.js
@@ -137,7 +137,7 @@ class Tokenizer {
         result += arr[i];
         continue;
       } else if (arr[i] === "." && lastIsWWW){
-        console.log('yes');
+        //console.log('yes');
         nextNoSpace = true;
       } else if (thisLBracket) {
         result += delim;

diff --git a/test/tokenizer-tests.js b/test/tokenizer-tests.js
@@ -5,6 +5,24 @@ describe('RiTa.Tokenizer', () => {
 
   if (typeof module !== 'undefined') require('./before');
 
+  it ('Calling tokenize then untokenize', () =>{
+    let sentences = [
+      "this is www.google.com",
+      "it is 'hell'"
+    ];
+    let tokens = [
+      ["this", "is", "www", ".", "google", ".", "com"],
+      ["it", "is", "'", "hell", "'"]
+    ];
+    for (i = 0; i < sentences.length; i++){
+      let usingTokenize = RiTa.tokenize(sentences[i]);
+      expect(usingTokenize).eql(tokens[i]);
+      let usingUntokenize = RiTa.untokenize(usingTokenize);
+      expect(usingUntokenize).eq(sentences[i]);
+    };
+
+  });
+
   it('Should correctly call tokenize', () => {
 
     expect(RiTa.tokenize("")).eql([""]);