Skip to content
This repository
Browse code

Full-text search, oh my

  • Loading branch information...
commit f8d51b8bccbb275927e4408d86869a5218153bf2 1 parent e0852d9
Chris O'Hara authored May 15, 2011
11  README.md
Source Rendered
@@ -17,6 +17,7 @@ It also comes with the following advanced data structures:
17 17
 - **KeyPair** - Uses two hash structures and an auto-incrementing key to assign an ID to each unique value
18 18
 - **SocialGraph** - Similar to Twitter's (following vs. followers)
19 19
 - **CappedList** - A list with a fixed length
  20
+- **FullText** - A full text index with support for stop words, stemming and basic boolean search
20 21
 
21 22
 *More coming soon!*
22 23
 
@@ -25,16 +26,18 @@ It also comes with the following advanced data structures:
25 26
 ```javascript
26 27
 var redback = require('redback').createClient();
27 28
 
28  
-//redback.create<Structure>(key)
  29
+var user3 = redback.createSocialGraph(3);
  30
+user3.follow(1, callback);
  31
+
  32
+var text = redback.createFullText('my_index');
  33
+text.indexFile({1: 'file1.txt, 2: 'file2.txt}, callback);
  34
+text.search('foo bar -exclude -these -words', callback);
29 35
 
30 36
 var user1 = redback.createHash('user1');
31 37
 user.set({username:'chris', password:'redisisawesome'}, callback);
32 38
 
33 39
 var log = redback.createCappedList('log', 1000);
34 40
 log.push('Log message ...');
35  
-
36  
-var user3 = redback.createSocialGraph(3);
37  
-user3.follow(1, callback);
38 41
 ```
39 42
 
40 43
 ## Creating your own structures
3  lib/Redback.js
@@ -23,7 +23,8 @@ var base = ['Hash','List','Set','SortedSet','Bitfield'];
23 23
  * Define the available advanced structures.
24 24
  */
25 25
 
26  
-var advanced = ['KeyPair','DensitySet','CappedList','SocialGraph'];
  26
+var advanced = ['KeyPair','DensitySet','CappedList','SocialGraph',
  27
+                'FullText'];
27 28
 
28 29
 /**
29 30
  * The Redback object wraps the Redis client and acts as a factory
353  lib/advanced_structures/FullText.js
... ...
@@ -0,0 +1,353 @@
  1
+/*!
  2
+ * Redback
  3
+ * Copyright(c) 2011 Chris O'Hara <cohara87@gmail.com>
  4
+ * MIT Licensed
  5
+ */
  6
+
  7
+/**
  8
+ * Module dependencies.
  9
+ */
  10
+
  11
+var Structure = require('../Structure'),
  12
+    EventEmitter = require('events').EventEmitter,
  13
+    fs = require('fs');
  14
+
  15
+/**
  16
+ * Stop words - words that shouldn't be indexed.
  17
+ */
  18
+
  19
+var stopwords = {
  20
+    'a':1,'able':1,'about':1,'across':1,'after':1,'all':1,'almost':1,'also':1,'am':1,'among':1,
  21
+    'an':1,'and':1,'any':1,'are':1,'as':1,'at':1,'be':1,'because':1,'been':1,'but':1,'by':1,'can':1,
  22
+    'cannot':1,'could':1,'dear':1,'did':1,'do':1,'does':1,'either':1,'else':1,'ever':1,'every':1,
  23
+    'for':1,'from':1,'get':1,'got':1,'had':1,'has':1,'have':1,'he':1,'her':1,'hers':1,'him':1,'his':1,
  24
+    'how':1,'however':1,'i':1,'if':1,'in':1,'into':1,'is':1,'it':1,'its':1,'just':1,'least':1,'let':1,
  25
+    'like':1,'likely':1,'may':1,'me':1,'might':1,'most':1,'must':1,'my':1,'neither':1,'no':1,
  26
+    'nor':1,'not':1,'of':1,'off':1,'often':1,'on':1,'only':1,'or':1,'other':1,'our':1,'own':1,
  27
+    'rather':1,'said':1,'say':1,'says':1,'she':1,'should':1,'since':1,'so':1,'some':1,'than':1,
  28
+    'that':1,'the':1,'their':1,'them':1,'then':1,'there':1,'these':1,'they':1,'this':1,'tis':1,
  29
+    'to':1,'too':1,'twas':1,'us':1,'wants':1,'was':1,'we':1,'were':1,'what':1,'when':1,'where':1,
  30
+    'which':1,'while':1,'who':1,'whom':1,'why':1,'will':1,'with':1,'would':1,'yet':1,'you':1,
  31
+    'your':1,'':1
  32
+}
  33
+
  34
+/**
  35
+ * A full text index with support for stop words, stemming and
  36
+ * a basic boolean search syntax.
  37
+ *
  38
+ * Usage:
  39
+ *    `redback.createFullText(key);`
  40
+ *
  41
+ * Reference:
  42
+ *    http://redis.io/topics/data-types#sets
  43
+ *
  44
+ * Redis Structure:
  45
+ *    `(namespace:)key:<word1> = set(docs)`
  46
+ *    `(namespace:)key:<word2> = set(docs)`
  47
+ *    `(namespace:)key:<wordN> = set(docs)`
  48
+ */
  49
+
  50
+var FullText = exports.FullText = Structure.new();
  51
+
  52
+/**
  53
+ * Initialise the index. libstemmer bindings are required.
  54
+ *
  55
+ * @api private
  56
+ */
  57
+
  58
+FullText.prototype.init = function () {
  59
+    this.indexed_bytes = 0;
  60
+    try {
  61
+        this.stem = require('stem').stem;
  62
+    } catch (e) {
  63
+        console.err('Full text requires the libstemmer bindings: `npm install -g stem`');
  64
+        process.exit(1);
  65
+    }
  66
+}
  67
+
  68
+/**
  69
+ * Index one or more documents.
  70
+ *
  71
+ * Index One Document:
  72
+ *    `text.indexFile(1, 'document string ...', callback);`
  73
+ *
  74
+ * Index Many Documents:
  75
+ *    `text.indexFile({1:'docstr1', 2:'docstr2'}, callback);`
  76
+ *
  77
+ * @param {int|string} id - the document's unique identifier
  78
+ * @param {string|ReadableStream|Buffer} document
  79
+ * @param {Function} callback (optional)
  80
+ * @return this
  81
+ * @api public
  82
+ */
  83
+
  84
+FullText.prototype.index = function (id, document, callback) {
  85
+    if (typeof id === 'object') {
  86
+        return this.indexAll(id, document);
  87
+    } else if (document instanceof EventEmitter &&
  88
+            typeof document.readable !== 'undefined' && document.readable === true) {
  89
+        return indexStream(id, document, callback);
  90
+    } else if (document instanceof Buffer) {
  91
+        document = document.toString();
  92
+    }
  93
+    this.indexed_bytes += document.length;
  94
+    var stemmed = this.stemWords(this.extractWords(document));
  95
+    this.buildIndex(id, stemmed, callback || function () {});
  96
+}
  97
+
  98
+/**
  99
+ * Index multiple documents.
  100
+ *
  101
+ * @param {Array} documents
  102
+ * @param {Function} callback (optional)
  103
+ * @return this
  104
+ * @api private
  105
+ */
  106
+
  107
+FullText.prototype.indexAll = function (documents, callback) {
  108
+    var self = this, ids = Object.keys(documents),
  109
+        failed = false, remaining = ids.length;
  110
+    ids.forEach(function (id) {
  111
+        self.index(id, documents[id], function (err) {
  112
+            if (failed) {
  113
+                return;
  114
+            } else if (err) {
  115
+                return callback(err);
  116
+            } else {
  117
+                if (!--remaining) {
  118
+                    callback();
  119
+                }
  120
+            }
  121
+        });
  122
+    });
  123
+    return this;
  124
+}
  125
+
  126
+/**
  127
+ * Index one or more files.
  128
+ *
  129
+ * Index One Document:
  130
+ *    `text.indexFile(1, '/path/to/file', callback);`
  131
+ *
  132
+ * Index Many Documents:
  133
+ *    `text.indexFile({1:'file1', 2:'file2'}, callback);`
  134
+ *
  135
+ * @param {int|string|Object} id
  136
+ * @param {string} filename (optional)
  137
+ * @param {Function} callback (optional)
  138
+ * @return this
  139
+ * @api public
  140
+ */
  141
+
  142
+FullText.prototype.indexFile =
  143
+FullText.prototype.indexFiles = function (id, filename, callback) {
  144
+    if (typeof id === 'object') {
  145
+        callback = filename;
  146
+        var self = this, files = id, ids = Object.keys(files),
  147
+            failed = false, remaining = ids.length;
  148
+        ids.forEach(function (id) {
  149
+            self.indexStream(id, fs.createReadStream(files[id]), function (err) {
  150
+                if (failed) {
  151
+                    return;
  152
+                } else if (err) {
  153
+                    return callback(err);
  154
+                } else {
  155
+                    if (!--remaining) callback();
  156
+                }
  157
+            });
  158
+        });
  159
+        return this;
  160
+    } else {
  161
+        return this.indexStream(id, fs.createReadStream(filename), callback);
  162
+    }
  163
+}
  164
+
  165
+/**
  166
+ * Split a string into an array of words. Also strip punctuation and
  167
+ * lowercase all words.
  168
+ *
  169
+ * @param {string} str
  170
+ * @return {Array} words
  171
+ * @api private
  172
+ */
  173
+
  174
+FullText.prototype.extractWords = function (str) {
  175
+    return str.toLowerCase()
  176
+              .replace(/[^a-zA-Z0-9'\s\r\n\t-]/g, '')
  177
+              .split(/[\s\r\n\t]/);
  178
+}
  179
+
  180
+/**
  181
+ * Given an array of words, remove stop words and stem the remaining.
  182
+ *
  183
+ * @param {Array} words
  184
+ * @param {Array} stemmed (optional) - the array to add to
  185
+ * @return {Array} stemmed_words
  186
+ * @api private
  187
+ */
  188
+
  189
+FullText.prototype.stemWords = function (words, stemmed) {
  190
+    stemmed = stemmed || [];
  191
+    for (var i = 0, l = words.length; i < l; i++) {
  192
+        if (typeof stopwords[words[i]] === 'undefined') {
  193
+            stemmed.push(this.stem(words[i]));
  194
+        }
  195
+    }
  196
+    return stemmed;
  197
+}
  198
+
  199
+/**
  200
+ * Index a readable stream.
  201
+ *
  202
+ * @param {int} id
  203
+ * @param {string} document
  204
+ * @param {Function} callback (optional)
  205
+ * @api private
  206
+ */
  207
+
  208
+FullText.prototype.indexStream = function (id, stream, callback) {
  209
+    var self = this, indexChunk, words, stemmed = [], i, l, last = '';
  210
+    indexChunk = function (chunk, end) {
  211
+        words = self.extractWords(chunk);
  212
+        if (!end) last += words.pop();
  213
+        self.stemWords(words, stemmed);
  214
+    }
  215
+    stream.setEncoding('utf8');
  216
+    stream.on('data', function (chunk) {
  217
+        self.indexed_bytes += chunk.length;
  218
+        indexChunk(last + chunk);
  219
+    });
  220
+    stream.on('end', function () {
  221
+        indexChunk(last, true);
  222
+        self.buildIndex(id, stemmed, callback || function () {});
  223
+    });
  224
+}
  225
+
  226
+/**
  227
+ * Builds the reverse index of a document.
  228
+ *
  229
+ * @param {int|string} id
  230
+ * @param {Array} words
  231
+ * @param {Function} callback
  232
+ * @api private
  233
+ */
  234
+
  235
+FullText.prototype.buildIndex = function (id, words, callback) {
  236
+    words.push('__documents');
  237
+    var self = this,
  238
+        remaining = words.length,
  239
+        failed = false,
  240
+        word_count = 0;
  241
+    words.forEach(function (word) {
  242
+        self.client.sadd(self.key + ':' + word, id, function (err, added) {
  243
+            if (failed) {
  244
+                return;
  245
+            } else if (err) {
  246
+                failed = true;
  247
+                return callback(err);
  248
+            } else {
  249
+                if (added) word_count++;
  250
+                if (!--remaining) callback(null, word_count);
  251
+            }
  252
+        });
  253
+    });
  254
+    return this;
  255
+}
  256
+
  257
+/**
  258
+ * Search the full text index. Words will be extracted from the
  259
+ * search string and used to filter search results. To exclude certain
  260
+ * words, prefix them with a hyphen "-".
  261
+ *
  262
+ * Basic Search:
  263
+ *    `index.search('foo bar', callback);`
  264
+ *
  265
+ * Excluding Words:
  266
+ *    `index.search('foo -bar -cool', callback);`
  267
+ *
  268
+ * @param {string} search
  269
+ * @param {Function} callback
  270
+ * @api public
  271
+ */
  272
+
  273
+FullText.prototype.search = function (search, callback) {
  274
+    var include = [], exclude = [];
  275
+    this.stemWords(this.extractWords(search)).forEach(function (word) {
  276
+        if (word[0] === '-') {
  277
+            exclude.push(word.substr(1));
  278
+        } else {
  279
+            include.push(word);
  280
+        }
  281
+    });
  282
+    return this._search(include, exclude, callback);
  283
+}
  284
+
  285
+/**
  286
+ * Execute a search based on two arrays: a list of stemmed words to include,
  287
+ * and a list of stemmed words to exclude.
  288
+ *
  289
+ * @param {Array} include
  290
+ * @param {Array} exclude
  291
+ * @param {Function} callback
  292
+ * @api private
  293
+ */
  294
+
  295
+FullText.prototype._search = function (include, exclude, callback) {
  296
+    if (include.length === 0) {
  297
+        include = ['__documents']; //A set containing all doc IDs
  298
+    }
  299
+    var multi = this.client.multi(), i, l, result_offset = 0;
  300
+    for (i = 0, l = include.length; i < l; i++) {
  301
+        include[i] = this.key + ':' + include[i];
  302
+    }
  303
+    l = exclude.length;
  304
+    if (l === 0) {
  305
+        multi.sinter.apply(multi, include);
  306
+    } else {
  307
+        var tmp_key = Math.random();
  308
+        include.unshift(tmp_key);
  309
+        multi.sinterstore.apply(multi, include);
  310
+        for (i = 0; i < l; i++) {
  311
+            exclude[i] = this.key + ':' + exclude[i];
  312
+        }
  313
+        exclude.unshift(tmp_key);
  314
+        multi.sdiff.apply(multi, exclude);
  315
+        multi.del(tmp_key);
  316
+        result_offset = 1;
  317
+    }
  318
+    multi.exec(function (err, results) {
  319
+        if (err) return callback(err, null);
  320
+        return callback(null, results[result_offset]);
  321
+    });
  322
+    return this;
  323
+}
  324
+
  325
+/**
  326
+ * Clear the index.
  327
+ *
  328
+ * @param {Function} callback (optional)
  329
+ * @api public
  330
+ */
  331
+
  332
+FullText.prototype.clear = function (callback) {
  333
+    var self = this;
  334
+    callback = callback || function () {};
  335
+    this.client.keys(this.key + ':*', function (err, keys) {
  336
+        if (err) return callback(err, null);
  337
+        var rem_count = 0, failed = false, remaining = keys.length;
  338
+        keys.forEach(function (key) {
  339
+            self.client.del(key, function (err, removed) {
  340
+                if (failed) {
  341
+                    return;
  342
+                } else if (err) {
  343
+                    failed = true;
  344
+                    return callback(err);
  345
+                } else {
  346
+                    if (removed) rem_count++;
  347
+                    if (!--remaining) callback(null, rem_count);
  348
+                }
  349
+            });
  350
+        });
  351
+    });
  352
+    return this;
  353
+}
2  package.json
... ...
@@ -1,6 +1,6 @@
1 1
 { "name"          : "redback",
2 2
   "description"   : "A high-level Redis library",
3  
-  "version"       : "0.1.5",
  3
+  "version"       : "0.1.6",
4 4
   "homepage"      : "https://github.com/chriso/redback",
5 5
   "author"        : "Chris O'Hara <cohara87@gmail.com>",
6 6
   "main"          : "index",
237  site/api.html
@@ -135,7 +135,8 @@
135 135
  </p>
136 136
 </td>
137 137
 <td class="code">
138  
-<pre><code><span class="keyword">var</span> <span class="variable">advanced</span> = [<span class="string">'KeyPair'</span>,<span class="string">'DensitySet'</span>,<span class="string">'CappedList'</span>,<span class="string">'SocialGraph'</span>];</code></pre>
  138
+<pre><code><span class="keyword">var</span> <span class="variable">advanced</span> = [<span class="string">'KeyPair'</span>,<span class="string">'DensitySet'</span>,<span class="string">'CappedList'</span>,<span class="string">'SocialGraph'</span>,
  139
+                <span class="string">'FullText'</span>];</code></pre>
139 140
 </td>
140 141
 </tr>
141 142
 <tr class="code">
@@ -350,6 +351,24 @@
350 351
 </tr>
351 352
 <tr class="code">
352 353
 <td class="docs">
  354
+<p>Checks whether the structure has an expiry.</p>
  355
+
  356
+<h2></h2>
  357
+
  358
+<ul><li><p><strong>param</strong>: <em>Function</em>  callback</p></li><li><p><strong>return</strong>: <em>this</em></p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  359
+</td>
  360
+<td class="code">
  361
+<pre><code><span class="class">Structure</span>.<span class="variable">prototype</span>.<span class="variable">isVolatile</span> = <span class="keyword">function</span> (<span class="variable">callback</span>) {
  362
+    <span class="this">this</span>.<span class="variable">client</span>.<span class="variable">ttl</span>(<span class="this">this</span>.<span class="variable">key</span>, <span class="keyword">function</span> (<span class="variable">err</span>, <span class="variable">ttl</span>) {
  363
+        <span class="keyword">if</span> (<span class="variable">err</span>) <span class="keyword">return</span> <span class="variable">callback</span>(<span class="variable">err</span>, <span class="keyword">null</span>);
  364
+        <span class="variable">callback</span>(<span class="keyword">null</span>, <span class="variable">ttl</span> != -<span class="number integer">1</span>);
  365
+    });
  366
+    <span class="keyword">return</span> <span class="this">this</span>;
  367
+}</code></pre>
  368
+</td>
  369
+</tr>
  370
+<tr class="code">
  371
+<td class="docs">
353 372
 <p>Remove the structure's associated expiry.</p>
354 373
 
355 374
 <h2></h2>
@@ -2066,6 +2085,201 @@
2066 2085
     <span class="keyword">return</span> <span class="this">this</span>;
2067 2086
 }</code></pre>
2068 2087
 </td>
  2088
+</tr><tr class="filename"><td><h2 id="lib/advanced_structures/FullText.js"><a href="#">FullText</a></h2></td><td>lib/advanced_structures/FullText.js</td></tr><tr class="code">
  2089
+<td class="docs">
  2090
+<p>Module dependencies.
  2091
+ </p>
  2092
+</td>
  2093
+<td class="code">
  2094
+<pre><code><span class="keyword">var</span> <span class="class">Structure</span> = <span class="variable">require</span>(<span class="string">'../Structure'</span>),
  2095
+    <span class="class">EventEmitter</span> = <span class="variable">require</span>(<span class="string">'events'</span>).<span class="class">EventEmitter</span>,
  2096
+    <span class="variable">fs</span> = <span class="variable">require</span>(<span class="string">'fs'</span>);</code></pre>
  2097
+</td>
  2098
+</tr>
  2099
+<tr class="code">
  2100
+<td class="docs">
  2101
+<p>Stop words - words that shouldn't be indexed.
  2102
+ </p>
  2103
+</td>
  2104
+<td class="code">
  2105
+<pre><code><span class="keyword">var</span> <span class="variable">stopwords</span> = {
  2106
+    <span class="string">'a'</span>:<span class="number integer">1</span>,<span class="string">'able'</span>:<span class="number integer">1</span>,<span class="string">'about'</span>:<span class="number integer">1</span>,<span class="string">'across'</span>:<span class="number integer">1</span>,<span class="string">'after'</span>:<span class="number integer">1</span>,<span class="string">'all'</span>:<span class="number integer">1</span>,<span class="string">'almost'</span>:<span class="number integer">1</span>,<span class="string">'also'</span>:<span class="number integer">1</span>,<span class="string">'am'</span>:<span class="number integer">1</span>,<span class="string">'among'</span>:<span class="number integer">1</span>,
  2107
+    <span class="string">'an'</span>:<span class="number integer">1</span>,<span class="string">'and'</span>:<span class="number integer">1</span>,<span class="string">'any'</span>:<span class="number integer">1</span>,<span class="string">'are'</span>:<span class="number integer">1</span>,<span class="string">'as'</span>:<span class="number integer">1</span>,<span class="string">'at'</span>:<span class="number integer">1</span>,<span class="string">'be'</span>:<span class="number integer">1</span>,<span class="string">'because'</span>:<span class="number integer">1</span>,<span class="string">'been'</span>:<span class="number integer">1</span>,<span class="string">'but'</span>:<span class="number integer">1</span>,<span class="string">'by'</span>:<span class="number integer">1</span>,<span class="string">'can'</span>:<span class="number integer">1</span>,
  2108
+    <span class="string">'cannot'</span>:<span class="number integer">1</span>,<span class="string">'could'</span>:<span class="number integer">1</span>,<span class="string">'dear'</span>:<span class="number integer">1</span>,<span class="string">'did'</span>:<span class="number integer">1</span>,<span class="string">'do'</span>:<span class="number integer">1</span>,<span class="string">'does'</span>:<span class="number integer">1</span>,<span class="string">'either'</span>:<span class="number integer">1</span>,<span class="string">'else'</span>:<span class="number integer">1</span>,<span class="string">'ever'</span>:<span class="number integer">1</span>,<span class="string">'every'</span>:<span class="number integer">1</span>,
  2109
+    <span class="string">'for'</span>:<span class="number integer">1</span>,<span class="string">'from'</span>:<span class="number integer">1</span>,<span class="string">'get'</span>:<span class="number integer">1</span>,<span class="string">'got'</span>:<span class="number integer">1</span>,<span class="string">'had'</span>:<span class="number integer">1</span>,<span class="string">'has'</span>:<span class="number integer">1</span>,<span class="string">'have'</span>:<span class="number integer">1</span>,<span class="string">'he'</span>:<span class="number integer">1</span>,<span class="string">'her'</span>:<span class="number integer">1</span>,<span class="string">'hers'</span>:<span class="number integer">1</span>,<span class="string">'him'</span>:<span class="number integer">1</span>,<span class="string">'his'</span>:<span class="number integer">1</span>,
  2110
+    <span class="string">'how'</span>:<span class="number integer">1</span>,<span class="string">'however'</span>:<span class="number integer">1</span>,<span class="string">'i'</span>:<span class="number integer">1</span>,<span class="string">'if'</span>:<span class="number integer">1</span>,<span class="string">'in'</span>:<span class="number integer">1</span>,<span class="string">'into'</span>:<span class="number integer">1</span>,<span class="string">'is'</span>:<span class="number integer">1</span>,<span class="string">'it'</span>:<span class="number integer">1</span>,<span class="string">'its'</span>:<span class="number integer">1</span>,<span class="string">'just'</span>:<span class="number integer">1</span>,<span class="string">'least'</span>:<span class="number integer">1</span>,<span class="string">'let'</span>:<span class="number integer">1</span>,
  2111
+    <span class="string">'like'</span>:<span class="number integer">1</span>,<span class="string">'likely'</span>:<span class="number integer">1</span>,<span class="string">'may'</span>:<span class="number integer">1</span>,<span class="string">'me'</span>:<span class="number integer">1</span>,<span class="string">'might'</span>:<span class="number integer">1</span>,<span class="string">'most'</span>:<span class="number integer">1</span>,<span class="string">'must'</span>:<span class="number integer">1</span>,<span class="string">'my'</span>:<span class="number integer">1</span>,<span class="string">'neither'</span>:<span class="number integer">1</span>,<span class="string">'no'</span>:<span class="number integer">1</span>,
  2112
+    <span class="string">'nor'</span>:<span class="number integer">1</span>,<span class="string">'not'</span>:<span class="number integer">1</span>,<span class="string">'of'</span>:<span class="number integer">1</span>,<span class="string">'off'</span>:<span class="number integer">1</span>,<span class="string">'often'</span>:<span class="number integer">1</span>,<span class="string">'on'</span>:<span class="number integer">1</span>,<span class="string">'only'</span>:<span class="number integer">1</span>,<span class="string">'or'</span>:<span class="number integer">1</span>,<span class="string">'other'</span>:<span class="number integer">1</span>,<span class="string">'our'</span>:<span class="number integer">1</span>,<span class="string">'own'</span>:<span class="number integer">1</span>,
  2113
+    <span class="string">'rather'</span>:<span class="number integer">1</span>,<span class="string">'said'</span>:<span class="number integer">1</span>,<span class="string">'say'</span>:<span class="number integer">1</span>,<span class="string">'says'</span>:<span class="number integer">1</span>,<span class="string">'she'</span>:<span class="number integer">1</span>,<span class="string">'should'</span>:<span class="number integer">1</span>,<span class="string">'since'</span>:<span class="number integer">1</span>,<span class="string">'so'</span>:<span class="number integer">1</span>,<span class="string">'some'</span>:<span class="number integer">1</span>,<span class="string">'than'</span>:<span class="number integer">1</span>,
  2114
+    <span class="string">'that'</span>:<span class="number integer">1</span>,<span class="string">'the'</span>:<span class="number integer">1</span>,<span class="string">'their'</span>:<span class="number integer">1</span>,<span class="string">'them'</span>:<span class="number integer">1</span>,<span class="string">'then'</span>:<span class="number integer">1</span>,<span class="string">'there'</span>:<span class="number integer">1</span>,<span class="string">'these'</span>:<span class="number integer">1</span>,<span class="string">'they'</span>:<span class="number integer">1</span>,<span class="string">'this'</span>:<span class="number integer">1</span>,<span class="string">'tis'</span>:<span class="number integer">1</span>,
  2115
+    <span class="string">'to'</span>:<span class="number integer">1</span>,<span class="string">'too'</span>:<span class="number integer">1</span>,<span class="string">'twas'</span>:<span class="number integer">1</span>,<span class="string">'us'</span>:<span class="number integer">1</span>,<span class="string">'wants'</span>:<span class="number integer">1</span>,<span class="string">'was'</span>:<span class="number integer">1</span>,<span class="string">'we'</span>:<span class="number integer">1</span>,<span class="string">'were'</span>:<span class="number integer">1</span>,<span class="string">'what'</span>:<span class="number integer">1</span>,<span class="string">'when'</span>:<span class="number integer">1</span>,<span class="string">'where'</span>:<span class="number integer">1</span>,
  2116
+    <span class="string">'which'</span>:<span class="number integer">1</span>,<span class="string">'while'</span>:<span class="number integer">1</span>,<span class="string">'who'</span>:<span class="number integer">1</span>,<span class="string">'whom'</span>:<span class="number integer">1</span>,<span class="string">'why'</span>:<span class="number integer">1</span>,<span class="string">'will'</span>:<span class="number integer">1</span>,<span class="string">'with'</span>:<span class="number integer">1</span>,<span class="string">'would'</span>:<span class="number integer">1</span>,<span class="string">'yet'</span>:<span class="number integer">1</span>,<span class="string">'you'</span>:<span class="number integer">1</span>,
  2117
+    <span class="string">'your'</span>:<span class="number integer">1</span>,<span class="string">''</span>:<span class="number integer">1</span>
  2118
+}</code></pre>
  2119
+</td>
  2120
+</tr>
  2121
+<tr class="code">
  2122
+<td class="docs">
  2123
+<p>A full text index with support for stop words, stemming and
  2124
+a basic boolean search syntax.</p>
  2125
+
  2126
+<h2>Usage</h2>
  2127
+
  2128
+<p>   <code>redback.createFullText(key);</code></p>
  2129
+
  2130
+<h2>Reference</h2>
  2131
+
  2132
+<p>   http://redis.io/topics/data-types#sets</p>
  2133
+
  2134
+<h2>Redis Structure</h2>
  2135
+
  2136
+<p>   <code>(namespace:)key:&lt;word1&gt; = set(docs)</code>
  2137
+   <code>(namespace:)key:&lt;word2&gt; = set(docs)</code>
  2138
+   <code>(namespace:)key:&lt;wordN&gt; = set(docs)</code>
  2139
+ </p>
  2140
+</td>
  2141
+<td class="code">
  2142
+<pre><code><span class="keyword">var</span> <span class="class">FullText</span> = <span class="variable">exports</span>.<span class="class">FullText</span> = <span class="class">Structure</span>.<span class="keyword">new</span>();</code></pre>
  2143
+</td>
  2144
+</tr>
  2145
+<tr class="code">
  2146
+<td class="docs">
  2147
+<p>Index one or more documents.</p>
  2148
+
  2149
+<h2>Index One Document</h2>
  2150
+
  2151
+<p>   <code>text.indexFile(1, 'document string ...', callback);</code></p>
  2152
+
  2153
+<h2>Index Many Documents</h2>
  2154
+
  2155
+<p>   <code>text.indexFile({1:'docstr1', 2:'docstr2'}, callback);</code></p>
  2156
+
  2157
+<h2></h2>
  2158
+
  2159
+<ul><li><p><strong>param</strong>: <em>int | string</em>  id - the document's unique identifier</p></li><li><p><strong>param</strong>: <em>string | ReadableStream | Buffer</em>  document</p></li><li><p><strong>param</strong>: <em>Function</em>  callback (optional)</p></li><li><p><strong>return</strong>: <em>this</em></p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  2160
+</td>
  2161
+<td class="code">
  2162
+<pre><code><span class="class">FullText</span>.<span class="variable">prototype</span>.<span class="variable">index</span> = <span class="keyword">function</span> (<span class="variable">id</span>, <span class="variable">document</span>, <span class="variable">callback</span>) {
  2163
+    <span class="keyword">if</span> (<span class="keyword">typeof</span> <span class="variable">id</span> === <span class="string">'object'</span>) {
  2164
+        <span class="keyword">return</span> <span class="this">this</span>.<span class="variable">indexAll</span>(<span class="variable">id</span>, <span class="variable">document</span>);
  2165
+    } <span class="keyword">else</span> <span class="keyword">if</span> (<span class="variable">document</span> <span class="variable">instanceof</span> <span class="class">EventEmitter</span> &<span class="variable">amp</span>;&<span class="variable">amp</span>;
  2166
+            <span class="keyword">typeof</span> <span class="variable">document</span>.<span class="variable">readable</span> !== <span class="string">'undefined'</span> &<span class="variable">amp</span>;&<span class="variable">amp</span>; <span class="variable">document</span>.<span class="variable">readable</span> === <span class="variable">true</span>) {
  2167
+        <span class="keyword">return</span> <span class="variable">indexStream</span>(<span class="variable">id</span>, <span class="variable">document</span>, <span class="variable">callback</span>);
  2168
+    } <span class="keyword">else</span> <span class="keyword">if</span> (<span class="variable">document</span> <span class="variable">instanceof</span> <span class="class">Buffer</span>) {
  2169
+        <span class="variable">document</span> = <span class="variable">document</span>.<span class="variable">toString</span>();
  2170
+    }
  2171
+    <span class="this">this</span>.<span class="variable">indexed_bytes</span> += <span class="variable">document</span>.<span class="variable">length</span>;
  2172
+    <span class="keyword">var</span> <span class="variable">stemmed</span> = <span class="this">this</span>.<span class="variable">stemWords</span>(<span class="this">this</span>.<span class="variable">extractWords</span>(<span class="variable">document</span>));
  2173
+    <span class="this">this</span>.<span class="variable">buildIndex</span>(<span class="variable">id</span>, <span class="variable">stemmed</span>, <span class="variable">callback</span> || <span class="keyword">function</span> () {});
  2174
+}</code></pre>
  2175
+</td>
  2176
+</tr>
  2177
+<tr class="code">
  2178
+<td class="docs">
  2179
+<p>Index one or more files.</p>
  2180
+
  2181
+<h2>Index One Document</h2>
  2182
+
  2183
+<p>   <code>text.indexFile(1, '/path/to/file', callback);</code></p>
  2184
+
  2185
+<h2>Index Many Documents</h2>
  2186
+
  2187
+<p>   <code>text.indexFile({1:'file1', 2:'file2'}, callback);</code></p>
  2188
+
  2189
+<h2></h2>
  2190
+
  2191
+<ul><li><p><strong>param</strong>: <em>int | string | Object</em>  id</p></li><li><p><strong>param</strong>: <em>string</em>  filename (optional)</p></li><li><p><strong>param</strong>: <em>Function</em>  callback (optional)</p></li><li><p><strong>return</strong>: <em>this</em></p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  2192
+</td>
  2193
+<td class="code">
  2194
+<pre><code><span class="class">FullText</span>.<span class="variable">prototype</span>.<span class="variable">indexFile</span> =
  2195
+<span class="class">FullText</span>.<span class="variable">prototype</span>.<span class="variable">indexFiles</span> = <span class="keyword">function</span> (<span class="variable">id</span>, <span class="variable">filename</span>, <span class="variable">callback</span>) {
  2196
+    <span class="keyword">if</span> (<span class="keyword">typeof</span> <span class="variable">id</span> === <span class="string">'object'</span>) {
  2197
+        <span class="variable">callback</span> = <span class="variable">filename</span>;
  2198
+        <span class="keyword">var</span> <span class="variable">self</span> = <span class="this">this</span>, <span class="variable">files</span> = <span class="variable">id</span>, <span class="variable">ids</span> = <span class="class">Object</span>.<span class="variable">keys</span>(<span class="variable">files</span>),
  2199
+            <span class="variable">failed</span> = <span class="variable">false</span>, <span class="variable">remaining</span> = <span class="variable">ids</span>.<span class="variable">length</span>;
  2200
+        <span class="variable">ids</span>.<span class="variable">forEach</span>(<span class="keyword">function</span> (<span class="variable">id</span>) {
  2201
+            <span class="variable">self</span>.<span class="variable">indexStream</span>(<span class="variable">id</span>, <span class="variable">fs</span>.<span class="variable">createReadStream</span>(<span class="variable">files</span>[<span class="variable">id</span>]), <span class="keyword">function</span> (<span class="variable">err</span>) {
  2202
+                <span class="keyword">if</span> (<span class="variable">failed</span>) {
  2203
+                    <span class="keyword">return</span>;
  2204
+                } <span class="keyword">else</span> <span class="keyword">if</span> (<span class="variable">err</span>) {
  2205
+                    <span class="keyword">return</span> <span class="variable">callback</span>(<span class="variable">err</span>);
  2206
+                } <span class="keyword">else</span> {
  2207
+                    <span class="keyword">if</span> (!--<span class="variable">remaining</span>) <span class="variable">callback</span>();
  2208
+                }
  2209
+            });
  2210
+        });
  2211
+        <span class="keyword">return</span> <span class="this">this</span>;
  2212
+    } <span class="keyword">else</span> {
  2213
+        <span class="keyword">return</span> <span class="this">this</span>.<span class="variable">indexStream</span>(<span class="variable">id</span>, <span class="variable">fs</span>.<span class="variable">createReadStream</span>(<span class="variable">filename</span>), <span class="variable">callback</span>);
  2214
+    }
  2215
+}</code></pre>
  2216
+</td>
  2217
+</tr>
  2218
+<tr class="code">
  2219
+<td class="docs">
  2220
+<p>Search the full text index. Words will be extracted from the
  2221
+search string and used to filter search results. To exclude certain
  2222
+words, prefix them with a hyphen "-".</p>
  2223
+
  2224
+<h2>Basic Search</h2>
  2225
+
  2226
+<p>   <code>index.search('foo bar', callback);</code></p>
  2227
+
  2228
+<h2>Excluding Words</h2>
  2229
+
  2230
+<p>   <code>index.search('foo -bar -cool', callback);</code></p>
  2231
+
  2232
+<h2></h2>
  2233
+
  2234
+<ul><li><p><strong>param</strong>: <em>string</em>  search</p></li><li><p><strong>param</strong>: <em>Function</em>  callback</p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  2235
+</td>
  2236
+<td class="code">
  2237
+<pre><code><span class="class">FullText</span>.<span class="variable">prototype</span>.<span class="variable">search</span> = <span class="keyword">function</span> (<span class="variable">search</span>, <span class="variable">callback</span>) {
  2238
+    <span class="keyword">var</span> <span class="variable">include</span> = [], <span class="variable">exclude</span> = [];
  2239
+    <span class="this">this</span>.<span class="variable">stemWords</span>(<span class="this">this</span>.<span class="variable">extractWords</span>(<span class="variable">search</span>)).<span class="variable">forEach</span>(<span class="keyword">function</span> (<span class="variable">word</span>) {
  2240
+        <span class="keyword">if</span> (<span class="variable">word</span>[<span class="number integer">0</span>] === <span class="string">'-'</span>) {
  2241
+            <span class="variable">exclude</span>.<span class="variable">push</span>(<span class="variable">word</span>.<span class="variable">substr</span>(<span class="number integer">1</span>));
  2242
+        } <span class="keyword">else</span> {
  2243
+            <span class="variable">include</span>.<span class="variable">push</span>(<span class="variable">word</span>);
  2244
+        }
  2245
+    });
  2246
+    <span class="keyword">return</span> <span class="this">this</span>.<span class="variable">_search</span>(<span class="variable">include</span>, <span class="variable">exclude</span>, <span class="variable">callback</span>);
  2247
+}</code></pre>
  2248
+</td>
  2249
+</tr>
  2250
+<tr class="code">
  2251
+<td class="docs">
  2252
+<p>Clear the index.</p>
  2253
+
  2254
+<h2></h2>
  2255
+
  2256
+<ul><li><p><strong>param</strong>: <em>Function</em>  callback (optional)</p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  2257
+</td>
  2258
+<td class="code">
  2259
+<pre><code><span class="class">FullText</span>.<span class="variable">prototype</span>.<span class="variable">clear</span> = <span class="keyword">function</span> (<span class="variable">callback</span>) {
  2260
+    <span class="keyword">var</span> <span class="variable">self</span> = <span class="this">this</span>;
  2261
+    <span class="variable">callback</span> = <span class="variable">callback</span> || <span class="keyword">function</span> () {};
  2262
+    <span class="this">this</span>.<span class="variable">client</span>.<span class="variable">keys</span>(<span class="this">this</span>.<span class="variable">key</span> + <span class="string">':*'</span>, <span class="keyword">function</span> (<span class="variable">err</span>, <span class="variable">keys</span>) {
  2263
+        <span class="keyword">if</span> (<span class="variable">err</span>) <span class="keyword">return</span> <span class="variable">callback</span>(<span class="variable">err</span>, <span class="keyword">null</span>);
  2264
+        <span class="keyword">var</span> <span class="variable">rem_count</span> = <span class="number integer">0</span>, <span class="variable">failed</span> = <span class="variable">false</span>, <span class="variable">remaining</span> = <span class="variable">keys</span>.<span class="variable">length</span>;
  2265
+        <span class="variable">keys</span>.<span class="variable">forEach</span>(<span class="keyword">function</span> (<span class="variable">key</span>) {
  2266
+            <span class="variable">self</span>.<span class="variable">client</span>.<span class="variable">del</span>(<span class="variable">key</span>, <span class="keyword">function</span> (<span class="variable">err</span>, <span class="variable">removed</span>) {
  2267
+                <span class="keyword">if</span> (<span class="variable">failed</span>) {
  2268
+                    <span class="keyword">return</span>;
  2269
+                } <span class="keyword">else</span> <span class="keyword">if</span> (<span class="variable">err</span>) {
  2270
+                    <span class="variable">failed</span> = <span class="variable">true</span>;
  2271
+                    <span class="keyword">return</span> <span class="variable">callback</span>(<span class="variable">err</span>);
  2272
+                } <span class="keyword">else</span> {
  2273
+                    <span class="keyword">if</span> (<span class="variable">removed</span>) <span class="variable">rem_count</span>++;
  2274
+                    <span class="keyword">if</span> (!--<span class="variable">remaining</span>) <span class="variable">callback</span>(<span class="keyword">null</span>, <span class="variable">rem_count</span>);
  2275
+                }
  2276
+            });
  2277
+        });
  2278
+    });
  2279
+    <span class="keyword">return</span> <span class="this">this</span>;
  2280
+}
  2281
+</code></pre>
  2282
+</td>
2069 2283
 </tr><tr class="filename"><td><h2 id="lib/advanced_structures/KeyPair.js"><a href="#">KeyPair</a></h2></td><td>lib/advanced_structures/KeyPair.js</td></tr><tr class="code">
2070 2284
 <td class="docs">
2071 2285
 <p>Module dependencies.
@@ -2372,7 +2586,8 @@
2372 2586
 </tr>
2373 2587
 <tr class="code">
2374 2588
 <td class="docs">
2375  
-<p>Build a social graph similar to Twitter's.</p>
  2589
+<p>Build a social graph similar to Twitter's. User ID can be a string or
  2590
+integer, as long as they're unique.</p>
2376 2591
 
2377 2592
 <h2>Usage</h2>
2378 2593
 
@@ -3057,6 +3272,24 @@
3057 3272
 </tr>
3058 3273
 <tr class="code">
3059 3274
 <td class="docs">
  3275
+<p>Checks whether a cache key has an expiry.</p>
  3276
+
  3277
+<h2></h2>
  3278
+
  3279
+<ul><li><p><strong>param</strong>: <em>string</em>  key</p></li><li><p><strong>param</strong>: <em>Function</em>  callback</p></li><li><p><strong>return</strong>: <em>this</em></p></li><li><p><strong>api</strong>: <em>public</em></p></li></ul>
  3280
+</td>
  3281
+<td class="code">
  3282
+<pre><code><span class="class">Cache</span>.<span class="variable">prototype</span>.<span class="variable">isVolatile</span> = <span class="keyword">function</span> (<span class="variable">key</span>, <span class="variable">callback</span>) {
  3283
+    <span class="this">this</span>.<span class="variable">client</span>.<span class="variable">ttl</span>(<span class="this">this</span>.<span class="variable">getKey</span>(<span class="variable">key</span>), <span class="keyword">function</span> (<span class="variable">err</span>, <span class="variable">ttl</span>) {
  3284
+        <span class="keyword">if</span> (<span class="variable">err</span>) <span class="keyword">return</span> <span class="variable">callback</span>(<span class="variable">err</span>, <span class="keyword">null</span>);
  3285
+        <span class="variable">callback</span>(<span class="keyword">null</span>, <span class="variable">ttl</span> != -<span class="number integer">1</span>);
  3286
+    });
  3287
+    <span class="keyword">return</span> <span class="this">this</span>;
  3288
+}</code></pre>
  3289
+</td>
  3290
+</tr>
  3291
+<tr class="code">
  3292
+<td class="docs">
3060 3293
 <p>Remove the cache key's associated expiry.</p>
3061 3294
 
3062 3295
 <h2></h2>
8  site/index.html
@@ -185,10 +185,10 @@
185 185
 
186 186
                 <p>
187 187
                     Redback provides an accessible and extensible interface to the Redis <a href="http://redis.io/topics/data-types">data types</a>.
188  
-                    You can use Redback as a Cache backend, a Pub/Sub provider, or to create complex data structures such as the
189  
-                        <a href="#">SocialGraph</a>,
190  
-                        <a href="#">KeyPair</a> or
191  
-                        <a href="#">DensitySet</a>.
  188
+                    You can use Redback as a Cache backend, a Pub/Sub provider, or use complex built-in structures such as the
  189
+                        <a href="#">Social Graph</a>,
  190
+                        <a href="#">Key Pair</a> or
  191
+                        <a href="#">Full Text Index</a>. You can also build your own Redis-backed structures with ease.
192 192
                 </p>
193 193
 
194 194
             <h2>How do I get it?</h2>
1  test/files/doc1.txt
... ...
@@ -0,0 +1 @@
  1
+This is a sentence containing some words
1  test/files/doc2.txt
... ...
@@ -0,0 +1 @@
  1
+This contains some punctuation? and StRaNge CaSiNg Issues which won't affect the index
1  test/files/doc3.txt
... ...
@@ -0,0 +1 @@
  1
+This contains punctuation too@*& !&@*(&(      and some excess whitespace
199  test/fulltext.test.js
... ...
@@ -0,0 +1,199 @@
  1
+var redback = require('redback').createClient(),
  2
+    assert = require('assert');
  3
+
  4
+//Flush the DB and close the Redis connection after 500ms
  5
+setTimeout(function () {
  6
+    redback.client.flushdb(function (err) {
  7
+        redback.client.quit();
  8
+    });
  9
+}, 500);
  10
+
  11
+module.exports = {
  12
+
  13
+    'test full text': function() {
  14
+        var text = redback.createFullText('test_fulltext');
  15
+
  16
+        var doc1 = 'This is a sentence containing some words',
  17
+            doc2 = 'This contains some punctuation? and StRaNge CaSiNg Issues which won\' affect the index',
  18
+            doc3 = 'This contains punctuation too@*& !&@*(&(      and some excess whitespace';
  19
+
  20
+        text.index(1, doc1, function (err, added) {
  21
+            text.index(2, doc2, function (err, added) {
  22
+                text.index(3, doc3, function (err, added) {
  23
+                    text.search('contains', function (err, results) {
  24
+                        var expected = [1,2,3], l = expected.length;
  25
+                        while (l--) assert.equal(expected.shift(), results.shift());
  26
+                    });
  27
+                    text.search('-whitespace', function (err, results) {
  28
+                        var expected = [1,2], l = expected.length;
  29
+                        while (l--) assert.equal(expected.shift(), results.shift());
  30
+                    });
  31
+                    text.search('contains -punctuation', function (err, results) {
  32
+                        var expected = [1], l = expected.length;
  33
+                        while (l--) assert.equal(expected.shift(), results.shift());
  34
+                    });
  35
+                    text.search('containing -excess', function (err, results) {
  36
+                        var expected = [1, 2], l = expected.length;
  37
+                        while (l--) assert.equal(expected.shift(), results.shift());
  38
+                    });
  39
+                    text.search('containing -excess -issue', function (err, results) {
  40
+                        var expected = [1], l = expected.length;
  41
+                        while (l--) assert.equal(expected.shift(), results.shift());
  42
+                    });
  43
+                    text.search('nothere', function (err, results) {
  44
+                        assert.equal(0, results.length);
  45
+                    });
  46
+                    text.search('-contains', function (err, results) {
  47
+                        assert.equal(0, results.length);
  48
+                    });
  49
+                });
  50
+            });
  51
+        });
  52
+    },
  53
+
  54
+    'test file index': function () {
  55
+        var text = redback.createFullText('test_fulltext_file_index');
  56
+
  57
+        text.indexFile(1, __dirname + '/files/doc1.txt', function (err, added) {
  58
+            text.indexFile(2, __dirname + '/files/doc2.txt', function (err, added) {
  59
+                text.indexFile(3, __dirname + '/files/doc3.txt', function (err, added) {
  60
+                    text.search('contains', function (err, results) {
  61
+                        var expected = [1,2,3], l = expected.length;
  62
+                        while (l--) assert.equal(expected.shift(), results.shift());
  63
+                    });
  64
+                    text.search('-whitespace', function (err, results) {
  65
+                        var expected = [1,2], l = expected.length;
  66
+                        while (l--) assert.equal(expected.shift(), results.shift());
  67
+                    });
  68
+                    text.search('contains -punctuation', function (err, results) {
  69
+                        var expected = [1], l = expected.length;
  70
+                        while (l--) assert.equal(expected.shift(), results.shift());
  71
+                    });
  72
+                    text.search('containing -excess', function (err, results) {
  73
+                        var expected = [1, 2], l = expected.length;
  74
+                        while (l--) assert.equal(expected.shift(), results.shift());
  75
+                    });
  76
+                    text.search('containing -excess -issue', function (err, results) {
  77
+                        var expected = [1], l = expected.length;
  78
+                        while (l--) assert.equal(expected.shift(), results.shift());
  79
+                    });
  80
+                    text.search('nothere', function (err, results) {
  81
+                        assert.equal(0, results.length);
  82
+                    });
  83
+                    text.search('-contains', function (err, results) {
  84
+                        assert.equal(0, results.length);
  85
+                    });
  86
+                });
  87
+            });
  88
+        });
  89
+    },
  90
+
  91
+    'test indexing many': function () {
  92
+        var text = redback.createFullText('test_fulltext_many');
  93
+
  94
+        var documents = {
  95
+            1: 'This is a sentence containing some words',
  96
+            2: 'This contains some punctuation? and StRaNge CaSiNg Issues which won\' affect the index',
  97
+            3: 'This contains punctuation too@*& !&@*(&(      and some excess whitespace'
  98
+        }
  99
+
  100
+        text.index(documents, function (err, added) {
  101
+            text.search('contains', function (err, results) {
  102
+                var expected = [1,2,3], l = expected.length;
  103
+                while (l--) assert.equal(expected.shift(), results.shift());
  104
+            });
  105
+            text.search('-whitespace', function (err, results) {
  106
+                var expected = [1,2], l = expected.length;
  107
+                while (l--) assert.equal(expected.shift(), results.shift());
  108
+            });
  109
+            text.search('contains -punctuation', function (err, results) {
  110
+                var expected = [1], l = expected.length;
  111
+                while (l--) assert.equal(expected.shift(), results.shift());
  112
+            });
  113
+            text.search('containing -excess', function (err, results) {
  114
+                var expected = [1, 2], l = expected.length;
  115
+                while (l--) assert.equal(expected.shift(), results.shift());
  116
+            });
  117
+            text.search('containing -excess -issue', function (err, results) {
  118
+                var expected = [1], l = expected.length;
  119
+                while (l--) assert.equal(expected.shift(), results.shift());
  120
+            });
  121
+            text.search('nothere', function (err, results) {
  122
+                assert.equal(0, results.length);
  123
+            });
  124
+            text.search('-contains', function (err, results) {
  125
+                assert.equal(0, results.length);
  126
+            });
  127
+        });
  128
+    },
  129
+
  130
+    'test file index using obj': function () {
  131
+        var text = redback.createFullText('test_fulltext_file_index_many');
  132
+
  133
+        var files = {
  134
+            1: __dirname + '/files/doc1.txt',
  135
+            2: __dirname + '/files/doc2.txt',
  136
+            3: __dirname + '/files/doc3.txt'
  137
+        }
  138
+
  139
+        text.indexFiles(files, function (err, added) {
  140
+            text.search('contains', function (err, results) {
  141
+                var expected = [1,2,3], l = expected.length;
  142
+                while (l--) assert.equal(expected.shift(), results.shift());
  143
+            });
  144
+            text.search('-whitespace', function (err, results) {
  145
+                var expected = [1,2], l = expected.length;
  146
+                while (l--) assert.equal(expected.shift(), results.shift());
  147
+            });
  148
+            text.search('contains -punctuation', function (err, results) {
  149
+                var expected = [1], l = expected.length;
  150
+                while (l--) assert.equal(expected.shift(), results.shift());
  151
+            });
  152
+            text.search('containing -excess', function (err, results) {
  153
+                var expected = [1, 2], l = expected.length;
  154
+                while (l--) assert.equal(expected.shift(), results.shift());
  155
+            });
  156
+            text.search('containing -excess -issue', function (err, results) {
  157
+                var expected = [1], l = expected.length;
  158
+                while (l--) assert.equal(expected.shift(), results.shift());
  159
+            });
  160
+            text.search('nothere', function (err, results) {
  161
+                assert.equal(0, results.length);
  162
+            });
  163
+            text.search('-contains', function (err, results) {
  164
+                assert.equal(0, results.length);
  165
+            });
  166
+        });
  167
+    },
  168
+
  169
+    'test clearing the index': function () {
  170
+    var text = redback.createFullText('test_fulltext_clear');
  171
+
  172
+        var documents = {
  173
+            1: 'This is a sentence containing some words',
  174
+            2: 'This contains some punctuation? and StRaNge CaSiNg Issues which won\' affect the index',
  175
+            3: 'This contains punctuation too@*& !&@*(&(      and some excess whitespace'
  176
+        }
  177
+
  178
+        text.index(documents, function (err, added) {
  179
+            text.search('contains', function (err, results) {
  180
+                var expected = [1,2,3], l = expected.length;
  181
+                while (l--) assert.equal(expected.shift(), results.shift());
  182
+            });
  183
+
  184
+            text.clear(function (err) {
  185
+                text.search('contains', function (err, results) {
  186
+                    assert.equal(0, results.length);
  187
+
  188
+                    text.index(documents, function (err, added) {
  189
+                        text.search('contains', function (err, results) {
  190
+                            var expected = [1,2,3], l = expected.length;
  191
+                            while (l--) assert.equal(expected.shift(), results.shift());
  192
+                        });
  193
+                    });
  194
+                });
  195
+            });
  196
+        });
  197
+    }
  198
+
  199
+}

0 notes on commit f8d51b8

Please sign in to comment.
Something went wrong with that request. Please try again.