Major revamp of the client query processing and support

Added example queries to README Enhanced the demo app Fixes #101: bugs with paginating (size/from) Fixes #95: Wrong variable assignment in SeachQueue Fixes #91: Passing escaped JSON search string Merge #72, update to latest revisions of Firebase and ES.
googlearchive · Dec 2, 2016 · dcc520c · dcc520c
1 parent d1731d4
commit dcc520c
Show file tree

Hide file tree

Showing 9 changed files with 312 additions and 123 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,5 @@
 /.idea
 /node_modules
 service-account.json
-config.js
+config.js
+app.yaml
diff --git a/README.md b/README.md
@@ -32,21 +32,10 @@ Client Implementations
 
 Read `example/index.html` and `example/example.js` for a client implementation. It works like this:
 
- - Push an object to `/search/request` which has the following keys: `index`, `type`, and `query`
+ - Push an object to `/search/request` which has the following keys: `index`, `type`, and `q` (or `body` for advanced queries)
  - Listen on `/search/response` for the reply from the server
 
-The query object can be any valid ElasticSearch DSL structure (see More on Queries).
-
-More on Queries
----------------
-
-The full ElasticSearch API is supported. For example, you can control the number of matches (defaults to 10) and initial offset for paginating search results:
-
-```
-queryObj : { "from" : 0, "size" : 50 , "query": queryObj }; 
-```
-
-Check out [this great tutorial](http://okfnlabs.org/blog/2013/07/01/elasticsearch-query-tutorial.html) on querying ElasticSearch. And be sure to read the [ElasticSearch API Reference](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/).
+The `body` object can be any valid ElasticSearch DSL structure (see More on Queries).
 
 Deploy to Heroku
 ================
@@ -67,6 +56,93 @@ Deploy to Heroku
 After you've deployed to Heroku, you need to create your initial index name to prevent IndexMissingException error from Bonsai. Create an index called "firebase" via curl using the BONSAI_URL that you copied during Heroku deployment.
 
  - `curl -X POST <BONSAI_URL>/firebase` (ex: https://user:pass@yourbonsai.bonsai.io/firebase)
+
+ More on Queries
+ ---------------
+
+ The full ElasticSearch API is supported. Check out [this great tutorial](http://okfnlabs.org/blog/2013/07/01/elasticsearch-query-tutorial.html) on querying ElasticSearch. And be sure to read the [ElasticSearch API Reference](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/).
+
+ ### Example: Simple text search
+
+```
+ {
+   "q": "foo*"
+ }
+ ```
+
+ ### Example: Paginate
+
+ You can control the number of matches (defaults to 10) and initial offset for paginating search results:
+ ```
+ {
+   "from" : 0, 
+   "size" : 50, 
+   "body": {
+     "query": {
+        "match": {
+           "_all": "foo"
+        }
+     }
+   }
+ }; 
+ ```
+
+ #### Example: Search for multiple tags or categories
+
+ ```
+ {
+   "body": {
+     "query": {
+       { "tag": [ "foo", "bar" ] }
+     }
+   }
+ }
+ ```
+
+ [read more](https://www.elastic.co/guide/en/elasticsearch/guide/current/complex-core-fields.html)
+
+### Example: Search only specific fields
+```
+ {
+   "body": {
+     "query": {
+       "match": {
+         "field":  "foo",
+       }
+     }
+   }
+ }
+```
+
+### Example: Give more weight to specific fields
+```
+ {
+   "body": {
+     "query": {
+       "multi_match": {
+         "query":  "foo",
+         "type":   "most_fields", 
+         "fields": [ 
+            "important_field^10", // adding ^10 makes this field relatively more important 
+            "trivial_field" 
+         ]
+       }
+     }
+   }
+ }
+```
+
+[read more](https://www.elastic.co/guide/en/elasticsearch/guide/current/most-fields.html)
+
+#### Helpful section of ES docs
+
+ [Search lite (simple text searches with `q`)](https://www.elastic.co/guide/en/elasticsearch/guide/current/search-lite.html)
+ [Finding exact values](https://www.elastic.co/guide/en/elasticsearch/guide/current/_finding_exact_values.html)
+ [Sorting and relevance](https://www.elastic.co/guide/en/elasticsearch/guide/current/sorting.html)
+ [Partial matching](https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-matching.html)
+ [Wildcards and regexp](https://www.elastic.co/guide/en/elasticsearch/guide/current/_wildcard_and_regexp_queries.html)
+ [Proximity matching](https://www.elastic.co/guide/en/elasticsearch/guide/current/proximity-matching.html)
+ [Dealing with human language](https://www.elastic.co/guide/en/elasticsearch/guide/current/languages.html)
 
 Support
 =======

diff --git a/example/example.js b/example/example.js
@@ -7,7 +7,7 @@
   // Set the configuration for your app
   // TODO: Replace with your project's config object
   var config = {
-    databaseURL: "https://flashlight.firebaseio.com"
+    databaseURL: "https://kato-flashlight.firebaseio.com"
   };
 
   // TODO: Replace this with the path to your ElasticSearch queue
@@ -24,45 +24,99 @@
   // Get a reference to the database service
   var database = firebase.database();
 
-  // handle form submits
+  // handle form submits and conduct a search
+  // this is mostly DOM manipulation and not very
+  // interesting; you're probably interested in
+  // doSearch() and buildQuery()
   $('form').on('submit', function(e) {
     e.preventDefault();
     var $form = $(this);
+    $('#results').text('');
+    $('#total').text('');
+    $('#query').text('');
+    if( $form.find('[name=term]').val() ) {
+      doSearch(buildQuery($form));
+    }
+  });
+
+  function buildQuery($form) {
+    // this just gets data out of the form
+    var index = $form.find('[name=index]').val();
+    var type = $form.find('[name="type"]:checked').val();
     var term = $form.find('[name="term"]').val();
-    var words = $form.find('[name="words"]').is(':checked');
-    if( term ) {
-      doSearch($form.find('[name="index"]').val(), $form.find('[name="type"]:checked').val(), makeTerm(term, words));
+    var matchWholePhrase = $form.find('[name="exact"]').is(':checked');
+    var size = parseInt($form.find('[name="size"]').val());
+    var from = parseInt($form.find('[name="from"]').val());
+
+    // skeleton of the JSON object we will write to DB
+    var query = {
+      index: index,
+      type: type
+    };
+
+    // size and from are used for pagination
+    if( !isNaN(size) ) { query.size = size; }
+    if( !isNaN(from) ) { query.from = from; }
+
+    buildQueryBody(query, term, matchWholePhrase);
+
+    return query;
+  }
+
+  function buildQueryBody(query, term, matchWholePhrase) {
+    if( matchWholePhrase ) {
+      var body = query.body = {};
+      body.query = {
+        // match_phrase matches the phrase exactly instead of breaking it
+        // into individual words
+        "match_phrase": {
+          // this is the field name, _all is a meta indicating any field
+          "_all": term
+        }
+        /**
+         * Match breaks up individual words and matches any
+         * This is the equivalent of the `q` string below
+        "match": {
+          "_all": term
+        }
+        */
+      }
     }
     else {
-      $('#results').text('');
+      query.q = term;
     }
-  });
+  }
 
-  // display search results
-  function doSearch(index, type, query) {
+  // conduct a search by writing it to the search/request path
+  function doSearch(query) {
     var ref = database.ref().child(PATH);
-    var key = ref.child('request').push({
-      index: index,
-      type: type,
-      query: query
-      // Our example just turns query into a string. However, it can be a string or an object.
-      // If passed as an object, it is declared as the `body` parameter.
-      // If passed as a string, it is declared as the `q` parameter.
-      // See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-search
-    }).key;
-
-    console.log('search', key, {index: index, type: type, query: query});
+    var key = ref.child('request').push(query).key;
+
+    console.log('search', key, query);
+    $('#query').text(JSON.stringify(query, null, 2));
     ref.child('response/'+key).on('value', showResults);
   }
 
+  // when results are written to the database, read them and display
   function showResults(snap) {
-    if( !snap.exists() ) { return; } // wait until we get data
     var dat = snap.val();
+    if( dat === null ) { return; } // wait until we get data
+
+    // when a value arrives from the database, stop listening
+    // and remove the temporary data from the database
     snap.ref.off('value', showResults);
     snap.ref.remove();
+
+    // the rest of this just displays data in our demo and probably
+    // isn't very interesting
+    var totalText = dat.total;
+    if( dat.hits && dat.hits.length !== dat.total ) {
+      totalText = dat.hits.length + ' of ' + dat.total;
+    }
+    $('#total').text('(' + totalText + ')');
+
     var $pair = $('#results')
       .text(JSON.stringify(dat, null, 2))
-      .add( $('#total').text(dat.total) )
       .removeClass('error zero');
     if( dat.error ) {
       $pair.addClass('error');
@@ -72,15 +126,8 @@
     }
   }
 
-  function makeTerm(term, matchWholeWords) {
-    if( !matchWholeWords ) {
-      if( !term.match(/^\*/) ) { term = '*'+term; }
-      if( !term.match(/\*$/) ) { term += '*'; }
-    }
-    return term;
-  }
-
-  // display raw data for reference
+  // display raw data for reference, this is just for the demo
+  // and probably not very interesting
   database.ref().on('value', setRawData);
   function setRawData(snap) {
     $('#raw').text(JSON.stringify(snap.val(), null, 2));

diff --git a/example/index.html b/example/index.html
@@ -34,16 +34,21 @@ <h1>Flashlight: Search library built on Firebase and ElasticSearch</h1>
                <!-- turn this into a select to search multiple indexes -->
                <input type="hidden" name="index" value="firebase" />
                <div class="input-group">
-                   <input type="text" class="form-control" name="term" placeholder="Enter search term">
+                   <input type="text" class="form-control" name="term" value="you" placeholder="Enter search term">
                    <span class="input-group-btn">
                        <button class="btn btn-default" type="submit">Search!</button>
                    </span>
                </div>
                <ul class="list-inline">
-                   <li><label><input type="radio" name="type" value="user" checked> users</label></li>
-                   <li><label><input type="radio" name="type" value="message"> messages</label></li>
+                   <li><label><input type="radio" name="type" value="user"> users</label></li>
+                   <li><label><input type="radio" name="type" value="message" checked> messages</label></li>
                    <li class="divider"></li>
-                   <li><label><input type="checkbox" name="words"> whole words</label></li>
+                   <li><label><input type="checkbox" name="exact"> match whole phrase</label></li>
+               </ul>
+               <ul class="list-inline paginate-fields">
+                   <li>Pagination: </li>
+                   <li><label>size <input type="number" name="size" /></label></li>
+                   <li><label>from <input type="number" name="from" /></label></li>
                </ul>
                <p class="help-block">You may use * or ? for wild cards in your search.</p>
            </form>
@@ -53,6 +58,8 @@ <h1>Flashlight: Search library built on Firebase and ElasticSearch</h1>
         <div class="col-md-6">
            <h2>Results <span id="total" class="zero">0</span></h2>
            <pre id="results" class="pre-scrollable zero">(enter a search term)</pre>
+           <h2>The query used</h2>
+           <pre id="query" class="pre-scrollable"></pre>
         </div>
         <div class="col-md-6">
            <h2>Raw data</h2>
@@ -67,7 +74,7 @@ <h2>Raw data</h2>
 <a id="fork_on_github" href="https://github.com/firebase/flashlight" target="_blank"><img src="https://s3.amazonaws.com/github/ribbons/forkme_right_green_007200.png" alt="Fork me on GitHub"></a>
 
 <!-- libs and dependencies -->
-<script src="https://www.gstatic.com/firebasejs/3.5.2/firebase.js"></script>
+<script src="https://www.gstatic.com/firebasejs/3.6.2/firebase.js"></script>
 <script type="text/javascript" src="https://code.jquery.com/jquery-2.0.3.min.js"></script>
 <script type="text/javascript" src="https://netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js"></script>
 

diff --git a/example/seed/security_rules.json b/example/seed/security_rules.json
@@ -1,16 +1,15 @@
 {
    "rules": {
-
-      ".read": false,
-      ".write": false,
-      "search": {
+     ".read": false,
+     ".write": false,
+     "search": {
          "request": {
             "$recid": {
                // I can only read records assigned to me
                ".read": "auth.id === data.child('id').val() || auth.uid === data.child('id').val()",
                // I can only write new records that don't exist yet
                ".write": "!data.exists() && (newData.child('id').val() === auth.id || newData.child('id').val() === auth.uid)",
-               ".validate": "newData.hasChildren(['query', 'index', 'type'])",
+               ".validate": "newData.hasChildren(['index', 'type']) && (newData.hasChild('q') || newData.hasChild('query') || newData.hasChild('body'))",
                "index": {
                   // accepts arrays or strings
                   ".validate": "(newData.isString() && newData.val().length < 1000) || newData.hasChildren()",
@@ -26,23 +25,38 @@
                   }
                },
                "query": {
-                  // lucene formatted string, such as "title:search_term" or an object
-                  // see https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-search
+                  // lucene formatted string, such as "title:search_term" or a `body` attribute
+                  // see https://www.elastic.co/guide/en/elasticsearch/guide/current/query-dsl-intro.html
                   ".validate": "newData.isString() || newData.hasChildren()"
                },
+               "body": {
+                  // The `body` object of an ES search, such as { size: 25, from: 0, query: "*foo*" }, see
+                  // https://www.elastic.co/guide/en/elasticsearch/guide/current/query-dsl-intro.html
+                  ".validate": "newData.hasChildren()"
+               },
+               "q": {
+                  // lucene formatted 'lite' search string, such as "*foo*" or "+name:(mary john) +date:>2014-09-10", see
+                  // https://www.elastic.co/guide/en/elasticsearch/guide/current/search-lite.html
+                  ".validate": "newData.isString()"
+               },
+               "size": {
+                  ".validate": "newData.isNumber() && newData.val() >= 0"
+               },
+               "from": {
+                 ".validate": "newData.isNumber() && newData.val() >= 0"
+               },
                "$other": {
                   ".validate": false
                }
             }
          },
          "response": {
+           ".indexOn": "timestamp",
             "$recid": {
-               // I can only read/write records assigned to me
-               ".read": "auth.id === data.child('id').val() || auth.uid === data.child('id').val()",
-               ".write": "auth.id === data.child('id').val() || auth.uid === data.child('id').val()",
-               // Assumes that Flashlight will be writing the records using a secret or a token that has admin: true
-               // The only thing a logged in user needs to do is delete results after reading them
-               ".validate": false
+              // I can only read/write records assigned to me
+              ".read": "auth.uid === data.child('id').val()",
+              // delete only
+              ".write": "auth.uid === data.child('id').val() && !newData.exists()"
             }
          }
       }

diff --git a/example/style.css b/example/style.css
@@ -36,4 +36,8 @@ h2 span.error, .error {
 
 @media (max-width: 767px) {
    #fork_on_github { display: none }
+}
+
+.paginate-fields [type=number] {
+   width: 4em;
 }