query dsl + highlighting and pagination

opencharles · Oct 14, 2016 · 8460678 · 8460678
1 parent b721c72
commit 8460678
Show file tree

Hide file tree

Showing 7 changed files with 178 additions and 38 deletions.
diff --git a/charles-rest/src/main/java/com/amihaiemil/charles/aws/AmazonEsSearch.java b/charles-rest/src/main/java/com/amihaiemil/charles/aws/AmazonEsSearch.java
@@ -24,7 +24,9 @@
  */
 package com.amihaiemil.charles.aws;
 
+import java.io.ByteArrayInputStream;
 import java.net.URI;
+
 import com.amazonaws.DefaultRequest;
 import com.amazonaws.Request;
 import com.amazonaws.http.HttpMethodName;
@@ -83,19 +85,16 @@ public Request<Void> buildAwsSearchRequest() {
         if(esEndpoint == null || esEndpoint.isEmpty()) {
             throw new IllegalStateException("ElasticSearch endpoint needs to be specified!");
         }
-        String search = "_search/"
-            + indexName + "/" 
-            + query.getCategory() + "?q=textContent="
-            + query.getContent() + "&from="
-            + query.getIndex() + "&size="
-            + query.getNr();
+        String search = indexName + "/_search";
         if(esEndpoint.endsWith("/")) {
             esEndpoint += search;
         } else {
             esEndpoint += "/" + search;
         }
         request.setEndpoint(URI.create(esEndpoint));
-        request.setHttpMethod(HttpMethodName.GET);
+        String queryDsl = this.query.toJson().toString();
+        request.setContent(new ByteArrayInputStream(queryDsl.getBytes()));
+        request.setHttpMethod(HttpMethodName.POST);
         return request;
     }
 }
diff --git a/charles-rest/src/main/java/com/amihaiemil/charles/aws/SearchResponseHandler.java b/charles-rest/src/main/java/com/amihaiemil/charles/aws/SearchResponseHandler.java
@@ -33,7 +33,6 @@
 import javax.json.JsonObject;
 
 import com.amazonaws.AmazonServiceException;
-import com.amazonaws.Request;
 import com.amazonaws.http.HttpResponse;
 import com.amazonaws.http.HttpResponseHandler;
 import com.amihaiemil.charles.rest.model.SearchResult;
@@ -79,27 +78,18 @@ private SearchResultsPage buildResultsPage(HttpResponse response) {
             JsonArray hits = result.getJsonObject("hits").getJsonArray("hits");
             for(int i=0; i<hits.size(); i++) {
                 JsonObject hitSource = hits.getJsonObject(i).getJsonObject("_source");
+                JsonObject highlight = hits.getJsonObject(i).getJsonObject("highlight");
                 SearchResult res = new SearchResult(
-                    //TODO ES settings and logic for highlighting text content.
-                    hitSource.getString("url"), "TODO set highlight settings", hitSource.getString("category")
+                    hitSource.getString("url"),
+                    highlight.getJsonArray("textContent").getString(0),
+                    hitSource.getString("category")
                 );
                 searchResults.add(res);
             }
             page.setResults(searchResults);
             page.setTotalHits(totalHits);
-            this.setPagesInfo(page, response.getRequest());
         } 
         return page;
     }
-
-    /**
-     * Set the page number, next page, previous page and all pages' links
-     * on the results page. Use the original search request since parameters from and size
-     * are not part of the response. 
-     * @param page
-     * @param request
-     */
-    private void setPagesInfo(SearchResultsPage page, Request<?> request) {
-        //TODO be implemented after highlighting, since request logic might change then.
-    }
+
 }
diff --git a/charles-rest/src/main/java/com/amihaiemil/charles/rest/CharlesResource.java b/charles-rest/src/main/java/com/amihaiemil/charles/rest/CharlesResource.java
@@ -24,14 +24,20 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 */
 package com.amihaiemil.charles.rest;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+
 import com.amihaiemil.charles.aws.AmazonEsSearch;
 import com.amihaiemil.charles.rest.model.EsQuery;
 import com.amihaiemil.charles.rest.model.SearchResultsPage;
@@ -47,6 +53,12 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 @Path("/")
 public class CharlesResource {
 
+	/**
+	 * Http request.
+	 */
+	@Context
+	private HttpServletRequest servletRequest;
+
     /**
      * Endpoint for checking if the service is online.
      * @return ok response.
@@ -75,10 +87,38 @@ public Response search(
         @QueryParam("index") @DefaultValue("0") String index,
         @QueryParam("size") @DefaultValue("10") String size
     ) throws JsonProcessingException {
-        EsQuery query = new EsQuery(keywords, category, index, size);
+    	int idx = Integer.valueOf(index);
+    	int nr = Integer.valueOf(size);
+        EsQuery query = new EsQuery(keywords, category, Integer.valueOf(index), Integer.valueOf(size));
         String indexName = user.toLowerCase() + "x" + repo.toLowerCase();
         AmazonEsSearch aws = new AmazonEsSearch(query, indexName);
         SearchResultsPage results = aws.search();
+
+        String queryStringFormat = "?kw=%s&ctg=%s&index=%s&size=%s";
+        String requestUrl = servletRequest.getRequestURL().toString();
+        if(idx == 0) {
+        	results.setPreviousPage("-");
+        } else {
+            String queryString = String.format(queryStringFormat, keywords, category, idx - nr, nr);
+            results.setPreviousPage(requestUrl + queryString);
+        }
+        if(idx + nr >= results.getTotalHits()) {
+            results.setNextPage("-");
+        } else {
+        	String queryString = String.format(queryStringFormat, keywords, category, idx + nr, nr);
+            results.setNextPage(requestUrl + queryString);
+        }
+        results.setPageNr(idx/nr + 1);
+
+        int start = 0;
+        List<String> pagesLinks = new ArrayList<String>();
+        while(start < nr) {
+        	pagesLinks.add(
+        	    requestUrl + String.format(queryStringFormat, keywords, category, start, nr)
+        	);
+            start += nr;
+        }
+
         return Response.ok().entity(new ObjectMapper().writeValueAsString(results)).build();
     }
 }
diff --git a/charles-rest/src/main/java/com/amihaiemil/charles/rest/model/EsQuery.java b/charles-rest/src/main/java/com/amihaiemil/charles/rest/model/EsQuery.java
@@ -1,5 +1,9 @@
 package com.amihaiemil.charles.rest.model;
 
+import javax.json.Json;
+import javax.json.JsonArray;
+import javax.json.JsonObject;
+
 /**
  * Query for search.
  * @author Mihai Andronache (amihaiemil@gmail.com)
@@ -22,18 +26,18 @@ public class EsQuery {
     /**
      * Show results from this index.
      */
-    private String index;
+    private int index;
 
     /**
      * Show nr of results on a page.
      */
-    private String nr;
+    private int nr;
 
     /**
      * Default ctor.
      */
     public EsQuery() {
-        this("", "", "", "");
+        this("", "", 0, 10);
     }
     /**
      * Ctor.
@@ -42,7 +46,7 @@ public EsQuery() {
      * @param index
      * @param nr
      */
-    public EsQuery(String content, String category, String index, String nr) {
+    public EsQuery(String content, String category, int index, int nr) {
         this.content = content;
         this.category = category;
         this.index = index;
@@ -65,20 +69,95 @@ public void setCategory(String category) {
         this.category = category;
     }
 
-    public String getIndex() {
+    public int getIndex() {
         return index;
     }
 
-    public void setIndex(String index) {
+    public void setIndex(int index) {
         this.index = index;
     }
 
-    public String getNr() {
+    public int getNr() {
         return nr;
     }
 
-    public void setNr(String nr) {
+    public void setNr(int nr) {
         this.nr = nr;
     }
+
+    /**
+     * Turn this to json query for performing a search request
+     * (with query in the body; the only way to provide highlighting)
+     * <br><br>
+     * An example of json query:<br>
+     * <pre>
+     * {
+     *     "from": 10,
+     *     "size": 15,
+     *     "filter": {
+     *         "bool": {
+     *             "filter": [{
+     *                 "match_phrase_prefix": {
+     *                     "textContent": "string here"
+     *                  }
+     *              },
+     *              {
+     *                  "type": {
+     *                      "value": "tech"
+     *                   }
+     *              }]
+     *         }
+     *     }
+     * }
+     * </pre>
+     * 
+     * 
+     * @return This Query in Json format 
+     * @see 
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-query.html">
+     * ElasticSearch Query
+     * </a>
+     * @see 
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/2.4/term-level-queries.html">
+     * Term level queries
+     * </a>
+     */
+    public JsonObject toJson() {
+        JsonObject matcher = Json.createObjectBuilder()
+            .add(
+                "match_phrase_prefix",
+                Json.createObjectBuilder().add("textContent", this.content)    
+            ).build();
+        JsonObject type = Json.createObjectBuilder()
+            .add(
+                "type",
+                Json.createObjectBuilder().add("value", this.category)    
+            ).build();
+        JsonArray filter = Json.createArrayBuilder().add(matcher).add(type).build();
+        JsonObject bool = Json.createObjectBuilder().add(
+            "bool", Json.createObjectBuilder().add("filter", filter).build()
+        ).build();
+
+        JsonObject highlight = Json.createObjectBuilder()
+            .add(
+                "fields",
+                Json.createObjectBuilder()
+                    .add(
+                        "textContent",
+                        Json.createObjectBuilder()
+                            .add("fragment_size", 150)
+                            .build()
+                    )
+                    .build()
+             ).build();
+
+        JsonObject query = Json.createObjectBuilder()
+            .add("from", this.index)
+            .add("size", this.nr)
+            .add("query", bool)
+            .add("highlight", highlight)
+            .build();
+        return query;
+    }
 
 }
diff --git a/charles-rest/src/test/java/com/amihaiemil/charles/aws/AmazonEsSearchTestCase.java b/charles-rest/src/test/java/com/amihaiemil/charles/aws/AmazonEsSearchTestCase.java
@@ -68,7 +68,7 @@ public void searchWithResults() throws IOException {
 
         try {
             AmazonEsSearch es = new AmazonEsSearch(
-                new EsQuery("test", "page", "0", "10"),
+                new EsQuery("test", "page", 0, 10),
                 "amihaiemilxtestrepo"
             );
             SearchResultsPage srp = es.search();
@@ -103,7 +103,7 @@ public void searchWithNoResults() throws IOException {
 
         try {
             AmazonEsSearch es = new AmazonEsSearch(
-                new EsQuery("test", "page", "0", "10"),
+                new EsQuery("test", "page", 0, 10),
                 "amihaiemilxtestrepo"
             );
             SearchResultsPage srp = es.search();
@@ -128,8 +128,8 @@ public void missingEsEndpoint() {
         EsQuery query = new EsQuery();
         query.setCategory("");
         query.setContent("");
-        query.setIndex("0");
-        query.setNr("10");
+        query.setIndex(0);
+        query.setNr(10);
         AmazonEsSearch es = new AmazonEsSearch(query, "user/idx");
         try {
             es.search();

diff --git a/charles-rest/src/test/java/com/amihaiemil/charles/aws/SearchResponseHandlerTestCase.java b/charles-rest/src/test/java/com/amihaiemil/charles/aws/SearchResponseHandlerTestCase.java
@@ -24,14 +24,17 @@
  */
 package com.amihaiemil.charles.aws;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.net.HttpURLConnection;
+
 import org.junit.Test;
 import org.mockito.Mockito;
+
 import com.amazonaws.AmazonServiceException;
-import com.amazonaws.Request;
 import com.amazonaws.http.HttpResponse;
 import com.amihaiemil.charles.rest.model.SearchResult;
 import com.amihaiemil.charles.rest.model.SearchResultsPage;
@@ -58,7 +61,6 @@ public void buildsSearchResultsPage() throws Exception {
                 new File("src/test/resources/esSearchResponse.json")
              )
         );
-        Mockito.when(response.getRequest()).thenReturn(Mockito.mock(Request.class));
         SearchResultsPage page = new SearchResponseHandler().handle(response);
         assertTrue(page.getTotalHits() == 27);
         assertTrue(page.getResults().size() == 10);