Permalink
Browse files

Merge branch 'master' of https://github.com/Timpy/Glimmer into Timpy-…

…master

Conflicts:
	pom.xml
  • Loading branch information...
2 parents 27bdaad + 87a9db2 commit 0a5c23cc72f1be24a1561d1511864a5e538777fb @Timpy Timpy committed Jan 15, 2014
View
@@ -22,8 +22,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
<plugin>
@@ -74,7 +74,7 @@
<dependency>
<groupId>it.unimi.di</groupId>
<artifactId>mg4j-big</artifactId>
- <version>5.2</version>
+ <version>5.2.1</version>
<exclusions>
<exclusion>
<artifactId>jetty</artifactId>
@@ -85,8 +85,8 @@
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
- <artifactId>org.apache.hadoop</artifactId>
- <groupId>hadoop-core</groupId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -38,33 +38,55 @@
*/
private long rollingSum;
- private ThreadLocal<Long> threadLocalStartTime = new ThreadLocal<Long>();
-
- public void start() {
- threadLocalStartTime.set(System.currentTimeMillis());
+ public class QueryTimer {
+ private final long startTime = System.currentTimeMillis();
+ private int searchDuration;
+ private int duration;
+
+ public QueryTimer endSearch() {
+ if (searchDuration == 0) {
+ searchDuration = (int) (System.currentTimeMillis() - startTime);
+ }
+ return this;
+ }
+ public QueryTimer end() {
+ if (duration == 0) {
+ duration = (int) (System.currentTimeMillis() - startTime);
+ }
+ return this;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+ public int getSearchDuration() {
+ return searchDuration;
+ }
+ public int getDuration() {
+ return duration;
+ }
}
- public synchronized long endQuery(String query, int numResults) {
- Long startTime = threadLocalStartTime.get();
- if (startTime == null) {
- throw new IllegalStateException("start() wasn't called!!");
- }
- long duration = System.currentTimeMillis() - startTime;
+ public QueryTimer start() {
+ return new QueryTimer();
+ }
+ public int endQuery(QueryTimer timer, String query, int numResults) {
+ timer.endSearch().end();
+
numQueries += 1;
- sumTime += duration;
+ sumTime += timer.duration;
if (numQueries % ROLLING_WINDOW_SIZE == 0) {
- rollingSum = duration;
+ rollingSum = timer.duration;
rollingCount = 1;
} else {
- rollingSum += duration;
+ rollingSum += timer.duration;
rollingCount += 1;
}
- LOGGER.info("#" + LOG_SEPARATOR + numQueries + LOG_SEPARATOR + query + LOG_SEPARATOR + duration + LOG_SEPARATOR
+ LOGGER.info("#" + LOG_SEPARATOR + numQueries + LOG_SEPARATOR + query + LOG_SEPARATOR + timer.searchDuration + LOG_SEPARATOR + timer.duration + LOG_SEPARATOR
+ ((double) sumTime / (double) numQueries) + LOG_SEPARATOR + rollingSum + LOG_SEPARATOR + rollingCount + LOG_SEPARATOR
+ ((double) rollingSum / (double) rollingCount) + LOG_SEPARATOR + numResults);
- threadLocalStartTime.set(null);
- return duration;
+ return timer.duration;
}
}
@@ -17,6 +17,7 @@
import it.unimi.di.big.mg4j.query.nodes.AbstractQueryBuilderVisitor;
import it.unimi.di.big.mg4j.query.nodes.Align;
import it.unimi.di.big.mg4j.query.nodes.And;
+import it.unimi.di.big.mg4j.query.nodes.Annotation;
import it.unimi.di.big.mg4j.query.nodes.Consecutive;
import it.unimi.di.big.mg4j.query.nodes.Difference;
import it.unimi.di.big.mg4j.query.nodes.False;
@@ -344,5 +345,10 @@ public Query visitPost(Weight node, Query subNode) throws QueryBuilderVisitorExc
public MyVisitor copy() {
return new MyVisitor();
}
+
+ @Override
+ public Query visitPost(Annotation node, Query subNode) throws QueryBuilderVisitorException {
+ return new Annotation( subNode );
+ }
}
}
@@ -34,6 +34,7 @@
import org.semanticweb.yars.nx.parser.NxParser;
import com.yahoo.glimmer.query.QueryLogger;
+import com.yahoo.glimmer.query.QueryLogger.QueryTimer;
import com.yahoo.glimmer.query.RDFIndex;
import com.yahoo.glimmer.util.BySubjectRecord;
import com.yahoo.glimmer.util.Util;
@@ -46,25 +47,13 @@
private final static Logger LOGGER = Logger.getLogger(Querier.class);
private static final String DEFAULT_CONTEXT = "default:";
private static final int CACHE_SIZE = 10000;
- private static final int BIG_RESULTS_LIMIT = 100000;
- private final Map<Integer, ResultsCacheValue> queryHashToResultsCache;
private final Map<String, Long> objectsSubjectsIdCache;
private final Map<Long, String> objectLabelCache;
private QueryLogger queryLogger = new QueryLogger();
public Querier() {
- Map<Integer, ResultsCacheValue> resultsCache = new LinkedHashMap<Integer, ResultsCacheValue>(CACHE_SIZE + 1, 1.1f, true) {
- private static final long serialVersionUID = -8171861525079261380L;
-
- protected boolean removeEldestEntry(java.util.Map.Entry<Integer, ResultsCacheValue> eldest) {
- return size() > CACHE_SIZE;
- };
- };
-
- queryHashToResultsCache = Collections.synchronizedMap(resultsCache);
-
LinkedHashMap<String, Long> idCache = new LinkedHashMap<String, Long>(CACHE_SIZE + 1, 1.1f, true) {
private static final long serialVersionUID = -8171861525079261380L;
@@ -84,45 +73,21 @@ protected boolean removeEldestEntry(java.util.Map.Entry<Long, String> eldest) {
objectLabelCache = Collections.synchronizedMap(labelCache);
}
- private static int getHashForQuery(String indexName, String query, int startItem, int maxNumItems) {
- long l = indexName.hashCode();
- l += query.hashCode();
- l += startItem;
- l += maxNumItems * 1299827l;
- return (int) (l ^ (l >>> 32));
- }
-
public QueryResult doQuery(RDFIndex index, Query query, int startItem, int maxNumItems, boolean deref, Integer objectLengthLimit)
throws QueryBuilderVisitorException, IOException {
if (startItem < 0 || maxNumItems < 0 || maxNumItems > 10000) {
throw new IllegalArgumentException("Bad item range - start:" + startItem + " maxNumItems:" + maxNumItems);
}
- queryLogger.start();
-
- Integer queryHash = getHashForQuery(index.getIndexName(), query.toString(), startItem, maxNumItems);
+ QueryTimer timer = queryLogger.start();
ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results;
int numResults;
- // TODO: Caching results here maybe pointless now that queries by type only aren't scored. See RDFIndex.process();
- if (queryHashToResultsCache.containsKey(queryHash)) {
- ResultsCacheValue cachedResults = queryHashToResultsCache.get(queryHash);
- results = cachedResults.results;
- numResults = cachedResults.numResults;
- } else {
- results = new ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>>();
- numResults = index.process(startItem, maxNumItems, results, query);
-
- if (numResults > BIG_RESULTS_LIMIT) {
- // Queries that return lots of results are slow. Cache them.
- queryHashToResultsCache.put(queryHash, new ResultsCacheValue(numResults, results));
- }
+ results = new ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>>();
+ numResults = index.process(startItem, maxNumItems, results, query);
- if (results.size() > maxNumItems) {
- results.size(maxNumItems);
- }
- }
+ timer.endSearch();
ObjectArrayList<QueryResultItem> resultItems = new ObjectArrayList<QueryResultItem>();
if (!results.isEmpty()) {
@@ -139,23 +104,24 @@ public QueryResult doQuery(RDFIndex index, Query query, int startItem, int maxNu
}
}
- long time = queryLogger.endQuery(query.toString(), numResults);
- QueryResult result = new QueryResult(null, query != null ? query.toString() : "", numResults, startItem, maxNumItems, resultItems, (int) time);
+ queryLogger.endQuery(timer, query.toString(), numResults);
+ QueryResult result = new QueryResult(null, query != null ? query.toString() : "", numResults, startItem, maxNumItems, resultItems, timer.getDuration(), timer.getSearchDuration());
return result;
}
public QueryResult doQueryForDocId(RDFIndex index, long id, boolean deref, Integer objectLengthLimit) throws IOException {
- queryLogger.start();
+ QueryTimer timer = queryLogger.start();
+ timer.endSearch();
QueryResultItem resultItem = createRdfResultItem(index, id, 1.0d, deref, objectLengthLimit);
- long time = queryLogger.endQuery("getDoc " + Long.toString(id), 1);
+ queryLogger.endQuery(timer, "getDoc " + Long.toString(id), 1);
List<QueryResultItem> results;
if (resultItem != null) {
results = Collections.singletonList(resultItem);
} else {
results = Collections.emptyList();
}
- return new QueryResult("", null, results.size(), 0, 1, results, (int) time);
+ return new QueryResult("", null, results.size(), 0, 1, results, timer.getDuration(), timer.getSearchDuration());
}
private QueryResultItem createRdfResultItem(RDFIndex index, long docId, double score, boolean lookupObjectLabels, Integer objectLengthLimit)
@@ -171,7 +137,8 @@ private QueryResultItem createRdfResultItem(RDFIndex index, long docId, double s
BySubjectRecord record = new BySubjectRecord();
if (!record.parse(new InputStreamReader(docInputStream))) {
- throw new RuntimeException("Couldn't parse doc with id:" + docId);
+ return null;
+ // throw new RuntimeException("Couldn't parse doc with id:" + docId);
}
if (docId != record.getId()) {
@@ -268,14 +235,4 @@ private QueryResultItem createRdfResultItem(RDFIndex index, long docId, double s
return item;
}
-
- private static class ResultsCacheValue {
- public final ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results;
- public final int numResults;
-
- public ResultsCacheValue(int numResults, ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results) {
- this.results = results;
- this.numResults = numResults;
- }
- }
}
@@ -18,6 +18,7 @@
import java.io.IOException;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -30,6 +31,8 @@
import org.apache.log4j.Logger;
import org.springframework.http.converter.HttpMessageConversionException;
import org.springframework.stereotype.Controller;
+import org.springframework.validation.BindException;
+import org.springframework.validation.FieldError;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -123,7 +126,7 @@ public RDFIndex getIndex(@RequestParam(required = false) String index) {
try {
id = Long.parseLong(idOrSubject);
} catch (NumberFormatException e) {
- throw new IllegalArgumentException("Query " + query + " failed to parse as a numeric subject ID(int)");
+ throw new IllegalArgumentException("Query " + query + " failed to parse as a numeric subject ID.");
}
} else {
id = index.getSubjectId(idOrSubject);
@@ -153,8 +156,15 @@ public RDFIndex getIndex(@RequestParam(required = false) String index) {
if (!(ex instanceof HttpMessageConversionException)) {
LOGGER.error("Exception when processing:" + request.getQueryString(), ex);
}
- response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
- return Collections.singletonMap(OBJECT_KEY, ex.getMessage());
+ if (ex instanceof IllegalArgumentException) {
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ return Collections.singletonMap(OBJECT_KEY, ex.getMessage());
+ } else if (ex instanceof BindException) {
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ List<FieldError> errors = ((BindException) ex).getFieldErrors();
+ return Collections.singletonMap(OBJECT_KEY, "Error binding on parameter:" + errors.get(0).getField());
+ }
+ throw new RuntimeException(ex);
}
private static String decodeEntities(String query) {
@@ -22,16 +22,18 @@
private final int pageStart;
private final int pageSize;
private final int time;
+ private final int searchTime;
private final String query;
private final String parsedQuery;
- public QueryResult(String query, String parsedQuery, int numResults, int pageStart, int pageSize, List<QueryResultItem> resultItems, int time) {
+ public QueryResult(String query, String parsedQuery, int numResults, int pageStart, int pageSize, List<QueryResultItem> resultItems, int time, int searchTime) {
super();
this.resultItems = resultItems;
this.numResults = numResults;
this.pageStart = pageStart;
this.pageSize = pageSize;
this.time = time;
+ this.searchTime = searchTime;
this.query = query != null ? query : "";
this.parsedQuery = parsedQuery;
}
@@ -55,6 +57,10 @@ public int getPageSize() {
public long getTime() {
return time;
}
+
+ public long getSearchTime() {
+ return searchTime;
+ }
public String getQuery() {
return query;
@@ -0,0 +1 @@
+[ "\nHTTP page not found error - 404.\nOops, sorry!" ]
@@ -0,0 +1 @@
+[ "\nHTTP Internal server error - 500.\nOops, sorry!" ]
@@ -12,7 +12,7 @@
<servlet-name>glimmer</servlet-name>
<url-pattern>/ajax/*</url-pattern>
</servlet-mapping>
-
+
<servlet-mapping>
<servlet-name>glimmer</servlet-name>
<url-pattern>/v1/search</url-pattern>
@@ -25,4 +25,14 @@
<session-config>
<session-timeout>60</session-timeout>
</session-config>
+
+ <error-page>
+ <error-code>404</error-code>
+ <location>/404.json</location>
+ </error-page>
+
+ <error-page>
+ <error-code>500</error-code>
+ <location>/500.json</location>
+ </error-page>
</web-app>

0 comments on commit 0a5c23c

Please sign in to comment.