Permalink
Browse files

add hint of query syntax in AccessTracker log (qs=normal querystring,

sq=solr-querystring) to allow to filter simple text queries for processing,
remove toString for counter parameter
use more predefined constants in solrservlet
  • Loading branch information...
reger24 committed Feb 19, 2017
1 parent e5858bc commit cc770512d5d6b1eb5b88e266376ede7c779a70bd
@@ -261,7 +261,7 @@ private void respond(final HttpServletRequest header, final Switchboard sb, fina
} else if (rv != null && rv instanceof SolrDocumentList) {
matches = (int) ((SolrDocumentList) rv).getNumFound();
}
AccessTracker.addToDump(originalQuery, Integer.toString(matches));
AccessTracker.addToDump(originalQuery, matches);
ConcurrentLog.info("GSA Query", "results: " + matches + ", for query:" + post.toString());
}

@@ -39,7 +39,6 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import net.yacy.cora.federate.FederateSearchManager;
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
@@ -162,16 +161,16 @@ public void run() {
String q = mmsp.get(CommonParams.Q, "");
if (querystring.length() == 0) querystring = q;
if (!mmsp.getMap().containsKey(CommonParams.START)) {
int startRecord = mmsp.getFieldInt("startRecord", null, 0);
int startRecord = mmsp.getFieldInt("startRecord", null, CommonParams.START_DEFAULT);
mmsp.getMap().remove("startRecord");
mmsp.getMap().put(CommonParams.START, new String[]{Integer.toString(startRecord)}); // sru patch
}
if (!mmsp.getMap().containsKey(CommonParams.ROWS)) {
int maximumRecords = mmsp.getFieldInt("maximumRecords", null, 10);
int maximumRecords = mmsp.getFieldInt("maximumRecords", null, CommonParams.ROWS_DEFAULT);
mmsp.getMap().remove("maximumRecords");
mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(maximumRecords)}); // sru patch
}
mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100))});
mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, CommonParams.ROWS_DEFAULT), (authenticated) ? 100000000 : 100))});

// set ranking according to profile number if ranking attributes are not given in the request
Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr);
@@ -207,9 +206,9 @@ public void run() {
if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[]{"<b>"});
if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[]{"</b>"});
if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[]{Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)});
if (!mmsp.getMap().containsKey("fl")) mmsp.getMap().put("fl", new String[]{
if (!mmsp.getMap().containsKey(CommonParams.FL)) mmsp.getMap().put(CommonParams.FL, new String[]{
CollectionSchema.sku.getSolrFieldName() + "," +
CollectionSchema.title + "," +
CollectionSchema.title.getSolrFieldName() + "," +
CollectionSchema.description_txt.getSolrFieldName() + "," +
CollectionSchema.id.getSolrFieldName() + "," +
CollectionSchema.url_paths_sxt.getSolrFieldName() + "," +
@@ -254,7 +253,7 @@ public void run() {

// check error
if (rsp.getException() != null) {
AccessTracker.addToDump(querystring, "0", new Date());
AccessTracker.addToDump(querystring, 0, new Date(), "sq");
sendError(hresponse, rsp.getException());
return;
}
@@ -263,7 +262,7 @@ public void run() {
NamedList<?> values = rsp.getValues();
DocList r = ((ResultContext) values.get("response")).docs;
int numFound = r.matches();
AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date());
AccessTracker.addToDump(querystring, numFound, new Date(), "sq");

// write response header
final String contentType = responseWriter.getContentType(req, rsp);
@@ -154,21 +154,31 @@ public static long size(final Location location) {
private static void addToDump(final QueryParams query, int resultCount) {
String queryString = query.getQueryGoal().getQueryString(false);
if (queryString == null || queryString.isEmpty()) return;
addToDump(queryString, Integer.toString(resultCount), new Date(query.starttime));
addToDump(queryString, resultCount, new Date(query.starttime), "qs");
}

public static void addToDump(String querystring, String resultcount) {
addToDump(querystring, resultcount, new Date());
public static void addToDump(String querystring, int resultCount) {
addToDump(querystring, resultCount, new Date(), "qs");
}

public static void addToDump(String querystring, String resultcount, Date d) {
/**
* Add a line to the queries log
*
* @param querystring the original query
* @param resultcount found results
* @param d start time
* @param querySyntax used syntax (qs=normal querstring, sq=solr querystring,
*/
public static void addToDump(String querystring, int resultcount, Date d, String querySyntax) {
//if (query.resultcount == 0) return;
if (querystring == null || querystring.isEmpty()) return;
final StringBuilder sb = new StringBuilder(40);
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(d));
sb.append(' ');
sb.append(resultcount);
sb.append(' ');
sb.append(querySyntax);
sb.append(' ');
sb.append(querystring);
synchronized (log) {
log.add(sb.toString());
@@ -332,15 +342,15 @@ private static Date readDate(RandomAccessFile raf, long x) throws IOException {

public static void main(String[] args) {
// i.e. /Users/admin/git/rc1/DATA/LOG/queries.log 20140522135156 20140614223118
String file = args[0];
String file = "C:\\src\\github\\yacy_search_server\\DATA\\LOG\\queries.log";
Date from;
try {
from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime();
Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime();
// try {
from = new Date(0);//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime();
Date to = new Date();//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime();
List<EventTracker.Event> dump = readLog(new File(file), from, to);
for (EventTracker.Event s: dump) System.out.println(s.toString());
} catch (ParseException e) {
e.printStackTrace();
}
// } catch (ParseException e) {
// e.printStackTrace();
// }
}
}

0 comments on commit cc77051

Please sign in to comment.