Skip to content

Commit

Permalink
prevent too high search request frequency submitted from the same peer
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4813 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed May 17, 2008
1 parent 1127d62 commit 3aa69da
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 26 deletions.
32 changes: 28 additions & 4 deletions htroot/yacy/search.java
View file
Expand Up @@ -121,6 +121,29 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
return prop;
}

// check the search tracker
TreeSet<Long> trackerHandles = sb.remoteSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 6000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (block) {
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
return prop;
}

// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);

Expand Down Expand Up @@ -291,10 +314,11 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
sb.remoteSearches.add(theQuery);
TreeSet<Long> handles = sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.remoteSearchTracker.put(client, handles);

// update the search tracker
trackerHandles.add(theQuery.handle);
sb.remoteSearchTracker.put(client, trackerHandles);


// log
yacyCore.log.logInfo("EXIT HASH SEARCH: " +
Expand Down
31 changes: 24 additions & 7 deletions htroot/yacy/ui/result.java
View file
Expand Up @@ -58,7 +58,8 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";

String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search

// get query
String querystring = (post == null) ? "" : post.get("search", "").trim();
final serverObjects prop = new serverObjects();
Expand Down Expand Up @@ -143,7 +144,24 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;

if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }

if ((!block) && (post.get("cat", "href").equals("href"))) {

final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
Expand All @@ -166,7 +184,6 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));

// do the search
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
plasmaSearchQuery theQuery = new plasmaSearchQuery(
querystring,
Expand Down Expand Up @@ -226,10 +243,10 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);

// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);

int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);
Expand Down
31 changes: 24 additions & 7 deletions htroot/yacy/user/ysearch.java
View file
Expand Up @@ -58,7 +58,8 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";

String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search

// get query
String querystring = (post == null) ? "" : post.get("search", "").trim();
final serverObjects prop = new serverObjects();
Expand Down Expand Up @@ -143,7 +144,24 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;

if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }

if ((!block) && (post.get("cat", "href").equals("href"))) {

final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
Expand All @@ -166,7 +184,6 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));

// do the search
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
plasmaSearchQuery theQuery = new plasmaSearchQuery(
querystring,
Expand Down Expand Up @@ -226,10 +243,10 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);

// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);

int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);
Expand Down
31 changes: 23 additions & 8 deletions htroot/yacysearch.java
View file
Expand Up @@ -93,11 +93,11 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve

// get query
String querystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
final serverObjects prop = new serverObjects();

boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
// we create empty entries for template strings
final serverObjects prop = new serverObjects();
prop.put("searchagain", "0");
prop.put("input", input);
prop.put("display", display);
Expand Down Expand Up @@ -174,8 +174,24 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;

serverObjects prop = new serverObjects();
if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }

if ((!block) && (post.get("cat", "href").equals("href"))) {

final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
Expand Down Expand Up @@ -302,12 +318,11 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);

prop = new serverObjects();
// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);

int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);
prop.put("num-results_itemscount", "0");
Expand Down

0 comments on commit 3aa69da

Please sign in to comment.