Skip to content
Permalink
Browse files

Render a relevant message and status on blocked search requests

When unauthenticated (or with insufficient rights) client is blocked
either because blacklisted or excessive request rate, render an error
message and a relevant HTTP status for API requests, instead of an empty
response that appears broken.
  • Loading branch information...
luccioman committed Apr 5, 2019
1 parent a8316c7 commit 42c8a251c8d06f3d584b45a5af9e3fc600ee5ec3
Showing with 67 additions and 17 deletions.
  1. +15 −2 htroot/yacysearch.html
  2. +39 −7 htroot/yacysearch.java
  3. +2 −0 htroot/yacysearchtrailer.html
  4. +11 −8 htroot/yacysearchtrailer.java
@@ -92,7 +92,10 @@
<div id="api" style="top:58px;">
<script type="text/javascript">
//<![CDATA[
document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\" id=\"apilink\" target=\"_blank\"\><img src=\"env/grafics/rss.png\" width=\"16\" height=\"16\" alt=\"RSS\" /></a>")
var showRSSIcon = #(num-results)#false::true::true::true::false::false#(/num-results)#;
if(showRSSIcon) {
document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\" id=\"apilink\" target=\"_blank\"\><img src=\"env/grafics/rss.png\" width=\"16\" height=\"16\" alt=\"RSS\" /></a>");
}
//]]>
</script>
<span>This search result can also be retrieved as RSS/<a href="http://www.opensearch.org" target="_blank">opensearch</a> output.
@@ -160,7 +163,17 @@
</div>
</div>
::
<p>Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p>
<p class="alert alert-danger" role="alert">Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p>
::
<p class="alert alert-danger col-sm-10" role="alert">
#(blockReason)#::You are not allowed to search the web with this peer.
::You have reached the maximum allowed number of accesses to this search page within ten minutes.
Please try again later or log in as administrator or as a user with extended search right.
::You have reached the maximum allowed number of accesses to this search page within one minute.
Please try again later or log in as administrator or as a user with extended search right.
::You have reached the maximum allowed number of accesses to this search page within three seconds.
Please try again later or log in as administrator or as a user with extended search right.
</p>
#(/num-results)#

#(urlmaskerror)#::
@@ -44,6 +44,8 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.http.HttpStatus;

import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.document.encoding.UTF8;
@@ -67,6 +69,7 @@
import net.yacy.data.ymark.YMarkTables;
import net.yacy.document.LibraryProvider;
import net.yacy.document.Tokenizer;
import net.yacy.http.servlets.TemplateProcessingException;
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.util.Bitfield;
@@ -321,9 +324,15 @@ public static serverObjects respond(
snippetFetchStrategy = null;
}
block = true;
prop.put("num-results_blockReason", 1);
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM "
+ client
+ " gets no permission to search");
if (!"html".equals(EXT)) {
/* API request : return the relevant HTTP status */
throw new TemplateProcessingException("You are not allowed to search the web with this peer.",
HttpStatus.SC_FORBIDDEN);
}
} else if ( !extendedSearchRights && !localhostAccess && !intranetMode ) {
// in case that we do a global search or we want to fetch snippets, we check for DoS cases
final int accInThreeSeconds;
@@ -405,13 +414,24 @@ public static serverObjects respond(
}
}
// general load protection
String timePeriodMsg = "";
if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
block = true;
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) {
block = true;
timePeriodMsg = "ten minutes";
prop.put("num-results_blockReason", 2);
} else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) {
block = true;
timePeriodMsg = "one minute";
prop.put("num-results_blockReason", 3);
} else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
block = true;
timePeriodMsg = "three seconds";
prop.put("num-results_blockReason", 4);
}
if(block) {
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+ client
+ ": "
@@ -422,10 +442,22 @@ public static serverObjects respond(
+ accInTenMinutes
+ "/600s, "
+ " requests, disallowed search");
if (!"html".equals(EXT)) {
/*
* API request : return the relevant HTTP status (429 - Too Many Requests - see
* https://tools.ietf.org/html/rfc6585#section-4)
*/
throw new TemplateProcessingException(
"You have reached the maximum allowed number of accesses to this search service within "
+ timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.",
429);
}
}
}

if ( !block ) {
if (block) {
prop.put("num-results", 5);
} else {
String urlmask = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); // the expression must be a subset of the java Match syntax described in http://lucene.apache.org/core/4_4_0/core/org/apache/lucene/util/automaton/RegExp.html
String tld = null;
String inlink = null;
@@ -56,6 +56,7 @@
</script>
#(/resource-switches)#

#(ranking-switches)#::
<p class="navbutton"></p>
<form action="yacysearch.html" method="get" accept-charset="UTF-8" name="rankingSwitchForm">
#(authSearch)#::
@@ -88,6 +89,7 @@
</div>
</div>
</form>
#(/ranking-switches)#

#(searchdomswitches)#::<p class="navbutton"></p>
<form action="yacysearch.html" method="get" accept-charset="UTF-8" name="contentdomSwitchForm"
@@ -104,9 +104,6 @@ public static serverObjects respond(final RequestHeader header, final serverObje
boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
boolean stealthmode = p2pmode && !global;

/* Add information about the current navigators generation (number of updates since their initialization) */
prop.put("nav-generation", theSearch.getNavGeneration());

// compose search navigation
ContentDomain contentdom = theSearch.getQuery().contentdom;
prop.put("searchdomswitches",
@@ -130,11 +127,17 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("resource-switches_global", adminAuthenticated && global);
appendSearchFormValues("resource-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);

appendSearchFormValues("", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
prop.put("contextRanking", !former.contains(" /date"));
prop.put("contextRanking_formerWithoutDate", former.replace(" /date", ""));
prop.put("dateRanking", former.contains(" /date"));
prop.put("dateRanking_former", former);
/* The search event has been found : we can render ranking switches */
prop.put("ranking-switches", true);
appendSearchFormValues("ranking-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);

/* Add information about the current navigators generation (number of updates since their initialization) */
prop.put("ranking-switches_nav-generation", theSearch.getNavGeneration());

prop.put("ranking-switches_contextRanking", !former.contains(" /date"));
prop.put("ranking-switches_contextRanking_formerWithoutDate", former.replace(" /date", ""));
prop.put("ranking-switches_dateRanking", former.contains(" /date"));
prop.put("ranking-switches_dateRanking_former", former);

appendSearchFormValues("searchdomswitches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);

0 comments on commit 42c8a25

Please sign in to comment.
You can’t perform that action at this time.