Skip to content
Permalink
Browse files

Made /yacysearch access rate limitations user configurable

With a new admin page at /SearchAccessRate_p.html in menu Network Access
> Local Search > Access Rate Limitations
  • Loading branch information...
luccioman committed Apr 2, 2019
1 parent 5b7e412 commit 0ab2b49c3136b262c616f9702d8056a483734fc6
@@ -930,6 +930,21 @@ search.result.show.snapshots = false
# when true, display the raw ranking score value
search.result.show.ranking = false

# Maximum numbers of accesses within a given time period to the search interface for unauthenticated users and authenticated users with no extended search right
search.public.max.access.3s = 60
search.public.max.access.1mn = 600
search.public.max.access.10mn = 3000

# Maximum numbers of accesses within a given time period to the search interface in P2P mode for unauthenticated users and authenticated users with no extended search right
search.public.max.p2p.access.3s = 1
search.public.max.p2p.access.1mn = 6
search.public.max.p2p.access.10mn = 60

# Maximum number of accesses within a given time period to the search interface to support fetching remote results snippets for unauthenticated users and authenticated users with no extended search right
search.public.max.remoteSnippet.access.3s = 1
search.public.max.remoteSnippet.access.1mn = 4
search.public.max.remoteSnippet.access.10mn = 20


# search navigators: comma-separated list of default values for search navigation.
# By default navigators keys are sorted by descending counts. To sort by ascending displayed labels, add the :label suffix (example : hosts:label).
@@ -25,10 +25,6 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.time.Duration;
import java.util.Properties;
@@ -131,27 +127,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
}
if (post.containsKey("searchpage_default")) {
// load defaults from defaults/yacy.init file
final Properties config = new Properties();
final String mes = "ConfigPortal";
FileInputStream fis = null;
try {
fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init"));
config.load(fis);
} catch (final FileNotFoundException e) {
ConcurrentLog.severe(mes, "could not find configuration file.");
return prop;
} catch (final IOException e) {
ConcurrentLog.severe(mes, "could not read configuration file.");
return prop;
} finally {
if (fis != null) {
try {
fis.close();
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}
}
final Properties config = sb.loadDefaultConfig();
sb.setConfig(SwitchboardConstants.GREETING, config.getProperty(SwitchboardConstants.GREETING,"P2P Web Search"));
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, config.getProperty(SwitchboardConstants.GREETING_HOMEPAGE,"https://yacy.net"));
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, config.getProperty(SwitchboardConstants.GREETING_LARGE_IMAGE,"env/grafics/YaCyLogo_120ppi.png"));
@@ -25,10 +25,6 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.Date;
import java.util.HashSet;
import java.util.Map;
@@ -37,7 +33,6 @@

import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.TransactionManager;
import net.yacy.data.WorkTables;
import net.yacy.search.Switchboard;
@@ -163,27 +158,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje

if (post.containsKey("searchpage_default")) {
// load defaults from defaults/yacy.init file
final Properties config = new Properties();
final String mes = "ConfigSearchPage_p";
FileInputStream fis = null;
try {
fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init"));
config.load(fis);
} catch (final FileNotFoundException e) {
ConcurrentLog.severe(mes, "could not find configuration file.");
return prop;
} catch (final IOException e) {
ConcurrentLog.severe(mes, "could not read configuration file.");
return prop;
} finally {
if (fis != null) {
try {
fis.close();
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}
}
final Properties config = sb.loadDefaultConfig();
sb.setConfig("publicTopmenu", config.getProperty("publicTopmenu","true"));
sb.setConfig(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
config.getProperty(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
@@ -0,0 +1,158 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>YaCy '#[clientname]#': Local Search access rate
limitations</title> #%env/templates/metas.template%#
</head>
<body id="Settings">
#%env/templates/header.template%#
#%env/templates/submenuAccessTracker.template%#
<h2>Local Search access rate limitations</h2>
<p>
You can configure here limitations on access rate to this peer search
interface by unauthenticated users and users without extended search
right (see the <a href="ConfigAccounts_p.html">Accounts</a>
configuration page for details on users rights).
</p>

<form action="SearchAccessRate_p.html" method="post"
class="form-horizontal">
<input type="hidden" name="transactionToken"
value="#[transactionToken]#" />
<fieldset>
<legend>YaCy search</legend>

<p>Access rate limitations to this peer search interface. When a
user with limited rights (unauthenticated or without extended search
right) exceeds a limit, the search is blocked.</p>

<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.access.3s">Max searches in 3s</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.access.3s"
name="search.public.max.access.3s" type="number"
value="#[search.public.max.access.3s]#" min="0" max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.access.1mn">Max searches in 1mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.access.1mn"
name="search.public.max.access.1mn" type="number"
value="#[search.public.max.access.1mn]#" min="0" max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.access.10mn">Max searches in 10mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.access.10mn"
name="search.public.max.access.10mn" type="number"
value="#[search.public.max.access.10mn]#" min="0" max="2147483647" />
</div>
</div>
</fieldset>

<fieldset>
<legend>Peer-to-peer search</legend>

<p>Access rate limitations to the peer-to-peer search mode. When
a user with limited rights (unauthenticated or without extended
search right) exceeds a limit, the search scope falls back to only
this local peer index.</p>

<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.p2p.access.3s">Max searches in 3s</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.p2p.access.3s"
name="search.public.max.p2p.access.3s" type="number"
value="#[search.public.max.p2p.access.3s]#" min="0"
max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.p2p.access.1mn">Max searches in 1mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.p2p.access.1mn"
name="search.public.max.p2p.access.1mn" type="number"
value="#[search.public.max.p2p.access.1mn]#" min="0"
max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.p2p.access.10mn">Max searches in 10mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control" id="search.public.max.p2p.access.10mn"
name="search.public.max.p2p.access.10mn" type="number"
value="#[search.public.max.p2p.access.10mn]#" min="0"
max="2147483647" />
</div>
</div>
</fieldset>

<fieldset>
<legend>Remote snippet load</legend>

<p>Limitations on snippet loading from remote websites. When a
user with limited rights (unauthenticated or without extended search
right) exceeds a limit, the snippets fetch strategy falls back to
'CACHEONLY' (check the default Snippet Fetch Strategy on the <a href="ConfigPortal_p.html">Search Portal</a> configuration page).</p>

<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.remoteSnippet.access.3s">Max
searches in 3s</label>
<div class="col-sm-3 col-md-2">
<input class="form-control"
id="search.public.max.remoteSnippet.access.3s"
name="search.public.max.remoteSnippet.access.3s" type="number"
value="#[search.public.max.remoteSnippet.access.3s]#" min="0"
max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.remoteSnippet.access.1mn">Max
searches in 1mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control"
id="search.public.max.remoteSnippet.access.1mn"
name="search.public.max.remoteSnippet.access.1mn" type="number"
value="#[search.public.max.remoteSnippet.access.1mn]#" min="0"
max="2147483647" />
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-6 col-md-4 col-lg-3"
for="search.public.max.remoteSnippet.access.10mn">Max
searches in 10mn</label>
<div class="col-sm-3 col-md-2">
<input class="form-control"
id="search.public.max.remoteSnippet.access.10mn"
name="search.public.max.remoteSnippet.access.10mn" type="number"
value="#[search.public.max.remoteSnippet.access.10mn]#" min="0"
max="2147483647" />
</div>
</div>
</fieldset>

<div class="form-group">
<div class="col-xs-offset-1 col-sm-offset-3 col-md-offset-3 col-lg-offset-2">
<input type="submit" class="btn btn-primary" name="set"
value="Submit" aria-describedby="changeInfo" /> <input
type="submit" class="btn btn-default" name="setDefaults"
value="Set defaults" title="Reset to defaults settings"
aria-describedby="changeInfo" /> <em id="changeInfo">Changes
will take effect immediately.</em>
</div>
</div>
</form>

#%env/templates/footer.template%#
</body>
</html>
@@ -0,0 +1,85 @@

// SearchAccessRate_p.java
// Copyright 2019 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import java.util.Properties;

import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.TransactionManager;
import net.yacy.search.SearchAccessRateConstants;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;

/**
* Handle configuration of access rate limitations to the peer search interface
* through the SearchAccessRate_p.html page.
*/
public class SearchAccessRate_p {

/**
* @param header the current request headers
* @param post the request parameters
* @param env holds the server environment
* @return a serverObjects instance filled with the properties required by the
* related template
*/
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();

/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));

if (post != null) {
/*
* Check the transaction is valid : validation apply then for every uses of the
* post parameters
*/
TransactionManager.checkPostTransaction(header, post);

if (post.containsKey("set")) {
/* Setting configuration values */
for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
sb.setConfig(config.getKey(), Math.max(0, post.getInt(config.getKey(), config.getDefaultValue())));
}
} else if (post.containsKey("setDefaults")) {
/* Resetting to defaults */
final Properties defaultConfig = sb.loadDefaultConfig();

for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
sb.setConfig(config.getKey(),
defaultConfig.getProperty(config.getKey(), String.valueOf(config.getDefaultValue())));
}

}

}

for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
/* Fill prop for template rendering */
prop.put(config.getKey(), sb.getConfigInt(config.getKey(), config.getDefaultValue()));
}

return prop;
}

}
@@ -15,6 +15,7 @@
<ul class="SubMenu">
<li><a href="AccessTracker_p.html?page=2" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log</a></li>
<li><a href="AccessTracker_p.html?page=3" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Host Tracker</a></li>
<li><a href="SearchAccessRate_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Access Rate Limitations</a></li>
</ul>
</div>

0 comments on commit 0ab2b49

Please sign in to comment.
You can’t perform that action at this time.