Permalink
Browse files

Added a new Debug/Analysis advanced settings subsection.

As discussed in PR #93 with @JeremyRand and @reger24 this new advanced
settings page includes:
 - a new setting to control remote Solr responses encoding
 - some existing debug settings which could not be set through the admin
user interface
  • Loading branch information...
luccioman committed Feb 9, 2017
1 parent 526f2d6 commit 185765198887d3610258586ab22f0d0c6438eaee
@@ -890,6 +890,9 @@ remotesearch.result.store.maxsize=-1
remotesearch.maxload.rwi=8.0
remotesearch.maxload.solr=4.0
# Control whether remote Solr instances responses should be binary encoded. Responses are transferred as XML when set to false.
remote.solr.binaryResponse.enabled=true
# specifies if yacy should set it's own referer if no referer URL
# was set by the client.
useYacyReferer = false
@@ -190,7 +190,9 @@ <h2>Settings Receipt:</h2>
<p>the change will take effect after restart.</p>
<p>Note: the SSL option must be switched on, see <a href="ConfigBasic.html">Basic Configuration</a></p>
::<!-- 33: Url/Web proxy settings changed -->
<p>URL Proxy settings have been saved.</p>
<p>URL Proxy settings have been saved.</p>
::<!-- 34: Debug/Analysis settings changed -->
<p>Debug/Analysis settings have been saved.</p>
#(/info)#
<p></p>
#(needsRestart)#
@@ -42,6 +42,7 @@
import net.yacy.peers.operation.yacySeedUploader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -512,6 +513,39 @@ else if (!filter.equals("*")){
prop.put("info", "32");
return prop;
}
// Debug/Analysis settings
if (post.containsKey("debugAnalysisSettings")) {
boolean tickedCheckbox = post.containsKey("solrBinaryResponse");
env.setConfig(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, tickedCheckbox);
tickedCheckbox = post.containsKey("searchTestLocalDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, tickedCheckbox);
tickedCheckbox = post.containsKey("searchTestLocalSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, tickedCheckbox);
/* For easier user understanding, the following flags controlling data sources selection
* are rendered in the UI as checkboxes corresponding to enabled value when ticked */
tickedCheckbox = post.containsKey("searchLocalDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, !tickedCheckbox);
tickedCheckbox = post.containsKey("searchLocalSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, !tickedCheckbox);
tickedCheckbox = post.containsKey("searchRemoteDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, !tickedCheckbox);
tickedCheckbox = post.containsKey("searchRemoteSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, !tickedCheckbox);
/* Let's clean up all search events as these settings affect how search is performed and we don't
* want cached results obtained with the previous settings */
SearchEventCache.cleanupEvents(true);
prop.put("info", "34");
return prop;
}
// nothing made
prop.put("info", "1");//no information submitted
@@ -0,0 +1,125 @@
<h3>Debug/Analysis Settings</h3>
<div class="alert alert-warning" role="alert">
Be careful with these advanced settings, they can deeply affect the search process! You probably don't need to modify them for normal use.
</div>
<form action="SettingsAck_p.html" method="post" enctype="multipart/form-data" class="form-horizontal">
<fieldset>
<legend>Solr communication</legend>
<div class="form-group">
<div class="col-sm-4">
<div class="checkbox">
<label>
<input name="solrBinaryResponse" id="solrBinaryResponse"
type="checkbox" #(solrBinaryResponseChecked)#::checked#(/solrBinaryResponseChecked)#
aria-describedby="solrBinaryResponseInfo"/>
Enable remote Solr binary responses
</label>
</div>
</div>
<div class="col-sm-8" id="solrBinaryResponseInfo">
When checked (default), responses from remote Solr index instances are transfered using an efficient binary data format.
When unchecked, responses are transferred as <abbr title="Extensible Markup Language">XML</abbr>,
which can be captured and parsed by any external XML aware tool for debug/analysis.
</div>
</div>
</fieldset>
<fieldset>
<legend>Search data sources</legend>
<p>By default all data sources are enabled to obtain search results,
but you can here disable one or more ones to check the behavior of the process.</p>
<div class="form-group">
<div class="col-sm-12">
<div class="checkbox">
<label>
<input name="searchLocalDHT" id="searchLocalDHT"
type="checkbox" #(searchLocalDHTChecked)#::checked#(/searchLocalDHTChecked)# />
Local <abbr title="Distributed Hash Table">DHT</abbr>/<abbr title="Reverse Word Index">RWI</abbr>
</label>
</div>
</div>
</div>
<div class="form-group">
<div class="col-sm-12">
<div class="checkbox">
<label>
<input name="searchLocalSolr" id="searchLocalSolr"
type="checkbox" #(searchLocalSolrChecked)#::checked#(/searchLocalSolrChecked)# />
Local Solr index
</label>
</div>
</div>
</div>
<div class="form-group">
<div class="col-sm-12">
<div class="checkbox">
<label>
<input name="searchRemoteDHT" id="searchRemoteDHT"
type="checkbox" #(searchRemoteDHTChecked)#::checked#(/searchRemoteDHTChecked)# />
Remote <abbr title="Distributed Hash Table">DHT</abbr>/<abbr title="Reverse Word Index">RWI</abbr>
</label>
</div>
</div>
</div>
<div class="form-group">
<div class="col-sm-12">
<div class="checkbox">
<label>
<input name="searchRemoteSolr" id="searchRemoteSolr"
type="checkbox" #(searchRemoteSolrChecked)#::checked#(/searchRemoteSolrChecked)# />
Remote Solr indexes
</label>
</div>
</div>
</div>
</fieldset>
<fieldset>
<legend>Search testing tweaks</legend>
<div class="form-group">
<div class="col-sm-4">
<div class="checkbox">
<label>
<input name="searchTestLocalDHT" id="searchTestLocalDHT"
type="checkbox" #(searchTestLocalDHTChecked)#::checked#(/searchTestLocalDHTChecked)#
aria-describedby="searchTestLocalDHTInfo"/>
Override <abbr title="Distributed Hash Table">DHT</abbr> peers selection by local only
</label>
</div>
</div>
<div class="col-sm-8" id="searchTestLocalDHTInfo">
When checked, the remote <abbr title="Distributed Hash Table">DHT</abbr> peers selection is overriden and only the local peer is selected to provide remote DHT search results.
</div>
</div>
<div class="form-group">
<div class="col-sm-4">
<div class="checkbox">
<label>
<input name="searchTestLocalSolr" id="searchTestLocalSolr"
type="checkbox" #(searchTestLocalSolrChecked)#::checked#(/searchTestLocalSolrChecked)#
aria-describedby="searchTestLocalSolrInfo"/>
Override Solr peers selection by local only
</label>
</div>
</div>
<div class="col-sm-8" id="searchTestLocalSolrInfo">
When checked, the remote Solr peers selection is overriden and only this peer is selected to provide remote Solr search results.
</div>
</div>
</fieldset>
<div class="col-sm-6">
<input type="submit" class="btn btn-primary" name="debugAnalysisSettings" value="Submit" aria-describedby="submitInfo"/>
<em id="submitInfo">Changes will take effect immediately.</em>
</div>
</form>
@@ -1,5 +1,5 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<!DOCTYPE html>
<html lang="en">
<head>
<title>YaCy '#[clientname]#': Advanced Settings</title>
#%env/templates/metas.template%#
@@ -27,6 +27,7 @@ <h2>Advanced Settings</h2>
<li><a href="?page=ProxyAccess">Transparent Proxy Access Settings</a></li>
<li><a href="?page=UrlProxyAccess">URL/Web Proxy Access Settings</a></li>
<li><a href="?page=proxy">Remote Proxy (optional)</a></li>
<li><a href="?page=debug">Debug/Analysis Settings</a></li>
</ul>
</td>
</tr>
@@ -72,6 +72,8 @@ else if (page.equals("parser")) {
}
else if (page.equals("crawler")) {
prop.put("settingsTables", "Settings_Crawler.inc");
} else if (page.equals("debug")) {
prop.put("settingsTables", "Settings_Debug.inc");
} else {
prop.put("settingsTables", "");
}
@@ -196,6 +198,20 @@ else if (page.equals("crawler")) {
prop.put("server.https",sb.getConfigBool("server.https", false));
prop.put("server.https_port.ssl", sb.getConfig("port.ssl","8443"));
// debug/analysis
prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0);
/* For easier user understanding, the following flags controlling data sources selection
* are rendered in the UI as checkboxes corresponding to enabled value when ticked */
prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0);
prop.put("searchLocalSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false) ? 1 : 0);
prop.put("searchRemoteDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false) ? 1 : 0);
prop.put("searchRemoteSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false) ? 1 : 0);
prop.put("searchTestLocalDHTChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false) ? 1 : 0);
prop.put("searchTestLocalSolrChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false) ? 1 : 0);
// return rewrite properties
return prop;
}
@@ -29,6 +29,8 @@
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.QueryParams;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrDocument;
@@ -96,7 +98,13 @@ public void setCoreName(String core) {
try {
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000);
try {
SolrConnector solrConnector = new RemoteSolrConnector(instance, false, null);
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT);
}
SolrConnector solrConnector = new RemoteSolrConnector(instance, useBinaryResponseWriter);
try {
this.lastaccesstime = System.currentTimeMillis();
SolrDocumentList docList = solrConnector.getDocumentListByParams(msp);
@@ -33,6 +33,8 @@
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
public class InstanceMirror {
@@ -157,7 +159,13 @@ public RemoteSolrConnector getRemoteConnector(String corename) {
if (this.remoteSolrInstance == null) return null;
RemoteSolrConnector rsc = this.remoteConnectorCache.get(corename);
if (rsc != null) return rsc;
rsc = new RemoteSolrConnector(this.remoteSolrInstance, true, corename);
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT);
}
rsc = new RemoteSolrConnector(this.remoteSolrInstance, useBinaryResponseWriter, corename);
this.remoteConnectorCache.put(corename, rsc);
return rsc;
}
@@ -1077,7 +1077,13 @@ public void run() {
try {
this.instance = new RemoteInstance("http://" + this.address, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
try {
this.solrConnector = new RemoteSolrConnector(this.instance, this.mySeed ? true : this.target.getVersion() >= 1.63, "solr");
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT);
}
this.solrConnector = new RemoteSolrConnector(this.instance, useBinaryResponseWriter && (this.mySeed ? true : this.target.getVersion() >= 1.63), "solr");
if (!solrConnector.isClosed() && !this.closed) {
try {
this.rsp[0] = this.solrConnector.getResponseByParams(solrQuery);
@@ -250,7 +250,10 @@ private void processSingleTarget(Seed seed) {
try {
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection
try {
SolrConnector solrConnector = new RemoteSolrConnector(instance, true, null);
SolrConnector solrConnector = new RemoteSolrConnector(instance,
sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT),
null);
if (!solrConnector.isClosed()) {
try {
QueryResponse rsp = solrConnector.getResponseByParams(solrQuery);
@@ -286,6 +286,20 @@
public static final String REMOTESEARCH_RESULT_STORE_MAXSIZE= "remotesearch.result.store.maxsize";
public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi";
public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr";
/**
* Setting key to configure whether responses from remote Solr instances
* should be binary encoded :
* <ul>
* <li>true : more efficient, uses the solrj binary response parser</li>
* <li>false : responses are transferred as XML, which can be captured and
* parsed by any external XML aware tool for debug/analysis</li>
* </ul>
*/
public static final String REMOTE_SOLR_BINARY_RESPONSE_ENABLED = "remote.solr.binaryResponse.enabled";
/** Default configuration setting for remote Solr responses binary encoding */
public static final boolean REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT = true;
public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";
@@ -314,15 +328,25 @@
public static final String CRAWLER_USER_AGENT_MINIMUMDELTA = "crawler.userAgent.minimumdelta";
public static final String CRAWLER_USER_AGENT_CLIENTTIMEOUT = "crawler.userAgent.clienttimeout";
/**
* debug flags
*/
public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off"; // =true: do not use the local dht/rwi index (which is not done if we do remote searches)
public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; // =true: do not use solr
public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; // =true: do not use dht/rwi
public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server
public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off"; // =true: do not use solr
public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server
/* --- debug flags --- */
/** when set to true : do not use the local dht/rwi index (which is not done if we do remote searches) */
public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off";
/** when set to true : do not use local solr index */
public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off";
/** when set to true : do not use remote dht/rwi */
public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off";
/** when set to true : do not use remote solr indexes */
public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off";
/** when set to true : do not use dht, search local peer in a shortcut to the own server */
public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal";
/** when set to true : do not use dht, search local peer in a shortcut to the own server */
public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal";
/**
* <p><code>public static final String <strong>WORDCACHE_MAX_COUNT</strong> = "wordCacheMaxCount"</code></p>
@@ -71,6 +71,7 @@
import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.schema.CollectionConfiguration;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphConfiguration;
@@ -178,7 +179,13 @@ public SolrConnector getConnectorForRead(String corename) {
public RemoteSolrConnector getDefaultRemoteSolrConnector() {
try {
return this.solrInstances.getDefaultRemoteConnector(true);
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT);
}
return this.solrInstances.getDefaultRemoteConnector(useBinaryResponseWriter);
} catch (IOException e) {
return null;
}

0 comments on commit 1857651

Please sign in to comment.