Skip to content

Commit

Permalink
Full redesign of solr connection architecture. This was done to support
Browse files Browse the repository at this point in the history
multiple solr cores instead of just one. Therefore it is now necessary
to distuingish between solr server connections (called an 'Instance')
and a connection to a single solr core. One Instance may now have
multiple connector classes assigned to it, each connecting to a single
core.
To support multiple cores it is also necessary to distinguish between
the connection configuration and the configuration of the index schema.
We will have multiple schema configurations in the future, each for
every solr core. This caused that the IndexFederated servlet had to be
split into two parts, the new Servlet for the Schema editor is now in
the IndexSchema Servlet.
  • Loading branch information
Orbiter committed Feb 15, 2013
1 parent 4111606 commit b6de1f4
Show file tree
Hide file tree
Showing 35 changed files with 932 additions and 513 deletions.
16 changes: 8 additions & 8 deletions htroot/ConfigHeuristics_p.java
Expand Up @@ -72,7 +72,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}
if (post.containsKey("opensearch_off")) sb.setConfig("heuristic.opensearch", false);
if (post.containsKey("discoverosd")) {
final boolean metafieldNOTavailable = sb.index.fulltext().getSolrScheme().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
final boolean metafieldNOTavailable = sb.index.fulltext().getSolrSchema().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
if (!metafieldNOTavailable) {
OpenSearchConnector osc = new OpenSearchConnector(sb, false);
if (osc.discoverFromSolrIndex(sb)) {
Expand Down Expand Up @@ -102,24 +102,24 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}

if (post.containsKey("setopensearch")) {
// read index scheme table flags
// read index schema table flags
writeopensearchcfg (sb,post);
}

if (post.containsKey("switchsolrfieldson")) {
final boolean metafieldNOTavailable = sb.index.fulltext().getSolrScheme().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
final boolean metafieldNOTavailable = sb.index.fulltext().getSolrSchema().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
if (metafieldNOTavailable) {
ConfigurationSet.Entry entry;
entry = sb.index.fulltext().getSolrScheme().get(YaCySchema.outboundlinks_tag_txt.name());
entry = sb.index.fulltext().getSolrSchema().get(YaCySchema.outboundlinks_tag_txt.name());
if (entry != null && !entry.enabled()) {
entry.setEnable(true);
}
entry = sb.index.fulltext().getSolrScheme().get(YaCySchema.inboundlinks_tag_txt.name());
entry = sb.index.fulltext().getSolrSchema().get(YaCySchema.inboundlinks_tag_txt.name());
if (entry != null && !entry.enabled()) {
entry.setEnable(true);
}
try {
sb.index.fulltext().getSolrScheme().commit();
sb.index.fulltext().getSolrSchema().commit();
} catch (IOException ex) {}
}
}
Expand All @@ -139,7 +139,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}
}

final boolean showmetafieldbutton = sb.index.fulltext().getSolrScheme().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
final boolean showmetafieldbutton = sb.index.fulltext().getSolrSchema().containsDisabled(YaCySchema.outboundlinks_tag_txt.name());
if (showmetafieldbutton) prop.put("osdsolrfieldswitch",1);
prop.put("site.checked", sb.getConfigBool("heuristic.site", false) ? 1 : 0);
prop.put("searchresult.checked", sb.getConfigBool("heuristic.searchresults", false) ? 1 : 0);
Expand Down Expand Up @@ -175,7 +175,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}

private static void writeopensearchcfg(final Switchboard sb, final serverObjects post) {
// read index scheme table flags
// read index schema table flags

final File f = new File(sb.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");
ConfigurationSet cfg = new ConfigurationSet(f);
Expand Down
2 changes: 1 addition & 1 deletion htroot/CrawlResults.java
Expand Up @@ -55,7 +55,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po
final serverObjects prop = new serverObjects();

int lines = 500;
boolean showCollection = sb.index.fulltext().getSolrScheme().isEmpty() || sb.index.fulltext().getSolrScheme().contains(YaCySchema.collection_sxt);
boolean showCollection = sb.index.fulltext().getSolrSchema().isEmpty() || sb.index.fulltext().getSolrSchema().contains(YaCySchema.collection_sxt);
boolean showInit = env.getConfigBool("IndexMonitorInit", false);
boolean showExec = env.getConfigBool("IndexMonitorExec", false);
boolean showDate = env.getConfigBool("IndexMonitorDate", true);
Expand Down
2 changes: 1 addition & 1 deletion htroot/CrawlStartExpert_p.java
Expand Up @@ -76,7 +76,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("xdstopwChecked", env.getConfigBool("xdstopw", true) ? "1" : "0");
prop.put("xpstopwChecked", env.getConfigBool("xpstopw", true) ? "1" : "0");

boolean collectionEnabled = sb.index.fulltext().getSolrScheme().isEmpty() || sb.index.fulltext().getSolrScheme().contains(YaCySchema.collection_sxt);
boolean collectionEnabled = sb.index.fulltext().getSolrSchema().isEmpty() || sb.index.fulltext().getSolrSchema().contains(YaCySchema.collection_sxt);
prop.put("collectionEnabled", collectionEnabled ? 1 : 0);
prop.put("collection", collectionEnabled ? "user" : "");

Expand Down
32 changes: 3 additions & 29 deletions htroot/IndexFederated_p.html
@@ -1,7 +1,7 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Federated Index</title>
<title>YaCy '#[clientname]#': Remote Solr Configuration</title>
#%env/templates/metas.template%#
</head>
<body id="IndexFederated_p">
Expand All @@ -12,7 +12,7 @@
</div>
#%env/templates/header.template%#
#%env/templates/submenuIndexControl.template%#
<h2>Federated Index</h2>
<h2>Remote Solr Configuration</h2>
<p>
YaCy supports multiple index storage locations. At this time only the YaCy-internal search index can be used for the YaCy search interface
A Solr index storage location is optional. The local index storage location can be disabled.
Expand Down Expand Up @@ -64,38 +64,12 @@ <h2>Federated Index</h2>
<dt class="TableCellDark">Solr URL(s)</dt>
<dd><textarea rows="2" cols="80" name="solr.indexing.url" id="solr.indexing.url"/>#[solr.indexing.url]#</textarea><br/>
You can set one or more Solr targets here which are accessed as a shard. For several targets, list them using a ',' (comma) as separator.</dd>
<dt class="TableCellDark">Commit-Within (milliseconds)</dt>
<dd><input type="text" size="6" maxlength="6" value="#[solr.indexing.commitWithinMs]#" name="solr.indexing.commitWithinMs" id="solr.indexing.commitWithinMs"/> (increase this value to i.e. 180000 - 3 minutes - for more performance)</dd>
<dt class="TableCellDark">Lazy Value Initialization</dt>
<dd><input type="checkbox" name="solr.indexing.lazy" id="solr.indexing.lazy" #(solr.indexing.lazy.checked)#:: checked="checked"#(/solr.indexing.lazy.checked)# /> (if checked, only non-zero values and non-empty strings are written)</dd>
<dt class="TableCellDark">Sharding Method</dt>
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.sharding]#" name="solr.indexing.sharding" id="solr.indexing.sharding" disabled="disabled"/></dd>
<dt class="TableCellDark">Scheme</dt>
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.schemefile]#" name="solr.indexing.schemefile" id="solr.indexing.schemefile" disabled="disabled"/></dd>
<dt></dt><dd><input type="submit" name="set" value="Set" /></dd>
</dl>
</div>
<div>
<h3>Index Scheme</h3><p>If you use a custom Solr schema you may enter a different field name in the column 'Custom Solr Field Name' of the YaCy default attribute name</p>
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Active</td>
<td>Attribute</td>
<td>Custom Solr Field Name</td>
<td>Comment</td>
</tr>
#{scheme}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"><input type="checkbox" name="scheme_#[key]#" value="checked" #(checked)#::checked="checked"#(/checked)#/></td>
<td align="left">#[key]#</td>
<td align="left"><input type="text" name="scheme_solrfieldname_#[key]#" value="#[solrfieldname]#"/></td>
<td align="left">#[comment]#</td>
</tr>
#{/scheme}#
</table>
</div>
</fieldset>
<input type="submit" name="set" value="Set" />
</fieldset>
</form>

#%env/templates/footer.template%#
Expand Down
71 changes: 6 additions & 65 deletions htroot/IndexFederated_p.java
Expand Up @@ -3,10 +3,6 @@
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 25.05.2011 at http://yacy.net
*
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
* $LastChangedRevision: 7653 $
* $LastChangedBy: orbiter $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
Expand All @@ -26,17 +22,16 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.ArrayList;

import org.apache.solr.common.SolrException;

import net.yacy.cora.document.UTF8;
import net.yacy.cora.federate.solr.YaCySchema;
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
import net.yacy.cora.federate.solr.connector.ShardSelection;
import net.yacy.cora.federate.solr.connector.ShardSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.yacy.ConfigurationSet;
import net.yacy.cora.federate.solr.instance.SolrRemoteInstance;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.OS;
Expand Down Expand Up @@ -95,7 +90,6 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
boolean lazy = post.getBoolean("solr.indexing.lazy");
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
final StringBuilder s = new StringBuilder();
String s0;
Expand All @@ -113,11 +107,8 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}
solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, schemename);


if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off
try {
Expand All @@ -133,7 +124,8 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
try {
if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
ArrayList<SolrRemoteInstance> instances = ShardSolrConnector.getShardInstances(solrurls);
ShardSolrConnector solr = new ShardSolrConnector(instances, ShardSelection.Method.MODULO_HOST_MD5, true);
sb.index.fulltext().connectRemoteSolr(solr);
} else {
sb.index.fulltext().disconnectRemoteSolr();
Expand All @@ -149,35 +141,6 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
} catch (SolrException e) {
Log.logSevere("IndexFederated_p", "change of solr connection failed", e);
}

// read index scheme table flags
final Iterator<ConfigurationSet.Entry> i = sb.index.fulltext().getSolrScheme().entryIterator();
ConfigurationSet.Entry entry;
boolean modified = false; // flag to remember changes
while (i.hasNext()) {
entry = i.next();
final String v = post.get("scheme_" + entry.key());
final String sfn = post.get("scheme_solrfieldname_" + entry.key());
if (sfn != null ) {
// set custom solr field name
if (!sfn.equals(entry.getValue())) {
entry.setValue(sfn);
modified = true;
}
}
// set enable flag
final boolean c = v != null && v.equals("checked");
if (entry.enabled() != c) {
entry.setEnable(c);
modified = true;
}
}
if (modified) { // save settings to config file if modified
try {
sb.index.fulltext().getSolrScheme().commit();
modified = false;
} catch (IOException ex) {}
}
}

// show solr host table
Expand All @@ -187,7 +150,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("table", 1);
final SolrConnector solr = sb.index.fulltext().getRemoteSolr();
final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((RemoteSolrConnector) solr).getSize()};
final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((RemoteSolrConnector) solr).getAdminInterface()};
final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SolrRemoteInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterface()};
boolean dark = false;
for (int i = 0; i < size.length; i++) {
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;
Expand All @@ -197,34 +160,12 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("table_list", size.length);
}

// write scheme
final String schemename = sb.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list");

int c = 0;
boolean dark = false;
// use enum SolrField to keep defined order
for(YaCySchema field : YaCySchema.values()) {
prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("scheme_" + c + "_checked", sb.index.fulltext().getSolrScheme().contains(field.name()) ? 1 : 0);
prop.putHTML("scheme_" + c + "_key", field.name());
prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName());
if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment());
c++;
}
prop.put("scheme", c);

// fill attribute fields
// allowed values are: classic, solr, off
// federated.service.yacy.indexing.engine = classic

prop.put(SwitchboardConstants.CORE_SERVICE_FULLTEXT + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_RWI + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
prop.put("solr.indexing.schemefile", schemename);

if ((sb.index.fulltext().connectedURLDb())) {
prop.put("migrateUrlDbtoSolr", 1);
Expand Down
48 changes: 48 additions & 0 deletions htroot/IndexSchema_p.html
@@ -0,0 +1,48 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Solr Schema Editor</title>
#%env/templates/metas.template%#
</head>
<body id="IndexSchema_p">
<div id="api">
<a href="/api/schema.xml">
<img src="env/grafics/api.png" width="60" height="40" alt="API" /></a>
<span>The solr schema can also be retrieved as xml here. Click the API icon to see the xml. Just copy this xml to solr/conf/schema.xml to configure solr.</span>
</div>
#%env/templates/header.template%#
#%env/templates/submenuIndexControl.template%#
<h2>Solr Schema Editor</h2>
<p>If you use a custom Solr schema you may enter a different field name in the column 'Custom Solr Field Name' of the YaCy default attribute name</p>

<form action="IndexSchema_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<fieldset>
<div>
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Active</td>
<td>Attribute</td>
<td>Custom Solr Field Name</td>
<td>Comment</td>
</tr>
#{schema}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"><input type="checkbox" name="schema_#[key]#" value="checked" #(checked)#::checked="checked"#(/checked)#/></td>
<td align="left">#[key]#</td>
<td align="left"><input type="text" name="schema_solrfieldname_#[key]#" value="#[solrfieldname]#"/></td>
<td align="left">#[comment]#</td>
</tr>
#{/schema}#
</table>
<dl>
<dt class="TableCellDark">Lazy Value Initialization</dt>
<dd><input type="checkbox" name="lazy" id="lazy" #(lazy.checked)#:: checked="checked"#(/lazy.checked)# /> (if checked, only non-zero values and non-empty strings are written)</dd>
</dl>
</div>
</fieldset>
<input type="submit" name="set" value="Set" />
</form>

#%env/templates/footer.template%#
</body>
</html>

0 comments on commit b6de1f4

Please sign in to comment.