Skip to content
Permalink
Browse files

Optionally allow external Solr server with self-signed certificate

This is necessary when you want to attach to a dedicated external Solr
server protected with basic http authentication and requested over https
but having only a self-signed certificate.
  • Loading branch information...
luccioman committed Apr 4, 2018
1 parent 1e4ceaa commit 69690c13a01367976e13b2b6c932fa43c3df96ae
@@ -1285,6 +1285,8 @@ federated.service.solr.indexing.sharding = MODULO_HOST_MD5
federated.service.solr.indexing.lazy = true
federated.service.solr.indexing.timeout = 60000
federated.service.solr.indexing.writeEnabled = true
# Set to true when TLS is enabled with a self-signed certificated on a main external Solr with authentication required ( its URL looks like https://user:password@localhost:8984/solr/)
federated.service.solr.indexing.authenticated.allowSelfSigned = false

# temporary definition of backend services to use.
# After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db.
@@ -36,6 +36,10 @@ <h2>Index Sources &amp; Targets</h2>
<dt>Use remote Solr server(s)&nbsp;<input type="checkbox" name="solr.indexing.solrremote" id="solr_indexing_solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# onclick="if(!document.getElementById('config').solr_indexing_solrremote.checked) {document.getElementById('config').core_service_fulltext.checked = true;}"/></dt>
<dd>It's easy to <a href="http://www.yacy-websearch.net/wiki/index.php/Dev:Solr" target="_blank">attach an external Solr to YaCy</a>.
This external Solr can be used instead the internal Solr. It can also be used additionally to the internal Solr, then both Solr indexes are mirrored.
</dd>

<dt>Allow self-signed certificates <input type="checkbox" name="federated.service.solr.indexing.authenticated.allowSelfSigned" #(federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#:: checked="checked"#(/federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#/></dt>
<dd>Tick this when the remote Solr server is password protected and is requested over HTTPS but provides only a self-signed certificate (not a validated one by an official Certificate Authority). The Solr URL could be for example something like <i>https://user:password@localhost:8984/solr</i>.
</dd>

#(table)#::
@@ -104,6 +104,10 @@ public static serverObjects respond(final RequestHeader header, final serverObje
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);

env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post
.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));

final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
final StringBuilder s = new StringBuilder();
String s0;
@@ -142,10 +146,13 @@ public static serverObjects respond(final RequestHeader header, final serverObje
// switch on
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = sb.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 10000);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);

try {
if (usesolr) {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@@ -191,6 +198,9 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",
env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
int mc = 0;
@@ -96,7 +96,13 @@ public void setCoreName(String core) {
msp.add(CommonParams.QT, "/"); // important to override default append of /select
msp.add(CommonParams.ROWS, Integer.toString(query.itemsPerPage));
try {
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000);
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
}
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000, trustSelfSignedOnAuthenticatedServer);
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
@@ -135,7 +135,7 @@ public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOE
public static void main(final String args[]) {
RemoteSolrConnector solr;
try {
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000);
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000, false);
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
instances.add(instance);
solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, true), true, "solr");
@@ -77,18 +77,51 @@
private final Map<String, ConcurrentUpdateSolrClient> server;
private final int timeout;

public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
/**
* @param urlList
* the list of URLs of remote Solr shard instances. Must not be null.
* @param coreNames
* the Solr core names for the main collection and the webgraph
* @param defaultCoreName
* the core name of the main collection
* @param timeout
* the connection timeout in milliseconds
* @param trustSelfSignedOnAuthenticatedServer
* when true, self-signed certificates are accepcted for an https
* connection to a remote server with authentication credentials
* @throws IOException
* when a connection could not be opened to a remote Solr instance
*/
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames,
String defaultCoreName, final int timeout, final boolean trustSelfSignedOnAuthenticatedServer)
throws IOException {
urlList.replace(' ', ',');
String[] urls = CommonPattern.COMMA.split(urlList);
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
for (final String u: urls) {
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout);
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer);
instances.add(instance);
}
return instances;
}

public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
/**
* @param url
* the remote Solr URL. A default localhost URL is assumed when null.
* @param coreNames
* the Solr core names for the main collection and the webgraph
* @param defaultCoreName
* the core name of the main collection
* @param timeout
* the connection timeout in milliseconds
* @param trustSelfSignedOnAuthenticatedServer
* when true, self-signed certificates are accepcted for an https
* connection to a remote server with authentication credentials
* @throws IOException
* when a connection could not be opened to the remote Solr instance
*/
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer) throws IOException {
this.timeout = timeout;
this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
@@ -140,8 +173,7 @@ public RemoteInstance(final String url, final Collection<String> coreNames, fina
}
}
if (solraccount.length() > 0) {
/* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, trustSelfSignedOnAuthenticatedServer);
} else if(u.isHTTPS()){
/* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);
@@ -1102,7 +1102,14 @@ private void logError(String messageBegin, Exception ex) {
@Override
public void run() {
try {
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
}

this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
@@ -255,7 +255,11 @@ private void processSingleTarget(Seed seed) {
solrQuery.set(CommonParams.ROWS, sb.getConfig(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, "20"));
this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")");
try {
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection
final boolean trustSelfSignedOnAuthenticatedServer = this.sb.getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);

RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
try {
SolrConnector solrConnector = new RemoteSolrConnector(instance,
sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
@@ -585,10 +585,13 @@ public void run() {
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);

if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
@@ -1486,10 +1489,13 @@ public void switchNetwork(final String networkDefinition) throws FileNotFoundExc
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);

if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
@@ -316,6 +316,13 @@
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT = "federated.service.solr.indexing.timeout";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED = "federated.service.solr.indexing.writeEnabled";

/** Setting key controlling whether a self-signed certificate is acceptable from a remote Solr instance requested with authentication credentials.
* This has no impact on connections to remote Solr instances used in p2p search for which self-signed certificates are always accepted. */
public static final String FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED = "federated.service.solr.indexing.authenticated.allowSelfSigned";

/** Default value controlling whether a self-signed certificate is acceptable from a remote Solr instance with authentication credentials. */
public static final boolean FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT = false;

public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext";
public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp";

0 comments on commit 69690c1

Please sign in to comment.
You can’t perform that action at this time.