made the solr connection more generic

yacy · May 9, 2012 · c00efc2 · c00efc2
1 parent ea2bd43
commit c00efc2
Show file tree

Hide file tree

Showing 12 changed files with 377 additions and 364 deletions.
diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java
@@ -32,14 +32,15 @@
 import net.yacy.cora.document.UTF8;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.services.federated.solr.SolrConnector;
-import net.yacy.cora.services.federated.solr.SolrScheme;
 import net.yacy.cora.services.federated.solr.SolrShardingConnector;
 import net.yacy.cora.services.federated.solr.SolrShardingSelection;
 import net.yacy.cora.services.federated.solr.SolrSingleConnector;
 import net.yacy.cora.storage.ConfigurationSet;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.search.Switchboard;
 import net.yacy.search.index.Segments;
+import net.yacy.search.index.SolrField;
+import net.yacy.search.index.SolrScheme;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -92,7 +93,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                 // switch on
                 final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
                 try {
-                    sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrShardingConnector(solrurls, scheme, SolrShardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
+                    sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrShardingConnector(solrurls, SolrShardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
                 } catch (final IOException e) {
                     Log.logException(e);
                     sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(null);
@@ -138,21 +139,17 @@ public static serverObjects respond(final RequestHeader header, final serverObje
         }
 
         // write scheme
-        SolrScheme scheme = (sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr() == null) ? null : sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr().getScheme();
         final String schemename = sb.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list");
-        if (scheme == null) {
-            scheme = new SolrScheme(new File(env.getDataPath(), "DATA/SETTINGS/" + schemename));
-        }
-        final Iterator<ConfigurationSet.Entry> i = scheme.allIterator();
+        final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.allIterator();
 
         int c = 0;
         boolean dark = false;
         ConfigurationSet.Entry entry;
-        SolrScheme.Field field;
+        SolrField field;
         while (i.hasNext()) {
             entry = i.next();
             try {
-                field = SolrScheme.Field.valueOf(entry.key());
+                field = SolrField.valueOf(entry.key());
             } catch (IllegalArgumentException e) {
                 continue;
             }

diff --git a/htroot/api/schema_p.java b/htroot/api/schema_p.java
@@ -22,15 +22,12 @@
  *  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import java.io.File;
 import java.util.Iterator;
 
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.services.federated.solr.SolrScheme;
-import net.yacy.cora.services.federated.solr.SolrScheme.Field;
 import net.yacy.cora.storage.ConfigurationSet;
 import net.yacy.search.Switchboard;
-import net.yacy.search.index.Segments;
+import net.yacy.search.index.SolrField;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -42,21 +39,16 @@ public static serverObjects respond(final RequestHeader header, final serverObje
         final Switchboard sb = (Switchboard) env;
 
         // write scheme
-        SolrScheme scheme = (sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr() == null) ? null : sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr().getScheme();
-        final String schemename = sb.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list");
-        if (scheme == null) {
-            scheme = new SolrScheme(new File(env.getDataPath(), "DATA/SETTINGS/" + schemename));
-        }
-        final Iterator<ConfigurationSet.Entry> i = scheme.allIterator();
+        final Iterator<ConfigurationSet.Entry> i = sb.solrScheme.allIterator();
 
         int c = 0;
         ConfigurationSet.Entry entry;
-        SolrScheme.Field field = null;
+        SolrField field = null;
         while (i.hasNext()) {
             entry = i.next();
             if (!entry.enabled()) continue; //scheme.contains(entry.key())
             try {
-                field = Field.valueOf(entry.key());
+                field = SolrField.valueOf(entry.key());
             } catch (IllegalArgumentException e) {
                 continue;
             }

diff --git a/source/net/yacy/cora/services/federated/solr/SolrConnector.java b/source/net/yacy/cora/services/federated/solr/SolrConnector.java
@@ -27,8 +27,6 @@
 import java.io.IOException;
 import java.util.List;
 
-import net.yacy.cora.protocol.ResponseHeader;
-import net.yacy.document.Document;
 import net.yacy.kelondro.data.meta.DigestURI;
 
 import org.apache.solr.common.SolrDocumentList;
@@ -37,12 +35,6 @@
 
 public interface SolrConnector {
 
-    /**
-     * with a scheme the fields of a SolrDocument can be translated to actual data values
-     * @return the solr scheme that can translate the SolrDocument
-     */
-    public SolrScheme getScheme();
-
     public void close();
 
     /**
@@ -73,15 +65,6 @@ public interface SolrConnector {
      */
     public boolean exists(final String id) throws IOException;
 
-    /**
-     * add a YaCy document. This calls the scheme processor to add the document as solr document
-     * @param id the url hash of the entry
-     * @param header the http response header
-     * @param doc the YaCy document
-     * @throws IOException
-     */
-    public void add(final String id, final ResponseHeader header, final Document doc) throws IOException;
-
     /**
      * add a solr input document
      * @param solrdoc

diff --git a/source/net/yacy/cora/services/federated/solr/SolrField.java b/source/net/yacy/cora/services/federated/solr/SolrField.java
@@ -0,0 +1,32 @@
+/**
+ *  SolrField
+ *  Copyright 2011 by Michael Peter Christen
+ *  First released 14.04.2011 at http://yacy.net
+ *
+ *  $LastChangedDate: 2011-04-14 22:05:04 +0200 (Do, 14 Apr 2011) $
+ *  $LastChangedRevision: 7654 $
+ *  $LastChangedBy: orbiter $
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.cora.services.federated.solr;
+
+
+public interface SolrField {
+
+    public String name();
+
+}
diff --git a/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java b/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java
@@ -27,8 +27,6 @@
 import java.io.IOException;
 import java.util.List;
 
-import net.yacy.cora.protocol.ResponseHeader;
-import net.yacy.document.Document;
 import net.yacy.kelondro.data.meta.DigestURI;
 
 import org.apache.solr.common.SolrDocumentList;
@@ -45,11 +43,6 @@ public SolrRetryConnector(final SolrConnector solrConnector, final long retryMax
         this.retryMaxTime = retryMaxTime;
     }
 
-    @Override
-    public SolrScheme getScheme() {
-        return this.solrConnector.getScheme();
-    }
-
     @Override
     public void close() {
         this.solrConnector.close();
@@ -115,21 +108,6 @@ public boolean exists(final String id) throws IOException {
         return false;
     }
 
-    @Override
-    public void add(final String id, final ResponseHeader header, final Document doc) throws IOException {
-        final long t = System.currentTimeMillis() + this.retryMaxTime;
-        Throwable ee = null;
-        while (System.currentTimeMillis() < t) try {
-            this.solrConnector.add(id, header, doc);
-            return;
-        } catch (final Throwable e) {
-            ee = e;
-            try {Thread.sleep(10);} catch (final InterruptedException e1) {}
-            continue;
-        }
-        if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
-    }
-
     @Override
     public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
         final long t = System.currentTimeMillis() + this.retryMaxTime;

diff --git a/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java b/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java
@@ -31,8 +31,7 @@
 import java.util.List;
 
 import net.yacy.cora.protocol.Domains;
-import net.yacy.cora.protocol.ResponseHeader;
-import net.yacy.document.Document;
+import net.yacy.cora.services.federated.solr.SolrShardingSelection.Method;
 import net.yacy.kelondro.data.meta.DigestURI;
 
 import org.apache.solr.common.SolrDocument;
@@ -43,25 +42,20 @@
 public class SolrShardingConnector implements SolrConnector {
 
     private final List<SolrConnector> connectors;
-    private final SolrScheme scheme;
     private final SolrShardingSelection sharding;
     private final String[] urls;
 
-    public SolrShardingConnector(final String urlList, final SolrScheme scheme, final SolrShardingSelection.Method method, final long timeout) throws IOException {
+    public SolrShardingConnector(final String urlList, final SolrShardingSelection.Method method, final long timeout) throws IOException {
         urlList.replace(' ', ',');
         this.urls = urlList.split(",");
         this.connectors = new ArrayList<SolrConnector>();
         for (final String u: this.urls) {
-            this.connectors.add(new SolrRetryConnector(new SolrSingleConnector(u.trim(), scheme), timeout));
+            this.connectors.add(new SolrRetryConnector(new SolrSingleConnector(u.trim()), timeout));
         }
         this.sharding = new SolrShardingSelection(method, this.urls.length);
-        this.scheme = scheme;
-    }
-
-    public SolrScheme getScheme() {
-        return this.scheme;
     }
 
+    @Override
     public void close() {
         for (final SolrConnector connector: this.connectors) connector.close();
     }
@@ -70,6 +64,7 @@ public void close() {
      * delete everything in the solr index
      * @throws IOException
      */
+    @Override
     public void clear() throws IOException {
         for (final SolrConnector connector: this.connectors) connector.clear();
     }
@@ -79,6 +74,7 @@ public void clear() throws IOException {
      * @param id the url hash of the entry
      * @throws IOException
      */
+    @Override
     public void delete(final String id) throws IOException {
         for (final SolrConnector connector: this.connectors) connector.delete(id);
     }
@@ -88,6 +84,7 @@ public void delete(final String id) throws IOException {
      * @param ids a list of url hashes
      * @throws IOException
      */
+    @Override
     public void delete(final List<String> ids) throws IOException {
         for (final SolrConnector connector: this.connectors) connector.delete(ids);
     }
@@ -98,29 +95,20 @@ public void delete(final List<String> ids) throws IOException {
      * @return true if any entry in solr exists
      * @throws IOException
      */
+    @Override
     public boolean exists(final String id) throws IOException {
         for (final SolrConnector connector: this.connectors) {
             if (connector.exists(id)) return true;
         }
         return false;
     }
 
-    /**
-     * add a YaCy document. This calls the scheme processor to add the document as solr document
-     * @param id the url hash of the entry
-     * @param header the http response header
-     * @param doc the YaCy document
-     * @throws IOException
-     */
-    public void add(final String id, final ResponseHeader header, final Document doc) throws IOException {
-        add(this.scheme.yacy2solr(id, header, doc));
-    }
-
     /**
      * add a Solr document
      * @param solrdoc
      * @throws IOException
      */
+    @Override
     public void add(final SolrInputDocument solrdoc) throws IOException {
         this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc);
     }
@@ -141,6 +129,7 @@ protected void addSolr(final Collection<SolrInputDocument> docs) throws IOExcept
      * @param httpstatus
      * @throws IOException
      */
+    @Override
     public void err(final DigestURI digestURI, final String failReason, final int httpstatus) throws IOException {
         this.connectors.get(this.sharding.selectURL(digestURI.toNormalform(true, false))).err(digestURI, failReason, httpstatus);
     }
@@ -152,6 +141,7 @@ public void err(final DigestURI digestURI, final String failReason, final int ht
      * @param querystring
      * @throws IOException
      */
+    @Override
     public SolrDocumentList get(final String querystring, final int offset, final int count) throws IOException {
         final SolrDocumentList list = new SolrDocumentList();
         for (final SolrConnector connector: this.connectors) {
@@ -181,6 +171,7 @@ public long[] getSizeList() {
         return size;
     }
 
+    @Override
     public long getSize() {
         final long[] size = getSizeList();
         long s = 0;

diff --git a/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java b/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java
@@ -36,8 +36,6 @@
 import net.yacy.cora.document.ASCII;
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.cora.protocol.Domains;
-import net.yacy.cora.protocol.ResponseHeader;
-import net.yacy.document.Document;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.logging.Log;
 
@@ -65,7 +63,6 @@ public class SolrSingleConnector implements SolrConnector {
     private final String solrurl, host, solrpath, solraccount, solrpw;
     private final int port;
     private HttpSolrServer server;
-    private final SolrScheme scheme;
 
     private final static int transmissionQueueCount = 4; // allow concurrent http sessions to solr
     private final static int transmissionQueueSize = 50; // number of documents that are collected until a commit is sent
@@ -80,9 +77,8 @@ public class SolrSingleConnector implements SolrConnector {
      * @throws IOException
      */
     @SuppressWarnings("unchecked")
-    public SolrSingleConnector(final String url, final SolrScheme scheme) throws IOException {
+    public SolrSingleConnector(final String url) throws IOException {
         this.solrurl = url;
-        this.scheme = scheme;
         this.transmissionRoundRobinCounter = 0;
         this.transmissionQueue = new ArrayBlockingQueue[transmissionQueueCount];
         for (int i = 0; i < transmissionQueueCount; i++) {
@@ -187,11 +183,6 @@ public void close() {
         }
     }
 
-    @Override
-    public SolrScheme getScheme() {
-        return this.scheme;
-    }
-
     @Override
     public long getSize() {
         try {
@@ -261,11 +252,6 @@ public void add(final File file, final String solrId) throws IOException {
         }
     }
 
-    @Override
-    public void add(final String id, final ResponseHeader header, final Document doc) throws IOException, SolrException {
-        add(this.scheme.yacy2solr(id, header, doc));
-    }
-
     @Override
     public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
         int thisrrc = this.transmissionRoundRobinCounter;
@@ -384,7 +370,8 @@ public String getAdminInterface() {
     public static void main(final String args[]) {
         SolrSingleConnector solr;
         try {
-            solr = new SolrSingleConnector("http://127.0.0.1:8983/solr", new SolrScheme());
+            //SolrScheme scheme = new SolrScheme();
+            solr = new SolrSingleConnector("http://127.0.0.1:8983/solr");
             solr.clear();
             final File exampleDir = new File("/Data/workspace2/yacy/test/parsertest/");
             long t, t0, a = 0;