Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Add support for BigCouch update sequences. (EXPERIMENTAL)

  • Loading branch information...
commit 467a5494c112d5b301f33418a88d879b94702cb9 1 parent 85bf6fa
Robert Newson authored
10 pom.xml
View
@@ -38,6 +38,11 @@
<version>1.6</version>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.5</version>
+ </dependency>
+ <dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20090211</version>
@@ -73,6 +78,11 @@
<version>${tika-version}</version>
</dependency>
<dependency>
+ <groupId>org.erlang.otp</groupId>
+ <artifactId>jinterface</artifactId>
+ <version>1.5.3.2</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.5.6</version>
47 src/main/java/com/github/rnewson/couchdb/lucene/DatabaseIndexer.java
View
@@ -63,6 +63,7 @@
import com.github.rnewson.couchdb.lucene.couchdb.CouchDocument;
import com.github.rnewson.couchdb.lucene.couchdb.Database;
import com.github.rnewson.couchdb.lucene.couchdb.DesignDocument;
+import com.github.rnewson.couchdb.lucene.couchdb.UpdateSequence;
import com.github.rnewson.couchdb.lucene.couchdb.View;
import com.github.rnewson.couchdb.lucene.util.Analyzers;
import com.github.rnewson.couchdb.lucene.util.Constants;
@@ -79,7 +80,7 @@
private String etag;
private final Analyzer analyzer;
- private long pending_seq;
+ private UpdateSequence pending_seq;
private IndexReader reader;
private final IndexWriter writer;
private final Database database;
@@ -161,10 +162,10 @@ private void blockForLatest(final boolean staleOk) throws IOException, JSONExcep
if (staleOk) {
return;
}
- final long latest = database.getInfo().getUpdateSequence();
+ final UpdateSequence latest = database.getInfo().getUpdateSequence();
synchronized (this) {
long timeout = getSearchTimeout();
- while (pending_seq < latest) {
+ while (pending_seq.isEarlierThan(latest)) {
try {
final long start = System.currentTimeMillis();
wait(timeout);
@@ -179,8 +180,8 @@ private void blockForLatest(final boolean staleOk) throws IOException, JSONExcep
}
}
- private synchronized void setPendingSequence(final long newSequence) {
- pending_seq = newSequence;
+ private synchronized void setPendingSequence(final UpdateSequence seq) {
+ pending_seq = seq;
notifyAll();
}
@@ -223,7 +224,7 @@ private static long now() {
private final Database database;
- private long ddoc_seq;
+ private UpdateSequence ddoc_seq;
private long lastCommit;
@@ -237,7 +238,7 @@ private static long now() {
private final File root;
- private long since;
+ private UpdateSequence since;
private final Map<View, IndexState> states = Collections
.synchronizedMap(new HashMap<View, IndexState>());
@@ -319,7 +320,7 @@ public Void handleResponse(final HttpResponse response)
break loop;
}
- final long seq = json.getLong("seq");
+ final UpdateSequence seq = new UpdateSequence(json.getString("seq"));
final String id = json.getString("id");
CouchDocument doc;
if (!json.isNull("doc")) {
@@ -341,7 +342,7 @@ public Void handleResponse(final HttpResponse response)
}
if (id.startsWith("_design")) {
- if (seq > ddoc_seq) {
+ if (ddoc_seq.isEarlierThan(seq)) {
logger.info("Exiting due to design document change.");
break loop;
}
@@ -358,7 +359,7 @@ public Void handleResponse(final HttpResponse response)
final View view = entry.getKey();
final IndexState state = entry.getValue();
- if (seq > state.pending_seq) {
+ if (state.pending_seq.isEarlierThan(seq)) {
final Document[] docs;
try {
docs = state.converter.convert(doc, view
@@ -667,9 +668,9 @@ private void commitAll() throws IOException {
final View view = entry.getKey();
final IndexState state = entry.getValue();
- if (state.pending_seq > getUpdateSequence(state.writer)) {
+ if (getUpdateSequence(state.writer).isEarlierThan(state.pending_seq)) {
final Map<String, String> userData = new HashMap<String, String>();
- userData.put("last_seq", Long.toString(state.pending_seq));
+ userData.put("last_seq", state.pending_seq.toString());
state.writer.commit(userData);
logger.info(view + " now at update_seq " + state.pending_seq);
}
@@ -703,22 +704,22 @@ private IndexState getState(final HttpServletRequest req,
return result;
}
- private long getUpdateSequence(final Directory dir) throws IOException {
+ private UpdateSequence getUpdateSequence(final Directory dir) throws IOException {
if (!IndexReader.indexExists(dir)) {
- return 0L;
+ return UpdateSequence.BOTTOM;
}
return getUpdateSequence(IndexReader.getCommitUserData(dir));
}
- private long getUpdateSequence(final IndexWriter writer) throws IOException {
+ private UpdateSequence getUpdateSequence(final IndexWriter writer) throws IOException {
return getUpdateSequence(writer.getDirectory());
}
- private long getUpdateSequence(final Map<String, String> userData) {
+ private UpdateSequence getUpdateSequence(final Map<String, String> userData) {
if (userData != null && userData.containsKey("last_seq")) {
- return Long.parseLong(userData.get("last_seq"));
+ return new UpdateSequence(userData.get("last_seq"));
}
- return 0L;
+ return UpdateSequence.BOTTOM;
}
private void init() throws IOException, JSONException {
@@ -729,7 +730,7 @@ private void init() throws IOException, JSONException {
context.setOptimizationLevel(9);
this.ddoc_seq = database.getInfo().getUpdateSequence();
- this.since = -1L;
+ this.since = null;
for (final DesignDocument ddoc : database.getAllDesignDocuments()) {
for (final Entry<String, View> entry : ddoc.getAllViews()
@@ -741,11 +742,13 @@ private void init() throws IOException, JSONException {
if (!states.containsKey(view)) {
final Directory dir = FSDirectory.open(viewDir(view, true),
new SingleInstanceLockFactory());
- final long seq = getUpdateSequence(dir);
- if (since == -1) {
+ final UpdateSequence seq = getUpdateSequence(dir);
+ if (since == null) {
+ since = seq;
+ }
+ if (seq.isEarlierThan(since)) {
since = seq;
}
- since = Math.min(since, seq);
logger.debug(dir + " bumped since to " + since);
final DocumentConverter converter = new DocumentConverter(
2  src/main/java/com/github/rnewson/couchdb/lucene/couchdb/Database.java
View
@@ -105,7 +105,7 @@ public DatabaseInfo getInfo() throws IOException, JSONException {
return httpClient.execute(get, handler);
}
- public HttpUriRequest getChangesRequest(final long since)
+ public HttpUriRequest getChangesRequest(final UpdateSequence since)
throws IOException {
return new HttpGet(
url
4 src/main/java/com/github/rnewson/couchdb/lucene/couchdb/DatabaseInfo.java
View
@@ -11,8 +11,8 @@ public DatabaseInfo(final JSONObject json) {
this.json = json;
}
- public long getUpdateSequence() throws JSONException {
- return json.getLong("update_seq");
+ public UpdateSequence getUpdateSequence() throws JSONException {
+ return new UpdateSequence(json.getString("update_seq"));
}
public String getName() throws JSONException {
84 src/main/java/com/github/rnewson/couchdb/lucene/couchdb/UpdateSequence.java
View
@@ -0,0 +1,84 @@
+package com.github.rnewson.couchdb.lucene.couchdb;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.codec.binary.Base64;
+
+import com.ericsson.otp.erlang.OtpErlangDecodeException;
+import com.ericsson.otp.erlang.OtpErlangList;
+import com.ericsson.otp.erlang.OtpErlangLong;
+import com.ericsson.otp.erlang.OtpErlangObject;
+import com.ericsson.otp.erlang.OtpErlangTuple;
+import com.ericsson.otp.erlang.OtpInputStream;
+
+public final class UpdateSequence {
+
+ public static final UpdateSequence BOTTOM = new UpdateSequence("0");
+
+ private long seq;
+ private Map<String, Long> vector;
+ private final String asString;
+
+ public UpdateSequence(final String seq) {
+ this.asString = seq;
+
+ if (seq.matches("[0-9]+")) {
+ this.seq = Long.parseLong(seq);
+ return;
+ }
+
+ if (seq.matches("[0-9]+-[0-9a-zA-Z_-]+")) {
+ final String packedSeqs = seq.split("-", 2)[1];
+ final byte[] bytes = new Base64(true).decode(packedSeqs);
+ final OtpInputStream stream = new OtpInputStream(bytes);
+ try {
+ final OtpErlangList list = (OtpErlangList) stream.read_any();
+ this.vector = new HashMap<String, Long>();
+ for (int i = 0, arity = list.arity(); i < arity; i++) {
+ final OtpErlangTuple tuple = (OtpErlangTuple) list
+ .elementAt(i);
+ final OtpErlangObject node = tuple.elementAt(0);
+ final OtpErlangObject range = tuple.elementAt(1);
+ final OtpErlangLong node_seq = (OtpErlangLong) tuple
+ .elementAt(2);
+ vector.put(node + "-" + range, node_seq.longValue());
+ }
+ } catch (final OtpErlangDecodeException e) {
+ throw new IllegalArgumentException(seq + " not valid.");
+ }
+ return;
+ }
+
+ throw new IllegalArgumentException(seq + " not recognized.");
+ }
+
+ public boolean isEarlierThan(final UpdateSequence other) {
+ if (this == BOTTOM) {
+ return true;
+ }
+
+ if (vector == null && other.vector == null) {
+ return this.seq < other.seq;
+ } else if (vector != null && other.vector != null) {
+ final Iterator<Entry<String, Long>> it = this.vector.entrySet()
+ .iterator();
+ while (it.hasNext()) {
+ final Entry<String, Long> entry = it.next();
+ final Long otherValue = other.vector.get(entry.getKey());
+ if (otherValue != null && otherValue >= entry.getValue()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ throw new IllegalArgumentException(other + " is not compatible.");
+ }
+
+ public String toString() {
+ return asString;
+ }
+
+}
Please sign in to comment.
Something went wrong with that request. Please try again.