Skip to content

Commit

Permalink
Add an affinity tagging strategy to all keys used to store entries re…
Browse files Browse the repository at this point in the history 
…presenting Lucene index data
  • Loading branch information
@Sanne
Sanne committed Jan 19, 2015
1 parent b79e171 commit 2ee4fd9
Show file tree
Hide file tree
Showing 45 changed files with 368 additions and 234 deletions.
15 changes: 12 additions & 3 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/ChunkCacheKey.java
View file
Expand Up @@ -23,14 +23,16 @@ public final class ChunkCacheKey implements IndexScopedKey {
private final String fileName;
private final int bufferSize;
private final int hashCode;
private final int affinitySegmentId;

public ChunkCacheKey(final String indexName, final String fileName, final int chunkId, final int bufferSize) {
public ChunkCacheKey(final String indexName, final String fileName, final int chunkId, final int bufferSize, final int affinitySegmentId) {
if (fileName == null)
throw new IllegalArgumentException("filename must not be null");
this.indexName = indexName;
this.fileName = fileName;
this.chunkId = chunkId;
this.bufferSize = bufferSize;
this.affinitySegmentId = affinitySegmentId;
this.hashCode = generatedHashCode();
}

Expand Down Expand Up @@ -62,6 +64,11 @@ public String getIndexName() {
return indexName;
}

@Override
public int getAffinitySegmentId() {
return affinitySegmentId;
}

@Override
public Object accept(KeyVisitor visitor) throws Exception {
return visitor.visit(this);
Expand Down Expand Up @@ -111,7 +118,7 @@ public boolean equals(Object obj) {
*/
@Override
public String toString() {
return fileName + "|" + chunkId + "|" + bufferSize + "|" + indexName;
return "C|" + fileName + "|" + chunkId + "|" + bufferSize + "|" + indexName + "|" + affinitySegmentId;
}

public static final class Externalizer extends AbstractExternalizer<ChunkCacheKey> {
Expand All @@ -122,6 +129,7 @@ public void writeObject(final ObjectOutput output, final ChunkCacheKey key) thro
output.writeUTF(key.fileName);
UnsignedNumeric.writeUnsignedInt(output, key.chunkId);
UnsignedNumeric.writeUnsignedInt(output, key.bufferSize);
UnsignedNumeric.writeUnsignedInt(output, key.affinitySegmentId);
}

@Override
Expand All @@ -130,7 +138,8 @@ public ChunkCacheKey readObject(final ObjectInput input) throws IOException {
final String fileName = input.readUTF();
final int chunkId = UnsignedNumeric.readUnsignedInt(input);
final int bufferSize = UnsignedNumeric.readUnsignedInt(input);
return new ChunkCacheKey(indexName, fileName, chunkId, bufferSize);
final int affinitySegmentId = UnsignedNumeric.readUnsignedInt(input);
return new ChunkCacheKey(indexName, fileName, chunkId, bufferSize, affinitySegmentId);
}

@Override
Expand Down
16 changes: 13 additions & 3 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/FileCacheKey.java
View file
Expand Up @@ -5,6 +5,7 @@
import java.io.ObjectOutput;
import java.util.Set;

import org.infinispan.commons.io.UnsignedNumeric;
import org.infinispan.commons.marshall.AbstractExternalizer;
import org.infinispan.commons.util.Util;

Expand All @@ -19,13 +20,15 @@ public final class FileCacheKey implements IndexScopedKey {

private final String indexName;
private final String fileName;
private final int affinitySegmentId;
private final int hashCode;

public FileCacheKey(final String indexName, final String fileName) {
public FileCacheKey(final String indexName, final String fileName, final int affinitySegmentId) {
if (fileName == null)
throw new IllegalArgumentException("filename must not be null");
this.indexName = indexName;
this.fileName = fileName;
this.affinitySegmentId = affinitySegmentId;
this.hashCode = generatedHashCode();
}

Expand All @@ -38,6 +41,11 @@ public String getIndexName() {
return indexName;
}

@Override
public int getAffinitySegmentId() {
return affinitySegmentId;
}

@Override
public Object accept(KeyVisitor visitor) throws Exception {
return visitor.visit(this);
Expand Down Expand Up @@ -83,7 +91,7 @@ public boolean equals(Object obj) {
*/
@Override
public String toString() {
return fileName + "|M|"+ indexName;
return "M|" + fileName + "|"+ indexName + "|" + affinitySegmentId;
}

public static final class Externalizer extends AbstractExternalizer<FileCacheKey> {
Expand All @@ -92,13 +100,15 @@ public static final class Externalizer extends AbstractExternalizer<FileCacheKey
public void writeObject(final ObjectOutput output, final FileCacheKey key) throws IOException {
output.writeUTF(key.indexName);
output.writeUTF(key.fileName);
UnsignedNumeric.writeUnsignedInt(output, key.affinitySegmentId);
}

@Override
public FileCacheKey readObject(final ObjectInput input) throws IOException {
String indexName = input.readUTF();
String fileName = input.readUTF();
return new FileCacheKey(indexName, fileName);
int affinitySegmentId = UnsignedNumeric.readUnsignedInt(input);
return new FileCacheKey(indexName, fileName, affinitySegmentId);
}

@Override
Expand Down
18 changes: 14 additions & 4 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/FileListCacheKey.java
View file
Expand Up @@ -5,6 +5,7 @@
import java.io.ObjectOutput;
import java.util.Set;

import org.infinispan.commons.io.UnsignedNumeric;
import org.infinispan.commons.marshall.AbstractExternalizer;
import org.infinispan.commons.util.Util;

Expand All @@ -18,10 +19,12 @@
public final class FileListCacheKey implements IndexScopedKey {

private final String indexName;
private final int affinitySegmentId;
private final int hashCode;

public FileListCacheKey(String indexName) {
public FileListCacheKey(String indexName, final int affinitySegmentId) {
this.indexName = indexName;
this.affinitySegmentId = affinitySegmentId;
this.hashCode = generatedHashCode();
}

Expand All @@ -35,6 +38,11 @@ public String getIndexName() {
return indexName;
}

@Override
public int getAffinitySegmentId() {
return affinitySegmentId;
}

@Override
public Object accept(KeyVisitor visitor) throws Exception {
return visitor.visit(this);
Expand Down Expand Up @@ -67,20 +75,22 @@ public boolean equals(Object obj) {
*/
@Override
public String toString() {
return "*|" + indexName;
return "*|" + indexName + "|" + affinitySegmentId;
}

public static final class Externalizer extends AbstractExternalizer<FileListCacheKey> {

@Override
public void writeObject(final ObjectOutput output, final FileListCacheKey key) throws IOException {
output.writeUTF(key.indexName);
UnsignedNumeric.writeUnsignedInt(output, key.affinitySegmentId);
}

@Override
public FileListCacheKey readObject(final ObjectInput input) throws IOException {
String indexName = input.readUTF();
return new FileListCacheKey(indexName);
final String indexName = input.readUTF();
final int affinitySegmentId = UnsignedNumeric.readUnsignedInt(input);
return new FileListCacheKey(indexName, affinitySegmentId);
}

@Override
Expand Down
16 changes: 13 additions & 3 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/FileReadLockKey.java
View file
Expand Up @@ -5,6 +5,7 @@
import java.io.ObjectOutput;
import java.util.Set;

import org.infinispan.commons.io.UnsignedNumeric;
import org.infinispan.commons.marshall.AbstractExternalizer;
import org.infinispan.commons.util.Util;

Expand All @@ -22,14 +23,16 @@ public final class FileReadLockKey implements IndexScopedKey {
private final String indexName;
private final String fileName;
private final int hashCode;
private final int affinitySegmentId;

public FileReadLockKey(final String indexName, final String fileName) {
public FileReadLockKey(final String indexName, final String fileName, final int affinitySegmentId) {
if (indexName == null)
throw new IllegalArgumentException("indexName shall not be null");
if (fileName == null)
throw new IllegalArgumentException("fileName shall not be null");
this.indexName = indexName;
this.fileName = fileName;
this.affinitySegmentId = affinitySegmentId;
this.hashCode = generateHashCode();
}

Expand All @@ -43,6 +46,11 @@ public String getIndexName() {
return indexName;
}

@Override
public int getAffinitySegmentId() {
return affinitySegmentId;
}

/**
* Get the fileName.
*
Expand Down Expand Up @@ -82,7 +90,7 @@ public boolean equals(Object obj) {

@Override
public String toString() {
return fileName + "|RL|"+ indexName;
return "RL|" + fileName + "|"+ indexName + "|" + affinitySegmentId;
}

public static final class Externalizer extends AbstractExternalizer<FileReadLockKey> {
Expand All @@ -91,13 +99,15 @@ public static final class Externalizer extends AbstractExternalizer<FileReadLock
public void writeObject(final ObjectOutput output, final FileReadLockKey key) throws IOException {
output.writeUTF(key.indexName);
output.writeUTF(key.fileName);
UnsignedNumeric.writeUnsignedInt(output, key.affinitySegmentId);
}

@Override
public FileReadLockKey readObject(final ObjectInput input) throws IOException {
String indexName = input.readUTF();
String fileName = input.readUTF();
return new FileReadLockKey(indexName, fileName);
final int affinitySegmentId = UnsignedNumeric.readUnsignedInt(input);
return new FileReadLockKey(indexName, fileName, affinitySegmentId);
}

@Override
Expand Down
11 changes: 11 additions & 0 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/IndexScopedKey.java
View file
Expand Up @@ -8,8 +8,19 @@
*/
public interface IndexScopedKey {

/**
* Different indexes are required to use different names
* @return
*/
String getIndexName();

/**
* This numeric id is used exclusively for storage affinity in Infinispan.
* It is not included in the equals and hashcode implementations!
* @return the segment id as defined in {@link org.infinispan.lucene.directory.BuildContext#affinityLocationIntoSegment(int)}, or -1 when not explicitly set.
*/
int getAffinitySegmentId();

<T> T accept(KeyVisitor<T> visitor) throws Exception;

}
82 changes: 48 additions & 34 deletions lucene/lucene-directory/src/main/java/org/infinispan/lucene/LuceneKey2StringMapper.java
View file
Expand Up @@ -19,8 +19,8 @@
public final class LuceneKey2StringMapper implements TwoWayKey2StringMapper {

/**
* The pipe character was chosen as it's illegal to have a pipe in a filename, so we only have to
* check for the indexnames.
* The pipe character was chosen as it's illegal to have a pipe in a filename, so we
* don't have to escape the filenames generated by Lucene.
*/
static final Pattern singlePipePattern = Pattern.compile("\\|");

Expand Down Expand Up @@ -54,40 +54,54 @@ public Object getKeyMapping(String key) {
if (key == null) {
throw new IllegalArgumentException("Not supporting null keys");
}
// ChunkCacheKey: fileName + "|" + chunkId + "|" + bufferSize "|" + indexName
// FileCacheKey : fileName + "|M|"+ indexName;
// FileListCacheKey : "*|" + indexName;
// FileReadLockKey : fileName + "|RL|"+ indexName;
if (key.startsWith("*|")) {
return new FileListCacheKey(key.substring(2));
}
else {
String[] split = singlePipePattern.split(key);
if (split.length != 3 && split.length != 4) {
throw log.keyMappperUnexpectedStringFormat(key);
}
else {
switch (split[1]) {
case "M":
if (split.length != 3) {
throw log.keyMappperUnexpectedStringFormat(key);
}
return new FileCacheKey(split[2], split[0]);
case "RL":
if (split.length != 3) throw log.keyMappperUnexpectedStringFormat(key);
return new FileReadLockKey(split[2], split[0]);
default:
if (split.length != 4) throw log.keyMappperUnexpectedStringFormat(key);
try {
int chunkId = Integer.parseInt(split[1]);
int bufferSize = Integer.parseInt(split[2]);
return new ChunkCacheKey(split[3], split[0], chunkId, bufferSize);
}
catch (NumberFormatException nfe) {
throw log.keyMappperUnexpectedStringFormat(key);
}
// ChunkCacheKey: "C|" + fileName + "|" + chunkId + "|" + bufferSize "|" + indexName + "|" + affinitySegmentId;
// FileCacheKey : "M|" + fileName + "|"+ indexName + "|" + affinitySegmentId;
// FileListCacheKey : "*|" + indexName + "|" + affinitySegmentId;
// FileReadLockKey : "RL|" + fileName + "|"+ indexName + "|" + affinitySegmentId;
String[] split = singlePipePattern.split(key);
switch (split[0]) {
case "C": {
if (split.length != 6) {
throw log.keyMappperUnexpectedStringFormat(key);
}
final String indexName = split[4];
final String fileName = split[1];
final int chunkId = toInt(split[2], key);
final int bufferSize = toInt(split[3], key);
final int affinitySegmentId = toInt(split[5], key);
return new ChunkCacheKey(indexName, fileName, chunkId, bufferSize, affinitySegmentId);
}
case "M": {
if (split.length != 4) throw log.keyMappperUnexpectedStringFormat(key);
final String indexName = split[2];
final String fileName = split[1];
final int affinitySegmentId = toInt(split[3], key);
return new FileCacheKey(indexName, fileName, affinitySegmentId);
}
case "*": {
if (split.length != 3) throw log.keyMappperUnexpectedStringFormat(key);
final String indexName = split[1];
final int affinitySegmentId = toInt(split[2], key);
return new FileListCacheKey(indexName, affinitySegmentId);
}
case "RL": {
if (split.length != 4) throw log.keyMappperUnexpectedStringFormat(key);
final String indexName = split[2];
final String fileName = split[1];
final int affinitySegmentId = toInt(split[3], key);
return new FileReadLockKey(indexName, fileName, affinitySegmentId);
}
default:
throw log.keyMappperUnexpectedStringFormat(key);
}
}

private int toInt(String token, String fullInput) {
try {
return Integer.parseInt(token);
}
catch (NumberFormatException nfe) {
throw log.keyMappperUnexpectedStringFormat(fullInput);
}
}

Expand Down

0 comments on commit 2ee4fd9

Please sign in to comment.