Skip to content

Commit

Permalink
- shifted some computation out of synchronization to allow more concu…
Browse files Browse the repository at this point in the history 
…rrency

- removed synchronization where not necessary

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6814 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Apr 14, 2010
1 parent f204076 commit dde394a
Show file tree
Hide file tree
Showing 4 changed files with 206 additions and 185 deletions.
10 changes: 6 additions & 4 deletions source/de/anomic/search/MetadataRepository.java
View file
Expand Up @@ -109,13 +109,13 @@ public void close() {
}
}

public synchronized int writeCacheSize() {
public int writeCacheSize() {
if (urlIndexFile instanceof SplitTable) return ((SplitTable) urlIndexFile).writeBufferSize();
if (urlIndexFile instanceof Cache) return ((Cache) urlIndexFile).writeBufferSize();
return 0;
}

public synchronized URIMetadataRow load(final byte[] urlHash, final WordReferenceVars searchedWord, final long ranking) {
public URIMetadataRow load(final byte[] urlHash, final WordReferenceVars searchedWord, final long ranking) {
// generates an plasmaLURLEntry using the url hash
// if the url cannot be found, this returns null
if (urlHash == null) return null;
Expand All @@ -129,9 +129,10 @@ public synchronized URIMetadataRow load(final byte[] urlHash, final WordReferenc
}
}

public synchronized void store(final URIMetadataRow entry) throws IOException {
public void store(final URIMetadataRow entry) throws IOException {
// Check if there is a more recent Entry already in the DB
URIMetadataRow oldEntry;
synchronized (this) {
try {
Row.Entry oe = (urlIndexFile == null) ? null : urlIndexFile.get(entry.hash());
oldEntry = (oe == null) ? null : new URIMetadataRow(oe, null, 0);
Expand All @@ -153,10 +154,11 @@ public synchronized void store(final URIMetadataRow entry) throws IOException {
} catch (RowSpaceExceededException e) {
throw new IOException("RowSpaceExceededException in " + this.urlIndexFile.filename() + ": " + e.getMessage());
}
}
statsDump = null;
}

public synchronized boolean remove(final byte[] urlHashBytes) {
public boolean remove(final byte[] urlHashBytes) {
if (urlHashBytes == null) return false;
try {
final Row.Entry r = urlIndexFile.remove(urlHashBytes);
Expand Down
119 changes: 64 additions & 55 deletions source/net/yacy/kelondro/blob/Heap.java
View file
Expand Up @@ -31,6 +31,7 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.io.AbstractWriter;
Expand All @@ -42,7 +43,7 @@

public final class Heap extends HeapModifier implements BLOB {

private HashMap<String, byte[]> buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key
private TreeMap<byte[], byte[]> buffer; // a write buffer to limit IO to the file
private int buffersize; // bytes that are buffered in buffer
private final int buffermax; // maximum size of the buffer

Expand Down Expand Up @@ -83,7 +84,7 @@ public Heap(
int buffermax) throws IOException {
super(heapFile, keylength, ordering);
this.buffermax = buffermax;
this.buffer = new HashMap<String, byte[]>();
this.buffer = new TreeMap<byte[], byte[]>(ordering);
this.buffersize = 0;
/*
// DEBUG
Expand Down Expand Up @@ -119,12 +120,14 @@ public synchronized int size() {
* @return true if the key exists, false otherwise
*/
@Override
public synchronized boolean has(byte[] key) {
public boolean has(byte[] key) {
assert index != null;
key = normalizeKey(key);
// check the buffer
if (this.buffer.containsKey(new String(key))) return true;
return super.has(key);
synchronized (this) {
// check the buffer
if (this.buffer.containsKey(key)) return true;
return super.has(key);
}
}

/**
Expand All @@ -138,7 +141,6 @@ private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExce
assert blob.length > 0;
if ((blob == null) || (blob.length == 0)) return;
final int pos = (int) file.length();
key = normalizeKey(key);
index.put(key, pos);
file.seek(pos);
file.writeInt(this.keylength + blob.length);
Expand All @@ -154,19 +156,19 @@ private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExce
*/
private void flushBuffer() throws IOException, RowSpaceExceededException {
// check size of buffer
Iterator<Map.Entry<String, byte[]>> i = this.buffer.entrySet().iterator();
Iterator<Map.Entry<byte[], byte[]>> i = this.buffer.entrySet().iterator();
int l = 0;
while (i.hasNext()) l += i.next().getValue().length;
assert l == this.buffersize;

// simulate write: this whole code block is only here to test the assert at the end of the block; remove after testing
i = this.buffer.entrySet().iterator();
int posBuffer = 0;
Map.Entry<String, byte[]> entry;
Map.Entry<byte[], byte[]> entry;
byte[] key, blob;
while (i.hasNext()) {
entry = i.next();
key = normalizeKey(entry.getKey().getBytes());
key = normalizeKey(entry.getKey());
blob = entry.getValue();
posBuffer += 4 + this.keylength + blob.length;
}
Expand All @@ -181,7 +183,7 @@ private void flushBuffer() throws IOException, RowSpaceExceededException {
byte[] b;
while (i.hasNext()) {
entry = i.next();
key = normalizeKey(entry.getKey().getBytes());
key = normalizeKey(entry.getKey());
blob = entry.getValue();
index.put(key, posFile);
b = AbstractWriter.int2array(this.keylength + blob.length);
Expand Down Expand Up @@ -211,14 +213,16 @@ private void flushBuffer() throws IOException, RowSpaceExceededException {
* @throws IOException
*/
@Override
public synchronized byte[] get(byte[] key) throws IOException {
public byte[] get(byte[] key) throws IOException {
key = normalizeKey(key);

// check the buffer
byte[] blob = this.buffer.get(new String(key));
if (blob != null) return blob;

return super.get(key);
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.get(key);
if (blob != null) return blob;

return super.get(key);
}
}

/**
Expand All @@ -228,13 +232,16 @@ public synchronized byte[] get(byte[] key) throws IOException {
* @throws IOException
*/
@Override
public synchronized long length(byte[] key) throws IOException {
public long length(byte[] key) throws IOException {
key = normalizeKey(key);
// check the buffer
byte[] blob = this.buffer.get(new String(key));
if (blob != null) return blob.length;

return super.length(key);
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.get(key);
if (blob != null) return blob.length;

return super.length(key);
}
}

/**
Expand Down Expand Up @@ -289,41 +296,41 @@ public int getBuffermax() {
* @throws RowSpaceExceededException
*/
@Override
public synchronized void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
public void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);

// we do not write records of length 0 into the BLOB
if (b.length == 0) return;

// first remove the old entry (removes from buffer and file)
// TODO: this can be enhanced!
this.remove(key);

// then look if we can use a free entry
if (putToGap(key, b)) return;

// if there is not enough space in the buffer, flush all
if (this.buffersize + b.length > buffermax) {
// this is too big. Flush everything
super.shrinkWithGapsAtEnd();
flushBuffer();
if (b.length > buffermax) {
this.add(key, b);
} else {
this.buffer.put(new String(key), b);
this.buffersize += b.length;
synchronized (this) {
// first remove the old entry (removes from buffer and file)
// TODO: this can be enhanced!
this.remove(key);

// then look if we can use a free entry
if (putToGap(key, b)) return;

// if there is not enough space in the buffer, flush all
if (this.buffersize + b.length > buffermax) {
// this is too big. Flush everything
super.shrinkWithGapsAtEnd();
flushBuffer();
if (b.length > buffermax) {
this.add(key, b);
} else {
this.buffer.put(key, b);
this.buffersize += b.length;
}
return;
}
return;

// add entry to buffer
this.buffer.put(key, b);
this.buffersize += b.length;
}

// add entry to buffer
this.buffer.put(new String(key), b);
this.buffersize += b.length;
}

private boolean putToGap(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);

// we do not write records of length 0 into the BLOB
if (b.length == 0) return true;

Expand Down Expand Up @@ -415,17 +422,19 @@ private boolean putToGap(byte[] key, final byte[] b) throws IOException, RowSpac
* @throws IOException
*/
@Override
public synchronized void remove(byte[] key) throws IOException {
public void remove(byte[] key) throws IOException {
key = normalizeKey(key);

// check the buffer
byte[] blob = this.buffer.remove(new String(key));
if (blob != null) {
this.buffersize -= blob.length;
return;
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.remove(key);
if (blob != null) {
this.buffersize -= blob.length;
return;
}

super.remove(key);
}

super.remove(key);
}

/**
Expand Down

0 comments on commit dde394a

Please sign in to comment.