Skip to content

Commit

Permalink
performance hacks
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6845 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Apr 28, 2010
1 parent 40a8d13 commit 455a763
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 90 deletions.
19 changes: 12 additions & 7 deletions source/net/yacy/kelondro/index/RowCollection.java
View file
Expand Up @@ -324,7 +324,7 @@ protected synchronized final byte[] getKey(final int index) {
if ((chunkcache == null) || (rowdef == null)) return null; // case may appear during shutdown
if (index >= chunkcount) return null;
if ((index + 1) * rowdef.objectsize > chunkcache.length) return null; // the whole chunk does not fit into the chunkcache
final byte[] b = new byte[this.rowdef.width(0)];
final byte[] b = new byte[this.rowdef.primaryKeyLength];
System.arraycopy(chunkcache, index * rowdef.objectsize, b, 0, b.length);
return b;
}
Expand All @@ -348,7 +348,10 @@ public synchronized final Row.Entry get(final int index, final boolean clone) {
public synchronized final void set(final int index, final Row.Entry a) throws RowSpaceExceededException {
assert (index >= 0) : "set: access with index " + index + " is below zero";
ensureSize(index + 1);
final boolean sameKey = match(a.bytes(), 0, a.cellwidth(0), index);
byte[] column = a.bytes();
assert a.cellwidth(0) == this.rowdef.primaryKeyLength;
assert column.length >= this.rowdef.primaryKeyLength;
final boolean sameKey = match(column, 0, index);
//if (sameKey) System.out.print("$");
a.writeToArray(chunkcache, index * rowdef.objectsize);
if (index >= this.chunkcount) this.chunkcount = index + 1;
Expand Down Expand Up @@ -1000,10 +1003,11 @@ private final int compare(final int i, final int j) {
return c;
}

protected synchronized int compare(final byte[] a, final int astart, final int alength, final int chunknumber) {
protected synchronized int compare(final byte[] a, final int astart, final int chunknumber) {
assert (chunknumber < chunkcount);
final int l = Math.min(this.rowdef.primaryKeyLength, Math.min(a.length - astart, alength));
return rowdef.objectOrder.compare(a, astart, chunkcache, chunknumber * this.rowdef.objectsize, l);
assert a.length - astart >= this.rowdef.primaryKeyLength;
final int len = Math.min(a.length - astart, this.rowdef.primaryKeyLength);
return rowdef.objectOrder.compare(a, astart, chunkcache, chunknumber * this.rowdef.objectsize, len);
}

protected final boolean match(final int i, final int j) {
Expand All @@ -1023,10 +1027,11 @@ protected final boolean match(final int i, final int j) {
return true;
}

protected synchronized boolean match(final byte[] a, int astart, final int alength, final int chunknumber) {
protected synchronized boolean match(final byte[] a, int astart, final int chunknumber) {
if (chunknumber >= chunkcount) return false;
int p = chunknumber * this.rowdef.objectsize;
int len = Math.min(this.rowdef.primaryKeyLength, Math.min(alength, a.length - astart));
assert a.length - astart >= this.rowdef.primaryKeyLength;
int len = Math.min(a.length - astart, this.rowdef.primaryKeyLength);
while (len-- != 0) {
if (a[astart++] != chunkcache[p++]) return false;
}
Expand Down
56 changes: 34 additions & 22 deletions source/net/yacy/kelondro/index/RowSet.java
View file
Expand Up @@ -101,12 +101,14 @@ public void reset() {
}

public final synchronized boolean has(final byte[] key) {
final int index = find(key, 0, key.length);
assert key.length == this.rowdef.primaryKeyLength;
final int index = find(key, 0);
return index >= 0;
}

public final synchronized Row.Entry get(final byte[] key) {
final int index = find(key, 0, key.length);
assert key.length == this.rowdef.primaryKeyLength;
final int index = find(key, 0);
if (index < 0) return null;
return get(index, true);
}
Expand All @@ -118,7 +120,8 @@ public final synchronized void put(final Row.Entry entry) throws RowSpaceExceede
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
}
final int index = find(entry.bytes(), 0, super.rowdef.primaryKeyLength);
assert entry.bytes().length >= this.rowdef.primaryKeyLength;
final int index = find(entry.bytes(), 0);
if (index < 0) {
super.addUnique(entry);
} else {
Expand All @@ -137,7 +140,8 @@ public final synchronized Row.Entry replace(final Row.Entry entry) throws RowSpa
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
}
index = find(entry.bytes(), 0, super.rowdef.primaryKeyLength);
assert entry.bytes().length >= this.rowdef.primaryKeyLength;
index = find(entry.bytes(), 0);
if (index < 0) {
super.addUnique(entry);
} else {
Expand All @@ -150,7 +154,8 @@ public final synchronized Row.Entry replace(final Row.Entry entry) throws RowSpa
}

public final synchronized long inc(final byte[] key, final int col, final long add, final Row.Entry initrow) throws RowSpaceExceededException {
final int index = find(key, 0, key.length);
assert key.length == this.rowdef.primaryKeyLength;
final int index = find(key, 0);
if (index >= 0) {
// the entry existed before
final Row.Entry entry = get(index, false); // no clone necessary
Expand All @@ -176,8 +181,9 @@ public final synchronized long inc(final byte[] key, final int col, final long a
public final synchronized boolean delete(final byte[] a) {
boolean exists = false;
int index;
assert a.length == this.rowdef.primaryKeyLength;
while (true) {
index = find(a, 0, a.length);
index = find(a, 0);
if (index < 0) {
return exists;
} else {
Expand All @@ -190,8 +196,9 @@ public final synchronized boolean delete(final byte[] a) {
public final synchronized Row.Entry remove(final byte[] a) {
Row.Entry entry = null;
int index;
assert a.length == this.rowdef.primaryKeyLength;
while (true) {
index = find(a, 0, a.length);
index = find(a, 0);
if (index < 0) {
return entry;
} else {
Expand All @@ -201,37 +208,38 @@ public final synchronized Row.Entry remove(final byte[] a) {
}
}

private final int find(final byte[] a, final int astart, final int alength) {
private final int find(final byte[] a, final int astart) {
// returns the chunknumber; -1 if not found

if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount);
if (rowdef.objectOrder == null) return iterativeSearch(a, astart, 0, this.chunkcount);

if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
}

if (this.rowdef.objectOrder != null && this.rowdef.objectOrder instanceof Base64Order) {
// first try to find in sorted area
assert this.rowdef.objectOrder.wellformed(a, astart, alength) : "not wellformed: " + new String(a, astart, alength);
assert this.rowdef.objectOrder.wellformed(a, astart, this.rowdef.primaryKeyLength) : "not wellformed: " + new String(a, astart, this.rowdef.primaryKeyLength);
}

// first try to find in sorted area
final int p = binarySearch(a, astart, alength);
final int p = binarySearch(a, astart);
if (p >= 0) return p;

// then find in unsorted area
return iterativeSearch(a, astart, alength, this.sortBound, this.chunkcount);
return iterativeSearch(a, astart, this.sortBound, this.chunkcount);
}

private final int iterativeSearch(final byte[] key, final int astart, final int alength, final int leftBorder, final int rightBound) {
private final int iterativeSearch(final byte[] key, final int astart, final int leftBorder, final int rightBound) {
// returns the chunknumber
for (int i = leftBorder; i < rightBound; i++) {
if (match(key, astart, alength, i)) return i;
assert key.length - astart >= this.rowdef.primaryKeyLength;
if (match(key, astart, i)) return i;
}
return -1;
}

private final int binarySearch(final byte[] key, final int astart, final int alength) {
private final int binarySearch(final byte[] key, final int astart) {
// returns the exact position of the key if the key exists,
// or -1 if the key does not exist
assert (rowdef.objectOrder != null);
Expand All @@ -240,15 +248,16 @@ private final int binarySearch(final byte[] key, final int astart, final int ale
int p = 0;
int d;
while (l < rbound) {
p = l + ((rbound - l) >> 1);
d = compare(key, astart, alength, p);
p = (l + rbound) >> 1;
assert key.length - astart >= this.rowdef.primaryKeyLength;
d = compare(key, astart, p);
if (d == 0) return p;
if (d < 0) rbound = p; else l = p + 1;
}
return -1;
}

protected final int binaryPosition(final byte[] key, final int astart, final int alength) {
protected final int binaryPosition(final byte[] key, final int astart) {
// returns the exact position of the key if the key exists,
// or a position of an entry that is greater than the key if the
// key does not exist
Expand All @@ -258,8 +267,9 @@ protected final int binaryPosition(final byte[] key, final int astart, final int
int p = 0;
int d;
while (l < rbound) {
p = l + ((rbound - l) >> 1);
d = compare(key, astart, alength, p);
p = (l + rbound) >> 1;
assert key.length - astart >= this.rowdef.primaryKeyLength;
d = compare(key, astart, p);
if (d == 0) return p;
if (d < 0) rbound = p; else l = p + 1;
}
Expand Down Expand Up @@ -291,7 +301,8 @@ public keyIterator(final boolean up, final byte[] firstKey) {
if (first == null) {
p = 0;
} else {
p = binaryPosition(first, 0, first.length); // check this to find bug in DHT selection enumeration
assert first.length == rowdef.primaryKeyLength;
p = binaryPosition(first, 0); // check this to find bug in DHT selection enumeration
}
}

Expand Down Expand Up @@ -350,7 +361,8 @@ public rowIterator(final boolean up, final byte[] firstKey) {
if (first == null) {
p = 0;
} else {
p = binaryPosition(first, 0, first.length); // check this to find bug in DHT selection enumeration
assert first.length == rowdef.primaryKeyLength;
p = binaryPosition(first, 0); // check this to find bug in DHT selection enumeration
}
}

Expand Down
56 changes: 0 additions & 56 deletions source/net/yacy/kelondro/order/Base64Order.java
View file
Expand Up @@ -520,68 +520,12 @@ private final int compares(final byte[] a, final int aoffset, final byte[] b, fi
i++;
continue;
}
//acc = ahpla[ac];
//assert (acc >= 0) : "acc = " + acc + ", a = " + NaturalOrder.arrayList(a, aoffset, al) + "/" + new String(a, aoffset, al) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + NaturalOrder.table(a, 16, aoffset);
//bcc = ahpla[bc];
//assert (bcc >= 0) : "bcc = " + bcc + ", b = " + NaturalOrder.arrayList(b, boffset, bl) + "/" + new String(b, boffset, bl) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + NaturalOrder.table(b, 16, boffset);
//if (acc > bcc) c = 1;
//if (acc < bcc) c = -1;
//assert c != 0;
//assert ab[(ac << 7) | bc] == c;
//return c;
return ab[(ac << 7) | bc];
}
// they are equal
return 0;
}
/*
public final int comparePivot(final byte[] compiledPivot, final byte[] b, final int boffset, final int blength) {
assert zero == null;
assert asc;
assert (boffset + blength <= b.length) : "b.length = " + b.length + ", boffset = " + boffset + ", blength = " + blength;
int i = 0;
final int bl = Math.min(blength, b.length - boffset);
byte acc, bcc;
assert boffset >= 0;
assert boffset < b.length;
assert boffset + Math.min(bl, compiledPivot.length) - 1 >= 0;
assert boffset + Math.min(bl, compiledPivot.length) - 1 < b.length;
byte bb;
while ((i < compiledPivot.length) && (i < bl)) {
acc = compiledPivot[i];
assert boffset + i >= 0;
assert boffset + i < b.length;
bb = b[boffset + i];
assert bb >= 0;
assert bb < 128;
bcc = ahpla[bb];
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + NaturalOrder.arrayList(b, boffset, bl) + "/" + new String(b, boffset, bl) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + NaturalOrder.table(b, 16, boffset);
if (acc > bcc) return 1;
if (acc < bcc) return -1;
// else the bytes are equal and it may go on yet undecided
i++;
}
// compare length
if (compiledPivot.length > bl) return 1;
if (compiledPivot.length < bl) return -1;
// they are equal
return 0;
}

public final byte[] compilePivot(final byte[] a, final int aoffset, final int alength) {
assert (aoffset + alength <= a.length) : "a.length = " + a.length + ", aoffset = " + aoffset + ", alength = " + alength;
final byte[] cp = new byte[Math.min(alength, a.length - aoffset)];
byte aa;
for (int i = cp.length - 1; i >= 0; i--) {
aa = a[aoffset + i];
assert aa >= 0;
assert aa < 128;
cp[i] = ahpla[aa];
assert cp[i] != -1;
}
return cp;
}
*/
public static void main(final String[] s) {
// java -classpath classes de.anomic.kelondro.kelondroBase64Order
final Base64Order b64 = new Base64Order(true, true);
Expand Down
6 changes: 3 additions & 3 deletions source/net/yacy/kelondro/rwi/IODispatcher.java
View file
Expand Up @@ -83,7 +83,7 @@ public void terminate() {
protected synchronized void dump(ReferenceContainerCache<? extends Reference> cache, File file, ReferenceContainerArray<? extends Reference> array) {
if (dumpQueue == null || controlQueue == null || !this.isAlive()) {
Log.logWarning("IODispatcher", "emergency dump of file " + file.getName());
if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize));
if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize), true);
} else {
DumpJob<? extends Reference> job = (DumpJob<? extends Reference>)new DumpJob(cache, file, array);
try {
Expand All @@ -98,7 +98,7 @@ protected synchronized void dump(ReferenceContainerCache<? extends Reference> ca
}
} catch (InterruptedException e) {
Log.logException(e);
cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize));
cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize), true);
}
}
}
Expand Down Expand Up @@ -224,7 +224,7 @@ private DumpJob(ReferenceContainerCache<ReferenceType> cache, File file, Referen
}
private void dump() {
try {
if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize));
if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, writeBufferSize), true);
array.mountBLOBFile(file);
} catch (IOException e) {
Log.logException(e);
Expand Down
2 changes: 1 addition & 1 deletion source/net/yacy/kelondro/rwi/IndexCell.java
View file
Expand Up @@ -328,7 +328,7 @@ public synchronized void clear() throws IOException {
* and is composed of the current date and the cell salt
*/
public synchronized void close() {
if (!this.ram.isEmpty()) this.ram.dump(this.array.newContainerBLOBFile(), (int) Math.min(MemoryControl.available() / 3, writeBufferSize));
if (!this.ram.isEmpty()) this.ram.dump(this.array.newContainerBLOBFile(), (int) Math.min(MemoryControl.available() / 3, writeBufferSize), true);
// close all
this.ram.close();
this.array.close();
Expand Down
12 changes: 11 additions & 1 deletion source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
View file
Expand Up @@ -89,7 +89,16 @@ public void close() {
this.cache = null;
}

public void dump(final File heapFile, int writeBuffer) {
/**
* dump the cache to a file. This method can be used in a destructive way
* which means that memory can be freed during the dump. This may be important
* because the dump is done in such situations when memory gets low. To get more
* memory during the dump helps to solve tight memory situations.
* @param heapFile
* @param writeBuffer
* @param destructive - if true then the cache is cleaned during the dump causing to free memory
*/
public void dump(final File heapFile, int writeBuffer, boolean destructive) {
assert this.cache != null;
Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's");
if (heapFile.exists()) FileUtils.deletedelete(heapFile);
Expand Down Expand Up @@ -127,6 +136,7 @@ public void dump(final File heapFile, int writeBuffer) {
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
if (destructive) container.clear(); // this memory is not needed any more
urlcount += container.size();
}
wordcount++;
Expand Down

0 comments on commit 455a763

Please sign in to comment.