Skip to content

Commit

Permalink
update of cache logging
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2917 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Nov 5, 2006
1 parent 8385557 commit d454ca4
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 29 deletions.
119 changes: 91 additions & 28 deletions source/de/anomic/kelondro/kelondroCache.java
View file
Expand Up @@ -165,7 +165,10 @@ private void flushUnique() throws IOException {
if (writeBufferUnique == null) return;
synchronized (writeBufferUnique) {
Iterator i = writeBufferUnique.rows();
while (i.hasNext()) index.addUnique((kelondroRow.Entry) i.next());
while (i.hasNext()) {
this.index.addUnique((kelondroRow.Entry) i.next());
this.cacheFlush++;
}
writeBufferUnique.clear();
writeBufferUnique.trim();
}
Expand All @@ -181,7 +184,8 @@ private void flushUnique(int maxcount) throws IOException {
while ((i.hasNext()) && (maxcount-- > 0)) {
row = (kelondroRow.Entry) i.next();
delete.add(row.getColBytes(index.primarykey()));
index.addUnique(row);
this.index.addUnique(row);
this.cacheFlush++;
}
i = delete.rows();
while (i.hasNext()) writeBufferUnique.remove(((kelondroRow.Entry) i.next()).getColBytes(0));
Expand All @@ -194,7 +198,10 @@ private void flushDoubles() throws IOException {
if (writeBufferDoubles == null) return;
synchronized (writeBufferDoubles) {
Iterator i = writeBufferDoubles.rows();
while (i.hasNext()) index.put((kelondroRow.Entry) i.next());
while (i.hasNext()) {
this.index.put((kelondroRow.Entry) i.next());
this.cacheFlush++;
}
writeBufferDoubles.clear();
writeBufferDoubles.trim();
}
Expand All @@ -210,7 +217,8 @@ private void flushDoubles(int maxcount) throws IOException {
while ((i.hasNext()) && (maxcount-- > 0)) {
row = (kelondroRow.Entry) i.next();
delete.add(row.getColBytes(index.primarykey()));
index.addUnique(row);
this.index.addUnique(row);
this.cacheFlush++;
}
i = delete.rows();
while (i.hasNext()) writeBufferDoubles.remove(((kelondroRow.Entry) i.next()).getColBytes(0));
Expand Down Expand Up @@ -273,10 +281,10 @@ public synchronized Entry get(byte[] key) throws IOException {
// first look into the miss cache
if (readMissCache != null) {
if (readMissCache.get(key) != null) {
hasnotHit++;
this.hasnotHit++;
return null;
} else {
hasnotMiss++;
this.hasnotMiss++;
}
}

Expand All @@ -286,36 +294,42 @@ public synchronized Entry get(byte[] key) throws IOException {
if (readHitCache != null) {
entry = readHitCache.get(key);
if (entry != null) {
readHit++;
this.readHit++;
return entry;
}
}
if (writeBufferUnique != null) {
entry = writeBufferUnique.get(key);
if (entry != null) {
readHit++;
this.readHit++;
return entry;
}
}
if (writeBufferDoubles != null) {
entry = writeBufferDoubles.get(key);
if (entry != null) {
readHit++;
this.readHit++;
return entry;
}
}

// finally ask the backend index
readMiss++;
this.readMiss++;
entry = index.get(key);
// learn from result
if (entry == null) {
checkMissSpace();
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key));
if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
return null;
} else {
checkHitSpace();
if (readHitCache != null) readHitCache.put(entry);
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(entry);
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry;
}
}
Expand Down Expand Up @@ -348,7 +362,10 @@ public synchronized Entry put(Entry row) throws IOException {
}
assert (writeBufferDoubles == null);
index.put(row); // write to backend
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return null;
}
}
Expand All @@ -362,13 +379,15 @@ public synchronized Entry put(Entry row) throws IOException {
if (writeBufferDoubles != null) {
// because the entry exists, it must be written in the doubles buffer
readHitCache.remove(key);
this.cacheDelete++;
writeBufferDoubles.put(row);
return entry;
} else {
// write directly to backend index
index.put(row);
// learn from situation
readHitCache.put(row); // overwrite old entry
kelondroRow.Entry dummy = readHitCache.put(row); // overwrite old entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
return entry;
}
}
Expand Down Expand Up @@ -406,7 +425,10 @@ public synchronized Entry put(Entry row) throws IOException {

// the worst case: we must write to the back-end directly
entry = index.put(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry;
}

Expand All @@ -426,11 +448,17 @@ public synchronized Entry put(Entry row, Date entryDate) throws IOException {
checkHitSpace();

// remove entry from miss- and hit-cache
if (readMissCache != null) readMissCache.remove(key);
if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
}

// the worst case: we must write to the backend directly
Entry entry = index.put(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry;
}

Expand All @@ -444,7 +472,8 @@ public synchronized void addUnique(Entry row) throws IOException {

// remove entry from miss- and hit-cache
if (readMissCache != null) {
readMissCache.remove(key);
this.readMissCache.remove(key);
this.hasnotDelete++;
// the entry does not exist before
if (writeBufferUnique != null) {
// since we know that the entry does not exist, we know that new
Expand All @@ -454,7 +483,10 @@ public synchronized void addUnique(Entry row) throws IOException {
}
assert (writeBufferDoubles == null);
index.addUnique(row); // write to backend
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return;
}

Expand All @@ -469,7 +501,10 @@ public synchronized void addUnique(Entry row) throws IOException {

// the worst case: we must write to the back-end directly
index.addUnique(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
}

public synchronized void addUnique(Entry row, Date entryDate) throws IOException {
Expand All @@ -488,11 +523,17 @@ public synchronized void addUnique(Entry row, Date entryDate) throws IOException
checkHitSpace();

// remove entry from miss- and hit-cache
if (readMissCache != null) readMissCache.remove(key);
if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
}

// the worst case: we must write to the backend directly
index.addUnique(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
}

public synchronized Entry remove(byte[] key) throws IOException {
Expand All @@ -502,13 +543,23 @@ public synchronized Entry remove(byte[] key) throws IOException {
// add entry to miss-cache
if (readMissCache != null) {
// set the miss cache; if there was already an entry we know that the return value must be null
if (readMissCache.put(readMissCache.row().newEntry(key)) != null) return null;
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) {
this.hasnotUnique++;
} else {
this.hasnotDouble++;
return null;
}
}

// remove entry from hit-cache
if (readHitCache != null) {
Entry entry = readHitCache.remove(key);
if (entry != null) {
if (entry == null) {
this.readMiss++;
} else {
this.readHit++;
this.cacheDelete++;
index.remove(key);
return entry;
}
Expand Down Expand Up @@ -536,23 +587,35 @@ public synchronized Entry removeOne() throws IOException {

if ((writeBufferUnique != null) && (writeBufferUnique.size() > 0)) {
Entry entry = writeBufferUnique.removeOne();
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(entry.getColBytes(index.primarykey())));
if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(entry.getColBytes(index.primarykey())));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
return entry;
}

if ((writeBufferDoubles != null) && (writeBufferDoubles.size() > 0)) {
Entry entry = writeBufferDoubles.removeOne();
byte[] key = entry.getColBytes(index.primarykey());
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key));
if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
index.remove(key);
return entry;
}

Entry entry = index.removeOne();
if (entry == null) return null;
byte[] key = entry.getColBytes(index.primarykey());
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key));
if (readHitCache != null) readHitCache.remove(key);
if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.remove(key);
if (dummy != null) this.cacheDelete++;
}
return entry;
}

Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/kelondro/kelondroRowCollection.java
View file
Expand Up @@ -150,7 +150,7 @@ public kelondroRow row() {
private final void ensureSize(int elements) {
int needed = elements * rowdef.objectsize();
if (chunkcache.length >= needed) return;
byte[] newChunkcache = new byte[needed * 2];
byte[] newChunkcache = new byte[needed * 12 / 10]; // increase space by 20%
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache;
newChunkcache = null;
Expand Down

0 comments on commit d454ca4

Please sign in to comment.