Skip to content

Commit

Permalink
better self-healing of corrupted databases
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5406 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Dec 22, 2008
1 parent 963da8c commit fc8189f
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 21 deletions.
26 changes: 19 additions & 7 deletions source/de/anomic/kelondro/kelondroAbstractRecords.java
View file
Expand Up @@ -219,7 +219,7 @@ protected synchronized int allocatePayload(byte[] chunk) throws IOException {
if (USAGE.FREEC == 0) {
// generate new entry
final int index = USAGE.allCount();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, othervise the USAGE computaton does not work
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, otherwise the USAGE computation does not work
USAGE.USEDC++;
writeused(false);
return index;
Expand All @@ -230,13 +230,14 @@ protected synchronized int allocatePayload(byte[] chunk) throws IOException {
// take link
int index = USAGE.FREEH.index;
if (index == kelondroHandle.NUL) {
serverLog.logSevere("kelondroTray/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = USAGE.allCount() + 1;
USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0;
index = USAGE.USEDC - 1;
//entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
} else {
//System.out.println("*DEBUG* ALLOCATED DELETED INDEX " + index);
// check for valid seek position
final long seekp = seekpos(USAGE.FREEH);
if (seekp >= entryFile.length()) {
Expand All @@ -249,12 +250,23 @@ protected synchronized int allocatePayload(byte[] chunk) throws IOException {
} else {
// read link to next element of FREEH chain
USAGE.FREEH.index = entryFile.readInt(seekp);
assert ((USAGE.FREEH.index == kelondroHandle.NUL) && (USAGE.FREEC == 0)) || seekpos(USAGE.FREEH) < entryFile.length() : "allocatePayload: USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length();
// check consistency
if (((USAGE.FREEH.index != kelondroHandle.NUL) || (USAGE.FREEC != 0)) && seekpos(USAGE.FREEH) >= entryFile.length()) {
// the FREEH pointer cannot be correct, because it points to a place outside of the file.
// to correct this, we reset the FREH pointer and return a index that has been calculated as if USAGE.FREE == 0
serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length() + "; wrong FREEH has been patched, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0;
//entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
}
}
}
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
USAGE.writeused(false);
USAGE.writefree();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
return index;
}
//}
Expand Down
34 changes: 20 additions & 14 deletions source/de/anomic/kelondro/kelondroCollectionIndex.java
View file
Expand Up @@ -373,20 +373,26 @@ private kelondroRow.Entry array_new(
arrayEntry.setCol(1, collection.exportCollection());

// write a new entry in this array
final int newRowNumber = array.add(arrayEntry);
try {
final int newRowNumber = array.add(arrayEntry);

// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize);
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize);
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));

// after calling this method there must be an index.addUnique(indexrow);
return indexrow;
// after calling this method there must be an index.addUnique(indexrow);
return indexrow;
} catch (Exception e) {
// the index appears to be corrupted at a particular point
serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, this.payloadrow.objectsize, partitionNumber, serialNumber).toString() + " has errors \"" + e.getMessage() + "\" (error #" + indexErrors + ")");
return null;
}
}

private void array_add(
Expand Down Expand Up @@ -451,7 +457,7 @@ public synchronized void put(final byte[] key, final kelondroRowCollection colle
// create new row and index entry
if ((collection != null) && (collection.size() > 0)) {
indexrow = array_new(key, collection); // modifies indexrow
index.addUnique(indexrow);
if (indexrow != null) index.addUnique(indexrow);
}
return;
}
Expand Down Expand Up @@ -497,7 +503,7 @@ public synchronized void merge(final indexContainer container) throws IOExceptio
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) {
indexrow = array_new(key, container); // modifies indexrow
index.addUnique(indexrow); // write modified indexrow
if (indexrow != null) index.addUnique(indexrow); // write modified indexrow
} else {
// merge with the old collection
// attention! this modifies the indexrow entry which must be written with index.put(indexrow) afterwards!
Expand Down

0 comments on commit fc8189f

Please sign in to comment.