Skip to content

Commit

Permalink
fixed datatabase inconsistency bugs
Browse files Browse the repository at this point in the history 
inserted many debug lines
added a huge number of asserts
extended database test methods


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3579 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Apr 19, 2007
1 parent ca79362 commit 595ee10
Show file tree
Hide file tree
Showing 19 changed files with 323 additions and 218 deletions.
65 changes: 52 additions & 13 deletions source/dbtest.java
View file
Expand Up @@ -11,6 +11,7 @@
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
Expand All @@ -23,15 +24,18 @@
import de.anomic.kelondro.kelondroFlexSplitTable;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroIntBytesMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroOrder;
import de.anomic.kelondro.kelondroProfile;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroSplittedTree;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow.Entry;
import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog;
import de.anomic.ymage.ymageChart;

public class dbtest {
Expand All @@ -50,8 +54,9 @@ public class dbtest {
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) +
kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes();
String s = (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
return s.getBytes();
}

public static byte[] randomHash(Random r) {
Expand Down Expand Up @@ -124,6 +129,7 @@ public WriteJob(final kelondroIndex aTable, final long aSource) {

public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("write: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (IOException e) {
Expand All @@ -141,6 +147,7 @@ public RemoveJob(final kelondroIndex aTable, final long aSource) {

public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable().remove(entry.getKey());
} catch (IOException e) {
Expand Down Expand Up @@ -194,6 +201,9 @@ public static void main(String[] args) {

// create the database access
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0);
if (dbe.equals("kelondroRowSet")) {
table = new kelondroRowSet(testRow, 0);
}
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow), true, false);
Expand Down Expand Up @@ -376,21 +386,31 @@ public static void main(String[] args) {
long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
long r;
int p;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
r = random.nextLong() % 1000;
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes());
serverInstantThread.oneTimeJob(new WriteJob(table, r), 0, 50);
if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) {
serverInstantThread.oneTimeJob(new ReadJob(table, random.nextLong() % writeCount), random.nextLong() % 1000, 20);
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt()) % ra.size();
System.out.println("remove: " + ((Long) ra.get(p)).longValue());
R = (Long) ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
System.out.println("remove: " + R.longValue());
serverInstantThread.oneTimeJob(new RemoveJob(table, ((Long) ra.remove(p)).longValue()), 0, 50);
}
}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
while (serverInstantThread.instantThreadCounter > 0) {
try {Thread.sleep(1000);} catch (InterruptedException e) {} // wait for all tasks to finish
System.out.println("count: " + serverInstantThread.instantThreadCounter + ", jobs: " + serverInstantThread.jobs.toString());
Expand All @@ -404,24 +424,42 @@ public static void main(String[] args) {
long writeCount = Long.parseLong(args[3]);
long readCount = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
Random random = new Random(randomstart);
long r;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
new WriteJob(table, i).run();
//if (i == 30) random = new Random(randomstart);
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes());
new WriteJob(table, r).run();
if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) {
new ReadJob(table, random.nextLong() % writeCount).run();
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt()) % ra.size();
R = (Long) ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
new RemoveJob(table, ((Long) ra.remove(p)).longValue()).run();
}
}
try {Thread.sleep(1000);} catch (InterruptedException e) {}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
}

long aftercommand = System.currentTimeMillis();
// final report
System.out.println("Database size = " + table.size() + " unique entries.");

// finally close the database/table
if (table instanceof kelondroTree) ((kelondroTree) table).close();
if (table instanceof kelondroFlexTable) ((kelondroFlexTable) table).close();
if (table instanceof kelondroSplittedTree) ((kelondroSplittedTree) table).close();
if (table instanceof dbTable) ((dbTable)table).close();
table.close();

long afterclose = System.currentTimeMillis();

Expand Down Expand Up @@ -501,7 +539,7 @@ public void close() {
this.theDBConnection = null;
}

public int size() throws IOException {
public int size() {
int size = -1;
try {
String sqlQuery = new String
Expand All @@ -521,7 +559,8 @@ public int size() throws IOException {

return size;
} catch (Exception e) {
throw new IOException(e.getMessage());
e.printStackTrace();
return -1;
}
}

Expand Down
30 changes: 13 additions & 17 deletions source/de/anomic/kelondro/kelondroBase64Order.java
View file
Expand Up @@ -299,8 +299,8 @@ public final int compare(byte[] a, int aoffset, int alength, byte[] b, int boffs
public final int compare0(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) {
if (zero == null) return compares(a, aoffset, alength, b, boffset, blength);
// we have an artificial start point. check all combinations
int az = compares(a, aoffset, alength, zero, 0, zero.length); // -1 if a < z; 0 if a == z; 1 if a > z
int bz = compares(b, boffset, blength, zero, 0, zero.length); // -1 if b < z; 0 if b == z; 1 if b > z
int az = compares(a, aoffset, alength, zero, 0, Math.min(alength, zero.length)); // -1 if a < z; 0 if a == z; 1 if a > z
int bz = compares(b, boffset, blength, zero, 0, Math.min(blength, zero.length)); // -1 if b < z; 0 if b == z; 1 if b > z
if ((az == 0) && (bz == 0)) return 0;
if (az == 0) return -1;
if (bz == 0) return 1;
Expand All @@ -315,32 +315,28 @@ public final int compares(byte[] a, int aoffset, int alength, byte[] b, int boff
int i = 0;
final int al = Math.min(alength, a.length - aoffset);
final int bl = Math.min(blength, b.length - boffset);
final int len = (al > bl) ? bl : al;
if (al > bl) return 1;
if (al < bl) return -1;
byte ac, bc;
byte acc, bcc;
while (i < len) {
assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, len);
assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, len);
while (i < al) {
assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, al);
assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, al);
ac = a[aoffset + i];
assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, len);
assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, al);
bc = b[boffset + i];
assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, len);
if ((ac == 0) && (bc == 0)) return 0; // zero-terminated length
assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, al);
acc = ahpla[ac];
assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, len) + "/" + new String(a, aoffset, len) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset);
assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, al) + "/" + new String(a, aoffset, al) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset);
bcc = ahpla[bc];
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, len) + "/" + new String(b, boffset, len) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset);
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, al) + "/" + new String(b, boffset, al) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset);
if (acc > bcc) return 1;
if (acc < bcc) return -1;
// else the bytes are equal and it may go on yet undecided
i++;
}
// check if we have a zero-terminated equality
if ((i == al) && (i < bl) && (b[i + boffset] == 0)) return 0;
if ((i == bl) && (i < al) && (a[i + aoffset] == 0)) return 0;
// no, decide by length
if (al > bl) return 1;
if (al < bl) return -1;
// no, they are equal
// they are equal
return 0;
}

Expand Down
4 changes: 3 additions & 1 deletion source/de/anomic/kelondro/kelondroBytesIntMap.java
View file
Expand Up @@ -49,6 +49,7 @@ public synchronized int geti(byte[] key) throws IOException {
}

public synchronized int puti(byte[] key, int i) throws IOException {
assert i >= 0 : "i = " + i;
assert (key != null);
//assert (!(serverLog.allZero(key)));
kelondroRow.Entry newentry = ki.row().newEntry();
Expand All @@ -60,6 +61,7 @@ public synchronized int puti(byte[] key, int i) throws IOException {
}

public synchronized void addi(byte[] key, int i) throws IOException {
assert i >= 0 : "i = " + i;
assert (key != null);
//assert (!(serverLog.allZero(key)));
kelondroRow.Entry newentry = ki.row().newEntry();
Expand Down Expand Up @@ -87,7 +89,7 @@ public synchronized int removeonei() throws IOException {
return (int) indexentry.getColLong(1);
}

public synchronized int size() throws IOException {
public synchronized int size() {
return ki.size();
}

Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/kelondro/kelondroCache.java
View file
Expand Up @@ -638,7 +638,7 @@ public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey)
return index.rows(up, firstKey);
}

public int size() throws IOException {
public int size() {
return index.size() + ((writeBufferUnique == null) ? 0 : writeBufferUnique.size());
}

Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/kelondro/kelondroFlexSplitTable.java
View file
Expand Up @@ -137,7 +137,7 @@ public static final String dateSuffix(Date date) {
return new String(suffix);
}

public int size() throws IOException {
public int size() {
Iterator i = tables.values().iterator();
int s = 0;
while (i.hasNext()) {
Expand Down

0 comments on commit 595ee10

Please sign in to comment.