Skip to content

Commit

Permalink
* simplified initialization of database objects
Browse files Browse the repository at this point in the history 
* replaced kelondroTree for NURLs by kelondroFlex
* replaced kelondroTree for EURLs by kelondroFlex
take care, may be very buggy
please finish crawls before updating. crawls will be lost.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2452 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Aug 24, 2006
1 parent 0b73f2b commit db1eae0
Show file tree
Hide file tree
Showing 42 changed files with 395 additions and 644 deletions.
5 changes: 3 additions & 2 deletions doc/News.html
View file
Expand Up @@ -84,7 +84,7 @@ <h2>News</h2>
</ul>
-->

<br>v0.46_20060823_2442<p>
<br><p>v0.46_20060823_2442
<ul>
<li>Web Interface Enhancements</li>
<ul>
Expand Down Expand Up @@ -130,7 +130,8 @@ <h2>News</h2>
</ul>
</ul>

<br><p>v0.45_20060501_2046<ul>
<br><p>v0.45_20060501_2046
<ul>
<li>Enhanced Search Functions</li>
<ul>
<li>Re-designed search page: this shows now a simplified page without the left navigation column. The column is only shown if the user is logged-in.</li>
Expand Down
13 changes: 4 additions & 9 deletions source/dbtest.java
View file
Expand Up @@ -173,23 +173,18 @@ public static void main(String[] args) {
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength);
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent);
} else {
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow, true);
}
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow);
}
if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName();
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
table = new kelondroSplittedTree(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer, preload,
8, testRow, 1, 80,
true);
8, testRow, 1, 80);
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow, true);
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow);
}
if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow);
Expand Down
8 changes: 1 addition & 7 deletions source/de/anomic/data/blogBoard.java
View file
Expand Up @@ -83,13 +83,7 @@ public class blogBoard {
public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_'));
}
}

Expand Down
72 changes: 15 additions & 57 deletions source/de/anomic/data/bookmarksDB.java
View file
Expand Up @@ -120,67 +120,25 @@ public static Date iso8601ToDate(String iso8601){
}

public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) {
//bookmarks
//check if database exists
// bookmarks
tagCache=new HashMap();
bookmarkCache=new HashMap();
if(bookmarksFile.exists()){
try {
//open it
this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//database reset :-((
bookmarksFile.delete();
bookmarksFile.getParentFile().mkdirs();
//urlHash is 12 bytes long
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
}else{
//new database
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
//tags
//check if database exists
if(tagsFile.exists()){
try {
//open it
this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
tagsFile.delete();
tagsFile.getParentFile().mkdirs();
// max. 128 byte long tags
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
}else{
//new database
tagsFile.getParentFile().mkdirs();
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_'));

// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
if (!tagsFileExisted) rebuildTags();

// dates
//check if database exists
if(datesFile.exists()){
try {
//open it
this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
datesFile.delete();
datesFile.getParentFile().mkdirs();
//YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
}else{
//new database
datesFile.getParentFile().mkdirs();
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_'));
if (!datesExisted) rebuildDates();

}

public void close(){
try {
flushBookmarkCache();
Expand Down
9 changes: 1 addition & 8 deletions source/de/anomic/data/messageBoard.java
View file
Expand Up @@ -70,14 +70,7 @@ public class messageBoard {
public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
if (path.exists()) try {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, '_'));
} catch (IOException e) {
path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
}
database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_'));
}
sn = 0;
}
Expand Down
20 changes: 3 additions & 17 deletions source/de/anomic/data/userDB.java
View file
Expand Up @@ -76,22 +76,8 @@ public userDB(File userTableFile, int bufferkb, long preloadTime) {
this.userTableFile = userTableFile;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (userTableFile.exists()) {
try {
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} catch (IOException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
} else {
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_'));
}

public int dbCacheNodeChunkSize() {
Expand All @@ -109,7 +95,7 @@ void resetDatabase() {
} catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
}

public void close() {
Expand Down
20 changes: 4 additions & 16 deletions source/de/anomic/data/wikiBoard.java
View file
Expand Up @@ -70,25 +70,13 @@ public class wikiBoard {
private static HashMap authors = new HashMap();

public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_'));
}
new File(bkppath.getParent()).mkdir();
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
if (bkppath.exists()) try {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, '_'));
} catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
} else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
}
bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_'));
}
}

Expand Down
4 changes: 4 additions & 0 deletions source/de/anomic/index/indexContainerOrder.java
View file
Expand Up @@ -53,4 +53,8 @@ public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, in
return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength);
}

public boolean equals(kelondroOrder otherOrder) {
if (!(otherOrder instanceof indexContainerOrder)) return false;
return this.embeddedOrder.equals(((indexContainerOrder) otherOrder).embeddedOrder);
}
}
4 changes: 2 additions & 2 deletions source/de/anomic/index/indexRAMCacheRI.java
View file
Expand Up @@ -96,7 +96,7 @@ private void dump(int waitingSeconds) throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
Expand Down Expand Up @@ -175,7 +175,7 @@ private void dump(int waitingSeconds) throws IOException {
private long restore() throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (!(indexDumpFile.exists())) return 0;
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis);
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/index/indexURL.java
View file
Expand Up @@ -451,7 +451,7 @@ public int cacheObjectChunkSize() {

public long[] cacheObjectStatus() {
if (urlHashCache instanceof kelondroTree) return ((kelondroTree) urlHashCache).cacheObjectStatus();
return null;
return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
}

public static final int flagTypeID(String hash) {
Expand Down
8 changes: 8 additions & 0 deletions source/de/anomic/kelondro/kelondroAbstractOrder.java
View file
Expand Up @@ -83,4 +83,12 @@ public byte[] zero() {
public void rotate(byte[] newzero) {
this.zero = newzero;
}

public boolean equals(kelondroOrder otherOrder) {
String thisSig = this.signature();
String otherSig = otherOrder.signature();
if ((thisSig == null) || (otherSig == null)) return false;
return thisSig.equals(otherSig);
}

}
13 changes: 4 additions & 9 deletions source/de/anomic/kelondro/kelondroCollectionIndex.java
View file
Expand Up @@ -91,7 +91,7 @@ public kelondroCollectionIndex(File path, String filenameStub, int keyLength, ke
this.loadfactor = loadfactor;

// create index table
index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength), true);
index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength));

// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize());
Expand Down Expand Up @@ -119,13 +119,8 @@ private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNum
"byte[] key-" + index.row().width(0) + "," +
"byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.playloadrow.objectsize())
);
if (f.exists()) {
return new kelondroFixedWidthArray(f, rowdef);
} else if (create) {
return new kelondroFixedWidthArray(f, rowdef, 0, true);
} else {
return null;
}
if ((!(f.exists())) && (!create)) return null;
return new kelondroFixedWidthArray(f, rowdef, 0);
}

private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) {
Expand Down Expand Up @@ -460,7 +455,7 @@ public static void main(String[] args) {
collectionIndex.close();

// printout of index
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9), true);
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9));
index.print();
index.close();
} catch (IOException e) {
Expand Down

0 comments on commit db1eae0

Please sign in to comment.