Skip to content

Commit

Permalink
replaced kelondro storage core; much less objects in kelondro cache n…
Browse files Browse the repository at this point in the history 
…ow; less IO from DB

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@724 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Sep 14, 2005
1 parent dff9660 commit 2148c0c
Show file tree
Hide file tree
Showing 14 changed files with 990 additions and 271 deletions.
2 changes: 1 addition & 1 deletion build.properties
View file
Expand Up @@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4

# Release Configuration
releaseVersion=0.402
releaseVersion=0.403
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}
Expand Down
9 changes: 7 additions & 2 deletions source/de/anomic/kelondro/kelondroAbstractRA.java
View file
Expand Up @@ -66,7 +66,12 @@ public String name() {
abstract public void seek(long pos) throws IOException;
abstract public void close() throws IOException;

// derivated methods:
// derived methods:
public void readFully(byte[] b, int off, int len) throws IOException {
int r = read(b, off, len);
if (r < len) readFully(b, off + r, len - r);
}

public byte readByte() throws IOException {
int ch = this.read();
if (ch < 0) throw new IOException();
Expand Down Expand Up @@ -112,7 +117,7 @@ public void writeLong(long v) throws IOException {
this.write((int) (v >>> 24) & 0xFF); this.write((int) (v >>> 16) & 0xFF);
this.write((int) (v >>> 8) & 0xFF); this.write((int) (v >>> 0) & 0xFF);
}

public void write(byte[] b) throws IOException {
this.write(b, 0, b.length);
}
Expand Down
30 changes: 22 additions & 8 deletions source/de/anomic/kelondro/kelondroArray.java
View file
Expand Up @@ -67,16 +67,22 @@ public kelondroArray(File file) throws IOException{
}

public synchronized byte[][] set(int index, byte[][] row) throws IOException {
if (row.length != columns()) throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns());
if (row.length != columns())
throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns());

// make room for element
if (size() <= index) while (newNode() <= index) {}
Node n;
while (size() <= index) {
n = newNode();
n.commit(CP_NONE);
}

// get the node at position index
Node n = getNode(new Handle(index));
n = getNode(new Handle(index));

// write the row
byte[][] before = n.setValues(row);
n.commit(CP_NONE);

return before;
}
Expand All @@ -103,7 +109,7 @@ public void print() throws IOException {
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(new String(row[j]) + ", ");
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", ");
System.out.println();
}
System.out.println("EndOfTable");
Expand All @@ -123,9 +129,7 @@ private static void cmd(String[] args) {
// create <filename> <valuelen>
File f = new File(args[1]);
if (f.exists()) f.delete();
int[] lens = new int[1];
lens[0] = Integer.parseInt(args[2]);
kelondroArray fm = new kelondroArray(f, lens, 2);
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2);
fm.close();
} else
if ((args.length == 2) && (args[0].equals("-v"))) {
Expand All @@ -149,7 +153,17 @@ private static void cmd(String[] args) {
byte[][] row = new byte[][]{args[3].getBytes()};
fm.set(Integer.parseInt(args[2]), row);
fm.close();
} else {
} else
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9);
for (int i = 0; i < 100; i++) {
fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()});
}
fm.close();
} else
{
System.err.println("usage: kelondroArray -c|-v|-s|-g [file]|[index [value]] <db-file>");
System.err.println("( create, view, set, get)");
System.exit(0);
Expand Down
153 changes: 74 additions & 79 deletions source/de/anomic/kelondro/kelondroBufferedRA.java
View file
Expand Up @@ -3,8 +3,8 @@
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 06.10.2004
// Frankfurt, Germany, 2005
// last major change: 13.09.2005
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
Expand Down Expand Up @@ -43,124 +43,105 @@

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;

public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA {

private static final int bufferSizeExp = 10;
private static final int bufferSize = 1 << bufferSizeExp;
private static final int bufferOffsetFilter = bufferSize - 1;

protected kelondroRA ra;
protected kelondroMScoreCluster bufferScore;
protected HashMap bufferMemory;
private int bufferMaxElements;
private int bufferElementSize;
protected byte[] buffer;
protected int bufferPage;
protected boolean bufferWritten;
private long seekpos;

public kelondroBufferedRA(kelondroRA ra, int buffersize, int elementsize) throws FileNotFoundException {
public kelondroBufferedRA(kelondroRA ra) throws FileNotFoundException {
this.ra = ra;
this.name = ra.name();
this.bufferMemory = new HashMap();
this.bufferScore = new kelondroMScoreCluster();
this.bufferElementSize = elementsize;
this.bufferMaxElements = (int) (buffersize / bufferElementSize);
this.buffer = new byte[bufferSize];
this.seekpos = 0;
this.bufferPage = -1;
this.bufferWritten = true;
}

private int bufferElementNumber(long address) {
return (int) address / bufferElementSize;
private void readBuffer(int newPageNr) throws IOException {
if (newPageNr == bufferPage) return;
bufferPage = newPageNr;
ra.seek(bufferPage << bufferSizeExp);
ra.readFully(buffer, 0, bufferSize);
bufferWritten = true;
}

private int bufferElementOffset(long address) {
return (int) address % bufferElementSize;

private void writeBuffer() throws IOException {
if ((bufferWritten) || (bufferPage < 0)) return;
ra.seek(bufferPage << bufferSizeExp);
ra.write(buffer, 0, bufferSize);
bufferWritten = true;
}

private byte[] readBuffer(int bufferNr) throws IOException {
Integer bufferNrI = new Integer(bufferNr);
byte[] buffer = (byte[]) bufferMemory.get(bufferNrI);
if (buffer == null) {
if (bufferMemory.size() >= bufferMaxElements) {
// delete elements in buffer if buffer too big
Iterator it = bufferScore.scores(true);
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
int age = bufferScore.deleteScore(element);
de.anomic.server.logging.serverLog.logFine("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
}
// add new element
buffer = new byte[bufferElementSize];
//System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize);
ra.seek(bufferNr * bufferElementSize);
ra.read(buffer, 0, bufferElementSize);
bufferMemory.put(bufferNrI, buffer);
private void updateToBuffer(int newPageNr) throws IOException {
if (newPageNr != bufferPage) {
writeBuffer();
readBuffer(newPageNr);
}
bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
return buffer;
}

private void writeBuffer(byte[] buffer, int bufferNr) throws IOException {
if (buffer == null) return;
Integer bufferNrI = new Integer(bufferNr);
ra.seek(bufferNr * bufferElementSize);
ra.write(buffer, 0, bufferElementSize);
bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
}

// pseudo-native method read
public int read() throws IOException {
int bn = bufferElementNumber(seekpos);
int offset = bufferElementOffset(seekpos);
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
return 0xFF & readBuffer(bn)[offset];
updateToBuffer(bn);
return 0xFF & buffer[offset];
}

// pseudo-native method write
public void write(int b) throws IOException {
int bn = bufferElementNumber(seekpos);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn);
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
updateToBuffer(bn);
buffer[offset] = (byte) b;
//writeBuffer(buffer, bn);
bufferWritten = false;
}

public int read(byte[] b, int off, int len) throws IOException {
int bn1 = bufferElementNumber(seekpos);
int bn2 = bufferElementNumber(seekpos + len - 1);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn1);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
//System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len);
System.arraycopy(buffer, offset, b, off, len);
seekpos += len;
return len;
} else {
// do recursively
int thislen = bufferElementSize - offset;
//System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen);
int thislen = bufferSize - offset;
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, thislen, len - thislen);
return thislen + read(b, off + thislen, len - thislen);
}
}

public void write(byte[] b, int off, int len) throws IOException {
int bn1 = bufferElementNumber(seekpos);
int bn2 = bufferElementNumber(seekpos + len - 1);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn1);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(b, off, buffer, offset, len);
bufferWritten = false;
seekpos += len;
//writeBuffer(buffer, bn1);
} else {
// do recursively
int thislen = bufferElementSize - offset;
int thislen = bufferSize - offset;
System.arraycopy(b, off, buffer, offset, thislen);
bufferWritten = false;
seekpos += thislen;
//writeBuffer(buffer, bn1);
write(b, thislen, len - thislen);
write(b, off + thislen, len - thislen);
}
}

Expand All @@ -169,16 +150,30 @@ public void seek(long pos) throws IOException {
}

public void close() throws IOException {
// write all unwritten buffers
Iterator it = bufferScore.scores(true);
while (it.hasNext()) {
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
}
// write unwritten buffer
if (buffer == null) return;
writeBuffer();
ra.close();
bufferScore = null;
bufferMemory = null;
buffer = null;
}

public void finalize() {
try {
close();
} catch (IOException e) {}
}

public static void main(String[] args) {
try {
kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx"));
file.seek(bufferSize - 2);
byte[] b = new byte[]{65, 66, 77, 88};
file.write(b);
file.seek(bufferSize * 2 - 30);
for (int i = 65; i < 150; i++) file.write(i);
file.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

0 comments on commit 2148c0c

Please sign in to comment.