Skip to content

Commit

Permalink
code cleanup
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1173 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Dec 6, 2005
1 parent 8f1f2da commit 7661844
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 95 deletions.
18 changes: 9 additions & 9 deletions source/de/anomic/htmlFilter/htmlFilterContentScraper.java
View file
Expand Up @@ -86,7 +86,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
private HashMap images;
private String title;
private String headline;
private serverByteBuffer text;
private serverByteBuffer content;
private URL root;

public htmlFilterContentScraper(URL root) {
Expand All @@ -98,13 +98,13 @@ public htmlFilterContentScraper(URL root) {
this.images = new HashMap();
this.title = "";
this.headline = "";
this.text = new serverByteBuffer(1024);
this.content = new serverByteBuffer(1024);
}

public void scrapeText(byte[] newtext) {
// System.out.println("SCRAPE: " + new String(newtext));
if ((text.length() != 0) && (text.byteAt(text.length() - 1) != 32)) text.append(32);
text.append(super.stripAll(new serverByteBuffer(newtext, newtext.length + 1)).trim()).append(32);
if ((content.length() != 0) && (content.byteAt(content.length() - 1) != 32)) content.append(32);
content.append(super.stripAll(new serverByteBuffer(newtext, newtext.length + 1)).trim()).append(32);
}


Expand Down Expand Up @@ -222,8 +222,8 @@ public String getHeadline() {
// extract headline from content
if (title.length() > 0) hl = title.trim();
else if (headline.length() > 0) hl = headline.trim();
else if (text.length() > 80) hl = new String(text.getBytes(), 0, 80).trim();
else hl = text.trim().toString();
else if (content.length() > 80) hl = new String(content.getBytes(), 0, 80).trim();
else hl = content.trim().toString();

// clean the line: may contain too many funny symbols
for (int i = 0; i < hl.length(); i++)
Expand All @@ -238,7 +238,7 @@ public String getHeadline() {
}

public byte[] getText() {
return text.getBytes();
return content.getBytes();
}

public Map getAnchors() {
Expand All @@ -256,7 +256,7 @@ public void close() {
images = null;
title = null;
headline = null;
text = null;
content = null;
root = null;
}

Expand All @@ -265,7 +265,7 @@ public void print() {
System.out.println("HEADLINE:" + headline);
System.out.println("ANCHORS :" + anchors.toString());
System.out.println("IMAGES :" + images.toString());
System.out.println("TEXT :" + new String(text.getBytes()));
System.out.println("TEXT :" + new String(content.getBytes()));
}

public static void main(String[] args) {
Expand Down
140 changes: 70 additions & 70 deletions source/de/anomic/http/httpc.java
View file
Expand Up @@ -131,7 +131,7 @@ public final class httpc {
// class variables
private Socket socket = null; // client socket for commands
private Thread socketOwner = null;
String host = null;
private String host = null;
//private long timeout;

// output and input streams for client control connection
Expand Down Expand Up @@ -872,15 +872,17 @@ public response POST(String path, httpHeader requestHeader, InputStream ins) thr

/**
* Call the server with the CONNECT-method.
* This is used to establish https-connections through a https-proxy
*
* @param host To which host should a connection be made?
* @param port Which port should be connected?
* @param requestHeader prefilled httpHeader.
* @return Instance of response with the content.
*/
public response CONNECT(String host, int port, httpHeader requestHeader) throws IOException {

public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException {
try {
send(httpHeader.METHOD_CONNECT, host + ":" + port, requestHeader, false);
send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false);
return new response(false);
} catch (SocketException e) {
throw new IOException(e.getMessage());
Expand Down Expand Up @@ -1520,6 +1522,67 @@ public static ArrayList getRegisteredOpenSockets(Thread thread) {
return openSockets;
}
}

/**
* This method outputs the input stream to either an output socket or an
* file or both. If the length of the input stream is given in the
* header, exactly that lenght is written. Otherwise the stream is
* written, till it is closed. If this instance is zipped, stream the
* input stream through gzip to unzip it on the fly.
*
* @param procOS OutputStream where the stream is to be written. If null
* no write happens.
* @param bufferOS OutputStream where the stream is to be written too.
* If null no write happens.
* @param clientInput InputStream where the content is to be read from.
* @throws IOException
*/
public static void writeContentX(InputStream clientInput, boolean usegzip, long length, OutputStream procOS, OutputStream bufferOS) throws IOException {
// we write length bytes, but if length == -1 (or < 0) then we
// write until the input stream closes
// procOS == null -> no write to procOS
// file == null -> no write to file
// If the Content-Encoding is gzip, we gunzip on-the-fly
// and change the Content-Encoding and Content-Length attributes in the header
byte[] buffer = new byte[2048];
int l;
long len = 0;

// using the proper intput stream
InputStream dis = (usegzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;

// we have three methods of reading: length-based, length-based gzip and connection-close-based
try {
if (length > 0) {
// we read exactly 'length' bytes
while ((len < length) && ((l = dis.read(buffer)) >= 0)) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
len += l;
}
} else {
// no content-length was given, thus we read until the connection closes
while ((l = dis.read(buffer, 0, buffer.length)) >= 0) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
}
}
} catch (java.net.SocketException e) {
throw new IOException("Socket exception: " + e.getMessage());
} catch (java.net.SocketTimeoutException e) {
throw new IOException("Socket time-out: " + e.getMessage());
} finally {
// close the streams
if (procOS != null) {
if (procOS instanceof httpChunkedOutputStream)
((httpChunkedOutputStream)procOS).finish();
procOS.flush();
}
if (bufferOS != null) bufferOS.flush();
buffer = null;
}
}


/**
* Inner Class to get the response of an http-request and parse it.
Expand Down Expand Up @@ -1669,7 +1732,7 @@ public boolean success() {
public byte[] writeContent() throws IOException {
int contentLength = (int) this.responseHeader.contentLength();
serverByteBuffer sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength);
writeContentX(null, sbb, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), null, sbb);
return sbb.getBytes();
}

Expand All @@ -1684,7 +1747,7 @@ public byte[] writeContent() throws IOException {
public byte[] writeContent(OutputStream procOS) throws IOException {
int contentLength = (int) this.responseHeader.contentLength();
serverByteBuffer sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength);
writeContentX(procOS, sbb, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, sbb);
return sbb.getBytes();
}

Expand All @@ -1702,78 +1765,15 @@ public void writeContent(OutputStream procOS, File file) throws IOException {
FileOutputStream bufferOS = null;
try {
if (file != null) bufferOS = new FileOutputStream(file);
writeContentX(procOS, bufferOS, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS);
} finally {
if (bufferOS != null) {
bufferOS.close();
if (file.length() == 0) file.delete();
}
}
}

/**
* This method outputs the input stream to either an output socket or an
* file or both. If the length of the input stream is given in the
* header, exactly that lenght is written. Otherwise the stream is
* written, till it is closed. If this instance is zipped, stream the
* input stream through gzip to unzip it on the fly.
*
* @param procOS OutputStream where the stream is to be written. If null
* no write happens.
* @param bufferOS OutputStream where the stream is to be written too.
* If null no write happens.
* @param clientInput InputStream where the content is to be read from.
* @throws IOException
*/
public void writeContentX(OutputStream procOS, OutputStream bufferOS, InputStream clientInput) throws IOException {
// we write length bytes, but if length == -1 (or < 0) then we
// write until the input stream closes
// procOS == null -> no write to procOS
// file == null -> no write to file
// If the Content-Encoding is gzip, we gunzip on-the-fly
// and change the Content-Encoding and Content-Length attributes in the header
byte[] buffer = new byte[2048];
int l;
long len = 0;

// find out length
long length = this.responseHeader.contentLength();

// using the proper intput stream
InputStream dis = (this.gzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;

// we have three methods of reading: length-based, length-based gzip and connection-close-based
try {
if (length > 0) {
// we read exactly 'length' bytes
while ((len < length) && ((l = dis.read(buffer)) >= 0)) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
len += l;
}
} else {
// no content-length was given, thus we read until the connection closes
while ((l = dis.read(buffer, 0, buffer.length)) >= 0) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
}
}
} catch (java.net.SocketException e) {
throw new IOException("Socket exception: " + e.getMessage());
} catch (java.net.SocketTimeoutException e) {
throw new IOException("Socket time-out: " + e.getMessage());
} finally {
// close the streams
if (procOS != null) {
if (procOS instanceof httpChunkedOutputStream)
((httpChunkedOutputStream)procOS).finish();
procOS.flush();
}
if (bufferOS != null) bufferOS.flush();
buffer = null;
}
}


/**
* This method outputs a logline to the serverlog with the current
* status of this instance.
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/http/httpd.java
View file
Expand Up @@ -172,8 +172,8 @@ public void reset() {
* Must be called at least once, but can be called again to re-use the object.
* @see de.anomic.server.serverHandler#initSession(de.anomic.server.serverCore.Session)
*/
public void initSession(serverCore.Session session) throws IOException {
this.session = session;
public void initSession(serverCore.Session newsession) throws IOException {
this.session = newsession;
this.userAddress = session.userAddress; // client InetAddress
this.clientIP = this.userAddress.getHostAddress();
if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost";
Expand Down
1 change: 1 addition & 0 deletions source/de/anomic/http/httpdFileHandler.java
View file
Expand Up @@ -211,6 +211,7 @@ public httpdFileHandler(serverSwitch switchboard) {
serverLog.logWarning("HTTPDFileHandler", "Content-MD5 support not availabel ...");
}
}

/*
* Returns the path of a (existing) localized File, or the english one, if not availible.
* @param path is relative from htroot
Expand Down
14 changes: 7 additions & 7 deletions source/de/anomic/kelondro/kelondroAttrSeq.java
View file
Expand Up @@ -109,13 +109,13 @@ public void logWarning(String message) {
this.theLogger.warning("ATTRSEQ WARNING for file " + this.file + ": " + message);
}

private void readAttrFile(File file) throws IOException {
private void readAttrFile(File loadfile) throws IOException {
BufferedReader br = null;
int p;
if (file.toString().endsWith(".gz")) {
br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file))));
if (loadfile.toString().endsWith(".gz")) {
br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(loadfile))));
} else {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
br = new BufferedReader(new InputStreamReader(new FileInputStream(loadfile)));
}
String line;
String key;
Expand All @@ -137,7 +137,7 @@ private void readAttrFile(File file) throws IOException {
if ((p = line.indexOf('=')) > 0) {
key = line.substring(0, p).trim();
if (entries.containsKey(key)) {
logWarning("read PropFile " + file.toString() + ", key " + key + ": double occurrence");
logWarning("read PropFile " + loadfile.toString() + ", key " + key + ": double occurrence");
} else {
entries.put(key, line.substring(p + 1).trim());
}
Expand Down Expand Up @@ -309,10 +309,10 @@ private Object[] atom(String a) {
} else return null;
int p = a.indexOf('-');
if (p < 0) return null;
String name = a.substring(0, p);
String atomname = a.substring(0, p);
try {
int x = Integer.parseInt(a.substring(p + 1));
return new Object[]{name, new Integer(x)};
return new Object[]{atomname, new Integer(x)};
} catch (NumberFormatException e) {
return null;
}
Expand Down
6 changes: 3 additions & 3 deletions source/de/anomic/kelondro/kelondroTree.java
View file
Expand Up @@ -941,19 +941,19 @@ public Object next() {
nextNode = null;
} else {
Object[] stacktop;
Node parent = null;
Node parentNode = null;
int parentpointer = (up) ? rightchild : leftchild;
while ((nodeStack.size() != 0) && (parentpointer == ((up) ? rightchild : leftchild))) {
//System.out.println("step up");
// go on, walk up further
stacktop = (Object[]) nodeStack.removeLast(); // top of stack: Node/parentpointer pair
parent = (Node) stacktop[0];
parentNode = (Node) stacktop[0];
parentpointer = ((Integer) stacktop[1]).intValue();
}
if ((nodeStack.size() == 0) && (parentpointer == ((up) ? rightchild : leftchild))) {
nextNode = null;
} else {
nextNode = parent;
nextNode = parentNode;
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions source/yacy.java
View file
Expand Up @@ -142,8 +142,8 @@ public final class yacy {
public static String combinedVersionString2PrettyString(String s) {
long svn;
try {svn = (long) (100000000.0 * Double.parseDouble(s));} catch (NumberFormatException ee) {svn = 0;}
double version = (Math.floor((double) svn / (double) 100000) / (double) 1000);
String vStr = (version < 0.11) ? "dev" : Double.toString(version);
double v = (Math.floor((double) svn / (double) 100000) / (double) 1000);
String vStr = (v < 0.11) ? "dev" : Double.toString(v);
//while (vStr.length() < 5) vStr = vStr + "0";
svn = svn % 100000;
if (svn > 4000) svn=svn / 10; // fix a previous bug online
Expand All @@ -160,8 +160,8 @@ public static String combinedVersionString2PrettyString(String s) {
* @param svn Current version given from svn.
* @return String with the combined version
*/
public static float versvn2combinedVersion(float version, int svn) {
return (float) (((double) version * 100000000.0 + ((double) svn)) / 100000000.0);
public static float versvn2combinedVersion(float v, int svn) {
return (float) (((double) v * 100000000.0 + ((double) svn)) / 100000000.0);
}

/**
Expand Down

0 comments on commit 7661844

Please sign in to comment.