Skip to content

Commit

Permalink
Domain-lists gzip-compressable and sendable via cr-send/receive
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1883 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
rramthun committed Mar 13, 2006
1 parent 1a13c8b commit 9f979d4
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 10 deletions.
2 changes: 1 addition & 1 deletion htroot/yacy/transfer.java
View file
Expand Up @@ -94,7 +94,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve

if (process.equals("permission")) {
prop.put("process", 0);
if ((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) {
if (((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) || ((filename.startsWith("domlist")) && (filename.endsWith(".txt.gz") || filename.endsWith(".zip")))) {
// consolidation of cr files
//System.out.println("yacy/transfer:post=" + post.toString());
//String cansendprotocol = (String) post.get("can-send-protocol", "http");
Expand Down
43 changes: 34 additions & 9 deletions source/yacy.java
View file
Expand Up @@ -63,6 +63,7 @@
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import java.util.zip.GZIPOutputStream;

import de.anomic.data.translator;
import de.anomic.http.httpHeader;
Expand Down Expand Up @@ -146,8 +147,8 @@ public final class yacy {
* FIXME: Why is this so complicated?
*
* @param s Combined version string
* @return Pretty string where version and svn-Version are separated by a
* slash
* @return Pretty string where version and SVN-version are separated by a
* slash, e.g. "0.435/01818"
*/
public static String combinedVersionString2PrettyString(String s) {
long svn;
Expand Down Expand Up @@ -196,7 +197,7 @@ private static void startup(String homePath, long startupMemFree, long startupMe
try {
/*String[] check =*/ "a,b".split(","); // split needs java 1.4
} catch (NoSuchMethodError e) {
System.err.println("STARTUP: Java Version too low. You need at least Java 1.4.2 to run YACY");
System.err.println("STARTUP: Java Version too low. You need at least Java 1.4.2 to run YaCy");
Thread.sleep(3000);
System.exit(-1);
}
Expand Down Expand Up @@ -239,7 +240,7 @@ private static void startup(String homePath, long startupMemFree, long startupMe
if (!unzipTest.exists()) {
String errorMsg = "The archive file containing YaCy was not unpacked correctly. " +
"Please use 'GNU-Tar' or upgrade to a newer version of your unzip software.\n" +
"For detailed informations on this bug see: " +
"For detailed information on this bug see: " +
"http://www.yacy-forum.de/viewtopic.php?t=715";
System.err.println(errorMsg);
serverLog.logSevere("STARTUP", errorMsg);
Expand Down Expand Up @@ -426,7 +427,7 @@ private static void startup(String homePath, long startupMemFree, long startupMe
//Error
}

try{ //seperate try, because we want this, even when the File "version2 does not exist.
try{ //seperate try, because we want this, even if the file "version" does not exist.
if(! currentRev.equals(sb.getConfig("svnRevision", "")) ){ //is this another version?!
final File sourceDir = new File(sb.getConfig("htRootPath", "htroot"));
final File destDir = new File(sb.getConfig("htLocalePath", "DATA/HTDOCS/locale"), lang);
Expand Down Expand Up @@ -461,7 +462,7 @@ private static void startup(String homePath, long startupMemFree, long startupMe
try {
sb.waitForShutdown();
} catch (Exception e) {
serverLog.logSevere("MAIN CONTROL LOOP", "PANIK: " + e.getMessage(),e);
serverLog.logSevere("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
}
restart = sb.getConfig("restart", "false");

Expand Down Expand Up @@ -1177,9 +1178,14 @@ private static void transferCR(String targetaddress, String crfile) {
serverLog.logInfo("TRANSFER-CR", "could not read file " + crfile);
}
}

/**
* Generates a text file containing all domains in this peer's DB.
*
* @param format String which determines format of the text file. Possible values: "html", "zip", "gzip" or "plain"
*/
private static void domlist(String homePath, String format, String targetName) {
File root = new File(homePath);

File root = new File(homePath);
try {
plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), 16000, 1000, 1000);
Iterator eiter = pool.loadedURL.entries(true, false);
Expand All @@ -1190,18 +1196,24 @@ private static void domlist(String homePath, String format, String targetName) {
if ((entry != null) && (entry.url() != null)) doms.add(entry.url().getHost());
}

// output file
// output file in HTML format
if (format.equals("html")) {
File file = new File(root, targetName + ".html");
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file));
Iterator i = doms.iterator();
String key;
bos.write(("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">").getBytes());
bos.write(serverCore.crlf);
bos.write(("<html><head><title>YaCy domainlist</title></head><body>").getBytes());
bos.write(serverCore.crlf);
while (i.hasNext()) {
key = i.next().toString();
bos.write(("<a href=\"http://" + key + "\">" + key + "</a><br>").getBytes());
bos.write(serverCore.crlf);
}
bos.write(("</body></html>").getBytes());
bos.close();
//output file in plain text but compressed with ZIP
} else if (format.equals("zip")) {
ZipEntry zipEntry = new ZipEntry(targetName + ".txt");
File file = new File(root, targetName + ".zip");
Expand All @@ -1215,6 +1227,18 @@ private static void domlist(String homePath, String format, String targetName) {
bos.write(serverCore.crlf);
}
bos.close();
//output file in plain text but compressed with GZIP
} else if (format.equals("gzip")) {
File file = new File(root, targetName + ".txt.gz");
GZIPOutputStream bos = new GZIPOutputStream(new FileOutputStream(file));
Iterator i = doms.iterator();
String key;
while (i.hasNext()) {
key = i.next().toString();
bos.write((key).getBytes());
bos.write(serverCore.crlf);
}
bos.close();
}
else {
// plain text list
Expand Down Expand Up @@ -1552,6 +1576,7 @@ public static void main(String args[]) {
if (args.length >= 3 && args[1].equals("-format")) {
if (args[2].equals("html")) format = args[2];
if (args[2].equals("zip")) format = args[2];
if (args[2].equals("gzip")) format = args[2];
args = shift(args, 1, 2);
}
if (args.length == 2) applicationRoot= args[1];
Expand Down

0 comments on commit 9f979d4

Please sign in to comment.