Skip to content

Commit

Permalink
- fixed charset problem in ConfigProfil_p.html (use accept-charset="U…
Browse files Browse the repository at this point in the history 
…TF-8" in forms)

- fixed wrong XML output if no peers are known in Network.xml
- simplified parsing of table properties in wikiCode and ZTableToken
- reimplemented GC heuristics. They are needed to constantly ensure that an amount of free memory is available which is higher than Java's max. limit for performing a Full GC (please use serverMemory.request(long, boolean) rather than serverMemory.available(long, boolean) to provide data for averaging over the last GCs)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3793 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
karlchenofhell committed Jun 5, 2007
1 parent 6c9df13 commit 30c3d90
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 148 deletions.
2 changes: 1 addition & 1 deletion htroot/ConfigProfile_p.html
View file
Expand Up @@ -14,7 +14,7 @@ <h2>Your Personal Profile</h2>
<p>
You do not need to provide any personal data here, but if you want to distribute your contact information, you can do that here.
</p>
<form action="ConfigProfile_p.html" accept-charset="ascii">
<form action="ConfigProfile_p.html" accept-charset="UTF-8">
<fieldset>
<dl>
<dt><label for="name">Name</label></dt>
Expand Down
52 changes: 25 additions & 27 deletions htroot/Network.xml
View file
@@ -1,7 +1,6 @@
<?xml version="1.0"?>
#(table)#
::
<peers>
#(table)#::
#{list}#
<peer>
<hash>#[hash]#</hash>
Expand Down Expand Up @@ -33,9 +32,7 @@
#(/complete)#
</peer>
#{/list}#
</peers>
::
<peers>
<active>
<count>#[active-count]#</count>
<links>#[active-links]#</links>
Expand All @@ -57,36 +54,37 @@
<words>#[all-words]#</words>
</all>
<your>
<name>#[my-name]#</name>
<hash>#[my-hash]#</hash>
<type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>
<version>#[my-version]#</version>
<utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime>
<links>#[my-links]#</links>
<words>#[my-words]#</words>
<acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex>
<sentwords>#[my-sI]#</sentwords>
<senturls>#[my-sU]#</senturls>
<receivedwords>#[my-rI]#</receivedwords>
<receivedurls>#[my-rU]#</receivedurls>
<ppm>#[my-ppm]#</ppm>
<qph>#[my-qph]#</qph>
<seeds>#[my-seeds]#</seeds>
<connects>#[my-connects]#</connects>
<location>#[my-location]#</location>
<seedurl>#[my-url]#</seedurl>
<name>#[my-name]#</name>
<hash>#[my-hash]#</hash>
<type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>
<version>#[my-version]#</version>
<utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime>
<links>#[my-links]#</links>
<words>#[my-words]#</words>
<acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex>
<acceptranking>#[my-rankingreceive]#</acceptranking>
<sentwords>#[my-sI]#</sentwords>
<senturls>#[my-sU]#</senturls>
<receivedwords>#[my-rI]#</receivedwords>
<receivedurls>#[my-rU]#</receivedurls>
<ppm>#[my-ppm]#</ppm>
<qph>#[my-qph]#</qph>
<seeds>#[my-seeds]#</seeds>
<connects>#[my-connects]#</connects>
<location>#[my-location]#</location>
<seedurl>#[my-url]#</seedurl>
</your>
<cluster>
<ppm>#[gppm]#</ppm>
<qph>#[gqph]#</qph>
<ppm>#[gppm]#</ppm>
<qph>#[gqph]#</qph>
</cluster>
<status>#[comment]#</status>
<!--
0 - nothing
1 - online mode, but no Internet available
2 - not in online mode
-->
</peers>
#(/table)#
</peers>
34 changes: 17 additions & 17 deletions source/de/anomic/data/wiki/tokens/TableToken.java
View file
Expand Up @@ -47,6 +47,7 @@

package de.anomic.data.wiki.tokens;

import java.util.Arrays;
import java.util.HashMap;
import java.util.regex.Pattern;

Expand Down Expand Up @@ -92,10 +93,16 @@ protected void parse() {
private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
private static final HashMap/* <String,String[]> */ ps = new HashMap();
static {
ps.put("frame", new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("rules", new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("valign", new String[] { "top", "middle", "bottom", "baseline" });
ps.put("align", new String[] { "left", "right", "center" });
Arrays.sort(tps);
String[] array;
Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("frame", array);
Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("rules", array);
Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });
ps.put("valign", array);
Arrays.sort(array = new String[] { "left", "right", "center" });
ps.put("align", array);
}

// contributed by [MN]
Expand All @@ -106,42 +113,35 @@ protected void parse() {
* @param properties A string that may contain several table properties and/or junk.
* @return A string that only contains table properties.
*/
private static StringBuffer parseTableProperties(final String properties){
private static StringBuffer parseTableProperties(final String properties) {
String[] values = properties.replaceAll("&quot;", "").split("[= ]"); //splitting the string at = and blanks
StringBuffer sb = new StringBuffer(properties.length());
String key, value;
String[] posVals;
int numberofvalues = values.length;
for (int i=0; i<numberofvalues; i++) {
key = values[i].trim();
if (i + 1 < numberofvalues) {
if (key.equals("nowrap")) {
addPair("nowrap", "nowrap", sb);
} else if (i + 1 < numberofvalues) {
value = values[++i].trim();
if (
(key.equals("summary")) ||
(key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||
((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||
((posVals = (String[])ps.get(key)) != null && isInArray(posVals, value)) ||
(isInArray(tps, key) && value.matches("\\d+"))
((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||
(Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))
) {
addPair(key, value, sb);
continue;
}
}
if (key.equals("nowrap"))
addPair("nowrap", "nowrap", sb);
}
return sb;
}

private static StringBuffer addPair(String key, String value, StringBuffer sb) {
return sb.append(" ").append(key).append("=\"").append(value).append("\"");
}

private static boolean isInArray(Object[] array, Object find) {
for (int i=array.length-1; i>-1; i--)
if (array[i].equals(find)) return true;
return false;
}

public Pattern[] getRegex() { return pattern; }
public String[] getBlockElementNames() { return blockElementNames; }
Expand Down
149 changes: 50 additions & 99 deletions source/de/anomic/data/wikiCode.java
View file
Expand Up @@ -49,6 +49,8 @@
import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;

import de.anomic.data.wiki.abstractWikiParser;
import de.anomic.data.wiki.wikiParser;
Expand All @@ -60,6 +62,23 @@
* parts of YaCy that use this class, like the blog or the profile.
*/
public class wikiCode extends abstractWikiParser implements wikiParser {

/* Table properties */
private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
private static final HashMap/* <String,String[]> */ ps = new HashMap();
static {
Arrays.sort(tps);
String[] array;
Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("frame", array);
Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("rules", array);
Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });
ps.put("valign", array);
Arrays.sort(array = new String[] { "left", "right", "center" });
ps.put("align", array);
}

private String numListLevel="";
private String ListLevel="";
private String defListLevel="";
Expand Down Expand Up @@ -119,7 +138,7 @@ private String processTable(String result, plasmaSwitchboard switchboard){
newrowstart=true;
line="<table";
if (result.trim().length()>lenTableStart) {
line+=parseTableProperties(result.substring(lenTableStart).trim());
line+=parseTableProperties(result.substring(lenTableStart).trim()).toString();
}
line+=">";
result=line;
Expand Down Expand Up @@ -157,7 +176,7 @@ else if ((result.startsWith(cellDivider)) && (table)) {
propEnd=lenCellDivider;
}
else {
line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim());
line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim()).toString();
}
// quick&dirty fix for http://www.yacy-forum.de/viewtopic.php?t=2825 [MN]
if(propEnd > cellEnd){
Expand All @@ -178,112 +197,44 @@ else if (result.startsWith(tableEnd) && (table)) { // Table end
}
return result;
}

//contributed by [MN]
// contributed by [MN], changes by [FB]
/** This method takes possible table properties and tests if they are valid.
* Valid in this case means if they are a property for the table, tr or td
* tag as stated in the HTML Pocket Reference by Jennifer Niederst (1st edition)
* The method is important to avoid XSS attacks on the wiki via table properties.
* @param str A string that may contain several table properties and/or junk.
* @param properties A string that may contain several table properties and/or junk.
* @return A string that only contains table properties.
*/
private String parseTableProperties(String str){
str = str.replaceAll("&quot;", ""); //killing all quotationmarks
String[] values = str.split("[= ]"); //splitting the string at = and blanks
str=""; //recycling... ;-)
int numberofvalues = values.length;
for(int i=0;i<numberofvalues;i++){
if((values[i].equals("rowspan")) ||
(values[i].equals("colspan")) ||
(values[i].equals("vspace")) ||
(values[i].equals("hspace")) ||
(values[i].equals("cellspacing")) ||
(values[i].equals("cellpadding")) ||
(values[i].equals("border"))){
if(i+1<numberofvalues){
if(values[i+1].matches("\\d+")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if((values[i].equals("width"))||(values[i].equals("height"))){
if(i+1<numberofvalues){
if(values[i+1].matches("\\d+%{0,1}")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("align")){
if(i+1<numberofvalues) {
if((values[i+1].equals("left")) ||
(values[i+1].equals("right")) ||
(values[i+1].equals("center"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("valign")){
if(i+1<numberofvalues) {
if((values[i+1].equals("top")) ||
(values[i+1].equals("middle")) ||
(values[i+1].equals("bottom")) ||
(values[i+1].equals("baseline"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
private static StringBuffer parseTableProperties(final String properties) {
final String[] values = properties.replaceAll("&quot;", "").split("[= ]"); //splitting the string at = and blanks
final StringBuffer sb = new StringBuffer(properties.length());
String key, value;
String[] posVals;
final int numberofvalues = values.length;
for (int i=0; i<numberofvalues; i++) {
key = values[i].trim();
if (key.equals("nowrap")) {
addPair("nowrap", "nowrap", sb);
} else if (i + 1 < numberofvalues) {
value = values[++i].trim();
if (
(key.equals("summary")) ||
(key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||
((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||
((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||
(Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))
) {
addPair(key, value, sb);
}
}
else if(values[i].equals("bgcolor")){
if(i+1<numberofvalues){
if(values[i+1].matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("rules")){
if(i+1<numberofvalues) {
if((values[i+1].equals("none")) ||
(values[i+1].equals("groups")) ||
(values[i+1].equals("rows")) ||
(values[i+1].equals("cols")) ||
(values[i+1].equals("all"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("frame")){
if(i+1<numberofvalues) {
if((values[i+1].equals("void")) ||
(values[i+1].equals("above")) ||
(values[i+1].equals("below")) ||
(values[i+1].equals("hsides")) ||
(values[i+1].equals("lhs")) ||
(values[i+1].equals("rhs")) ||
(values[i+1].equals("vsides")) ||
(values[i+1].equals("box")) ||
(values[i+1].equals("border"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("summary")){
if(i+1<numberofvalues){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
else if(values[i].equals("nowrap")){
str = str + " nowrap";
}
}
return str;
} //end contrib [MN]
return sb;
}

private static StringBuffer addPair(String key, String value, StringBuffer sb) {
return sb.append(" ").append(key).append("=\"").append(value).append("\"");
}

/** This method processes ordered lists.
*/
Expand Down
7 changes: 4 additions & 3 deletions source/de/anomic/server/serverAbstractThread.java
View file
Expand Up @@ -250,7 +250,7 @@ public void run() {
logSystem("thread '" + this.getName() + "' deployed, " + ((this.busyPause < 0) ? "starting job." : "starting loop."));
}
long timestamp;
long memstamp0, memstamp1, memnow;
long memstamp0, memstamp1;
boolean isBusy;
//Runtime rt = Runtime.getRuntime();

Expand All @@ -273,7 +273,8 @@ public void run() {
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
} else if ((memnow = serverMemory.available()) > memprereq) try {
//} else if ((memnow = serverMemory.available()) > memprereq) try {
} else if (serverMemory.request(memprereq, false)) try {
// do job
timestamp = System.currentTimeMillis();
memstamp0 = serverMemory.used();
Expand Down Expand Up @@ -307,7 +308,7 @@ public void run() {
busyCycles++;
} else {
log.logWarning("Thread '" + this.getName() + "' runs short memory cycle. Free mem: " +
(memnow / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");
(serverMemory.available() / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");
// omit job, not enough memory
// process scheduled pause
timestamp = System.currentTimeMillis();
Expand Down

0 comments on commit 30c3d90

Please sign in to comment.