Basic WatchCrawler.html, which loades the Queues using AJAX.

ajax.js: generic Ajax-Functions xml/queues/indexing_p*: the Backend git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1002 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Oct 29, 2005 · 5526d2f · 5526d2f
1 parent 92c49b4
commit 5526d2f
Show file tree

Hide file tree

Showing 6 changed files with 219 additions and 0 deletions.
diff --git a/htroot/WatchCrawler.html b/htroot/WatchCrawler.html
@@ -0,0 +1,29 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+<html>
+<head>
+<title>YaCy '#[clientname]#': Watch Crawler</title>
+#[metas]#
+<script src="/js/ajax.js"></script>
+<script src="/js/WatchCrawler.js"></script>
+</head>
+<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
+#[header]#
+<br>
+<h2>Watch Crawler</h2>
+<p />
+Indexing Queue:<br />
+<table border="0" cellpadding="2" cellspacing="1" id="indexingTable">
+<tr class="TableHeader">
+<th class="small">Initiator</th>
+<th class="small">Depth</th>
+<th class="small">Modified Date</th>
+<th class="small">Anchor Name</th>
+<th class="small">URL</th>
+<th class="small">Size</th>
+<th class="small">Delete</th>
+</tr>
+</table>
+
+#[footer]#
+</body>
+</html>
diff --git a/htroot/js/WatchCrawler.js b/htroot/js/WatchCrawler.js
@@ -0,0 +1,26 @@
+function handleResponse(){
+    if(http.readyState == 4){
+        var response = http.responseXML;
+        indexingTable=document.getElementById("indexingTable");
+        entries=response.getElementsByTagName("entry");
+        for(i=0;i<entries.length;i++){
+            row=document.createElement("tr");
+
+            //simply add all fields chronologically
+            //TODO: add them by Name
+            field=entries[i].firstChild;
+            while(field != null){
+                if(field.nodeType == 1 && field.nodeName != "inProcess"){//Element
+                    col=document.createElement("td");
+                    text=document.createTextNode(field.firstChild.nodeValue);
+                    col.appendChild(text);
+                    row.appendChild(col);
+                }
+                field=field.nextSibling;
+            }
+
+            indexingTable.appendChild(row);
+        }
+    }
+}
+sndReq("/xml/queues/indexing_p.xml");
diff --git a/htroot/js/ajax.js b/htroot/js/ajax.js
@@ -0,0 +1,18 @@
+function createRequestObject() {
+    var ro;
+    var browser = navigator.appName;
+    if(browser == "Microsoft Internet Explorer"){
+        ro = new ActiveXObject("Microsoft.XMLHTTP");
+    }else{
+        ro = new XMLHttpRequest();
+    }
+    return ro;
+}
+var http = createRequestObject();
+
+function sndReq(action) {
+    //http.open('get', 'rpc.php?action='+action);
+    http.open('get', action);
+    http.onreadystatechange = handleResponse;
+    http.send(null);
+}
diff --git a/htroot/xml/queues/indexing_p.java b/htroot/xml/queues/indexing_p.java
@@ -0,0 +1,129 @@
+// /xml/queues/indexing_p.java
+// -------------------------------
+// part of the AnomicHTTPD caching proxy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2004, 2005
+// last major change: 28.10.2005
+// this file is contributed by Alexander Schier
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notive above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+// You must compile this file with
+// javac -classpath .:../classes IndexCreate_p.java
+// if the shell's current path is HTROOT
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
+import java.util.Iterator;
+import java.io.IOException;
+
+import de.anomic.data.wikiCode;
+import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.plasma.plasmaSwitchboardQueue;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacySeed;
+
+public class indexing_p {
+
+    private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
+    private static String daydate(Date date) {
+        if (date == null) return ""; else return dayFormatter.format(date);
+    }
+
+    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
+        // return variable that accumulates replacements
+        plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+        wikiCode wikiTransformer = new wikiCode(switchboard);
+        serverObjects prop = new serverObjects();
+        prop.put("rejected", 0);
+        int showRejectedCount = 10;
+
+        yacySeed initiator;
+
+        if ((switchboard.sbQueue.size() == 0) && (switchboard.indexingTasksInProcess.size() == 0)) {
+            prop.put("list", 0); //is empty
+        } else {
+            plasmaSwitchboardQueue.Entry pcentry;
+            int inProcessCount = 0;
+            long totalSize = 0;
+            int i=0; //counter
+            try {
+                ArrayList entryList = new ArrayList();
+
+                // getting all entries that are currently in process
+                synchronized (switchboard.indexingTasksInProcess) {
+                    inProcessCount = switchboard.indexingTasksInProcess.size();
+                    entryList.addAll(switchboard.indexingTasksInProcess.values());
+                }
+
+                // getting all enqueued entries
+                if ((switchboard.sbQueue.size() > 0)) {
+                    entryList.addAll(switchboard.sbQueue.list(0));
+                }
+
+                for (i = 0; i < entryList.size(); i++) {
+                    boolean inProcess = i < inProcessCount;
+                    pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
+                    long entrySize = pcentry.size();
+                    totalSize += entrySize;
+                    if ((pcentry != null)&&(pcentry.url() != null)) {
+                        initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
+                        prop.put("list_"+i+"_initiator", ((initiator == null) ? "proxy" : wikiTransformer.replaceHTML(initiator.getName())));
+                        prop.put("list_"+i+"_depth", pcentry.depth());
+                        prop.put("list_"+i+"_modified", (pcentry.responseHeader() == null) ? "" : daydate(pcentry.responseHeader().lastModified()));
+                        prop.put("list_"+i+"_anchor", (pcentry.anchorName()==null)?"":wikiTransformer.replaceHTML(pcentry.anchorName()));
+                        prop.put("list_"+i+"_url", wikiTransformer.replaceHTML(pcentry.normalizedURLString()));
+                        prop.put("list_"+i+"_size", entrySize);
+                        prop.put("list_"+i+"_inProcess", (inProcess)?1:0);
+                        prop.put("list_"+i+"_hash", pcentry.urlHash());
+                    }
+                }
+                prop.put("list", i);
+            } catch (IOException e) {}
+        }
+
+        // return rewrite properties
+        return prop;
+    }
+
+}
+
+
+
diff --git a/htroot/xml/queues/indexing_p.xml b/htroot/xml/queues/indexing_p.xml
@@ -0,0 +1,16 @@
+<xml version="1.0">
+<indexingQueue>
+#{list}#
+  <entry>
+    <initiator>#[initiator]#</initiator>
+    <depth>#[depth]#</depth>
+    <modified>#[modified]#</modified>
+    <anchor>#[anchor]#</anchor>
+    <url>#[url]#</url>
+    <size>#[size]#</size>
+    <hash>#[hash]#</hash>
+    <inProcess>#(inProcess)#false::true#(/inProcess)#</inProcess>
+  </entry>
+#{/list}#
+</indexingQueue>
+</xml>
diff --git a/source/de/anomic/data/wikiCode.java b/source/de/anomic/data/wikiCode.java
@@ -111,6 +111,7 @@ public static String replaceHTML(String result) {
 	// avoide html inside
 	//p0 = 0; while ((p0 = result.indexOf("&", p0)) >= 0) result = result.substring(0, p0) + "&amp;" + result.substring(p0 + 1);
 	p0 = 0; while ((p0 = result.indexOf('"', p0)) >= 0) result = result.substring(0, p0) + "&quot;" + result.substring(p0 + 1);
+	p0 = 0; while ((p0 = result.indexOf('&', p0)) >= 0) result = result.substring(0, p0) + "&amp;" + result.substring(p0 + 1);
 	p0 = 0; while ((p0 = result.indexOf("<", p0)) >= 0) result = result.substring(0, p0) + "&lt;" + result.substring(p0 + 1);
 	p0 = 0; while ((p0 = result.indexOf(">", p0)) >= 0) result = result.substring(0, p0) + "&gt;" + result.substring(p0 + 1);
 	//p0 = 0; while ((p0 = result.indexOf("*", p0)) >= 0) result = result.substring(0, p0) + "&#149;" + result.substring(p0 + 1);