Skip to content

Commit

Permalink
fixed some bugs for auto-filter and added monitor in profile list
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1959 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Mar 24, 2006
1 parent 1f32ad8 commit 708cc6c
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 4 deletions.
8 changes: 8 additions & 0 deletions htroot/IndexCreate_p.html
View file
Expand Up @@ -272,6 +272,10 @@ <h2>Index Creation</h2>
<td class="small"><b>Start URL</b></td>
<td width="16" class="small"><b>Depth</b></td>
<td width="60" class="small"><b>Filter</b></td>
<td width="10" class="small"><b>MaxAge</b></td>
<td width="10" class="small"><b>Auto Filter Depth</b></td>
<td class="small"><b>Auto Filter Content</b></td>
<td width="10" class="small"><b>Max Page Per Domain</b></td>
<td width="10" class="small"><b>Accept "?" URLs</b></td>
<td width="10" class="small"><b>Fill Proxy Cache</b></td>
<td width="10" class="small"><b>Local Indexing</b></td>
Expand All @@ -283,6 +287,10 @@ <h2>Index Creation</h2>
<td class="small"><a class="small" href="#[startURL]#">#[startURL]#</a></td>
<td class="small">#[depth]#</td>
<td class="small">#[filter]#</td>
<td class="small">#[crawlingIfOlder]#</td>
<td class="small">#[crawlingDomFilterDepth]#</td>
<td class="small">#[crawlingDomFilterContent]#</td>
<td class="small">#[crawlingDomMaxPages]#</td>
<td class="small">#(withQuery)#no::yes#(/withQuery)#</td>
<td class="small">#(storeCache)#no::yes#(/storeCache)#</td>
<td class="small">#(localIndexing)#no::yes#(/localIndexing)#</td>
Expand Down
14 changes: 11 additions & 3 deletions htroot/IndexCreate_p.java
View file
Expand Up @@ -95,11 +95,11 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
env.setConfig("crawlingFilter", newcrawlingfilter);
int newcrawlingdepth = Integer.parseInt(post.get("crawlingDepth", "0"));
env.setConfig("crawlingDepth", Integer.toString(newcrawlingdepth));
int recrawlIfOlder = Integer.parseInt(post.get("recrawlIfOlder", "-1"));
int recrawlIfOlder = Integer.parseInt(post.get("crawlingIfOlder", "-1"));
env.setConfig("crawlingIfOlder", recrawlIfOlder);
int domFilterDepth = Integer.parseInt(post.get("domFilterDepth", "-1"));
int domFilterDepth = Integer.parseInt(post.get("crawlingDomFilterDepth", "-1"));
env.setConfig("crawlingDomFilterDepth", Integer.toString(domFilterDepth));
int domMaxPages = Integer.parseInt(post.get("domMaxPages", "-1"));
int domMaxPages = Integer.parseInt(post.get("crawlingDomMaxPages", "-1"));
env.setConfig("crawlingDomMaxPages", Integer.toString(domMaxPages));
boolean crawlingQ = post.get("crawlingQ", "").equals("on");
env.setConfig("crawlingQ", (crawlingQ) ? "true" : "false");
Expand Down Expand Up @@ -353,6 +353,8 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
Iterator it = switchboard.profiles.profiles(true);
plasmaCrawlProfile.entry profile;
dark = true;
Iterator domnamesi;
String domnames;
while (it.hasNext()) {
profile = (plasmaCrawlProfile.entry) it.next();
//table += profile.map().toString() + "<br>";
Expand All @@ -362,6 +364,12 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
prop.put("crawlProfiles_"+count+"_handle", wikiCode.replaceHTML(profile.handle()));
prop.put("crawlProfiles_"+count+"_depth", profile.generalDepth());
prop.put("crawlProfiles_"+count+"_filter", profile.generalFilter());
prop.put("crawlProfiles_"+count+"_crawlingIfOlder", (profile.recrawlIfOlder() == Long.MAX_VALUE) ? "no re-crawl" : ""+profile.recrawlIfOlder());
prop.put("crawlProfiles_"+count+"_crawlingDomFilterDepth", (profile.domFilterDepth() == Integer.MAX_VALUE) ? "inactive" : ""+profile.domFilterDepth());
domnamesi = profile.domNames();
domnames=""; while (domnamesi.hasNext()) domnames += ((String) domnamesi.next()) + ", ";
prop.put("crawlProfiles_"+count+"_crawlingDomFilterContent", domnames);
prop.put("crawlProfiles_"+count+"_crawlingDomMaxPages", (profile.domMaxPages() == Integer.MAX_VALUE) ? "unlimited" : ""+profile.domMaxPages());
prop.put("crawlProfiles_"+count+"_withQuery", ((profile.crawlingQ()) ? 1 : 0));
prop.put("crawlProfiles_"+count+"_storeCache", ((profile.storeHTCache()) ? 1 : 0));
prop.put("crawlProfiles_"+count+"_localIndexing", ((profile.localIndexing()) ? 1 : 0));
Expand Down
6 changes: 5 additions & 1 deletion source/de/anomic/plasma/plasmaCrawlProfile.java
View file
Expand Up @@ -57,6 +57,7 @@
public class plasmaCrawlProfile {

private kelondroMap profileTable;
private HashMap domsCache;
private File profileTableFile;
private int bufferkb;

Expand All @@ -73,6 +74,7 @@ public plasmaCrawlProfile(File file, int bufferkb) {
dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, '#', true);
}
profileTable = new kelondroMap(dyn);
domsCache = new HashMap();
}

public int[] dbCacheChunkSize() {
Expand Down Expand Up @@ -270,7 +272,8 @@ public String toString() {

public entry(Map mem) {
this.mem = mem;
this.doms = new HashMap();
this.doms = (HashMap) domsCache.get(this.mem.get("handle"));
if (this.doms == null) this.doms = new HashMap();
}

public Map map() {
Expand Down Expand Up @@ -393,6 +396,7 @@ public void domInc(String domain) {
// increase counter
doms.put(domain, new Integer(c.intValue() + 1));
}
domsCache.put(this.mem.get("handle"), doms);
}
public int domCount(String domain) {
Integer c = (Integer) doms.get(domain);
Expand Down

0 comments on commit 708cc6c

Please sign in to comment.