Skip to content

Commit

Permalink
Crawler hostbalancer does not delete finished queue files,
Browse files Browse the repository at this point in the history
use alternative delete to fight the sympthom (and fix deletion of host dirs on startup)
Root cause (which class holds a lock on .stack) not found.
http://mantis.tokeek.de/view.php?id=404
  • Loading branch information
reger committed Jun 5, 2014
1 parent e64be5d commit 92d1604
Showing 1 changed file with 12 additions and 11 deletions.
23 changes: 12 additions & 11 deletions source/net/yacy/crawler/HostQueue.java
Expand Up @@ -50,6 +50,7 @@
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowHandleSet;
import net.yacy.kelondro.table.Table;
import static net.yacy.kelondro.util.FileUtils.deletedelete;
import net.yacy.kelondro.util.kelondroException;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
Expand Down Expand Up @@ -134,14 +135,14 @@ private int openAllStacks() {
int sz = depthStack.size();
if (sz == 0) {
depthStack.close();
stackFile.delete();
deletedelete(stackFile);
} else {
this.depthStacks.put(depth, depthStack);
c += sz;
}
}
} catch (NumberFormatException e) {}
}
}
return c;
}

Expand All @@ -154,7 +155,7 @@ public synchronized int getLowestStackDepth() {
if (entry == null) return 0; // happens only if map is empty
if (entry.getValue().size() == 0) {
entry.getValue().close();
getFile(entry.getKey()).delete();
deletedelete(getFile(entry.getKey()));
this.depthStacks.remove(entry.getKey());
continue;
}
Expand All @@ -174,7 +175,7 @@ private Index getLowestStack() {
if (entry == null) return null; // happens only if map is empty
if (entry.getValue().size() == 0) {
entry.getValue().close();
getFile(entry.getKey()).delete();
deletedelete(getFile(entry.getKey()));
this.depthStacks.remove(entry.getKey());
continue;
}
Expand Down Expand Up @@ -222,7 +223,7 @@ private Index openStack(File f, int depth) {
ConcurrentLog.logException(e);
}
} else {
try {
try {
return new BufferedObjectIndex(new Table(f, Request.rowdef, EcoFSBufferSize, 0, false, exceed134217727, true), objectIndexBufferSize);
} catch (final SpaceExceededException e) {
try {
Expand All @@ -233,9 +234,9 @@ private Index openStack(File f, int depth) {
} catch (kelondroException e) {
// possibly the file was closed meanwhile
ConcurrentLog.logException(e);
}
}
}
}
return null;
}

Expand All @@ -244,25 +245,25 @@ public synchronized void close() {
for (Map.Entry<Integer, Index> entry: this.depthStacks.entrySet()) {
int size = entry.getValue().size();
entry.getValue().close();
if (size == 0) getFile(entry.getKey()).delete();
if (size == 0) deletedelete(getFile(entry.getKey()));
}
this.depthStacks.clear();
String[] l = this.hostPath.list();
if ((l == null || l.length == 0) && this.hostPath != null) this.hostPath.delete();
if ((l == null || l.length == 0) && this.hostPath != null) deletedelete(this.hostPath);
}

@Override
public synchronized void clear() {
for (Map.Entry<Integer, Index> entry: this.depthStacks.entrySet()) {
entry.getValue().close();
getFile(entry.getKey()).delete();
deletedelete(getFile(entry.getKey()));
}
this.depthStacks.clear();
String[] l = this.hostPath.list();
if (l != null) for (String s: l) {
new File(this.hostPath, s).delete();
deletedelete(new File(this.hostPath, s));
}
this.hostPath.delete();
deletedelete(this.hostPath);
}

@Override
Expand Down

0 comments on commit 92d1604

Please sign in to comment.