Skip to content

Commit

Permalink
*) better handling of server shutdown
Browse files Browse the repository at this point in the history 
   See: e.g. http://www.yacy-forum.de/viewtopic.php?p=25234

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2470 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
theli committed Sep 4, 2006
1 parent 7df5727 commit b4acbda
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 44 deletions.
22 changes: 17 additions & 5 deletions source/de/anomic/plasma/plasmaRankingDistribution.java
View file
Expand Up @@ -106,7 +106,7 @@ public int size() {
if ((sourcePath.exists()) && (sourcePath.isDirectory())) return sourcePath.list().length; else return 0;
}

public boolean transferRanking(int count) {
public boolean transferRanking(int count) throws InterruptedException {

if (method == METHOD_NONE) {
log.logFine("no ranking distribution: no transfer method given");
Expand Down Expand Up @@ -140,6 +140,10 @@ public boolean transferRanking(int count) {
File crfile = null;

for (int i = 0; i < count; i++) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");

// getting the next file to transfer
crfile = new File(sourcePath, outfiles[i]);

if ((method == METHOD_ANYSENIOR) || (method == METHOD_ANYPRINCIPAL)) {
Expand All @@ -161,9 +165,13 @@ public boolean transferRanking(int count) {
return false;
}

private boolean transferRankingAnySeed(File crfile, int trycount) {
private boolean transferRankingAnySeed(File crfile, int trycount) throws InterruptedException {
yacySeed target = null;
for (int j = 0; j < trycount; j++) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");

// get next target
target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);

if (target == null) continue;
Expand All @@ -173,10 +181,14 @@ private boolean transferRankingAnySeed(File crfile, int trycount) {
return false;
}

private boolean transferRankingAddress(File crfile) {
private boolean transferRankingAddress(File crfile) throws InterruptedException {
// try all addresses
for (int i = 0; i < address.length; i++) {
if (transferRankingAddress(crfile, address[i])) return true;
for (int i = 0; i < this.address.length; i++) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");

// try to transfer ranking address using the next address
if (transferRankingAddress(crfile, this.address[i])) return true;
}
return false;
}
Expand Down
79 changes: 46 additions & 33 deletions source/de/anomic/plasma/plasmaSwitchboard.java
View file
Expand Up @@ -542,7 +542,7 @@ ramEURL, getConfigBool("useFlexTableForEURL", true),

// clean up profiles
this.log.logConfig("Cleaning Profiles");
cleanProfiles();
try { cleanProfiles(); } catch (InterruptedException e) { /* Ignore this here */ }

// init ranking transmission
/*
Expand Down Expand Up @@ -769,13 +769,17 @@ private void resetProfiles() {
initProfiles();
}

public boolean cleanProfiles() {
public boolean cleanProfiles() throws InterruptedException {
if ((sbQueue.size() > 0) || (cacheLoader.size() > 0) || (urlPool.noticeURL.stackSize() > 0)) return false;
final Iterator iter = profiles.profiles(true);
plasmaCrawlProfile.entry entry;
boolean hasDoneSomething = false;
try {
while (iter.hasNext()) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");

// getting next profile
entry = (plasmaCrawlProfile.entry) iter.next();
if (!((entry.name().equals("proxy")) || (entry.name().equals("remote")))) {
iter.remove();
Expand Down Expand Up @@ -1085,40 +1089,49 @@ public int cleanupJobSize() {
}

public boolean cleanupJob() {

boolean hasDoneSomething = false;

// do transmission of cr-files
int count = rankingOwnDistribution.size() / 100;
if (count == 0) count = 1;
if (count > 5) count = 5;
rankingOwnDistribution.transferRanking(count);
rankingOtherDistribution.transferRanking(1);

// clean up error stack
if ((urlPool.errorURL.stackSize() > 1000)) {
log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack");
urlPool.errorURL.clearStack();
hasDoneSomething = true;
}
// clean up loadedURL stack
for (int i = 1; i <= 6; i++) {
if (urlPool.loadedURL.getStackSize(i) > 1000) {
log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i);
urlPool.loadedURL.clearStack(i);
try {
boolean hasDoneSomething = false;

// do transmission of cr-files
checkInterruption();
int count = rankingOwnDistribution.size() / 100;
if (count == 0) count = 1;
if (count > 5) count = 5;
rankingOwnDistribution.transferRanking(count);
rankingOtherDistribution.transferRanking(1);

// clean up error stack
checkInterruption();
if ((urlPool.errorURL.stackSize() > 1000)) {
log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack");
urlPool.errorURL.clearStack();
hasDoneSomething = true;
}
}
// clean up profiles
if (cleanProfiles()) hasDoneSomething = true;
// clean up loadedURL stack
for (int i = 1; i <= 6; i++) {
checkInterruption();
if (urlPool.loadedURL.getStackSize(i) > 1000) {
log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i);
urlPool.loadedURL.clearStack(i);
hasDoneSomething = true;
}
}
// clean up profiles
checkInterruption();
if (cleanProfiles()) hasDoneSomething = true;

// clean up news
try {
log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
} catch (IOException e) {}

return hasDoneSomething;
// clean up news
checkInterruption();
try {
log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
} catch (IOException e) {}

return hasDoneSomething;
} catch (InterruptedException e) {
this.log.logInfo("cleanupJob: Shutdown detected");
return false;
}
}

/**
Expand Down
16 changes: 10 additions & 6 deletions source/de/anomic/yacy/yacyNewsPool.java
View file
Expand Up @@ -160,17 +160,21 @@ public int size(int dbKey) {
return switchQueue(dbKey).size();
}

public int automaticProcess() throws IOException {
public int automaticProcess() throws IOException, InterruptedException {
// processes news in the incoming-db
// returns number of processes
yacyNewsRecord record;
int pc = 0;
synchronized (incomingNews) {
for (int i = incomingNews.size() - 1; i >= 0; i--) {
record = incomingNews.top(i);
synchronized (this.incomingNews) {
for (int i = this.incomingNews.size() - 1; i >= 0; i--) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");

// get next news record
record = this.incomingNews.top(i);
if ((i > 500) || (automaticProcessP(record))) {
incomingNews.pop(i);
processedNews.push(record);
this.incomingNews.pop(i);
this.processedNews.push(record);
//newsDB.remove(id);
pc++;
}
Expand Down

0 comments on commit b4acbda

Please sign in to comment.