Permalink
Browse files

added a cache to prevent too many seed enumerations

  • Loading branch information...
Orbiter committed May 17, 2017
1 parent bd7411a commit 7de7879f1321156250b319f9d8b538ad724cd7ad
Showing with 36 additions and 16 deletions.
  1. +11 −0 source/net/yacy/peers/Seed.java
  2. +25 −16 source/net/yacy/peers/SeedDB.java
@@ -55,6 +55,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
@@ -225,6 +226,16 @@
private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations.
Bitfield bitfield = null;
private static ConcurrentMap<String, String> map2concurrentMap(Map<String, String> dna0) {
ConcurrentMap<String, String> dna = new ConcurrentHashMap<String, String>();
dna.putAll(dna0);
return dna;
}
public Seed(Map.Entry<byte[], Map<String, String>> dna0) {
this(UTF8.String(dna0.getKey()), dna0.getValue() instanceof ConcurrentMap ? (ConcurrentMap<String, String>) dna0.getValue() : map2concurrentMap(dna0.getValue()));
}
public Seed(final String theHash, final ConcurrentMap<String, String> theDna) {
// create a seed with a pre-defined hash map
assert theHash != null;
@@ -42,7 +42,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.federate.yacy.Distribution;
import net.yacy.cora.order.Base64Order;
@@ -414,12 +413,30 @@ public Seed anySeedVersion(final double minVersion) {
return e.next();
}
private final Map<Long, Integer> sizeActiveSince_result = new ConcurrentHashMap<>();
private final Map<Long, Long> sizeActiveSince_time = new ConcurrentHashMap<>();
/**
* count the number of peers that had been seen within the given time limit
* @param limitMinutes the time limit in minutes. 1440 minutes is a day
* @return the number of peers seen in the given time
*/
public int sizeActiveSince(final long limitMinutes) {
// buffering because that iteration takes a long time
long now = System.currentTimeMillis();
Integer s0 = sizeActiveSince_result.get(limitMinutes);
Long t0 = sizeActiveSince_time.get(limitMinutes);
if (s0 == null || t0 == null || now - t0 > 60000) {
int s1 = sizeActiveSinceInt(limitMinutes);
sizeActiveSince_result.put(limitMinutes, s1);
sizeActiveSince_time.put(limitMinutes, now);
return s1;
}
return s0;
}
private int sizeActiveSinceInt(final long limitMinutes) {
int c = this.seedActiveDB.size();
Seed seed;
Iterator<Seed> i = seedsSortedDisconnected(false, Seed.LASTSEEN);
@@ -1039,26 +1056,18 @@ public boolean hasNext() {
private Seed internalNext() {
if (this.it == null || !(this.it.hasNext())) return null;
try {
Map.Entry<byte[], Map<String, String>> dna0;
ConcurrentHashMap<String, String> dna;
Map.Entry<byte[], Map<String, String>> dna;
while (this.it.hasNext()) {
try {
dna0 = this.it.next();
dna = this.it.next();
} catch (final OutOfMemoryError e) {
ConcurrentLog.logException(e);
dna0 = null;
}
assert dna0 != null;
if (dna0 == null) continue;
if (dna0.getValue() instanceof ConcurrentHashMap) {
dna = (ConcurrentHashMap<String, String>) dna0.getValue();
} else {
dna = new ConcurrentHashMap<String, String>();
dna.putAll(dna0.getValue());
dna = null;
}
if (dna0.getKey() == null) continue; // bad seed
final String hash = UTF8.String(dna0.getKey());
return new Seed(hash, dna);
assert dna != null;
if (dna == null) continue;
if (dna.getKey() == null) continue; // bad seed
return new Seed(dna);
}
return null;
} catch (final Exception e) {

0 comments on commit 7de7879

Please sign in to comment.