Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace deprecated routines in guava #50

Merged
merged 18 commits into from
Mar 18, 2014
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions commons/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,6 @@
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>r08</version>
</dependency>

<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@

import org.archive.bdb.KryoBinding;

import com.google.common.collect.MapEvictionListener;
import com.google.common.collect.MapMaker;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.bind.tuple.TupleBinding;
Expand Down Expand Up @@ -64,8 +65,9 @@
* @author paul baclace (conversion to ConcurrentMap)
*
*/
public class ObjectIdentityBdbManualCache<V extends IdentityCacheable>
implements ObjectIdentityCache<V>, Closeable, Serializable, MapEvictionListener<String, V> {
@SuppressWarnings("ALL")
public class ObjectIdentityBdbManualCache<V extends IdentityCacheable>
implements ObjectIdentityCache<V>, Closeable, Serializable {
private static final long serialVersionUID = 1L;
private static final Logger logger =
Logger.getLogger(ObjectIdentityBdbManualCache.class.getName());
Expand Down Expand Up @@ -110,6 +112,18 @@ public class ObjectIdentityBdbManualCache<V extends IdentityCacheable>
*/
public ObjectIdentityBdbManualCache() {
super();
dirtyItems = CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(5, TimeUnit.MINUTES)
.removalListener(new RemovalListener<String, V>() {
@Override
public void onRemoval(RemovalNotification<String, V> stringVRemovalNotification) {
evictions.incrementAndGet();
diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue());
}
})
.<String, V>build()
.asMap();
}

/**
Expand All @@ -127,16 +141,18 @@ public ObjectIdentityBdbManualCache() {
public void initialize(final Environment env, String dbName,
final Class valueClass, final StoredClassCatalog classCatalog)
throws DatabaseException {
// TODO: tune capacity for actual threads, expected size of key caches?
this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap();
// TODO: tune capacity for actual threads, expected size of key caches?
this.memMap = CacheBuilder.newBuilder()
.concurrencyLevel(64)
.initialCapacity(8192)
.softValues()
.<String, V>build()
.asMap();
this.db = openDatabase(env, dbName);
this.diskMap = createDiskMap(this.db, classCatalog, valueClass);
// keep a record of items that must be persisted; auto-persist if
// unchanged after 5 minutes, or more than 10K would collect
this.dirtyItems = new MapMaker().concurrencyLevel(64)
.maximumSize(10000).expireAfterWrite(5,TimeUnit.MINUTES)
.evictionListener(this).makeMap();


this.count = new AtomicLong(diskMap.size());
}

Expand Down Expand Up @@ -363,9 +379,9 @@ public void dirtyKey(String key) {
dirtyItems.put(key,val);
}

@Override
public void onEviction(String key, V val) {
/*@Override
public void onRemoval(RemovalNotification<String, V> stringVRemovalNotification) {
evictions.incrementAndGet();
diskMap.put(key, val);
}
diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue());
}*/
}
2 changes: 1 addition & 1 deletion commons/src/main/java/org/archive/util/TestUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ private static void scanSuite(TestSuite suite, File start, File dir)
}
cname = cname.replace(File.separatorChar, '.');
cname = cname.substring(0, cname.length() - 5);
suite.addTestSuite(Class.forName(cname));
suite.addTestSuite((Class<? extends TestCase>) Class.forName(cname));
}
}
}
Expand Down
5 changes: 2 additions & 3 deletions engine/src/main/java/org/archive/crawler/util/TopNSet.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,9 @@
import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;

import com.google.common.cache.CacheBuilder;
import org.archive.util.Histotable;

import com.google.common.collect.MapMaker;

/**
* Counting Set which only remembers the 'top N' of all String values
* reported (with counts) to it. Precise if counts reported for a
Expand Down Expand Up @@ -56,7 +55,7 @@ public class TopNSet implements Serializable {

public TopNSet(int size){
maxsize = size;
set = new MapMaker().concurrencyLevel(64).makeMap();
set = CacheBuilder.newBuilder().concurrencyLevel(64).<String, Long>build().asMap();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.archive.checkpointing.Checkpointable;
Expand All @@ -42,9 +45,6 @@
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;

import com.google.common.base.Function;
import com.google.common.collect.MapMaker;

/**
* A step, post-ExtractorHTMLForms, where a followup CrawlURI to
* attempt a form submission may be synthesized.
Expand Down Expand Up @@ -119,22 +119,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
Logger.getLogger(FormLoginProcessor.class.getName());

// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsSeenCount =
new MapMaker().makeComputingMap(new Function<String, AtomicLong>() {
@Override
public AtomicLong apply(String arg0) {
return new AtomicLong(0L);
}
});
ConcurrentMap<String, AtomicLong> eligibleFormsSeenCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();

// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsAttemptsCount =
new MapMaker().makeComputingMap(new Function<String, AtomicLong>() {
@Override
public AtomicLong apply(String arg0) {
return new AtomicLong(0L);
}
});
ConcurrentMap<String, AtomicLong> eligibleFormsAttemptsCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();

/**
* SURT prefix against which configured username/password is
Expand Down