Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: internetarchive/heritrix3
base: 20502f5
...
head fork: internetarchive/heritrix3
compare: 4e05c89
Checking mergeability… Don't worry, you can still create the pull request.
  • 2 commits
  • 4 files changed
  • 0 commit comments
  • 1 contributor
Commits on Jan 19, 2012
@nlevitt nlevitt HER-1984 save script state - implement by adding a map to the applica…
…tion context for arbitrary use, and make sure the app context is available in all scripting environments

* PathSharingContext.java
    new member variable ConcurrentHashMap data and accessor getData()
* ScriptedProcessor.java, ScriptedDecideRule.java
    make appCtx available to scripts; also remove unused member sharedMap
* ActionDirectory.java
    formatting fix
803f9ca
@nlevitt nlevitt Merge branch 'master' of github.com:internetarchive/heritrix3 4e05c89
View
15 commons/src/main/java/org/archive/spring/PathSharingContext.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -178,4 +179,18 @@ protected void initLifecycleProcessor() {
}
super.initLifecycleProcessor();
}
+
+ protected ConcurrentHashMap<Object, Object> data;
+
+ /**
+ * @return a shared map for arbitrary use during a crawl; for example, could
+ * be used for state persisting for the duration of the crawl,
+ * shared among ScriptedProcessor, scripting console, etc scripts
+ */
+ public ConcurrentHashMap<Object, Object> getData() {
+ if (data == null) {
+ data = new ConcurrentHashMap<Object, Object>();
+ }
+ return data;
+ }
}
View
2  engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java
@@ -312,7 +312,7 @@ protected boolean tryAsScript(File actionFile, String timestamp) {
PrintWriter rawOut = new PrintWriter(rawString);
Exception ex = null;
engine.put("rawOut", rawOut);
- engine.put("appCtx",appCtx);
+ engine.put("appCtx", appCtx);
// evaluate and record any exception
try {
View
6 modules/src/main/java/org/archive/modules/ScriptedProcessor.java
@@ -20,8 +20,6 @@
package org.archive.modules;
import java.io.Reader;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -106,8 +104,6 @@ public void setApplicationContext(ApplicationContext applicationContext) throws
transient protected ThreadLocal<ScriptEngine> threadEngine =
new ThreadLocal<ScriptEngine>();
protected ScriptEngine sharedEngine;
- /** map for optional use by scripts */
- public Map<Object,Object> sharedMap = new ConcurrentHashMap<Object,Object>();
/**
* Constructor.
@@ -135,12 +131,14 @@ protected void innerProcess(CrawlURI curi) {
// synchronization is harmless for local thread engine,
// necessary for shared engine
engine.put("curi",curi);
+ engine.put("appCtx", appCtx);
try {
engine.eval("process(curi)");
} catch (ScriptException e) {
logger.log(Level.WARNING,e.getMessage(),e);
} finally {
engine.put("curi", null);
+ engine.put("appCtx", null);
}
}
}
View
6 modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java
@@ -20,8 +20,6 @@
package org.archive.modules.deciderules;
import java.io.Reader;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -106,8 +104,6 @@ public void setApplicationContext(ApplicationContext applicationContext) throws
transient protected ThreadLocal<ScriptEngine> threadEngine =
new ThreadLocal<ScriptEngine>();
transient protected ScriptEngine sharedEngine;
- /** map for optional use by scripts */
- public Map<Object,Object> sharedMap = new ConcurrentHashMap<Object,Object>();
public ScriptedDecideRule() {
}
@@ -128,12 +124,14 @@ public DecideResult innerDecide(CrawlURI uri) {
// necessary for shared engine
try {
engine.put("object",uri);
+ engine.put("appCtx", appCtx);
return (DecideResult)engine.eval("decisionFor(object)");
} catch (ScriptException e) {
logger.log(Level.WARNING,e.getMessage(),e);
return DecideResult.NONE;
} finally {
engine.put("object", null);
+ engine.put("appCtx", null);
}
}
}

No commit comments for this range

Something went wrong with that request. Please try again.