diff --git a/files/cloudgene b/files/cloudgene
index a6667a58..902ecd68 100644
--- a/files/cloudgene
+++ b/files/cloudgene
@@ -5,8 +5,11 @@
# The path to the folder containing MyDaemon.jar
FILE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+# Hadoop path with all jars
+HADOOP_CLASS_PATH=`hadoop classpath`
+
# Our classpath including our jar file
-CLASS_PATH="$FILE_PATH/cloudgene.jar"
+CLASS_PATH="$FILE_PATH/cloudgene.jar:$HADOOP_CLASS_PATH"
# The fully qualified name of the class to execute
CLASS="cloudgene.mapred.CommandLineInterface"
diff --git a/pom.xml b/pom.xml
index 15e2f64c..5a014d6a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
cloudgene
cloudgene
- 2.0.0-rc1
+ 2.0.0-rc2
maven-restlet
@@ -215,7 +215,7 @@
genepi
genepi-hadoop
- mr1-1.2.2
+ mr1-1.2.4
com.sun.jersey
@@ -225,6 +225,10 @@
com.sun.jersey
jersey-server
+
+ org.apache.hadoop
+ hadoop-client
+
@@ -298,7 +302,7 @@
org.apache.hadoop
hadoop-hdfs
2.6.0-cdh5.10.0
- compile
+ test
test-jar
@@ -311,12 +315,68 @@
-
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ 2.6.0-cdh5.10.0
+ test
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ net.java.dev.jets3t
+ jets3t
+
+
+ com.sun.jersey
+ jersey-core
+
+
+ com.sun.jersey
+ jersey-server
+
+
+ com.sun.jersey
+ jersey-json
+
+
+
org.apache.hadoop
hadoop-common
2.6.0-cdh5.10.0
- compile
+ test
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ net.java.dev.jets3t
+ jets3t
+
+
+ com.sun.jersey
+ jersey-core
+
+
+ com.sun.jersey
+ jersey-server
+
+
+ com.sun.jersey
+ jersey-json
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ 2.6.0-cdh5.10.0
+ test
test-jar
@@ -352,6 +412,7 @@
jets3t
+ test
@@ -405,6 +466,12 @@
1.2.14
+
+ commons-io
+ commons-io
+ 2.4
+
+
com.esotericsoftware.yamlbeans
yamlbeans
@@ -471,6 +538,7 @@
dumbster
dumbster
1.6
+ test
diff --git a/src/main/html/webapp/components/core/dashboard/dashboard.js b/src/main/html/webapp/components/core/dashboard/dashboard.js
index 3b2ad6b0..e70a3d58 100644
--- a/src/main/html/webapp/components/core/dashboard/dashboard.js
+++ b/src/main/html/webapp/components/core/dashboard/dashboard.js
@@ -8,24 +8,23 @@ export default Control.extend({
"init": function(element, options) {
var url = 'static/home.stache';
- fetch(url).then(function(response) {
- return response.text();
- }).then(function(data) {
+ $.get(url,
+ function(data) {
- var template = stache(data);
+ var template = stache(data);
- Counter.findOne({}, function(counter) {
- $(element).html(template({
- counter: counter,
- loggedIn: options.appState.loggedIn
- }));
- }, function(message) {
- $(element).html(template({
- counter: undefined,
- loggedIn: options.loggedIn
- }));
- });
+ Counter.findOne({}, function(counter) {
+ $(element).html(template({
+ counter: counter,
+ loggedIn: options.appState.loggedIn
+ }));
+ }, function(message) {
+ $(element).html(template({
+ counter: undefined,
+ loggedIn: options.loggedIn
+ }));
+ });
- });
+ });
}
});
diff --git a/src/main/html/webapp/helpers/router.js b/src/main/html/webapp/helpers/router.js
index 6dac5cbf..c48dc10c 100644
--- a/src/main/html/webapp/helpers/router.js
+++ b/src/main/html/webapp/helpers/router.js
@@ -90,7 +90,7 @@ export default Control.extend({
} else {
view.addClass(this.options.classes);
}
- view.html('Loading...');
+ view.html('');
this.element.append(view);
new Control(view[0], data);
diff --git a/src/main/html/webapp/helpers/static-page.js b/src/main/html/webapp/helpers/static-page.js
index dd176fd6..806fc255 100644
--- a/src/main/html/webapp/helpers/static-page.js
+++ b/src/main/html/webapp/helpers/static-page.js
@@ -9,23 +9,22 @@ export default Control.extend({
"init": function(element, options) {
try {
- if (options.page){
+ if (options.page) {
options.template = "static/" + options.page + ".stache";
}
- fetch(options.template).then(function(response) {
- return response.text();
- }).then(function(data) {
- var view = stache(data);
- if (view) {
- $(element).html(view());
- } else {
- new ErrorPage(element, {
- status: "404",
- message: "Oops, Sorry We Can't Find That Page!"
- });
- }
- });
+ $.get(options.template,
+ function(data) {
+ var view = stache(data);
+ if (view) {
+ $(element).html(view());
+ } else {
+ new ErrorPage(element, {
+ status: "404",
+ message: "Oops, Sorry We Can't Find That Page!"
+ });
+ }
+ });
} catch (e) {
new ErrorPage(element, {
diff --git a/src/main/html/webapp/package-lock.json b/src/main/html/webapp/package-lock.json
index b02d2464..93488b73 100644
--- a/src/main/html/webapp/package-lock.json
+++ b/src/main/html/webapp/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "Cloudgene",
- "version": "2.0.0-rc1",
+ "version": "2.0.0-rc2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
diff --git a/src/main/html/webapp/package.json b/src/main/html/webapp/package.json
index 5a6a44ad..c5c974b1 100644
--- a/src/main/html/webapp/package.json
+++ b/src/main/html/webapp/package.json
@@ -1,6 +1,6 @@
{
"name": "Cloudgene",
- "version": "2.0.0-rc1",
+ "version": "2.0.0-rc2",
"dependencies": {
"@fortawesome/fontawesome-free": "5.3.1",
"bootbox": "4.4.0",
diff --git a/src/main/java/cloudgene/mapred/Main.java b/src/main/java/cloudgene/mapred/Main.java
index 0826019b..ead2e8e3 100644
--- a/src/main/java/cloudgene/mapred/Main.java
+++ b/src/main/java/cloudgene/mapred/Main.java
@@ -37,7 +37,7 @@
public class Main implements Daemon {
- public static final String VERSION = "2.0.0-rc1";
+ public static final String VERSION = "2.0.0-rc2";
private Database database;
@@ -71,17 +71,17 @@ public void runCloudgene(Settings settings, String[] args) throws Exception {
settings.checkTechnologies();
// configure logger
- if (new File("config/log4j.properties").exists()) {
+ if (new File("log4j.properties").exists()) {
- PropertyConfigurator.configure("config/log4j.properties");
+ PropertyConfigurator.configure("log4j.properties");
Slf4jLoggerFacade loggerFacade = new Slf4jLoggerFacade();
Engine.getInstance().setLoggerFacade(loggerFacade);
} else {
- if (new File("log4j.properties").exists()) {
- PropertyConfigurator.configure("log4j.properties");
+ if (new File("config/log4j.properties").exists()) {
+ PropertyConfigurator.configure("config/log4j.properties");
Slf4jLoggerFacade loggerFacade = new Slf4jLoggerFacade();
Engine.getInstance().setLoggerFacade(loggerFacade);
diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java
index 59ffd975..2479224e 100644
--- a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java
+++ b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java
@@ -57,12 +57,14 @@ public Representation get() {
// delete local directory and hdfs directory
String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId());
+ FileUtil.deleteDirectory(localOutput);
+ try {
String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId()));
-
- FileUtil.deleteDirectory(localOutput);
HdfsUtil.delete(hdfsOutput);
-
+ }catch (NoClassDefFoundError e) {
+
+ }
job.setState(AbstractJob.STATE_RETIRED);
dao.update(job);
diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java b/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java
index c412f3e3..c8050fcc 100644
--- a/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java
+++ b/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java
@@ -8,7 +8,6 @@
import java.util.jar.Attributes;
import java.util.jar.Manifest;
-import org.apache.hadoop.mapred.ClusterStatus;
import org.restlet.data.MediaType;
import org.restlet.data.Status;
import org.restlet.representation.Representation;
@@ -23,9 +22,9 @@
import cloudgene.mapred.Main;
import cloudgene.mapred.core.User;
import cloudgene.mapred.util.BaseResource;
-import cloudgene.mapred.util.HadoopCluster;
import cloudgene.mapred.util.RBinary;
import cloudgene.mapred.util.Technology;
+import genepi.hadoop.HadoopCluster;
import genepi.hadoop.HadoopUtil;
import net.sf.json.JSONObject;
@@ -90,31 +89,30 @@ public Representation get() {
object.put("hadoop_safemode", false);
}
- ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails();
StringBuffer state = new StringBuffer();
state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n");
state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n");
- state.append("State: " + cluster.getJobTrackerStatus().toString() + "\n");
- state.append("MapTask: " + cluster.getMaxMapTasks() + "\n");
- state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n");
+ state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n");
+ state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n");
+ state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n");
state.append("Nodes\n");
- for (String tracker : cluster.getActiveTrackerNames()) {
+ for (String tracker : HadoopCluster.getActiveTrackerNames()) {
state.append(" " + tracker + "\n");
}
state.append("Blacklist:\n");
- for (String tracker : cluster.getBlacklistedTrackerNames()) {
+ for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) {
state.append(" " + tracker + "\n");
}
object.put("hadoop_details", state.toString());
object.put("hadoop_enabled", true);
object.put("hadoop_jobtracker", HadoopCluster.getJobTracker());
object.put("hadoop_hdfs", HadoopCluster.getDefaultFS());
- object.put("hadoop_map_tasks", cluster.getMaxMapTasks());
- object.put("hadoop_reduce_tasks", cluster.getMaxReduceTasks());
- object.put("hadoop_active_nodes", cluster.getActiveTrackerNames().size());
- object.put("hadoop_inactive_nodes", cluster.getBlacklistedTrackerNames().size());
+ object.put("hadoop_map_tasks", HadoopCluster.getMaxMapTasks());
+ object.put("hadoop_reduce_tasks", HadoopCluster.getMaxReduceTasks());
+ object.put("hadoop_active_nodes", HadoopCluster.getActiveTrackerNames().size());
+ object.put("hadoop_inactive_nodes", HadoopCluster.getBlacklistedTrackerNames().size());
object.put("hadoop_nodes",
- cluster.getActiveTrackerNames().size() + cluster.getBlacklistedTrackerNames().size());
+ HadoopCluster.getActiveTrackerNames().size() + HadoopCluster.getBlacklistedTrackerNames().size());
} catch (Exception e) {
object.put("hadoop_enabled", false);
object.put("hadoop_error", "Hadoop cluster is unreachable");
diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java
index 17566f75..3634104b 100644
--- a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java
+++ b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java
@@ -4,6 +4,7 @@
import java.util.Vector;
import org.restlet.data.MediaType;
+import org.restlet.data.Status;
import org.restlet.representation.Representation;
import org.restlet.representation.StringRepresentation;
import org.restlet.representation.Variant;
@@ -27,9 +28,13 @@ public class GetJobDetails extends BaseResource {
@Get
public Representation get(Representation entity, Variant variant) {
-
+
User user = getAuthUser();
+ if (getSettings().isMaintenance() && (user == null || !user.isAdmin())) {
+ return error(Status.SERVER_ERROR_SERVICE_UNAVAILABLE, "This functionality is currently under maintenance.");
+ }
+
String id = getAttribute("job");
if (id == null) {
@@ -126,11 +131,15 @@ public Representation deleteJob(Representation entity) {
String localOutput = FileUtil.path(getSettings().getLocalWorkspace(),
job.getId());
+ FileUtil.deleteDirectory(localOutput);
+
+ try {
String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(getSettings()
.getHdfsWorkspace(), job.getId()));
-
- FileUtil.deleteDirectory(localOutput);
HdfsUtil.delete(hdfsOutput);
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+ }
// delete job from database
job.setState(AbstractJob.STATE_DELETED);
diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java b/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java
index 268cc89e..e0a5eff8 100644
--- a/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java
+++ b/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java
@@ -47,7 +47,13 @@ public Representation get(Representation entity) {
if (job.getState() == AbstractJob.STATE_DEAD) {
- String hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id);
+ String hdfsWorkspace = "";
+ try {
+ HdfsUtil.path(getSettings().getHdfsWorkspace(), id);
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+
+ }
String localWorkspace = FileUtil.path(getSettings().getLocalWorkspace(), id);
job.setLocalWorkspace(localWorkspace);
diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java b/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java
index 6c626b8e..f82041b6 100644
--- a/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java
+++ b/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java
@@ -82,7 +82,14 @@ public Representation post(Representation entity) {
}
- String hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id);
+ String hdfsWorkspace = "";
+
+ try {
+ hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id);
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+ }
+
String localWorkspace = FileUtil.path(getSettings().getLocalWorkspace(), id);
FileUtil.createDirectory(localWorkspace);
@@ -252,6 +259,7 @@ private Map parseAndUpdateInputParams(Representation entity, Wdl
return null;
}
+ try {
for (WdlParameterInput input : app.getWorkflow().getInputs()) {
if (!params.containsKey(input.getId())) {
if (props.containsKey(input.getId())) {
@@ -269,6 +277,11 @@ private Map parseAndUpdateInputParams(Representation entity, Wdl
}
}
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+
+ }
return params;
}
diff --git a/src/main/java/cloudgene/mapred/api/v2/server/Apps.java b/src/main/java/cloudgene/mapred/api/v2/server/Apps.java
index a473dd30..3d621acc 100644
--- a/src/main/java/cloudgene/mapred/api/v2/server/Apps.java
+++ b/src/main/java/cloudgene/mapred/api/v2/server/Apps.java
@@ -159,12 +159,16 @@ private void updateState(Application app, JSONObject jsonObject) {
WdlApp wdlApp = app.getWdlApp();
if (wdlApp != null) {
if (wdlApp.needsInstallation()) {
+ try {
boolean installed = ApplicationInstaller.isInstalled(wdlApp, getSettings());
if (installed) {
jsonObject.put("state", "completed");
} else {
jsonObject.put("state", "on demand");
}
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+ }
} else {
jsonObject.put("state", "n/a");
}
diff --git a/src/main/java/cloudgene/mapred/cli/CloneApplications.java b/src/main/java/cloudgene/mapred/cli/CloneApplications.java
index c91f8937..ee09d691 100644
--- a/src/main/java/cloudgene/mapred/cli/CloneApplications.java
+++ b/src/main/java/cloudgene/mapred/cli/CloneApplications.java
@@ -8,8 +8,6 @@
import java.util.Vector;
import org.apache.commons.io.FileUtils;
-import org.restlet.data.Status;
-import org.restlet.representation.StringRepresentation;
import com.esotericsoftware.yamlbeans.YamlReader;
diff --git a/src/main/java/cloudgene/mapred/cli/RunApplication.java b/src/main/java/cloudgene/mapred/cli/RunApplication.java
index c9f00480..642ef481 100644
--- a/src/main/java/cloudgene/mapred/cli/RunApplication.java
+++ b/src/main/java/cloudgene/mapred/cli/RunApplication.java
@@ -12,13 +12,8 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
-import org.apache.hadoop.mapred.ClusterStatus;
import com.esotericsoftware.yamlbeans.YamlException;
-import com.spotify.docker.client.DefaultDockerClient;
-import com.spotify.docker.client.DockerClient;
-import com.spotify.docker.client.exceptions.DockerCertificateException;
-import com.spotify.docker.client.exceptions.DockerException;
import cloudgene.mapred.core.User;
import cloudgene.mapred.jobs.CloudgeneJob;
@@ -26,14 +21,12 @@
import cloudgene.mapred.jobs.Message;
import cloudgene.mapred.jobs.WorkflowEngine;
import cloudgene.mapred.util.Application;
-import cloudgene.mapred.util.HadoopCluster;
-import cloudgene.mapred.util.RBinary;
import cloudgene.mapred.util.Technology;
import cloudgene.mapred.wdl.WdlApp;
import cloudgene.mapred.wdl.WdlParameterInput;
import cloudgene.mapred.wdl.WdlParameterOutput;
import cloudgene.mapred.wdl.WdlReader;
-import genepi.hadoop.HadoopUtil;
+import genepi.hadoop.HadoopCluster;
import genepi.hadoop.HdfsUtil;
import genepi.io.FileUtil;
@@ -211,10 +204,9 @@ public int start() {
// print summary and warnigns
if (settings.isEnable(Technology.HADOOP_CLUSTER)) {
- ClusterStatus details = HadoopUtil.getInstance().getClusterDetails();
- int nodes = details.getActiveTrackerNames().size();
- printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + details.getMaxMapTasks()
- + " map tasks and " + details.getMaxReduceTasks() + " reduce tasks");
+ int nodes = HadoopCluster.getActiveTrackerNames().size();
+ printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + HadoopCluster.getMaxMapTasks()
+ + " map tasks and " + HadoopCluster.getMaxReduceTasks() + " reduce tasks");
} else {
printText(0, spaces("[WARN]", 8) + "Cluster seems unreachable. Hadoop support disabled.");
}
@@ -240,7 +232,12 @@ public int start() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd-HHmmss");
String id = "job-" + sdf.format(new Date());
- String hdfs = HdfsUtil.path("cloudgene-cli", id);
+ String hdfs = "";
+ try {
+ hdfs = HdfsUtil.path("cloudgene-cli", id);
+ } catch (NoClassDefFoundError e) {
+
+ }
String local = FileUtil.path(id);
if (line.hasOption("output")) {
diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java
index 02acbbe6..684d3681 100644
--- a/src/main/java/cloudgene/mapred/cli/StartServer.java
+++ b/src/main/java/cloudgene/mapred/cli/StartServer.java
@@ -1,11 +1,9 @@
package cloudgene.mapred.cli;
-import org.apache.hadoop.mapred.ClusterStatus;
-
import cloudgene.mapred.Main;
-import cloudgene.mapred.util.HadoopCluster;
import cloudgene.mapred.util.Technology;
import genepi.base.Tool;
+import genepi.hadoop.HadoopCluster;
import genepi.hadoop.HadoopUtil;
public class StartServer extends BaseTool {
@@ -61,10 +59,9 @@ public int run() {
// print summary and warnigns
if (settings.isEnable(Technology.HADOOP_CLUSTER)) {
- ClusterStatus details = HadoopUtil.getInstance().getClusterDetails();
- int nodes = details.getActiveTrackerNames().size();
- printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + details.getMaxMapTasks()
- + " map tasks and " + details.getMaxReduceTasks() + " reduce tasks");
+ int nodes = HadoopCluster.getActiveTrackerNames().size();
+ printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + HadoopCluster.getMaxMapTasks()
+ + " map tasks and " + HadoopCluster.getMaxReduceTasks() + " reduce tasks");
} else {
printText(0, spaces("[WARN]", 8) + "Cluster seems unreachable. Hadoop support disabled.");
}
diff --git a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java
index 089e6f61..84bf5f94 100644
--- a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java
+++ b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java
@@ -1,8 +1,6 @@
package cloudgene.mapred.cli;
-import org.apache.hadoop.mapred.ClusterStatus;
-
-import cloudgene.mapred.util.HadoopCluster;
+import genepi.hadoop.HadoopCluster;
import genepi.hadoop.HadoopUtil;
public class VerifyCluster extends BaseTool {
@@ -21,20 +19,19 @@ public int run() {
try {
HadoopCluster.verifyCluster();
- ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails();
StringBuffer state = new StringBuffer();
state.append("Mode: " + (HadoopUtil.getInstance().isInSafeMode() ? "Safe Mode" : "Running"));
state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n");
state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n");
- state.append("State: " + cluster.getJobTrackerStatus().toString() + "\n");
- state.append("MapTask: " + cluster.getMaxMapTasks() + "\n");
- state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n");
+ state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n");
+ state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n");
+ state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n");
state.append("Nodes\n");
- for (String tracker : cluster.getActiveTrackerNames()) {
+ for (String tracker : HadoopCluster.getActiveTrackerNames()) {
state.append(" " + tracker + "\n");
}
state.append("Blacklist:\n");
- for (String tracker : cluster.getBlacklistedTrackerNames()) {
+ for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) {
state.append(" " + tracker + "\n");
}
System.out.println(state.toString());
diff --git a/src/main/java/cloudgene/mapred/cron/AlertJob.java b/src/main/java/cloudgene/mapred/cron/AlertJob.java
index df75ca09..844d6c44 100644
--- a/src/main/java/cloudgene/mapred/cron/AlertJob.java
+++ b/src/main/java/cloudgene/mapred/cron/AlertJob.java
@@ -20,6 +20,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
// check namenode state
+ try {
boolean safemode = HadoopUtil.getInstance().isInSafeMode();
if (safemode) {
@@ -46,6 +47,9 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
}
}
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+ }
// if (cluster.getJobTrackerStatus() == )
diff --git a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java
index 0d2a9940..a0129bad 100644
--- a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java
+++ b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java
@@ -33,13 +33,17 @@ public static int executeRetire(Database database, Settings settings) {
// delete local directory and hdfs directory
String localOutput = FileUtil.path(
settings.getLocalWorkspace(), job.getId());
-
- String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(
- settings.getHdfsWorkspace(), job.getId()));
-
FileUtil.deleteDirectory(localOutput);
+
+ try {
+ String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(
+ settings.getHdfsWorkspace(), job.getId()));
HdfsUtil.delete(hdfsOutput);
-
+ }catch (NoClassDefFoundError e) {
+ // TODO: handle exception
+ }
+
+
job.setState(AbstractJob.STATE_RETIRED);
dao.update(job);
diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java
index 0bb53ae5..cf845b36 100644
--- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java
+++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java
@@ -1,9 +1,5 @@
package cloudgene.mapred.jobs;
-import genepi.hadoop.HdfsUtil;
-import genepi.hadoop.common.WorkflowContext;
-import genepi.io.FileUtil;
-
import java.io.File;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -16,8 +12,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.util.MailUtil;
import cloudgene.mapred.util.Settings;
-import cloudgene.mapred.wdl.WdlParameter;
-import cloudgene.mapred.wdl.WdlParameterOutputType;
+import genepi.hadoop.HdfsUtil;
+import genepi.hadoop.common.WorkflowContext;
+import genepi.io.FileUtil;
public class CloudgeneContext extends WorkflowContext {
@@ -58,7 +55,7 @@ public class CloudgeneContext extends WorkflowContext {
private Map config;
private int chunks = 0;
-
+
public CloudgeneContext(CloudgeneJob job) {
this.workingDirectory = job.getWorkingDirectory();
@@ -71,11 +68,13 @@ public CloudgeneContext(CloudgeneJob job) {
workspace = job.getHdfsWorkspace();
- hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp"));
-
- hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace));
-
- hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace));
+ try {
+ hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp"));
+ hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace));
+ hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace));
+ } catch (Error e) {
+ log("No hdfs folders created.");
+ }
localOutput = new File(job.getLocalWorkspace()).getAbsolutePath();
@@ -113,6 +112,8 @@ public void setupOutputParameters(boolean hasHdfsOutputs) {
HdfsUtil.delete(getHdfsTemp());
} catch (Exception e) {
System.out.println("Warning: problems during hdfs init.");
+ } catch (Error e) {
+ System.out.println("Warning: problems during hdfs init.");
}
}
@@ -456,7 +457,7 @@ public String getConfig(String param) {
return null;
}
}
-
+
public void addFile(String filename) {
chunks++;
String chunkFolder = FileUtil.path(getLocalOutput(), "chunks");
@@ -467,4 +468,5 @@ public void addFile(String filename) {
message(chunkFilename, 27);
}
+
}
diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java
index b1b55d20..6447149b 100644
--- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java
+++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java
@@ -40,6 +40,8 @@ public class CloudgeneJob extends AbstractJob {
private Executor executor;
+ public static final int MAX_DOWNLOAD = 10;
+
private static final Log log = LogFactory.getLog(CloudgeneJob.class);
public CloudgeneJob() {
@@ -370,14 +372,17 @@ public boolean cleanUp() {
// delete hdfs workspace
if (isRemoveHdfsWorkspace()) {
+ if (context.getHdfsOutput() != null){
writeLog("Cleaning up hdfs files...");
HdfsUtil.delete(context.getHdfsOutput());
HdfsUtil.delete(context.getHdfsInput());
+ }
}
} catch (Exception e) {
log.warn("Warning: problems during hdfs cleanup.");
+ } catch (Error e) {
+ log.warn("Warning: problems during hdfs cleanup.");
}
-
}
return true;
@@ -510,8 +515,7 @@ public boolean exportParameter(CloudgeneParameterOutput out) {
download.setSize(item.getSize());
download.setHash(hash);
download.setParameter(out);
- int maxDownloads = getSettings().getMaxDownloads();
- download.setCount(maxDownloads);
+ download.setCount(MAX_DOWNLOAD);
files.add(download);
}
Collections.sort(files);
diff --git a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java
index 47e12331..bdad7bcd 100644
--- a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java
+++ b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java
@@ -10,19 +10,14 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.RunningJob;
-
import cloudgene.mapred.jobs.CloudgeneContext;
import cloudgene.mapred.jobs.CloudgeneStep;
import cloudgene.mapred.jobs.Message;
import cloudgene.mapred.util.Technology;
import cloudgene.mapred.wdl.WdlStep;
import genepi.hadoop.HadoopUtil;
-import genepi.hadoop.HdfsUtil;
import genepi.hadoop.common.WorkflowContext;
import genepi.io.FileUtil;
-import genepi.io.text.LineWriter;
public class HadoopMapReduceStep extends CloudgeneStep {
@@ -84,36 +79,7 @@ public boolean run(WdlStep step, CloudgeneContext context) {
List command = new Vector();
command.add(hadoop);
- // TODO: write config and set to hadoop config
- try {
- FileUtil.createDirectory("temp-conf");
- LineWriter writer = new LineWriter("temp-conf/mapred-site.xml");
- Configuration configuration = HdfsUtil.getConfiguration();
- writer.write("");
- writer.write("");
- writer.write("mapred.job.tracker" + configuration.get("mapred.job.tracker")
- + "");
- writer.write("");
- writer.close();
-
- writer = new LineWriter("temp-conf/core-site.xml");
- writer.write("");
- writer.write("");
- writer.write("fs.default.name" + configuration.get("fs.defaultFS")
- + "");
- writer.write("");
- writer.close();
-
- // HdfsUtil.getConfiguration().writeXml(new
- // FileOutputStream("temp-conf/mapred-default.xml"));
-
- command.add("--config");
- command.add(new File("temp-conf").getAbsolutePath());
-
- } catch (IOException e1) {
- // TODO Auto-generated catch block
- e1.printStackTrace();
- }
+
// -fs and -jt and -config are supported by generic tool runner.
command.add("jar");
@@ -262,7 +228,7 @@ protected boolean executeCommand(List command, WorkflowContext context)
@Override
public void updateProgress() {
- RunningJob job = HadoopUtil.getInstance().getJob(jobId);
+ /*RunningJob job = HadoopUtil.getInstance().getJob(jobId);
if (job != null) {
try {
@@ -283,7 +249,7 @@ public void updateProgress() {
} else {
map = 0;
reduce = 0;
- }
+ }*/
}
diff --git a/src/main/java/cloudgene/mapred/util/ClusterInformation.java b/src/main/java/cloudgene/mapred/util/ClusterInformation.java
deleted file mode 100644
index 6aa11381..00000000
--- a/src/main/java/cloudgene/mapred/util/ClusterInformation.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package cloudgene.mapred.util;
-
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.util.VersionInfo;
-
-public class ClusterInformation {
-
- private ClusterStatus status;
-
- private String version;
-
- private String date;
-
- public ClusterInformation(ClusterStatus status) {
- this.status = status;
- }
-
- public ClusterStatus getStatus() {
- return status;
- }
-
- public void setStatus(ClusterStatus status) {
- this.status = status;
- }
-
- public String getVersion() {
- return VersionInfo.getVersion();
- }
-
- public String getDate() {
- return VersionInfo.getDate();
- }
-
- public void setDate(String date) {
- this.date = date;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-}
diff --git a/src/main/java/cloudgene/mapred/util/HadoopCluster.java b/src/main/java/cloudgene/mapred/util/HadoopCluster.java
deleted file mode 100644
index af3295ce..00000000
--- a/src/main/java/cloudgene/mapred/util/HadoopCluster.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package cloudgene.mapred.util;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.ClusterStatus;
-
-import genepi.hadoop.HadoopUtil;
-import genepi.hadoop.HdfsUtil;
-import genepi.io.FileUtil;
-
-public class HadoopCluster {
-
- private static String username;
-
- private static String conf;
-
- private static String name;
-
- public static void setConfPath(String name, String path, String username) {
- if (username != null) {
- System.setProperty("HADOOP_USER_NAME", username);
- HadoopCluster.username = username;
- }
- HadoopCluster.conf = path;
- HadoopCluster.name = name;
- if (new File(path).exists()) {
- Configuration configuration = new Configuration();
- // add all xml files from hadoop conf folder to default
- // configuration
- String[] xmlFiles = FileUtil.getFiles(path, "*.xml");
- for (String xmlFile : xmlFiles) {
- configuration.addResource(new Path(xmlFile));
- }
-
- HdfsUtil.setDefaultConfiguration(configuration);
- }
-
- }
-
- public static String getUsername() {
- return username;
- }
-
- public static String getConf() {
- return conf;
- }
-
- public static String getName() {
- return name;
- }
-
- public static String getJobTracker() {
- Configuration configuration = HdfsUtil.getConfiguration();
- return configuration.get("mapred.job.tracker");
- }
-
- public static String getDefaultFS() {
- Configuration configuration = HdfsUtil.getConfiguration();
- return configuration.get("fs.defaultFS");
- }
-
- public static boolean verifyCluster() throws Exception {
-
- String confFolder = HadoopCluster.getConf();
-
- if (confFolder == null) {
- throw new Exception("Please define a cluster in file settings.yaml.");
- }
-
- if (new File(confFolder).exists()) {
- String configName = "mapred-site.xml";
- String configFile = FileUtil.path(confFolder, configName);
- if (!new File(configFile).exists()) {
- throw new Exception("No '" + configName + "' file found in configuration folder '" + confFolder + "'.");
- }
- } else {
- throw new Exception("Configuration folder '" + confFolder + "' not found.");
- }
-
- ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails();
- if (cluster.getActiveTrackerNames().isEmpty()) {
- throw new Exception("No active nodes founds.");
- }
- Configuration configuration = HdfsUtil.getConfiguration();
- FileSystem fileSystem = FileSystem.get(configuration);
-
- try {
- Path path = fileSystem.getHomeDirectory();
-
- if (fileSystem.exists(path)) {
- return true;
- } else {
- throw new Exception(
- "Home directory '" + path + "' for user " + HadoopCluster.getUsername() + " not found.");
- }
-
- } catch (Exception e) {
- throw new Exception(
- "Problems find the home directory for user " + HadoopCluster.getUsername() + ". " + e.getMessage());
- }
- }
-
-}
diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java
index 37b3e2b8..e7f2fa2a 100644
--- a/src/main/java/cloudgene/mapred/util/Settings.java
+++ b/src/main/java/cloudgene/mapred/util/Settings.java
@@ -31,6 +31,7 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.util.GitHubUtil.Repository;
import cloudgene.mapred.wdl.WdlApp;
+import genepi.hadoop.HadoopCluster;
import genepi.io.FileUtil;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
@@ -38,7 +39,7 @@
public class Settings {
- private String hadoopPath = "/usr/bin/hadoop";
+ private String hadoopPath = "/usr/";
private String pigPath = "/usr/";
@@ -208,7 +209,10 @@ public static Settings load(Config config) throws FileNotFoundException, YamlExc
if (conf != null) {
log.info("Use Haddop configuration folder '" + conf + "'"
+ (username != null ? " with username " + username : ""));
- HadoopCluster.setConfPath(name, conf, username);
+ try {
+ HadoopCluster.setConfPath(name, conf, username);
+ }catch (NoClassDefFoundError e) {
+ }
}
}
diff --git a/src/main/java/cloudgene/sample/SampleStep.java b/src/main/java/cloudgene/sample/SampleStep.java
deleted file mode 100644
index f985084c..00000000
--- a/src/main/java/cloudgene/sample/SampleStep.java
+++ /dev/null
@@ -1,151 +0,0 @@
-package cloudgene.sample;
-
-import genepi.hadoop.HadoopUtil;
-import genepi.hadoop.HdfsUtil;
-import genepi.hadoop.common.WorkflowContext;
-import genepi.hadoop.common.WorkflowStep;
-import genepi.io.FileUtil;
-
-import java.io.File;
-
-import org.apache.hadoop.mapred.ClusterStatus;
-
-import cloudgene.mapred.jobs.CloudgeneContext;
-import cloudgene.mapred.util.Settings;
-
-public class SampleStep extends WorkflowStep {
-
- @Override
- public boolean run(WorkflowContext context) {
-
- Settings settings = ((CloudgeneContext) context).getSettings();
-
- context.ok("Cloudgene runs with the following Hadoop configuration:");
-
- ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails();
- StringBuffer state = new StringBuffer();
- state.append("Hadoop Cluster\n");
- state.append("State: " + cluster.getJobTrackerStatus().toString()
- + "\n");
- state.append("MapTask: " + cluster.getMaxMapTasks() + "\n");
- state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n");
- state.append("Nodes\n");
- for (String tracker : cluster.getActiveTrackerNames()) {
- state.append(tracker + "\n");
- }
-
- String filename = context.getInput("testfile");
- context.ok(FileUtil.readFileAsString(filename));
-
- context.ok(state.toString());
- context.beginTask("waiting....");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e1) {
- // TODO Auto-generated catch block
- e1.printStackTrace();
- }
-
- String hadoopPath = settings.getHadoopPath();
-
- if (hadoopPath.trim().isEmpty()) {
- context.error("Hadoop Binary was not set. Please set the correct path in the admin panel.");
- return false;
- }
-
- File path = new File(hadoopPath);
-
- if (!path.exists()) {
- context.error("Hadoop Binary "
- + path
- + "
was not found. Please set the correct path in the admin panel.");
- return false;
- }
-
- String hadoop = "";
-
- if (path.isDirectory()) {
- hadoop = FileUtil.path(hadoopPath, "bin", "hadoop");
- } else {
- hadoop = hadoopPath;
- }
-
- File file = new File(hadoop);
-
- if (!file.exists()) {
- context.error("Hadoop Binary "
- + hadoop
- + "
was not found. Please set the correct path in the admin panel.");
- return false;
- }
-
- if (!file.canExecute()) {
- context.error("Hadoop Binary "
- + hadoop
- + "
was found, but it can not be executed. Please check the permissions.");
- return false;
- }
-
- context.ok("Hadoop Binary was found in " + hadoop
- + "
and is executable.");
-
- // TODO: write r script which checks packages
-
- context.ok("R was found and all packages are installed.");
-
- // TODO: write file to hdfs temp directory
-
- HdfsUtil.delete("cloudgene-test/test-file.txt");
- FileUtil.writeStringBufferToFile("lf.txt", new StringBuffer("lf-yeah!"));
- HdfsUtil.put("lf.txt", "cloudgene-test/test-file.txt");
-
- context.ok("HDFS File System check (check if file cloudgene-test/test-file.txt exists in hdfs!)");
-
- // TODO: write file to local temp directory
-
- context.ok("Local File System check");
-
- String outputFolder = context.getOutput("stats");
-
- FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "1.txt"),
- new StringBuffer("file 1"));
- FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "2.txt"),
- new StringBuffer("file 2"));
- FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "3.txt"),
- new StringBuffer("file 3"));
- FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "4.txt"),
- new StringBuffer("file 4"));
- FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "5.txt"),
- new StringBuffer("file 5"));
-
- /*
- * // Mail Server.... String mail = context.get("cloudgene.user.mail");
- * if (mail != null) {
- *
- * String subject = "Mail Server Test"; String message =
- * "This email was sent by Cloudgene to test your mail-server settings."
- * ;
- *
- * try {
- *
- * context.sendMail(subject, message);
- *
- * context.ok("We have sent a test-email to " + mail + ".");
- *
- * } catch (Exception e) { context.error("Sending mail failed: " +
- * e.getMessage()); return false; }
- *
- * } else {
- *
- * context.error(
- * "No email address found. Please enter your email address (Account -> Profile)."
- * ); return false;
- *
- * }
- */
- context.ok("Congratulations. Cloudgene works properly on your Hadoop Cluster!");
-
- return true;
-
- }
-}
diff --git a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java
index 2fb78ccf..df45d2d2 100644
--- a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java
@@ -10,10 +10,10 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.LoginToken;
-import cloudgene.mapred.util.junit.TestMailServer;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.LoginToken;
+import cloudgene.mapred.util.TestMailServer;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class ChangeGroupTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java
index a2eec21b..7c9b2163 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java
@@ -5,9 +5,9 @@
import org.restlet.resource.ClientResource;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
public class CancelJobTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java
index 35334ae9..278005ca 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java
@@ -12,8 +12,8 @@
import org.restlet.resource.ClientResource;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestServer;
public class DeleteJobTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java
index 0cbb2ebd..bc4a3356 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java
@@ -8,9 +8,9 @@
import org.restlet.resource.ClientResource;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
public class DownloadResultsTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java
index f66a17f3..0cc424bf 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java
@@ -10,10 +10,10 @@
import org.restlet.resource.ClientResource;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.LoginToken;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.LoginToken;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
public class GetJobsTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java
index 80e532c4..48ac43a1 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java
@@ -10,9 +10,9 @@
import org.restlet.representation.FileRepresentation;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
import genepi.io.FileUtil;
public class GetLogsTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java
index 83c29c4a..4c43a29b 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java
@@ -4,9 +4,9 @@
import org.restlet.ext.html.FormDataSet;
import cloudgene.mapred.jobs.AbstractJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
public class RestartJobTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java
index 67976fe4..35a36587 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java
@@ -9,9 +9,9 @@
import cloudgene.mapred.jobs.AbstractJob;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
+import cloudgene.mapred.util.TestServer;
public class ShareResultsTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java
index 7b55b2c6..8b61d457 100644
--- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java
@@ -13,10 +13,10 @@
import org.restlet.resource.ClientResource;
import cloudgene.mapred.jobs.AbstractJob;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestCluster;
import cloudgene.mapred.util.TestSFTPServer;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestCluster;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.TestServer;
import genepi.hadoop.common.WorkflowContext;
import genepi.io.FileUtil;
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java
index b417cc3a..10fd21f1 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java
@@ -11,9 +11,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestMailServer;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestMailServer;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class ActivateUserTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java
index 455f0ab9..8f295241 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java
@@ -16,9 +16,9 @@
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.jobs.AbstractJob;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.LoginToken;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.LoginToken;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
import genepi.io.FileUtil;
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java
index 29d59187..8e8fe44a 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java
@@ -11,8 +11,8 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class LoginUserTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java
index 565fe631..98ed8824 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java
@@ -8,9 +8,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.LoginToken;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.LoginToken;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class LogoutUserTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java
index 1cc3c1f5..6a4a67c9 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java
@@ -7,9 +7,9 @@
import org.restlet.data.Form;
import org.restlet.resource.ClientResource;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestMailServer;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestMailServer;
+import cloudgene.mapred.util.TestServer;
public class RegisterUserTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java
index 6f14fabe..b15d990b 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java
@@ -12,9 +12,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestMailServer;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestMailServer;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class ResetPasswordTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java
index aee8583f..c3f21352 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java
@@ -10,9 +10,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestMailServer;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestMailServer;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class UpdatePasswordTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java
index 2005f8b9..5b528407 100644
--- a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java
+++ b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java
@@ -10,9 +10,9 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.UserDao;
import cloudgene.mapred.util.HashUtil;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.LoginToken;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.LoginToken;
+import cloudgene.mapred.util.TestServer;
import genepi.db.Database;
public class UserProfileTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/database/JobDaoTest.java b/src/test/java/cloudgene/mapred/database/JobDaoTest.java
index e85ac479..e615a12a 100644
--- a/src/test/java/cloudgene/mapred/database/JobDaoTest.java
+++ b/src/test/java/cloudgene/mapred/database/JobDaoTest.java
@@ -9,8 +9,8 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.jobs.AbstractJob;
import cloudgene.mapred.jobs.CloudgeneJob;
-import cloudgene.mapred.util.junit.JobsApiTestCase;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.JobsApiTestCase;
+import cloudgene.mapred.util.TestServer;
public class JobDaoTest extends JobsApiTestCase {
diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java
index 9b4150c2..7abefac0 100644
--- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java
+++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java
@@ -6,7 +6,7 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.util.Settings;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.TestServer;
import cloudgene.mapred.wdl.WdlApp;
import cloudgene.mapred.wdl.WdlReader;
import genepi.hadoop.HdfsUtil;
diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java
index 1a347a82..92399f17 100644
--- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java
+++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java
@@ -11,7 +11,7 @@
import junit.framework.TestCase;
import cloudgene.mapred.core.User;
import cloudgene.mapred.util.Settings;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.TestServer;
import cloudgene.mapred.wdl.WdlApp;
import cloudgene.mapred.wdl.WdlReader;
diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java
index 20121e17..979bd891 100644
--- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java
+++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java
@@ -10,7 +10,7 @@
import cloudgene.mapred.core.User;
import cloudgene.mapred.database.JobDao;
import cloudgene.mapred.util.Settings;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.TestServer;
import cloudgene.mapred.wdl.WdlApp;
import cloudgene.mapred.wdl.WdlReader;
diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java
index 989e5e77..67c93789 100644
--- a/src/test/java/cloudgene/mapred/steps/TestCommand.java
+++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java
@@ -16,7 +16,7 @@
import cloudgene.mapred.jobs.Message;
import cloudgene.mapred.jobs.WorkflowEngine;
import cloudgene.mapred.util.Settings;
-import cloudgene.mapred.util.junit.TestServer;
+import cloudgene.mapred.util.TestServer;
import cloudgene.mapred.wdl.WdlApp;
import cloudgene.mapred.wdl.WdlReader;
diff --git a/src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java b/src/test/java/cloudgene/mapred/util/JobsApiTestCase.java
similarity index 99%
rename from src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java
rename to src/test/java/cloudgene/mapred/util/JobsApiTestCase.java
index 034a2929..e5a1a6b9 100644
--- a/src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java
+++ b/src/test/java/cloudgene/mapred/util/JobsApiTestCase.java
@@ -1,8 +1,7 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import java.io.IOException;
-import org.apache.avro.data.Json;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -16,7 +15,7 @@
import junit.framework.TestCase;
-public class JobsApiTestCase extends TestCase {
+public abstract class JobsApiTestCase extends TestCase {
public ClientResource createClientResource(String path) {
diff --git a/src/main/java/cloudgene/mapred/util/junit/LoginToken.java b/src/test/java/cloudgene/mapred/util/LoginToken.java
similarity index 91%
rename from src/main/java/cloudgene/mapred/util/junit/LoginToken.java
rename to src/test/java/cloudgene/mapred/util/LoginToken.java
index a46b0056..786ec70a 100644
--- a/src/main/java/cloudgene/mapred/util/junit/LoginToken.java
+++ b/src/test/java/cloudgene/mapred/util/LoginToken.java
@@ -1,4 +1,4 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import org.restlet.data.CookieSetting;
diff --git a/src/main/java/cloudgene/mapred/util/junit/TestCluster.java b/src/test/java/cloudgene/mapred/util/TestCluster.java
similarity index 98%
rename from src/main/java/cloudgene/mapred/util/junit/TestCluster.java
rename to src/test/java/cloudgene/mapred/util/TestCluster.java
index f180618a..070eda3d 100644
--- a/src/main/java/cloudgene/mapred/util/junit/TestCluster.java
+++ b/src/test/java/cloudgene/mapred/util/TestCluster.java
@@ -1,4 +1,4 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import genepi.hadoop.HdfsUtil;
diff --git a/src/main/java/cloudgene/mapred/util/junit/TestMailServer.java b/src/test/java/cloudgene/mapred/util/TestMailServer.java
similarity index 96%
rename from src/main/java/cloudgene/mapred/util/junit/TestMailServer.java
rename to src/test/java/cloudgene/mapred/util/TestMailServer.java
index a15e1cd8..f9c7c8b5 100644
--- a/src/main/java/cloudgene/mapred/util/junit/TestMailServer.java
+++ b/src/test/java/cloudgene/mapred/util/TestMailServer.java
@@ -1,4 +1,4 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import java.util.Iterator;
import java.util.List;
diff --git a/src/main/java/cloudgene/mapred/util/junit/TestServer.java b/src/test/java/cloudgene/mapred/util/TestServer.java
similarity index 99%
rename from src/main/java/cloudgene/mapred/util/junit/TestServer.java
rename to src/test/java/cloudgene/mapred/util/TestServer.java
index 3a791b00..39ce2440 100644
--- a/src/main/java/cloudgene/mapred/util/junit/TestServer.java
+++ b/src/test/java/cloudgene/mapred/util/TestServer.java
@@ -1,4 +1,4 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import java.io.File;
import java.io.InputStream;
diff --git a/src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java b/src/test/java/cloudgene/mapred/util/WdlAppTestCase.java
similarity index 97%
rename from src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java
rename to src/test/java/cloudgene/mapred/util/WdlAppTestCase.java
index af5fe85e..6c456599 100644
--- a/src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java
+++ b/src/test/java/cloudgene/mapred/util/WdlAppTestCase.java
@@ -1,4 +1,4 @@
-package cloudgene.mapred.util.junit;
+package cloudgene.mapred.util;
import genepi.hadoop.HdfsUtil;
import genepi.io.FileUtil;