diff --git a/files/cloudgene b/files/cloudgene index a6667a58..902ecd68 100644 --- a/files/cloudgene +++ b/files/cloudgene @@ -5,8 +5,11 @@ # The path to the folder containing MyDaemon.jar FILE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +# Hadoop path with all jars +HADOOP_CLASS_PATH=`hadoop classpath` + # Our classpath including our jar file -CLASS_PATH="$FILE_PATH/cloudgene.jar" +CLASS_PATH="$FILE_PATH/cloudgene.jar:$HADOOP_CLASS_PATH" # The fully qualified name of the class to execute CLASS="cloudgene.mapred.CommandLineInterface" diff --git a/pom.xml b/pom.xml index 15e2f64c..5a014d6a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cloudgene cloudgene - 2.0.0-rc1 + 2.0.0-rc2 maven-restlet @@ -215,7 +215,7 @@ genepi genepi-hadoop - mr1-1.2.2 + mr1-1.2.4 com.sun.jersey @@ -225,6 +225,10 @@ com.sun.jersey jersey-server + + org.apache.hadoop + hadoop-client + @@ -298,7 +302,7 @@ org.apache.hadoop hadoop-hdfs 2.6.0-cdh5.10.0 - compile + test test-jar @@ -311,12 +315,68 @@ - + + + org.apache.hadoop + hadoop-hdfs + 2.6.0-cdh5.10.0 + test + + + org.apache.httpcomponents + httpcore + + + net.java.dev.jets3t + jets3t + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + org.apache.hadoop hadoop-common 2.6.0-cdh5.10.0 - compile + test + + + org.apache.httpcomponents + httpcore + + + net.java.dev.jets3t + jets3t + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + + + org.apache.hadoop + hadoop-common + 2.6.0-cdh5.10.0 + test test-jar @@ -352,6 +412,7 @@ jets3t + test @@ -405,6 +466,12 @@ 1.2.14 + + commons-io + commons-io + 2.4 + + com.esotericsoftware.yamlbeans yamlbeans @@ -471,6 +538,7 @@ dumbster dumbster 1.6 + test diff --git a/src/main/html/webapp/components/core/dashboard/dashboard.js b/src/main/html/webapp/components/core/dashboard/dashboard.js index 3b2ad6b0..e70a3d58 100644 --- a/src/main/html/webapp/components/core/dashboard/dashboard.js +++ b/src/main/html/webapp/components/core/dashboard/dashboard.js @@ -8,24 +8,23 @@ export default Control.extend({ "init": function(element, options) { var url = 'static/home.stache'; - fetch(url).then(function(response) { - return response.text(); - }).then(function(data) { + $.get(url, + function(data) { - var template = stache(data); + var template = stache(data); - Counter.findOne({}, function(counter) { - $(element).html(template({ - counter: counter, - loggedIn: options.appState.loggedIn - })); - }, function(message) { - $(element).html(template({ - counter: undefined, - loggedIn: options.loggedIn - })); - }); + Counter.findOne({}, function(counter) { + $(element).html(template({ + counter: counter, + loggedIn: options.appState.loggedIn + })); + }, function(message) { + $(element).html(template({ + counter: undefined, + loggedIn: options.loggedIn + })); + }); - }); + }); } }); diff --git a/src/main/html/webapp/helpers/router.js b/src/main/html/webapp/helpers/router.js index 6dac5cbf..c48dc10c 100644 --- a/src/main/html/webapp/helpers/router.js +++ b/src/main/html/webapp/helpers/router.js @@ -90,7 +90,7 @@ export default Control.extend({ } else { view.addClass(this.options.classes); } - view.html('Loading...'); + view.html(''); this.element.append(view); new Control(view[0], data); diff --git a/src/main/html/webapp/helpers/static-page.js b/src/main/html/webapp/helpers/static-page.js index dd176fd6..806fc255 100644 --- a/src/main/html/webapp/helpers/static-page.js +++ b/src/main/html/webapp/helpers/static-page.js @@ -9,23 +9,22 @@ export default Control.extend({ "init": function(element, options) { try { - if (options.page){ + if (options.page) { options.template = "static/" + options.page + ".stache"; } - fetch(options.template).then(function(response) { - return response.text(); - }).then(function(data) { - var view = stache(data); - if (view) { - $(element).html(view()); - } else { - new ErrorPage(element, { - status: "404", - message: "Oops, Sorry We Can't Find That Page!" - }); - } - }); + $.get(options.template, + function(data) { + var view = stache(data); + if (view) { + $(element).html(view()); + } else { + new ErrorPage(element, { + status: "404", + message: "Oops, Sorry We Can't Find That Page!" + }); + } + }); } catch (e) { new ErrorPage(element, { diff --git a/src/main/html/webapp/package-lock.json b/src/main/html/webapp/package-lock.json index b02d2464..93488b73 100644 --- a/src/main/html/webapp/package-lock.json +++ b/src/main/html/webapp/package-lock.json @@ -1,6 +1,6 @@ { "name": "Cloudgene", - "version": "2.0.0-rc1", + "version": "2.0.0-rc2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/main/html/webapp/package.json b/src/main/html/webapp/package.json index 5a6a44ad..c5c974b1 100644 --- a/src/main/html/webapp/package.json +++ b/src/main/html/webapp/package.json @@ -1,6 +1,6 @@ { "name": "Cloudgene", - "version": "2.0.0-rc1", + "version": "2.0.0-rc2", "dependencies": { "@fortawesome/fontawesome-free": "5.3.1", "bootbox": "4.4.0", diff --git a/src/main/java/cloudgene/mapred/Main.java b/src/main/java/cloudgene/mapred/Main.java index 0826019b..ead2e8e3 100644 --- a/src/main/java/cloudgene/mapred/Main.java +++ b/src/main/java/cloudgene/mapred/Main.java @@ -37,7 +37,7 @@ public class Main implements Daemon { - public static final String VERSION = "2.0.0-rc1"; + public static final String VERSION = "2.0.0-rc2"; private Database database; @@ -71,17 +71,17 @@ public void runCloudgene(Settings settings, String[] args) throws Exception { settings.checkTechnologies(); // configure logger - if (new File("config/log4j.properties").exists()) { + if (new File("log4j.properties").exists()) { - PropertyConfigurator.configure("config/log4j.properties"); + PropertyConfigurator.configure("log4j.properties"); Slf4jLoggerFacade loggerFacade = new Slf4jLoggerFacade(); Engine.getInstance().setLoggerFacade(loggerFacade); } else { - if (new File("log4j.properties").exists()) { - PropertyConfigurator.configure("log4j.properties"); + if (new File("config/log4j.properties").exists()) { + PropertyConfigurator.configure("config/log4j.properties"); Slf4jLoggerFacade loggerFacade = new Slf4jLoggerFacade(); Engine.getInstance().setLoggerFacade(loggerFacade); diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java index 59ffd975..2479224e 100644 --- a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java +++ b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java @@ -57,12 +57,14 @@ public Representation get() { // delete local directory and hdfs directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); + FileUtil.deleteDirectory(localOutput); + try { String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - - FileUtil.deleteDirectory(localOutput); HdfsUtil.delete(hdfsOutput); - + }catch (NoClassDefFoundError e) { + + } job.setState(AbstractJob.STATE_RETIRED); dao.update(job); diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java b/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java index c412f3e3..c8050fcc 100644 --- a/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java +++ b/src/main/java/cloudgene/mapred/api/v2/admin/server/GetClusterDetails.java @@ -8,7 +8,6 @@ import java.util.jar.Attributes; import java.util.jar.Manifest; -import org.apache.hadoop.mapred.ClusterStatus; import org.restlet.data.MediaType; import org.restlet.data.Status; import org.restlet.representation.Representation; @@ -23,9 +22,9 @@ import cloudgene.mapred.Main; import cloudgene.mapred.core.User; import cloudgene.mapred.util.BaseResource; -import cloudgene.mapred.util.HadoopCluster; import cloudgene.mapred.util.RBinary; import cloudgene.mapred.util.Technology; +import genepi.hadoop.HadoopCluster; import genepi.hadoop.HadoopUtil; import net.sf.json.JSONObject; @@ -90,31 +89,30 @@ public Representation get() { object.put("hadoop_safemode", false); } - ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails(); StringBuffer state = new StringBuffer(); state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + cluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + cluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n"); + state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); + state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); + state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); state.append("Nodes\n"); - for (String tracker : cluster.getActiveTrackerNames()) { + for (String tracker : HadoopCluster.getActiveTrackerNames()) { state.append(" " + tracker + "\n"); } state.append("Blacklist:\n"); - for (String tracker : cluster.getBlacklistedTrackerNames()) { + for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { state.append(" " + tracker + "\n"); } object.put("hadoop_details", state.toString()); object.put("hadoop_enabled", true); object.put("hadoop_jobtracker", HadoopCluster.getJobTracker()); object.put("hadoop_hdfs", HadoopCluster.getDefaultFS()); - object.put("hadoop_map_tasks", cluster.getMaxMapTasks()); - object.put("hadoop_reduce_tasks", cluster.getMaxReduceTasks()); - object.put("hadoop_active_nodes", cluster.getActiveTrackerNames().size()); - object.put("hadoop_inactive_nodes", cluster.getBlacklistedTrackerNames().size()); + object.put("hadoop_map_tasks", HadoopCluster.getMaxMapTasks()); + object.put("hadoop_reduce_tasks", HadoopCluster.getMaxReduceTasks()); + object.put("hadoop_active_nodes", HadoopCluster.getActiveTrackerNames().size()); + object.put("hadoop_inactive_nodes", HadoopCluster.getBlacklistedTrackerNames().size()); object.put("hadoop_nodes", - cluster.getActiveTrackerNames().size() + cluster.getBlacklistedTrackerNames().size()); + HadoopCluster.getActiveTrackerNames().size() + HadoopCluster.getBlacklistedTrackerNames().size()); } catch (Exception e) { object.put("hadoop_enabled", false); object.put("hadoop_error", "Hadoop cluster is unreachable"); diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java index 17566f75..3634104b 100644 --- a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java +++ b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java @@ -4,6 +4,7 @@ import java.util.Vector; import org.restlet.data.MediaType; +import org.restlet.data.Status; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; import org.restlet.representation.Variant; @@ -27,9 +28,13 @@ public class GetJobDetails extends BaseResource { @Get public Representation get(Representation entity, Variant variant) { - + User user = getAuthUser(); + if (getSettings().isMaintenance() && (user == null || !user.isAdmin())) { + return error(Status.SERVER_ERROR_SERVICE_UNAVAILABLE, "This functionality is currently under maintenance."); + } + String id = getAttribute("job"); if (id == null) { @@ -126,11 +131,15 @@ public Representation deleteJob(Representation entity) { String localOutput = FileUtil.path(getSettings().getLocalWorkspace(), job.getId()); + FileUtil.deleteDirectory(localOutput); + + try { String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(getSettings() .getHdfsWorkspace(), job.getId())); - - FileUtil.deleteDirectory(localOutput); HdfsUtil.delete(hdfsOutput); + }catch (NoClassDefFoundError e) { + // TODO: handle exception + } // delete job from database job.setState(AbstractJob.STATE_DELETED); diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java b/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java index 268cc89e..e0a5eff8 100644 --- a/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java +++ b/src/main/java/cloudgene/mapred/api/v2/jobs/RestartJob.java @@ -47,7 +47,13 @@ public Representation get(Representation entity) { if (job.getState() == AbstractJob.STATE_DEAD) { - String hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id); + String hdfsWorkspace = ""; + try { + HdfsUtil.path(getSettings().getHdfsWorkspace(), id); + }catch (NoClassDefFoundError e) { + // TODO: handle exception + + } String localWorkspace = FileUtil.path(getSettings().getLocalWorkspace(), id); job.setLocalWorkspace(localWorkspace); diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java b/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java index 6c626b8e..f82041b6 100644 --- a/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java +++ b/src/main/java/cloudgene/mapred/api/v2/jobs/SubmitJob.java @@ -82,7 +82,14 @@ public Representation post(Representation entity) { } - String hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id); + String hdfsWorkspace = ""; + + try { + hdfsWorkspace = HdfsUtil.path(getSettings().getHdfsWorkspace(), id); + }catch (NoClassDefFoundError e) { + // TODO: handle exception + } + String localWorkspace = FileUtil.path(getSettings().getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); @@ -252,6 +259,7 @@ private Map parseAndUpdateInputParams(Representation entity, Wdl return null; } + try { for (WdlParameterInput input : app.getWorkflow().getInputs()) { if (!params.containsKey(input.getId())) { if (props.containsKey(input.getId())) { @@ -269,6 +277,11 @@ private Map parseAndUpdateInputParams(Representation entity, Wdl } } + } catch (Exception e) { + e.printStackTrace(); + throw e; + + } return params; } diff --git a/src/main/java/cloudgene/mapred/api/v2/server/Apps.java b/src/main/java/cloudgene/mapred/api/v2/server/Apps.java index a473dd30..3d621acc 100644 --- a/src/main/java/cloudgene/mapred/api/v2/server/Apps.java +++ b/src/main/java/cloudgene/mapred/api/v2/server/Apps.java @@ -159,12 +159,16 @@ private void updateState(Application app, JSONObject jsonObject) { WdlApp wdlApp = app.getWdlApp(); if (wdlApp != null) { if (wdlApp.needsInstallation()) { + try { boolean installed = ApplicationInstaller.isInstalled(wdlApp, getSettings()); if (installed) { jsonObject.put("state", "completed"); } else { jsonObject.put("state", "on demand"); } + }catch (NoClassDefFoundError e) { + // TODO: handle exception + } } else { jsonObject.put("state", "n/a"); } diff --git a/src/main/java/cloudgene/mapred/cli/CloneApplications.java b/src/main/java/cloudgene/mapred/cli/CloneApplications.java index c91f8937..ee09d691 100644 --- a/src/main/java/cloudgene/mapred/cli/CloneApplications.java +++ b/src/main/java/cloudgene/mapred/cli/CloneApplications.java @@ -8,8 +8,6 @@ import java.util.Vector; import org.apache.commons.io.FileUtils; -import org.restlet.data.Status; -import org.restlet.representation.StringRepresentation; import com.esotericsoftware.yamlbeans.YamlReader; diff --git a/src/main/java/cloudgene/mapred/cli/RunApplication.java b/src/main/java/cloudgene/mapred/cli/RunApplication.java index c9f00480..642ef481 100644 --- a/src/main/java/cloudgene/mapred/cli/RunApplication.java +++ b/src/main/java/cloudgene/mapred/cli/RunApplication.java @@ -12,13 +12,8 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; -import org.apache.hadoop.mapred.ClusterStatus; import com.esotericsoftware.yamlbeans.YamlException; -import com.spotify.docker.client.DefaultDockerClient; -import com.spotify.docker.client.DockerClient; -import com.spotify.docker.client.exceptions.DockerCertificateException; -import com.spotify.docker.client.exceptions.DockerException; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.CloudgeneJob; @@ -26,14 +21,12 @@ import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.util.Application; -import cloudgene.mapred.util.HadoopCluster; -import cloudgene.mapred.util.RBinary; import cloudgene.mapred.util.Technology; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HadoopUtil; +import genepi.hadoop.HadoopCluster; import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; @@ -211,10 +204,9 @@ public int start() { // print summary and warnigns if (settings.isEnable(Technology.HADOOP_CLUSTER)) { - ClusterStatus details = HadoopUtil.getInstance().getClusterDetails(); - int nodes = details.getActiveTrackerNames().size(); - printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + details.getMaxMapTasks() - + " map tasks and " + details.getMaxReduceTasks() + " reduce tasks"); + int nodes = HadoopCluster.getActiveTrackerNames().size(); + printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + HadoopCluster.getMaxMapTasks() + + " map tasks and " + HadoopCluster.getMaxReduceTasks() + " reduce tasks"); } else { printText(0, spaces("[WARN]", 8) + "Cluster seems unreachable. Hadoop support disabled."); } @@ -240,7 +232,12 @@ public int start() { SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd-HHmmss"); String id = "job-" + sdf.format(new Date()); - String hdfs = HdfsUtil.path("cloudgene-cli", id); + String hdfs = ""; + try { + hdfs = HdfsUtil.path("cloudgene-cli", id); + } catch (NoClassDefFoundError e) { + + } String local = FileUtil.path(id); if (line.hasOption("output")) { diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java index 02acbbe6..684d3681 100644 --- a/src/main/java/cloudgene/mapred/cli/StartServer.java +++ b/src/main/java/cloudgene/mapred/cli/StartServer.java @@ -1,11 +1,9 @@ package cloudgene.mapred.cli; -import org.apache.hadoop.mapred.ClusterStatus; - import cloudgene.mapred.Main; -import cloudgene.mapred.util.HadoopCluster; import cloudgene.mapred.util.Technology; import genepi.base.Tool; +import genepi.hadoop.HadoopCluster; import genepi.hadoop.HadoopUtil; public class StartServer extends BaseTool { @@ -61,10 +59,9 @@ public int run() { // print summary and warnigns if (settings.isEnable(Technology.HADOOP_CLUSTER)) { - ClusterStatus details = HadoopUtil.getInstance().getClusterDetails(); - int nodes = details.getActiveTrackerNames().size(); - printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + details.getMaxMapTasks() - + " map tasks and " + details.getMaxReduceTasks() + " reduce tasks"); + int nodes = HadoopCluster.getActiveTrackerNames().size(); + printText(0, spaces("[INFO]", 8) + "Cluster has " + nodes + " nodes, " + HadoopCluster.getMaxMapTasks() + + " map tasks and " + HadoopCluster.getMaxReduceTasks() + " reduce tasks"); } else { printText(0, spaces("[WARN]", 8) + "Cluster seems unreachable. Hadoop support disabled."); } diff --git a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java index 089e6f61..84bf5f94 100644 --- a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java +++ b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java @@ -1,8 +1,6 @@ package cloudgene.mapred.cli; -import org.apache.hadoop.mapred.ClusterStatus; - -import cloudgene.mapred.util.HadoopCluster; +import genepi.hadoop.HadoopCluster; import genepi.hadoop.HadoopUtil; public class VerifyCluster extends BaseTool { @@ -21,20 +19,19 @@ public int run() { try { HadoopCluster.verifyCluster(); - ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails(); StringBuffer state = new StringBuffer(); state.append("Mode: " + (HadoopUtil.getInstance().isInSafeMode() ? "Safe Mode" : "Running")); state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + cluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + cluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n"); + state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); + state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); + state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); state.append("Nodes\n"); - for (String tracker : cluster.getActiveTrackerNames()) { + for (String tracker : HadoopCluster.getActiveTrackerNames()) { state.append(" " + tracker + "\n"); } state.append("Blacklist:\n"); - for (String tracker : cluster.getBlacklistedTrackerNames()) { + for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { state.append(" " + tracker + "\n"); } System.out.println(state.toString()); diff --git a/src/main/java/cloudgene/mapred/cron/AlertJob.java b/src/main/java/cloudgene/mapred/cron/AlertJob.java index df75ca09..844d6c44 100644 --- a/src/main/java/cloudgene/mapred/cron/AlertJob.java +++ b/src/main/java/cloudgene/mapred/cron/AlertJob.java @@ -20,6 +20,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException { // check namenode state + try { boolean safemode = HadoopUtil.getInstance().isInSafeMode(); if (safemode) { @@ -46,6 +47,9 @@ public void execute(JobExecutionContext context) throws JobExecutionException { } } + }catch (NoClassDefFoundError e) { + // TODO: handle exception + } // if (cluster.getJobTrackerStatus() == ) diff --git a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java index 0d2a9940..a0129bad 100644 --- a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java +++ b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java @@ -33,13 +33,17 @@ public static int executeRetire(Database database, Settings settings) { // delete local directory and hdfs directory String localOutput = FileUtil.path( settings.getLocalWorkspace(), job.getId()); - - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path( - settings.getHdfsWorkspace(), job.getId())); - FileUtil.deleteDirectory(localOutput); + + try { + String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path( + settings.getHdfsWorkspace(), job.getId())); HdfsUtil.delete(hdfsOutput); - + }catch (NoClassDefFoundError e) { + // TODO: handle exception + } + + job.setState(AbstractJob.STATE_RETIRED); dao.update(job); diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index 0bb53ae5..cf845b36 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -1,9 +1,5 @@ package cloudgene.mapred.jobs; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.common.WorkflowContext; -import genepi.io.FileUtil; - import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; @@ -16,8 +12,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.wdl.WdlParameter; -import cloudgene.mapred.wdl.WdlParameterOutputType; +import genepi.hadoop.HdfsUtil; +import genepi.hadoop.common.WorkflowContext; +import genepi.io.FileUtil; public class CloudgeneContext extends WorkflowContext { @@ -58,7 +55,7 @@ public class CloudgeneContext extends WorkflowContext { private Map config; private int chunks = 0; - + public CloudgeneContext(CloudgeneJob job) { this.workingDirectory = job.getWorkingDirectory(); @@ -71,11 +68,13 @@ public CloudgeneContext(CloudgeneJob job) { workspace = job.getHdfsWorkspace(); - hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp")); - - hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); - - hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); + try { + hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp")); + hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); + hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); + } catch (Error e) { + log("No hdfs folders created."); + } localOutput = new File(job.getLocalWorkspace()).getAbsolutePath(); @@ -113,6 +112,8 @@ public void setupOutputParameters(boolean hasHdfsOutputs) { HdfsUtil.delete(getHdfsTemp()); } catch (Exception e) { System.out.println("Warning: problems during hdfs init."); + } catch (Error e) { + System.out.println("Warning: problems during hdfs init."); } } @@ -456,7 +457,7 @@ public String getConfig(String param) { return null; } } - + public void addFile(String filename) { chunks++; String chunkFolder = FileUtil.path(getLocalOutput(), "chunks"); @@ -467,4 +468,5 @@ public void addFile(String filename) { message(chunkFilename, 27); } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index b1b55d20..6447149b 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -40,6 +40,8 @@ public class CloudgeneJob extends AbstractJob { private Executor executor; + public static final int MAX_DOWNLOAD = 10; + private static final Log log = LogFactory.getLog(CloudgeneJob.class); public CloudgeneJob() { @@ -370,14 +372,17 @@ public boolean cleanUp() { // delete hdfs workspace if (isRemoveHdfsWorkspace()) { + if (context.getHdfsOutput() != null){ writeLog("Cleaning up hdfs files..."); HdfsUtil.delete(context.getHdfsOutput()); HdfsUtil.delete(context.getHdfsInput()); + } } } catch (Exception e) { log.warn("Warning: problems during hdfs cleanup."); + } catch (Error e) { + log.warn("Warning: problems during hdfs cleanup."); } - } return true; @@ -510,8 +515,7 @@ public boolean exportParameter(CloudgeneParameterOutput out) { download.setSize(item.getSize()); download.setHash(hash); download.setParameter(out); - int maxDownloads = getSettings().getMaxDownloads(); - download.setCount(maxDownloads); + download.setCount(MAX_DOWNLOAD); files.add(download); } Collections.sort(files); diff --git a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java index 47e12331..bdad7bcd 100644 --- a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java +++ b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java @@ -10,19 +10,14 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.RunningJob; - import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.util.Technology; import cloudgene.mapred.wdl.WdlStep; import genepi.hadoop.HadoopUtil; -import genepi.hadoop.HdfsUtil; import genepi.hadoop.common.WorkflowContext; import genepi.io.FileUtil; -import genepi.io.text.LineWriter; public class HadoopMapReduceStep extends CloudgeneStep { @@ -84,36 +79,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { List command = new Vector(); command.add(hadoop); - // TODO: write config and set to hadoop config - try { - FileUtil.createDirectory("temp-conf"); - LineWriter writer = new LineWriter("temp-conf/mapred-site.xml"); - Configuration configuration = HdfsUtil.getConfiguration(); - writer.write(""); - writer.write(""); - writer.write("mapred.job.tracker" + configuration.get("mapred.job.tracker") - + ""); - writer.write(""); - writer.close(); - - writer = new LineWriter("temp-conf/core-site.xml"); - writer.write(""); - writer.write(""); - writer.write("fs.default.name" + configuration.get("fs.defaultFS") - + ""); - writer.write(""); - writer.close(); - - // HdfsUtil.getConfiguration().writeXml(new - // FileOutputStream("temp-conf/mapred-default.xml")); - - command.add("--config"); - command.add(new File("temp-conf").getAbsolutePath()); - - } catch (IOException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } + // -fs and -jt and -config are supported by generic tool runner. command.add("jar"); @@ -262,7 +228,7 @@ protected boolean executeCommand(List command, WorkflowContext context) @Override public void updateProgress() { - RunningJob job = HadoopUtil.getInstance().getJob(jobId); + /*RunningJob job = HadoopUtil.getInstance().getJob(jobId); if (job != null) { try { @@ -283,7 +249,7 @@ public void updateProgress() { } else { map = 0; reduce = 0; - } + }*/ } diff --git a/src/main/java/cloudgene/mapred/util/ClusterInformation.java b/src/main/java/cloudgene/mapred/util/ClusterInformation.java deleted file mode 100644 index 6aa11381..00000000 --- a/src/main/java/cloudgene/mapred/util/ClusterInformation.java +++ /dev/null @@ -1,41 +0,0 @@ -package cloudgene.mapred.util; - -import org.apache.hadoop.mapred.ClusterStatus; -import org.apache.hadoop.util.VersionInfo; - -public class ClusterInformation { - - private ClusterStatus status; - - private String version; - - private String date; - - public ClusterInformation(ClusterStatus status) { - this.status = status; - } - - public ClusterStatus getStatus() { - return status; - } - - public void setStatus(ClusterStatus status) { - this.status = status; - } - - public String getVersion() { - return VersionInfo.getVersion(); - } - - public String getDate() { - return VersionInfo.getDate(); - } - - public void setDate(String date) { - this.date = date; - } - - public void setVersion(String version) { - this.version = version; - } -} diff --git a/src/main/java/cloudgene/mapred/util/HadoopCluster.java b/src/main/java/cloudgene/mapred/util/HadoopCluster.java deleted file mode 100644 index af3295ce..00000000 --- a/src/main/java/cloudgene/mapred/util/HadoopCluster.java +++ /dev/null @@ -1,106 +0,0 @@ -package cloudgene.mapred.util; - -import java.io.File; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.ClusterStatus; - -import genepi.hadoop.HadoopUtil; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class HadoopCluster { - - private static String username; - - private static String conf; - - private static String name; - - public static void setConfPath(String name, String path, String username) { - if (username != null) { - System.setProperty("HADOOP_USER_NAME", username); - HadoopCluster.username = username; - } - HadoopCluster.conf = path; - HadoopCluster.name = name; - if (new File(path).exists()) { - Configuration configuration = new Configuration(); - // add all xml files from hadoop conf folder to default - // configuration - String[] xmlFiles = FileUtil.getFiles(path, "*.xml"); - for (String xmlFile : xmlFiles) { - configuration.addResource(new Path(xmlFile)); - } - - HdfsUtil.setDefaultConfiguration(configuration); - } - - } - - public static String getUsername() { - return username; - } - - public static String getConf() { - return conf; - } - - public static String getName() { - return name; - } - - public static String getJobTracker() { - Configuration configuration = HdfsUtil.getConfiguration(); - return configuration.get("mapred.job.tracker"); - } - - public static String getDefaultFS() { - Configuration configuration = HdfsUtil.getConfiguration(); - return configuration.get("fs.defaultFS"); - } - - public static boolean verifyCluster() throws Exception { - - String confFolder = HadoopCluster.getConf(); - - if (confFolder == null) { - throw new Exception("Please define a cluster in file settings.yaml."); - } - - if (new File(confFolder).exists()) { - String configName = "mapred-site.xml"; - String configFile = FileUtil.path(confFolder, configName); - if (!new File(configFile).exists()) { - throw new Exception("No '" + configName + "' file found in configuration folder '" + confFolder + "'."); - } - } else { - throw new Exception("Configuration folder '" + confFolder + "' not found."); - } - - ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails(); - if (cluster.getActiveTrackerNames().isEmpty()) { - throw new Exception("No active nodes founds."); - } - Configuration configuration = HdfsUtil.getConfiguration(); - FileSystem fileSystem = FileSystem.get(configuration); - - try { - Path path = fileSystem.getHomeDirectory(); - - if (fileSystem.exists(path)) { - return true; - } else { - throw new Exception( - "Home directory '" + path + "' for user " + HadoopCluster.getUsername() + " not found."); - } - - } catch (Exception e) { - throw new Exception( - "Problems find the home directory for user " + HadoopCluster.getUsername() + ". " + e.getMessage()); - } - } - -} diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 37b3e2b8..e7f2fa2a 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -31,6 +31,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.util.GitHubUtil.Repository; import cloudgene.mapred.wdl.WdlApp; +import genepi.hadoop.HadoopCluster; import genepi.io.FileUtil; import net.lingala.zip4j.core.ZipFile; import net.lingala.zip4j.exception.ZipException; @@ -38,7 +39,7 @@ public class Settings { - private String hadoopPath = "/usr/bin/hadoop"; + private String hadoopPath = "/usr/"; private String pigPath = "/usr/"; @@ -208,7 +209,10 @@ public static Settings load(Config config) throws FileNotFoundException, YamlExc if (conf != null) { log.info("Use Haddop configuration folder '" + conf + "'" + (username != null ? " with username " + username : "")); - HadoopCluster.setConfPath(name, conf, username); + try { + HadoopCluster.setConfPath(name, conf, username); + }catch (NoClassDefFoundError e) { + } } } diff --git a/src/main/java/cloudgene/sample/SampleStep.java b/src/main/java/cloudgene/sample/SampleStep.java deleted file mode 100644 index f985084c..00000000 --- a/src/main/java/cloudgene/sample/SampleStep.java +++ /dev/null @@ -1,151 +0,0 @@ -package cloudgene.sample; - -import genepi.hadoop.HadoopUtil; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.common.WorkflowContext; -import genepi.hadoop.common.WorkflowStep; -import genepi.io.FileUtil; - -import java.io.File; - -import org.apache.hadoop.mapred.ClusterStatus; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.util.Settings; - -public class SampleStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - Settings settings = ((CloudgeneContext) context).getSettings(); - - context.ok("Cloudgene runs with the following Hadoop configuration:"); - - ClusterStatus cluster = HadoopUtil.getInstance().getClusterDetails(); - StringBuffer state = new StringBuffer(); - state.append("Hadoop Cluster\n"); - state.append("State: " + cluster.getJobTrackerStatus().toString() - + "\n"); - state.append("MapTask: " + cluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + cluster.getMaxReduceTasks() + "\n"); - state.append("Nodes\n"); - for (String tracker : cluster.getActiveTrackerNames()) { - state.append(tracker + "\n"); - } - - String filename = context.getInput("testfile"); - context.ok(FileUtil.readFileAsString(filename)); - - context.ok(state.toString()); - context.beginTask("waiting...."); - try { - Thread.sleep(5000); - } catch (InterruptedException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - - String hadoopPath = settings.getHadoopPath(); - - if (hadoopPath.trim().isEmpty()) { - context.error("Hadoop Binary was not set. Please set the correct path in the admin panel."); - return false; - } - - File path = new File(hadoopPath); - - if (!path.exists()) { - context.error("Hadoop Binary " - + path - + " was not found. Please set the correct path in the admin panel."); - return false; - } - - String hadoop = ""; - - if (path.isDirectory()) { - hadoop = FileUtil.path(hadoopPath, "bin", "hadoop"); - } else { - hadoop = hadoopPath; - } - - File file = new File(hadoop); - - if (!file.exists()) { - context.error("Hadoop Binary " - + hadoop - + " was not found. Please set the correct path in the admin panel."); - return false; - } - - if (!file.canExecute()) { - context.error("Hadoop Binary " - + hadoop - + " was found, but it can not be executed. Please check the permissions."); - return false; - } - - context.ok("Hadoop Binary was found in " + hadoop - + " and is executable."); - - // TODO: write r script which checks packages - - context.ok("R was found and all packages are installed."); - - // TODO: write file to hdfs temp directory - - HdfsUtil.delete("cloudgene-test/test-file.txt"); - FileUtil.writeStringBufferToFile("lf.txt", new StringBuffer("lf-yeah!")); - HdfsUtil.put("lf.txt", "cloudgene-test/test-file.txt"); - - context.ok("HDFS File System check (check if file cloudgene-test/test-file.txt exists in hdfs!)"); - - // TODO: write file to local temp directory - - context.ok("Local File System check"); - - String outputFolder = context.getOutput("stats"); - - FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "1.txt"), - new StringBuffer("file 1")); - FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "2.txt"), - new StringBuffer("file 2")); - FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "3.txt"), - new StringBuffer("file 3")); - FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "4.txt"), - new StringBuffer("file 4")); - FileUtil.writeStringBufferToFile(FileUtil.path(outputFolder, "5.txt"), - new StringBuffer("file 5")); - - /* - * // Mail Server.... String mail = context.get("cloudgene.user.mail"); - * if (mail != null) { - * - * String subject = "Mail Server Test"; String message = - * "This email was sent by Cloudgene to test your mail-server settings." - * ; - * - * try { - * - * context.sendMail(subject, message); - * - * context.ok("We have sent a test-email to " + mail + "."); - * - * } catch (Exception e) { context.error("Sending mail failed: " + - * e.getMessage()); return false; } - * - * } else { - * - * context.error( - * "No email address found. Please enter your email address (Account -> Profile)." - * ); return false; - * - * } - */ - context.ok("Congratulations. Cloudgene works properly on your Hadoop Cluster!"); - - return true; - - } -} diff --git a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java index 2fb78ccf..df45d2d2 100644 --- a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java @@ -10,10 +10,10 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.LoginToken; -import cloudgene.mapred.util.junit.TestMailServer; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.LoginToken; +import cloudgene.mapred.util.TestMailServer; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class ChangeGroupTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java index a2eec21b..7c9b2163 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/CancelJobTest.java @@ -5,9 +5,9 @@ import org.restlet.resource.ClientResource; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; public class CancelJobTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java index 35334ae9..278005ca 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DeleteJobTest.java @@ -12,8 +12,8 @@ import org.restlet.resource.ClientResource; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestServer; public class DeleteJobTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java index 0cbb2ebd..bc4a3356 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java @@ -8,9 +8,9 @@ import org.restlet.resource.ClientResource; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; public class DownloadResultsTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java index f66a17f3..0cc424bf 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/GetJobsTest.java @@ -10,10 +10,10 @@ import org.restlet.resource.ClientResource; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.LoginToken; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.LoginToken; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; public class GetJobsTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java index 80e532c4..48ac43a1 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/GetLogsTest.java @@ -10,9 +10,9 @@ import org.restlet.representation.FileRepresentation; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; import genepi.io.FileUtil; public class GetLogsTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java index 83c29c4a..4c43a29b 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/RestartJobTest.java @@ -4,9 +4,9 @@ import org.restlet.ext.html.FormDataSet; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; public class RestartJobTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java index 67976fe4..35a36587 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/ShareResultsTest.java @@ -9,9 +9,9 @@ import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; +import cloudgene.mapred.util.TestServer; public class ShareResultsTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 7b55b2c6..8b61d457 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -13,10 +13,10 @@ import org.restlet.resource.ClientResource; import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestCluster; import cloudgene.mapred.util.TestSFTPServer; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestCluster; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.TestServer; import genepi.hadoop.common.WorkflowContext; import genepi.io.FileUtil; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java index b417cc3a..10fd21f1 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java @@ -11,9 +11,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestMailServer; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestMailServer; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class ActivateUserTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java index 455f0ab9..8f295241 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java @@ -16,9 +16,9 @@ import cloudgene.mapred.database.UserDao; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.LoginToken; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.LoginToken; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; import genepi.io.FileUtil; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java index 29d59187..8e8fe44a 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java @@ -11,8 +11,8 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class LoginUserTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java index 565fe631..98ed8824 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/LogoutUserTest.java @@ -8,9 +8,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.LoginToken; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.LoginToken; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class LogoutUserTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java index 1cc3c1f5..6a4a67c9 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/RegisterUserTest.java @@ -7,9 +7,9 @@ import org.restlet.data.Form; import org.restlet.resource.ClientResource; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestMailServer; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestMailServer; +import cloudgene.mapred.util.TestServer; public class RegisterUserTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java index 6f14fabe..b15d990b 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java @@ -12,9 +12,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestMailServer; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestMailServer; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class ResetPasswordTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java index aee8583f..c3f21352 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java @@ -10,9 +10,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestMailServer; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestMailServer; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class UpdatePasswordTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java index 2005f8b9..5b528407 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java @@ -10,9 +10,9 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.LoginToken; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.LoginToken; +import cloudgene.mapred.util.TestServer; import genepi.db.Database; public class UserProfileTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/database/JobDaoTest.java b/src/test/java/cloudgene/mapred/database/JobDaoTest.java index e85ac479..e615a12a 100644 --- a/src/test/java/cloudgene/mapred/database/JobDaoTest.java +++ b/src/test/java/cloudgene/mapred/database/JobDaoTest.java @@ -9,8 +9,8 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneJob; -import cloudgene.mapred.util.junit.JobsApiTestCase; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.JobsApiTestCase; +import cloudgene.mapred.util.TestServer; public class JobDaoTest extends JobsApiTestCase { diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java index 9b4150c2..7abefac0 100644 --- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java +++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java @@ -6,7 +6,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.TestServer; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; import genepi.hadoop.HdfsUtil; diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index 1a347a82..92399f17 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -11,7 +11,7 @@ import junit.framework.TestCase; import cloudgene.mapred.core.User; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.TestServer; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java index 20121e17..979bd891 100644 --- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java @@ -10,7 +10,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.JobDao; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.TestServer; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index 989e5e77..67c93789 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -16,7 +16,7 @@ import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.junit.TestServer; +import cloudgene.mapred.util.TestServer; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; diff --git a/src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java b/src/test/java/cloudgene/mapred/util/JobsApiTestCase.java similarity index 99% rename from src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java rename to src/test/java/cloudgene/mapred/util/JobsApiTestCase.java index 034a2929..e5a1a6b9 100644 --- a/src/main/java/cloudgene/mapred/util/junit/JobsApiTestCase.java +++ b/src/test/java/cloudgene/mapred/util/JobsApiTestCase.java @@ -1,8 +1,7 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import java.io.IOException; -import org.apache.avro.data.Json; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -16,7 +15,7 @@ import junit.framework.TestCase; -public class JobsApiTestCase extends TestCase { +public abstract class JobsApiTestCase extends TestCase { public ClientResource createClientResource(String path) { diff --git a/src/main/java/cloudgene/mapred/util/junit/LoginToken.java b/src/test/java/cloudgene/mapred/util/LoginToken.java similarity index 91% rename from src/main/java/cloudgene/mapred/util/junit/LoginToken.java rename to src/test/java/cloudgene/mapred/util/LoginToken.java index a46b0056..786ec70a 100644 --- a/src/main/java/cloudgene/mapred/util/junit/LoginToken.java +++ b/src/test/java/cloudgene/mapred/util/LoginToken.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import org.restlet.data.CookieSetting; diff --git a/src/main/java/cloudgene/mapred/util/junit/TestCluster.java b/src/test/java/cloudgene/mapred/util/TestCluster.java similarity index 98% rename from src/main/java/cloudgene/mapred/util/junit/TestCluster.java rename to src/test/java/cloudgene/mapred/util/TestCluster.java index f180618a..070eda3d 100644 --- a/src/main/java/cloudgene/mapred/util/junit/TestCluster.java +++ b/src/test/java/cloudgene/mapred/util/TestCluster.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import genepi.hadoop.HdfsUtil; diff --git a/src/main/java/cloudgene/mapred/util/junit/TestMailServer.java b/src/test/java/cloudgene/mapred/util/TestMailServer.java similarity index 96% rename from src/main/java/cloudgene/mapred/util/junit/TestMailServer.java rename to src/test/java/cloudgene/mapred/util/TestMailServer.java index a15e1cd8..f9c7c8b5 100644 --- a/src/main/java/cloudgene/mapred/util/junit/TestMailServer.java +++ b/src/test/java/cloudgene/mapred/util/TestMailServer.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import java.util.Iterator; import java.util.List; diff --git a/src/main/java/cloudgene/mapred/util/junit/TestServer.java b/src/test/java/cloudgene/mapred/util/TestServer.java similarity index 99% rename from src/main/java/cloudgene/mapred/util/junit/TestServer.java rename to src/test/java/cloudgene/mapred/util/TestServer.java index 3a791b00..39ce2440 100644 --- a/src/main/java/cloudgene/mapred/util/junit/TestServer.java +++ b/src/test/java/cloudgene/mapred/util/TestServer.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import java.io.File; import java.io.InputStream; diff --git a/src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java b/src/test/java/cloudgene/mapred/util/WdlAppTestCase.java similarity index 97% rename from src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java rename to src/test/java/cloudgene/mapred/util/WdlAppTestCase.java index af5fe85e..6c456599 100644 --- a/src/main/java/cloudgene/mapred/util/junit/WdlAppTestCase.java +++ b/src/test/java/cloudgene/mapred/util/WdlAppTestCase.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.util.junit; +package cloudgene.mapred.util; import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil;