Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

IMPALA-28 remove default value for JT and NN

  • Loading branch information...
commit 5df6e6b096ae1d4d7b547142655d611e8b9cd206 1 parent 4809da9
@leejianwei leejianwei authored
View
20 ...on/src/main/java/org/springframework/data/hadoop/impala/common/ConfigurationCommands.java
@@ -15,6 +15,8 @@
*/
package org.springframework.data.hadoop.impala.common;
+import java.net.URI;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
@@ -105,10 +107,20 @@ public String info() {
sb.append(VersionInfo.getVersion());
sb.append(" rev.");
sb.append(VersionInfo.getRevision());
- sb.append("][fs=");
- sb.append(FileSystem.getDefaultUri(hadoopConfiguration));
- sb.append("][jt=");
- sb.append(hadoopConfiguration.get("mapred.job.tracker"));
+ sb.append("]");
+
+ sb.append("[fs=");
+ String fs = hadoopConfiguration.get("fs.default.name");
+ if(fs != null && fs.length() > 0){
+ sb.append(fs);
+ }
+ sb.append("]");
+
+ sb.append("[jt=");
+ String jt = hadoopConfiguration.get("mapred.job.tracker");
+ if(jt != null && jt.length() > 0){
+ sb.append(jt);
+ }
sb.append("]");
// TODO: potentially add a check to see whether HDFS is running
View
4 plugin-common/src/main/resources/META-INF/spring/spring-shell-plugin.xml
@@ -10,8 +10,8 @@
<context:component-scan base-package="org.springframework.data.hadoop.impala.provider"/>
<hdp:configuration>
- fs.default.name=webhdfs://localhost:50070
- mapred.job.tracker=localhost:9001
+ fs.default.name=
+ mapred.job.tracker=
</hdp:configuration>
<context:component-scan base-package="org.springframework.data.hadoop.impala.common"/>
View
17 plugin-hdfs/src/main/java/org/springframework/data/hadoop/impala/hdfs/FsShellCommands.java
@@ -15,14 +15,18 @@
*/
package org.springframework.data.hadoop.impala.hdfs;
+import java.net.URI;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.PostConstruct;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.springframework.data.hadoop.impala.common.ConfigurationAware;
+import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
@@ -341,6 +345,19 @@ public void touchz(
run(argv.toArray(new String[0]));
}
+ @CliAvailabilityIndicator({PREFIX + "ls", PREFIX + "cat", PREFIX + "chgrp",
+ PREFIX + "chown", PREFIX + "chmod", PREFIX + "copyFromLocal", PREFIX + "put", PREFIX + "moveFromLocal",
+ PREFIX + "copyToLocal", PREFIX + "get", PREFIX + "count", PREFIX + "cp", PREFIX + "mv",
+ PREFIX + "du", PREFIX + "expunge", PREFIX + "mkdir", PREFIX + "rm",
+ PREFIX + "setrep", PREFIX + "tail", PREFIX + "text", PREFIX + "touchz"})
+ public boolean isCmdAvailable() {
+ String fs = getHadoopConfiguration().get("fs.default.name");
+ if(fs != null && fs.length() > 0){
+ return true;
+ }
+ return false;
+ }
+
/**
* @param value
*/
View
10 plugin-hive/src/main/java/org/springframework/data/hadoop/impala/hive/HiveCommands.java
@@ -30,6 +30,7 @@
import org.springframework.data.hadoop.hive.HiveClientFactoryBean;
import org.springframework.data.hadoop.hive.HiveScriptRunner;
import org.springframework.shell.core.CommandMarker;
+import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.shell.support.util.StringUtils;
@@ -132,6 +133,15 @@ public String script(@CliOption(key = { "", "location" }, mandatory = true, help
return sb.append(StringUtils.LINE_SEPARATOR).append("Script [" + uri + "] executed succesfully").toString();
}
+
+ @CliAvailabilityIndicator({PREFIX + "script"})
+ public boolean isCmdAvailable() {
+ if(host != null && host.length() > 0){
+ return true;
+ }
+ return false;
+ }
+
private static String fixLocation(String location) {
if (StringUtils.hasText(location) && !location.contains(":")) {
View
31 ...uce/src/main/java/org/springframework/data/hadoop/impala/mapreduce/MapReduceCommands.java
@@ -23,6 +23,7 @@
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.net.URI;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
@@ -32,6 +33,7 @@
import java.util.jar.JarFile;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
@@ -40,6 +42,7 @@
import org.springframework.data.hadoop.impala.common.ConfigurationAware;
import org.springframework.data.hadoop.impala.common.util.SecurityUtil;
import org.springframework.data.hadoop.impala.common.util.SecurityUtil.ExitTrappedException;
+import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.shell.event.ParseResult;
@@ -57,6 +60,8 @@
private JobClient jobClient;
+ private static final String PREFIX = "mr job ";
+
@Autowired
private SecurityUtil securityUtil;
@@ -96,7 +101,7 @@ protected boolean configurationChanged() throws Exception {
return true;
}
- @CliCommand(value = "mr job submit", help = "submit Map Reduce Jobs.")
+ @CliCommand(value = PREFIX + "submit", help = "submit Map Reduce Jobs.")
public void submit(@CliOption(key = { "jobfile" }, mandatory = true, help = "the configuration file for MR job") final String jobFile) {
List<String> argv = new ArrayList<String>();
argv.add("-submit");
@@ -104,7 +109,7 @@ public void submit(@CliOption(key = { "jobfile" }, mandatory = true, help = "the
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job status", help = "query Map Reduce status.")
+ @CliCommand(value = PREFIX + "status", help = "query Map Reduce status.")
public void status(@CliOption(key = { "jobid" }, mandatory = true, help = "the job Id") final String jobid) {
List<String> argv = new ArrayList<String>();
argv.add("-status");
@@ -112,7 +117,7 @@ public void status(@CliOption(key = { "jobid" }, mandatory = true, help = "the j
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job counter", help = "print the counter value")
+ @CliCommand(value = PREFIX + "counter", help = "print the counter value")
public void counter(@CliOption(key = { "jobid" }, mandatory = true, help = "the job Id") final String jobid, @CliOption(key = { "groupname" }, mandatory = true, help = "the job Id") final String groupName, @CliOption(key = { "countername" }, mandatory = true, help = "the job Id") final String counterName) {
List<String> argv = new ArrayList<String>();
argv.add("-counter");
@@ -122,7 +127,7 @@ public void counter(@CliOption(key = { "jobid" }, mandatory = true, help = "the
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job kill", help = "kill Map Reduce job.")
+ @CliCommand(value = PREFIX + "kill", help = "kill Map Reduce job.")
public void kill(@CliOption(key = { "jobid" }, mandatory = true, help = "the job Id") final String jobid) {
List<String> argv = new ArrayList<String>();
argv.add("-kill");
@@ -130,7 +135,7 @@ public void kill(@CliOption(key = { "jobid" }, mandatory = true, help = "the job
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job events", help = "print the events' details received by jobtracker for the given range")
+ @CliCommand(value = PREFIX + "events", help = "print the events' details received by jobtracker for the given range")
public void events(@CliOption(key = { "jobid" }, mandatory = true, help = "the job Id") final String jobid, @CliOption(key = { "from" }, mandatory = true, help = "from event number") final String from, @CliOption(key = { "number" }, mandatory = true, help = "total number of events") final String number) {
List<String> argv = new ArrayList<String>();
argv.add("-events");
@@ -140,7 +145,7 @@ public void events(@CliOption(key = { "jobid" }, mandatory = true, help = "the j
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job history", help = "print job details, failed and killed tip details")
+ @CliCommand(value = PREFIX + "history", help = "print job details, failed and killed tip details")
public void history(@CliOption(key = { "all" }, mandatory = false, specifiedDefaultValue = "true", unspecifiedDefaultValue = "false", help = "list all jobs") final boolean all, @CliOption(key = { "" }, mandatory = true, help = "job output directory") final String outputDir) {
List<String> argv = new ArrayList<String>();
argv.add("-history");
@@ -151,7 +156,7 @@ public void history(@CliOption(key = { "all" }, mandatory = false, specifiedDefa
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job list", help = "list MapReduce Jobs.")
+ @CliCommand(value = PREFIX + "list", help = "list MapReduce Jobs.")
public void list(@CliOption(key = { "all" }, mandatory = false, specifiedDefaultValue = "true", unspecifiedDefaultValue = "false", help = "list all jobs") final boolean all) {
List<String> argv = new ArrayList<String>();
argv.add("-list");
@@ -178,7 +183,7 @@ public void failTask(@CliOption(key = { "taskid" }, mandatory = true, help = "th
run(argv.toArray(new String[0]));
}
- @CliCommand(value = "mr job set priority", help = "change the priority of the job.")
+ @CliCommand(value = PREFIX + "set priority", help = "change the priority of the job.")
public void setPriority(@CliOption(key = { "jobid" }, mandatory = true, help = "the job Id") final String jobid, @CliOption(key = { "priority" }, mandatory = true, help = "the job priority") final JobPriority priority) {
List<String> argv = new ArrayList<String>();
argv.add("-set-priority");
@@ -212,6 +217,16 @@ public void jar(@CliOption(key = { "jarfile" }, mandatory = true, help = "jar fi
securityUtil.enableSystemExitCall();
}
}
+
+ @CliAvailabilityIndicator({PREFIX + "submit", PREFIX + "status", PREFIX + "counter", PREFIX + "kill", PREFIX + "events",
+ PREFIX + "history", PREFIX + "list", "mr task kill", "mr task fail", PREFIX + "set priority", "mr jar"})
+ public boolean isCmdAvailable() {
+ String jobTracker = getHadoopConfiguration().get("mapred.job.tracker");
+ if(jobTracker != null && jobTracker.length() > 0){
+ return true;
+ }
+ return false;
+ }
/**
* @param jarFileName
View
10 plugin-pig/src/main/java/org/springframework/data/hadoop/impala/pig/PigCommands.java
@@ -37,6 +37,7 @@
import org.springframework.data.hadoop.pig.PigContextFactoryBean;
import org.springframework.data.hadoop.pig.PigServerFactoryBean;
import org.springframework.shell.core.CommandMarker;
+import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
@@ -199,6 +200,15 @@ public String script(@CliOption(key = { "", "location" }, mandatory = true, help
pig.shutdown();
}
}
+
+ @CliAvailabilityIndicator({PREFIX + "script"})
+ public boolean isCmdAvailable() {
+ String jobTracker = hadoopConfiguration.get("mapred.job.tracker");
+ if(jobTracker != null && jobTracker.length() > 0){
+ return true;
+ }
+ return false;
+ }
private static String fixLocation(String location) {
if (StringUtils.hasText(location) && !location.contains(":")) {
Please sign in to comment.
Something went wrong with that request. Please try again.