Permalink
Browse files

HIVE-1264. Make Hive work with Hadoop security

(Todd Lipcon via jvs)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/hive/trunk@1021549 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent c379f7f commit 36e9d61e107bf32e2ee9398f87c77f5c9f9ff7e7 John Sichi committed Oct 11, 2010
View
@@ -181,6 +181,9 @@ Trunk - Unreleased
HIVE-1697. Migration scripts should increase size of PARAM_VALUE in
PARTITION_PARAMS (Paul Yang via namit)
+ HIVE-1264. Make Hive work with Hadoop security
+ (Todd Lipcon via jvs)
+
OPTIMIZATIONS
BUG FIXES
View
@@ -218,8 +218,10 @@
<!-- the normal classpath -->
<path id="common-classpath">
- <pathelement location="${hadoop.jar}"/>
- <pathelement location="${hadoop.tools.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
<pathelement location="${build.dir.hive}/classes"/>
<fileset dir="${build.dir.hive}" includes="*/*.jar"/>
<fileset dir="${hive.root}/lib" includes="*.jar"/>
View
@@ -20,9 +20,15 @@ build.dir.hadoop=${build.dir.hive}/hadoopcore
hadoop.version.ant-internal=${hadoop.version}
hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
hadoop.root=${hadoop.root.default}
-hadoop.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
-hadoop.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
-hadoop.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
+# Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test}
+# We will add both styles to the classpath and it will pick up whichever is there
+hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
+hadoop.oldstyle-name.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
+hadoop.oldstyle-name.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
+hadoop.newstyle-name.jar=${hadoop.root}/hadoop-core-${hadoop.version.ant-internal}.jar
+hadoop.newstyle-name.test.jar=${hadoop.root}/hadoop-test-${hadoop.version.ant-internal}.jar
+hadoop.newstyle-name.tools.jar=${hadoop.root}/hadoop-tools-${hadoop.version.ant-internal}.jar
+
jetty.test.jar=${hadoop.root}/lib/jetty-5.1.4.jar
servlet.test.jar=${hadoop.root}/lib/servlet-api.jar
jasper.test.jar=${hadoop.root}/lib/jetty-ext/jasper-runtime.jar
@@ -34,3 +40,8 @@ common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar
# Data nucleus repository - needed for jdo2-api-2.3-ec.jar download
#
datanucleus.repo=http://www.datanucleus.org/downloads/maven2
+
+# URLs pointing to a built tarball of a secure hadoop release
+hadoop.security.url=http://mirror.facebook.net/facebook/hive-deps/hadoop/core/hadoop-0.20.3-CDH3-SNAPSHOT/hadoop-0.20.3-CDH3-SNAPSHOT.tar.gz
+hadoop.security.version=0.20.3-CDH3-SNAPSHOT
+
View
@@ -112,7 +112,8 @@
<!-- the normal classpath -->
<path id="common-classpath">
- <pathelement location="${hadoop.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.jar}"/>
<pathelement location="${build.dir.hive}/classes"/>
<fileset dir="${hive.root}" includes="hive-*.jar"/>
<fileset dir="${hive.root}/lib" includes="*.jar"/>
@@ -591,13 +591,11 @@ public void setAuxJars(String auxJars) {
*/
public String getUser() throws IOException {
try {
- UserGroupInformation ugi = UserGroupInformation.readFrom(this);
- if (ugi == null) {
- ugi = UserGroupInformation.login(this);
- }
+ UserGroupInformation ugi = ShimLoader.getHadoopShims()
+ .getUGIForConf(this);
return ugi.getUserName();
- } catch (LoginException e) {
- throw (IOException) new IOException().initCause(e);
+ } catch (LoginException le) {
+ throw new IOException(le);
}
}
View
@@ -36,7 +36,6 @@
<pathelement location="${test.src.data.dir}/conf"/>
<pathelement location="${hive.conf.dir}"/>
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
- <pathelement location="${hadoop.test.jar}"/>
<pathelement location="${jetty.test.jar}"/>
<pathelement location="${servlet.test.jar}"/>
<pathelement location="${jasper.test.jar}"/>
View
@@ -37,7 +37,8 @@
<pathelement location="${test.src.data.dir}/conf"/>
<pathelement location="${hive.conf.dir}"/>
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
- <pathelement location="${hadoop.test.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.test.jar}"/>
<pathelement location="${jetty.test.jar}"/>
<pathelement location="${servlet.test.jar}"/>
<pathelement location="${jasper.test.jar}"/>
View
@@ -40,7 +40,8 @@
<pathelement location="${test.src.data.dir}/conf"/>
<pathelement location="${hive.conf.dir}"/>
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
- <pathelement location="${hadoop.test.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.test.jar}"/>
<pathelement location="${jetty.test.jar}"/>
<pathelement location="${servlet.test.jar}"/>
<pathelement location="${jasper.test.jar}"/>
@@ -85,10 +85,10 @@
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
public class Driver implements CommandProcessor {
@@ -285,21 +285,11 @@ public boolean hasReduceTasks(List<Task<? extends Serializable>> tasks) {
*/
public Driver(HiveConf conf) {
this.conf = conf;
- try {
- UnixUserGroupInformation.login(conf, true);
- } catch (Exception e) {
- LOG.warn("Ignoring " + e.getMessage());
- }
}
public Driver() {
if (SessionState.get() != null) {
conf = SessionState.get().getConf();
- try {
- UnixUserGroupInformation.login(conf, true);
- } catch (Exception e) {
- LOG.warn("Ignoring " + e.getMessage());
- }
}
}
@@ -739,8 +729,7 @@ public int execute() {
// Get all the pre execution hooks and execute them.
for (PreExecute peh : getPreExecHooks()) {
peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
- UnixUserGroupInformation.readFromConf(conf,
- UnixUserGroupInformation.UGI_PROPERTY_NAME));
+ ShimLoader.getHadoopShims().getUGIForConf(conf));
}
int jobs = Utilities.getMRTasks(plan.getRootTasks()).size();
@@ -822,8 +811,7 @@ public int execute() {
for (PostExecute peh : getPostExecHooks()) {
peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
(SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null),
- UnixUserGroupInformation.readFromConf(conf,
- UnixUserGroupInformation.UGI_PROPERTY_NAME));
+ ShimLoader.getHadoopShims().getUGIForConf(conf));
}
if (SessionState.get() != null) {
View
@@ -26,9 +26,12 @@ to call at top-level: ant deploy-contrib compile-core-test
<import file="../build-common.xml"/>
<path id="classpath">
- <pathelement location="${hadoop.jar}"/>
- <pathelement location="${hadoop.tools.jar}"/>
- <pathelement location="${hadoop.test.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
+ <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.test.jar}"/>
+ <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
<fileset dir="${hadoop.root}/lib">
<include name="**/*.jar" />
<exclude name="**/excluded/" />
@@ -66,6 +69,10 @@ to call at top-level: ant deploy-contrib compile-core-test
<antcall target="build_shims" inheritRefs="false" inheritAll="false">
<param name="hadoop.version.ant-internal" value="0.20.0" />
</antcall>
+ <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
+ <param name="hadoop.version.ant-internal.prefix" value="0.20S" />
+ </antcall>
<getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version}" />
<javac
encoding="${build.encoding}"
View
@@ -13,6 +13,10 @@
<dependency org="hadoop" name="core" rev="0.20.0">
<artifact name="hadoop" type="source" ext="tar.gz"/>
</dependency>
+ <dependency org="hadoop" name="core" rev="${hadoop.security.version}">
+ <artifact name="hadoop" type="source" ext="tar.gz"
+ url="${hadoop.security.url}" />
+ </dependency>
<conflict manager="all" />
</dependencies>
</ivy-module>
@@ -29,6 +29,9 @@
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
import java.io.IOException;
@@ -134,4 +137,13 @@ public int createHadoopArchive(Configuration conf, Path parentDir, Path destDir,
throw new RuntimeException("Not implemented in this Hadoop version");
}
+ @Override
+ public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+ UserGroupInformation ugi =
+ UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+ if(ugi == null) {
+ ugi = UserGroupInformation.login(conf);
+ }
+ return ugi;
+ }
}
@@ -31,6 +31,9 @@
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
import java.io.IOException;
@@ -137,4 +140,14 @@ public int createHadoopArchive(Configuration conf, Path parentDir, Path destDir,
public void setNullOutputFormat(JobConf conf) {
conf.setOutputFormat(NullOutputFormat.class);
}
+
+ @Override
+ public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+ UserGroupInformation ugi =
+ UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+ if(ugi == null) {
+ ugi = UserGroupInformation.login(conf);
+ }
+ return ugi;
+ }
}
@@ -37,6 +37,9 @@
import org.apache.hadoop.mapred.TaskAttemptContext;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -512,4 +515,15 @@ public void setNullOutputFormat(JobConf conf) {
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
}
+
+
+ @Override
+ public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+ UserGroupInformation ugi =
+ UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+ if(ugi == null) {
+ ugi = UserGroupInformation.login(conf);
+ }
+ return ugi;
+ }
}
@@ -23,6 +23,7 @@
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.List;
+import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -48,6 +49,8 @@
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.tools.HadoopArchives;
import org.apache.hadoop.util.ToolRunner;
@@ -436,4 +439,14 @@ public void setNullOutputFormat(JobConf conf) {
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
}
+
+ @Override
+ public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+ UserGroupInformation ugi =
+ UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+ if(ugi == null) {
+ ugi = UserGroupInformation.login(conf);
+ }
+ return ugi;
+ }
}
Oops, something went wrong.

0 comments on commit 36e9d61

Please sign in to comment.