conf/core-site.xml.template conf/hdfs-site.xml.template conf/mapred-site.xml.template src/c++/libhdfs/tests/conf/core-site.xml src/c++/libhdfs/tests/conf/hdfs-site.xml src/c++/libhdfs/tests/conf/mapred-site.xml src/contrib/test/core-site.xml src/contrib/test/hdfs-site.xml src/contrib/test/mapred-site.xml src/core/core-default.xml src/hdfs/hdfs-default.xml src/mapred/mapred-default.xml src/test/core-site.xml src/test/hdfs-site.xml src/test/mapred-site.xml @@ -23,6 +23,9 @@ conf/masters conf/slaves conf/hadoop-env.sh conf/hadoop-site.xml +conf/core-site.xml +conf/mapred-site.xml +conf/hdfs-site.xml conf/hadoop-policy.xml conf/capacity-scheduler.xml docs/api/ .gitignore @@ -54,6 +54,9 @@ Trunk (unreleased changes) percentage in capacity scheduler UI. (Sreekanth Ramakrishnan via yhemanth) + HADOOP-4631. Splits the configuration into three parts - one for core, + one for mapred and the last one for HDFS. (Sharad Agarwal via cdouglas) + NEW FEATURES HADOOP-4575. Add a proxy service for relaying HsftpFileSystem requests. CHANGES.txt @@ -342,6 +342,7 @@ <copy todir="${build.classes}"> <fileset dir="${core.src.dir}" includes="**/*.properties"/> + <fileset dir="${core.src.dir}" includes="core-default.xml"/> </copy> </target> @@ -378,6 +379,7 @@ <copy todir="${build.classes}"> <fileset dir="${mapred.src.dir}" includes="**/*.properties"/> + <fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/> </copy> </target> @@ -413,6 +415,7 @@ <copy todir="${build.classes}"> <fileset dir="${hdfs.src.dir}" includes="**/*.properties"/> + <fileset dir="${hdfs.src.dir}" includes="hdfs-default.xml"/> </copy> </target> @@ -547,7 +550,6 @@ <attribute name="Implementation-Vendor" value="Apache"/> </section> </manifest> - <fileset file="${conf.dir}/hadoop-default.xml"/> <fileset file="${conf.dir}/commons-logging.properties"/> <fileset file="${conf.dir}/log4j.properties"/> <fileset file="${conf.dir}/hadoop-metrics.properties"/> @@ -860,8 +862,12 @@ <copy todir="${docs.dir}"> <fileset dir="${docs.src}/build/site/" /> </copy> - <style basedir="${conf.dir}" destdir="${docs.dir}" - includes="hadoop-default.xml" style="conf/configuration.xsl"/> + <style basedir="${core.src.dir}" destdir="${docs.dir}" + includes="core-default.xml" style="conf/configuration.xsl"/> + <style basedir="${hdfs.src.dir}" destdir="${docs.dir}" + includes="hdfs-default.xml" style="conf/configuration.xsl"/> + <style basedir="${mapred.src.dir}" destdir="${docs.dir}" + includes="mapred-default.xml" style="conf/configuration.xsl"/> <antcall target="changes-to-html"/> <antcall target="cn-docs"/> </target> @@ -876,8 +882,12 @@ <copy todir="${cndocs.dir}"> <fileset dir="${cndocs.src}/build/site/" /> </copy> - <style basedir="${conf.dir}" destdir="${cndocs.dir}" - includes="hadoop-default.xml" style="conf/configuration.xsl"/> + <style basedir="${core.src.dir}" destdir="${cndocs.dir}" + includes="core-default.xml" style="conf/configuration.xsl"/> + <style basedir="${hdfs.src.dir}" destdir="${cndocs.dir}" + includes="hdfs-default.xml" style="conf/configuration.xsl"/> + <style basedir="${mapred.src.dir}" destdir="${cndocs.dir}" + includes="mapred-default.xml" style="conf/configuration.xsl"/> <antcall target="changes-to-html"/> </target> build.xml @@ -7,27 +7,5 @@ <configuration> -<property> - <name>hadoop.tmp.dir</name> - <value>build/test/libhdfs</value> - <description>A base for other temporary directories.</description> -</property> - - -<property> - <name>fs.default.name</name> - <value>localhost:23000</value> - <description>The name of the default file system. Either the - literal string "local" or a host:port for DFS.</description> -</property> - -<property> - <name>dfs.replication</name> - <value>1</value> - <description>Default block replication. - The actual number of replications can be specified when the file is created. - The default is used if replication is not specified in create time. - </description> -</property> </configuration> src/c++/libhdfs/tests/conf/hadoop-site.xml @@ -8,16 +8,5 @@ <configuration> -<property> - <name>hadoop.tmp.dir</name> - <value>${build.test}</value> - <description>A base for other temporary directories.</description> -</property> - -<property> - <name>mapred.system.dir</name> - <value>build/contrib/${contrib.name}/test/system</value> -</property> - </configuration> src/contrib/test/hadoop-site.xml @@ -41,6 +41,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.StringTokenizer; +import java.util.WeakHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -81,9 +82,9 @@ import org.xml.sax.SAXException; * * <p>Unless explicitly turned off, Hadoop by default specifies two * resources, loaded in-order from the classpath: <ol> - * <li><tt><a href="{@docRoot}/../hadoop-default.html">hadoop-default.xml</a> + * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a> * </tt>: Read-only defaults for hadoop.</li> - * <li><tt>hadoop-site.xml</tt>: Site-specific configuration for a given hadoop + * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop * installation.</li> * </ol> * Applications may add additional resources, which are loaded @@ -103,7 +104,7 @@ import org.xml.sax.SAXException; * &lt;/property&gt;</pre></tt> * * Administrators typically define parameters as final in - * <tt>hadoop-site.xml</tt> for values that user applications may not alter. + * <tt>core-site.xml</tt> for values that user applications may not alter. * * <h4 id="VariableExpansion">Variable Expansion</h4> * @@ -149,6 +150,38 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, */ private Set<String> finalParameters = new HashSet<String>(); + private boolean loadDefaults = true; + + /** + * Configurtion objects + */ + private static final WeakHashMap<Configuration,Object> REGISTRY = + new WeakHashMap<Configuration,Object>(); + + /** + * List of default Resources. Resources are loaded in the order of the list + * entries + */ + private static final ArrayList<String> defaultResources = + new ArrayList<String>(); + + static{ + //print deprecation warning if hadoop-site.xml is found in classpath + ClassLoader cL = Thread.currentThread().getContextClassLoader(); + if (cL == null) { + cL = Configuration.class.getClassLoader(); + } + if(cL.getResource("hadoop-site.xml")!=null) { + LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + + "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " + + "mapred-site.xml and hdfs-site.xml to override properties of " + + "core-default.xml, mapred-default.xml and hdfs-default.xml " + + "respectively"); + } + addDefaultResource("core-default.xml"); + addDefaultResource("core-site.xml"); + } + private Properties properties; private Properties overlay; private ClassLoader classLoader; @@ -172,12 +205,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, * @param loadDefaults specifies whether to load from the default files */ public Configuration(boolean loadDefaults) { + this.loadDefaults = loadDefaults; if (LOG.isDebugEnabled()) { LOG.debug(StringUtils.stringifyException(new IOException("config()"))); } - if (loadDefaults) { - resources.add("hadoop-default.xml"); - resources.add("hadoop-site.xml"); + synchronized(Configuration.class) { + REGISTRY.put(this, null); } } @@ -205,6 +238,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, } this.finalParameters = new HashSet<String>(other.finalParameters); + synchronized(Configuration.class) { + REGISTRY.put(this, null); + } + } + + /** + * Add a default resource. Resources are loaded in the order of the resources + * added. + * @param name file name. File should be present in the classpath. + */ + public static synchronized void addDefaultResource(String name) { + if(!defaultResources.contains(name)) { + defaultResources.add(name); + for(Configuration conf : REGISTRY.keySet()) { + if(conf.loadDefaults) { + conf.reloadConfiguration(); + } + } + } } /** @@ -972,6 +1024,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, private void loadResources(Properties properties, ArrayList resources, boolean quiet) { + if(loadDefaults) { + for (String resource : defaultResources) { + loadResource(properties, resource, quiet); + } + + //support the hadoop-site.xml as a deprecated case + if(getResource("hadoop-site.xml")!=null) { + loadResource(properties, "hadoop-site.xml", quiet); + } + } + for (Object resource : resources) { loadResource(properties, resource, quiet); } @@ -1158,6 +1221,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, public String toString() { StringBuffer sb = new StringBuffer(); sb.append("Configuration: "); + if(loadDefaults) { + toString(defaultResources, sb); + if(resources.size()>0) { + sb.append(", "); + } + } toString(resources, sb); return sb.toString(); } src/core/org/apache/hadoop/conf/Configuration.java @@ -1256,9 +1256,8 @@ public class FsShell extends Configured implements Tool { String fs = "-fs [local | <file system URI>]: \tSpecify the file system to use.\n" + "\t\tIf not specified, the current configuration is used, \n" + "\t\ttaken from the following, in increasing precedence: \n" + - "\t\t\thadoop-default.xml inside the hadoop jar file \n" + - "\t\t\thadoop-default.xml in $HADOOP_CONF_DIR \n" + - "\t\t\thadoop-site.xml in $HADOOP_CONF_DIR \n" + + "\t\t\tcore-default.xml inside the hadoop jar file \n" + + "\t\t\tcore-site.xml in $HADOOP_CONF_DIR \n" + "\t\t'local' means use the local file system as your DFS. \n" + "\t\t<file system URI> specifies a particular file system to \n" + "\t\tcontact. This argument is optional but if used must appear\n" + src/core/org/apache/hadoop/fs/FsShell.java @@ -49,6 +49,11 @@ public class DistributedFileSystem extends FileSystem { DFSClient dfs; private boolean verifyChecksum = true; + + static{ + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } public DistributedFileSystem() { } @@ -449,4 +454,6 @@ public class DistributedFileSystem extends FileSystem { ) throws IOException { dfs.setTimes(getPathName(p), mtime, atime); } + + } src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -126,6 +126,11 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; public class DataNode extends Configured implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants, Runnable { public static final Log LOG = LogFactory.getLog(DataNode.class); + + static{ + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } public static final String DN_CLIENTTRACE_FORMAT = "src: %s" + // src IP src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -97,6 +97,11 @@ import java.util.Iterator; public class NameNode implements ClientProtocol, DatanodeProtocol, NamenodeProtocol, FSConstants, RefreshAuthorizationPolicyProtocol { + static{ + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -161,6 +161,11 @@ public class JobClient extends Configured implements MRConstants, Tool { private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; private static final long MAX_JOBPROFILE_AGE = 1000 * 2; + static{ + Configuration.addDefaultResource("mapred-default.xml"); + Configuration.addDefaultResource("mapred-site.xml"); + } + /** * A NetworkedJob is an implementation of RunningJob. It holds * a JobProfile object to provide some info, and interacts with the src/mapred/org/apache/hadoop/mapred/JobClient.java @@ -106,6 +106,11 @@ public class JobConf extends Configuration { private static final Log LOG = LogFactory.getLog(JobConf.class); + static{ + Configuration.addDefaultResource("mapred-default.xml"); + Configuration.addDefaultResource("mapred-site.xml"); + } + /** * A value which if set for memory related configuration options, * indicates that the options are turned off. src/mapred/org/apache/hadoop/mapred/JobConf.java @@ -86,6 +86,11 @@ import org.apache.hadoop.util.VersionInfo; public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmissionProtocol, TaskTrackerManager, RefreshAuthorizationPolicyProtocol { + static{ + Configuration.addDefaultResource("mapred-default.xml"); + Configuration.addDefaultResource("mapred-site.xml"); + } + static long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000; static long RETIRE_JOB_INTERVAL; static long RETIRE_JOB_CHECK_INTERVAL; src/mapred/org/apache/hadoop/mapred/JobTracker.java @@ -101,6 +101,11 @@ public class TaskTracker static enum State {NORMAL, STALE, INTERRUPTED, DENIED} + static{ + Configuration.addDefaultResource("mapred-default.xml"); + Configuration.addDefaultResource("mapred-site.xml"); + } + public static final Log LOG = LogFactory.getLog(TaskTracker.class); src/mapred/org/apache/hadoop/mapred/TaskTracker.java @@ -8,43 +8,5 @@ <configuration> -<property> - <name>hadoop.tmp.dir</name> - <value>build/test</value> - <description>A base for other temporary directories.</description> - <final>true</final> -</property> - -<property> - <name>test.fs.s3.name</name> - <value>s3:///</value> - <description>The name of the s3 file system for testing.</description> -</property> - -<property> - <name>fs.s3.block.size</name> - <value>128</value> - <description>Size of a block in bytes.</description> -</property> - -<property> - <name>fs.ftp.user.localhost</name> - <value>user</value> - <description>The username for connecting to FTP server running on localhost. - This is required by FTPFileSystem</description> -</property> - -<property> - <name>fs.ftp.password.localhost</name> - <value>password</value> - <description>The password for connecting to FTP server running on localhost. - This is required by FTPFileSystem</description> -</property> - -<property> - <name>test.fs.s3n.name</name> - <value>s3n:///</value> - <description>The name of the s3n file system for testing.</description> -</property> </configuration> src/test/hadoop-site.xml @@ -224,7 +224,7 @@ public class TestConfiguration extends TestCase { conf.addResource(fileResource); String expectedOutput = - "Configuration: hadoop-default.xml, hadoop-site.xml, " + + "Configuration: core-default.xml, core-site.xml, " + fileResource.toString(); assertEquals(expectedOutput, conf.toString()); } src/test/org/apache/hadoop/conf/TestConfiguration.java @@ -60,10 +60,10 @@ public class TestDFSShellGenericOptions extends TestCase { } private void testConfOption(String[] args, String namenode) { - // prepare configuration hadoop-site.xml + // prepare configuration hdfs-site.xml File configDir = new File(new File("build", "test"), "minidfs"); assertTrue(configDir.mkdirs()); - File siteFile = new File(configDir, "hadoop-site.xml"); + File siteFile = new File(configDir, "hdfs-site.xml"); PrintWriter pw; try { pw = new PrintWriter(siteFile); src/test/org/apache/hadoop/hdfs/TestDFSShellGenericOptions.java @@ -606,7 +606,7 @@ public class MiniMRCluster { stopJobTracker(); } finally { File configDir = new File("build", "minimr"); - File siteFile = new File(configDir, "hadoop-site.xml"); + File siteFile = new File(configDir, "mapred-site.xml"); siteFile.delete(); } } src/test/org/apache/hadoop/mapred/MiniMRCluster.java conf/hadoop-default.xml conf/hadoop-site.xml.template daa2257dd9ae39daeec43a4d214aaae981e0e08d Christopher Douglas cdouglas@apache.org http://github.com/yahoo/hadoop/commit/7efc2338c82771011e153d63997e977d6b797be5 7efc2338c82771011e153d63997e977d6b797be5 2008-12-15T16:05:03-08:00 2008-12-15T16:05:03-08:00 HADOOP-4631. Splits the configuration into three parts - one for core, one for mapred and the last one for HDFS. Contributed by Sharad Agarwal. git-svn-id: https://svn.eu.apache.org/repos/asf/hadoop/core/trunk@726883 13f79535-47bb-0310-9956-ffa450edef68 a8f3baf4e964f6dd24eba722ea3665e31b165bed Christopher Douglas cdouglas@apache.org