Browse files

HADOOP-4631. Splits the configuration into three parts - one for core,

one for mapred and the last one for HDFS. Contributed by Sharad Agarwal.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@726883 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent eb8cecb commit 3baf0757d95e546589ce1ee2617de0930f7b76ab @cdouglas cdouglas committed Dec 16, 2008
Showing with 1,117 additions and 865 deletions.
  1. +3 −0 .gitignore
  2. +3 −0 CHANGES.txt
  3. +15 −5 build.xml
  4. 0 conf/{hadoop-site.xml.template → core-site.xml.template}
  5. +8 −0 conf/hdfs-site.xml.template
  6. +8 −0 conf/mapred-site.xml.template
  7. +24 −0 src/c++/libhdfs/tests/conf/core-site.xml
  8. +0 −22 src/c++/libhdfs/tests/conf/hadoop-site.xml
  9. +17 −0 src/c++/libhdfs/tests/conf/hdfs-site.xml
  10. +8 −0 src/c++/libhdfs/tests/conf/mapred-site.xml
  11. +16 −0 src/contrib/test/core-site.xml
  12. +0 −11 src/contrib/test/hadoop-site.xml
  13. +9 −0 src/contrib/test/hdfs-site.xml
  14. +13 −0 src/contrib/test/mapred-site.xml
  15. +434 −0 src/core/core-default.xml
  16. +75 −6 src/core/org/apache/hadoop/conf/Configuration.java
  17. +2 −3 src/core/org/apache/hadoop/fs/FsShell.java
  18. +356 −0 src/hdfs/hdfs-default.xml
  19. +7 −0 src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
  20. +5 −0 src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
  21. +5 −0 src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
  22. +18 −776 conf/hadoop-default.xml → src/mapred/mapred-default.xml
  23. +5 −0 src/mapred/org/apache/hadoop/mapred/JobClient.java
  24. +5 −0 src/mapred/org/apache/hadoop/mapred/JobConf.java
  25. +5 −0 src/mapred/org/apache/hadoop/mapred/JobTracker.java
  26. +5 −0 src/mapred/org/apache/hadoop/mapred/TaskTracker.java
  27. +50 −0 src/test/core-site.xml
  28. +0 −38 src/test/hadoop-site.xml
  29. +9 −0 src/test/hdfs-site.xml
  30. +8 −0 src/test/mapred-site.xml
  31. +1 −1 src/test/org/apache/hadoop/conf/TestConfiguration.java
  32. +2 −2 src/test/org/apache/hadoop/hdfs/TestDFSShellGenericOptions.java
  33. +1 −1 src/test/org/apache/hadoop/mapred/MiniMRCluster.java
View
3 .gitignore
@@ -23,6 +23,9 @@ conf/masters
conf/slaves
conf/hadoop-env.sh
conf/hadoop-site.xml
+conf/core-site.xml
+conf/mapred-site.xml
+conf/hdfs-site.xml
conf/hadoop-policy.xml
conf/capacity-scheduler.xml
docs/api/
View
3 CHANGES.txt
@@ -54,6 +54,9 @@ Trunk (unreleased changes)
percentage in capacity scheduler UI. (Sreekanth Ramakrishnan via
yhemanth)
+ HADOOP-4631. Splits the configuration into three parts - one for core,
+ one for mapred and the last one for HDFS. (Sharad Agarwal via cdouglas)
+
NEW FEATURES
HADOOP-4575. Add a proxy service for relaying HsftpFileSystem requests.
View
20 build.xml
@@ -342,6 +342,7 @@
<copy todir="${build.classes}">
<fileset dir="${core.src.dir}" includes="**/*.properties"/>
+ <fileset dir="${core.src.dir}" includes="core-default.xml"/>
</copy>
</target>
@@ -378,6 +379,7 @@
<copy todir="${build.classes}">
<fileset dir="${mapred.src.dir}" includes="**/*.properties"/>
+ <fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/>
</copy>
</target>
@@ -413,6 +415,7 @@
<copy todir="${build.classes}">
<fileset dir="${hdfs.src.dir}" includes="**/*.properties"/>
+ <fileset dir="${hdfs.src.dir}" includes="hdfs-default.xml"/>
</copy>
</target>
@@ -547,7 +550,6 @@
<attribute name="Implementation-Vendor" value="Apache"/>
</section>
</manifest>
- <fileset file="${conf.dir}/hadoop-default.xml"/>
<fileset file="${conf.dir}/commons-logging.properties"/>
<fileset file="${conf.dir}/log4j.properties"/>
<fileset file="${conf.dir}/hadoop-metrics.properties"/>
@@ -860,8 +862,12 @@
<copy todir="${docs.dir}">
<fileset dir="${docs.src}/build/site/" />
</copy>
- <style basedir="${conf.dir}" destdir="${docs.dir}"
- includes="hadoop-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${core.src.dir}" destdir="${docs.dir}"
+ includes="core-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${hdfs.src.dir}" destdir="${docs.dir}"
+ includes="hdfs-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${mapred.src.dir}" destdir="${docs.dir}"
+ includes="mapred-default.xml" style="conf/configuration.xsl"/>
<antcall target="changes-to-html"/>
<antcall target="cn-docs"/>
</target>
@@ -876,8 +882,12 @@
<copy todir="${cndocs.dir}">
<fileset dir="${cndocs.src}/build/site/" />
</copy>
- <style basedir="${conf.dir}" destdir="${cndocs.dir}"
- includes="hadoop-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${core.src.dir}" destdir="${cndocs.dir}"
+ includes="core-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${hdfs.src.dir}" destdir="${cndocs.dir}"
+ includes="hdfs-default.xml" style="conf/configuration.xsl"/>
+ <style basedir="${mapred.src.dir}" destdir="${cndocs.dir}"
+ includes="mapred-default.xml" style="conf/configuration.xsl"/>
<antcall target="changes-to-html"/>
</target>
View
0 conf/hadoop-site.xml.template → conf/core-site.xml.template
File renamed without changes.
View
8 conf/hdfs-site.xml.template
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
View
8 conf/mapred-site.xml.template
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
View
24 src/c++/libhdfs/tests/conf/core-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running libhdfs unit tests. -->
+<!-- This is mostly empty, to use the default values, overriding the -->
+<!-- potentially user-editted core-site.xml in the conf/ directory. -->
+
+<configuration>
+
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>build/test/libhdfs</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+
+<property>
+ <name>fs.default.name</name>
+ <value>localhost:23000</value>
+ <description>The name of the default file system. Either the
+ literal string "local" or a host:port for DFS.</description>
+</property>
+
+</configuration>
View
22 src/c++/libhdfs/tests/conf/hadoop-site.xml
@@ -7,27 +7,5 @@
<configuration>
-<property>
- <name>hadoop.tmp.dir</name>
- <value>build/test/libhdfs</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-
-<property>
- <name>fs.default.name</name>
- <value>localhost:23000</value>
- <description>The name of the default file system. Either the
- literal string "local" or a host:port for DFS.</description>
-</property>
-
-<property>
- <name>dfs.replication</name>
- <value>1</value>
- <description>Default block replication.
- The actual number of replications can be specified when the file is created.
- The default is used if replication is not specified in create time.
- </description>
-</property>
</configuration>
View
17 src/c++/libhdfs/tests/conf/hdfs-site.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ <description>Default block replication.
+ The actual number of replications can be specified when the file is created.
+ The default is used if replication is not specified in create time.
+ </description>
+</property>
+
+</configuration>
View
8 src/c++/libhdfs/tests/conf/mapred-site.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
View
16 src/contrib/test/core-site.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running unit tests. This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory. -->
+
+<configuration>
+
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>${build.test}</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+</configuration>
View
11 src/contrib/test/hadoop-site.xml
@@ -8,16 +8,5 @@
<configuration>
-<property>
- <name>hadoop.tmp.dir</name>
- <value>${build.test}</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-<property>
- <name>mapred.system.dir</name>
- <value>build/contrib/${contrib.name}/test/system</value>
-</property>
-
</configuration>
View
9 src/contrib/test/hdfs-site.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+
+</configuration>
View
13 src/contrib/test/mapred-site.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+ <name>mapred.system.dir</name>
+ <value>build/contrib/${contrib.name}/test/system</value>
+</property>
+
+</configuration>
View
434 src/core/core-default.xml
@@ -0,0 +1,434 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into core-site.xml and change them -->
+<!-- there. If core-site.xml does not already exist, create it. -->
+
+<configuration>
+
+<!--- global properties -->
+
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>/tmp/hadoop-${user.name}</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+<property>
+ <name>hadoop.native.lib</name>
+ <value>true</value>
+ <description>Should native hadoop libraries, if present, be used.</description>
+</property>
+
+<property>
+ <name>hadoop.http.filter.initializers</name>
+ <value></value>
+ <description>A comma separated list of class names. Each class in the list
+ must extend org.apache.hadoop.http.FilterInitializer. The corresponding
+ Filter will be initialized. Then, the Filter will be applied to all user
+ facing jsp and servlet web pages. The ordering of the list defines the
+ ordering of the filters.</description>
+</property>
+
+<property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ <description>Is service-level authorization enabled?</description>
+</property>
+
+<!--- logging properties -->
+
+<property>
+ <name>hadoop.logfile.size</name>
+ <value>10000000</value>
+ <description>The max size of each log file</description>
+</property>
+
+<property>
+ <name>hadoop.logfile.count</name>
+ <value>10</value>
+ <description>The max number of log files</description>
+</property>
+
+<!-- i/o properties -->
+<property>
+ <name>io.file.buffer.size</name>
+ <value>4096</value>
+ <description>The size of buffer for use in sequence files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+</property>
+
+<property>
+ <name>io.bytes.per.checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ io.file.buffer.size.</description>
+</property>
+
+<property>
+ <name>io.skip.checksum.errors</name>
+ <value>false</value>
+ <description>If true, when a checksum error is encountered while
+ reading a sequence file, entries are skipped, instead of throwing an
+ exception.</description>
+</property>
+
+<property>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
+ <description>A list of the compression codec classes that can be used
+ for compression/decompression.</description>
+</property>
+
+<property>
+ <name>io.serializations</name>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ <description>A list of serialization classes that can be used for
+ obtaining serializers and deserializers.</description>
+</property>
+
+<!-- file system properties -->
+
+<property>
+ <name>fs.default.name</name>
+ <value>file:///</value>
+ <description>The name of the default file system. A URI whose
+ scheme and authority determine the FileSystem implementation. The
+ uri's scheme determines the config property (fs.SCHEME.impl) naming
+ the FileSystem implementation class. The uri's authority is used to
+ determine the host, port, etc. for a filesystem.</description>
+</property>
+
+<property>
+ <name>fs.trash.interval</name>
+ <value>0</value>
+ <description>Number of minutes between trash checkpoints.
+ If zero, the trash feature is disabled.
+ </description>
+</property>
+
+<property>
+ <name>fs.file.impl</name>
+ <value>org.apache.hadoop.fs.LocalFileSystem</value>
+ <description>The FileSystem for file: uris.</description>
+</property>
+
+<property>
+ <name>fs.hdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+ <description>The FileSystem for hdfs: uris.</description>
+</property>
+
+<property>
+ <name>fs.s3.impl</name>
+ <value>org.apache.hadoop.fs.s3.S3FileSystem</value>
+ <description>The FileSystem for s3: uris.</description>
+</property>
+
+<property>
+ <name>fs.s3n.impl</name>
+ <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
+ <description>The FileSystem for s3n: (Native S3) uris.</description>
+</property>
+
+<property>
+ <name>fs.kfs.impl</name>
+ <value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value>
+ <description>The FileSystem for kfs: uris.</description>
+</property>
+
+<property>
+ <name>fs.hftp.impl</name>
+ <value>org.apache.hadoop.hdfs.HftpFileSystem</value>
+</property>
+
+<property>
+ <name>fs.hsftp.impl</name>
+ <value>org.apache.hadoop.hdfs.HsftpFileSystem</value>
+</property>
+
+<property>
+ <name>fs.ftp.impl</name>
+ <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
+ <description>The FileSystem for ftp: uris.</description>
+</property>
+
+<property>
+ <name>fs.ramfs.impl</name>
+ <value>org.apache.hadoop.fs.InMemoryFileSystem</value>
+ <description>The FileSystem for ramfs: uris.</description>
+</property>
+
+<property>
+ <name>fs.har.impl</name>
+ <value>org.apache.hadoop.fs.HarFileSystem</value>
+ <description>The filesystem for Hadoop archives. </description>
+</property>
+
+<property>
+ <name>fs.checkpoint.dir</name>
+ <value>${hadoop.tmp.dir}/dfs/namesecondary</value>
+ <description>Determines where on the local filesystem the DFS secondary
+ name node should store the temporary images to merge.
+ If this is a comma-delimited list of directories then the image is
+ replicated in all of the directories for redundancy.
+ </description>
+</property>
+
+<property>
+ <name>fs.checkpoint.edits.dir</name>
+ <value>${fs.checkpoint.dir}</value>
+ <description>Determines where on the local filesystem the DFS secondary
+ name node should store the temporary edits to merge.
+ If this is a comma-delimited list of directoires then teh edits is
+ replicated in all of the directoires for redundancy.
+ Default value is same as fs.checkpoint.dir
+ </description>
+</property>
+
+<property>
+ <name>fs.checkpoint.period</name>
+ <value>3600</value>
+ <description>The number of seconds between two periodic checkpoints.
+ </description>
+</property>
+
+<property>
+ <name>fs.checkpoint.size</name>
+ <value>67108864</value>
+ <description>The size of the current edit log (in bytes) that triggers
+ a periodic checkpoint even if the fs.checkpoint.period hasn't expired.
+ </description>
+</property>
+
+
+
+<property>
+ <name>fs.s3.block.size</name>
+ <value>67108864</value>
+ <description>Block size to use when writing files to S3.</description>
+</property>
+
+<property>
+ <name>fs.s3.buffer.dir</name>
+ <value>${hadoop.tmp.dir}/s3</value>
+ <description>Determines where on the local filesystem the S3 filesystem
+ should store files before sending them to S3
+ (or after retrieving them from S3).
+ </description>
+</property>
+
+<property>
+ <name>fs.s3.maxRetries</name>
+ <value>4</value>
+ <description>The maximum number of retries for reading or writing files to S3,
+ before we signal failure to the application.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3.sleepTimeSeconds</name>
+ <value>10</value>
+ <description>The number of seconds to sleep between each S3 retry.
+ </description>
+</property>
+
+
+<property>
+ <name>local.cache.size</name>
+ <value>10737418240</value>
+ <description>The limit on the size of cache you want to keep, set by default
+ to 10GB. This will act as a soft limit on the cache directory for out of band data.
+ </description>
+</property>
+
+<property>
+ <name>io.seqfile.compress.blocksize</name>
+ <value>1000000</value>
+ <description>The minimum block size for compression in block compressed
+ SequenceFiles.
+ </description>
+</property>
+
+<property>
+ <name>io.seqfile.lazydecompress</name>
+ <value>true</value>
+ <description>Should values of block-compressed SequenceFiles be decompressed
+ only when necessary.
+ </description>
+</property>
+
+<property>
+ <name>io.seqfile.sorter.recordlimit</name>
+ <value>1000000</value>
+ <description>The limit on number of records to be kept in memory in a spill
+ in SequenceFiles.Sorter
+ </description>
+</property>
+
+ <property>
+ <name>io.mapfile.bloom.size</name>
+ <value>1048576</value>
+ <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
+ keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
+ Larger values minimize the number of filters, which slightly increases the performance,
+ but may waste too much space if the total number of keys is usually much smaller
+ than this number.
+ </description>
+</property>
+
+<property>
+ <name>io.mapfile.bloom.error.rate</name>
+ <value>0.005</value>
+ <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
+ As this value decreases, the size of BloomFilter-s increases exponentially. This
+ value is the probability of encountering false positives (default is 0.5%).
+ </description>
+</property>
+
+<property>
+ <name>hadoop.util.hash.type</name>
+ <value>murmur</value>
+ <description>The default implementation of Hash. Currently this can take one of the
+ two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
+ </description>
+</property>
+
+
+<!-- ipc properties -->
+
+<property>
+ <name>ipc.client.idlethreshold</name>
+ <value>4000</value>
+ <description>Defines the threshold number of connections after which
+ connections will be inspected for idleness.
+ </description>
+</property>
+
+<property>
+ <name>ipc.client.kill.max</name>
+ <value>10</value>
+ <description>Defines the maximum number of clients to disconnect in one go.
+ </description>
+</property>
+
+<property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>10000</value>
+ <description>The maximum time in msec after which a client will bring down the
+ connection to the server.
+ </description>
+</property>
+
+<property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>10</value>
+ <description>Indicates the number of retries a client will make to establish
+ a server connection.
+ </description>
+</property>
+
+<property>
+ <name>ipc.server.listen.queue.size</name>
+ <value>128</value>
+ <description>Indicates the length of the listen queue for servers accepting
+ client connections.
+ </description>
+</property>
+
+<property>
+ <name>ipc.server.tcpnodelay</name>
+ <value>false</value>
+ <description>Turn on/off Nagle's algorithm for the TCP socket connection on
+ the server. Setting to true disables the algorithm and may decrease latency
+ with a cost of more/smaller packets.
+ </description>
+</property>
+
+<property>
+ <name>ipc.client.tcpnodelay</name>
+ <value>false</value>
+ <description>Turn on/off Nagle's algorithm for the TCP socket connection on
+ the client. Setting to true disables the algorithm and may decrease latency
+ with a cost of more/smaller packets.
+ </description>
+</property>
+
+
+<!-- Web Interface Configuration -->
+
+<property>
+ <name>webinterface.private.actions</name>
+ <value>false</value>
+ <description> If set to true, the web interfaces of JT and NN may contain
+ actions, such as kill job, delete file, etc., that should
+ not be exposed to public. Enable this option if the interfaces
+ are only reachable by those who have the right authorization.
+ </description>
+</property>
+
+<!-- Proxy Configuration -->
+
+<property>
+ <name>hadoop.rpc.socket.factory.class.default</name>
+ <value>org.apache.hadoop.net.StandardSocketFactory</value>
+ <description> Default SocketFactory to use. This parameter is expected to be
+ formatted as "package.FactoryClassName".
+ </description>
+</property>
+
+<property>
+ <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
+ <value></value>
+ <description> SocketFactory to use to connect to a DFS. If null or empty, use
+ hadoop.rpc.socket.class.default. This socket factory is also used by
+ DFSClient to create sockets to DataNodes.
+ </description>
+</property>
+
+
+
+<property>
+ <name>hadoop.socks.server</name>
+ <value></value>
+ <description> Address (host:port) of the SOCKS server to be used by the
+ SocksSocketFactory.
+ </description>
+</property>
+
+<!-- Rack Configuration -->
+
+<property>
+ <name>topology.node.switch.mapping.impl</name>
+ <value>org.apache.hadoop.net.ScriptBasedMapping</value>
+ <description> The default implementation of the DNSToSwitchMapping. It
+ invokes a script specified in topology.script.file.name to resolve
+ node names. If the value for topology.script.file.name is not set, the
+ default value of DEFAULT_RACK is returned for all node names.
+ </description>
+</property>
+
+<property>
+ <name>topology.script.file.name</name>
+ <value></value>
+ <description> The script name that should be invoked to resolve DNS names to
+ NetworkTopology names. Example: the script would take host.foo.bar as an
+ argument, and return /rack1 as the output.
+ </description>
+</property>
+
+<property>
+ <name>topology.script.number.args</name>
+ <value>100</value>
+ <description> The max number of args that the script configured with
+ topology.script.file.name should be run with. Each arg is an
+ IP address.
+ </description>
+</property>
+
+
+
+</configuration>
View
81 src/core/org/apache/hadoop/conf/Configuration.java
@@ -41,6 +41,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
+import java.util.WeakHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -81,9 +82,9 @@
*
* <p>Unless explicitly turned off, Hadoop by default specifies two
* resources, loaded in-order from the classpath: <ol>
- * <li><tt><a href="{@docRoot}/../hadoop-default.html">hadoop-default.xml</a>
+ * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
* </tt>: Read-only defaults for hadoop.</li>
- * <li><tt>hadoop-site.xml</tt>: Site-specific configuration for a given hadoop
+ * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
* installation.</li>
* </ol>
* Applications may add additional resources, which are loaded
@@ -103,7 +104,7 @@
* &lt;/property&gt;</pre></tt>
*
* Administrators typically define parameters as final in
- * <tt>hadoop-site.xml</tt> for values that user applications may not alter.
+ * <tt>core-site.xml</tt> for values that user applications may not alter.
*
* <h4 id="VariableExpansion">Variable Expansion</h4>
*
@@ -149,6 +150,38 @@
*/
private Set<String> finalParameters = new HashSet<String>();
+ private boolean loadDefaults = true;
+
+ /**
+ * Configurtion objects
+ */
+ private static final WeakHashMap<Configuration,Object> REGISTRY =
+ new WeakHashMap<Configuration,Object>();
+
+ /**
+ * List of default Resources. Resources are loaded in the order of the list
+ * entries
+ */
+ private static final ArrayList<String> defaultResources =
+ new ArrayList<String>();
+
+ static{
+ //print deprecation warning if hadoop-site.xml is found in classpath
+ ClassLoader cL = Thread.currentThread().getContextClassLoader();
+ if (cL == null) {
+ cL = Configuration.class.getClassLoader();
+ }
+ if(cL.getResource("hadoop-site.xml")!=null) {
+ LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
+ "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
+ + "mapred-site.xml and hdfs-site.xml to override properties of " +
+ "core-default.xml, mapred-default.xml and hdfs-default.xml " +
+ "respectively");
+ }
+ addDefaultResource("core-default.xml");
+ addDefaultResource("core-site.xml");
+ }
+
private Properties properties;
private Properties overlay;
private ClassLoader classLoader;
@@ -172,12 +205,12 @@ public Configuration() {
* @param loadDefaults specifies whether to load from the default files
*/
public Configuration(boolean loadDefaults) {
+ this.loadDefaults = loadDefaults;
if (LOG.isDebugEnabled()) {
LOG.debug(StringUtils.stringifyException(new IOException("config()")));
}
- if (loadDefaults) {
- resources.add("hadoop-default.xml");
- resources.add("hadoop-site.xml");
+ synchronized(Configuration.class) {
+ REGISTRY.put(this, null);
}
}
@@ -205,6 +238,25 @@ public Configuration(Configuration other) {
}
this.finalParameters = new HashSet<String>(other.finalParameters);
+ synchronized(Configuration.class) {
+ REGISTRY.put(this, null);
+ }
+ }
+
+ /**
+ * Add a default resource. Resources are loaded in the order of the resources
+ * added.
+ * @param name file name. File should be present in the classpath.
+ */
+ public static synchronized void addDefaultResource(String name) {
+ if(!defaultResources.contains(name)) {
+ defaultResources.add(name);
+ for(Configuration conf : REGISTRY.keySet()) {
+ if(conf.loadDefaults) {
+ conf.reloadConfiguration();
+ }
+ }
+ }
}
/**
@@ -972,6 +1024,17 @@ public void clear() {
private void loadResources(Properties properties,
ArrayList resources,
boolean quiet) {
+ if(loadDefaults) {
+ for (String resource : defaultResources) {
+ loadResource(properties, resource, quiet);
+ }
+
+ //support the hadoop-site.xml as a deprecated case
+ if(getResource("hadoop-site.xml")!=null) {
+ loadResource(properties, "hadoop-site.xml", quiet);
+ }
+ }
+
for (Object resource : resources) {
loadResource(properties, resource, quiet);
}
@@ -1158,6 +1221,12 @@ public void setClassLoader(ClassLoader classLoader) {
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("Configuration: ");
+ if(loadDefaults) {
+ toString(defaultResources, sb);
+ if(resources.size()>0) {
+ sb.append(", ");
+ }
+ }
toString(resources, sb);
return sb.toString();
}
View
5 src/core/org/apache/hadoop/fs/FsShell.java
@@ -1256,9 +1256,8 @@ private void printHelp(String cmd) {
String fs = "-fs [local | <file system URI>]: \tSpecify the file system to use.\n" +
"\t\tIf not specified, the current configuration is used, \n" +
"\t\ttaken from the following, in increasing precedence: \n" +
- "\t\t\thadoop-default.xml inside the hadoop jar file \n" +
- "\t\t\thadoop-default.xml in $HADOOP_CONF_DIR \n" +
- "\t\t\thadoop-site.xml in $HADOOP_CONF_DIR \n" +
+ "\t\t\tcore-default.xml inside the hadoop jar file \n" +
+ "\t\t\tcore-site.xml in $HADOOP_CONF_DIR \n" +
"\t\t'local' means use the local file system as your DFS. \n" +
"\t\t<file system URI> specifies a particular file system to \n" +
"\t\tcontact. This argument is optional but if used must appear\n" +
View
356 src/hdfs/hdfs-default.xml
@@ -0,0 +1,356 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into hdfs-site.xml and change them -->
+<!-- there. If hdfs-site.xml does not already exist, create it. -->
+
+<configuration>
+
+<property>
+ <name>dfs.namenode.logging.level</name>
+ <value>info</value>
+ <description>The logging level for dfs namenode. Other values are "dir"(trac
+e namespace mutations), "block"(trace block under/over replications and block
+creations/deletions), or "all".</description>
+</property>
+
+<property>
+ <name>dfs.secondary.http.address</name>
+ <value>0.0.0.0:50090</value>
+ <description>
+ The secondary namenode http server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:50010</value>
+ <description>
+ The address where the datanode server will listen to.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:50075</value>
+ <description>
+ The datanode http server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:50020</value>
+ <description>
+ The datanode ipc server address and port.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.handler.count</name>
+ <value>3</value>
+ <description>The number of server threads for the datanode.</description>
+</property>
+
+<property>
+ <name>dfs.http.address</name>
+ <value>0.0.0.0:50070</value>
+ <description>
+ The address and the base port where the dfs namenode web ui will listen on.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.https.enable</name>
+ <value>false</value>
+ <description>Decide if HTTPS(SSL) is supported on HDFS
+ </description>
+</property>
+
+<property>
+ <name>dfs.https.need.client.auth</name>
+ <value>false</value>
+ <description>Whether SSL client certificate authentication is required
+ </description>
+</property>
+
+<property>
+ <name>dfs.https.server.keystore.resource</name>
+ <value>ssl-server.xml</value>
+ <description>Resource file from which ssl server keystore
+ information will be extracted
+ </description>
+</property>
+
+<property>
+ <name>dfs.https.client.keystore.resource</name>
+ <value>ssl-client.xml</value>
+ <description>Resource file from which ssl client keystore
+ information will be extracted
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.https.address</name>
+ <value>0.0.0.0:50475</value>
+</property>
+
+<property>
+ <name>dfs.https.address</name>
+ <value>0.0.0.0:50470</value>
+</property>
+
+ <property>
+ <name>dfs.datanode.dns.interface</name>
+ <value>default</value>
+ <description>The name of the Network Interface from which a data node should
+ report its IP address.
+ </description>
+ </property>
+
+<property>
+ <name>dfs.datanode.dns.nameserver</name>
+ <value>default</value>
+ <description>The host name or IP address of the name server (DNS)
+ which a DataNode should use to determine the host name used by the
+ NameNode for communication and display purposes.
+ </description>
+ </property>
+
+
+
+<property>
+ <name>dfs.replication.considerLoad</name>
+ <value>true</value>
+ <description>Decide if chooseTarget considers the target's load or not
+ </description>
+</property>
+<property>
+ <name>dfs.default.chunk.view.size</name>
+ <value>32768</value>
+ <description>The number of bytes to view for a file on the browser.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.du.reserved</name>
+ <value>0</value>
+ <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
+ </description>
+</property>
+
+<property>
+ <name>dfs.name.dir</name>
+ <value>${hadoop.tmp.dir}/dfs/name</value>
+ <description>Determines where on the local filesystem the DFS name node
+ should store the name table(fsimage). If this is a comma-delimited list
+ of directories then the name table is replicated in all of the
+ directories, for redundancy. </description>
+</property>
+
+<property>
+ <name>dfs.name.edits.dir</name>
+ <value>${dfs.name.dir}</value>
+ <description>Determines where on the local filesystem the DFS name node
+ should store the transaction (edits) file. If this is a comma-delimited list
+ of directories then the transaction file is replicated in all of the
+ directories, for redundancy. Default value is same as dfs.name.dir
+ </description>
+</property>
+<property>
+ <name>dfs.web.ugi</name>
+ <value>webuser,webgroup</value>
+ <description>The user account used by the web interface.
+ Syntax: USERNAME,GROUP1,GROUP2, ...
+ </description>
+</property>
+
+<property>
+ <name>dfs.permissions</name>
+ <value>true</value>
+ <description>
+ If "true", enable permission checking in HDFS.
+ If "false", permission checking is turned off,
+ but all other behavior is unchanged.
+ Switching from one parameter value to the other does not change the mode,
+ owner or group of files or directories.
+ </description>
+</property>
+
+<property>
+ <name>dfs.permissions.supergroup</name>
+ <value>supergroup</value>
+ <description>The name of the group of super-users.</description>
+</property>
+
+<property>
+ <name>dfs.data.dir</name>
+ <value>${hadoop.tmp.dir}/dfs/data</value>
+ <description>Determines where on the local filesystem an DFS data node
+ should store its blocks. If this is a comma-delimited
+ list of directories, then data will be stored in all named
+ directories, typically on different devices.
+ Directories that do not exist are ignored.
+ </description>
+</property>
+
+<property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ <description>Default block replication.
+ The actual number of replications can be specified when the file is created.
+ The default is used if replication is not specified in create time.
+ </description>
+</property>
+
+<property>
+ <name>dfs.replication.max</name>
+ <value>512</value>
+ <description>Maximal block replication.
+ </description>
+</property>
+
+<property>
+ <name>dfs.replication.min</name>
+ <value>1</value>
+ <description>Minimal block replication.
+ </description>
+</property>
+
+<property>
+ <name>dfs.block.size</name>
+ <value>67108864</value>
+ <description>The default block size for new files.</description>
+</property>
+
+<property>
+ <name>dfs.df.interval</name>
+ <value>60000</value>
+ <description>Disk usage statistics refresh interval in msec.</description>
+</property>
+
+<property>
+ <name>dfs.client.block.write.retries</name>
+ <value>3</value>
+ <description>The number of retries for writing blocks to the data nodes,
+ before we signal failure to the application.
+ </description>
+</property>
+
+<property>
+ <name>dfs.blockreport.intervalMsec</name>
+ <value>3600000</value>
+ <description>Determines block reporting interval in milliseconds.</description>
+</property>
+
+<property>
+ <name>dfs.blockreport.initialDelay</name> <value>0</value>
+ <description>Delay for first block report in seconds.</description>
+</property>
+
+<property>
+ <name>dfs.heartbeat.interval</name>
+ <value>3</value>
+ <description>Determines datanode heartbeat interval in seconds.</description>
+</property>
+
+<property>
+ <name>dfs.namenode.handler.count</name>
+ <value>10</value>
+ <description>The number of server threads for the namenode.</description>
+</property>
+
+<property>
+ <name>dfs.safemode.threshold.pct</name>
+ <value>0.999f</value>
+ <description>
+ Specifies the percentage of blocks that should satisfy
+ the minimal replication requirement defined by dfs.replication.min.
+ Values less than or equal to 0 mean not to start in safe mode.
+ Values greater than 1 will make safe mode permanent.
+ </description>
+</property>
+
+<property>
+ <name>dfs.safemode.extension</name>
+ <value>30000</value>
+ <description>
+ Determines extension of safe mode in milliseconds
+ after the threshold level is reached.
+ </description>
+</property>
+
+<property>
+ <name>dfs.balance.bandwidthPerSec</name>
+ <value>1048576</value>
+ <description>
+ Specifies the maximum amount of bandwidth that each datanode
+ can utilize for the balancing purpose in term of
+ the number of bytes per second.
+ </description>
+</property>
+
+<property>
+ <name>dfs.hosts</name>
+ <value></value>
+ <description>Names a file that contains a list of hosts that are
+ permitted to connect to the namenode. The full pathname of the file
+ must be specified. If the value is empty, all hosts are
+ permitted.</description>
+</property>
+
+<property>
+ <name>dfs.hosts.exclude</name>
+ <value></value>
+ <description>Names a file that contains a list of hosts that are
+ not permitted to connect to the namenode. The full pathname of the
+ file must be specified. If the value is empty, no hosts are
+ excluded.</description>
+</property>
+
+<property>
+ <name>dfs.max.objects</name>
+ <value>0</value>
+ <description>The maximum number of files, directories and blocks
+ dfs supports. A value of zero indicates no limit to the number
+ of objects that dfs supports.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.decommission.interval</name>
+ <value>30</value>
+ <description>Namenode periodicity in seconds to check if decommission is
+ complete.</description>
+</property>
+
+<property>
+ <name>dfs.namenode.decommission.nodes.per.interval</name>
+ <value>5</value>
+ <description>The number of nodes namenode checks if decommission is complete
+ in each dfs.namenode.decommission.interval.</description>
+</property>
+
+<property>
+ <name>dfs.replication.interval</name>
+ <value>3</value>
+ <description>The periodicity in seconds with which the namenode computes
+ repliaction work for datanodes. </description>
+</property>
+
+<property>
+ <name>dfs.access.time.precision</name>
+ <value>3600000</value>
+ <description>The access time for HDFS file is precise upto this value.
+ The default value is 1 hour. Setting a value of 0 disables
+ access times for HDFS.
+ </description>
+</property>
+
+</configuration>
View
7 src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -49,6 +49,11 @@
DFSClient dfs;
private boolean verifyChecksum = true;
+
+ static{
+ Configuration.addDefaultResource("hdfs-default.xml");
+ Configuration.addDefaultResource("hdfs-site.xml");
+ }
public DistributedFileSystem() {
}
@@ -449,4 +454,6 @@ public void setTimes(Path p, long mtime, long atime
) throws IOException {
dfs.setTimes(getPathName(p), mtime, atime);
}
+
+
}
View
5 src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -126,6 +126,11 @@
public class DataNode extends Configured
implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants, Runnable {
public static final Log LOG = LogFactory.getLog(DataNode.class);
+
+ static{
+ Configuration.addDefaultResource("hdfs-default.xml");
+ Configuration.addDefaultResource("hdfs-site.xml");
+ }
public static final String DN_CLIENTTRACE_FORMAT =
"src: %s" + // src IP
View
5 src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -97,6 +97,11 @@
public class NameNode implements ClientProtocol, DatanodeProtocol,
NamenodeProtocol, FSConstants,
RefreshAuthorizationPolicyProtocol {
+ static{
+ Configuration.addDefaultResource("hdfs-default.xml");
+ Configuration.addDefaultResource("hdfs-site.xml");
+ }
+
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
if (protocol.equals(ClientProtocol.class.getName())) {
View
794 conf/hadoop-default.xml → src/mapred/mapred-default.xml
@@ -2,55 +2,11 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Do not modify this file directly. Instead, copy entries that you -->
-<!-- wish to modify from this file into hadoop-site.xml and change them -->
-<!-- there. If hadoop-site.xml does not already exist, create it. -->
+<!-- wish to modify from this file into mapred-site.xml and change them -->
+<!-- there. If mapred-site.xml does not already exist, create it. -->
<configuration>
-<!--- global properties -->
-
-<property>
- <name>hadoop.tmp.dir</name>
- <value>/tmp/hadoop-${user.name}</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-<property>
- <name>hadoop.native.lib</name>
- <value>true</value>
- <description>Should native hadoop libraries, if present, be used.</description>
-</property>
-
-<property>
- <name>hadoop.http.filter.initializers</name>
- <value></value>
- <description>A comma separated list of class names. Each class in the list
- must extend org.apache.hadoop.http.FilterInitializer. The corresponding
- Filter will be initialized. Then, the Filter will be applied to all user
- facing jsp and servlet web pages. The ordering of the list defines the
- ordering of the filters.</description>
-</property>
-
-<property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
- <description>Is service-level authorization enabled?</description>
-</property>
-
-<!--- logging properties -->
-
-<property>
- <name>hadoop.logfile.size</name>
- <value>10000000</value>
- <description>The max size of each log file</description>
-</property>
-
-<property>
- <name>hadoop.logfile.count</name>
- <value>10</value>
- <description>The max number of log files</description>
-</property>
-
<property>
<name>hadoop.job.history.location</name>
<value></value>
@@ -70,14 +26,6 @@
</description>
</property>
-<property>
- <name>dfs.namenode.logging.level</name>
- <value>info</value>
- <description>The logging level for dfs namenode. Other values are "dir"(trac
-e namespace mutations), "block"(trace block under/over replications and block
-creations/deletions), or "all".</description>
-</property>
-
<!-- i/o properties -->
<property>
@@ -114,30 +62,6 @@ creations/deletions), or "all".</description>
</property>
<property>
- <name>io.file.buffer.size</name>
- <value>4096</value>
- <description>The size of buffer for use in sequence files.
- The size of this buffer should probably be a multiple of hardware
- page size (4096 on Intel x86), and it determines how much data is
- buffered during read and write operations.</description>
-</property>
-
-<property>
- <name>io.bytes.per.checksum</name>
- <value>512</value>
- <description>The number of bytes per checksum. Must not be larger than
- io.file.buffer.size.</description>
-</property>
-
-<property>
- <name>io.skip.checksum.errors</name>
- <value>false</value>
- <description>If true, when a checksum error is encountered while
- reading a sequence file, entries are skipped, instead of throwing an
- exception.</description>
-</property>
-
-<property>
<name>io.map.index.skip</name>
<value>0</value>
<description>Number of index entries to skip between each entry.
@@ -146,502 +70,6 @@ creations/deletions), or "all".</description>
</property>
<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
- <description>A list of the compression codec classes that can be used
- for compression/decompression.</description>
-</property>
-
-<property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
- <description>A list of serialization classes that can be used for
- obtaining serializers and deserializers.</description>
-</property>
-
-<!-- file system properties -->
-
-<property>
- <name>fs.default.name</name>
- <value>file:///</value>
- <description>The name of the default file system. A URI whose
- scheme and authority determine the FileSystem implementation. The
- uri's scheme determines the config property (fs.SCHEME.impl) naming
- the FileSystem implementation class. The uri's authority is used to
- determine the host, port, etc. for a filesystem.</description>
-</property>
-
-<property>
- <name>fs.trash.interval</name>
- <value>0</value>
- <description>Number of minutes between trash checkpoints.
- If zero, the trash feature is disabled.
- </description>
-</property>
-
-<property>
- <name>fs.file.impl</name>
- <value>org.apache.hadoop.fs.LocalFileSystem</value>
- <description>The FileSystem for file: uris.</description>
-</property>
-
-<property>
- <name>fs.hdfs.impl</name>
- <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
- <description>The FileSystem for hdfs: uris.</description>
-</property>
-
-<property>
- <name>fs.s3.impl</name>
- <value>org.apache.hadoop.fs.s3.S3FileSystem</value>
- <description>The FileSystem for s3: uris.</description>
-</property>
-
-<property>
- <name>fs.s3n.impl</name>
- <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
- <description>The FileSystem for s3n: (Native S3) uris.</description>
-</property>
-
-<property>
- <name>fs.kfs.impl</name>
- <value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value>
- <description>The FileSystem for kfs: uris.</description>
-</property>
-
-<property>
- <name>fs.hftp.impl</name>
- <value>org.apache.hadoop.hdfs.HftpFileSystem</value>
-</property>
-
-<property>
- <name>fs.hsftp.impl</name>
- <value>org.apache.hadoop.hdfs.HsftpFileSystem</value>
-</property>
-
-<property>
- <name>fs.ftp.impl</name>
- <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
- <description>The FileSystem for ftp: uris.</description>
-</property>
-
-<property>
- <name>fs.ramfs.impl</name>
- <value>org.apache.hadoop.fs.InMemoryFileSystem</value>
- <description>The FileSystem for ramfs: uris.</description>
-</property>
-
-<property>
- <name>fs.har.impl</name>
- <value>org.apache.hadoop.fs.HarFileSystem</value>
- <description>The filesystem for Hadoop archives. </description>
-</property>
-
-<property>
- <name>fs.checkpoint.dir</name>
- <value>${hadoop.tmp.dir}/dfs/namesecondary</value>
- <description>Determines where on the local filesystem the DFS secondary
- name node should store the temporary images to merge.
- If this is a comma-delimited list of directories then the image is
- replicated in all of the directories for redundancy.
- </description>
-</property>
-
-<property>
- <name>fs.checkpoint.edits.dir</name>
- <value>${fs.checkpoint.dir}</value>
- <description>Determines where on the local filesystem the DFS secondary
- name node should store the temporary edits to merge.
- If this is a comma-delimited list of directoires then teh edits is
- replicated in all of the directoires for redundancy.
- Default value is same as fs.checkpoint.dir
- </description>
-</property>
-
-<property>
- <name>fs.checkpoint.period</name>
- <value>3600</value>
- <description>The number of seconds between two periodic checkpoints.
- </description>
-</property>
-
-<property>
- <name>fs.checkpoint.size</name>
- <value>67108864</value>
- <description>The size of the current edit log (in bytes) that triggers
- a periodic checkpoint even if the fs.checkpoint.period hasn't expired.
- </description>
-</property>
-
-<property>
- <name>dfs.secondary.http.address</name>
- <value>0.0.0.0:50090</value>
- <description>
- The secondary namenode http server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- <description>
- The address where the datanode server will listen to.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- <description>
- The datanode http server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:50020</value>
- <description>
- The datanode ipc server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.handler.count</name>
- <value>3</value>
- <description>The number of server threads for the datanode.</description>
-</property>
-
-<property>
- <name>dfs.http.address</name>
- <value>0.0.0.0:50070</value>
- <description>
- The address and the base port where the dfs namenode web ui will listen on.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>dfs.https.enable</name>
- <value>false</value>
- <description>Decide if HTTPS(SSL) is supported on HDFS
- </description>
-</property>
-
-<property>
- <name>dfs.https.need.client.auth</name>
- <value>false</value>
- <description>Whether SSL client certificate authentication is required
- </description>
-</property>
-
-<property>
- <name>dfs.https.server.keystore.resource</name>
- <value>ssl-server.xml</value>
- <description>Resource file from which ssl server keystore
- information will be extracted
- </description>
-</property>
-
-<property>
- <name>dfs.https.client.keystore.resource</name>
- <value>ssl-client.xml</value>
- <description>Resource file from which ssl client keystore
- information will be extracted
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
-</property>
-
-<property>
- <name>dfs.https.address</name>
- <value>0.0.0.0:50470</value>
-</property>
-
- <property>
- <name>dfs.datanode.dns.interface</name>
- <value>default</value>
- <description>The name of the Network Interface from which a data node should
- report its IP address.
- </description>
- </property>
-
-<property>
- <name>dfs.datanode.dns.nameserver</name>
- <value>default</value>
- <description>The host name or IP address of the name server (DNS)
- which a DataNode should use to determine the host name used by the
- NameNode for communication and display purposes.
- </description>
- </property>
-
-<property>
- <name>dfs.replication.considerLoad</name>
- <value>true</value>
- <description>Decide if chooseTarget considers the target's load or not
- </description>
-</property>
-<property>
- <name>dfs.default.chunk.view.size</name>
- <value>32768</value>
- <description>The number of bytes to view for a file on the browser.
- </description>
-</property>
-
-<property>
- <name>dfs.datanode.du.reserved</name>
- <value>0</value>
- <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
- </description>
-</property>
-
-<property>
- <name>dfs.name.dir</name>
- <value>${hadoop.tmp.dir}/dfs/name</value>
- <description>Determines where on the local filesystem the DFS name node
- should store the name table(fsimage). If this is a comma-delimited list
- of directories then the name table is replicated in all of the
- directories, for redundancy. </description>
-</property>
-
-<property>
- <name>dfs.name.edits.dir</name>
- <value>${dfs.name.dir}</value>
- <description>Determines where on the local filesystem the DFS name node
- should store the transaction (edits) file. If this is a comma-delimited list
- of directories then the transaction file is replicated in all of the
- directories, for redundancy. Default value is same as dfs.name.dir
- </description>
-</property>
-<property>
- <name>dfs.web.ugi</name>
- <value>webuser,webgroup</value>
- <description>The user account used by the web interface.
- Syntax: USERNAME,GROUP1,GROUP2, ...
- </description>
-</property>
-
-<property>
- <name>dfs.permissions</name>
- <value>true</value>
- <description>
- If "true", enable permission checking in HDFS.
- If "false", permission checking is turned off,
- but all other behavior is unchanged.
- Switching from one parameter value to the other does not change the mode,
- owner or group of files or directories.
- </description>
-</property>
-
-<property>
- <name>dfs.permissions.supergroup</name>
- <value>supergroup</value>
- <description>The name of the group of super-users.</description>
-</property>
-
-<property>
- <name>dfs.data.dir</name>
- <value>${hadoop.tmp.dir}/dfs/data</value>
- <description>Determines where on the local filesystem an DFS data node
- should store its blocks. If this is a comma-delimited
- list of directories, then data will be stored in all named
- directories, typically on different devices.
- Directories that do not exist are ignored.
- </description>
-</property>
-
-<property>
- <name>dfs.replication</name>
- <value>3</value>
- <description>Default block replication.
- The actual number of replications can be specified when the file is created.
- The default is used if replication is not specified in create time.
- </description>
-</property>
-
-<property>
- <name>dfs.replication.max</name>
- <value>512</value>
- <description>Maximal block replication.
- </description>
-</property>
-
-<property>
- <name>dfs.replication.min</name>
- <value>1</value>
- <description>Minimal block replication.
- </description>
-</property>
-
-<property>
- <name>dfs.block.size</name>
- <value>67108864</value>
- <description>The default block size for new files.</description>
-</property>
-
-<property>
- <name>dfs.df.interval</name>
- <value>60000</value>
- <description>Disk usage statistics refresh interval in msec.</description>
-</property>
-
-<property>
- <name>dfs.client.block.write.retries</name>
- <value>3</value>
- <description>The number of retries for writing blocks to the data nodes,
- before we signal failure to the application.
- </description>
-</property>
-
-<property>
- <name>dfs.blockreport.intervalMsec</name>
- <value>3600000</value>
- <description>Determines block reporting interval in milliseconds.</description>
-</property>
-
-<property>
- <name>dfs.blockreport.initialDelay</name> <value>0</value>
- <description>Delay for first block report in seconds.</description>
-</property>
-
-<property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
-</property>
-
-<property>
- <name>dfs.namenode.handler.count</name>
- <value>10</value>
- <description>The number of server threads for the namenode.</description>
-</property>
-
-<property>
- <name>dfs.safemode.threshold.pct</name>
- <value>0.999f</value>
- <description>
- Specifies the percentage of blocks that should satisfy
- the minimal replication requirement defined by dfs.replication.min.
- Values less than or equal to 0 mean not to start in safe mode.
- Values greater than 1 will make safe mode permanent.
- </description>
-</property>
-
-<property>
- <name>dfs.safemode.extension</name>
- <value>30000</value>
- <description>
- Determines extension of safe mode in milliseconds
- after the threshold level is reached.
- </description>
-</property>
-
-<property>
- <name>dfs.balance.bandwidthPerSec</name>
- <value>1048576</value>
- <description>
- Specifies the maximum amount of bandwidth that each datanode
- can utilize for the balancing purpose in term of
- the number of bytes per second.
- </description>
-</property>
-
-<property>
- <name>dfs.hosts</name>
- <value></value>
- <description>Names a file that contains a list of hosts that are
- permitted to connect to the namenode. The full pathname of the file
- must be specified. If the value is empty, all hosts are
- permitted.</description>
-</property>
-
-<property>
- <name>dfs.hosts.exclude</name>
- <value></value>
- <description>Names a file that contains a list of hosts that are
- not permitted to connect to the namenode. The full pathname of the
- file must be specified. If the value is empty, no hosts are
- excluded.</description>
-</property>
-
-<property>
- <name>dfs.max.objects</name>
- <value>0</value>
- <description>The maximum number of files, directories and blocks
- dfs supports. A value of zero indicates no limit to the number
- of objects that dfs supports.
- </description>
-</property>
-
-<property>
- <name>dfs.namenode.decommission.interval</name>
- <value>30</value>
- <description>Namenode periodicity in seconds to check if decommission is
- complete.</description>
-</property>
-
-<property>
- <name>dfs.namenode.decommission.nodes.per.interval</name>
- <value>5</value>
- <description>The number of nodes namenode checks if decommission is complete
- in each dfs.namenode.decommission.interval.</description>
-</property>
-
-<property>
- <name>dfs.replication.interval</name>
- <value>3</value>
- <description>The periodicity in seconds with which the namenode computes
- repliaction work for datanodes. </description>
-</property>
-
-<property>
- <name>dfs.access.time.precision</name>
- <value>3600000</value>
- <description>The access time for HDFS file is precise upto this value.
- The default value is 1 hour. Setting a value of 0 disables
- access times for HDFS.
- </description>
-</property>
-
-<property>
- <name>fs.s3.block.size</name>
- <value>67108864</value>
- <description>Block size to use when writing files to S3.</description>
-</property>
-
-<property>
- <name>fs.s3.buffer.dir</name>
- <value>${hadoop.tmp.dir}/s3</value>
- <description>Determines where on the local filesystem the S3 filesystem
- should store files before sending them to S3
- (or after retrieving them from S3).
- </description>
-</property>
-
-<property>
- <name>fs.s3.maxRetries</name>
- <value>4</value>
- <description>The maximum number of retries for reading or writing files to S3,
- before we signal failure to the application.
- </description>
-</property>
-
-<property>
- <name>fs.s3.sleepTimeSeconds</name>
- <value>10</value>
- <description>The number of seconds to sleep between each S3 retry.
- </description>
-</property>
-
-<!-- map/reduce properties -->
-
-<property>
<name>mapred.job.tracker</name>
<value>local</value>
<description>The host and port that the MapReduce job tracker runs
@@ -688,14 +116,6 @@ creations/deletions), or "all".</description>
</property>
<property>
- <name>local.cache.size</name>
- <value>10737418240</value>
- <description>The limit on the size of cache you want to keep, set by default
- to 10GB. This will act as a soft limit on the cache directory for out of band data.
- </description>
-</property>
-
-<property>
<name>mapred.system.dir</name>
<value>${hadoop.tmp.dir}/mapred/system</value>
<description>The shared directory where MapReduce stores control files.
@@ -722,11 +142,11 @@ creations/deletions), or "all".</description>
<name>mapred.local.dir.minspacekill</name>
<value>0</value>
<description>If the space in mapred.local.dir drops under this,
- do not ask more tasks until all the current ones have finished and
- cleaned up. Also, to save the rest of the tasks we have running,
- kill one of them, to clean up some space. Start with the reduce tasks,
- then go with the ones that have finished the least.
- Value in bytes.
+ do not ask more tasks until all the current ones have finished and
+ cleaned up. Also, to save the rest of the tasks we have running,
+ kill one of them, to clean up some space. Start with the reduce tasks,
+ then go with the ones that have finished the least.
+ Value in bytes.
</description>
</property>
@@ -1192,6 +612,7 @@ creations/deletions), or "all".</description>
from the reduce directory as they are consumed.</description>
</property>
+
<!--
<property>
<name>keep.task.files.pattern</name>
@@ -1240,58 +661,6 @@ creations/deletions), or "all".</description>
</property>
<property>
- <name>io.seqfile.compress.blocksize</name>
- <value>1000000</value>
- <description>The minimum block size for compression in block compressed
- SequenceFiles.
- </description>
-</property>
-
-<property>
- <name>io.seqfile.lazydecompress</name>
- <value>true</value>
- <description>Should values of block-compressed SequenceFiles be decompressed
- only when necessary.
- </description>
-</property>
-
-<property>
- <name>io.seqfile.sorter.recordlimit</name>
- <value>1000000</value>
- <description>The limit on number of records to be kept in memory in a spill
- in SequenceFiles.Sorter
- </description>
-</property>
-
-<property>
- <name>io.mapfile.bloom.size</name>
- <value>1048576</value>
- <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
- keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
- Larger values minimize the number of filters, which slightly increases the performance,
- but may waste too much space if the total number of keys is usually much smaller
- than this number.
- </description>
-</property>
-
-<property>
- <name>io.mapfile.bloom.error.rate</name>
- <value>0.005</value>
- <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
- As this value decreases, the size of BloomFilter-s increases exponentially. This
- value is the probability of encountering false positives (default is 0.5%).
- </description>
-</property>
-
-<property>
- <name>hadoop.util.hash.type</name>
- <value>murmur</value>
- <description>The default implementation of Hash. Currently this can take one of the
- two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
- </description>
-</property>
-
-<property>
<name>map.sort.class</name>
<value>org.apache.hadoop.util.QuickSort</value>
<description>The default sort class for sorting keys.
@@ -1309,7 +678,7 @@ creations/deletions), or "all".</description>
<name>mapred.userlog.retain.hours</name>
<value>24</value>
<description>The maximum time, in hours, for which the user-logs are to be
- retained.
+ retained.
</description>
</property>
@@ -1327,18 +696,18 @@ creations/deletions), or "all".</description>
<description>Names a file that contains the list of hosts that
should be excluded by the jobtracker. If the value is empty, no
hosts are excluded.</description>
-</property>
+</property>
<property>
<name>mapred.max.tracker.blacklists</name>
<value>4</value>
- <description>The number of blacklists for a taskTracker by various jobs
+ <description>The number of blacklists for a taskTracker by various jobs
after which the task tracker could be blacklisted across
- all jobs. The tracker will be given a tasks later
- (after a day). The tracker will become a healthy
- tracker after a restart.
+ all jobs. The tracker will be given a tasks later
+ (after a day). The tracker will become a healthy
+ tracker after a restart.
</description>
-</property>
+</property>
<property>
<name>mapred.max.tracker.failures</name>
@@ -1491,66 +860,7 @@ creations/deletions), or "all".</description>
acceptable.
</description>
</property>
-
-<!-- ipc properties -->
-
-<property>
- <name>ipc.client.idlethreshold</name>
- <value>4000</value>
- <description>Defines the threshold number of connections after which
- connections will be inspected for idleness.
- </description>
-</property>
-
-<property>
- <name>ipc.client.kill.max</name>
- <value>10</value>
- <description>Defines the maximum number of clients to disconnect in one go.
- </description>
-</property>
-
-<property>
- <name>ipc.client.connection.maxidletime</name>
- <value>10000</value>
- <description>The maximum time in msec after which a client will bring down the
- connection to the server.
- </description>
-</property>
-
-<property>
- <name>ipc.client.connect.max.retries</name>
- <value>10</value>
- <description>Indicates the number of retries a client will make to establish
- a server connection.
- </description>
-</property>
-
-<property>
- <name>ipc.server.listen.queue.size</name>
- <value>128</value>
- <description>Indicates the length of the listen queue for servers accepting
- client connections.
- </description>
-</property>
-
-<property>
- <name>ipc.server.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the server. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
- </description>
-</property>
-
-<property>
- <name>ipc.client.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the client. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
- </description>
-</property>
-
+
<!-- Job Notification Configuration -->
<!--
@@ -1579,38 +889,8 @@ creations/deletions), or "all".</description>
<description>Indicates time in milliseconds between notification URL retry
calls</description>
</property>
-
-<!-- Web Interface Configuration -->
-
-<property>
- <name>webinterface.private.actions</name>
- <value>false</value>
- <description> If set to true, the web interfaces of JT and NN may contain
- actions, such as kill job, delete file, etc., that should
- not be exposed to public. Enable this option if the interfaces
- are only reachable by those who have the right authorization.
- </description>
-</property>
-
+
<!-- Proxy Configuration -->
-
-<property>
- <name>hadoop.rpc.socket.factory.class.default</name>
- <value>org.apache.hadoop.net.StandardSocketFactory</value>
- <description> Default SocketFactory to use. This parameter is expected to be
- formatted as "package.FactoryClassName".
- </description>
-</property>
-
-<property>
- <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
- <value></value>
- <description> SocketFactory to use to connect to a DFS. If null or empty, use
- hadoop.rpc.socket.class.default. This socket factory is also used by
- DFSClient to create sockets to DataNodes.
- </description>
-</property>
-
<property>
<name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
<value></value>
@@ -1620,44 +900,6 @@ creations/deletions), or "all".</description>
</property>
<property>
- <name>hadoop.socks.server</name>
- <value></value>
- <description> Address (host:port) of the SOCKS server to be used by the
- SocksSocketFactory.
- </description>
-</property>
-
-<!-- Rack Configuration -->
-
-<property>
- <name>topology.node.switch.mapping.impl</name>
- <value>org.apache.hadoop.net.ScriptBasedMapping</value>
- <description> The default implementation of the DNSToSwitchMapping. It
- invokes a script specified in topology.script.file.name to resolve
- node names. If the value for topology.script.file.name is not set, the
- default value of DEFAULT_RACK is returned for all node names.
- </description>
-</property>
-
-<property>
- <name>topology.script.file.name</name>
- <value></value>
- <description> The script name that should be invoked to resolve DNS names to
- NetworkTopology names. Example: the script would take host.foo.bar as an
- argument, and return /rack1 as the output.
- </description>
-</property>
-
-<property>
- <name>topology.script.number.args</name>
- <value>100</value>
- <description> The max number of args that the script configured with
- topology.script.file.name should be run with. Each arg is an
- IP address.
- </description>
-</property>
-
-<property>
<name>mapred.task.cache.levels</name>
<value>2</value>
<description> This is the max level of the task cache. For example, if
@@ -1743,4 +985,4 @@ creations/deletions), or "all".</description>
</description>
</property>
-</configuration>
+</configuration>
View
5 src/mapred/org/apache/hadoop/mapred/JobClient.java
@@ -161,6 +161,11 @@
private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED;
private static final long MAX_JOBPROFILE_AGE = 1000 * 2;
+ static{
+ Configuration.addDefaultResource("mapred-default.xml");
+ Configuration.addDefaultResource("mapred-site.xml");
+ }
+
/**
* A NetworkedJob is an implementation of RunningJob. It holds
* a JobProfile object to provide some info, and interacts with the
View
5 src/mapred/org/apache/hadoop/mapred/JobConf.java
@@ -106,6 +106,11 @@
private static final Log LOG = LogFactory.getLog(JobConf.class);
+ static{
+ Configuration.addDefaultResource("mapred-default.xml");
+ Configuration.addDefaultResource("mapred-site.xml");
+ }
+
/**
* A value which if set for memory related configuration options,
* indicates that the options are turned off.
View
5 src/mapred/org/apache/hadoop/mapred/JobTracker.java
@@ -86,6 +86,11 @@
public class JobTracker implements MRConstants, InterTrackerProtocol,
JobSubmissionProtocol, TaskTrackerManager, RefreshAuthorizationPolicyProtocol {
+ static{
+ Configuration.addDefaultResource("mapred-default.xml");
+ Configuration.addDefaultResource("mapred-site.xml");
+ }
+
static long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000;
static long RETIRE_JOB_INTERVAL;
static long RETIRE_JOB_CHECK_INTERVAL;
View
5 src/mapred/org/apache/hadoop/mapred/TaskTracker.java
@@ -101,6 +101,11 @@
static enum State {NORMAL, STALE, INTERRUPTED, DENIED}
+ static{
+ Configuration.addDefaultResource("mapred-default.xml");
+ Configuration.addDefaultResource("mapred-site.xml");
+ }
+
public static final Log LOG =
LogFactory.getLog(TaskTracker.class);
View
50 src/test/core-site.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running unit tests. This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory. -->
+
+<configuration>
+
+
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>build/test</value>
+ <description>A base for other temporary directories.</description>
+ <final>true</final>
+</property>
+
+<property>
+ <name>test.fs.s3.name</name>
+ <value>s3:///</value>
+ <description>The name of the s3 file system for testing.</description>
+</property>
+
+<property>
+ <name>fs.s3.block.size</name>
+ <value>128</value>
+ <description>Size of a block in bytes.</description>
+</property>
+
+<property>
+ <name>fs.ftp.user.localhost</name>
+ <value>user</value>
+ <description>The username for connecting to FTP server running on localhost.
+ This is required by FTPFileSystem</description>
+</property>
+
+<property>
+ <name>fs.ftp.password.localhost</name>
+ <value>password</value>
+ <description>The password for connecting to FTP server running on localhost.
+ This is required by FTPFileSystem</description>
+</property>
+