hadoop config 详解

1.配置文件：

-rw-r--r-- 1 hadoop  4436 Jun  2 17:47 capacity-scheduler.xml
-rw-r--r-- 1 hadoop 61163 Jun  2 17:47 core-default.xml
-rw-r--r-- 1 hadoop  1779 Jun  2 17:47 core-site.xml
-rwxr-xr-x 1 hadoop  4328 Jun  2 17:47 hadoop-env.sh
-rw-r--r-- 1 hadoop  9683 Jun  2 17:47 hadoop-policy.xml
-rw-r--r-- 1 hadoop 81473 Jun  2 17:47 hdfs-default.xml
-rw-r--r-- 1 hadoop  3305 Jun  2 17:47 hdfs-site.xml
-rwxr-xr-x 1 hadoop  1449 Jun  2 17:47 httpfs-env.sh
-rw-r--r-- 1 hadoop   620 Jun  2 17:47 httpfs-site.xml
-rw-r--r-- 1 hadoop  3518 Jun  2 17:47 kms-acls.xml
-rwxr-xr-x 1 hadoop  1527 Jun  2 17:47 kms-env.sh
-rw-r--r-- 1 hadoop  5511 Jun  2 17:47 kms-site.xml
-rw-r--r-- 1 hadoop 76016 Jun  2 17:47 mapred-default.xml
-rwxr-xr-x 1 hadoop  1383 Jun  2 17:47 mapred-env.sh
-rw-r--r-- 1 hadoop  2644 Jun  2 17:47 mapred-site.xml
-rw-r--r-- 1 hadoop     6 Jun  2 17:47 masters
-rw-r--r-- 1 hadoop    26 Jun  2 17:47 slaves
-rw-r--r-- 1 hadoop 63142 Jun  2 17:47 yarn-default.xml
-rwxr-xr-x 1 hadoop  4601 Jun  2 17:47 yarn-env.sh
-rw-r--r-- 1 hadoop  4448 Jun  2 17:47 yarn-site.xml

2.core-site.xml

<configuration>
<!--- global properties -->
<property>
  <name>hadoop.tmp.dir</name>
  <value>/xwtec/hadoop</value>
  <description>A base for other temporary directories.</description>
</property>
<!-- i/o properties -->
<property>
  <name>io.file.buffer.size</name>
  <value>131072</value>
  <description>The size of buffer for use in sequence files.
  The size of this buffer should probably be a multiple of hardware
  page size (4096 on Intel x86), and it determines how much data is
  buffered during read and write operations.</description>
</property>
<!-- file system properties -->
<property>
  <name>fs.defaultFS</name>
  <value>hdfs://earth:9000</value>
  <description>The name of the default file system.  A URI whose
  scheme and authority determine the FileSystem implementation.  The
  uri's scheme determines the config property (fs.SCHEME.impl) naming
  the FileSystem implementation class.  The uri's authority is used to
  determine the host, port, etc. for a filesystem.</description>
</property>
</configuration>

3.hdfs-site.xml

<property>
  <name>dfs.namenode.name.dir</name>
  <value>file:///xwtec/hadoop/dfs/name</value>
  <description>Determines where on the local filesystem the DFS name node
      should store the name table(fsimage).  If this is a comma-delimited list
      of directories then the name table is replicated in all of the
      directories, for redundancy. </description>
</property>
<property>
  <name>dfs.permissions.enabled</name>
  <value>false</value>
  <description>
    If "true", enable permission checking in HDFS.
    If "false", permission checking is turned off,
    but all other behavior is unchanged.
    Switching from one parameter value to the other does not change the mode,
    owner or group of files or directories.
  </description>
</property>
<property>
  <name>dfs.datanode.data.dir</name>
  <value>/xwtec/hadoop/dfs/data</value>
  <description>Determines where on the local filesystem an DFS data node
  should store its blocks.  If this is a comma-delimited
  list of directories, then data will be stored in all named
  directories, typically on different devices. The directories should be tagged
  with corresponding storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS
  storage policies. The default storage type will be DISK if the directory does
  not have a storage type tagged explicitly. Directories that do not exist will
  be created if local filesystem permission allows.
  </description>
<property>
  <name>dfs.blocksize</name>
  <value>134217728</value>
  <description>
      The default block size for new files, in bytes.
      You can use the following suffix (case insensitive):
      k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa) to specify the size (such as 128k, 512m, 1g, etc.),
      Or provide complete size in bytes (such as 134217728 for 128 MB).
  </description>
</property>
</property>
<property>
  <name>dfs.namenode.secondary.http-address</name>
  <value>mercury:50090</value>
  <description>
    The secondary namenode http server address and port.
  </description>
</property>
<property>
  <name>dfs.namenode.secondary.https-address</name>
  <value>mercury:50091</value>
  <description>
    The secondary namenode HTTPS server address and port.
  </description>
</property>
<property>
  <name>dfs.replication</name>
  <value>2</value>
  <description>Default block replication. 
  The actual number of replications can be specified when the file is created.
  The default is used if replication is not specified in create time.
  </description>
</property>

4. yarn-site.xml

  <!-- Resource Manager Configs -->
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>earth</value>
    <description>The hostname of the RM.</description>
  </property>    
  <property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>${yarn.resourcemanager.hostname}:8030</value>
    <description>The address of the scheduler interface.</description>
  </property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>${yarn.resourcemanager.hostname}:8031</value>
  </property>
  <property>
    <name>yarn.resourcemanager.address</name>
    <value>${yarn.resourcemanager.hostname}:8032</value>
    <description>The address of the applications manager interface in the RM.</description>
  </property>
  <property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>${yarn.resourcemanager.hostname}:8033</value>
    <description>The address of the RM admin interface.</description>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>${yarn.resourcemanager.hostname}:8088</value>
    <description>The http address of the RM web application.</description>
  </property>
  <property>
    <name>yarn.resourcemanager.scheduler.class</name>
    <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
    <description>The class to use as the resource scheduler.</description>
  </property>
  <!-- Node Manager Configs -->
  <property>
    <name>yarn.nodemanager.hostname</name>
    <value>0.0.0.0</value>
    <description>The hostname of the NM.</description>
  </property>
  <property>
    <name>yarn.nodemanager.local-dirs</name>
    <value>${hadoop.tmp.dir}/nm-local-dir</value>
    <description>List of directories to store localized files in. An 
      application's localized file directory will be found in:
      ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
      Individual containers' work directories, called container_${contid}, will
      be subdirectories of this.
   </description>
  </property>
  <property>
    <!-- where's yarn.log.dir?-->
    <name>yarn.nodemanager.log-dirs</name>
    <value>${yarn.log.dir}/userlogs</value>
    <description>
      Where to store container logs. An application's localized log directory 
      will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
      Individual containers' log directories will be below this, in directories 
      named container_{$contid}. Each container directory will contain the files
      stderr, stdin, and syslog generated by that container.
    </description>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
    <description>A comma separated list of services where service name should only
      contain a-zA-Z0-9_ and can not start with numbers</description>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <description>The minimum allocation for every container request at the RM,
    in MBs. Memory requests lower than this will throw a
    InvalidResourceRequestException.</description>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>1024</value>
  </property>
  <property>
    <description>The maximum allocation for every container request at the RM,
    in MBs. Memory requests higher than this will throw a
    InvalidResourceRequestException.</description>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>6666</value>
  </property>
  <property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>7168</value>
  </property>

5. mapred-site.xml

<!-- MR YARN Application properties -->
<property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
  <description>The runtime framework for executing MapReduce jobs.
  Can be one of local, classic or yarn.
  </description>
</property>
<!-- jobhistory properties -->
<property>
  <name>mapreduce.jobhistory.address</name>
  <value>earth:10020</value>
  <description>MapReduce JobHistory Server IPC host:port</description>
</property>
<property>
  <name>mapreduce.jobhistory.webapp.address</name>
  <value>earth:19888</value>
  <description>MapReduce JobHistory Server Web UI host:port</description>
</property>
<property>
  <name>mapreduce.jobhistory.intermediate-done-dir</name>
  <value>${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate</value>
  <description></description>
</property>
<property>
  <name>mapreduce.jobhistory.done-dir</name>
  <value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>
  <description></description>
</property>
<property>
  <name>mapreduce.job.ubertask.enable</name>
  <value>false</value>
  <description>Whether to enable the small-jobs "ubertask" optimization,
  which runs "sufficiently small" jobs sequentially within a single JVM.
  "Small" is defined by the following maxmaps, maxreduces, and maxbytes
  settings. Note that configurations for application masters also affect
  the "Small" definition - yarn.app.mapreduce.am.resource.mb must be
  larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb,
  and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than
  both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable
  ubertask. Users may override this value.
  </description>
</property>
<property>
 <name>mapreduce.map.memory.mb</name>
 <value>1024</value>
</property>
<property>
  <name>mapreduce.reduce.memory.mb</name>
  <value>1024</value>
</property>

6.yarn-env.sh

# default log directory & file
if [ "$YARN_LOG_DIR" = "" ]; then
  YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
fi
if [ "$YARN_LOGFILE" = "" ]; then
  YARN_LOGFILE='yarn.log'
fi
# default policy file for service-level authorization
if [ "$YARN_POLICYFILE" = "" ]; then
  YARN_POLICYFILE="hadoop-policy.xml"
fi
# restore ordinary behaviour
unset IFS
YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
  YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi
YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"

7.hadoop-env.xml

# Set Hadoop-specific environment variables here.
# The only required environment variable is JAVA_HOME.  All others are
# optional.  When running a distributed configuration it is best to
# set JAVA_HOME in this file, so that it is correctly defined on
# remote nodes.
# The java implementation to use.
JAVA_HOME=/opt/jdk1.7.0_79
export JAVA_HOME=${JAVA_HOME}
# The jsvc implementation to use. Jsvc is required to run secure datanodes
# that bind to privileged ports to provide authentication of data transfer
# protocol.  Jsvc is not required if SASL is configured for authentication of
# data transfer protocol using non-privileged ports.
#export JSVC_HOME=${JSVC_HOME}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}

# Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
  if [ "$HADOOP_CLASSPATH" ]; then
    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
  else
    export HADOOP_CLASSPATH=$f
  fi
done
# The maximum amount of heap to use, in MB. Default is 1000.
#export HADOOP_HEAPSIZE=
#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
# Extra Java runtime options.  Empty by default.
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
export HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=/opt/hadoop/lib/native"
# Command specific options appended to HADOOP_OPTS when specified
export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
# On secure datanodes, user to run the datanode as after dropping privileges.
# This **MUST** be uncommented to enable secure HDFS if using privileged ports
# to provide authentication of data transfer protocol.  This **MUST NOT** be
# defined if SASL is configured for authentication of data transfer protocol
# using non-privileged ports.
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
# Where log files are stored.  $HADOOP_HOME/logs by default.
#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
# Where log files are stored in the secure data environment.
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
###
# HDFS Mover specific parameters
###
# Specify the JVM options to be used when starting the HDFS Mover.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HADOOP_MOVER_OPTS=""
###
# Advanced Users Only!
###
# The directory where pid files are stored. /tmp by default.
# NOTE: this should be set to a directory that can only be written to by 
#       the user that will run the hadoop daemons.  Otherwise there is the
#       potential for a symlink attack.
export HADOOP_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
# A string representing this instance of hadoop. $USER by default.
export HADOOP_IDENT_STRING=$USER

来自为知笔记(Wiz)