zoukankan html css js c++ java

hadoop-2.0.0-cdh4.2.1安装手册

1. 角色分配：

ocdata16: 10.1.253.26 :NameNode JournalNode Zookeeper DFSZKFailoverController
ocdata17: 10.1.253.27 :NameNode JournalNode Zookeeper DFSZKFailoverController
ocdata18: 10.1.253.28 :DataNode JournalNode Zookeeper
ocdata19: 10.1.253.29 :DataNode

2. 添加SSH互信

登录10.1.253.2610.1.253.2710.1.253.2810.1.253.29

    ssh-keygen -t rsa -P "" 
    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
    chmod go-w $HOME $HOME/.ssh
    chmod 600 $HOME/.ssh/authorized_keys
    chown `whoami` $HOME/.ssh/authorized_keys

登录10.1.253.26,执行

    ssh stream_dev@ocdata17 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
    ssh stream_dev@ocdata18 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
    ssh stream_dev@ocdata19 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

    scp ~/.ssh/authorized_keys stream_dev@ocdata17:~/.ssh/authorized_keys
    scp ~/.ssh/authorized_keys stream_dev@ocdata18:~/.ssh/authorized_keys
    scp ~/.ssh/authorized_keys stream_dev@ocdata19:~/.ssh/authorized_keys

3. 安装

下载安装包
- wget http://archive.cloudera.com/cdh4/cdh/4/zookeeper-3.4.5-cdh4.2.1.tar.gz
- wget http://archive.cloudera.com/cdh4/cdh/4/hadoop-2.0.0-cdh4.2.1.tar.gz

解压，配置

配置zookeeper

    cd /home/stream_dev/shaoaq/zookeeper-3.4.5-cdh4.2.1/conf
    cp zoo_sample.cfg zoo.cfg
    修改：
        dataDir=/home/stream_dev/shaoaq/zookeeper
        clientPort=2183
        server.16=ocdata16:2898:3898
        server.17=ocdata17:2898:3898
        server.18=ocdata18:2898:3898

    在 ocdata16ocdata17ocdata18上，
        mkdir /home/stream_dev/shaoaq/zookeeper,
        然后vim /home/stream_dev/shaoaq/zookeeper/myid,分别输入16，17，18

配置hadoop

hdfs-site.xml

       <property>
                <name>dfs.nameservices</name>
                <value>mycluster</value>
        </property>
        <property>
                <name>dfs.ha.namenodes.mycluster</name>
                <value>nn1,nn2</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.mycluster.nn1</name>
                <value>ocdata16:8020</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.mycluster.nn2</name>
                <value>ocdata17:8020</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.mycluster.nn1</name>
                <value>ocdata16:50070</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.mycluster.nn2</name>
                <value>ocdata17:50070</value>
        </property>
        <property>
                <name>dfs.namenode.shared.edits.dir</name>
                <value>qjournal://ocdata16:9485;ocdata17:9485;ocdata18:9485/mycluster</value>
        </property>
        <property>
                <name>dfs.client.failover.proxy.provider.mycluster</name>
                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        <property>
                <name>dfs.ha.fencing.methods</name>
                <value>shell(/bin/true)</value>
        </property>
        <property>
                <name>dfs.ha.fencing.ssh.connect-timeout</name>
                <value>10000</value>
        </property>
        <property>
                <name>dfs.journalnode.edits.dir</name>
                <value>/home/stream_dev/shaoaq/journal/node/local/data</value>
        </property>
        <property>
                <name>dfs.journalnode.http-address</name>
                <value>0.0.0.0:9480</value>
        </property>
        <property>
                <name>dfs.journalnode.rpc-address</name>
                <value>0.0.0.0:9485</value>
        </property>
        <property>
                <name>dfs.ha.automatic-failover.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>ha.zookeeper.quorum</name>
                <value>ocdata16:2183,ocdata17:2183,ocdata18:2183</value>
        </property>

core-site.xml

        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://mycluster</value>
        </property>

slaves
```
ocdata16
ocdata17
ocdata18
ocdata19
```

hadoop-env.sh

        export JAVA_HOME=/home/stream_dev/stream/soft/jdk1.7.0_21

新建sbin/start-dfs-journalnode.sh

#!/usr/bin/env bash

bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`

DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh

#---------------------------------------------------------
# quorumjournal nodes (if any)

SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)

case "$SHARED_EDITS_DIR" in
qjournal://*)
  JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://([^/]*)/.*,1,g; s/;/ /g; s/:[0-9]*//g')
  echo "Starting journal nodes [$JOURNAL_NODES]" 
  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" 
      --config "$HADOOP_CONF_DIR" 
      --hostnames "$JOURNAL_NODES" 
      --script "$bin/hdfs" start journalnode ;;
esac

新建sbin/stop-dfs-journalnode.sh

#!/usr/bin/env bash

bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`

DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh

#---------------------------------------------------------
# quorumjournal nodes (if any)

SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)

case "$SHARED_EDITS_DIR" in
qjournal://*)
  JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://([^/]*)/.*,1,g; s/;/ /g; s/:[0-9]*//g')
  echo "Stopping journal nodes [$JOURNAL_NODES]" 
  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" 
      --config "$HADOOP_CONF_DIR" 
      --hostnames "$JOURNAL_NODES" 
      --script "$bin/hdfs" stop journalnode ;;
esac

分发到其他机器

    scp -r ~/shaoaq stream_dev@ocdata17:~/
    scp -r ~/shaoaq stream_dev@ocdata18:~/
    scp -r ~/shaoaq stream_dev@ocdata19:~/

4. 初始化

启动HDFS

登录ocdata16,执行

    ./hdfs namenode -format
    ./hdfs namenode

登录ocdata17,执行
```
    ./hdfs namenode -bootstrapStandby
```
在ocdata16上，ctrl+c 停止namenode
停止journalnode
```
    ./stop-dfs-journalnode.sh
```

5. 日常启动停止¶

启动Zookeeper

分别在ocdata16ocdata17ocdata18上执行：
```
        ./zkServer.sh start
```

判断是否执行成功：

        echo ruok | nc ocdata16 2183
        echo ruok | nc ocdata17 2183
        echo ruok | nc ocdata18 2183

如果都返回imok,则代表启动成功

启动HDFS
```
   cd sbin
   ./start-dfs.sh
```
停止HDFS
```
   cd sbin
   ./stop-dfs.sh
```

6. Yarn

mapred-site.xm

cp ../etc/hadoop/mapred-site.xml.template ../etc/hadoop/mapred-site.xm

        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>

yarn-site.xml

        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce.shuffle</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
                <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
        <property>
                <description>The address of the applications manager interface in the RM.</description>
                <name>yarn.resourcemanager.address</name>
                <value>ocnosql76:8032</value>
        </property>
        <property>
                <description>The address of the scheduler interface.</description>
                <name>yarn.resourcemanager.scheduler.address</name>
                <value>ocnosql76:8030</value>
        </property>
        <property>
                <description>The address of the RM web application.</description>
                <name>yarn.resourcemanager.webapp.address</name>
                <value>ocnosql76:8088</value>
        </property>
        <property>
                <name>yarn.resourcemanager.resource-tracker.address</name>
                <value>ocnosql76:8031</value>
        </property>
        <property>
                <description>The address of the RM admin interface.</description>
                <name>yarn.resourcemanager.admin.address</name>
                <value>ocnosql76:8033</value>
        </property>

Yarn的启动停止

   cd sbin
   ./start-yarn.sh
   ./stop-yarn.sh

7. 其他

在安装前，最好在每台机器上确认hostname -i 返回该机器的IP地址
在安装前，确保每个机器上hosts里包含其他机器的主机名和IP的对应关系
如果节点一直在备节点，而不是自动切换到主节点，请重新格式化zkfc：./hdfs zkfc -formatZK
注意：主机名不能含有下划线

查看全文

相关阅读:
.Net解析html文档类库HtmlAgilityPack完整使用说明
 C# 中使用JSON
Property 'submit' of object #<HTMLFormElement> is not a function
bin/...的访问被拒绝被拒绝的问题
 vs2012 ultimate 密钥
 ASP.NET-自定义HttpModule与HttpHandler
jQuery同步Ajax带来的UI线程阻塞问题及解决办法
 Windows API
服务器不安装Excel，实现导出Excel功能
 内存或磁盘空间不足，Microsoft Office Excel 无法再次打开或保存任何文档。 [问题点数：20分，结帖人wenyang2004]

原文地址：https://www.cnblogs.com/java20130722/p/3206920.html