清晰安装Hadoop/Hbase/Thrift

//以下是本人在CentOS 6.4系统版本下的实践对Hadoop/Hbase/Thrift安装记录和总结

 

1，准备安装包：
    http://ohse.de/uwe/software/lrzsz.html

    http://archive.apache.org/dist/hadoop/core/hadoop-1.0.4/

    http://mirrors.cnnic.cn/apache/hbase/stable/

    http://mirrors.cnnic.cn/apache/hbase/hbase-0.94.9/

    https://github.com/downloads/libevent/libevent/libevent-2.0.20-stable.tar.gz

    http://www.python.org/ftp/python/2.7.5/Python-2.7.5.tar.bz2

    http://sourceforge.net/projects/boost/files/boost/1.52.0/boost_1_52_0.tar.gz/download

    http://thrift.apache.org/download/

2, 安装与配置Hadoop
    http://book.51cto.com/art/201110/298590.htm

    1> 安装jdk:
        #tar -xvf targz/jdk-7u25-linux-x64.tar.gz
        #mkdir /usr/local/jvm/java-7-sun -p
        #mv jdk1.7.0_25/* /usr/local/jvm/java-7-sun/
        #vi ~/.bashrc
            export JAVA_HOME=/usr/local/jvm/java-7-sun
            export JRE_HOME=${JAVA_HOME}/jre
            export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib:${CLASSPATH}
            export PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin:${PYTHONHOME}/bin:${THRIFT_HOME}/bin:${PATH}
        #. ~/.bashrc
        #java -version
            java version "1.7.0_25"
            Java(TM) SE Runtime Environment (build 1.7.0_25-b15)
            Java HotSpot(TM) 64-Bit Server VM (build 23.25-b01, mixed mode)
        // 说明安装成功

    2> 配置SSH免密码登录
        # //修改hosts文件
        #vi /etc/hosts
            192.168.0.20 master
            192.168.0.21 slave1
            192.168.0.22 slave1

        #// 找到以下内容，并去掉注释符”#“
        #vi /etc/ssh/sshd_config
            RSAAuthentication yes
            PubkeyAuthentication yes
            AuthorizedKeysFile      .ssh/authorized_keys
        #// 若ssh端口为非22，则修改脚本内容
        #vi /usr/bin/ssh-copy-id
            { eval "$GET_ID" ; } | ssh -p myport $1 "umask 077; test -d ~/.ssh || mkdir
        #// 接下来就生成免密码登录态文件
            ssh-keygen -t rsa -P ''
            ssh-copy-id -i ~/.ssh/id_rsa.pub root@master
            ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave2
            ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave1

    3> 安装hadoop
        #//解压hadoop-1.0.4.tar.gz
        #tar -xvf hadoop-1.0.4.tar.gz
        #mv hadoop-1.0.4/ /usr/local/hadoop/
        #//修改文件句柄限制
        #vi /etc/security/limits.conf
            *               -        nofile          65535
            *               soft    nproc           32000
            *               hard   nproc           32000
        #//修改当前主机名，不然在格式化时会出现这种错误，如下：
        #//SHUTDOWN_MSG: Shutting down NameNode at java.net.UnknownHostException: hadoop1: hadoop1
        #vi /etc/sysconfig/network
            NETWORKING=yes
            HOSTNAME=master
        #//修改用户环境变量
        #vi ~/.bashrc
            export HADOOP_HOME=/usr/local/hadoop/hadoop-1.0.4
            #将$HADOOP_HOME目录下的jar包全部添加
            for i in $HADOOP_HOME/*.jar
            do
            CLASSPATH=$CLASSPATH:$i
            done

            #将$HADOOP_HOME/lib目录下的jar全部添加
            for i in $HADOOP_HOME/lib/*.jar
            do
            CLASSPATH=$CLASSPATH:$i
            done

        #//修改hadoop-env.sh配置文件   
        #cd /usr/local/hadoop/hadoop-1.0.4/
        #vi conf/hadoop-env.sh
            export JAVA_HOME=/usr/local/jvm/java-7-sun/
        #mkdir /usr/local/hadoop/hadoop-1.0.4/pids
            export HADOOP_PID_DIR=/usr/local/hadoop/hadoop-1.0.4/pids
            #取消环境变量$HADOOP_HOME过时的警告
            export HADOOP_HOME_WARN_SUPPRESS=1
            #修改对应的免登录的ssh端口，因为上面配置ssh免登录时的端口修改了
            #http://blog.chenlb.com/2008/12/change-hadoop-ssh-port.html
            export HADOOP_SSH_OPTS="-p myport"
        #//修改core-site.xml配置文件 
        #vi conf/core-site.xml
            <configuration>
                <property>
                    <name>fs.default.name</name>
                    <value>hdfs://master:9000</value>
                </property>

                <!-- 这个建议加上，否则机器重启后，所有数据将会丢失 -->
                <property>
                    <name>hadoop.tmp.dir</name>
                    <value>/usr/local/hadoop/hadoop-1.0.4/tmp</value>
                </property>
            </configuration>
        #vi conf/hdfs-site.xml
            <property>
                <name>dfs.name.dir</name>
                <value>/usr/local/hadoop/hadoop-1.0.4/name</value>
                <description>
                    这里有的说要创建name目录有的说不要创建(建议不创建)
                </description>
            </property>

            <property>
                <name>dfs.data.dir</name>
                <value>/u01/hadoopdata,/u02/hadoopdata,/u03/hadoopdata,/u04/hadoopdata</value>
                <description>
                    DataNode上存储数据块的地方。如果指定多个目录(逗号分割)，则数据库被随机的存放。
                </description>
            </property>

            <property>
                <name>dfs.replication</name>
                <value>3</value>
            </property>

            <property>
                <name>dfs.datanode.max.xcievers</name>
                <value>4096</value>
                <description>
                    datanode同时打开的文件上限。默认256太小。
                </description>
            </property>
        #vi conf/mapred-site.xml
            <property> 
                <name>mapred.job.tracker</name>
                <value>hdfs://master:9001</value>
            </property>
        #//格式化namenode
        #./bin/hadoop namenode -format
        #//若是分布式系统，则要把当前目录下的所有文件scp到各个datanode机器上
        #scp -r * root@slave1:/usr/local/hadoop/hadoop-1.0.4

        注意：所有的slave机器上的配置建议都与master一致，即把上述配置文件和hadoop目录都scp到所有的slave机器上  

    4> 运行
        #cd /usr/local/hadoop/hadoop-1.0.4/bin
        #./start-all.sh

    5> 页面验证
        http://192.168.100.20:50030 (MapReduce的Web页面) 
        http://192.168.100.20:50070 (HDFS的Web页面)
        http://192.168.100.20:50060 (Task Tracker Status)

    6> 后的进程名称：
        * NameNode
        * SecondaryNameNode
        * DataNode
        * JobTracker
        * TaskTracker

3, 安装与配置Hbase
    1>安装Hbase
        #//解压hadoop-1.0.4.tar.gz
        #tar -xvf hbase-0.94.9.tar.gz
        #mv hbase-0.94.9/ /usr/local/hadoop/
        #vi ~/.bashrc
            export HBASE_HOME=/usr/local/hadoop/hbase-0.94.9
        #cd /usr/local/hadoop/hbase-0.94.9
        #vi conf/hbase-env.sh
            export JAVA_HOME=/usr/local/jvm/java-7-sun/
        #//下面的HBASE_CLASSPATH是方便hbase和hadoop进行关联
            export HBASE_CLASSPATH=/usr/local/hadoop/hadoop-1.0.4/conf
            export HBASE_PID_DIR=/usr/local/hadoop/hbase-0.94.9/pids
            export HBASE_MANAGES_ZK=true
        #修改对应的免登录的ssh端口，因为上面配置ssh免登录时的端口修改了
        # http://blog.chenlb.com/2008/12/change-hadoop-ssh-port.html
            export HBASE_SSH_OPTS="-p"myport
        #vi conf/hbase-site.xml
            <property>
                <name>hbase.rootdir</name>
                <value>hdfs://master:9000/hbase</value>
                <description>
                    设置hbase在hdfs上的目录，主机名为hdfs的namenode节点所在的主机
                </description>
            </property>
            <property>
                <name>hbase.cluster.distributed</name>
                <value>true</value>
                <description>
                    设置为true，表明是完全分布式的hbase集群,伪分布式模式下也需要设置为true
                </description>
            </property>
            <property>
                <name>hbase.zookeeper.property.dataDir</name>
                <value>/usr/local/hadoop/hbase-0.94.9/zookeeper</value>    <description>
                    ZooKeeper保存数据的目录地址
                </description>
            </property>
            <property>
                <name>hbase.zookeeper.quorum</name>
                <value>slave1,slave2</value>
                <description>
                    设置zookeeper的主机，建议使用单数
                </description>
            </property>
            <property>
                <name>hbase.client.write.buffer</name>
                <value>209715200</value>
                <description>
                    HTable客户端的写缓冲的默认大小
                </description>
            </property>
            <property>
                <name>hbase.client.keyvalue.maxsize</name>
                <value>209715200</value>
                <description>
                    一个KeyValue实例的最大size
                </description>
            </property>
        #vi regionservers
            slave1

    2> 运行
        #cd /usr/local/hadoop/hbase-0.94.9/bin
        #./start-hbase.sh

    3> 页面验证
        http://192.168.100.20:60010/master-status

    4> 后的进程名称：
        * HQuorumPeer
        * HMaster
        * HRegionServer

    5> 伪分布式配置（以上是完全分布式架构的搭建，如果是伪分布，则要修改下面内容，其他与完全分布式一样)
        #vi hbase-site.xml
            <property>
                <name>hbase.zookeeper.quorum</name>
                <value>localhost</value>
                <description>
                    设置zookeeper的主机，建议使用单数
                </description>
            </property>
        #vi regionservers
            master

        如果不按照上面的配置，会出现各种奇怪问题，如下日志：
            FATAL org.apache.hadoop.hbase.master.HMaster: Unhandled exception
            org.apache.hadoop.hbase.master.HMaster: Unhandled exception. Starting shutdown

 

4, 上述所涉及到的配置文件列表

　　

5, 安装thrift及依赖库
    #tar -xvf libevent-2.0.20-stable.tar.gz
    #./configure --prefix=/usr/local/libevent
    #mk
    #mk install

    #tar -xvf Python-2.7.5.tar.bz2
    #./configure --prefix=/usr/local/python
    #mk
    #mk install

    #tar -xvf boost_1_52_0.tar.gz
    #./bootstrap.sh --prefix=/usr/local/boost --with-python=/usr/local/python
    #./b2
    #./b2 install

    ============================================
    boost编译问题：
    wrap_python.hpp:50:23: error: pyconfig.h: No such file or directory
    有可能是做过一次./bootstrap.sh(后面没有带参数)导致，这时要删除目录再来一次带参数的配置化
    ============================================

    #tar -xvf thrift-0.9.0.tar.gz
    #export CPPFLAGS="-DHAVE_NETDB_H=1 -fpermissive"
    #./configure --with-boost=/usr/local/boost/ --with-qt4=false --with-java=/usr/local/jvm/java-7-sun/ --with-libevent=/usr/local/libevent/ --prefix=/usr/local/thrift
    #mk
    #mk install

6, 其他
    1>启动hadoop的thrift服务(用于HDFS的访问)
        a> 先需修改start_thrift_server.sh
            #vi /usr/local/hadoop/hadoop-1.0.4/src/contrib/thriftfs/scripts/start_thrift_server.sh
            修改1：
                将
                    for f in $TOP/build/*.jar ; do
                改为:
                    for f in $TOP/*.jar ; do
            修改2：
                将
                    for f in $TOP/build/contrib/thriftfs/*.jar ; do
                改为:
                    for f in $TOP/src/contrib/thriftfs/*.jar ; do
        b> 启动thrift服务进程
            #/usr/local/hadoop/hadoop-1.0.4/src/contrib/thriftfs/scripts/start_thrift_server.sh #thriftport#
    2> 启动hbase的thrift服务
        a> 先确保hadoop的的状态为：Safe mode is OFF
            #/usr/local/hadoop/hadoop-1.0.4/bin/hadoop dfsadmin -safemode leave
        b> 启动thrift服务进程
            #/home/hadoop/hbase/bin/hbase thrift start

            或者

            #./hbase-daemon.sh start thrift

 

     以上内容也使用evernote作了分享，可以下载到上述中的配置文件《清晰安装Hadoop/Hbase/Thrift--EverNote版》。

    说明：配置并安装好hadoop/hbase/thrift，就可以使用各种编程语言进行对hbase或hadoop的访问的开发了。开发步骤稍候详见另外一篇帖子《CPP/PHP访问hbase的代码开发》。