#Hadoop集群部署

#

1. 配置JAVA环境（vi/etc/profile.d/java.sh）

[root@master ~]# yum install java-1.8.0-openjdk-devel

 

[root@master ~]# vi /etc/profile.d/java.sh
添加JAVA环境变量
export JAVA_HOME=/usr

或者编译安装。

export JAVA_HOME=/usr/java/jdk1.8.0_151
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin

node节点也需要配置JAVA环境。

2.创建hadoop用户

[root@master hadoop]# useradd hadoop
[root@master hadoop]#echo 'hadoop'|passwd -stdin hadoop

node2，node3，node4都需要创建。

3. 配置hosts（/etc/hosts）

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.15.128 master192.168.15.129 node2
192.168.15.130  node3
192.168.15.131 node4

node2，node3，node4都需要配置。

 

4. 配置master免密登录node节点

[root@master ~]# su - hadoop

[hadoop@master ~]$ ssh-keygen -t rsa -P ''

[hadoop@master ~]$ for i in 2 3 4;do ssh-copy-id -i .ssh/id_rsa_pub hadoop@node${i};done 

 

5.  解压hadoop安装包

[root@master ~]# tar -zxvf hadoop-2.8.2.tar.gz -C /usr/local/
[root@master ~]# cd /usr/local
[root@master ~]# ln -sv hadoop-2.8.2 hadoop

 

6. 创建数据目录和日志目录

[root@master ~]# mkdir -pv /data/hadoop/hdfs/{name,sname,data}
[root@master ~]# chown -R hadoop:hadoop /data
创建日志目录
[root@master ~]# mkdir /usr/local/hadoop/logs
[root@master ~]# chown -R hadoop:hadoop /usr/local/hadoop/
[root@master ~]# chmod g+x /usr/local/hadoop/logs

 

7. 配置hadoop配置文件（/usr/local/hadoop/etc/hadoop/）下面的配置Node节点也需要同样的配置，master配置完，

可以通过命令[hadoop@master ~]$ scp /usr/local/hadoop/etc/hadoop/*  hadoop@node2:/usr/local/hadoop/etc/hadoop/

core-site.xml

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master:8082</value>
        </property>
</configuration>

hdfs-site.xml

<configuration>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>file:///data/hadoop/hdfs/name</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>file:///data/hadoop/hdfs/data</value>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>fs.checkpoint.dir</name>
                <value>file:///data/hadoop/hdfs/sname</value>
        </property>
        <property>
                <name>fs.checkpoint.edits.dir</name>
                <value>file:///data/hadoop/hdfs/sname</value>
        </property>
</configuration>

 yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
 <property>
         <name>yarn.resourcemanager.address</name>
        <value>master:8032</value>
</property>
<property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>master:8030</value>
</property>
<property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>master:8031</value>
</property>
<property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>master:8033</value>
</property>
<property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>master:8088</value>
</property>
<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
</property>
<property>
        <name>yarn.nodemanager.auxservices.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
        <name>yarn.resourcemanager.scheduler.class</name>
        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>
</configuration>

mapred-site.xml

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

 slaves

node2
node3
node4

hadoop-env.sh(需要添加JAVA环境变量)

export JAVA_HOME=/usr/java/jdk1.8.0_151

8. 格式化HDFS

[root@master ~]# su - hadoop
[hadoop@master ~]$ hdfs namenode -format

输出结果为 17/11/13 23:13:47 INFO common.Storage: Storage directory /data/hadoop/hdfs/name has been successfully formatted.
说明格式化成功了。

9. 启动dfs

[hadoop@master ~]$ start-dfs.sh
Starting namenodes on [master]
master: starting namenode, logging to /bdapps/hadoop/logs/hadoop-hadoop-namenode-master.out
node2: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node2.out
node3: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node3.out
node4: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node4.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /bdapps/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out

10. 上传一个文件（/etc/fstab）

[hadoop@master ~]$ hdfs dfs -mkdir /test
[hadoop@master ~]$ hdfs dfs -put /etc/fstab /test
[hadoop@master ~]$ hdfs dfs -ls /test
Found 1 items
-rw-r--r--   2 hadoop supergroup        779 2017-11-14 00:03 /test/fstab
[hadoop@master ~]$ hdfs dfs -cat /test/fstab

#
# /etc/fstab
# Created by anaconda on Mon Nov 13 08:43:43 2017
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/VolGroup-lv_root /                       ext4    defaults        1 1
UUID=98ed038f-f133-4809-bad8-c4139402b11b /boot                   ext4    defaults        1 2
/dev/mapper/VolGroup-lv_swap swap                    swap    defaults        0 0
tmpfs                   /dev/shm                tmpfs   defaults        0 0
devpts                  /dev/pts                devpts  gid=5,mode=620  0 0
sysfs                   /sys                    sysfs   defaults        0 0
proc                    /proc                   proc    defaults        0 0
[hadoop@master ~]$ 

11. 启动yarn

[hadoop@master ~]$ start-yarn.sh 
starting yarn daemons
starting resourcemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-resourcemanager-master.out
node4: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node4.out
node3: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node3.out
node2: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node2.out

可以通过命令jps查看java进程

Master节点

[hadoop@master ~]$ jps
7600 ResourceManager
7106 NameNode
8483 Jps
7304 SecondaryNameNode

Node节点

[root@node4 ~]# jps
2802 Jps
2678 NodeManager
2553 DataNode

12. 访问

hadoop总览
http://192.168.15.128:50070

查看hadoop的应用
http://192.168.15.128:8088

#