[Hadoop][Hadoop]Cluster and HA

#!/bin/bash

 

# 时间服务器

yum install -y ntp

systemctl enable ntpd

systemctl start ntpd

 

# 免密登录

ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa

cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

chmod 0600 ~/.ssh/authorized_keys

ssh-copy-id -i /home/hadoop/.ssh/id_dsa.pub $USER@$HOSTNAME

 

# HADOOP HOST

zookeeper01=10.211.55.101

zookeeper02=10.211.55.102

zookeeper03=10.211.55.103

journalnode01=10.211.55.101

journalnode02=10.211.55.102

journalnode03=10.211.55.103

namenode01=10.211.55.101

namenode02=10.211.55.102

datanode01=10.211.55.101

datanode02=10.211.55.102

datanode03=10.211.55.103

resourcemanager01=10.211.55.101

resourcemanager02=10.211.55.102

 

# HOSTS 文件

cat <<EOF | tee /etc/hosts

$zookeeper01    zookeeper01

$zookeeper02    zookeeper02

$zookeeper03    zookeeper03

$journalnode01    journalnode01

$journalnode02    journalnode02

$journalnode03    journalnode03

$namenode01    namenode01

$namenode02    namenode02

$datanode01    datanode01

$datanode02    datanode02

$datanode03    datanode03

$resourcemanager01    resourcemanager01

$resourcemanager02    resourcemanager02

EOF

 

# JAVA JDK

rpm -ivh /tmp/jdk-8u131-linux-x64.rpm

 

# JAVA 环境变量

echo 'JAVA_HOME=/usr/java/default' >> /etc/profile

echo 'JRE_HOME=$JAVA_HOME/jre' >> /etc/profile

echo 'CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib' >> /etc/profile

echo 'PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin' >> /etc/profile

echo 'export JAVA_HOME JRE_HOME PATH CLASSPATH' >> /etc/profile

source /etc/profile

 

# 修改 JAVA DNS 缓存

sed -i 's/#networkaddress.cache.ttl=-1/networkaddress.cache.ttl=60/' $JRE_HOME/lib/security/java.security

 

# ZOOKEEPER 国内安装文件下载地址

# ZOOKEEPER_WEB_FILE=https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz

 

# 是否下载 ZOOKEEPER 安装文件

# wget -P /tmp $ZOOKEEPER_WEB_FILE

 

# ZOOKEEPER 安装文件

# ZOOKEEPER_INSTALL_FILE=/tmp/zookeeper-3.4.10.tar.gz

 

# ZOOKEEPER 目录

# ZOOKEEPER_INSTALL_DIR=/opt/zookeeper-3.4.10

# ZOOKEEPER_HOME=/opt/zookeeper

 

# 安装 ZOOKEEPER

tar -C /opt -xf $ZOOKEEPER_INSTALL_FILE

ln -s $ZOOKEEPER_INSTALL_DIR $ZOOKEEPER_HOME

 

# 创建组和用户

groupadd zookeeper

useradd -g zookeeper zookeeper -s /sbin/nologin

 

# 文件存放目录

mkdir -p /mnt/zookeeper/data

mkdir -p /var/log/zookeeper

chown zookeeper:zookeeper -R /mnt/zookeeper/data

chown zookeeper:zookeeper -R /var/log/zookeeper

 

#创建配置文件

cat <<EOF | tee $ZOOKEEPER_HOME/conf/zoo.cfg

tickTime=2000

initLimit=10

syncLimit=5

dataDir=/mnt/zookeeper/data

dataLogDir=/var/log/zookeeper

clientPort=2181

server.1=zookeeper01:2888:3888

server.2=zookeeper02:2888:3888

server.3=zookeeper03:2888:3888

EOF

 

chown zookeeper:zookeeper -R $ZOOKEEPER_INSTALL_DIR

chown zookeeper:zookeeper -R $ZOOKEEPER_LINK_DIR

 

#systemd开机配置

cat <<EOF | tee /usr/lib/systemd/system/zookeeper.service 

[Unit]

Description=zookeeper

After=syslog.target network.target

 

[Service]

Type=forking

Environment=ZOO_LOG_DIR=/var/log/zookeeper

# Environment=ZOO_LOG4J_PROP="INFO,ROLLINGFILE"

ExecStart=/opt/zookeeper/bin/zkServer.sh start

ExecStop=/opt/zookeeper/bin/zkServer.sh stop

Restart=always

User=zookeeper

Group=zookeeper

 

[Install]

WantedBy=multi-user.target

EOF

 

#每台zookeeper的id不一样

# echo "1" > /mnt/zookeeper/data/myid

# echo "2" > /mnt/zookeeper/data/myid

# echo "3" > /mnt/zookeeper/data/myid

 

systemctl enable zookeeper

systemctl start zookeeper

systemctl status zookeeper

 

firewall-cmd --zone=public --add-port=2181/tcp --permanent

firewall-cmd --zone=public --add-port=2888/tcp --permanent

firewall-cmd --zone=public --add-port=3888/tcp --permanent

firewall-cmd --reload

 

#检测是否可用

#echo conf | nc $HOSTNAME 2181

 

 

# HADOOP 国内安装文件下载地址

# HADOOPO_WEB_FILE=https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz

# HADOOPO_WEB_FILE=https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.8.0/hadoop-2.8.0.tar.gz

 

# 是否下载 HADOOP 安装文件

# wget -P /tmp $HADOOPO_WEB_FILE

 

# HADOOP 安装文件

# HADOOP_INSTALL_FILE=/tmp/hadoop-2.8.0.tar.gz

 

# HADOOP 目录

# HADOOP_INSTALL_DIR=/opt/hadoop-2.8.0

# HADOOP_HOME=/opt/hadoop

# HADOOP_WORK_DIR=/mnt/hadoop

 

# 安装依赖包

# OPEN SSL

yum install -y openssl-devel

 

# 安装 HADOOP

if [ -f $HADOOP_INSTALL_FILE ]; then

    tar -C /opt -xf $HADOOP_INSTALL_FILE

    ln -s $HADOOP_INSTALL_DIR /opt/hadoop

else

    echo 'require Hadoop install file'

    return

fi

 

# 创建组和用户

groupadd hadoop

useradd -g hadoop hadoop -s /bin/bash

 

# 文件存放目录

mkdir -p $HADOOP_WORK_DIR/name

mkdir -p $HADOOP_WORK_DIR/data

mkdir -p $HADOOP_WORK_DIR/journal

mkdir -p $HADOOP_WORK_DIR/tmp

mkdir -p $HADOOP_HOME/run

chown hadoop:hadoop -R $HADOOP_INSTALL_DIR

chown hadoop:hadoop -R $HADOOP_HOME

chown hadoop:hadoop -R $HADOOP_WORK_DIR

 

 

# 系统变量

if [ -z $HADOOP_PREFIX ]; then 

    echo "HADOOP_HOME=/opt/hadoop" >> /etc/profile

    echo 'HADOOP_PREFIX=$HADOOP_HOME' >> /etc/profile

    echo 'HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadooop' >> /etc/profile

    echo 'PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' >> /etc/profile

    echo 'HADOOP_PID_DIR=$HADOOP_PREFIX/run' >> /etc/profile

    echo 'YARN_PID_DIR=$HADOOP_PREFIX/run' >> /etc/profile

    echo "export HADOOP_HOME HADOOP_PREFIX HADOOP_CONF_DIR HADOOP_PID_DIR YARN_PID_DIR PATH" >> /etc/profile

    source /etc/profile

else

    echo "HADOOP_PREFIX=$HADOOP_PREFIX"

fi

 

# core-site.xml 初始配置

cat <<EOF | tee $HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://nameservices</value>

    </property>

    <property>

        <name>hadoop.tmp.dir</name>

        <value>/mnt/hadoop/tmp</value>

    </property>

    <property>

        <name>dfs.journalnode.edits.dir</name>

        <value>/mnt/hadoop/journal</value>

    </property>

    <property>

        <name>ha.zookeeper.quorum</name>

        <value>zookeeper01:2181,zookeeper02:2181,zookeeper03:2181</value>

    </property>

</configuration>

EOF

 

# hdfs-site.xml初始配置

cat <<EOF | tee $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>

    <property>

        <name>dfs.namenode.name.dir</name>

        <value>/mnt/hadoop/name</value>

    </property>

    <property>

        <name>dfs.datanode.data.dir</name>

        <value>/mnt/hadoop/data</value>

    </property>

    <property>

        <name>dfs.replication</name>

        <value>3</value>

    </property>

    <property>

        <name>dfs.nameservices</name>

        <value>nameservices</value>

    </property>

    <property>

        <name>dfs.ha.namenodes.nameservices</name>

        <value>nn1,nn2</value>

    </property>

    <property>

        <name>dfs.namenode.rpc-address.nameservices.nn1</name>

        <value>namenode01:8020</value>

    </property>

    <property>

        <name>dfs.namenode.rpc-address.nameservices.nn2</name>

        <value>namenode02:8020</value>

    </property>

    <property>

        <name>dfs.namenode.http-address.nameservices.nn1</name>

        <value>namenode01:50070</value>

    </property>

    <property>

        <name>dfs.namenode.http-address.nameservices.nn2</name>

        <value>namenode02:50070</value>

    </property>

    <property>

        <name>dfs.namenode.shared.edits.dir</name>

        <value>qjournal://journalnode01:8485;journalnode02;journalnode03:8485/nameservices</value>

    </property>

    <property>

        <name>dfs.client.failover.proxy.provider.nameservices</name>

        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

    </property>

    <property>

        <name>dfs.ha.fencing.methods</name>

        <value>sshfence</value>

    </property>

    <property>

        <name>dfs.ha.fencing.ssh.private-key-files</name>

        <value>/home/hadoop/.ssh/id_rsa</value>

    </property>

     <property>

        <name>dfs.ha.automatic-failover.enabled</name>

        <value>true</value>

     </property>

</configuration>

EOF

 

# mapred-site.xml初始配置

cat <<EOF | tee $HADOOP_HOME/etc/hadoop/mapred-site.xml

<configuration>

    <property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

    </property>

</configuration>

EOF

 

# yarn-site.xml初始配置

cat <<EOF | tee $HADOOP_HOME/etc/hadoop/yarn-site.xml

<configuration>

    <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.enabled</name>

        <value>true</value>

    </property>

    <property>

        <name>yarn.resourcemanager.cluster-id</name>

        <value>resouceCluster</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.rm-ids</name>

        <value>rm1,rm2</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm1</name>

        <value>resourcemanager01</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm2</name>

        <value>resourcemanager02</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm1</name>

        <value>resourcemanager01:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm2</name>

        <value>resourcemanager02:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.zk-address</name>

        <value>zookeeper01:2181,zookeeper02:2181,zookeeper03:2181</value>

</property>

</configuration>

EOF

 

# systems 开机配置 journalnode

cat <<EOF | tee /usr/lib/systemd/system/hdfs-journalnode.service

[Unit]

Description=Hadoop DFS Journal Node

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs start journalnode

ExecStop=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs stop journalnode

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PID_DIR=$HADOOP_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=8485/tcp --permanent

firewall-cmd --zone=public --add-port=8480/tcp --permanent

firewall-cmd --reload

 

systemctl enable hdfs-journalnode

systemctl start hdfs-journalnode

systemctl status hdfs-journalnode

 

# systems 开机配置 zkfc

cat <<EOF | tee /usr/lib/systemd/system/hdfs-zkfc.service

[Unit]

Description=Hadoop DFS HA state in ZooKeeper

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs start zkfc

ExecStop=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs stop zkfc

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PID_DIR=$HADOOP_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=8019/tcp --permanent

firewall-cmd --reload

 

su hadoop - -c "hdfs zkfc -formatZK"

 

systemctl enable hdfs-zkfc

systemctl start hdfs-zkfc

systemctl status hdfs-zkfc

 

# systems 开机配置 namenode

cat <<EOF | tee /usr/lib/systemd/system/hdfs-nameode.service

[Unit]

Description=Hadoop DFS Name Node

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs start namenode

ExecStop=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs stop namenode

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PID_DIR=$HADOOP_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=8020/tcp --permanent

firewall-cmd --zone=public --add-port=50070/tcp --permanent

firewall-cmd --reload

 

# 每个NameNode执行

su hadoop - -c "hdfs namenode -format"

 

systemctl enable hdfs-nameode

systemctl start hdfs-nameode

systemctl status hdfs-nameode

 

su hadoop - -c "hdfs haadmin -getServiceState nn1"

su hadoop - -c "hdfs haadmin -getServiceState nn2"

 

 

# systems 开机配置 datanode

cat <<EOF | tee /usr/lib/systemd/system/hdfs-datanode.service

[Unit]

Description=Hadoop DFS Data Node

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs start datanode

ExecStop=/opt/hadoop/sbin/hadoop-daemon.sh --script hdfs stop datanode

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PID_DIR=$HADOOP_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=50010/tcp --permanent

firewall-cmd --zone=public --add-port=50020/tcp --permanent

firewall-cmd --zone=public --add-port=50075/tcp --permanent

firewall-cmd --reload

 

systemctl enable hdfs-datanode

systemctl start hdfs-datanode

systemctl status hdfs-datanode

 

# systems 开机配置 resourcemanager

cat <<EOF | tee /usr/lib/systemd/system/yarn-resourcemanager.service

[Unit]

Description=Hadoop YARN Resource Manager

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/yarn-daemon.sh start resourcemanager

ExecStop=/opt/hadoop/sbin/yarn-daemon.sh stop resourcemanager

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=YARN_PID_DIR=$YARN_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=8030/tcp --permanent

firewall-cmd --zone=public --add-port=8031/tcp --permanent

firewall-cmd --zone=public --add-port=8032/tcp --permanent

firewall-cmd --zone=public --add-port=8033/tcp --permanent

firewall-cmd --zone=public --add-port=8088/tcp --permanent

firewall-cmd --reload

 

systemctl enable yarn-resourcemanager

systemctl start yarn-resourcemanager

systemctl status yarn-resourcemanager

 

# systems 开机配置 nodemanager

cat <<EOF | tee /usr/lib/systemd/system/yarn-nodemanager.service

[Unit]

Description=Hadoop YARN Node Manager

After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target

Requires=network-online.target

 

[Service]

User=hadoop

Group=hadoop

Type=forking

ExecStart=/opt/hadoop/sbin/yarn-daemon.sh start nodemanager

ExecStop=/opt/hadoop/sbin/yarn-daemon.sh stop nodemanager

Environment=JAVA_HOME=$JAVA_HOME

Environment=HADOOP_HOME=$HADOOP_HOME

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=HADOOP_PREFIX=$HADOOP_PREFIX

Environment=YARN_PID_DIR=$YARN_PID_DIR

Restart=on-failure

 

[Install]

WantedBy=multi-user.target

EOF

 

firewall-cmd --zone=public --add-port=8040/tcp --permanent

firewall-cmd --zone=public --add-port=8041/tcp --permanent

firewall-cmd --zone=public --add-port=8042/tcp --permanent

firewall-cmd --reload

 

systemctl enable yarn-nodemanager

systemctl start yarn-nodemanager

systemctl status yarn-nodemanager