lvs+keepalive高可用web应用部署

环境：

192.168.92.183  7.3.1611
192.168.92.184  7.3.1611
192.168.92.185  7.3.1611

一、系统标准化

1、安装必要软件

yum -y install wget vim lrzsz unzip

2、下载标准版包

$ cd /usr/local/src
$ wget http://182.138.101.48:51280/package/tar/nn_sys_init-20180605.tar.gz

3、修改config.ini配置文件

vim config.ini

#[SYSTEM]
NN_HOSTNAME='SZTW_CMS_node3'                          # 主机名，必须填写

NTP_SERVER=0.pool.ntp.org                  # NTP服务器地址，默认为 cn.pool.ntp.org
NTP_CRON="*/30 * * * *"                 # NTP同步周期，规则同计划任务一致，默认30分钟同步一次

TIMEZONE="Asia/Shanghai"                # 时区注意大小写

DATA_DIST_MOUNT_DIR="/data"             # 数据存放目录，不建议修改

YUM_REPO_FILE_PATH="http://182.138.101.48:21689/centos/7"                        # 自定义YUM源URL: http://192.168.95.51/centos
SYSTEM_MANAGE_TOOLS="iotop lsof"                # 需要安装的管理工具，用空格隔开

#[ZABBIX]
ZABBIX_SERVER_IP=""               # zabbix服务器/代理服务器地址,留空将不安装zabbix

STARCOR_WORK_DIR_TREE="live m3u8 starcor/server store vod logs starcor/www"

#[SSH]
ENABLE_INIT_SSH_KEY="1"                 # 是否初始化密钥登录，1为初始化，0为不初始化

3、执行脚本

bash nn_install.sh

#!/bin/env bash
#
# 系统初始化、主要功能包括：NTP、用户管理、时区、服务、zabbix、历史记录
#
# FileName: nn_starcor_init
# Author: zhongcheng.yang@starcor.cn
# Date: 2015-10-19
# Dependencies: Null
# ChangeLog:
# 2015-10-19: 新建
# 2015-11-05: 调整执行逻辑，执行时询问初始化信息，增加zabbix初始化
# 2016-02-19: 修改程序逻辑，所有配置信息改为配置文件
# 版本：20160219

#初始化默认配置信息
. config.ini

while getopts tfra:h:n:c:m:y:s:z: opt; do
    # h: NN_HOSTNAME
    # n: NTP_SERVER
    # c: NTP_CRON
    # t: TIMEZONE
    # y: YUM_REPO_FILE_PATH
    # s: SYSTEM_MANAGE_TOOLS
    # z: ZABBIX_SERVER_IP
    # i: INIT_STARCOR_WORK_TYPE
    case ${opt} in
        a) {
            ACTION=${OPTARG}
        };;
        h) {
            NN_HOSTNAME=${OPTARG}
        };;
        n) {
            NTP_SERVER=${OPTARG}
        };;
        c) {
            NTP_CRON=${OPTARG}
        };;
        m) {
            TIMEZONE=${OPTARG}
        };;
        y) {
            YUM_REPO_FILE_PATH=${OPTARG}
        };;
        s) {
            SYSTEM_MANAGE_TOOLS=${OPTARG-3306}
        };;
        z) {
        	ZABBIX_SERVER_IP=${OPTARG}
        };;
        t) {
            EXEC_FROM_OPM=opm
            ACTION=install
        };;
        f) {
            FORCE_INSTALL=yes
        };;
        r) {
            NEED_REBOOT=yes
        };;
    esac
done

sys_release=$(uname -r)

echo ${sys_release} |grep "el6" |grep -v grep >/dev/null && sys_version="el6"
echo ${sys_release} |grep "el7" |grep -v grep >/dev/null && sys_version="el7"

[ -z "${sys_version}" ] && { echo "不支持的系统版本"; exit 1; }

err_log_file=/tmp/stand_$$.log
touch ${err_log_file}
#install_dir=/usr/local/redis

# 设置错误和标准输出重定向
set_redirect() {
    exec 3>&1 1>/dev/null
    exec 4>&2 2>${err_log_file}
}
# 恢复错误和标准输出重定向
unset_redirect() {
    exec 1>&3 3>&-
    exec 2>&4 4>&-
}

# 设置提示颜色
color_err(){
    if [ "${EXEC_FROM_OPM}" = "opm" ];then
        unset_redirect
        echo '[WORK_RESULT]{"status":"failure", "message":"'"$1"'","errorDetail":"'"$(cat ${err_log_file})"'"}'
        rm -rf ${err_log_file}
    else
        echo -e "33[1;31m$133[0m"
    fi
}

color_ok(){
    if [ "${EXEC_FROM_OPM}" = "opm" ];then
        unset_redirect
        echo '[WORK_RESULT]{"status":"success", "message":"'"$1"'"}'
    else
        echo -e "33[1;35m$133[0m"
    fi
}

color_start(){
    [ "${EXEC_FROM_OPM}" = "opm" ] || { echo -e "33[1;35m$133[0m"; }
}

#is_package_installed(){
#    [ "${FORCE_INSTALL}" != "yes" -a -d "${install_dir}" ] && color_err "目录${install_dir}已经存在，退出安装"
#}

exit_code_check(){
    [ $? -ne 0 ] && color_err $1
}

# 基础判断，权限、参数
[ $(id -u) -ne 0 ] && { color_err '请以root身份执行该脚本'; }

[ "${EXEC_FROM_OPM}" = "opm" ] && set_redirect

[ -z "${NN_HOSTNAME}" ] && { color_err "未指定主机名，退出初始化"; }

# 清空现有历史记录
> ~/.bash_history

readonly WORK_DIR=$(pwd)
readonly SERVER_VERSION=$(awk '{print $3}' /etc/redhat-release)
BACKUP_DIR=${WORK_DIR}'/backup/'
NOT_USED_SERVICE="auditd netfs postfix"
NOT_USED_SERVICE_EL7="auditd postfix"
REQUIRED_TOOLS_DEFAULT="dmidecode net-tools bc make libpcap-devel wget lrzsz rsync vim sysstat ntpdate openssh-clients net-snmp-devel openssl libselinux-python"
TOOLS_DIR=${WORK_DIR}'/tools/'
ZABBIX_DIR=${WORK_DIR}'/zabbix/'
HOST_NAME=${NN_HOSTNAME}
LOG_FILE="result.log"
# 备份功能
[ ! -d ${BACKUP_DIR} ] && mkdir ${BACKUP_DIR}
function backup_file(){
    cp $1 ${BACKUP_DIR}
}

# 系统基础初始化开始
function sys_init(){

    # 更新YUM源
    if [ 'a'${YUM_REPO_FILE_PATH} != 'a' ];then
        BS_REPO="/etc/yum.repos.d/CentOS-Base.repo"
        backup_file ${BS_REPO}
#        cp -Rf ${YUM_REPO_FILE_PATH} ${BS_REPO} && log_echo "YUM源配置: 成功" ok || { log_echo "YUM源配置: 失败, 程序退出" err; exit 1; }
        echo "[base]" >${BS_REPO}
        echo "name=${HOST_NAME}" >>${BS_REPO}
        echo "baseurl=${YUM_REPO_FILE_PATH}" >>${BS_REPO}
        echo "gpgcheck=0" >>${BS_REPO}
        yum clean all
    fi

    # 安装基础包
#    color_start "安装基础工具包..."
    REQUIRED_TOOLS=${REQUIRED_TOOLS_DEFAULT}" "${SYSTEM_MANAGE_TOOLS}
    yum install ${REQUIRED_TOOLS} -y
    [ $? -eq 0 ] && log_echo "基础工具${REQUIRED_TOOLS}: 安装成功" ok || { log_echo "基础工具${REQUIRED_TOOLS}: 安装失败" err; exit; }
    cp ${TOOLS_DIR}'iftop' /usr/local/sbin/iftop && log_echo '工具iftop: 安装成功' ok || log_echo '工具iftop: 安装失败' err

    # 更新时区
    [ 'a'${TIMEZONE} = 'a' ] && TIMEZONE='UTC'
    TZ="/usr/share/zoneinfo/${TIMEZONE}"
    [ ! -f "${TZ}" ] && log_echo '指定时区不存在: 失败' err
#    cp -Rf ${TZ} /etc/localtime && log_echo '初始化时区: 成功' ok || log_echo '初始化时区: 失败' err
    ln -sfT ${TZ} /etc/localtime && log_echo '初始化时区: 成功' ok || log_echo '初始化时区: 失败' err

    # 设置NTP任务
    [ -z "${NTP_SERVER}" ] && NTP_SERVER='cn.pool.ntp.org'
    [ -z "${NTP_CRON}" ] && NTP_CRON='*/30 * * * *'
    CRON_TAB="${NTP_CRON} root /usr/sbin/ntpdate ${NTP_SERVER}"
    /usr/sbin/ntpdate ${NTP_SERVER}
    if [ $(grep '/usr/sbin/ntpdate' /etc/crontab |wc -l) -eq 0 ];then
        echo "${CRON_TAB}" >>/etc/crontab && log_echo 'NTP: 成功' ok || log_echo 'NTP: 失败'err
    else
        sed -i "s#.*/usr/sbin/ntpdate.*#${CRON_TAB}#g" /etc/crontab
    fi

    # 设备主机名
    backup_file /etc/hosts
    echo "127.0.0.1    ${HOST_NAME}" >>/etc/hosts

    # 禁用selinux
#    backup_file /etc/selinux/config
#    sed -i "s/SELINUX.*$/SELINUX=disabled/g" /etc/selinux/config && log_echo 'selinux: 禁用成功' ok || log_echo 'selinux: 禁用失败' err


    # 初始化工作目录
    init_work_dir

    log_echo "
基本功能初始化完成..." ok
#    color_start "开始系统优化..."
    [ ! -f /usr/sbin/scfg ] && cp -f scfg /usr/sbin/
    chmod +x /usr/sbin/scfg
    bash /usr/sbin/scfg update
    [ $? -eq 0 ] && log_echo "系统优化: 成功" ok || log_echo "系统优化: 失败" err

    # 设置日志格式
#    backup_file /etc/profile
    [ ! -d '/var/tmp' ] && { mkdir /var/tmp;chmod 777 !$; } || chmod 777 /var/tmp
    cp -f 'files/bash_history.sh' /etc/profile.d/bash_history_profile.sh && log_echo '格式化历史记录: 成功' ok || log_echo '格式化历史记录: 失败' err
    chmod +x /etc/profile.d/bash_history_profile.sh

    # 安装DELL工具
    megacli_install

    # 安装zabbix
    [ -n "${ZABBIX_SERVER_IP}" ] && { zabbix_install; }

    # 初始化SSH KEY
    [ "${ENABLE_INIT_SSH_KEY}" -eq 1 ] && { cd ${WORK_DIR}; bash nn_ssh_install.sh; }
}

function sys_init_el6() {
    cd ${WORK_DIR}
    backup_file /etc/sysconfig/network
    sed -i "s/HOSTNAME.*$/HOSTNAME=${HOST_NAME}/g" /etc/sysconfig/network

    # 关闭无用服务
    for i in ${NOT_USED_SERVICE}
    do
        chkconfig ${i} off && log_echo "服务 ${i}: 关闭成功" ok || log_echo "服务 ${i}: 关闭失败" err
    done

}

function sys_init_el7() {
    cd ${WORK_DIR}
    hostnamectl set-hostname "${HOST_NAME}"

    # 关闭无用服务
    for i in ${NOT_USED_SERVICE_EL7}
    do
        systemctl disable ${i} >/dev/null 2>&1 && log_echo "服务 ${i}: 关闭成功" ok || log_echo "服务 ${i}: 关闭失败" err
    done

}

function init_work_dir() {
    for x in ${STARCOR_WORK_DIR_TREE};do
        td=${DATA_DIST_MOUNT_DIR}/${x}
        mkdir -p ${td}
    done
}

# zabbix安装
function zabbix_install() {
    cd ${ZABBIX_DIR}
#    color_start "开始安装zabbix-agent: "
    /bin/bash nn_install.sh ${ZABBIX_SERVER_IP} ${HOST_NAME}
    [ $? -eq 0 ] && log_echo "安装zabbix-agent: 成功" ok || log_echo "安装zabbix-agent: 失败" err
    /etc/init.d/zabbix_agentd restart
}

# 日志记录，并回显
function log_echo() {
    [ $2 = 'ok' ] || color_err "$1"
    echo "$1" >> ${LOG_FILE}
}

# DELL工具安装
function megacli_install() {
    dmidecode |grep -i dell |grep Vendor >/dev/null
    if [ $? -eq 0 ]; then
        cd ${TOOLS_DIR}
        rpm -ivh MegaCli-8.07.07-1.noarch.rpm
        ln -sfT /opt/MegaRAID/MegaCli/MegaCli64 /bin/MegaCli64
    fi
}

function echo_help() {
    cat <<EOF
$(basename $0):
    -a:                   执行操作
        help:             查看使用帮助
        check:            检查初始化状态
        install:          执行初始化
EOF
}

function echo_check() {
    [ ! -f ${LOG_FILE} ] && { color_err "还未进行初始化！"; exit 1; } || cat ${LOG_FILE}
}

[ -z "${ACTION}" ] && ACTION=$1
case ${ACTION} in
    'help') {
        echo_help
        exit 0
    };;
    'check') {
        echo_check
        exit 0
    };;
    'install') {
        echo "----------------------------------start------------------------------------" >> ${LOG_FILE}
        now=$(date +"%Y-%m-%d %H:%M:%S")
        echo "操作时间：${now}" >>${LOG_FILE}
        sys_init ${HOST_NAME}

        case "${sys_version}" in
            'el6') { sys_init_el6; };;
            'el7') { sys_init_el7; };;
        esac


        echo "----------------------------------end------------------------------------" >> ${LOG_FILE}
        echo >>${LOG_FILE}
        color_ok "初始化成功"
        [ "${NEED_REBOOT}" = "yes" ] && reboot
    };;
    *) {
        echo_help
        exit 0
    };;
esac

对于修改终端配色可注释

$ vim bashrc

4、重启测试配置

$ reboot

检查配置

$ bash nn_install.sh check
$ date
$ cat /etc/crontab

二、基本组件安装

１８５、１８４端

2.1 LNMP安装

$ cd /usr/local/src
$ wget http://182.138.101.48:51280/package/rpm/centos7/lnmp/lnmp-el7-20180515.tar.gz
$ tar zxf lnmp-el7-20180515.tar.gz
$ cd lnmp-el7-20180515
$　bash nn_install.sh 

确定80、3306.9000端口是否启动

1、验证

ip/index.html

echo "192.168.92.184" >  /data/starcor/www/index.html
echo "192.168.92.183" >  /data/starcor/www/index.html

2.2 LVS+Keepalive安装

1、下载lvs安装包

$ cd /usr/local/src
$ wget http://182.138.101.48:51280/package/lvs_realserver.tar.gz
$ tar zxf lvs_realserver.tar.gz
$ lvs_realserver /etc/init.d/
$ chmod +x /etc/init.d/lvs_realserver

2、修改vip

vim /etc/init.d/lva_relserver

[root@sztw_cms_node3 keepalived-el7-20180525]# cat /etc/init.d/lvs_realserver 
#!/usr/bin/env bash
#real_server
#chkconfig 235 26 26
VIP=192.168.92.248
. /etc/rc.d/init.d/functions
case "$1" in
"start"){
    echo " start LVS of REALServer"
    /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
    /sbin/route add -host $VIP dev lo:0
    echo "1" > /proc/sys/net/ipv4/ip_forward
    echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
    };;
"stop"){
    /sbin/ifconfig lo:0 down
    echo "close LVS Directorserver"
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
    };;
*){
    echo "Usage: $0 {start|stop}"
    exit 1
    };;
esac

3、启动服务

$ /etc/init.d/lva_relserver start 

4、验证

5、下载keepalive安装包

wget http://182.138.101.48:51280/package/rpm/centos7/keepalived/keepalived-el7-20180525.tar.gz
tar zxf keepalived-el7-20180525.tar.gz
cd keepalived-el7-20180525
bash nn_install.sh

6、修改keepalive配置文件

vim /etc/keepalive/keepalive.conf

global_defs {
   notification_email {
     zhongcheng.yang@starcor.cn
     congqin.li@starcor.cn
   }
   notification_email_from starcor_bug@starcor.cn
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_MASTER
}

vrrp_instance VI_185 {
    interface ens160
    virtual_router_id 185
    priority 100
    nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass starcor_keepalived
    }
    virtual_ipaddress {
        192.168.92.248
    }
}

virtual_server 192.168.92.248 80 {
    delay_loop 1
    lb_algo wrr
    lb_kind DR
    persistence_timeout 3
    protocol TCP
#    real_server 192.168.92.183 80 {
#       weight 1
#	HTTP_GET {
#            url {
#                path /health_check.php
#                digest 4845f01eaa8068384625e302e9a4eb05
#            }
#            connect_timeout 5
#            nb_get_retry 3
#            delay_before_retry 1
#        }
#    }
    real_server 192.168.92.184 80 {
        weight 2
        TCP_CHECK {
            connect_timeout 5
            nb_get_retry 3
            delay_before_retry 1
	    connect_port 80

        }

    }


    real_server 192.168.92.183 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 1
	    connect_port 80

        }

  }
}

7、启动服务

$ /etc/init.d/keepalived start

8、验证

9、在183、184也分别部署lvs_reaserver并启动服务(注意要给执行权限x)

scp /etc/init.d/lvs_realserver 192.168.92.184:/etc/init.d/
scp /etc/init.d/lvs_realserver 192.168.92.183:/etc/init.d/

10、验证

关闭其中任意依然看是否会切到另外主机内容上

关闭184上的nginx服务

$ pkill ngin

关闭183上nginx服务

pkill nginx 

4.3 配置MySQL主从

主库配置

1、修改配置文件

修改如下配置、并重启

server-id = 10 //数据库ID号
log-slave-updates //把更新的记录写到二进制文件中

[root@sztw_cms_node1 lnmp-el7-20180515]# vim /usr/local/mysql/my.cnf 

#-------------------- client ---------------------------
[client]
server-id=183
log-slave-update
port = 3306
socket = /data/mysql/data/mysql.sock
user = root
#-------------------- client end ---------------------------


[mysqld]
#---------------------- paths -----------------------
bind_address = 0.0.0.0
port = 3306
socket = /data/mysql/data/mysql.sock
pid-file = /data/mysql/data/mysql3306.pid
datadir = /data/mysql/data
tmpdir = /data/mysql/data
character-set-server = utf8
skip-host-cache
skip-name-resolve
server-id = 4515
replicate-same-server-id = 0
auto-increment-increment = 1
auto-increment-offset    = 1
log-slave-updates
log-bin = /data/mysql/data/mysql-bin.log
relay_log_purge = 1
relay-log = /data/mysql/data/slave-relay.log
relay-log-index = /data/mysql/data/slave-relay-log.index
expire_logs_days        = 3
         

2、 登入mysql，查看master状态

$ mysql -uroot -pstarcor -h127.0.0.1
mysql> show master statusG;

3、创建授权同步账号

grant replication slave on *.* to 'replication'@'192.168.92.184' identified by 'starcor';
flsh privieges;

4、重启

/etc/init.d/mysqld restat 

从库配置

1、修改配置文件

server-id = 20 //数据库ID号，一定不要和主ID相同

log-slave-updates //把更新的记录写到二进制文件中

2、重启数据库

3、更改Slave连接信息

mysql> CHANGE MASTER TO
    ->   MASTER_HOST='192.168.92.183',
    ->   MASTER_USER='replication',
    ->   MASTER_PASSWORD='starcor',
    ->   MASTER_PORT=3306,
    ->   MASTER_LOG_FILE='mysql-bin.000005',
    ->   MASTER_LOG_POS=120;
Query OK, 0 rows affected, 2 warnings (0.00 sec)
mysql> start slave

4、状态检查

​ 执行：show slave statusG;

​ 检查：IO threading和SQL Threading 是否为running， 如果是即主从同步正常

​ 如其它某一个不为running，请根据相应错误信息进行排查

4.4 多实例配置redis主从

1、下载文件

$　wget http://182.138.101.48:51280/package/rpm/centos7/redis/redis-el7-20180523.tar.gz．
＄tar zxf redis-el7-20180523.tar.gz 
$ cd redis-el7-20180523
$bash nn_install.sh 

$ /usr/local/redis/bin/redis-server

２、配置文件

​ 配置文件： /data/redis/conf/redis.conf

参数	值	说明
daemonize	yes	启用守护进程
pidfile	/var/run/redis.pid	pid文件
port	6379	监听端口
timeout	0	客户端闲置不关闭连接
loglevel	notice	日志记录级别
logfile	/data/redis/log/redis.log	日志目录
databases	16	数据库的数量
rdbcompression	yes	存储至本地数据库时压缩数据
dbfilename	dump.rdb	指定本地数据库文件名，默认值为dump.rdb
dir	/data/redis/dump	本地数据库存放目录

３、配置多实例

多实例简单，直接在配置后加端口号即可

＄　cp /usr/local/redis/conf/redis.conf /usr/local/redis/conf/redis6380.conf

４、修改配置文件

pidfile /var/run/redis/redis6380.pid
port 6380
logfile /data/logs/redis/redis6380.log
slaveof 192.168.92.185 6379

６、启动程序

ln -s /usr/local/redis/bin/* /usr/bin/
redis-server /usr/local/redis/conf/redis6380.conf

７、验证主从

redis-cli -p 6380