jenkins服务器使用python脚本rabbitmqadmin和shell对目标服务器进行管理

jenkins服务器使用python脚本rabbitmqadmin和shell对目标服务器进行管理

准备工作：
1.jenkins服务器，安装rabbitmqadmi命令
rabbitmqadmin管理脚本下载地址：https://raw.githubusercontent.com/rabbitmq/rabbitmq-management/v3.7.8/bin/rabbitmqadmin
直接将python脚本拷贝到 /usr/bin/rabbitmqadmin下，赋权 
chmod +x /usr/bin/rabbitmqadmin


# 测试
# su apache
[apache@jenkins:/root]$ rabbitmqadmin -H 1.1.1.1 -P 15672 -u chinasoft -p chinasoft2018 -f tsv -q list queues


2.打通jenkins和jumper_server的ssh，让jenkins能自动登录跳板机

跳板机和后端的windows也需要打通，后端windows安装openssh for windows并且配置互信，跳板机能自动登录windows机器

3.需要修改的变量，获取正确的key值
 rabbitmq_queue=chinasoft*

# 检测Rabbitmq 消息队列数量 如果消息队列内数量大于200 则删除队列内消息
rabbitmq_host=1.1.1.1
rabbitmq_port=15672
rabbitmq_username=chinasoft
rabbitmq_password=pass
rabbitmq_queue=chinasoft*
rabbitmq_queue_max=200
drop_queue=()

rabbitmqadmin -H $rabbitmq_host -P $rabbitmq_port -u $rabbitmq_username -p $rabbitmq_password -f tsv -q list queues | while read queue message_total ; do  if [ "$queue" == "$rabbitmq_queue" ] && [ "$message_total" -gt $rabbitmq_queue_max ]; then rabbitmqadmin -H $rabbitmq_host -P $rabbitmq_port -u $rabbitmq_username -p $rabbitmq_password delete queue name=$queue; drop_queue+=($queue);  fi  ; done

echo ${drop_queue[@]}

if [ ${#drop_queue[*]} -gt 0 ] 
then
    echo "warning 启动构建的服务"
    dir_name=bak.$(date +%Y-%m-%d)
    cp -a /data/www/vhosts/chinasoft.com /data/data_backup/chinasoft.com_$dir_name
    push_dir=/data/www/vhosts/chinasoft.com/httpdocs
    mkdir -p $push_dir

    ###生产环境的变量

    ENVIRONMENT_NAME="convert_env"
    ENVIRONMENT_VARIABLE="prod"
    WIN_ROOT="/cygdrive/d"
    WIN_RESTART_SLEEP=20
    WIN_ACCOUNT="Administrator"

    win_back_ips=('172.17.0.33' '172.17.0.32')
    execute_jump_ip="1.1.1.1"
    middle_server_ip="172.17.0.36"
    back_server_ip="172.17.0.35"
    restart_middle_group_name="online2:"
    restart_back_group_name=("chinasoft_back:" "chinasoft_server:" )

    # $1 - jump_ip
    # $2 - win_account
    # $3 - win_ip
    # $4 - win_root
    # $5 - push_dir
    # $6 - sleep
    function rsync_windows_server()
    {
        __jump_ip="$1"
        __win_account="$2"
        __win_ip="$3"
        __win_root="$4"
        __push_dir="$5"
        __sleep=$6

        shift
        shift
        shift
        shift
        shift
        shift

        __exclude_params=()
        for exclude_dir in "$@"; do
            __exclude_params+=(--exclude="$exclude_dir")
        done
        ssh -p2008 root@$execute_jump_ip "ssh -p 2008 $__win_account@$__win_ip "export $ENVIRONMENT_NAME=$ENVIRONMENT_VARIABLE; $__win_root$__push_dir/converter_db/scripts/back_stop_consumer.bat""
        rsync -avzP --bwlimit=1000 "${__exclude_params[@]}" $__push_dir/converter_db root@$__win_ip::root --chmod=ugo=rwX
        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 $__win_account@$__win_ip shutdown -r -t 0"
        sleep $__sleep
    } 


    rsync -vau --delete --exclude=.git/ --exclude=.gitignore/ --exclude=node_modules/ $WORKSPACE/ $push_dir
    #chmod -R ugo+r $push_dir
    chmod +x -R $push_dir/converter_db/scripts
    chmod +x -R $push_dir/converter_db/core_api

    __linux_exclude_params=()
    __linux_exclude_params+=(--exclude="converter_db/core_api/Windows")

    __windows_exclude_dir=()
    __windows_exclude_dir+=("converter_db/core_api/Linux")

    if [ "$PASS_CORE" == "true" ];then
        __linux_exclude_params+=(--exclude="converter_db/core_api")
        __windows_exclude_dir+=("converter_db/core_api")
    fi

    rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_middle apache@$middle_server_ip::apache

    if [ "$BUILD_TASK_SERVICE" == "true" ];then
        # stop consumer & rsync & restart windows
        for server_ip in "${win_back_ip[@]}"; do
            rsync_windows_server "$execute_jump_ip" "$WIN_ACCOUNT" $server_ip "$WIN_ROOT" "$push_dir" "$WIN_RESTART_SLEEP" "${__windows_exclude_dir[@]}"
        done

        rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_db apache@$middle_server_ip::apache
        rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_db apache@$back_server_ip::apache |true
        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$middle_server_ip supervisorctl restart $middle_celery_group"
        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$back_server_ip supervisorctl restart ${back_celery_group[@]}"
    fi
fi



#############################
#第一部分：硬重启
#PE.ali_www.chinasoft.com_monitor-new
#############################

# 每隔两小时检查一下rabbitmq，监控chinasoft.com的rabbitmq队列，如果超过200的阈值，则触发重启动作


Build periodically

两小时执行一次
H H/2 * * * 

whoami
# 检测Rabbitmq 消息队列数量 如果消息队列内数量大于200 则删除队列内消息
rabbitmq_host=1.1.1.2
rabbitmq_port=15672
rabbitmq_username=chinasoft
rabbitmq_password=pass
rabbitmq_queue=online_*
rabbitmq_queue_max=200
drop_queue=()

rabbitmqadmin -H $rabbitmq_host -P $rabbitmq_port -u $rabbitmq_username -p $rabbitmq_password -f tsv -q list queues | while read queue message_total ; do  if [ "$queue" == "$rabbitmq_queue" ] && [ "$message_total" -gt $rabbitmq_queue_max ]; then rabbitmqadmin -H $rabbitmq_host -P $rabbitmq_port -u $rabbitmq_username -p $rabbitmq_password delete queue name=$queue; drop_queue+=($queue);  fi  ; done

echo ${drop_queue[@]}

if [ ${#drop_queue[*]} -gt 0 ] 
then
    echo "warning 启动重启windows服务器的脚本"
    push_dir="/data/www/vhosts/chinasoft.com/httpdocs"
    ###生产环境的变量

    ENVIRONMENT_NAME="convert_env"
    ENVIRONMENT_VARIABLE="prod"
    WIN_ROOT="/cygdrive/d"
    WIN_RESTART_SLEEP=20
    WIN_ACCOUNT="Administrator"

    win_back_ips=('172.17.0.33' '172.17.0.32')
    execute_jump_ip="1.1.1.1"
    middle_server_ip="172.17.0.36"
    back_server_ip="172.17.0.35"
    restart_middle_group_name="online2:"
    restart_back_group_name=("chinasoft_back:" "chinasoft_server:" )

    # $1 - jump_ip
    # $2 - win_account
    # $3 - win_ip
    # $4 - win_root
    # $5 - push_dir
    # $6 - sleep
    function rsync_windows_server()
    {
        __jump_ip="$1"
        __win_account="$2"
        __win_ip="$3"
        __win_root="$4"
        __push_dir="$5"
        __sleep=$6

        shift
        shift
        shift
        shift
        shift
        shift

        __exclude_params=()
        for exclude_dir in "$@"; do
            __exclude_params+=(--exclude="$exclude_dir")
        done
        ssh -p2008 root@$execute_jump_ip "ssh -p 2008 $__win_account@$__win_ip "export $ENVIRONMENT_NAME=$ENVIRONMENT_VARIABLE; $__win_root$__push_dir/converter_db/scripts/back_stop_consumer.bat""
        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 $__win_account@$__win_ip shutdown -r -t 0"
        sleep $__sleep
    } 


    #chmod -R ugo+r $push_dir
    chmod +x -R $push_dir/converter_db/scripts
    chmod +x -R $push_dir/converter_db/core_api

    __linux_exclude_params=()
    __linux_exclude_params+=(--exclude="converter_db/core_api/Windows")

    __windows_exclude_dir=()
    __windows_exclude_dir+=("converter_db/core_api/Linux")

    if [ "$PASS_CORE" == "true" ];then
        __linux_exclude_params+=(--exclude="converter_db/core_api")
        __windows_exclude_dir+=("converter_db/core_api")
    fi


    if [ "$BUILD_TASK_SERVICE" == "true" ];then
        # stop consumer & rsync & restart windows
        for server_ip in "${win_back_ip[@]}"; do
            rsync_windows_server "$execute_jump_ip" "$WIN_ACCOUNT" $server_ip "$WIN_ROOT" "$push_dir" "$WIN_RESTART_SLEEP" "${__windows_exclude_dir[@]}"
        done

        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$middle_server_ip supervisorctl restart $middle_celery_group"
        ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$back_server_ip supervisorctl restart ${back_celery_group[@]}"
    fi
fi

#############################################

# 软重启，热更新

dir_name=bak.$(date +%Y-%m-%d)
cp -a /data/www/vhosts/chinasoft.com /data/data_backup/chinasoft.com_$dir_name
push_dir=/data/www/vhosts/chinasoft.com/httpdocs
mkdir -p $push_dir

###生产环境的变量

ENVIRONMENT_NAME="convert_env"
ENVIRONMENT_VARIABLE="prod"
WIN_ROOT="/cygdrive/d"
WIN_RESTART_SLEEP=20
WIN_ACCOUNT="Administrator"

win_back_ips=('172.17.0.33' '172.17.0.32')
execute_jump_ip="1.1.1.1"
middle_server_ip="172.17.0.36"
back_server_ip="172.17.0.35"
restart_middle_group_name="online2:"
restart_back_group_name=("chinasoft_back:" "chinasoft_server:" )

# $1 - jump_ip
# $2 - win_account
# $3 - win_ip
# $4 - win_root
# $5 - push_dir
# $6 - sleep
function rsync_windows_server()
{
    __jump_ip="$1"
    __win_account="$2"
    __win_ip="$3"
    __win_root="$4"
    __push_dir="$5"
    __sleep=$6

    shift
    shift
    shift
    shift
    shift
    shift

    __exclude_params=()
    for exclude_dir in "$@"; do
        __exclude_params+=(--exclude="$exclude_dir")
    done
    ssh -p2008 root@$execute_jump_ip "ssh -p 2008 $__win_account@$__win_ip "export $ENVIRONMENT_NAME=$ENVIRONMENT_VARIABLE; $__win_root$__push_dir/converter_db/scripts/back_stop_consumer.bat""
    rsync -avzP --bwlimit=1000 "${__exclude_params[@]}" $__push_dir/converter_db root@$__win_ip::root --chmod=ugo=rwX
    ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 $__win_account@$__win_ip shutdown -r -t 0"
    sleep $__sleep
} 


rsync -vau --delete --exclude=.git/ --exclude=.gitignore/ --exclude=node_modules/ $WORKSPACE/ $push_dir
#chmod -R ugo+r $push_dir
chmod +x -R $push_dir/converter_db/scripts
chmod +x -R $push_dir/converter_db/core_api

__linux_exclude_params=()
__linux_exclude_params+=(--exclude="converter_db/core_api/Windows")

__windows_exclude_dir=()
__windows_exclude_dir+=("converter_db/core_api/Linux")

if [ "$PASS_CORE" == "true" ];then
    __linux_exclude_params+=(--exclude="converter_db/core_api")
    __windows_exclude_dir+=("converter_db/core_api")
fi

rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_middle apache@$middle_server_ip::apache

if [ "$BUILD_TASK_SERVICE" == "true" ];then
    # stop consumer & rsync & restart windows
    for server_ip in "${win_back_ip[@]}"; do
        rsync_windows_server "$execute_jump_ip" "$WIN_ACCOUNT" $server_ip "$WIN_ROOT" "$push_dir" "$WIN_RESTART_SLEEP" "${__windows_exclude_dir[@]}"
    done

    rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_db apache@$middle_server_ip::apache
    rsync -vaRzP --progress --bwlimit=1000 "${__linux_exclude_params[@]}" --password-file=/usr/local/rsync/etc/pe_rsyncd.passwd $push_dir/converter_db apache@$back_server_ip::apache |true
    ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$middle_server_ip supervisorctl restart $middle_celery_group"
    ssh -p2008 root@$execute_jump_ip  "ssh -p 2008 root@$back_server_ip supervisorctl restart ${back_celery_group[@]}"
fi
fi

# 发布代码以后删除缓存操作
# 要配置http://jenkins2.chinasoft.cn/jenkins/view/pe.chinasoft/job/web_html/configure  删除nginx的缓存
ssh -p2008 devadmin@1.1.1.1 "sudo rm -rf /dev/shm/nginx/hipdf/cache/*"