环境: ubuntu 16.04
k8s版本: 1.11.5
目标:mysql master挂机后切换到mysql slave上运行(以mysql-open为例)
需要root免密 sudo免密
devtest-node112 mysql-open-master
devtest-node121 mysql-open-slave
一、主从切换 (大约为1-2分钟左右切换完成)
1.检测mysql master 运行情况(每分钟执行一次,失败5次而且slave工作正常的情况下进行切换)
2.给老的标签打上SchedulingDisabled ,命令如下
kubectl cordon devtest-node112
3.去除mysql slave的slave设置,使用mysql客户端执行如下命令:
stop slave;reset master;reset slave all;
4.删除mysql slave的deploy
kubectl delete deploy mysql-open-deploy
kubectl delete po mysql-open-slave-deploy-xxxxx --grace-period=0 --force
5.删除slave标签
kubectl label nodes devtest-node121 ToSlave-
6.给slave节点打上master的label
kubectl label nodes devtest-node121 ToMysql=mysql-open
7.删除原有的myql-open-master 的pod
kubectl delete po mysql-open-deploy-xxxxx --grace-period=0 --force
完成主切换到slave节点上,如果密码和用户不一致请修正脚本内容
S_MYSQL_U="root"
S_MYSQL_P="123456"
汇总的脚本如下:
二、恢复slave节点 (恢复时间的长短,取决于数据库备份和scp的时间)
现在的情况是 devtest-node121 为主节点,devtest-node112 已经可以正常运行
1.备份devtest-node121 上的数据,为从节点完成初始化数据(使用percona-xtrabackup,由于使用的mysql 5.7的 ,xtrabackup请使用2.x的版本)
红色部分根据实际情况进行调整
innobackupex --user=USERNAME --password=PASSWORD --datadir=mysqldata_path -S mysqldata_path/mysql.sock /data/bak
2.复制备份的文件到需要创建slave的节点上这里是 devtest-node138
scp -r /data/bak/2020-xxxx-xxxx-xxxx devtest-node112:/data/mysql-open-slave
chown -R 999:999 /data/mysql-open-slave
3.打上slave的label,并去除原来的master的label
kubectl label nodes devtest-node112 ToSlave=mysql-open ToMysql-
4.恢复调度的机制
kubectl uncordon devtest-node112
5.重新创建slave
kubectl apply -f path/mysql-open-slave-deploy.yaml
6.等pod起来后,使用mysql客户端执行如下命令,红色部分为根据实际情况选填
CHANGE MASTER TO MASTER_HOST='mysql-open-master',MASTER_USER='slaveuser',MASTER_PASSWORD='123456',MASTER_LOG_FILE='master.000001',MASTER_LOG_POS=154;start slave;
mysql-open-master-----》mysql-master的主机名或svc地址
slaveuser -----》建立主从的用户名
123456 -----》建立主从的密码
master.000001 -----》二进制文件的名称 (cat /data/mysql-open-slave/xtrabackup_binlog_info,第一列)
154 -----》二进制文件的执行的字符数 (cat /data/mysql-open-slave/xtrabackup_binlog_info,第二列)
![](http://192.168.0.200:8090/download/attachments/6652157/image2020-1-3%2011%3A13%3A43.png?version=1&modificationDate=1578021328000&api=v2)
7.查看slave的状态,使用mysql客户端执行
show slave statusG;
汇总恢复部分脚本1-6的步骤
#!/bin/bash
set -e
USERNAME= "root"
PASSWORD= "MThjN2Y3Zj"
if [ ! -f /usr/bin/xtrabackup ]; then
sudo apt install -y percona-xtrabackup-24 --allow-unauthenticated
fi
DOCKERID=$( sudo docker ps | grep mysql| grep - v pause| awk '{print $1}' )
if [ "${DOCKERID}" == "" -o $( echo ${DOCKERID}| sed 's? ?
?g' | wc -l) - ne 1 ]; then
echo "no mysql server or mysql server is not 1,please check"
exit 0
fi
sudo docker inspect -f {{.State.Pid}} ${DOCKERID}
MPATH=$( sudo docker inspect $DOCKERID | grep : /var/lib/mysql | sed 's?"??g' | awk -F ":" '{print $1}' | awk '{print $1}' )
mkdir -p /data/bak
sudo chmod -R 777 /data/bak
innobackupex --user=${USERNAME} --password=${PASSWORD} --datadir=${MPATH} -S ${MPATH} /mysql .sock /data/bak
i=$( sudo ls /data/bak/ -l| grep ^d| awk '{print $NF}' | grep ^20| tail -1)
innobackupex --apply-log /data/bak/ "$i"
echo 'finish'
echo 'finish' > /tmp/mysql-bak-process .txt
|
#!/bin/bash
set -e
BASEDIR= "../yaml"
ROOTPASSWORD= "MThjN2Y3Zj"
alias ssh = "ssh -o ConnectTimeout=10"
kubectl get po -o wide --all-namespaces| grep -w mysql| grep Running| grep - v slave| awk '{print $1,$2,$(NF-1)}' > /tmp/mysql-restore .txt
ALLMYSQLNU=$( cat /tmp/mysql-restore .txt| wc -l)
for ((m=1;m<=ALLMYSQLNU;m++)); do
i=$( cat /tmp/mysql-restore .txt| awk NR==$m '{print}' )
NS=$( echo $i| awk '{print $1}' )
MYSQLNAME=$( echo $i| awk '{print $2}' | awk -F "-deploy" '{print $1}' )
STATUSSLAVE=$(kubectl -n ${NS} get po| grep ${MYSQLNAME}-slave| wc -l)
if [ ${STATUSSLAVE} - eq 0 ]; then
if [ $( echo ${MYSQLNAME}| grep terminal| wc -l) - eq 1 ]; then
MYSQLLABEL= "mysql-terminal"
else
MYSQLLABEL= "${MYSQLNAME}"
fi
OLDMASTERNODE=$(kubectl get nodes --show-labels| grep SchedulingDisabled| grep -w "${MYSQLLABEL}" | awk '{print $1}' )
if [ "${OLDMASTERNODE}" != "" ]; then
MYSQL_DATA_BAK= "$(ssh ${OLDMASTERNODE} sudo ls -d /data/mysql*)"
ssh ${OLDMASTERNODE} "sudo mkdir -p /data/oldbak/`date +%F`"
ssh ${OLDMASTERNODE} "sudo mv /data/mysql* /data/oldbak/`date +%F`/"
if [ "$(ssh ${OLDMASTERNODE} sudo systemctl status docker|grep Active|awk '{print $2}')" != "active" ]; then
ssh ${OLDMASTERNODE} sudo systemctl start docker
fi
if [ "$(ssh ${OLDMASTERNODE} sudo systemctl status kubelet|grep Active|awk '{print $2}')" != "active" ]; then
ssh ${OLDMASTERNODE} sudo systemctl start kubelet
fi
else
echo "no node can use,please check"
exit 0
fi
CMASTERNODE=$(kubectl -n ${NS} get po -o wide| grep ${MYSQLNAME}-deploy| grep Run| awk '{print $(NF-1)}' )
if [ "${CMASTERNODE}" != "" ]; then
scp mysql-bak.sh ${CMASTERNODE}: /tmp/mysql-bak .sh
ssh ${CMASTERNODE} 'sudo bash -x /tmp/mysql-bak.sh'
else
echo "no ${MYSQLNAME} master"
fi
BAKPATH=$( ssh ${CMASTERNODE} "sudo ls /data/bak/ -l|grep ^d|awk '{print $NF}'|grep ^20|tail -1" )
if [ "${BAKPATH}" != "" ]; then
ssh ${CMASTERNODE} sudo chmod 755 -R /data/bak
ssh ${OLDMASTERNODE} sudo mkdir -p /data/ ${MYSQLNAME}-slave
ssh ${OLDMASTERNODE} sudo chmod -R 777 /data/ ${MYSQLNAME}-slave
ssh -t ${OLDMASTERNODE} scp -r ${CMASTERNODE}: /data/bak/ ${BAKPATH}/* /data/ ${MYSQLNAME}-slave
ssh ${OLDMASTERNODE} sudo chown -R 999:999 /data/ ${MYSQLNAME}-slave
fi
kubectl label nodes ${OLDMASTERNODE} ToMysql-
kubectl label nodes ${OLDMASTERNODE} ToSlave=${MYSQLLABEL} --overwrite
kubectl uncordon ${OLDMASTERNODE}
sleep 10
kubectl apply -f ${BASEDIR}/${MYSQLNAME}-slave-deploy.yaml
sleep 5
kubectl -n ${NS} get po| grep -w ${MYSQLNAME}
sleep 15
BINNAME=$( ssh ${OLDMASTERNODE} cat /data/ ${MYSQLNAME}-slave /xtrabackup_binlog_info | awk '{print $1}' )
BINPOS=$( ssh ${OLDMASTERNODE} cat /data/ ${MYSQLNAME}-slave /xtrabackup_binlog_info | awk '{print $2}' )
NSLAVENAME=$(kubectl -n ${NS} get po -o wide| grep ${MYSQLNAME}-slave| awk '{print $1}' )
SLAVECMD= "CHANGE MASTER TO MASTER_HOST='${MYSQLNAME}-master',MASTER_USER='slave_db',MASTER_PASSWORD='ujmyhn@Sdn1',MASTER_LOG_FILE='${BINNAME}',MASTER_LOG_POS=${BINPOS};start slave;"
kubectl -n ${NS} exec -it ${NSLAVENAME} -- mysql -u "root" -p "$ROOTPASSWORD" -e "${SLAVECMD}"
echo "${MYSQLNAME} restore finish"
fi
done
echo -e "
"
echo "all finished"
|