1. 部署galera集群
利用四台主机cicd、node1、node2、node3来搭建galera集群。
1> 上传压缩包至cicd，解压得到ansible配置文件
[root@cicd ~]# ls anaconda-ks.cfg ansible-galera.tar.gz [root@cicd ~]# tar zxf ansible-galera.tar.gz [root@cicd ~]# ls anaconda-ks.cfg ansible-galera ansible-galera.tar.gz [root@cicd ~]# cd ansible-galera/ [root@cicd ansible-galera]# ls bin delete.yml hosts install.yml roles
2> 安装ansible
[root@cicd ~]# mount /dev/cdrom /mnt mount: /dev/sr0 is write-protected, mounting read-only [root@cicd ~]# yum install ansible -y
3> 清空ansible目录文件，将解压得到的文件复制到ansible目录下
[root@cicd ~]# cd /etc/ansible/ [root@cicd ansible]# ls ansible.cfg hosts roles [root@cicd ansible]# rm -rf * [root@cicd ansible]# ls [root@cicd ansible]# mv /root/ansible-galera/* . [root@cicd ansible]# ls bin delete.yml hosts install.yml roles
4> 集群进行域名解析，使每个节点都能静态解析。
[root@cicd ansible]# vim /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.16.95 cicd 192.168.16.96 node1 192.168.16.98 node2 192.168.16.99 node3 [root@cicd ansible]# scp /etc/hosts 192.168.16.96:/etc/hosts [root@cicd ansible]# scp /etc/hosts 192.168.16.98:/etc/hosts [root@cicd ansible]# scp /etc/hosts 192.168.16.99:/etc/hosts
5> 在部署节点设置无秘钥登录
[root@cicd ansible]# ssh-keygen [root@cicd ansible]# ssh-copy-id node1 [root@cicd ansible]# ssh-copy-id node2 [root@cicd ansible]# ssh-copy-id node3 [root@cicd ansible]# ssh-copy-id cicd
6> bin目录用来存放配置文件以及镜像、安装脚本，因此要将配置文件、镜像和安装版本放在该目录下。
[root@cicd ansible]# ls bin delete.yml hosts install.yml roles [root@cicd ansible]# cd bin [root@cicd bin]# ls VERSION.md [root@cicd bin]# cd [root@cicd ~]# ls anaconda-ks.cfg ansible-galera ansible-galera.tar.gz galera.tar.gz [root@cicd ~]# tar zxf galera.tar.gz [root@cicd ~]# ls anaconda-ks.cfg ansible-galera.tar.gz galera.tar.gz ansible-galera config.tar.gz mariadb-galera.ta [root@cicd ~]# mv config.tar.gz mariadb-galera.tar /etc/ansible/bin/ [root@cicd ~]# cd /etc/ansible/bin/ [root@cicd bin]# ls config.tar.gz mariadb-galera.tar VERSION.md
7> 在四台机器上部署docker
[root@cicd ~]# wget http://download2.yunwei.edu/shell/docker.tar.gz [root@cicd ~]# tar zxf docker.tar.gz [root@cicd ~]# cd docker [root@cicd docker]# bash docker.sh
四台均部署。
8> 编辑配置文件hosts
[root@cicd ~]# cd /etc/ansible [root@cicd ansible]# ls bin delete.yml hosts install.yml roles [root@cicd ansible]# vim hosts [all] #集群节点 192.168.16.96 192.168.16.98 192.168.16.99 [master] #主节点 192.168.16.96 [slave1] #从节点 192.168.16.98 [slave2] #从节点 192.168.16.99 [all:vars] #变量 #定义集群ip master="192.168.16.96" slave1="192.168.16.98" slave2="192.168.16.99" #部署目录，即 ansible 工作目录 base_dir="/etc/ansible" #默认软件安装目录 opt_dir="/opt"
9> 执行yml文件，部署ansible集群，得到docker-galera集群
[root@cicd ~]# cd /etc/ansible/
[root@cicd ansible]# ls
bin delete.yml hosts install.yml roles
[root@cicd ansible]# ansible-playbook install.yml
10> 查看容器即数据库结果，验证是否安装成功。
[root@node1 ~]# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 10cda5ecde16 panubo/mariadb-galera "/galera-entrypoint.…" 5 minutes ago Up 5 minutes galera1 [root@node2 ~]# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES bc1dca8dea63 panubo/mariadb-galera "/galera-entrypoint.…" 6 minutes ago Up 34 seconds galera2 [root@node3 ~]# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES fd542150b5cc panubo/mariadb-galera "/galera-entrypoint.…" 7 minutes ago Up 22 seconds galera3 [root@node1 ~]# docker exec -it 10cda5ecde16 /bin/bash root@node1:/# mysql -uroot -p123 MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+ [root@node2 opt]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 9abff0435fa2 panubo/mariadb-galera "/galera-entrypoint.…" 2 hours ago Up 2 hours galera2 [root@node2 opt]# docker exec -it 9abff0435fa2 /bin/bash root@node2:/# mysql -uroot -p123 Welcome to the MariaDB monitor. Commands end with ; or g. Your MariaDB connection id is 4 Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or 'h' for help. Type 'c' to clear the current input statement. MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+ [root@node3 opt]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES [root@node3 opt]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 26b3b9dee6ea panubo/mariadb-galera "/galera-entrypoint.…" 2 hours ago Up 2 hours galera3 [root@node3 opt]# docker exec -it 26b3b9dee6ea /bin/bash root@node3:/# mysql -uroot -p123; Welcome to the MariaDB monitor. Commands end with ; or g. Your MariaDB connection id is 4 Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or 'h' for help. Type 'c' to clear the current input statement. MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+ 1 row in set (0.01 sec)
11> 各文件内容
[root@node ansible]# cat install.yml - hosts: - all roles: - ready
- hosts: - master roles: - master - hosts: - slave1 roles: - slave1 - hosts: - slave2 roles: - slave2
对应的main.yml文件在目录roles下面 [root@node ansible]# cd roles [root@node roles]# ls master ready slave1 slave2 [root@node ansible]# cd roles [root@node roles]# ls master ready slave1 slave2 [root@node roles]# cd master/ [root@node master]# ls tasks [root@node master]# cd tasks/ [root@node tasks]# ls main.yml [root@node tasks]# cat main.yml #部署wsrep进程mariadb-master - name: 启动wsrep进程 shell: "cd {{ opt_dir }}/mariadb/ && sh master.sh" [root@node tasks]# cd /etc/ansible/roles/ready/ [root@node ready]# ls tasks templates [root@node ready]# cd tasks/ [root@node tasks]# ls main.yml [root@node tasks]# cat main.yml #准备工作目录： - name: prepare some dirs file: name={{ item }} state=directory mode=0755 with_items: - "{{ opt_dir }}/mariadb" ## ---------下载 配置文件及镜像 ----------- - name: download configfile and iamge copy: src={{ base_dir }}/bin/{{ item }} dest={{ opt_dir }}/mariadb/{{ item }} mode=0755 with_items: - mariadb-galera.tar - config.tar.gz #解压配置文件 - name: 解压config shell: "tar zxf {{ opt_dir }}/mariadb/config.tar.gz -C {{ opt_dir }}/mariadb/" #导入mariadb-galera镜像 - name: 导入mariadb-galera shell: "cd {{ opt_dir }}/mariadb/ && docker load -i mariadb-galera.tar" ## ---------分发 脚本文件 ----------- - name: download configfile and iamge template: src={{ item }} dest={{ opt_dir }}/mariadb/{{ item }} mode=0755 with_items: - master.sh - slave1.sh - slave2.sh [root@node roles]# ls master ready slave1 slave2 [root@node roles]# cd ready/ [root@node ready]# ls tasks templates [root@node ready]# cd templates/ [root@node templates]# ls master.sh slave1.sh slave2.sh [root@node templates]# cat master.sh docker run -d --net host --name galera1 -e WSREP_NODE_ADDRESS={{ master }} -e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 -e MYSQL_ROOT_PASSWORD=123 -p 3306:3306 -p 4567:4567/udp -p 4567-4568:4567-4568 -p 4444:4444 -v /opt/mariadb/config/mysql:/etc/mysql -v /opt/mariadb/config/data:/var/lib/mysql:Z --restart=always panubo/mariadb-galera mysqld --wsrep-new-cluster [root@node templates]# cat slave1.sh docker run -d --net host --name galera2 -e WSREP_NODE_ADDRESS={{ slave1 }} -e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 -p 3306:3306 -p 4567:4567/udp -p 4567-4568:4567-4568 -p 4444:4444 -v /opt/mariadb/config/mysql:/etc/mysql -v /opt/mariadb/config/data:/var/lib/mysql:Z --restart=always panubo/mariadb-galera mysqld [root@node templates]# cat slave2.sh docker run -d --net host --name galera3 -e WSREP_NODE_ADDRESS={{ slave2 }} -e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 -p 3306:3306 -p 4567:4567/udp -p 4567-4568:4567-4568 -p 4444:4444 -v /opt/mariadb/config/mysql:/etc/mysql -v /opt/mariadb/config/data:/var/lib/mysql:Z --restart=always panubo/mariadb-galera mysqld [root@node roles]# cd slave1 [root@node slave1]# ls tasks [root@node slave1]# cd tasks/ [root@node tasks]# ls main.yml [root@node tasks]# cat main.yml #部署wsrep进程mariadb-master - name: 启动salve进程 shell: "cd {{ opt_dir }}/mariadb/ && sh slave1.sh" [root@node tasks]# cat main.yml #部署wsrep进程mariadb-master - name: 启动salve进程 shell: "cd {{ opt_dir }}/mariadb/ && sh slave2.sh" [root@node tasks]#
11> 删除部署的docker-galance
删除该集群用/etc/ansible/delete.yum文件
文件内容 [root@node ansible]# cat delete.yml - hosts: - master tasks: - name: stop mariadb-galera-docker shell: "docker stop galera1 && docker rm galera1" - hosts: - slave1 tasks: - name: stop mariadb-galera-docker shell: "docker stop galera2 && docker rm galera2" - hosts: - slave2 tasks: - name: stop mariadb-galera-docker shell: "docker stop galera3 && docker rm galera3" - hosts: - all tasks: #清理文件及镜像 - name: 清理目录和文件 file: name={{ opt_dir }}/mariadb state=absent - name: rm image shell: "docker rmi 75feb1db734a" 执行删除 [root@node ansible]# ansible-playbook delete.yml
2. 集群故障恢复
1> 集群中某slave节点down机，重启该节点后重启docker容器即可；
[root@node ~]# mysql -h 192.168.16.96 -uroot -p123 #-h：指定节点数据库登录 Welcome to the MariaDB monitor. Commands end with ; or g. Your MariaDB connection id is 4 Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or 'h' for help. Type 'c' to clear the current input statement. MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+ 模拟节点node3 down机 [root@node3 ~]# powerof MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 2 | +--------------------+-------+ 再将节点重新启动，查看数据库 Last login: Wed Jul 17 04:57:57 2019 from 192.168.16.95 [root@node3 ~]# MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+
2> 集群中master节点down机，因为master节点已经切换重启该master节点后master-docker容器不能正常启动，需要将原先master-docker容器删除，再以非wrster方式启动一个普通容器；
[root@node1 ~]# shutdown -h now MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 2 | +--------------------+-------+ 重启node1 Last login: Wed Jul 17 04:57:41 2019 from 192.168.16.95 [root@node1 ~]# [root@node1 ~]# docker ps -a #由于被抢占了master，一直处于重启状态 CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e919dd79cb44 panubo/mariadb-galera "/galera-entrypoint.…" 34 minutes ago Restarting (1) 11 seconds ago galera1 MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 2 | +--------------------+-------+ 将该容器删除，然后再以普通容器的方式启动 [root@node1 ~]# docker stop e919dd79cb44 e919dd79cb44 [root@node1 ~]# docker rm e919dd79cb44 e919dd79cb44 修改从节点启动脚本，以普通容器形式启动 [root@node1 opt]# cd [root@node1 ~]# cd /opt/mariadb/ [root@node1 mariadb]# ls config config.tar.gz mariadb-galera.tar master.sh slave1.sh slave2.sh [root@node1 mariadb]# vim slave1.sh docker run -d --net host --name galera1 -e WSREP_NODE_ADDRESS=192.168.16.96 -e WSREP_CLUSTER_ADDRESS=gcomm://192.168.16.96:4567,192.168.16.98:4567,192.168.16.99:4567 -p 3306:3306 -p 4567:4567/udp -p 4567-4568:4567-4568 -p 4444:4444 -v /opt/mariadb/config/mysql:/etc/mysql -v /opt/mariadb/config/data:/var/lib/mysql:Z --restart=always panubo/mariadb-galera mysqld [root@node1 mariadb]# docker run -d --net host --name galera1 > -e WSREP_NODE_ADDRESS=192.168.16.96 > -e WSREP_CLUSTER_ADDRESS=gcomm://192.168.16.96:4567,192.168.16.98:4567,192.168.16.99:4567 > -p 3306:3306 > -p 4567:4567/udp > -p 4567-4568:4567-4568 > -p 4444:4444 > -v /opt/mariadb/config/mysql:/etc/mysql > -v /opt/mariadb/config/data:/var/lib/mysql:Z > --restart=always > panubo/mariadb-galera mysqld WARNING: Published ports are discarded when using host network mode 354383d0d45ee204e1cf5b9f5f0196216ea68a8df3b61a5618eb0cb1abe90e7b #重启成功 "slave1.sh" 11L, 383C written [root@node1 mariadb]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 354383d0d45e panubo/mariadb-galera "/galera-entrypoint.…" 10 minutes ago Up 10 minutes galera1 MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+
3> 集群全部意外down机（如断电），强制找一台节点作为master节点，将/opt/mariadb/data目录中的文件grastate.dat文件的safe_to_bootstrap:0改为1，之后在该节点启动wsrap进程的docker。
模拟三个节点down [root@node ~]# ansible all -m shell -a 'shutdown -h now' 192.168.16.96 | UNREACHABLE! => { "changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.96 closed.", "unreachable": true } 192.168.16.99 | UNREACHABLE! => { "changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.99 closed.", "unreachable": true } 192.168.16.98 | UNREACHABLE! => { "changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.98 closed.", "unreachable": true } 再将三个节点开机 Last login: Wed Jul 17 05:53:07 2019 from 192.168.16.95 [root@node1 ~]# Last login: Wed Jul 17 05:52:33 2019 from 192.168.16.95 [root@node2 ~]# Last login: Wed Jul 17 05:53:06 2019 from 192.168.16.95 [root@node3 ~]# 无法登陆数据库 [root@node ~]# mysql -h 192.168.16.99 -uroot -p123 ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111) [root@node ~]# mysql -h 192.168.16.98 -uroot -p123 ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111) [root@node ~]# mysql -h 192.168.16.96-uroot -p123 ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111) 但是各节点容器是启动的 [root@node ~]# ansible all -m shell -a 'docker ps' 192.168.16.96 | CHANGED | rc=0 >> CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 354383d0d45e panubo/mariadb-galera "/galera-entrypoint.…" 18 minutes ago Up 20 seconds galera1 192.168.16.99 | CHANGED | rc=0 >> CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 57f1d19388c2 panubo/mariadb-galera "/galera-entrypoint.…" About an hour ago Up 21 seconds galera3 192.168.16.98 | CHANGED | rc=0 >> CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e96ee6196980 panubo/mariadb-galera "/galera-entrypoint.…" About an hour ago Up 16 seconds galera2 修改文件grastate.dat [root@node1 ~]# cd /opt/mariadb/config/data/ [root@node1 data]# ls aria_log.00000001 grastate.dat ib_logfile1 performance_schema aria_log_control ibdata1 multi-master.info galera.cache ib_logfile0 mysql [root@node1 data]# vim grastate.dat # GALERA saved state version: 2.1 uuid: 8c85e11d-f45d-11e8-8ed7-dbc6d595b08d seqno: -1 safe_to_bootstrap: 1 删除galera1容器，由于做了映射。并不会删除数据库 [root@node1 data]# docker stop 354383d0d45e 354383d0d45e [root@node1 data]# docker rm 354383d0d45e 354383d0d45e 执行master.sh脚本，人为启动mode1为master [root@node1 ~]# cd /opt/mariadb/ [root@node1 mariadb]# ls config config.tar.gz mariadb-galera.tar master.sh slave1.sh slave2.sh [root@node1 mariadb]# bash master.sh WARNING: Published ports are discarded when using host network mode 2c9900b470a429484e1e959d9d4b4ab4bbfbd0238abaf67214c82cdf7388f404 [root@node1 mariadb]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 2c9900b470a4 panubo/mariadb-galera "/galera-entrypoint.…" 23 seconds ago Up 23 seconds galera1 查看数据库，其他两节点自动加入 [root@node ~]# mysql -h 192.168.16.96 -uroot -p123 MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size'; +--------------------+-------+ | Variable_name | Value | +--------------------+-------+ | wsrep_cluster_size | 3 | +--------------------+-------+