| 主机名 | IP |
| master |
eth0: 192.168.30.130(NAT) eth1: 192.168.17.130(VMNet4) |
| node-1 |
eth0: 192.168.30.131(NAT) eth1: 192.168.17.131(VMNet4) |
安装过程省略
master上的配置
[root@master ~]# vim /etc/tgt/targets.conf <target iqn.2018-06.cn.sishen.www:target_san1> backing-store /dev/sdb1 initiator-address 192.168.30.131 initiator-address 192.168.17.131 vendor_id sishen product_id target-01 </target> [root@master ~]# /etc/init.d/tgtd restart Stopping SCSI target daemon: initiators still connected [FAILED] Starting SCSI target daemon: [ OK ] [root@master ~]# tgt tgtadm tgt-admin tgtd tgtimg tgt-setup-lun [root@master ~]# tgt-admin --show Target 1: iqn.2018-06.cn.sishen.www:target_san1 System information: Driver: iscsi State: ready I_T nexus information: LUN information: LUN: 0 Type: controller SCSI ID: IET 00010000 SCSI SN: beaf10 Size: 0 MB, Block size: 1 Online: Yes Removable media: No Prevent removal: No Readonly: No Backing store type: null Backing store path: None Backing store flags: Account information: ACL information: 192.168.30.131 192.168.17.131
在客户端测试通讯
[root@node-1 ~]# ping -c 2 192.168.30.130 PING 192.168.30.130 (192.168.30.130) 56(84) bytes of data. 64 bytes from 192.168.30.130: icmp_seq=1 ttl=64 time=0.183 ms 64 bytes from 192.168.30.130: icmp_seq=2 ttl=64 time=0.267 ms --- 192.168.30.130 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1000ms rtt min/avg/max/mdev = 0.183/0.225/0.267/0.042 ms [root@node-1 ~]# ping -c 2 192.168.17.130 PING 192.168.17.130 (192.168.17.130) 56(84) bytes of data. 64 bytes from 192.168.17.130: icmp_seq=1 ttl=64 time=0.350 ms 64 bytes from 192.168.17.130: icmp_seq=2 ttl=64 time=0.154 ms --- 192.168.17.130 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1000ms rtt min/avg/max/mdev = 0.154/0.252/0.350/0.098 ms
配置线路1
[root@node-1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.30.130:3260 Starting iscsid: [ OK ] 192.168.30.130:3260,1 iqn.2018-06.cn.sishen.www:target_san1 [root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ] Starting iscsi: [ OK ] [root@node-1 ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sr0 11:0 1 3.6G 0 rom /media/cdrom sda 8:0 0 20G 0 disk ├─sda1 8:1 0 200M 0 part /boot ├─sda2 8:2 0 2G 0 part [SWAP] └─sda3 8:3 0 17.8G 0 part └─vg_master-LogVol00 (dm-0) 253:0 0 17.8G 0 lvm / sdb 8:16 0 5G 0 disk └─sdb1 8:17 0 5G 0 part
配置线路2
[root@node-1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.17.130:3260 192.168.17.130:3260,1 iqn.2018-06.cn.sishen.www:target_san1
[root@node-1 ~]# tree /var/lib/iscsi/ /var/lib/iscsi/ ├── ifaces ├── isns ├── nodes │ └── iqn.2018-06.cn.sishen.www:target_san1 │ ├── 192.168.17.130,3260,1 │ │ └── default │ └── 192.168.30.130,3260,1 │ └── default ├── send_targets │ ├── 192.168.17.130,3260 │ │ ├── iqn.2018-06.cn.sishen.www:target_san1,192.168.17.130,3260,1,default -> /var/lib/iscsi/nodes/iqn.2018-06.cn.sishen.www:target_san1/192.168.17.130,3260,1 │ │ └── st_config │ └── 192.168.30.130,3260 │ ├── iqn.2018-06.cn.sishen.www:target_san1,192.168.30.130,3260,1,default -> /var/lib/iscsi/nodes/iqn.2018-06.cn.sishen.www:target_san1/192.168.30.130,3260,1 │ └── st_config ├── slp └── static 13 directories, 4 files
客户端重启iscsi服务
[root@node-1 ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sr0 11:0 1 3.6G 0 rom /media/cdrom sda 8:0 0 20G 0 disk ├─sda1 8:1 0 200M 0 part /boot ├─sda2 8:2 0 2G 0 part [SWAP] └─sda3 8:3 0 17.8G 0 part └─vg_master-LogVol00 (dm-0) 253:0 0 17.8G 0 lvm / sdc 8:32 0 5G 0 disk └─sdc1 8:33 0 5G 0 part sdb 8:16 0 5G 0 disk └─sdb1 8:17 0 5G 0 part
挂在数据测试
[root@node-1 ~]# mount /dev/sdb1 /opt/ [root@node-1 ~]# df -h /opt/ Filesystem Size Used Avail Use% Mounted on /dev/sdb1 5.0G 139M 4.6G 3% /opt [root@node-1 ~]# ll /dev/sdb1 brw-rw---- 1 root disk 8, 17 Jun 4 11:15 /dev/sdb1 [root@node-1 ~]# ll /dev/sdc1 brw-rw---- 1 root disk 8, 33 Jun 4 11:15 /dev/sdc1
配置多路径实现冗余
[root@node-1 ~]# yum install -y device-mapper-multipath [root@node-1 ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/ [root@node-1 ~]# /etc/init.d/multipathd restart ux_socket_connect: No such file or directory Stopping multipathd daemon: [FAILED] Starting multipathd daemon: [ OK ] [root@node-1 ~]# multipath -ll #没有反应，卸载/opt/下面的挂载重启multipath服务 [root@node-1 ~]# umount /opt/ [root@node-1 ~]# /etc/init.d/multipathd restart ok Stopping multipathd daemon: [ OK ] Starting multipathd daemon: [ OK ] [root@node-1 ~]# multipath -ll mpatha (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='0' hwhandler='0' wp=rw |-+- policy='round-robin 0' prio=1 status=active | `- 9:0:0:1 sdc 8:32 active ready running `-+- policy='round-robin 0' prio=1 status=enabled `- 10:0:0:1 sdb 8:16 active ready running [root@node-1 ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/mapper/vg_master-LogVol00 18G 4.1G 13G 25% / tmpfs 2.0G 72K 2.0G 1% /dev/shm /dev/sda1 194M 35M 150M 19% /boot /dev/sr0 3.6G 3.6G 0 100% /media/cdrom /dev/mapper/mpathap1 5.0G 139M 4.6G 3% /opt [root@node-1 ~]# ll /dev/mapper/mpatha* lrwxrwxrwx 1 root root 7 Jun 4 11:20 /dev/mapper/mpatha -> ../dm-1 lrwxrwxrwx 1 root root 7 Jun 4 11:20 /dev/mapper/mpathap1 -> ../dm-2
配置负载均衡
[root@node-1 ~]# multipath -ll mpatha (1IET 00010001) dm-1 sishen,target-01 #远程存储设备的产品信息 dm-0 xuegod #厂商,target1 #产品ID size=5.0G features='0' hwhandler='0' wp=rw |-+- policy='round-robin 0' prio=1 status=active | `- 9:0:0:1 sdc 8:32 active ready running `-+- policy='round-robin 0' prio=1 status=enabled `- 10:0:0:1 sdb 8:16 active ready running [root@node-1 ~]# cp /etc/multipath.conf{,.bak} [root@node-1 ~]# vim /etc/multipath.conf ## Use user friendly names, instead of using WWIDs as names. #注释掉以下三行 #defaults { # user_friendly_names yes #} 62 multipaths { 63 multipath { 64 wwid "1IET 00010001" # 65 alias data-back 66 path_grouping_policy multibus 67 path_checker readsector0 68 path_selector "round-robin 0" 69 failback manual 70 rr_weight priorities 71 no_path_retry 5 72 } 73 } [root@node-1 ~]# /etc/init.d/multipathd restart ok Stopping multipathd daemon: [ OK ] Starting multipathd daemon: [ OK ] [root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ] Starting iscsi: [ OK ] [root@node-1 ~]# multipath -ll Jun 04 11:30:32 | multipath.conf line 67, invalid keyword: path_checker data-back (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw `-+- policy='round-robin 0' prio=1 status=active |- 12:0:0:1 sdb 8:16 active ready running `- 11:0:0:1 sdc 8:32 active ready running
[root@node-1 ~]# vim /etc/multipath.conf 24 ## Use user friendly names, instead of using WWIDs as names.， 25 #defaults { 26 # user_friendly_names yes 27 #} #在62行修改或添加如下内容 62 multipaths { 63 multipath { 64 wwid "1IET 00010001" 65 alias data-back 66 path_grouping_policy multibus 67 path_selector "round-robin 0" 68 failback manual 69 rr_weight priorities 70 no_path_retry 5 71 } 72 }
重启服务
[root@node-1 ~]# /etc/init.d/multipathd restart ok Stopping multipathd daemon: [ OK ] Starting multipathd daemon: [ OK ] [root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ] Starting iscsi: [ OK ] [root@node-1 ~]# multipath -ll datashare (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw `-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active ready running `- 5:0:0:1 sdc 8:32 active ready running [root@node-1 ~]# ll /dev/mapper/datashare* lrwxrwxrwx 1 root root 7 Jun 4 13:02 /dev/mapper/datashare -> ../dm-1 lrwxrwxrwx 1 root root 7 Jun 4 13:02 /dev/mapper/datasharep1 -> ../dm-2
挂载测试
[root@node-1 ~]# mount /dev/mapper/datashare datashare datasharep1 [root@node-1 ~]# mount /dev/mapper/datasharep1 /opt/ [root@node-1 ~]# df -h /opt/ Filesystem Size Used Avail Use% Mounted on /dev/mapper/datasharep1 5.0G 139M 4.6G 3% /opt [root@node-1 ~]# cp install.log /opt/ [root@node-1 ~]# ll /opt/ total 80 -rw-r--r-- 1 root root 61802 Jun 4 13:05 install.log drwx------ 2 root root 16384 Jun 4 12:37 lost+found
模拟故障测试冗余
[root@master ~]# ifdown eth1 #手动停掉eth1 Device state: 3 (disconnected) [root@node-1 ~]# multipath -ll #这里大约需要60秒来识别，所以会卡在这里，等待即可 datashare (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw `-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active faulty running `- 5:0:0:1 sdc 8:32 active ready running
在服务端 [root@master ~]# ifdown eth1 Device state: 3 (disconnected) 在客户端 [root@node-1 ~]# multipath -ll #回车之后等待大约60秒 datashare (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw `-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 failed faulty running `- 5:0:0:1 sdc 8:32 active ready running #这里也要等待大约60秒 测试数据写入 [root@node-1 ~]# cp /etc/passwd /opt/ [root@node-1 ~]# ll /opt/ total 84 -rw-r--r-- 1 root root 61802 Jun 4 13:05 install.log drwx------ 2 root root 16384 Jun 4 12:37 lost+found -rw-r--r-- 1 root root 1779 Jun 4 13:16 passwd 在服务端启动eth1 [root@master ~]# ifup eth1 Active connection state: activated Active connection path: /org/freedesktop/NetworkManager/ActiveConnection/2 在客户端查看负载均衡 [root@node-1 ~]# multipath -ll #立刻就会显示结果 datashare (1IET 00010001) dm-1 sishen,target-01 size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw `-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active ready running `- 5:0:0:1 sdc 8:32 active ready running #这里说明负载均衡正常