rac多路径下添加lun

 

以下步骤需要在2个节点上做同样的配置

 

1.虚拟机添加新的磁盘
虚拟机添加磁盘后，让系统扫描到新的磁盘，或是重启动系统

 

2.系统重启后相应的配置项会自动生成
[root@rac01 ~]# more /etc/multipath/wwids
# Multipath wwids, Version : 1.0
# NOTE: This file is automatically maintained by multipath and multipathd.
# You should not need to edit this file in normal circumstances.
#
# Valid WWIDs:
/VBOX_HARDDISK_VB7e0b2f07-0b02d751/
/VBOX_HARDDISK_VB36d0c6c4-81a819db/
/VBOX_HARDDISK_VB52bb81d4-c5ca964b/
/VBOX_HARDDISK_VBe7e3fc72-e130e0c8/
/VBOX_HARDDISK_VBfea867e8-d3c8d798/

同时可以使用如下命令获取设备的wwid
[root@rac01 rules.d]# /usr/lib/udev/scsi_id -g -u /dev/sdf
1ATA_VBOX_HARDDISK_VBfea867e8-d3c8d798

重启后自动会绑定,若不重启需要重新加载

[root@rac01 ~]# more /etc/multipath/bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha VBOX_HARDDISK_VB7e0b2f07-0b02d751
mpathb VBOX_HARDDISK_VB36d0c6c4-81a819db
mpathc VBOX_HARDDISK_VB52bb81d4-c5ca964b
mpathd VBOX_HARDDISK_VBe7e3fc72-e130e0c8
mpathe VBOX_HARDDISK_VBfea867e8-d3c8d798

3.multipath -ll也会列出新添加的磁盘,但是没有别名
[root@rac01 ~]# multipath -ll
mpathe (VBOX_HARDDISK_VBfea867e8-d3c8d798) dm-4 ATA ,VBOX HARDDISK
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
ocrdisk03 (VBOX_HARDDISK_VB52bb81d4-c5ca964b) dm-3 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (VBOX_HARDDISK_VBe7e3fc72-e130e0c8) dm-2 ATA ,VBOX HARDDISK
size=20G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (VBOX_HARDDISK_VB36d0c6c4-81a819db) dm-5 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
ocrdisk01 (VBOX_HARDDISK_VB7e0b2f07-0b02d751) dm-1 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running

/dev/mapper下面也会列出新增的设备,指向dm
[root@rac01 ~]# ls -al /dev/mapp*
total 0
drwxr-xr-x 2 root root 180 Jul 14 20:57 .
drwxr-xr-x 20 root root 3460 Jul 14 20:57 ..
crw------- 1 root root 10, 236 Jul 14 20:57 control
lrwxrwxrwx 1 root root 7 Jul 14 21:51 datadisk01 -> ../dm-2
lrwxrwxrwx 1 root root 7 Jul 14 20:58 mpathe -> ../dm-4
lrwxrwxrwx 1 root root 7 Jul 14 21:42 ocrdisk01 -> ../dm-1
lrwxrwxrwx 1 root root 7 Jul 14 21:51 ocrdisk02 -> ../dm-5
lrwxrwxrwx 1 root root 7 Jul 14 21:42 ocrdisk03 -> ../dm-3
lrwxrwxrwx 1 root root 7 Jul 14 20:58 vgroot-lvroot -> ../dm-0

 

但是dm下的设备权限不会自动修改为grid,下面需要重新加载udev才能修改权限
[root@rac01 ~]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Jul 14 20:58 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Jul 14 21:52 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 2 Jul 14 21:52 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Jul 14 21:52 /dev/dm-3
brw-rw---- 1 root disk 253, 4 Jul 14 20:58 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Jul 14 21:52 /dev/dm-5

4.修改配置文件,配置别名与wwid的对应关系,添加红色部分
注意rac环境,另外的节点也要做同样的修改配置

[root@rac01 ~]# more /etc/multipath.conf
blacklist {
    devnode "^sda"
}
defaults {
    user_friendly_names yes
    path_grouping_policy multibus
    failback immediate
    no_path_retry fail
}

multipaths {
       multipath {
               wwid                    VBOX_HARDDISK_VB7e0b2f07-0b02d751
               alias                   ocrdisk01
               path_grouping_policy    multibus
       }
       multipath {
               wwid                    VBOX_HARDDISK_VB36d0c6c4-81a819db
               alias                   ocrdisk02
               path_grouping_policy    multibus
       }
       multipath {
               wwid                    VBOX_HARDDISK_VB52bb81d4-c5ca964b
               alias                   ocrdisk03
               path_grouping_policy    multibus
       }       
       multipath {
               wwid                    VBOX_HARDDISK_VBe7e3fc72-e130e0c8
               alias                   datadisk01
               path_grouping_policy    multibus
       }  
       multipath {
               wwid                    VBOX_HARDDISK_VBfea867e8-d3c8d798
               alias                   datadisk02
               path_grouping_policy    multibus
       }
}

 

 

 

4.重新reload
[root@rac01 ~]# multipath -r

或是执行 multipath -v3

reload: ocrdisk01 (VBOX_HARDDISK_VB7e0b2f07-0b02d751) undef ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:0:0 sdb 8:16 active ready running
reload: ocrdisk02 (VBOX_HARDDISK_VB36d0c6c4-81a819db) undef ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 4:0:0:0 sdc 8:32 active ready running
reload: ocrdisk03 (VBOX_HARDDISK_VB52bb81d4-c5ca964b) undef ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 5:0:0:0 sdd 8:48 active ready running
reload: datadisk01 (VBOX_HARDDISK_VBe7e3fc72-e130e0c8) undef ATA ,VBOX HARDDISK
size=20G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 6:0:0:0 sde 8:64 active ready running
Jul 14 21:59:56 | VBOX_HARDDISK_VBfea867e8-d3c8d798: rename mpathe to datadisk02
: datadisk02 (VBOX_HARDDISK_VBfea867e8-d3c8d798) undef ATA ,VBOX HARDDISK
size=5.0G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 7:0:0:0 sdf 8:80 active ready running

这个时候输出是别名了
[root@rac01 ~]# multipath -ll
datadisk02 (VBOX_HARDDISK_VBfea867e8-d3c8d798) dm-4 ATA ,VBOX HARDDISK
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
ocrdisk03 (VBOX_HARDDISK_VB52bb81d4-c5ca964b) dm-3 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (VBOX_HARDDISK_VBe7e3fc72-e130e0c8) dm-2 ATA ,VBOX HARDDISK
size=20G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (VBOX_HARDDISK_VB36d0c6c4-81a819db) dm-5 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
ocrdisk01 (VBOX_HARDDISK_VB7e0b2f07-0b02d751) dm-1 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running

 

5.重新加载udev
[root@rac01 ~]# cd /etc/udev/rules.d/
添加新增项
[root@rac01 rules.d]# more 99-oracle-asmdevices.rules
ENV{DM_NAME}=="ocrdisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk03", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"

[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Jul 14 21:58 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Jul 14 22:11 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 2 Jul 14 22:11 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Jul 14 22:11 /dev/dm-3
brw-rw---- 1 root disk 253, 4 Jul 14 22:05 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Jul 14 22:11 /dev/dm-5

当前新设备的权限还是属于root,执行如下命令后,权限赋予grid

[root@rac01 rules.d]# udevadm trigger
[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Jul 14 22:11 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Jul 14 22:11 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 2 Jul 14 22:11 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Jul 14 22:11 /dev/dm-3
brw-rw---- 1 grid asmadmin 253, 4 Jul 14 22:11 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Jul 14 22:11 /dev/dm-5