本次操作是完成 centos68 + 11.2.0.4 + haip
oracle 11.2.0.2 开始提供 haip 来作为私有网段的冗余手段
haip简单介绍
gi最多支持4个有效地私网配置，所以定义4个以上没有任何意义。
ora.cluster_interconnect.haip资源为 rac、asm 和 acfs等通过私网进行内部通信的组件创建1到4个haip地址，这些IP地址表现为169.254..，因此需要保证169.254..在我们的网络环境里避免使用。在下面将会给出我的环境里haip内容。
当gi集群中第一个节点启动的时候，根据活动的私网适配器个数来决定haip地址数，尽管只有一个私有网络，GI依然会分配一个haip地址，如果有多余两个私网适配器，GI将创建4个haip。
4个是目前最多支持的个数，因此即使有更多的私网适配器被gi激活，haip个数也不再改变。
使用 haip 后，不需要在 /etc/hosts 里列出 private ip
之前的ip分配
rac1 rac2
scan ip 192.168.56.170
vip 192.168.56.160 192.168.56.161
public ip 192.168.56.150 192.168.56.151
private ip 192.168.165.150 192.168.165.151 公共网段为 192.168.56.x
私有网段为 192.168.165.x
分配新的私有IP
rac1 rac2
private ip 192.168.165.150 192.168.165.151
192.168.165.160 192.168.165.161rac1 待加新ip 192.168.165.160
rac2 待加新ip 192.168.165.161
添加私有ip
[root@rac1 ~]#cd /u01/app/grid/product/11.2.0/grid_1/bin
[root@rac1 bin]#./crsctl stat res -t -init注意 ora.cluster_interconnect.haip
[root@rac1 bin]#./oifcfg getif
eth0 192.168.56.0 global public
eth1 192.168.165.0 global cluster_interconnect
[root@rac1 bin]#./oifcfg iflist -p -n
eth0 192.168.56.0 PRIVATE 255.255.255.0
eth1 192.168.165.0 PRIVATE 255.255.255.0
eth1 169.254.0.0 UNKNOWN 255.255.0.0
eth2 192.168.165.0 PRIVATE 255.255.255.0其中，eth2 为我们在os层面新加的网卡
SQL> select name, ip_address from v$cluster_interconnects;
NAME IP_ADDRESS
--------------- ----------------
eth1:1 169.254.211.4可以看出，eth2 还没有增加到私有网段里
现将eth2网卡加入cluster_interconnect,rac1、rac2两个节点都需要执行
[root@rac1 bin]#./oifcfg setif -global eth2/192.168.165.0:cluster_interconnect
[root@rac2 bin]#./oifcfg setif -global eth2/192.168.165.0:cluster_interconnect此时 并未在eth2上绑定haip，需要重启集群。
[root@rac1 bin]# ./crsctl stop cluster -all
[root@rac1 bin]# ./crsctl start cluster -all查看v$cluster_interconnects
SQL> select name, ip_address from v$cluster_interconnects;
NAME IP_ADDRESS
--------------- ----------------
eth1:1 169.254.25.60
eth2:1 169.254.151.26
此时的ip情况如下
rac1 rac2
scan ip 192.168.56.170
vip 192.168.56.160 192.168.56.161
public ip 192.168.56.150 192.168.56.151
private ip 192.168.165.150 192.168.165.151
192.168.165.160 192.168.165.161删除私有ip
删除前 eth1,eth2 的信息
eth1 Link encap:Ethernet HWaddr 08:00:27:2A:E9:7B
inet addr:192.168.165.150 Bcast:192.168.165.255 Mask:255.255.255.0
inet6 addr: fe80::a00:27ff:fe2a:e97b/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:266013 errors:0 dropped:0 overruns:0 frame:0
TX packets:226720 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:201852294 (192.5 MiB) TX bytes:145679363 (138.9 MiB)
eth1:1 Link encap:Ethernet HWaddr 08:00:27:2A:E9:7B
inet addr:169.254.102.247 Bcast:169.254.127.255 Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
eth2 Link encap:Ethernet HWaddr 08:00:27:C9:ED:D8
inet addr:192.168.165.160 Bcast:192.168.165.255 Mask:255.255.255.0
inet6 addr: fe80::a00:27ff:fec9:edd8/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:40624 errors:0 dropped:0 overruns:0 frame:0
TX packets:35432 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:29982482 (28.5 MiB) TX bytes:23093002 (22.0 MiB)
eth2:1 Link encap:Ethernet HWaddr 08:00:27:C9:ED:D8
inet addr:169.254.239.245 Bcast:169.254.255.255 Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1删除 eth2
[root@rac1 bin]#./oifcfg delif -global eth2/192.168.165.0:cluster_interconnect
[root@rac2 bin]#./oifcfg delif -global eth2/192.168.165.0:cluster_interconnect这个时候绑定在 eth2 上的 169.254.. 会自动漂移到其它私有网卡
eth1 Link encap:Ethernet HWaddr 08:00:27:2A:E9:7B
inet addr:192.168.165.150 Bcast:192.168.165.255 Mask:255.255.255.0
inet6 addr: fe80::a00:27ff:fe2a:e97b/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:298442 errors:0 dropped:0 overruns:0 frame:0
TX packets:258338 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:225383098 (214.9 MiB) TX bytes:167837403 (160.0 MiB)
eth1:1 Link encap:Ethernet HWaddr 08:00:27:2A:E9:7B
inet addr:169.254.102.247 Bcast:169.254.127.255 Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
eth1:2 Link encap:Ethernet HWaddr 08:00:27:2A:E9:7B
inet addr:169.254.239.245 Bcast:169.254.255.255 Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
eth2 Link encap:Ethernet HWaddr 08:00:27:C9:ED:D8
inet addr:192.168.165.160 Bcast:192.168.165.255 Mask:255.255.255.0
inet6 addr: fe80::a00:27ff:fec9:edd8/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:68149 errors:0 dropped:0 overruns:0 frame:0
TX packets:61325 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:51272634 (48.8 MiB) TX bytes:41555876 (39.6 MiB)可以不用重启集群,也可以有机会再手动重启集群
[root@rac1 bin]# ./crsctl stop cluster -all
[root@rac1 bin]# ./crsctl start cluster -all此时的ip情况如下
rac1 rac2
scan ip 192.168.56.170
vip 192.168.56.160 192.168.56.161
public ip 192.168.56.150 192.168.56.151
private ip 192.168.165.150 192.168.165.151个人建议，gi + rac 架构，对公有网卡进行os的bonding，对私有网卡做 haip
参考:
Grid Infrastructure Redundant Interconnect and ora.cluster_interconnect.haip (文档 ID 1210883.1)