Oracle-部署19C RAC on RHEL7

Installing Oracle Database 12C+ Release 2 RAC on RHEL7.6

单实例文件系统部署跳过GI和ASM配置部分即可

1. 检查清单

1.1 硬件条件

1）检查内存

# grep MemTotal /proc/meminfo

2）检查swap空间

grep SwapTotal /proc/meminfo

3）检查/tmp目录空间

df -h /tmp

4）检查共享内存（/dev/shm）

df -h /dev/shm

cluvfy Fails with Error:"PRVE-0427 : Failed To Retrieve The Size Of In-memory File System Mounted As /dev/shm " (文档 ID 2243758.1)

5）存储容量需求

ASM存储平衡级别：

1.2 软件条件

2. 环境配置

2.1 配置虚拟文件系统

vi /etc/fstab
tmpfs                    /dev/shm                tmpfs    defaults,rw,exec,size=2G     0 0

-- 重新装载
# mount -o remount /dev/shm

2.1 disable Transparent HugePages

-- 检查
cat /sys/kernel/mm/transparent_hugepage/enabled

1. For Oracle Linux 7 and Red Hat Enterprise Linux 7, add or modify the transparent_hugepage=never parameter 
in the /etc/default/grub file:transparent_hugepage=never

For example:
RUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto quiet numa=off transparent_hugepage=never"
GRUB_DISABLE_RECOVERY="true"

2. Run the grub2–mkconfig command to regenerate the grub.cfg file.
# grub2-mkconfig -o /boot/grub2/grub.cfg

3. Restart the system to make the changes permanent.

# 4. 检查确认
cat /proc/cmdline
cat /sys/kernel/mm/transparent_hugepage/enabled

2.2 安装系统 rmp包

-- Packages for Red Hat Enterprise Linux 7:
gcc
gcc-c++
bc
binutils
compat-libcap1
compat-libstdc++-33
dtrace-modules
dtrace-modules-headers
dtrace-modules-provider-headers
dtrace-utils
elfutils-libelf
elfutils-libelf-devel
fontconfig-devel
glibc
glibc-devel
ksh
libaio
libaio-devel
libdtrace-ctf-devel
libX11
libXau
libXi
libXtst
libXrender
libXrender-devel
libgcc
librdmacm-devel
libstdc++
libstdc++-devel
libxcb
make
net-tools (for Oracle RAC and Oracle Clusterware)
nfs-utils (for Oracle ACFS)
python (for Oracle ACFS Remote)
python-configshell (for Oracle ACFS Remote)
python-rtslib (for Oracle ACFS Remote)
python-six (for Oracle ACFS Remote)
targetcli (for Oracle ACFS Remote)
smartmontools
sysstat
unixODBC
unixODBC-devel
rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})
" gcc gcc-c++ bc binutils compat-libcap1 compat-libstdc++-33 dtrace-modules dtrace-modules-headers dtrace-modules-provider-headers dtrace-utils elfutils-libelf elfutils-libelf-devel fontconfig-devel glibc glibc-devel ksh libaio libaio-devel libdtrace-ctf-devel libX11 libXau libXi libXtst libXrender libXrender-devel libgcc librdmacm-devel libstdc++ libstdc++-devel libxcb make net-tools smartmontools sysstat unixODBC unixODBC-devel



yum -y install compat-libstdc++-33.i686 glibc.i686 glibc-devel.i686 libaio.i686 libaio-devel.i686 libgcc.i686 libstdc++.i686 libstdc++-devel.i686 libXi.i686 libXtst.i686 unixODBC.i686 unixODBC-devel.i686
yum -y install gcc gcc-c++ bc binutils compat-libcap1 compat-libstdc++-33  dtrace-modules dtrace-modules-headers dtrace-modules-provider-headers dtrace-utils elfutils-libelf  elfutils-libelf-devel fontconfig-devel  glibc glibc-devel ksh libaio libaio-devel libdtrace-ctf-devel libX11 libXau libXi libXtst libXrender libXrender-devel  libgcc librdmacm-devel  libstdc++ libstdc++-devel  libxcb make net-tools smartmontools sysstat unixODBC unixODBC-devel

2.3 配置/etc/hosts文件

cp /etc/hosts /etc/hosts_$(date +%Y%d%m)
cat > /etc/hosts << EOF
127.0.0.1 loopback localhost.localdomain localhost localhost4
::1 loopback localhost.localdomain localhost localhost6

# Public
192.168.10.173 o19c1 o19c1.example.com
192.168.10.174 o19c2 o19c2.example.com

# Private
172.168.0.158 o19c1-priv o19c1.example.com-priv
172.168.0.159 o19c2-priv o19c2.example.com-priv

# Virtual
192.168.10.175 o19c1-vip o19c1.example.com-vip 
192.168.10.176 o19c2-vip o19c2.example.com-vip

# SCAN
#192.168.10.177 o19c-scan o19c-scan.example.com
#192.168.10.178 o19c-scan o19c-scan.example.com
#192.168.10.179 o19c-scan o19c-scan.example.com

EOF

2.4 . ssh配置 (LoginGraceTime 0)

cp /etc/ssh/sshd_config /etc/ssh/sshd_config_$(date +%Y%m%d) 
or
cp /etc/ssh/sshd_config{,_$(date +%Y%m%d)}

grep '^LoginGraceTime' /etc/ssh/sshd_config
echo "LoginGraceTime 0" >> /etc/ssh/sshd_config

2.5 关闭selinux

if [[ "$(getenforce)" = "Enforcing" ]]; then
    cp /etc/selinux/config /etc/selinux/config_$(date +%Y%m%d)
    setenforce 0
    # sed -i "/^SELINUX=enforcing/c#SELINUX=enforcing
SELINUX=disable" /etc/selinux/config
    sed -i "/^SELINUX=enforcing/aSELINUX=disable" /etc/selinux/config
fi

2.5 关闭防火墙

if [[ "$(ps -ef |grep -v grep|grep -ci firewalld)" = "1" ]]; then
    systemctl stop firewalld
    systemctl disable firewalld
fi

2.6 配置PAM

vi /etc/pam.d/login
session    required     pam_limits.so

cp /etc/pam.d/login{,_$(date +%Y%m%d)}
echo "session    required     pam_limits.so" >> /etc/pam.d/login

2.7 禁用avahi-daemon服务

--查看服务状态
systemctl status avahi-daemon
 
--停止服务
systemctl stop avahi-daemon
 
--禁止服务自启动
systemctl disable avahi-daemon

if [[ -f "/etc/systemd/system/dbus-org.freedesktop.Avahi.service" ]]; then
    systemctl stop avahi-dnsconfd
    systemctl stop avahi-daemon
    systemctl disable avahi-dnsconfd
    systemctl disable avahi-daemon
fi

2.8 屏蔽RemoveIPC参数 [ rhel 7.2 BUG Doc ID 2081410.1 ]

if [[ -f "/etc/systemd/logind.conf" ]]; then
    cp /etc/systemd/logind.conf /etc/systemd/logind.conf_$(date +%Y%m%d)
    sed -i "/#RemoveIPC=/aRemoveIPC=no" /etc/systemd/logind.conf
fi

grep '^RemoveIPC' /etc/systemd/logind.conf
[[ "$?" -eq "1" ]] && cp /etc/systemd/logind.conf{,_$(date +%Y%m%d)} && sed -i "/#RemoveIPC=/aRemoveIPC=no" /etc/systemd/logind.conf

-- 重启服务器或重启systemd-logind
systemctl daemon-reload
systemctl restart systemd-logind

2.9 配置系统资源限制

# the value of size in Kb
MEMTOTAL=$(grep MemTotal /proc/meminfo|awk '{print $2}')
MEMLOCK=$(( MEMTOTAL * 9/10 ))
cat > /etc/security/limits.d/99-grid-oracle-limits.conf << EOF
oracle soft nproc 16384 #Ora bug 15971421
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
oracle soft stack 10240
oracle hard stack 32768
# setting for memlock hard limit is maximum of 128GB on x86_64 or 3GB on x86 OR 90 % of RAM
oracle soft memlock ${MEMLOCK}
oracle hard memlock ${MEMLOCK}

grid soft nproc 16384 #Ora bug 15971421
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
grid soft stack 10240
grid hard stack 32768
grid soft memlock ${MEMLOCK}
grid hard memlock ${MEMLOCK}
EOF

2.9 配置系统内核参数

# the value of size in Kb
MEMTOTAL=$(grep MemTotal /proc/meminfo|awk '{print $2}')
SHMMAX=$(( MEMTOTAL * 1024 * 8 / 10 ))
SHMMNI=4096
PAGESIZE=$(getconf PAGE_SIZE)
SHMALL=$(( SHMMAX / 4096 ))
cat > /etc/sysctl.d/97-oracle-database-sysctl.conf << EOF
# -- The number of asynchronous IO requests at the same time(as per Note 579108.1),for example 1048576 = 1024 * 1024
fs.aio-max-nr = 3145728
# -- 512 * processes (for example 6815744 for 13312 processes)
fs.file-max = 6815744
# is maximum of sga in bytes
kernel.shmmax = ${SHMMAX}
kernel.shmall = ${SHMALL}
kernel.shmmni = ${SHMMNI}
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_wmem = 4096        16384   4194304
kernel.panic_on_oops = 1
vm.min_free_kbytes = 524288
EOF
# /sbin/sysctl --system
-- 检查确认
/sbin/sysctl -a |grep net.core.[wr]mem_max

Table A-1 Minimum Operating System Resource Parameter Settings

Parameter	Value	File
semmsl semmns semopm semmni	250 32000 100 128	/proc/sys/kernel/sem
shmall	Greater than or equal to the value of shmmax, in pages.	/proc/sys/kernel/shmall
shmmax	Half the size of physical memory in bytesSee My Oracle Support Note 567506.1 for additional information about configuring shmmax.	/proc/sys/kernel/shmmax
shmmni	4096	/proc/sys/kernel/shmmni
panic_on_oops	1	/proc/sys/kernel/panic_on_oops
file-max	6815744	/proc/sys/fs/file-max
aio-max-nr	1048576Note: This value limits concurrent outstanding requests and should be set to avoid I/O subsystem failures.	/proc/sys/fs/aio-max-nr
ip_local_port_range	Minimum: 9000Maximum: 65500	/proc/sys/net/ipv4/ip_local_port_range
rmem_default	262144	/proc/sys/net/core/rmem_default
rmem_max	4194304	/proc/sys/net/core/rmem_max
wmem_default	262144	/proc/sys/net/core/wmem_default
wmem_max	1048576	/proc/sys/net/core/wmem_max

2.9.1 UDP和TCP 内核参数配置(临时)

# 检查
cat /proc/sys/net/ipv4/ip_local_port_range
# 配置
echo 9000 65500 > /proc/sys/net/ipv4/ip_local_port_range
# 
/etc/rc.d/init.d/network restart

2.10 配置网络参数nozeroconf

cp /etc/sysconfig/network /etc/sysconfig/network_$(date +%Y%m%d)
cat >> /etc/sysconfig/network << EOF
NOZEROCONF=yes
EOF

cat >> /etc/sysconfig/network-scripts/ifcfg-lo << EOF
MTU=16436
EOF

2.11 系统时间校对

RHEL建议使用NTP [Tips on Troubleshooting NTP / chrony Issues](Doc ID 2068875.1)

# 检验时间和时区确认正确
date 
# 查看当前时区
timedatectl status
# 修改时区（若需要）
timedatectl set-timezone Asia/ShangHai
 
# 关闭chrony服务，移除chrony配置文件（虚机不配置时间同步服务，使用ctss）
systemctl list-unit-files|grep chronyd
systemctl status chronyd
 
systemctl disable chronyd
systemctl stop chronyd
 
# 移除方式备份chrony配置文件
mv /etc/chrony.conf /etc/chrony.conf_bak

# chronyd服务配置时间同步
# 配置时间服务器（若需要）
vi /etc/chrony.conf
server 192.168.10.166 iburst
# 检查确认
chronyc sources -v
chronyc sourcestats -v

# NTP服务方式配置同步时间
# 1. 关闭并禁用chrony服务
systemctl list-unit-files|grep chronyd
systemctl status chronyd
systemctl disable chronyd
systemctl stop chronyd

# 2. 删除其配置文件
mv /etc/chrony.conf /etc/chrony.conf_bak

# 3. 安装NTP服务
yum install -y ntp ntpdate
systemctl enable ntpd
systemctl start ntpd

# 手动同步时间
ntpdate -u <NTP Server>

# 4. 配置NTP，开启微调模式
# 编辑/etc/sysconfig/ntpd，在-g后面加上-x 和 -p参数
# Command line options for ntpd
OPTIONS="-g -x -p /var/run/ntpd.pid"

# 5. 检查确认
ntpq -p
ntpstat  # 查看这台服务器是否连接到NTP服务器

2.12 创建用户组和用户

# 1. 创建用户组
/usr/sbin/groupadd -g 54321 oinstall
/usr/sbin/groupadd -g 54322 dba
/usr/sbin/groupadd -g 54323 oper
/usr/sbin/groupadd -g 54324 backupdba
/usr/sbin/groupadd -g 54325 dgdba
/usr/sbin/groupadd -g 54326 kmdba
/usr/sbin/groupadd -g 54327 asmdba
/usr/sbin/groupadd -g 54328 asmoper
/usr/sbin/groupadd -g 54329 asmadmin
/usr/sbin/groupadd -g 54330 racdba

# 2. 创建用户
/usr/sbin/useradd -u 54321 -g oinstall -G dba,oper,asmdba,backupdba,dgdba,kmdba,racdba --comment "Oracle Software Owner" oracle
/usr/sbin/useradd -u 54322 -g oinstall -G asmadmin,asmdba,asmoper,dba --comment "Oracle Software Owner" grid

$ id oracle
uid=54321(oracle) gid=54321(oinstall) groups=54321(oinstall),54322(dba), 54323(oper),54324(backupdba),54325(dgdba),54326(kmdba),54327(asmdba),54330(racdba)

$ id grid
uid=54331(grid) gid=54321(oinstall) groups=54321(oinstall),54322(dba),54327(asmdba),54328(asmoper),54329(asmadmin),54330(racdba)

# 3. 配置密码
echo "oracle"|passwd --stdin  oracle 
echo "grid"|passwd --stdin  grid 

2.13 创建软件安装目录

# 1. 软件安装目录
mkdir -p /ups/oracle/grid
mkdir -p /ups/oracle/19c/grid
mkdir -p /ups/oracle/oraInventory
chown -R grid:oinstall /ups/oracle
mkdir -p /ups/oracle/database/db_1
chown -R oracle:oinstall /ups/oracle/database
chmod -R 775 /ups/oracle

# 安装文件目录（临时存储）
mkdir -p /ups/soft
chown -R grid:oinstall /ups/soft
chmod 775 /ups/soft

# 
cat >> /etc/oraInst.loc <<EOF
inventory_loc=/ups/oracle/oraInventory
inst_group=oinstall
EOF

chown grid:oinstall /etc/oraInst.loc
chmod 644 /etc/oraInst.loc

2.14 配置用户环境变量

su - grid
vi ~/.bash_profile
export ORACLE_SID=+ASM1
export ORACLE_BASE=/ups/oracle/grid
export ORACLE_HOME=/ups/oracle/19c/grid
export PATH=${ORACLE_HOME}/bin:${ORACLE_HOME}/OPatch:${PATH}
export INVENTORY_LOCATION=/ups/oracle/oraInventory
umask 022
if [ -t 0 ]; then
   stty intr ^C
fi
 
su - oracle
vi ~/.bash_profile
export ORACLE_SID=o19cdb1
export ORACLE_BASE=/ups/oracle/database
export ORACLE_HOME=${ORACLE_BASE}/db_1
export GRID_HOME=/ups/oracle/19c/grid
export LD_LIBRARY_PATH=${ORACLE_HOME}/lib
export PATH=${ORACLE_HOME}/bin:${ORACLE_HOME}/OPatch:${ORACLE_HOME}/perl/bin:${GRID_HOME}/bin:${HOME}/scripts/bin:${PATH}
export TNS_ADMIN=${GRID_HOME}/network/admin
export SQLPATH=${ORACLE_HOME}/rdbms/admin:${HOME}/scripts/sql
umask 022
if [ -t 0 ]; then
   stty intr ^C
fi

2.15 配置profile

cat > /etc/profile.d/oracle-grid.sh << EOF
#Setting the appropriate ulimits for oracle and grid user
if [ $USER = "oracle" ]; then
    if [ $SHELL = "/bin/ksh" ]; then
        ulimit -u 16384
        ulimit -n 65536
    else
        ulimit -u 16384 -n 65536
    fi
fi
if [ $USER = "grid" ]; then
    if [ $SHELL = "/bin/ksh" ]; then
        ulimit -u 16384
        ulimit -n 65536
    else
        ulimit -u 16384 -n 65536
    fi
fi
EOF

2.16 配置ssh互相

export SSH='ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
cat ~/.ssh/*.pub >> ~/.ssh/authorized_keys
ssh -o stricthostkeychecking=no 192.168.10.174  cat  ~/.ssh/*.pub >> ~/.ssh/authorized_keys
scp ~/.ssh/authorized_keys 192.168.10.174:~/.ssh/authorized_keys

scp -r ~/.ssh /home/grid/
scp -r ~/.ssh /home/oracle/
chown grid:oinstall /home/grid/.ssh -R
chown oracle:oinstall /home/oracle/.ssh -R

# 检查确认
export SSH='ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'
for ip in $(grep -Ev "^#|localhost|vip|scan" /etc/hosts); do
    ${SSH} ${ip} date
done

# 另一个方式使用Oracle自带脚本创建

2.17 配置dns

# yum 安装
yum -y install unbound
yum install -y bind-utils
 
# 配置配置文件/etc/unbound/unbound.conf
vi /etc/unbound/unbound.conf
……
38 # interface: 0.0.0.0
39 interface: 0.0.0.0
……
//找到38行，复制去掉注释行，打开监听全网功能。
177         # access-control: 0.0.0.0/0 refuse
178         access-control: 192.168.10.0/24 allow
179         # access-control: 127.0.0.0/8 allow
// 找到配置文件/etc/unbound/unbound.conf的第177行，缺省为注释行，且内容为拒绝访问。复制本行内容到下面一行，去掉注释“#“，改refuse为allow。然后保存退出，重启服务即可。
155         # do-ip6: yes
156          do-ip6: no
//找到155行内容，在其下复制一行并去除注释，改yes为no，重启服务即可去除对Ipv6的监听
 
# 创建解析文件

cat > /etc/unbound/local.d/example.conf << EOF
local-zone: "example.com." static
local-data: "example.com. 86400 IN SOA ns.example.com. root 1 1D 1H 1W 1H"
local-data: "ns.example.com.            IN A 192.168.10.173"
local-data: "o19c1.example.com.            IN A 192.168.10.173"
local-data: "o19c-scan.example.com. IN A 192.168.10.177"
local-data: "o19c-scan.example.com. IN A 192.168.10.178"
local-data: "o19c-scan.example.com. IN A 192.168.10.179"
local-data-ptr: "192.168.10.177 o19c-scan.example.com."
local-data-ptr: "192.168.10.178 o19c-scan.example.com."
local-data-ptr: "192.168.10.179 o19c-scan.example.com."
EOF
 
# 启动服务及检查
systemctl start unbound
systemctl restart unbound
systemctl status unbound
netstat -tunlp |grep unbound
ss -tunlp|grep unbound

cat /etc/resolv.conf 
# Generated by NetworkManager
search example.com
nameserver 192.168.10.173

-- 验证
nslookup o19c-scan

[root@o19c2 network-scripts]# nslookup o19c-scan
Server:		192.168.10.173
Address:	192.168.10.173#53

Name:	o19c-scan.example.com
Address: 192.168.10.178
Name:	o19c-scan.example.com
Address: 192.168.10.179
Name:	o19c-scan.example.com
Address: 192.168.10.177


# 若不启用DNS，则执行下面命令
mv /etc/resolv.conf /etc/resolv.conf_orig

3. 共享存储配置

3.1 虚拟机创建共享磁盘

# vmware 配置
set path=%path%;/ups/vmware
vmware-vdiskmanager.exe -c -s 2g  -a lsilogic -t 4 "o19c-crs1.vmdk"
vmware-vdiskmanager.exe -c -s 2g  -a lsilogic -t 4 "o19c-crs2.vmdk"
vmware-vdiskmanager.exe -c -s 2g  -a lsilogic -t 4 "o19c-crs3.vmdk"
vmware-vdiskmanager.exe -c -s 32g -a lsilogic -t 4 "o19c-mgmt.vmdk"
vmware-vdiskmanager.exe -c -s 24g  -a lsilogic -t 4 "o19c-data.vmdk"
vmware-vdiskmanager.exe -c -s 8g  -a lsilogic -t 4 "o19c-data1.vmdk"

-- 编辑o19c1.vmx和o19c2.vmx文件，可以使用uuid
#
# ----------------------------------------------------------------
# SHARED DISK SECTION - (BEGIN)
# ----------------------------------------------------------------
# -  The goal in meeting the hardware requirements is to have a
#    shared storage for the two nodes. The way to achieve this in
#    VMware is the creation of a NEW SCSI BUS. It has to be of
#    type "virtual" and we must have the disk.locking = "false"
#    option.
# -  Just dataCacheMaxSize = "0" should be sufficient with the
#    diskLib.* parameters, although I include all parameters for
#    documentation purposes.
# -  maxUnsyncedWrites should matter for sparse disks only, and
#    I certainly do not recommend using sparse disks for
#    clustering.
# -  dataCacheMaxSize=0 should disable cache size completely, so
#    other three dataCache options should do nothing (no harm,
#    but nothing good either).
# ----------------------------------------------------------------
#
diskLib.dataCacheMaxSize = "0"
diskLib.dataCacheMaxReadAheadSize = "0"
diskLib.dataCacheMinReadAheadSize = "0"
diskLib.dataCachePageSize = "4096"
diskLib.maxUnsyncedWrites = "0"
disk.locking = "false"
# ----------------------------------------------------------------
#   Create one HBA
# ----------------------------------------------------------------
scsi1.present = "TRUE"
scsi1.virtualDev = "lsilogic"
scsi1.sharedBus = "virtual"
scsi1:0.present = "TRUE"
scsi1:0.fileName = "D:VMSNodesStorageso19cdb-crs1.vmdk"
scsi1:0.mode = "independent-persistent"
scsi1:0.redo = ""
scsi1:1.present = "TRUE"
scsi1:1.fileName = "D:VMSNodesStorageso19cdb-crs2.vmdk"
scsi1:1.mode = "independent-persistent"
scsi1:1.redo = ""
scsi1:2.present = "TRUE"
scsi1:2.fileName = "D:VMSNodesStorageso19cdb-crs3.vmdk"
scsi1:2.mode = "independent-persistent"
scsi1:2.redo = ""
scsi1:3.present = "TRUE"
scsi1:3.fileName = "D:VMSNodesStorageso19cdb-data.vmdk"
scsi1:3.mode = "independent-persistent"
scsi1:3.redo = ""
#
# ----------------------------------------------------------------
# SHARED DISK SECTION - (END)
# ----------------------------------------------------------------
#

3.2 配置UDEV

cd /dev
for i in $(lsblk |grep disk|awk '{print $1}'|grep -v sda); do
	echo "KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="`/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$i`", SYMLINK+="oracleasm/asm-$i", OWNER="grid", GROUP="asmadmin", MODE="0660""  >>/etc/udev/rules.d/99-oracle-asmdevices.rules
done

vi /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="36000c291d17055c53a62f7abcf88455a", SYMLINK+="oracleasm/asm-crs1", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="36000c29d08a0342cdca087c34777f9f7", SYMLINK+="oracleasm/asm-crs2", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="36000c299b5061370eef6a9d6edcfb4ee", SYMLINK+="oracleasm/asm-crs3", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="36000c299792d931c3c73d16c6d25d7ac", SYMLINK+="oracleasm/asm-mgmt", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="36000c295f9d043986fc840f111d477d8", SYMLINK+="oracleasm/asm-data", OWNER="grid", GROUP="asmadmin", MODE="0660"

# 重启生效
/sbin/partprobe /dev/sdd
/sbin/udevadm control --reload-rules
/sbin/udevadm trigger --type=devices --action=change

3.3 Verifying the Disk I/O Scheduler on Linux

# cat /sys/block/${ASM_DISK}/queue/scheduler
noop [deadline] cfq

cd /dev
for i in $(lsblk |grep disk|awk '{print $1}'|grep -v sda); do
	echo $i; cat /sys/block/${i}/queue/scheduler
done

cat >> /etc/udev/rules.d/60-oracle-schedulers.rules <<EOF
ACTION=="add|change", KERNEL=="sd[b-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="deadline"
EOF

/sbin/udevadm control --reload-rules
/sbin/udevadm trigger --type=devices --action=change

3.4 格式化共享磁盘（可选）

cd /dev/
for dsk in $(ls asm-*); do
    echo "dd if=/dev/zero of=${dsk} bs=1024k count=100"
done

4. GI安装配置

4.1 响应文件安装（silent)

其中-silent指的是静默安装，-ignorePrereq忽略prerequisite的检查结果，showProgress显示进度

# 1. 解压软件到GI HOME
su - grid
cd /ups/soft
unzip -qo LINUX.X64_193000_grid_home.zip -d ${ORACLE_HOME}/

# 2. 安装CVU rpm包
su - root
. /home/grid/.bash_profile
cd ${ORACLE_HOME}/cv/rpm
rpm -ivh cvuqdisk-1.0.10-1.rpm
# -- 实际该rpm包含（cvuhelper  cvupackager.sh  exectask  exectask.sh  orarun.sh  runfixup.sh）这部分文件

# 3. CVU检查
su - grid

${ORACLE_HOME}/runcluvfy.sh stage -pre crsinst -n o19c1,o19c2 -fixup -verbose > ~/grid_env_check.txt

# 4. 安装GI软件
# 4.1 修改响应文件内容
cat > ~/gridsetup.rsp<<-EOF
oracle.install.responseFileVersion=/oracle/install/rspfmt_crsinstall_response_schema_v19.0.0
oracle.install.option=CRS_CONFIG
oracle.install.asm.OSDBA=asmdba
oracle.install.asm.OSOPER=asmoper
oracle.install.asm.OSASM=asmadmin
oracle.install.crs.config.scanType=LOCAL_SCAN
oracle.install.crs.config.SCANClientDataFile=
oracle.install.crs.config.gpnp.scanName=o19c-scan
oracle.install.crs.config.gpnp.scanPort=1533
oracle.install.crs.config.ClusterConfiguration=STANDALONE
oracle.install.crs.config.configureAsExtendedCluster=false
oracle.install.crs.config.memberClusterManifestFile=
oracle.install.crs.config.clusterName=o19c-cluster
oracle.install.crs.config.gpnp.configureGNS=false
oracle.install.crs.config.autoConfigureClusterNodeVIP=false
oracle.install.crs.config.gpnp.gnsOption=
oracle.install.crs.config.gpnp.gnsClientDataFile=
oracle.install.crs.config.gpnp.gnsSubDomain=
oracle.install.crs.config.gpnp.gnsVIPAddress=
oracle.install.crs.config.sites=
oracle.install.crs.config.clusterNodes=o19c1:o19c1-vip:HUB,o19c2:o19c2-vip:HUB
oracle.install.crs.config.networkInterfaceList=ens32:192.168.10.0:1,ens33:172.168.0.0:5
oracle.install.crs.configureGIMR=false
oracle.install.asm.configureGIMRDataDG=false
oracle.install.crs.config.storageOption=
oracle.install.crs.config.sharedFileSystemStorage.votingDiskLocations=
oracle.install.crs.config.sharedFileSystemStorage.ocrLocations=
oracle.install.crs.config.useIPMI=false
oracle.install.crs.config.ipmi.bmcUsername=
oracle.install.crs.config.ipmi.bmcPassword=
oracle.install.asm.SYSASMPassword=oracle
oracle.install.asm.diskGroup.name=CRSDG
oracle.install.asm.diskGroup.redundancy=NORMAL
oracle.install.asm.diskGroup.AUSize=4
oracle.install.asm.diskGroup.FailureGroups=
oracle.install.asm.diskGroup.disksWithFailureGroupNames=
oracle.install.asm.diskGroup.disks=/dev/asm-crs1,/dev/asm-crs2,/dev/asm-crs3
oracle.install.asm.diskGroup.quorumFailureGroupNames=
oracle.install.asm.diskGroup.diskDiscoveryString=/dev/asm-*
oracle.install.asm.monitorPassword=oracle
oracle.install.asm.gimrDG.name=MGMT
oracle.install.asm.gimrDG.redundancy=EXTERNAL
oracle.install.asm.gimrDG.AUSize=4
oracle.install.asm.gimrDG.FailureGroups=
oracle.install.asm.gimrDG.disksWithFailureGroupNames=
oracle.install.asm.gimrDG.disks=/dev/asm-mgmt
oracle.install.asm.gimrDG.quorumFailureGroupNames=
oracle.install.asm.configureAFD=false
oracle.install.crs.configureRHPS=false
oracle.install.crs.config.ignoreDownNodes=false
oracle.install.config.managementOption=NONE
oracle.install.config.omsHost=
oracle.install.config.omsPort=
oracle.install.config.emAdminUser=
oracle.install.config.emAdminPassword=
oracle.install.crs.rootconfig.executeRootScript=false
oracle.install.crs.rootconfig.configMethod=
oracle.install.crs.rootconfig.sudoPath=
oracle.install.crs.rootconfig.sudoUserName=
oracle.install.crs.config.batchinfo=
oracle.install.crs.app.applicationAddress=
oracle.install.crs.deleteNode.nodes=
EOF

# 执行命令
${ORACLE_HOME}/gridSetup.sh -silent -ignorePrereq -skipPrereqs -waitforcompletion -responseFile ~/gridsetup.rsp 

# 4.2 命令行指定需要参数
${GRID_HOME}/gridSetup.sh -silent -ignorePrereq -skipPrereqs -waitforcompletion 
        -responseFile ${GRID_HOME}/install/response/gridsetup.rsp 
        INVENTORY_LOCATION=/ups/oracle/oraInventory 
        SELECTED_LANGUAGES=${ORA_LANGUAGES} 
        oracle.install.option=CRS_CONFIG 
        ORACLE_BASE=${ORACLE_BASE} 
        oracle.install.asm.OSDBA=dba 
        oracle.install.asm.OSASM=dba 
        oracle.install.crs.config.scanType=LOCAL_SCAN 
        oracle.install.crs.config.gpnp.scanName=${SCAN_NAME} 
        oracle.install.crs.config.gpnp.scanPort=${SCAN_PORT} 
        oracle.install.crs.config.ClusterConfiguration=STANDALONE 
        oracle.install.crs.config.configureAsExtendedCluster=false 
        oracle.install.crs.config.clusterName=${CLUSTER_NAME} 
        oracle.install.crs.config.gpnp.configureGNS=false 
        oracle.install.crs.config.autoConfigureClusterNodeVIP=false 
        oracle.install.crs.config.clusterNodes=o19c1:o19c1-vip:HUB,o19c2:o19c2-vip:HUB 
        oracle.install.crs.config.networkInterfaceList=ens32:192.168.10.0:1,ens33:172.168.0.0:5 
        oracle.install.crs.configureGIMR=false 
        oracle.install.crs.config.storageOption=FLEX_ASM_STORAGE 
        oracle.install.crs.config.useIPMI=false 
        oracle.install.asm.storageOption=ASM 
        oracle.install.asm.diskGroup.name=CRS 
        oracle.install.asm.diskGroup.redundancy=NORMAL 
        oracle.install.asm.diskGroup.disksWithFailureGroupNames=/dev/oracleasm/asm-crs-disk1,CRSFG1,/dev/oracleasm/asm-crs-disk2,CRSFG2,/dev/oracleasm/asm-crs-disk3,CRSFG3 
        oracle.install.asm.diskGroup.disks=/dev/oracleasm/asm-crs-disk1,/dev/oracleasm/asm-crs-disk2,/dev/oracleasm/asm-crs-disk3 
        oracle.install.asm.diskGroup.diskDiscoveryString=/dev/oracleasm/* 
        oracle.install.asm.monitorPassword=${SYS_PASSWORD} 
        oracle.install.asm.SYSASMPassword=${SYS_PASSWORD} 
        oracle.install.asm.configureAFD=false 
        oracle.install.asm.monitorPassword=${SYS_PASSWORD} 
        oracle.install.crs.configureRHPS=false 
        oracle.install.crs.config.ignoreDownNodes=false 
        oracle.install.config.managementOption=NONE 
        oracle.install.config.omsPort=0 
        oracle.install.crs.rootconfig.executeRootScript=false


# 5. 创建MGMT管理资料库等配置(一个节点执行)【19C 移除MGMT必选项】
/ups/oracle/19.2/grid/gridSetup.sh -silent -executeConfigTools -all -responseFile /home/grid/gridsetup.rsp

${GRID_HOME}/gridSetup.sh -silent -executeConfigTools 
        -responseFile ${GRID_HOME}/install/response/gridsetup.rsp 
        INVENTORY_LOCATION=/ups/oracle/oraInventory 
        SELECTED_LANGUAGES=${ORA_LANGUAGES} 
        oracle.install.option=CRS_CONFIG 
        ORACLE_BASE=${ORACLE_BASE} 
        oracle.install.asm.OSDBA=dba 
        oracle.install.asm.OSASM=dba 
        oracle.install.crs.config.scanType=LOCAL_SCAN 
        oracle.install.crs.config.gpnp.scanName=${SCAN_NAME} 
        oracle.install.crs.config.gpnp.scanPort=${SCAN_PORT} 
        oracle.install.crs.config.ClusterConfiguration=STANDALONE 
        oracle.install.crs.config.configureAsExtendedCluster=false 
        oracle.install.crs.config.clusterName=${CLUSTER_NAME} 
        oracle.install.crs.config.gpnp.configureGNS=false 
        oracle.install.crs.config.autoConfigureClusterNodeVIP=false 
        oracle.install.crs.config.clusterNodes=o19c1:o19c1-vip:HUB,o19c2:o19c2-vip:HUB 
        oracle.install.crs.config.networkInterfaceList=ens32:192.168.10.0:1,ens33:172.168.0.0:5 
        oracle.install.crs.configureGIMR=false 
        oracle.install.crs.config.storageOption=FLEX_ASM_STORAGE 
        oracle.install.crs.config.useIPMI=false 
        oracle.install.asm.storageOption=ASM 
        oracle.install.asm.diskGroup.name=CRS 
        oracle.install.asm.diskGroup.redundancy=NORMAL 
        oracle.install.asm.diskGroup.disksWithFailureGroupNames=/dev/oracleasm/asm-crs-disk1,CRSFG1,/dev/oracleasm/asm-crs-disk2,CRSFG2,/dev/oracleasm/asm-crs-disk3,CRSFG3 
        oracle.install.asm.diskGroup.disks=/dev/oracleasm/asm-crs-disk1,/dev/oracleasm/asm-crs-disk2,/dev/oracleasm/asm-crs-disk3 
        oracle.install.asm.diskGroup.diskDiscoveryString=/dev/oracleasm/* 
        oracle.install.asm.monitorPassword=${SYS_PASSWORD} 
        oracle.install.asm.SYSASMPassword=${SYS_PASSWORD} 
        oracle.install.asm.configureAFD=false 
        oracle.install.asm.monitorPassword=${SYS_PASSWORD} 
        oracle.install.crs.configureRHPS=false 
        oracle.install.crs.config.ignoreDownNodes=false 
        oracle.install.config.managementOption=NONE 
        oracle.install.config.omsPort=0 
        oracle.install.crs.rootconfig.executeRootScript=false

4.2 图形界面安装配置

${ORACLE_HOME}/gridSetup.sh

4.3 创建ASM diskgroup

4.3.1 asmca -silent方式创建

asmca -silent -createDiskGroup -diskString '/dev/asm-*' -diskGroupName DATA -disk '/dev/asm-data' -redundancy EXTERNAL -au_size 4

4.3.2 图形界面方式运行asmca

5. 安装DB软件

5.1 静默安装(silent)

# 1. 使用说明
${ORACLE_HOME}/runInstaller -silent -h
Usage:  runInstaller [<flag>] [<option>]
Following are the possible flags: 
	-help - display help. 
	-silent - run in silent mode. The inputs can be a response file or a list of command line variable value pairs.
		[-ignorePrereqFailure - ignore all prerequisite checks failures.]
	-responseFile - specify the complete path of the response file to use.
	-logLevel - enable the log of messages up to the priority level provided in this argument. Valid options are: severe, warning, info, config, fine, finer, finest.
	-executePrereqs | -executeConfigTools | -createGoldImage
	-executePrereqs - execute the prerequisite checks only.
	-executeConfigTools - execute the config tools for an installed home.
	-createGoldImage - create a gold image from the current Oracle home.
		-destinationLocation - specify the complete path to where the created gold image will be located.
		[-exclFiles - specify the complete paths to the files to be excluded from the new gold image.]
	-debug - run in debug mode.
	-printdiskusage - log the debug information for the disk usage.
	-printmemory - log the debug information for the memory usage.
	-printtime - log the debug information for the time usage.
	-waitForCompletion - wait for the completion of the installation, instead of spawning the installer and returning the console prompt.
	-noconfig - do not execute the config tools.
	-noconsole - suppress the display of messages in the console. The console is not allocated.
	-ignoreInternalDriverError - ignore any internal driver errors.
	-noCopy - perform the configuration without copying the software on to the remote nodes.
	-applyRU - apply release update to the Oracle home.
	-applyOneOffs - apply one-off patch to the Oracle home. Multiple one-off patches can be passed as a comma separated list of locations.
	

# 2. 解压安装文件DB ORACLE_HOME
su - oracle
[oracle@o19c1 soft]$ mkdir -p /ups/oracle/database/db_1
[oracle@o19c1 soft]$ unzip -qo p6880880_190000_Linux-x86-64.zip -d /ups/oracle/database/db_1

# 3. 安装软件
# 3.1 修改配置影响文件方式
cat > /ups/soft/db_install.rsp <<-EOF
oracle.install.responseFileVersion=/oracle/install/rspfmt_dbinstall_response_schema_v19.0.0
oracle.install.option=INSTALL_DB_SWONLY
oracle.install.db.InstallEdition=EE
oracle.install.db.OSDBA_GROUP=dba
oracle.install.db.OSOPER_GROUP=oper
oracle.install.db.OSBACKUPDBA_GROUP=backupdba
oracle.install.db.OSDGDBA_GROUP=dgdba
oracle.install.db.OSKMDBA_GROUP=kmdba
oracle.install.db.OSRACDBA_GROUP=racdba
oracle.install.db.rootconfig.executeRootScript=false
oracle.install.db.rootconfig.configMethod=
oracle.install.db.rootconfig.sudoPath=
oracle.install.db.rootconfig.sudoUserName=
oracle.install.db.CLUSTER_NODES=o19c1,o19c2
oracle.install.db.config.starterdb.type=GENERAL_PURPOSE
oracle.install.db.config.starterdb.globalDBName=o19cdb
oracle.install.db.config.starterdb.SID=o19cdb
oracle.install.db.ConfigureAsContainerDB=true
oracle.install.db.config.PDBName=pdb_orcl
oracle.install.db.config.starterdb.characterSet=AL32UTF8
oracle.install.db.config.starterdb.memoryOption=false
oracle.install.db.config.starterdb.memoryLimit=2048
oracle.install.db.config.starterdb.installExampleSchemas=false
oracle.install.db.config.starterdb.password.ALL=oracle
oracle.install.db.config.starterdb.password.SYS=
oracle.install.db.config.starterdb.password.SYSTEM=
oracle.install.db.config.starterdb.password.DBSNMP=
oracle.install.db.config.starterdb.password.PDBADMIN=oracle
oracle.install.db.config.starterdb.managementOption=
oracle.install.db.config.starterdb.omsHost=
oracle.install.db.config.starterdb.omsPort=
oracle.install.db.config.starterdb.emAdminUser=
oracle.install.db.config.starterdb.emAdminPassword=
oracle.install.db.config.starterdb.enableRecovery=false
oracle.install.db.config.starterdb.storageType=ASM_STORAGE
oracle.install.db.config.starterdb.fileSystemStorage.dataLocation=
oracle.install.db.config.starterdb.fileSystemStorage.recoveryLocation=
oracle.install.db.config.asm.diskGroup=DATA
oracle.install.db.config.asm.ASMSNMPPassword=oracle
EOF

${ORACLE_HOME}/runInstaller -silent -waitForCompletion -ignorePrereqFailure -responseFile /ups/soft/db_install.rsp 


# 3.2 命令行参数
su - oracle
${ORACLE_HOME}/runInstaller -ignorePrereq -waitforcompletion -silent          
        -responseFile ${ORACLE_HOME}/install/response/db_install.rsp          
        oracle.install.option=${INSTALL_OPTION}                               
        ORACLE_HOSTNAME=${ORA_HOST_NAME}                                      
        UNIX_GROUP_NAME=${UNIX_GROUP_NAME}                                    
        INVENTORY_LOCATION=${ORA_INVENTORY}                                   
        SELECTED_LANGUAGES=en,en_US,zh_CN                                     
        ORACLE_HOME=${DB_HOME_DIR}                                            
        ORACLE_BASE=${DB_BASE_DIR}                                            
        oracle.install.db.InstallEdition=${INSTALL_EDITION}                   
        oracle.install.db.OSDBA_GROUP=${OSDBA_GROUP}                          
        oracle.install.db.OSOPER_GROUP=${OSOPER_GROUP}                        
        oracle.install.db.OSBACKUPDBA_GROUP=${OSBACKUPDBA_GROUP}              
        oracle.install.db.OSDGDBA_GROUP=${OSDGDBA_GROUP}                      
        oracle.install.db.OSKMDBA_GROUP=${OSKMDBA_GROUP}                      
        oracle.install.db.OSRACDBA_GROUP=${OSRACDBA_GROUP}                    
        SECURITY_UPDATES_VIA_MYORACLESUPPORT=false                            
        DECLINE_SECURITY_UPDATES=true                                         
        oracle.install.db.CLUSTER_NODES=${NODE_HOSTNAME_LIST}

5.2 图形界面安装

oracle用户执行命令：${ORACLE_HOME}/runInstaller

6. 安装软件补丁后建库

使用root用户执行补丁更新命令，GI打补丁需要用GI $ORACLE_HOME/OPatch/opatchauto，DB打补丁需要使用 DB $ORACLE_HOME/OPatch/opatchauto

su - root
/ups/app/19c/grid/OPatch/opatchauto apply /oracle/soft/<patchid> -oh /ups/app/19c/grid

# 由于没有创建实例，未能更新DB  HOME，需要单独指定目录更新补丁
su - root
/ups/app/oracle/database/19c/db_1/OPatch/opatchauto apply /oracle/soft/<patchid> -oh /ups/app/oracle/database/19c/db_1

7. dbca工具配置数据库

7.1 静默方式(silent)

# 默认响应文件
${ORACLE_HOME}/assistants/dbca/dbca.rsp

# 创建数据库
su - oracle
cat> ~/dbca.rsp <<-EOF
responseFileVersion=/oracle/assistants/rspfmt_dbca_response_schema_v19.0.0
gdbName=o19cdb
sid=o19cdb
databaseConfigType=RAC
policyManaged=
createServerPool=
serverPoolName=
cardinality=
force=
pqPoolName=
pqCardinality=
createAsContainerDatabase=true
numberOfPDBs=1
pdbName=PDB_ORCL
useLocalUndoForPDBs=TRUE
pdbAdminPassword=oracle
nodelist=o19c1,o19c2
templateName=General_Purpose.dbc
sysPassword=oracle
systemPassword=oracle
oracleHomeUserPassword=
emConfiguration=
emExpressPort=5500
runCVUChecks=FALSE
dbsnmpPassword=oracle
omsHost=
omsPort=
emUser=
emPassword=
dvConfiguration=
dvUserName=
dvUserPassword=
dvAccountManagerName=
dvAccountManagerPassword=
olsConfiguration=
datafileJarLocation={ORACLE_HOME}/assistants/dbca/templates/
datafileDestination=
recoveryAreaDestination=
storageType=ASM
diskGroupName=DATA
asmsnmpPassword=
recoveryGroupName=
characterSet=AL32UTF8
nationalCharacterSet=AL16UTF16
registerWithDirService=
dirServiceUserName=
dirServicePassword=
walletPassword=
listeners=
variablesFile=
variables=
initParams=dispatchers=,processes=600,open_cursors=600
sampleSchema=
memoryPercentage=
databaseType=MULTIPURPOSE
automaticMemoryManagement=false
totalMemory=1024
EOF

${ORACLE_HOME}/bin/dbca -silent -ignorePreReqs -ignorePrereqFailure -createDatabase -responseFile /tmp/dbca.rsp -redoLogFileSize 50
 


# 删除数据库
${ORACLE_HOME}/bin/dbca -silent -ignorePreReqs -ignorePrereqFailure -deleteDatabase -sourceDB o19c -sysDBAUserName sys -sysDBAPassword oracle

# 添加实例
${ORACLE_HOME}/bin/dbca -silent -addInstance -gdbName o19c -nodeName o19c2 -instanceName o19c2 -sysDBAUserName sys -sysDBAPassword oracle

7.2 图形界面安装配置

dbca

8. 收尾工作

8.1 传递oui-patch.xml到非软件安装的其它节点（19C)

# 1. 复制oui-patch.xml文件到其它节点，避免后期补丁升级异常
-- “During 19.x GI installation, the file 'oui-patch.xml' will be created under the central inventory directory on the OUI node (node where gridSetup.sh was invoked) but not on the other nodes.

# node1 (执行gridSetup.sh的节点)
export INV_DIR="$(grep 'inventory_loc' /etc/oraInst.loc|awk -F= '{print $NF}')/ContentsXML"
scp ${INV_DIR}/oui-patch.xml  o19c2:${INV_DIR}/oui-patch.xml
ssh o19c2 "chmod 660 ${INV_DIR}/oui-patch.xml && chown grid:oinstall ${INV_DIR}/oui-patch.xml"