视频：<Oracle11g RAC(openfiler +multipath +udev +补丁升级)>
或
视频：<Oracle11g RAC(openfiler +multipath +udev +补丁升级)>
五、安装Grid前的准备工作(rac1、rac2)
5.1 创建用户组，用户和目录
--用户组
# /usr/sbin/groupadd -g 1000 oinstall
# /usr/sbin/groupadd -g 1020 asmadmin
# /usr/sbin/groupadd -g 1021 asmdba
# /usr/sbin/groupadd -g 1022 asmoper
# /usr/sbin/groupadd -g 1031 dba
# /usr/sbin/groupadd -g 1032 oper
--用户
#/usr/sbin/useradd -u 1100 -g oinstall -G asmadmin,asmdba,asmoper grid
#passwd grid
#/usr/sbin/useradd -u 1101 -g oinstall -G dba,asmdba,oper oracle
#passwd oracle
--目录
#mkdir -p /u01/app/11.2.0/grid
#mkdir -p /u01/app/grid
#chown -R grid:oinstall /u01
#mkdir -p /u01/app/oracle
#chown oracle:oinstall /u01/app/oracle
#chmod -R 775 /u01
5.2 Checking Server Hardware and Memory Configuration
a. the size of the /dev/shm mount area on each server must be greater than the system global areal (SGA) and the program global area (PGA) of the databases on the servers
#修改/dev/shm的值，需要修改/etc/fstab，
tmpfs /dev/shm tmpfs size=1500M 0 0
mount -o remount /dev/shm
注：size=1500M 不能用1.5G 否则会报：mount: /dev/shm not mounted already, or bad option
b.
# grep MemTotal /proc/meminfo
# grep SwapTotal /proc/meminfo
# df -k /tmp
# free
具体值参考文档
5.3 检查网络需求
5.3.1 手动配置IP Address
[root@node1 app]# vi /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1 localhost
::1 localhost6.localdomain6 localhost6
#rac1
192.168.2.20 rac1
192.168.2.21 rac1-vip
10.0.2.10 rac1-priv
#rac2
192.168.2.30 rac2
192.168.2.31 rac2-vip
10.0.2.11 rac2-priv
#Scan IP
192.168.2.24 scan-cluster
说明: 如果这里使用/etc/hosts文件解析scan ip,因为不能做轮训的负载均衡，所以这时候scan ip只能有一个。
5.3.2 关闭SElinux和防火墙
在/etc/selinux/config修改：
SELINUX=disabled
然后重启
[root@rac1 etc]# vi /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of these two values:
[root@rac1 etc]# reboot
关闭iptables并关闭自动启动：
service iptables stop
chkconfig iptables off
chkconfig --list |grep iptables
5.4 检查软件需求
5.4.1所需系统软件包
使用如下命令来查询系统已经安装了哪些package，还缺什么package：
rpm -q --qf '%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n' binutils \
compat-libstdc++-33 \
glibc \
ksh \
libaio \
libgcc \
libstdc++ \
make \
compat-libcap1 \
gcc \
gcc-c++ \
glibc-devel \
libaio-devel \
libstdc++-devel \
sysstat
示例：
[root@rac2 ~]# rpm -q --qf '%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n' binutils \
> compat-libstdc++-33 \
> glibc \
> ksh \
> libaio \
> libgcc \
> libstdc++ \
> make \
> compat-libcap1 \
> gcc \
> gcc-c++ \
> glibc-devel \
> libaio-devel \
> libstdc++-devel \
> sysstat
binutils-2.20.51.0.2-5.48.el6 (x86_64)
compat-libstdc++-33-3.2.3-69.el6 (x86_64)
glibc-2.12-1.212.0.1.el6 (x86_64)
glibc-2.12-1.212.0.1.el6 (i686)
package ksh is not installed
libaio-0.3.107-10.el6 (x86_64)
libgcc-4.4.7-23.0.1.el6 (x86_64)
libgcc-4.4.7-23.0.1.el6 (i686)
libstdc++-4.4.7-23.0.1.el6 (x86_64)
make-3.81-23.el6 (x86_64)
compat-libcap1-1.10-1 (x86_64)
gcc-4.4.7-23.0.1.el6 (x86_64)
gcc-c++-4.4.7-23.0.1.el6 (x86_64)
glibc-devel-2.12-1.212.0.1.el6 (x86_64)
package libaio-devel is not installed
libstdc++-devel-4.4.7-23.0.1.el6 (x86_64)
sysstat-9.0.4-33.el6_9.1 (x86_64)
5.4.2网络时钟协议(NTP)设置
有两种网络时钟可供选择：一种是操作系统自带的NTP;一种是Oracle Cluster Time Synchronization Service.我们选用第二种，oracle自带的。
禁止系统自带网络时钟协议
# /sbin/service ntpd stop
# chkconfig ntpd off
# rm /etc/ntp.conf
# rm /var/run/ntpd.pid #这个文件可能不存在
5.5 设置集群软件用户的环境变量
5.5.1设置Grid用户创建文件的默认权限
[root@node1 Server]# su - grid
[grid@node1 ~]$ echo $SHELL
/bin/bash
实验环境使用的是bash 则修改.bash_profile,在此文件结尾加入:
保存 使其生效
[grid@node1 ~]$ . ./.bash_profile
[grid@node1 ~]$
5.5.2 修改内核参数
在/etc/sysctl.conf中添加如下内容：
cat >> /etc/sysctl.conf<<EOF
kernel.shmmax =
kernel.shmall =
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
fs.file-max = 6815744
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
vm.swappiness = 0
vm.dirty_background_ratio = 3
vm.dirty_ratio = 80
vm.dirty_expire_centisecs = 500
vm.dirty_writeback_centisecs = 100
vm.min_free_kbytes=65536
EOF
注：
ram > 8G then
vm.min_free_kbytes=262144
ram <8g
vm.min_free_kbytes=65536
[root@node1 ~]# sysctl -p
修改/etc/security/limits.conf
[root@node1 ~]# cat >> /etc/security/limits.conf<<EOF
> grid soft nproc 2047
> grid hard nproc 16384
> grid soft nofile 1024
> grid hard nofile 65536
> oracle soft nproc 2047
> oracle hard nproc 16384
> oracle soft nofile 1024
> oracle hard nofile 65536
> EOF
[root@node1 ~]#
修改/etc/pam.d/login
[root@node1 ~]# cat >> /etc/pam.d/login<<EOF
> session required pam_limits.so
> EOF
[root@node1 ~]#
5.7 创建节点间的互信机制
5.7.1以grid用户创建互信机制
节点node1
[root@node1 ~]# su - grid
[grid@node1 ~]$ pwd
/home/grid
[grid@node1 ~]$ ls -al
total 44
drwx------ 4 grid oinstall 4096 Aug 22 21:54 .
drwxr-xr-x 4 root root 4096 Aug 22 06:42 ..
-rw------- 1 grid oinstall 150 Aug 22 21:54 .bash_history
-rw-r--r-- 1 grid oinstall 33 Aug 22 06:41 .bash_logout
-rw-r--r-- 1 grid oinstall 188 Aug 22 17:25 .bash_profile
-rw-r--r-- 1 grid oinstall 124 Aug 22 06:41 .bashrc
-rw-r--r-- 1 grid oinstall 515 Aug 22 06:41 .emacs
drwxr-xr-x 3 grid oinstall 4096 Aug 22 06:41 .kde
drwxr-xr-x 4 grid oinstall 4096 Aug 22 06:41 .mozilla
-rw------- 1 grid oinstall 642 Aug 22 17:25 .viminfo
[grid@node1 ~]$ mkdir ~/.ssh
[grid@node1 ~]$ chmod 700 ~/.ssh
[grid@node1 ~]$ ls -al
total 48
drwx------ 5 grid oinstall 4096 Aug 25 06:05 .
drwxr-xr-x 4 root root 4096 Aug 22 06:42 ..
-rw------- 1 grid oinstall 150 Aug 22 21:54 .bash_history
-rw-r--r-- 1 grid oinstall 33 Aug 22 06:41 .bash_logout
-rw-r--r-- 1 grid oinstall 188 Aug 22 17:25 .bash_profile
-rw-r--r-- 1 grid oinstall 124 Aug 22 06:41 .bashrc
-rw-r--r-- 1 grid oinstall 515 Aug 22 06:41 .emacs
drwxr-xr-x 3 grid oinstall 4096 Aug 22 06:41 .kde
drwxr-xr-x 4 grid oinstall 4096 Aug 22 06:41 .mozilla
drwx------ 2 grid oinstall 4096 Aug 25 06:05 .ssh
-rw------- 1 grid oinstall 642 Aug 22 17:25 .viminfo
[grid@node1 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/grid/.ssh/id_rsa):
Enter passphrase (empty for no passphrase): --------默认
Enter same passphrase again: --------默认
Your identification has been saved in /home/grid/.ssh/id_rsa.
Your public key has been saved in /home/grid/.ssh/id_rsa.pub.
The key fingerprint is:
d6:9b:32:56:90:78:de:ff:d5:5e:ec:3a:12:cc:0d:6c grid@node1.localdomain
[grid@node1 ~]$ ssh-keygen -t dsa
Generating public/private dsa key pair.
Enter file in which to save the key (/home/grid/.ssh/id_dsa):
Enter passphrase (empty for no passphrase): --------默认
Enter same passphrase again: ---------默认
Your identification has been saved in /home/grid/.ssh/id_dsa.
Your public key has been saved in /home/grid/.ssh/id_dsa.pub.
The key fingerprint is:
6e:16:64:ee:2a:96:9a:04:a9:e5:b3:d8:ca:a9:db:67 grid@node1.localdomain
[grid@node1 ~]$
节点node2
[root@node2 ~]# su - grid
[grid@node2 ~]$ pwd
/home/grid
[grid@node2 ~]$ ls -al
total 44
drwx------ 4 grid oinstall 4096 Aug 22 21:54 .
drwxr-xr-x 4 root root 4096 Aug 22 06:42 ..
-rw------- 1 grid oinstall 150 Aug 22 21:54 .bash_history
-rw-r--r-- 1 grid oinstall 33 Aug 22 06:41 .bash_logout
-rw-r--r-- 1 grid oinstall 188 Aug 22 17:25 .bash_profile
-rw-r--r-- 1 grid oinstall 124 Aug 22 06:41 .bashrc
-rw-r--r-- 1 grid oinstall 515 Aug 22 06:41 .emacs
drwxr-xr-x 3 grid oinstall 4096 Aug 22 06:41 .kde
drwxr-xr-x 4 grid oinstall 4096 Aug 22 06:41 .mozilla
-rw------- 1 grid oinstall 642 Aug 22 17:25 .viminfo
[grid@node2 ~]$ mkdir ~/.ssh
[grid@node2 ~]$ chmod 700 ~/.ssh
[grid@node2 ~]$ ls -al
total 48
drwx------ 5 grid oinstall 4096 Aug 25 06:10 .
drwxr-xr-x 4 root root 4096 Aug 22 06:42 ..
-rw------- 1 grid oinstall 150 Aug 22 21:54 .bash_history
-rw-r--r-- 1 grid oinstall 33 Aug 22 06:41 .bash_logout
-rw-r--r-- 1 grid oinstall 188 Aug 22 17:25 .bash_profile
-rw-r--r-- 1 grid oinstall 124 Aug 22 06:41 .bashrc
-rw-r--r-- 1 grid oinstall 515 Aug 22 06:41 .emacs
drwxr-xr-x 3 grid oinstall 4096 Aug 22 06:41 .kde
drwxr-xr-x 4 grid oinstall 4096 Aug 22 06:41 .mozilla
drwx------ 2 grid oinstall 4096 Aug 25 06:10 .ssh
-rw------- 1 grid oinstall 642 Aug 22 17:25 .viminfo
[grid@node2 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/grid/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/grid/.ssh/id_rsa.
Your public key has been saved in /home/grid/.ssh/id_rsa.pub.
The key fingerprint is:
8c:fd:26:e2:3a:14:78:63:2c:6a:bf:3e:04:66:1d:22 grid@node2.localdomain
[grid@node2 ~]$ ssh-keygen -t dsa
Generating public/private dsa key pair.
Enter file in which to save the key (/home/grid/.ssh/id_dsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/grid/.ssh/id_dsa.
Your public key has been saved in /home/grid/.ssh/id_dsa.pub.
The key fingerprint is:
1b:21:fb:f3:55:4e:01:9d:8b:56:6b:2a:39:98:f9:c5 grid@node2.localdomain
[grid@node2 ~]$
返回节点node1
[grid@node1 ~]$ cat ~/.ssh/id_rsa.pub >>~/.ssh/authorized_keys
[grid@node1 ~]$ cat ~/.ssh/id_dsa.pub >>~/.ssh/authorized_keys
[grid@node1 ~]$ ssh node2 cat ~/.ssh/id_rsa.pub >>~/.ssh/authorized_keys
grid@node2's password:
[grid@node1 ~]$ ssh node2 cat ~/.ssh/id_dsa.pub >>~/.ssh/authorized_keys
grid@node2's password:
[grid@node1 ~]$ scp ~/.ssh/authorized_keys node2:~/.ssh/authorized_keys
grid@node2's password:
authorized_keys 100% 2032 2.0KB/s 00:00
[grid@node1 ~]$
验证
[grid@node1 ~]$ ssh node1 date
Mon Aug 25 06:39:46 EDT 2014
[grid@node1 ~]$ ssh node2 date
Mon Aug 25 06:39:50 EDT 2014
[grid@node1 ~]$
[grid@node2 .ssh]$ ssh node1 date
Mon Aug 25 06:40:03 EDT 2014
[grid@node2 .ssh]$ ssh node2 date
Mon Aug 25 06:40:07 EDT 2014
[grid@node2 .ssh]$
5.7.2以oracle用户创建互信机制
5.8 创建ASM磁盘
5.8.1利用udev绑定盘符
1. 创建UDEV规则，以硬盘为单位（也可以以分区为单位： /dev/sdb1 /dev/sdc1）。
[root@rac2 device-mapper-1.02.117]# pwd
/usr/share/doc/device-mapper-1.02.117
[root@rac2 device-mapper-1.02.117]# cp 12-dm-permissions.rules /etc/udev/rules.d/
[root@rac2 device-mapper-1.02.117]#
修改文件/etc/udev/rules.d/12-dm-permissions.rules，添加如下内容：
ENV{DM_NAME}=="asmocrvote1", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="asmocrvote2", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="asmocrvote3", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="asmr5datadisk1", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="asmr5datadisk2", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="asmr5archdisk1", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
5.8.2重启udev服务
[root@node1 rules.d]# start_udev
Starting udev: [ OK ]
[root@node1 rules.d]#
[root@node2 rules.d]# start_udev
Starting udev: [ OK ]
[root@node2 rules.d]#
查看结果：
[root@rac2 rules.d]# ls -l /dev/dm*
brw-rw---- 1 root disk 249, 0 10月
brw-rw---- 1 grid asmadmin 249, 1 10月
brw-rw---- 1 grid asmadmin 249, 2 10月
brw-rw---- 1 grid asmadmin 249, 3 10月
brw-rw---- 1 grid asmadmin 249, 4 10月
brw-rw---- 1 grid asmadmin 249, 5 10月
brw-rw---- 1 grid asmadmin 249, 6 10月
[root@rac2 rules.d]#
到此ASM共享存储配置完毕！！！！