[root@rhel11gr2rac1 bin]# ./crsctl status res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.DATA.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.FRA.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.LISTENER.lsnr
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.OCR_VOTING.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
ONLINE ONLINE rhel11gr2rac1 mounted on /golden_
gate
ONLINE ONLINE rhel11gr2rac2 mounted on /golden_
gate
ora.asm
ONLINE ONLINE rhel11gr2rac1 Started
ONLINE ONLINE rhel11gr2rac2 Started
ora.gsd
OFFLINE OFFLINE rhel11gr2rac1
OFFLINE OFFLINE rhel11gr2rac2
ora.net1.network
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.ons
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.registry.acfs
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
1 ONLINE ONLINE rhel11gr2rac2
goldengate-vip
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.cvu
1 ONLINE ONLINE rhel11gr2rac1
ora.ggdb.db
1 OFFLINE OFFLINE
ora.layatest.db
1 ONLINE ONLINE rhel11gr2rac1 Open
2 ONLINE ONLINE rhel11gr2rac2 Open
ora.oc4j
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac2.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan1.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan2.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan3.vip
1 ONLINE ONLINE rhel11gr2rac1
--
[root@rhel11gr2rac1 bin]# ./ocrconfig -export /u01/app/oracle/ocr.bkp_23oct
[root@rhel11gr2rac1 bin]# ls -lrt /u01/app/oracle/ocr.bkp_23oct
-rw------- 1 root root 131266 Oct 23 21:26 /u01/app/oracle/ocr.bkp_23oct
Stop crs on both nodes
Node 1
[root@rhel11gr2rac1 bin]# ./crsctl stop crs
Node 2
[root@rhel11gr2rac2 bin]# ./crsctl stop crs
NAME TOTAL_MB USABLE_FILE_MB STATE TYPE
------------------------------ ---------- -------------- ----------- ------
ACFDG 9759 581 MOUNTED EXTERN
DATA 3871 1977 CONNECTED EXTERN
FRA 1918 1816 CONNECTED EXTERN
OCR_VOTING 2868 436 MOUNTED NORMAL
NAME FAILGROUP PATH TOTAL_MB HEADER_STATU REDUNDA
------------------------------ ------------------------------ ---------------------------------------- ---------- ------------ -------
ACFSDISK ACFSDISK /dev/oracleasm/disks/ACFSDISK 9759 MEMBER UNKNOWN
FRA FRA /dev/oracleasm/disks/FRA 1918 MEMBER UNKNOWN
DATA DATA /dev/oracleasm/disks/DATA 3871 MEMBER UNKNOWN
OCR_VOTING_0002 OCR_VOTING_0002 /dev/oracleasm/disks/OCR_VOT_DISK_3 956 MEMBER UNKNOWN
OCR_VOTING_0001 OCR_VOTING_0001 /dev/oracleasm/disks/OCR_VOT_DISK_2 956 MEMBER UNKNOWN
OCR_VOTING_0000 OCR_VOTING_0000 /dev/oracleasm/disks/OCR_VOT_DISK_1 956 MEMBER UNKNOWN
[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOT_DISK_1
OCR_VOT_DISK_2
OCR_VOT_DISK_3
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm querydisk /dev/sd* | grep "OCR"
Device "/dev/sdc1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sdd1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sde1" is marked an ASM disk with the label "OCR_VOT_DISK_2"
Device "/dev/sdf1" is marked an ASM disk with the label "OCR_VOT_DISK_2"
Device "/dev/sdg1" is marked an ASM disk with the label "OCR_VOT_DISK_3"
Device "/dev/sdh1" is marked an ASM disk with the label "OCR_VOT_DISK_3"
[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOT_DISK_1
OCR_VOT_DISK_2
OCR_VOT_DISK_3
[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_3
Removing ASM disk "OCR_VOT_DISK_3": [ OK ]
[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm querydisk /dev/sd*
Device "/dev/sda" is not marked as an ASM disk
Device "/dev/sda1" is not marked as an ASM disk
Device "/dev/sda2" is not marked as an ASM disk
Device "/dev/sda3" is not marked as an ASM disk
Device "/dev/sdb" is not marked as an ASM disk
Device "/dev/sdb1" is not marked as an ASM disk
Device "/dev/sdc" is not marked as an ASM disk
Device "/dev/sdc1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sdd" is not marked as an ASM disk
Device "/dev/sdd1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sde" is not marked as an ASM disk
Device "/dev/sde1" is marked an ASM disk with the label "OCR_VOT_DISK_2"
Device "/dev/sdf" is not marked as an ASM disk
Device "/dev/sdf1" is marked an ASM disk with the label "OCR_VOT_DISK_2"
[root@rhel11gr2rac1 bin]# ./crsctl start crs
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
[ohasd(10125)]CRS-2807:Resource 'ora.crsd' failed to start automatically.
2013-10-23 05:44:24.116
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted.
Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".
2013-10-23 05:44:56.872
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted. Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".
2013-10-23 05:45:26.926
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted. Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE INTERMEDIATE rhel11gr2rac1 OCR not started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 ONLINE ONLINE rhel11gr2rac1
ora.crsd
1 ONLINE OFFLINE
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 ONLINE INTERMEDIATE rhel11gr2rac1
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[oracle@rhel11gr2rac1 ~]$ export ORACLE_HOME=/grid/11.2
[oracle@rhel11gr2rac1 ~]$ export PATH=$ORACLE_HOME/bin:$PATH
[oracle@rhel11gr2rac1 ~]$ export ORACLE_SID=+ASM1
[oracle@rhel11gr2rac1 ~]$ asmcmd
ASMCMD> lsdg
State Type Rebal Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 4096 1048576 9759 581 0 581 0 N ACFDG/
MOUNTED NORMAL N 512 4096 1048576 3821 1557 0 778 0 N DATA/
MOUNTED NORMAL N 512 4096 1048576 1876 1630 0 815 0 N FRA/
ASMCMD> exit
[oracle@rhel11gr2rac1 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.3.0 Production on Wed Oct 23 21:51:57 2013
Copyright (c) 1982, 2011, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> select name, state from V$asm_diskgroup;
NAME STATE
------------------------------ -----------
ACFDG MOUNTED
DATA MOUNTED
FRA MOUNTED
OCR_VOTING DISMOUNTED
SQL> alter diskgroup ocr_voting mount;
alter diskgroup ocr_voting mount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "2" is missing from group number "4"
SQL> select name, path , failgroup , header_status from V$asm_disk;
NAME PATH FAILGROUP HEADER_STATU
------------------------------ ---------------------------------------- ------------------------------ ------------
/dev/oracleasm/disks/OCR_VOT_DISK_2 MEMBER
/dev/oracleasm/disks/OCR_VOT_DISK_1 MEMBER
ACFSDISK /dev/oracleasm/disks/ACFSDISK ACFSDISK MEMBER
FRA_0001 /dev/oracleasm/disks/FRA2 FRA_0001 MEMBER
FRA_0000 /dev/oracleasm/disks/FRA1 FRA_0000 MEMBER
DATA_0001 /dev/oracleasm/disks/DATA2 DATA_0001 MEMBER
DATA_0000 /dev/oracleasm/disks/DATA1 DATA_0000 MEMBER
SQL> alter diskgroup ocr_voting mount force;
Diskgroup altered.
--
From Asm instance alert log.
SQL> alter diskgroup ocr_voting mount force
NOTE: cache registered group OCR_VOTING number=4 incarn=0xf1ec9b2a
NOTE: cache began mount (first) of group OCR_VOTING number=4 incarn=0xf1ec9b2a
NOTE: Assigning number (4,1) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_2)
NOTE: Assigning number (4,0) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_1)
Wed Oct 23 05:58:17 2013
NOTE: GMON heartbeating for grp 4
GMON querying group 4 at 25 for pid 23, osid 11366
NOTE: Assigning number (4,2) to disk ()
GMON querying group 4 at 26 for pid 23, osid 11366
NOTE: process _user11366_+asm1 (11366) initiating offline of disk 2.3916196865 () with mask 0x7e in group 4
NOTE: checking PST: grp = 4
GMON checking disk modes for group 4 at 27 for pid 23, osid 11366
NOTE: checking PST for grp 4 done.
WARNING: Disk 2 () in group 4 mode 0x7f is now being offlined
NOTE: initiating PST update: grp = 4, dsk = 2/0xe96c6c01, mask = 0x6a, op = clear
GMON updating disk modes for group 4 at 28 for pid 23, osid 11366
NOTE: group OCR_VOTING: updated PST location: disk 0000 (PST copy 0)
NOTE: group OCR_VOTING: updated PST location: disk 0001 (PST copy 1)
Wed Oct 23 21:58:30 2013
NOTE: PST update grp = 4 completed successfully
------------
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rhel11gr2rac1 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 ONLINE ONLINE rhel11gr2rac1
ora.crsd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 ONLINE ONLINE rhel11gr2rac1
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rhel11gr2rac1 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 ONLINE ONLINE rhel11gr2rac1
ora.crsd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 ONLINE ONLINE rhel11gr2rac1
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
ONLINE ONLINE rhel11gr2rac1
ora.DATA.dg
ONLINE ONLINE rhel11gr2rac1
ora.FRA.dg
ONLINE ONLINE rhel11gr2rac1
ora.LISTENER.lsnr
ONLINE OFFLINE rhel11gr2rac1 STARTING
ora.OCR_VOTING.dg
ONLINE ONLINE rhel11gr2rac1
ora.acfdg.acfsvolume.acfs
ONLINE OFFLINE rhel11gr2rac1 volume /golden_gate
offline,STARTING
ora.asm
ONLINE ONLINE rhel11gr2rac1 Started
ora.gsd
OFFLINE OFFLINE rhel11gr2rac1
ora.net1.network
ONLINE ONLINE rhel11gr2rac1
ora.ons
ONLINE OFFLINE rhel11gr2rac1 STARTING
ora.registry.acfs
ONLINE ONLINE rhel11gr2rac1
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
1 ONLINE UNKNOWN rhel11gr2rac1
goldengate-vip
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
1 ONLINE OFFLINE STARTING
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.cvu
1 ONLINE ONLINE rhel11gr2rac1
ora.ggdb.db
1 OFFLINE OFFLINE
ora.layatest.db
1 ONLINE OFFLINE Instance Shutdown
2 ONLINE OFFLINE
ora.oc4j
1 ONLINE OFFLINE STARTING
ora.rhel11gr2rac1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac2.vip
1 ONLINE INTERMEDIATE rhel11gr2rac1 FAILED OVER
ora.scan1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan2.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan3.vip
1 ONLINE ONLINE rhel11gr2rac1
Moving the OCR to another diskgroup.
--
Scenario - Now the OCR/voting is in the diskgroup OCR_VOTING with normal redundancy, we will
delete the disk so that this disk group cannot be mounted, then we will create an diskgroup
with same name with external redundancy and restore the ocr and voting disk
to that diskgroup.
Shutdown the crs on both the nodes.
Node 1
# crsctl stop crs
Node 2
# crsctl stop crs
Below are the two disks which are assinged to current ocr_voting diskgroup.
NOTE: Assigning number (4,1) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_2)
NOTE: Assigning number (4,0) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_1)
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK
OCR_VOT_DISK_1
OCR_VOT_DISK_2
You have new mail in /var/spool/mail/root
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_2
Removing ASM disk "OCR_VOT_DISK_2": [ OK ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_1
Removing ASM disk "OCR_VOT_DISK_1": [ OK ]
Now start the crs on the node 1 in exclusive mode without starting the crsd.
[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE INTERMEDIATE rhel11gr2rac1 OCR not started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 OFFLINE OFFLINE
ora.crsd
1 OFFLINE OFFLINE
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 OFFLINE OFFLINE
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# su - oracle
[oracle@rhel11gr2rac1 ~]$ . oraenv
ORACLE_SID = [oracle] ? +ASM1
The Oracle base remains unchanged with value /u01/app/oracle
[oracle@rhel11gr2rac1 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.3.0 Production on Sat Oct 26 09:44:14 2013
Copyright (c) 1982, 2011, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> col path form a40;
SQL> select path, header_status from V$asm_disk;
SQL> select path, header_status from V$asm_disk;
PATH HEADER_STATU
---------------------------------------- ------------
/dev/oracleasm/disks/OCR_VOTE_DISK PROVISIONED
/dev/oracleasm/disks/ACFSDISK MEMBER
/dev/oracleasm/disks/FRA2 MEMBER
/dev/oracleasm/disks/FRA1 MEMBER
/dev/oracleasm/disks/DATA2 MEMBER
/dev/oracleasm/disks/DATA1 MEMBER
6 rows selected.
SQL> ! cat /etc/oracle/ocr.loc
ocrconfig_loc=+OCR_VOTING
local_only=FALSE
SQL> create diskgroup OCR_VOTING EXTERNAL REDUNDANCY disk '/dev/oracleasm/disks/OCR_VOTE_DISK' attribute 'compatible.asm' = '11.2';
Diskgroup created.
Restore from backup.
[root@rhel11gr2rac1 bin]# ./ocrconfig -import /u01/app/oracle/ocr.bkp_23oct
[root@rhel11gr2rac1 bin]# ./ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 3
Total space (kbytes) : 262120
Used space (kbytes) : 3520
Available space (kbytes) : 258600
ID : 1241840421
Device/File Name : +OCR_VOTING
Device/File integrity check succeeded
Device/File not configured
Device/File not configured
Device/File not configured
Device/File not configured
Cluster registry integrity check succeeded
Logical corruption check succeeded
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rhel11gr2rac1 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 OFFLINE OFFLINE
ora.crsd
1 OFFLINE OFFLINE
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 OFFLINE OFFLINE
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 73095be13c524f62bf76a7ac90537b12 (/dev/oracleasm/disks/OCR_VOT_DISK_1) [OCR_VOTING]
2. OFFLINE 884ae51c216e4f0ebf368f1ca97e7120 () []
Located 2 voting disk(s).
From the above we could see that still the CSSD process is showing online cause still the voting disk
can be read from the old device /dev/oracleasm/disks/OCR_VOT_DISK_1
But we could see that the disk OCR_VOT_DISK_1 is already deleted (we have deleted the logical name but physically still the
device /dev/sd* is still accessable.
NoW restore the voting file.
Stop the crs and start again in exclusive mode.
[root@rhel11gr2rac1 bin]# ./crsctl stop crs
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded
[root@rhel11gr2rac1 bin]# ./crsctl replace votedisk +OCR_VOTING
Successful addition of voting disk 2fa636c2d1a74f11bffc43049f3fefc4.
Successfully replaced voting disk group with +OCR_VOTING.
CRS-4266: Voting file(s) successfully replaced
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 2fa636c2d1a74f11bffc43049f3fefc4 (ORCL:OCR_VOTE_DISK) [OCR_VOTING]
Located 1 voting disk(s).
Now stop and restart the crs on all the nodes.
# crsctl start crs
[root@rhel11gr2rac2 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.DATA.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.FRA.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.LISTENER.lsnr
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.OCR_VOTING.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
ONLINE ONLINE rhel11gr2rac1 mounted on /golden_
gate
ONLINE ONLINE rhel11gr2rac2 mounted on /golden_
gate
ora.asm
ONLINE ONLINE rhel11gr2rac1 Started
ONLINE ONLINE rhel11gr2rac2 Started
ora.gsd
OFFLINE OFFLINE rhel11gr2rac1
OFFLINE OFFLINE rhel11gr2rac2
ora.net1.network
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.ons
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.registry.acfs
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
1 ONLINE ONLINE rhel11gr2rac2
goldengate-vip
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.cvu
1 ONLINE ONLINE rhel11gr2rac1
ora.ggdb.db
1 OFFLINE OFFLINE
ora.layatest.db
1 ONLINE ONLINE rhel11gr2rac1 Open
2 ONLINE ONLINE rhel11gr2rac2 Open
ora.oc4j
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac2.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan1.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan2.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan3.vip
1 ONLINE ONLINE rhel11gr2rac1
Notes:-
1. We need to create the ASM diskgroup with same name to restore the OCR/VOTING disk in case of failure.
2. The compatiable.asm attribute should be 11.2
-----------------------
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK
Device "/dev/sdi1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdi2" is marked an ASM disk with the label "DATA2"
Device "/dev/sdj" is not marked as an ASM disk
Device "/dev/sdj1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdj2" is marked an ASM disk with the label "DATA2"
Node 2
SQL> alter diskgroup data dismount force;
Diskgroup altered.
SQL> select name, state from V$asm_diskgroup;
NAME STATE
------------------------------ -----------
OCR_VOTING MOUNTED
ACFDG MOUNTED
DATA MOUNTED
FRA MOUNTED
SQL> alter diskgroup data dismount;
Diskgroup altered.
SQL> select name, state from V$asm_diskgroup;
NAME STATE
------------------------------ -----------
OCR_VOTING MOUNTED
ACFDG MOUNTED
DATA DISMOUNTED
FRA MOUNTED
Node1
SQL> alter diskgroup data dismount;
alter diskgroup data dismount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15027: active use of diskgroup "DATA" precludes its dismount
SQL> alter diskgroup data dismount force;
Diskgroup altered.
SQL> ! ps -ef | grep pmon
oracle 5948 1 0 21:46 ? 00:00:00 asm_pmon_+ASM1
oracle 8256 8178 0 22:05 pts/1 00:00:00 /bin/bash -c ps -ef | grep pmon
SQL> select name, state from V$asm_diskgroup;
NAME STATE
------------------------------ -----------
OCR_VOTING MOUNTED
ACFDG MOUNTED
DATA MOUNTED
FRA MOUNTED
SQL> ! ps -ef | grep pmon
oracle 5948 1 0 21:46 ? 00:00:00 asm_pmon_+ASM1
oracle 8347 8178 0 22:07 pts/1 00:00:00 /bin/bash -c ps -ef | grep pmon
SQL> alter diskgroup data dismount;
Diskgroup altered.
[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm deletedisk DATA1
Removing ASM disk "DATA1": [ OK ]
SQL> alter diskgroup DAta MOUNT;
alter diskgroup DAta MOUNT
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "0" is missing from group number "4"
SQL> alter diskgroup data mount force;
Diskgroup altered.
SQL> select path, name, header_status from V$asm_disk;
PATH NAME HEADER_STATU
---------------------------------------- ------------------------------ ------------
ORCL:DATA1 CANDIDATE
DATA_0000 UNKNOWN
ORCL:ACFSDISK ACFSDISK MEMBER
ORCL:DATA2 DATA_0001 MEMBER
ORCL:FRA1 FRA_0000 MEMBER
ORCL:FRA2 FRA_0001 MEMBER
ORCL:OCR_VOTE_DISK OCR_VOTING_0000 MEMBER
Now we could see that for the deleted disk we are getting two entries ... one shows as candidate and another as unknown.
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA2
FRA1
FRA2
OCR_VOTE_DISK
[root@rhel11gr2rac1 bin]# ls -lrt /dev/oracleasm/disks/*
brw-rw---- 1 oracle dba 8, 97 Oct 30 21:43 /dev/oracleasm/disks/OCR_VOTE_DISK
brw-rw---- 1 oracle dba 8, 130 Oct 30 21:43 /dev/oracleasm/disks/DATA2
brw-rw---- 1 oracle dba 8, 161 Oct 30 21:43 /dev/oracleasm/disks/FRA1
brw-rw---- 1 oracle dba 8, 162 Oct 30 21:43 /dev/oracleasm/disks/FRA2
brw-rw---- 1 oracle dba 8, 193 Oct 30 21:43 /dev/oracleasm/disks/ACFSDISK
Using the kfed utility we can read the header of the disks, here the deleted disk DATA1 is /dev/sdi1, in the kfbh.type we
could see that it is showing diskhead, which is not corrupted.
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1
kfbh.endian: 1 ; 0x000: 0x01
kfbh.hard: 130 ; 0x001: 0x82
kfbh.type: 1 ; 0x002: KFBTYP_DISKHEAD
kfbh.datfmt: 1 ; 0x003: 0x01
kfbh.block.blk: 0 ; 0x004: blk=0
kfbh.block.obj: 2147483648 ; 0x008: disk=0
kfbh.check: 804703913 ; 0x00c: 0x2ff6cea9
kfbh.fcn.base: 7030 ; 0x010: 0x00001b76
kfbh.fcn.wrap: 0 ; 0x014: 0x00000000
kfbh.spare1: 0 ; 0x018: 0x00000000
kfbh.spare2: 0 ; 0x01c: 0x00000000
kfdhdb.driver.provstr: ORCLCLRD ; 0x000: length=8
kfdhdb.driver.reserved[0]: 0 ; 0x008: 0x00000000
kfdhdb.driver.reserved[1]: 0 ; 0x00c: 0x00000000
kfdhdb.driver.reserved[2]: 0 ; 0x010: 0x00000000
kfdhdb.driver.reserved[3]: 0 ; 0x014: 0x00000000
kfdhdb.driver.reserved[4]: 0 ; 0x018: 0x00000000
kfdhdb.driver.reserved[5]: 0 ; 0x01c: 0x00000000
kfdhdb.compat: 186646528 ; 0x020: 0x0b200000
kfdhdb.dsknum: 0 ; 0x024: 0x0000
kfdhdb.grptyp: 2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts: 3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname: DATA_0000 ; 0x028: length=9
kfdhdb.grpname: DATA ; 0x048: length=4
kfdhdb.fgname: DATA_0000 ; 0x068: length=9
kfdhdb.capname: ; 0x088: length=0
kfdhdb.crestmp.hi: 32991850 ; 0x0a8: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.crestmp.lo: 3718423552 ; 0x0ac: USEC=0x0 MSEC=0xa9 SECS=0x1a MINS=0x37
kfdhdb.mntstmp.hi: 32992197 ; 0x0b0: HOUR=0x5 DAYS=0x1e MNTH=0xa YEAR=0x7dd
kfdhdb.mntstmp.lo: 3175588864 ; 0x0b4: USEC=0x0 MSEC=0x1e9 SECS=0x14 MINS=0x2f
kfdhdb.secsize: 512 ; 0x0b8: 0x0200
kfdhdb.blksize: 4096 ; 0x0ba: 0x1000
kfdhdb.ausize: 1048576 ; 0x0bc: 0x00100000
kfdhdb.mfact: 113792 ; 0x0c0: 0x0001bc80
kfdhdb.dsksize: 1910 ; 0x0c4: 0x00000776
kfdhdb.pmcnt: 2 ; 0x0c8: 0x00000002
kfdhdb.fstlocn: 1 ; 0x0cc: 0x00000001
kfdhdb.altlocn: 2 ; 0x0d0: 0x00000002
kfdhdb.f1b1locn: 2 ; 0x0d4: 0x00000002
kfdhdb.redomirrors[0]: 0 ; 0x0d8: 0x0000
kfdhdb.redomirrors[1]: 1 ; 0x0da: 0x0001
kfdhdb.redomirrors[2]: 65535 ; 0x0dc: 0xffff
kfdhdb.redomirrors[3]: 65535 ; 0x0de: 0xffff
kfdhdb.dbcompat: 168820736 ; 0x0e0: 0x0a100000
kfdhdb.grpstmp.hi: 32991850 ; 0x0e4: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.grpstmp.lo: 3718281216 ; 0x0e8: USEC=0x0 MSEC=0x1e SECS=0x1a MINS=0x37
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk DATA1 /dev/sdi1
Marking disk "DATA1" as an ASM disk: [ OK ]
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1
kfbh.endian: 0 ; 0x000: 0x00
kfbh.hard: 0 ; 0x001: 0x00
kfbh.type: 0 ; 0x002: KFBTYP_INVALID
kfbh.datfmt: 0 ; 0x003: 0x00
kfbh.block.blk: 0 ; 0x004: blk=0
kfbh.block.obj: 0 ; 0x008: file=0
kfbh.check: 1856147420 ; 0x00c: 0x6ea28fdc
kfbh.fcn.base: 0 ; 0x010: 0x00000000
kfbh.fcn.wrap: 0 ; 0x014: 0x00000000
kfbh.spare1: 0 ; 0x018: 0x00000000
kfbh.spare2: 0 ; 0x01c: 0x00000000
2B1E9B240400 00000000 00000000 00000000 6EA28FDC [...............n]
2B1E9B240410 00000000 00000000 00000000 00000000 [................]
2B1E9B240420 4C43524F 4B534944 41544144 00000031 [ORCLDISKDATA1...]
2B1E9B240430 00000000 00000000 00000000 00000000 [................]
Repeat 252 times
KFED-00322: file not found; arguments: [kfbtTraverseBlock] [Invalid OSM block type] [] [0]
now the disk header is corrupted.
Now delete the disk
[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm deletedisk DATA1
Removing ASM disk "DATA1": [ OK ]
We havent taken the backup of the header before creating the disk again, but still
ASM contains the copy of ASM disk header in (Allocation unit# 1, Block# 254) of every ASM disk.
Let view the content of it.
[root@rhel11gr2rac1 ~]# kfed read /dev/sdi1 aun=1 blkn=256 | grep KFB
kfbh.type: 5 ; 0x002: KFBTYP_LISTHEAD
Its is showing as listhead and not the invalid. We can repair the header using the
kfed repair option.
[root@rhel11gr2rac1 ~]# kfed repair /dev/sdi1
Now lets check the header again.
[root@rhel11gr2rac1 ~]# /grid/11.2/bin/kfed read /dev/sdi1
kfbh.endian: 1 ; 0x000: 0x01
kfbh.hard: 130 ; 0x001: 0x82
kfbh.type: 1 ; 0x002: KFBTYP_DISKHEAD
kfbh.datfmt: 1 ; 0x003: 0x01
kfbh.block.blk: 0 ; 0x004: blk=0
kfbh.block.obj: 2147483648 ; 0x008: disk=0
kfbh.check: 804703913 ; 0x00c: 0x2ff6cea9
kfbh.fcn.base: 7030 ; 0x010: 0x00001b76
kfbh.fcn.wrap: 0 ; 0x014: 0x00000000
kfbh.spare1: 0 ; 0x018: 0x00000000
kfbh.spare2: 0 ; 0x01c: 0x00000000
kfdhdb.driver.provstr: ORCLDISKDATA1 ; 0x000: length=13
kfdhdb.driver.reserved[0]: 1096040772 ; 0x008: 0x41544144
kfdhdb.driver.reserved[1]: 49 ; 0x00c: 0x00000031
kfdhdb.driver.reserved[2]: 0 ; 0x010: 0x00000000
kfdhdb.driver.reserved[3]: 0 ; 0x014: 0x00000000
kfdhdb.driver.reserved[4]: 0 ; 0x018: 0x00000000
kfdhdb.driver.reserved[5]: 0 ; 0x01c: 0x00000000
kfdhdb.compat: 186646528 ; 0x020: 0x0b200000
kfdhdb.dsknum: 0 ; 0x024: 0x0000
kfdhdb.grptyp: 2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts: 3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname: DATA_0000 ; 0x028: length=9
kfdhdb.grpname: DATA ; 0x048: length=4
kfdhdb.fgname: DATA_0000 ; 0x068: length=9
kfdhdb.capname: ; 0x088: length=0
kfdhdb.crestmp.hi: 32991850 ; 0x0a8: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.crestmp.lo: 3718423552 ; 0x0ac: USEC=0x0 MSEC=0xa9 SECS=0x1a MINS=0x37
kfdhdb.mntstmp.hi: 32992197 ; 0x0b0: HOUR=0x5 DAYS=0x1e MNTH=0xa YEAR=0x7dd
kfdhdb.mntstmp.lo: 3175588864 ; 0x0b4: USEC=0x0 MSEC=0x1e9 SECS=0x14 MINS=0x2f
kfdhdb.secsize: 512 ; 0x0b8: 0x0200
kfdhdb.blksize: 4096 ; 0x0ba: 0x1000
kfdhdb.ausize: 1048576 ; 0x0bc: 0x00100000
kfdhdb.mfact: 113792 ; 0x0c0: 0x0001bc80
kfdhdb.dsksize: 1910 ; 0x0c4: 0x00000776
kfdhdb.pmcnt: 2 ; 0x0c8: 0x00000002
kfdhdb.fstlocn: 1 ; 0x0cc: 0x00000001
kfdhdb.altlocn: 2 ; 0x0d0: 0x00000002
kfdhdb.f1b1locn: 2 ; 0x0d4: 0x00000002
kfdhdb.redomirrors[0]: 0 ; 0x0d8: 0x0000
kfdhdb.redomirrors[1]: 1 ; 0x0da: 0x0001
kfdhdb.redomirrors[2]: 65535 ; 0x0dc: 0xffff
kfdhdb.redomirrors[3]: 65535 ; 0x0de: 0xffff
kfdhdb.dbcompat: 168820736 ; 0x0e0: 0x0a100000
kfdhdb.grpstmp.hi: 32991850 ; 0x0e4: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.grpstmp.lo: 3718281216 ; 0x0e8: USEC=0x0 MSEC=0x1e SECS=0x1a MINS=0x37
kfdhdb.vfstart: 0 ; 0x0ec: 0x00000000
kfdhdb.vfend: 0 ; 0x0f0: 0x00000000
kfdhdb.spfile: 0 ; 0x0f4: 0x00000000
kfdhdb.spfflg: 0 ; 0x0f8: 0x00000000
kfdhdb.ub4spare[0]: 0 ; 0x0fc: 0x00000000
kfdhdb.ub4spare[1]: 0 ; 0x100: 0x00000000
kfdhdb.ub4spare[2]: 0 ; 0x104: 0x00000000
kfdhdb.ub4spare[3]: 0 ; 0x108: 0x00000000
kfdhdb.ub4spare[4]: 0 ; 0x10c: 0x00000000
kfdhdb.ub4spare[5]: 0 ; 0x110: 0x00000000
kfdhdb.ub4spare[6]: 0 ; 0x114: 0x00000000
kfdhdb.ub4spare[7]: 0 ; 0x118: 0x00000000
kfdhdb.ub4spare[8]: 0 ; 0x11c: 0x00000000
kfdhdb.ub4spare[9]: 0 ; 0x120: 0x00000000
kfdhdb.ub4spare[10]: 0 ; 0x124: 0x00000000
kfdhdb.ub4spare[11]: 0 ; 0x128: 0x00000000
kfdhdb.ub4spare[12]: 0 ; 0x12c: 0x00000000
kfdhdb.ub4spare[13]: 0 ; 0x130: 0x00000000
kfdhdb.ub4spare[14]: 0 ; 0x134: 0x00000000
kfdhdb.ub4spare[15]: 0 ; 0x138: 0x00000000
kfdhdb.ub4spare[16]: 0 ; 0x13c: 0x00000000
kfdhdb.ub4spare[17]: 0 ; 0x140: 0x00000000
kfdhdb.ub4spare[18]: 0 ; 0x144: 0x00000000
kfdhdb.ub4spare[19]: 0 ; 0x148: 0x00000000
kfdhdb.ub4spare[20]: 0 ; 0x14c: 0x00000000
kfdhdb.ub4spare[21]: 0 ; 0x150: 0x00000000
kfdhdb.ub4spare[22]: 0 ; 0x154: 0x00000000
kfdhdb.ub4spare[23]: 0 ; 0x158: 0x00000000
kfdhdb.ub4spare[24]: 0 ; 0x15c: 0x00000000
kfdhdb.ub4spare[25]: 0 ; 0x160: 0x00000000
kfdhdb.ub4spare[26]: 0 ; 0x164: 0x00000000
kfdhdb.ub4spare[27]: 0 ; 0x168: 0x00000000
kfdhdb.ub4spare[28]: 0 ; 0x16c: 0x00000000
kfdhdb.ub4spare[29]: 0 ; 0x170: 0x00000000
kfdhdb.ub4spare[30]: 0 ; 0x174: 0x00000000
kfdhdb.ub4spare[31]: 0 ; 0x178: 0x00000000
kfdhdb.ub4spare[32]: 0 ; 0x17c: 0x00000000
kfdhdb.ub4spare[33]: 0 ; 0x180: 0x00000000
kfdhdb.ub4spare[34]: 0 ; 0x184: 0x00000000
kfdhdb.ub4spare[35]: 0 ; 0x188: 0x00000000
kfdhdb.ub4spare[36]: 0 ; 0x18c: 0x00000000
kfdhdb.ub4spare[37]: 0 ; 0x190: 0x00000000
kfdhdb.ub4spare[38]: 0 ; 0x194: 0x00000000
kfdhdb.ub4spare[39]: 0 ; 0x198: 0x00000000
kfdhdb.ub4spare[40]: 0 ; 0x19c: 0x00000000
kfdhdb.ub4spare[41]: 0 ; 0x1a0: 0x00000000
kfdhdb.ub4spare[42]: 0 ; 0x1a4: 0x00000000
kfdhdb.ub4spare[43]: 0 ; 0x1a8: 0x00000000
kfdhdb.ub4spare[44]: 0 ; 0x1ac: 0x00000000
kfdhdb.ub4spare[45]: 0 ; 0x1b0: 0x00000000
kfdhdb.ub4spare[46]: 0 ; 0x1b4: 0x00000000
kfdhdb.ub4spare[47]: 0 ; 0x1b8: 0x00000000
kfdhdb.ub4spare[48]: 0 ; 0x1bc: 0x00000000
kfdhdb.ub4spare[49]: 0 ; 0x1c0: 0x00000000
kfdhdb.ub4spare[50]: 0 ; 0x1c4: 0x00000000
kfdhdb.ub4spare[51]: 0 ; 0x1c8: 0x00000000
kfdhdb.ub4spare[52]: 0 ; 0x1cc: 0x00000000
kfdhdb.ub4spare[53]: 0 ; 0x1d0: 0x00000000
kfdhdb.acdb.aba.seq: 0 ; 0x1d4: 0x00000000
kfdhdb.acdb.aba.blk: 0 ; 0x1d8: 0x00000000
kfdhdb.acdb.ents: 0 ; 0x1dc: 0x0000
kfdhdb.acdb.ub2spare: 0 ; 0x1de: 0x0000
Its header has been restored properly.
Main things to see..
kfbh.type: 1 ; 0x002: KFBTYP_DISKHEAD
kfdhdb.driver.provstr: ORCLDISKDATA1 ; 0x000: length=13
kfdhdb.grptyp: 2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts: 3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname: DATA_0000 ; 0x028: length=9
kfdhdb.grpname: DATA ; 0x048: length=4
kfdhdb.fgname: DATA_0000 ; 0x068: length=9
KFBTYP_DISKHEAD - Heade is valid.
ORCLDISKDATA1 - This is the disk name which been created using oracleasm [DATA1]
DATA_0000 - Disk name and fail group name in asm instance.
DATA - Is the Diskgroup name.
Now we have the proper header details on the device. Now do the scandisk to list out this disk.
---------------------------------------------------------------------------------------------------------------
Note ---- If kfdhdb.driver.provstr: shows only ORCLDISK and not the full disk name i.e. ORCLDISKDATA1 then
you can run the below force-rename to rename the disk.. in our case it is listing properly.
[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm force-renamedisk /dev/sdi1 DATA1
---------------------------------------------------------------------------------------------------------------
[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm scandisks
Scanning the system for Oracle ASMLib disks: [ OK ]
[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK
Now the DATA1 disk is getting visible.
Below from the asm instance.
SQL> select path, name, state, failgroup, mode_status, total_mb, label , header_status from V$asm_disk;
PATH NAME STATE FAILGROUP MODE_ST TOTAL_MB LABEL HEADER_STATU
-------------------- ------------------------------ -------- ------------------------------ ------- ---------- ------------------------------- ------------
ORCL:DATA1 NORMAL ONLINE 0 DATA1 MEMBER
DATA_0000 NORMAL DATA_0000 OFFLINE 0 UNKNOWN
ORCL:ACFSDISK ACFSDISK NORMAL ACFSDISK ONLINE 9759 ACFSDISK MEMBER
ORCL:DATA2 DATA_0001 NORMAL DATA_0001 ONLINE 1911 DATA2 MEMBER
ORCL:FRA1 FRA_0000 NORMAL FRA_0000 ONLINE 955 FRA1 MEMBER
ORCL:FRA2 FRA_0001 NORMAL FRA_0001 ONLINE 921 FRA2 MEMBER
ORCL:OCR_VOTE_DISK OCR_VOTING_0000 NORMAL OCR_VOTING_0000 ONLINE 959 OCR_VOTE_DISK MEMBER
Now add this disk to the diskgroup data using force option.
Note- To add the existing disk which has been droped we can use the force option, but not for adding a new disk.
SQL> alter diskgroup DATA add disk 'ORCL:DATA1' force;
Diskgroup altered.
SQL> select path, name, state, failgroup, mode_status, total_mb, label , header_status from V$asm_disk;
PATH NAME STATE FAILGROUP MODE_ST TOTAL_MB LABEL HEADER_STATU
-------------------- ------------------------------ -------- ------------------------------ ------- ---------- ------------------------------- ------------
DATA_0000 NORMAL DATA_0000 OFFLINE 0 UNKNOWN
ORCL:ACFSDISK ACFSDISK NORMAL ACFSDISK ONLINE 9759 ACFSDISK MEMBER
ORCL:DATA1 DATA1 NORMAL DATA1 ONLINE 1910 DATA1 MEMBER
ORCL:DATA2 DATA_0001 NORMAL DATA_0001 ONLINE 1911 DATA2 MEMBER
ORCL:FRA1 FRA_0000 NORMAL FRA_0000 ONLINE 955 FRA1 MEMBER
ORCL:FRA2 FRA_0001 NORMAL FRA_0001 ONLINE 921 FRA2 MEMBER
ORCL:OCR_VOTE_DISK OCR_VOTING_0000 NORMAL OCR_VOTING_0000 ONLINE 959 OCR_VOTE_DISK MEMBER
7 rows selected.
SQL> alter diskgroup data drop disk 'ORCL:DATA1';
alter diskgroup data drop disk 'ORCL:DATA1'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15054: disk "ORCL:DATA1" does not exist in diskgroup "DATA"
SQL> alter diskgroup data drop disk 'DATA1';
Diskgroup altered.
Data 1 is showing as member.
Using this data we can create the oraleasm disk again.
From the asm instance lets check what are the disks are listing.
SQL> select path, name, header_status from V$asm_disk;
PATH NAME HEADER_STATU
---------------------------------------- ------------------------------ ------------
DATA_0000 UNKNOWN
ORCL:ACFSDISK ACFSDISK MEMBER
ORCL:DATA2 DATA_0001 MEMBER
ORCL:FRA1 FRA_0000 MEMBER
ORCL:FRA2 FRA_0001 MEMBER
ORCL:OCR_VOTE_DISK OCR_VOTING_0000 MEMBER
--- Here i have changed the asm_diskstring from default value null to "ORCL:*" and once this is done
my asm instance keep rebooting for every one sec...
-- Here just stop the instance and recreate the spfile from a pfile with modified parameter values and restart the instance.
http://www.usn-it.de/index.php/2011/07/12/oracle-grid-infrastructure-how-to-recover-from-a-messed-up-asmcssd-diskstring/
evice "/dev/sdi1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdi2" is marked an ASM disk with the label "DATA2"
Device "/dev/sdj" is not marked as an ASM disk
Device "/dev/sdj1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdj2" is marked an ASM disk with the label "DATA2"
Device "/dev/sdm1" is marked an ASM disk with the label "ACFSDISK"
Test scenario.
---
OCR_VOTING DISK is external redundancy.
[oracle@rhel11gr2rac1 bin]$ ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.DATA.dg
ONLINE OFFLINE rhel11gr2rac1
ONLINE OFFLINE rhel11gr2rac2
ora.FRA.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.LISTENER.lsnr
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.OCR_VOTING.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
ONLINE ONLINE rhel11gr2rac1 mounted on /golden_
gate
OFFLINE OFFLINE rhel11gr2rac2 volume /golden_gate
offline
ora.asm
ONLINE ONLINE rhel11gr2rac1 Started
ONLINE ONLINE rhel11gr2rac2 Started
ora.gsd
OFFLINE OFFLINE rhel11gr2rac1
OFFLINE OFFLINE rhel11gr2rac2
ora.net1.network
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.ons
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.registry.acfs
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
1 ONLINE UNKNOWN rhel11gr2rac2
goldengate-vip
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.cvu
1 ONLINE ONLINE rhel11gr2rac1
ora.ggdb.db
1 OFFLINE OFFLINE
ora.layatest.db
1 ONLINE OFFLINE Instance Shutdown
2 ONLINE OFFLINE
ora.oc4j
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac2.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan1.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan2.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan3.vip
1 ONLINE ONLINE rhel11gr2rac1
[oracle@rhel11gr2rac1 bin]$ ./ocrconfig -showbackup
rhel11gr2rac2 2013/10/01 22:56:35 /grid/11.2/cdata/urac1rac2/backup00.ocr
rhel11gr2rac1 2013/07/13 21:00:51 /grid/11.2/cdata/urac1rac2/backup01.ocr
rhel11gr2rac2 2013/10/01 22:56:35 /grid/11.2/cdata/urac1rac2/day.ocr
rhel11gr2rac2 2013/10/01 22:56:35 /grid/11.2/cdata/urac1rac2/week.ocr
PROT-25: Manual backups for the Oracle Cluster Registry are not available
[oracle@rhel11gr2rac1 bin]$ ./crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 7c8085fce8f24fb3bff9beb4ff52deb3 (ORCL:OCT_VOTE_DISK_1) [OCR_VOTING]
Located 1 voting disk(s).
Node 1
gpnptool output.
[oracle@rhel11gr2rac1 bin]$ ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
./gpnptool.bin get -o-
Success.
Node 2
gpnptool output.
[root@rhel11gr2rac2 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
./gpnptool.bin get -o-
Success.
SQL> select name, state , type from V$asm_diskgroup;
NAME STATE TYPE
------------------------------ ----------- ------
ACFDG MOUNTED EXTERN
FRA MOUNTED NORMAL
OCR_VOTING MOUNTED EXTERN
SQL> select name, group_number from V$asm_diskgroup;
NAME GROUP_NUMBER
------------------------------ ------------
ACFDG 1
FRA 3
OCR_VOTING 4
SQL> col path form a40;
SQL> select path, header_status, group_number from V$asm_disk;
PATH HEADER_STATU GROUP_NUMBER
---------------------------------------- ------------ ------------
ORCL:ACFSDISK MEMBER 1
ORCL:FRA1 MEMBER 3
ORCL:FRA2 MEMBER 3
ORCL:OCT_VOTE_DISK_1 MEMBER 4
/dev/sdg1 -- OCT_VOTE_DISK_1
-- Stopped crs on both the nodes.
sde1
kfed read /dev/sde1 | grep kfdhdb.dskname
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdc1 | grep kfdhdb.dskname
kfdhdb.dskname: OCR_VOTING_0000 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdd1 | grep kfdhdb.dskname
kfdhdb.dskname: OCR_VOTING_0000 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sde1 | grep kfdhdb.dskname
kfdhdb.dskname: OCR_VOTING_0001 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdf1 | grep kfdhdb.dskname
kfdhdb.dskname: OCR_VOTING_0001 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdg1 | grep kfdhdb.dskname
kfdhdb.dskname: OCT_VOTE_DISK_1 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdh1 | grep kfdhdb.dskname
kfdhdb.dskname: OCT_VOTE_DISK_1 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1 | grep kfdhdb.dskname
kfdhdb.dskname: DATA1 ; 0x028: length=5
Device "/dev/sdk1" is marked an ASM disk with the label "FRA1"
Device "/dev/sdk2" is marked an ASM disk with the label "FRA2"
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk FRA1
Removing ASM disk "FRA1": [ OK ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk FRA2
Removing ASM disk "FRA2": [ OK ]
create new ocr disk using /dev/sdc1 /dev/sde1 /dev/sdk1
format the above files...
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_1 /dev/sdc1
Marking disk "OCR_VOTING_DISK_1" as an ASM disk: [ OK ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_2 /dev/sde1
Marking disk "OCR_VOTING_DISK_2" as an ASM disk: [ OK ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_3 /dev/sdk1
Marking disk "OCR_VOTING_DISK_3" as an ASM disk: [ OK ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
OCR_VOTING_DISK_1
OCR_VOTING_DISK_2
OCR_VOTING_DISK_3
[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE INTERMEDIATE rhel11gr2rac1 OCR not started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 OFFLINE OFFLINE
ora.crsd
1 OFFLINE OFFLINE
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 OFFLINE OFFLINE
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
Located 0 voting disk(s).
SQL> select name , state from V$asm_diskgroup;
NAME STATE
------------------------------ -----------
ACFDG DISMOUNTED
SQL> select path, header_status, name from V$asm_disk;
PATH HEADER_STATU NAME
---------------------------------------- ------------ ------------------------------
ORCL:ACFSDISK MEMBER
ORCL:OCR_VOTING_DISK_3 PROVISIONED
ORCL:OCR_VOTING_DISK_2 PROVISIONED
ORCL:OCR_VOTING_DISK_1 PROVISIONED
SQL> create diskgroup OCR_VOTING normal redundancy disk 'ORCL:OCR_VOTING_DISK_1','ORCL:OCR_VOTING_DISK_2','ORCL:OCR_VOTING_DISK_3'
2 attribute 'compatible.asm' = '11.2';
Diskgroup created.
SQL> select path, header_status, name from V$asm_disk;
PATH HEADER_STATU NAME
---------------------------------------- ------------ ------------------------------
ORCL:ACFSDISK MEMBER
ORCL:OCR_VOTING_DISK_1 MEMBER OCR_VOTING_DISK_1
ORCL:OCR_VOTING_DISK_2 MEMBER OCR_VOTING_DISK_2
ORCL:OCR_VOTING_DISK_3 MEMBER OCR_VOTING_DISK_3
[root@rhel11gr2rac1 bin]# ./ocrconfig -restore /grid/11.2/cdata/urac1rac2/backup00.ocr
[root@rhel11gr2rac1 bin]# ./ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 3
Total space (kbytes) : 262120
Used space (kbytes) : 3520
Available space (kbytes) : 258600
ID : 845960811
Device/File Name : +OCR_VOTING
Device/File integrity check succeeded
Device/File not configured
Device/File not configured
Device/File not configured
Device/File not configured
Cluster registry integrity check succeeded
Logical corruption check succeeded
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rhel11gr2rac1 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE rhel11gr2rac1
ora.crf
1 OFFLINE OFFLINE
ora.crsd
1 OFFLINE OFFLINE
ora.cssd
1 ONLINE ONLINE rhel11gr2rac1
ora.cssdmonitor
1 ONLINE ONLINE rhel11gr2rac1
ora.ctssd
1 ONLINE ONLINE rhel11gr2rac1 ACTIVE:0
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE rhel11gr2rac1
ora.evmd
1 OFFLINE OFFLINE
ora.gipcd
1 ONLINE ONLINE rhel11gr2rac1
ora.gpnpd
1 ONLINE ONLINE rhel11gr2rac1
ora.mdnsd
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
Located 0 voting disk(s).
[root@rhel11gr2rac1 bin]# ./crsctl replace votedisk +OCR_VOTING
Successful addition of voting disk 9e87f77bd3064f87bf36380306d013ed.
Successful addition of voting disk b0de29805c974f12bffbf5f3a2564e99.
Successful addition of voting disk 007d0a5205bd4f0ebf97a2098200ba01.
Successfully replaced voting disk group with +OCR_VOTING.
CRS-4266: Voting file(s) successfully replaced
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 9e87f77bd3064f87bf36380306d013ed (ORCL:OCR_VOTING_DISK_1) [OCR_VOTING]
2. ONLINE b0de29805c974f12bffbf5f3a2564e99 (ORCL:OCR_VOTING_DISK_2) [OCR_VOTING]
3. ONLINE 007d0a5205bd4f0ebf97a2098200ba01 (ORCL:OCR_VOTING_DISK_3) [OCR_VOTING]
Located 3 voting disk(s).
[root@rhel11gr2rac1 bin]# ./crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
FRA1
FRA2
OCT_VOTE_DISK_1
You have new mail in /var/spool/mail/root
[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm scandisks
Scanning the system for Oracle ASMLib disks: [ OK ]
[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
OCR_VOTING_DISK_1
OCR_VOTING_DISK_2
OCR_VOTING_DISK_3
Now start the crs on both the nodes.
[root@rhel11gr2rac1 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@rhel11gr2rac2 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.DATA.dg
ONLINE OFFLINE rhel11gr2rac1
ONLINE OFFLINE rhel11gr2rac2
ora.FRA.dg
ONLINE OFFLINE rhel11gr2rac1
ONLINE OFFLINE rhel11gr2rac2
ora.LISTENER.lsnr
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.OCR_VOTING.dg
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
ONLINE ONLINE rhel11gr2rac1 mounted on /golden_
gate
ONLINE ONLINE rhel11gr2rac2 mounted on /golden_
gate
ora.asm
ONLINE ONLINE rhel11gr2rac1 Started
ONLINE ONLINE rhel11gr2rac2 Started
ora.gsd
OFFLINE OFFLINE rhel11gr2rac1
OFFLINE OFFLINE rhel11gr2rac2
ora.net1.network
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.ons
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
ora.registry.acfs
ONLINE ONLINE rhel11gr2rac1
ONLINE ONLINE rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
1 ONLINE UNKNOWN rhel11gr2rac1
goldengate-vip
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE rhel11gr2rac1
ora.cvu
1 ONLINE ONLINE rhel11gr2rac1
ora.ggdb.db
1 OFFLINE OFFLINE
ora.layatest.db
1 ONLINE OFFLINE Instance Shutdown
2 ONLINE OFFLINE
ora.oc4j
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac1.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.rhel11gr2rac2.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan1.vip
1 ONLINE ONLINE rhel11gr2rac2
ora.scan2.vip
1 ONLINE ONLINE rhel11gr2rac1
ora.scan3.vip
1 ONLINE ONLINE rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
./gpnptool.bin get -o-
Success.
[root@rhel11gr2rac2 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
./gpnptool.bin get -o-
Success.
No comments:
Post a Comment