Monday, 5 October 2015

Use of OCR in ASM diskgroup with Normal Redundancy

[root@rhel11gr2rac1 bin]# ./crsctl status res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.DATA.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.FRA.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER.lsnr
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.OCR_VOTING.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
               ONLINE  ONLINE       rhel11gr2rac1            mounted on /golden_
                                                             gate
               ONLINE  ONLINE       rhel11gr2rac2            mounted on /golden_
                                                             gate
ora.asm
               ONLINE  ONLINE       rhel11gr2rac1            Started
               ONLINE  ONLINE       rhel11gr2rac2            Started
ora.gsd
               OFFLINE OFFLINE      rhel11gr2rac1
               OFFLINE OFFLINE      rhel11gr2rac2
ora.net1.network
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.ons
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.registry.acfs
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
      1        ONLINE  ONLINE       rhel11gr2rac2
goldengate-vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cvu
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ggdb.db
      1        OFFLINE OFFLINE
ora.layatest.db
      1        ONLINE  ONLINE       rhel11gr2rac1            Open
      2        ONLINE  ONLINE       rhel11gr2rac2            Open
ora.oc4j
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac2.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan1.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan2.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan3.vip
      1        ONLINE  ONLINE       rhel11gr2rac1


--

[root@rhel11gr2rac1 bin]# ./ocrconfig -export /u01/app/oracle/ocr.bkp_23oct

[root@rhel11gr2rac1 bin]# ls -lrt /u01/app/oracle/ocr.bkp_23oct
-rw------- 1 root root 131266 Oct 23 21:26 /u01/app/oracle/ocr.bkp_23oct

Stop crs on both nodes

Node 1

[root@rhel11gr2rac1 bin]# ./crsctl stop crs

Node 2

[root@rhel11gr2rac2 bin]# ./crsctl stop crs

NAME                             TOTAL_MB USABLE_FILE_MB STATE       TYPE
------------------------------ ---------- -------------- ----------- ------
ACFDG                                9759            581 MOUNTED     EXTERN
DATA                                 3871           1977 CONNECTED   EXTERN
FRA                                  1918           1816 CONNECTED   EXTERN
OCR_VOTING                           2868            436 MOUNTED     NORMAL


NAME                           FAILGROUP                      PATH                                       TOTAL_MB HEADER_STATU REDUNDA
------------------------------ ------------------------------ ---------------------------------------- ---------- ------------ -------
ACFSDISK                       ACFSDISK                       /dev/oracleasm/disks/ACFSDISK                  9759 MEMBER       UNKNOWN
FRA                            FRA                            /dev/oracleasm/disks/FRA                       1918 MEMBER       UNKNOWN
DATA                           DATA                           /dev/oracleasm/disks/DATA                      3871 MEMBER       UNKNOWN
OCR_VOTING_0002                OCR_VOTING_0002                /dev/oracleasm/disks/OCR_VOT_DISK_3             956 MEMBER       UNKNOWN
OCR_VOTING_0001                OCR_VOTING_0001                /dev/oracleasm/disks/OCR_VOT_DISK_2             956 MEMBER       UNKNOWN
OCR_VOTING_0000                OCR_VOTING_0000                /dev/oracleasm/disks/OCR_VOT_DISK_1             956 MEMBER       UNKNOWN


[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOT_DISK_1
OCR_VOT_DISK_2
OCR_VOT_DISK_3


[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm querydisk /dev/sd* | grep "OCR"

Device "/dev/sdc1" is marked an ASM disk with the label "OCR_VOT_DISK_1"

Device "/dev/sdd1" is marked an ASM disk with the label "OCR_VOT_DISK_1"

Device "/dev/sde1" is marked an ASM disk with the label "OCR_VOT_DISK_2"

Device "/dev/sdf1" is marked an ASM disk with the label "OCR_VOT_DISK_2"

Device "/dev/sdg1" is marked an ASM disk with the label "OCR_VOT_DISK_3"

Device "/dev/sdh1" is marked an ASM disk with the label "OCR_VOT_DISK_3"


[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOT_DISK_1
OCR_VOT_DISK_2
OCR_VOT_DISK_3

[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_3
Removing ASM disk "OCR_VOT_DISK_3":                        [  OK  ]
[root@rhel11gr2rac1 disks]# /etc/init.d/oracleasm querydisk /dev/sd*
Device "/dev/sda" is not marked as an ASM disk
Device "/dev/sda1" is not marked as an ASM disk
Device "/dev/sda2" is not marked as an ASM disk
Device "/dev/sda3" is not marked as an ASM disk
Device "/dev/sdb" is not marked as an ASM disk
Device "/dev/sdb1" is not marked as an ASM disk
Device "/dev/sdc" is not marked as an ASM disk
Device "/dev/sdc1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sdd" is not marked as an ASM disk
Device "/dev/sdd1" is marked an ASM disk with the label "OCR_VOT_DISK_1"
Device "/dev/sde" is not marked as an ASM disk
Device "/dev/sde1" is marked an ASM disk with the label "OCR_VOT_DISK_2"
Device "/dev/sdf" is not marked as an ASM disk
Device "/dev/sdf1" is marked an ASM disk with the label "OCR_VOT_DISK_2"


[root@rhel11gr2rac1 bin]# ./crsctl start crs




[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.




[ohasd(10125)]CRS-2807:Resource 'ora.crsd' failed to start automatically.
2013-10-23 05:44:24.116
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted.
Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".
2013-10-23 05:44:56.872
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted. Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".
2013-10-23 05:45:26.926
[/grid/11.2/bin/oraagent.bin(10243)]CRS-5019:All OCR locations are on ASM disk groups [OCR_VOTING], and none of these disk groups are mounted. Details are at "(:CLSN00100:)" in "/grid/11.2/log/rhel11gr2rac1/agent/ohasd/oraagent_oracle/oraagent_oracle.log".


[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  INTERMEDIATE rhel11gr2rac1            OCR not started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crsd
      1        ONLINE  OFFLINE
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        ONLINE  INTERMEDIATE rhel11gr2rac1
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1



[oracle@rhel11gr2rac1 ~]$ export ORACLE_HOME=/grid/11.2
[oracle@rhel11gr2rac1 ~]$ export PATH=$ORACLE_HOME/bin:$PATH
[oracle@rhel11gr2rac1 ~]$ export ORACLE_SID=+ASM1
[oracle@rhel11gr2rac1 ~]$ asmcmd
ASMCMD> lsdg
State    Type    Rebal  Sector  Block       AU  Total_MB  Free_MB  Req_mir_free_MB  Usable_file_MB  Offline_disks  Voting_files  Name
MOUNTED  EXTERN  N         512   4096  1048576      9759      581                0             581              0             N  ACFDG/
MOUNTED  NORMAL  N         512   4096  1048576      3821     1557                0             778              0             N  DATA/
MOUNTED  NORMAL  N         512   4096  1048576      1876     1630                0             815              0             N  FRA/
ASMCMD> exit
[oracle@rhel11gr2rac1 ~]$ sqlplus / as sysasm

SQL*Plus: Release 11.2.0.3.0 Production on Wed Oct 23 21:51:57 2013

Copyright (c) 1982, 2011, Oracle.  All rights reserved.


Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options

SQL> select name, state from V$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
ACFDG                          MOUNTED
DATA                           MOUNTED
FRA                            MOUNTED
OCR_VOTING                     DISMOUNTED


SQL> alter diskgroup ocr_voting mount;
alter diskgroup ocr_voting mount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "2" is missing from group number "4"



SQL> select name, path , failgroup , header_status from V$asm_disk;

NAME                           PATH                                     FAILGROUP                      HEADER_STATU
------------------------------ ---------------------------------------- ------------------------------ ------------
                               /dev/oracleasm/disks/OCR_VOT_DISK_2                                     MEMBER
                               /dev/oracleasm/disks/OCR_VOT_DISK_1                                     MEMBER
ACFSDISK                       /dev/oracleasm/disks/ACFSDISK            ACFSDISK                       MEMBER
FRA_0001                       /dev/oracleasm/disks/FRA2                FRA_0001                       MEMBER
FRA_0000                       /dev/oracleasm/disks/FRA1                FRA_0000                       MEMBER
DATA_0001                      /dev/oracleasm/disks/DATA2               DATA_0001                      MEMBER
DATA_0000                      /dev/oracleasm/disks/DATA1               DATA_0000                      MEMBER


SQL> alter diskgroup ocr_voting mount force;

Diskgroup altered.


--

From Asm instance alert log.

SQL> alter diskgroup ocr_voting mount force
NOTE: cache registered group OCR_VOTING number=4 incarn=0xf1ec9b2a
NOTE: cache began mount (first) of group OCR_VOTING number=4 incarn=0xf1ec9b2a
NOTE: Assigning number (4,1) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_2)
NOTE: Assigning number (4,0) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_1)
Wed Oct 23 05:58:17 2013
NOTE: GMON heartbeating for grp 4
GMON querying group 4 at 25 for pid 23, osid 11366
NOTE: Assigning number (4,2) to disk ()
GMON querying group 4 at 26 for pid 23, osid 11366
NOTE: process _user11366_+asm1 (11366) initiating offline of disk 2.3916196865 () with mask 0x7e in group 4
NOTE: checking PST: grp = 4
GMON checking disk modes for group 4 at 27 for pid 23, osid 11366
NOTE: checking PST for grp 4 done.
WARNING: Disk 2 () in group 4 mode 0x7f is now being offlined
NOTE: initiating PST update: grp = 4, dsk = 2/0xe96c6c01, mask = 0x6a, op = clear
GMON updating disk modes for group 4 at 28 for pid 23, osid 11366
NOTE: group OCR_VOTING: updated PST location: disk 0000 (PST copy 0)
NOTE: group OCR_VOTING: updated PST location: disk 0001 (PST copy 1)
Wed Oct 23 21:58:30 2013
NOTE: PST update grp = 4 completed successfully


------------

[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  ONLINE       rhel11gr2rac1            Started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crsd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1



[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  ONLINE       rhel11gr2rac1            Started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crsd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1
[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
               ONLINE  ONLINE       rhel11gr2rac1
ora.DATA.dg
               ONLINE  ONLINE       rhel11gr2rac1
ora.FRA.dg
               ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER.lsnr
               ONLINE  OFFLINE      rhel11gr2rac1            STARTING
ora.OCR_VOTING.dg
               ONLINE  ONLINE       rhel11gr2rac1
ora.acfdg.acfsvolume.acfs
               ONLINE  OFFLINE      rhel11gr2rac1            volume /golden_gate
                                                             offline,STARTING
ora.asm
               ONLINE  ONLINE       rhel11gr2rac1            Started
ora.gsd
               OFFLINE OFFLINE      rhel11gr2rac1
ora.net1.network
               ONLINE  ONLINE       rhel11gr2rac1
ora.ons
               ONLINE  OFFLINE      rhel11gr2rac1            STARTING
ora.registry.acfs
               ONLINE  ONLINE       rhel11gr2rac1
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
      1        ONLINE  UNKNOWN      rhel11gr2rac1
goldengate-vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  OFFLINE                               STARTING
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cvu
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ggdb.db
      1        OFFLINE OFFLINE
ora.layatest.db
      1        ONLINE  OFFLINE                               Instance Shutdown
      2        ONLINE  OFFLINE
ora.oc4j
      1        ONLINE  OFFLINE                               STARTING
ora.rhel11gr2rac1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac2.vip
      1        ONLINE  INTERMEDIATE rhel11gr2rac1            FAILED OVER
ora.scan1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan2.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan3.vip
      1        ONLINE  ONLINE       rhel11gr2rac1





Moving the OCR to another diskgroup.

--

Scenario - Now the OCR/voting is in the diskgroup OCR_VOTING with normal redundancy, we will
delete the disk so that this disk group cannot be mounted, then we will create an diskgroup
with same name with external redundancy and restore the ocr and voting disk
to that diskgroup.



Shutdown the crs on both the nodes.


Node 1

# crsctl stop crs


Node 2

# crsctl stop crs


Below are the two disks which are assinged to current ocr_voting diskgroup.

NOTE: Assigning number (4,1) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_2)
NOTE: Assigning number (4,0) to disk (/dev/oracleasm/disks/OCR_VOT_DISK_1)


[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK
OCR_VOT_DISK_1
OCR_VOT_DISK_2
You have new mail in /var/spool/mail/root

[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_2
Removing ASM disk "OCR_VOT_DISK_2":                        [  OK  ]

[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk OCR_VOT_DISK_1
Removing ASM disk "OCR_VOT_DISK_1":                        [  OK  ]


Now start the crs on the node 1 in exclusive mode without starting the crsd.

[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded

[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  INTERMEDIATE rhel11gr2rac1            OCR not started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        OFFLINE OFFLINE
ora.crsd
      1        OFFLINE OFFLINE
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        OFFLINE OFFLINE
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1



[root@rhel11gr2rac1 bin]# su - oracle

[oracle@rhel11gr2rac1 ~]$ . oraenv
ORACLE_SID = [oracle] ? +ASM1
The Oracle base remains unchanged with value /u01/app/oracle

[oracle@rhel11gr2rac1 ~]$ sqlplus / as sysasm

SQL*Plus: Release 11.2.0.3.0 Production on Sat Oct 26 09:44:14 2013

Copyright (c) 1982, 2011, Oracle.  All rights reserved.


Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options

SQL> col path form a40;
SQL> select path, header_status from V$asm_disk;

SQL> select path, header_status from V$asm_disk;

PATH                                     HEADER_STATU
---------------------------------------- ------------
/dev/oracleasm/disks/OCR_VOTE_DISK       PROVISIONED
/dev/oracleasm/disks/ACFSDISK            MEMBER
/dev/oracleasm/disks/FRA2                MEMBER
/dev/oracleasm/disks/FRA1                MEMBER
/dev/oracleasm/disks/DATA2               MEMBER
/dev/oracleasm/disks/DATA1               MEMBER

6 rows selected.


SQL> ! cat /etc/oracle/ocr.loc
ocrconfig_loc=+OCR_VOTING
local_only=FALSE

SQL> create diskgroup OCR_VOTING EXTERNAL REDUNDANCY disk '/dev/oracleasm/disks/OCR_VOTE_DISK' attribute 'compatible.asm' = '11.2';

Diskgroup created.


Restore from backup.


[root@rhel11gr2rac1 bin]# ./ocrconfig -import /u01/app/oracle/ocr.bkp_23oct

[root@rhel11gr2rac1 bin]# ./ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          3
         Total space (kbytes)     :     262120
         Used space (kbytes)      :       3520
         Available space (kbytes) :     258600
         ID                       : 1241840421
         Device/File Name         : +OCR_VOTING
                                    Device/File integrity check succeeded

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

         Cluster registry integrity check succeeded

         Logical corruption check succeeded



[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  ONLINE       rhel11gr2rac1            Started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        OFFLINE OFFLINE
ora.crsd
      1        OFFLINE OFFLINE
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        OFFLINE OFFLINE
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1


[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   73095be13c524f62bf76a7ac90537b12 (/dev/oracleasm/disks/OCR_VOT_DISK_1) [OCR_VOTING]
 2. OFFLINE  884ae51c216e4f0ebf368f1ca97e7120 () []
Located 2 voting disk(s).


From the above we could see that still the CSSD process is showing online cause still the voting disk
can be read from the old device /dev/oracleasm/disks/OCR_VOT_DISK_1


But we could see that the disk OCR_VOT_DISK_1 is already deleted (we have deleted the logical name but physically still the
device /dev/sd* is still accessable.


NoW restore the voting file.

Stop the crs and start again in exclusive mode.


[root@rhel11gr2rac1 bin]# ./crsctl stop crs
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded
[root@rhel11gr2rac1 bin]# ./crsctl replace votedisk +OCR_VOTING
Successful addition of voting disk 2fa636c2d1a74f11bffc43049f3fefc4.
Successfully replaced voting disk group with +OCR_VOTING.
CRS-4266: Voting file(s) successfully replaced
[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   2fa636c2d1a74f11bffc43049f3fefc4 (ORCL:OCR_VOTE_DISK) [OCR_VOTING]
Located 1 voting disk(s).


Now stop and restart the crs on all the nodes.

# crsctl start crs


[root@rhel11gr2rac2 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.DATA.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.FRA.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER.lsnr
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.OCR_VOTING.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
               ONLINE  ONLINE       rhel11gr2rac1            mounted on /golden_
                                                             gate
               ONLINE  ONLINE       rhel11gr2rac2            mounted on /golden_
                                                             gate
ora.asm
               ONLINE  ONLINE       rhel11gr2rac1            Started
               ONLINE  ONLINE       rhel11gr2rac2            Started
ora.gsd
               OFFLINE OFFLINE      rhel11gr2rac1
               OFFLINE OFFLINE      rhel11gr2rac2
ora.net1.network
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.ons
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.registry.acfs
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
      1        ONLINE  ONLINE       rhel11gr2rac2
goldengate-vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cvu
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ggdb.db
      1        OFFLINE OFFLINE
ora.layatest.db
      1        ONLINE  ONLINE       rhel11gr2rac1            Open
      2        ONLINE  ONLINE       rhel11gr2rac2            Open
ora.oc4j
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac2.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan1.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan2.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan3.vip
      1        ONLINE  ONLINE       rhel11gr2rac1


Notes:-

1. We need to create the ASM diskgroup with same name to restore the OCR/VOTING disk in case of failure.
2. The compatiable.asm attribute should be 11.2



-----------------------


[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK



Device "/dev/sdi1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdi2" is marked an ASM disk with the label "DATA2"
Device "/dev/sdj" is not marked as an ASM disk
Device "/dev/sdj1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdj2" is marked an ASM disk with the label "DATA2"



Node 2


SQL> alter diskgroup data dismount force;

Diskgroup altered.

SQL> select name, state from V$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
OCR_VOTING                     MOUNTED
ACFDG                          MOUNTED
DATA                           MOUNTED
FRA                            MOUNTED

SQL> alter diskgroup data dismount;

Diskgroup altered.

SQL> select name, state from V$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
OCR_VOTING                     MOUNTED
ACFDG                          MOUNTED
DATA                           DISMOUNTED
FRA                            MOUNTED



Node1

SQL> alter diskgroup data dismount;
alter diskgroup data dismount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15027: active use of diskgroup "DATA" precludes its dismount


SQL> alter diskgroup data dismount force;

Diskgroup altered.

SQL> ! ps -ef | grep pmon
oracle    5948     1  0 21:46 ?        00:00:00 asm_pmon_+ASM1
oracle    8256  8178  0 22:05 pts/1    00:00:00 /bin/bash -c  ps -ef | grep pmon

SQL> select name, state from V$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
OCR_VOTING                     MOUNTED
ACFDG                          MOUNTED
DATA                           MOUNTED
FRA                            MOUNTED

SQL> ! ps -ef | grep pmon
oracle    5948     1  0 21:46 ?        00:00:00 asm_pmon_+ASM1
oracle    8347  8178  0 22:07 pts/1    00:00:00 /bin/bash -c  ps -ef | grep pmon



SQL> alter diskgroup data dismount;

Diskgroup altered.




[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm deletedisk DATA1
Removing ASM disk "DATA1":                                 [  OK  ]


SQL> alter diskgroup DAta MOUNT;
alter diskgroup DAta MOUNT
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "0" is missing from group number "4"



SQL> alter diskgroup data mount force;

Diskgroup altered.


SQL> select path, name, header_status from V$asm_disk;

PATH                                     NAME                           HEADER_STATU
---------------------------------------- ------------------------------ ------------
ORCL:DATA1                                                              CANDIDATE
                                         DATA_0000                      UNKNOWN
ORCL:ACFSDISK                            ACFSDISK                       MEMBER
ORCL:DATA2                               DATA_0001                      MEMBER
ORCL:FRA1                                FRA_0000                       MEMBER
ORCL:FRA2                                FRA_0001                       MEMBER
ORCL:OCR_VOTE_DISK                       OCR_VOTING_0000                MEMBER



Now we could see that for the deleted disk we are getting two entries ... one shows as candidate and another as unknown.



[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA2
FRA1
FRA2
OCR_VOTE_DISK

[root@rhel11gr2rac1 bin]# ls -lrt /dev/oracleasm/disks/*
brw-rw---- 1 oracle dba 8,  97 Oct 30 21:43 /dev/oracleasm/disks/OCR_VOTE_DISK
brw-rw---- 1 oracle dba 8, 130 Oct 30 21:43 /dev/oracleasm/disks/DATA2
brw-rw---- 1 oracle dba 8, 161 Oct 30 21:43 /dev/oracleasm/disks/FRA1
brw-rw---- 1 oracle dba 8, 162 Oct 30 21:43 /dev/oracleasm/disks/FRA2
brw-rw---- 1 oracle dba 8, 193 Oct 30 21:43 /dev/oracleasm/disks/ACFSDISK


Using the kfed utility we can read the header of the disks, here the deleted disk DATA1 is /dev/sdi1, in the kfbh.type we
could see that it is showing diskhead, which is not corrupted.


[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1
kfbh.endian:                          1 ; 0x000: 0x01
kfbh.hard:                          130 ; 0x001: 0x82
kfbh.type:                            1 ; 0x002: KFBTYP_DISKHEAD
kfbh.datfmt:                          1 ; 0x003: 0x01
kfbh.block.blk:                       0 ; 0x004: blk=0
kfbh.block.obj:              2147483648 ; 0x008: disk=0
kfbh.check:                   804703913 ; 0x00c: 0x2ff6cea9
kfbh.fcn.base:                     7030 ; 0x010: 0x00001b76
kfbh.fcn.wrap:                        0 ; 0x014: 0x00000000
kfbh.spare1:                          0 ; 0x018: 0x00000000
kfbh.spare2:                          0 ; 0x01c: 0x00000000
kfdhdb.driver.provstr:         ORCLCLRD ; 0x000: length=8
kfdhdb.driver.reserved[0]:            0 ; 0x008: 0x00000000
kfdhdb.driver.reserved[1]:            0 ; 0x00c: 0x00000000
kfdhdb.driver.reserved[2]:            0 ; 0x010: 0x00000000
kfdhdb.driver.reserved[3]:            0 ; 0x014: 0x00000000
kfdhdb.driver.reserved[4]:            0 ; 0x018: 0x00000000
kfdhdb.driver.reserved[5]:            0 ; 0x01c: 0x00000000
kfdhdb.compat:                186646528 ; 0x020: 0x0b200000
kfdhdb.dsknum:                        0 ; 0x024: 0x0000
kfdhdb.grptyp:                        2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts:                        3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname:               DATA_0000 ; 0x028: length=9
kfdhdb.grpname:                    DATA ; 0x048: length=4
kfdhdb.fgname:                DATA_0000 ; 0x068: length=9
kfdhdb.capname:                         ; 0x088: length=0
kfdhdb.crestmp.hi:             32991850 ; 0x0a8: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.crestmp.lo:           3718423552 ; 0x0ac: USEC=0x0 MSEC=0xa9 SECS=0x1a MINS=0x37
kfdhdb.mntstmp.hi:             32992197 ; 0x0b0: HOUR=0x5 DAYS=0x1e MNTH=0xa YEAR=0x7dd
kfdhdb.mntstmp.lo:           3175588864 ; 0x0b4: USEC=0x0 MSEC=0x1e9 SECS=0x14 MINS=0x2f
kfdhdb.secsize:                     512 ; 0x0b8: 0x0200
kfdhdb.blksize:                    4096 ; 0x0ba: 0x1000
kfdhdb.ausize:                  1048576 ; 0x0bc: 0x00100000
kfdhdb.mfact:                    113792 ; 0x0c0: 0x0001bc80
kfdhdb.dsksize:                    1910 ; 0x0c4: 0x00000776
kfdhdb.pmcnt:                         2 ; 0x0c8: 0x00000002
kfdhdb.fstlocn:                       1 ; 0x0cc: 0x00000001
kfdhdb.altlocn:                       2 ; 0x0d0: 0x00000002
kfdhdb.f1b1locn:                      2 ; 0x0d4: 0x00000002
kfdhdb.redomirrors[0]:                0 ; 0x0d8: 0x0000
kfdhdb.redomirrors[1]:                1 ; 0x0da: 0x0001
kfdhdb.redomirrors[2]:            65535 ; 0x0dc: 0xffff
kfdhdb.redomirrors[3]:            65535 ; 0x0de: 0xffff
kfdhdb.dbcompat:              168820736 ; 0x0e0: 0x0a100000
kfdhdb.grpstmp.hi:             32991850 ; 0x0e4: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.grpstmp.lo:           3718281216 ; 0x0e8: USEC=0x0 MSEC=0x1e SECS=0x1a MINS=0x37




[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk DATA1 /dev/sdi1
Marking disk "DATA1" as an ASM disk:                       [  OK  ]
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1
kfbh.endian:                          0 ; 0x000: 0x00
kfbh.hard:                            0 ; 0x001: 0x00
kfbh.type:                            0 ; 0x002: KFBTYP_INVALID
kfbh.datfmt:                          0 ; 0x003: 0x00
kfbh.block.blk:                       0 ; 0x004: blk=0
kfbh.block.obj:                       0 ; 0x008: file=0
kfbh.check:                  1856147420 ; 0x00c: 0x6ea28fdc
kfbh.fcn.base:                        0 ; 0x010: 0x00000000
kfbh.fcn.wrap:                        0 ; 0x014: 0x00000000
kfbh.spare1:                          0 ; 0x018: 0x00000000
kfbh.spare2:                          0 ; 0x01c: 0x00000000
2B1E9B240400 00000000 00000000 00000000 6EA28FDC  [...............n]
2B1E9B240410 00000000 00000000 00000000 00000000  [................]
2B1E9B240420 4C43524F 4B534944 41544144 00000031  [ORCLDISKDATA1...]
2B1E9B240430 00000000 00000000 00000000 00000000  [................]
  Repeat 252 times
KFED-00322: file not found; arguments: [kfbtTraverseBlock] [Invalid OSM block type] [] [0]


now the disk header is corrupted.


Now delete the disk

[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm deletedisk DATA1
Removing ASM disk "DATA1":                                 [  OK  ]



We havent taken the backup of the header before creating the disk again, but still
ASM contains the copy of ASM disk header in (Allocation unit# 1, Block# 254) of every ASM disk.

Let view the content of it.

[root@rhel11gr2rac1 ~]# kfed read /dev/sdi1 aun=1 blkn=256 | grep KFB
kfbh.type:                            5 ; 0x002: KFBTYP_LISTHEAD



Its is showing as listhead and not the invalid. We can repair the header using the
kfed repair option.



[root@rhel11gr2rac1 ~]# kfed repair /dev/sdi1


Now lets check the header again.

[root@rhel11gr2rac1 ~]# /grid/11.2/bin/kfed read /dev/sdi1
kfbh.endian:                          1 ; 0x000: 0x01
kfbh.hard:                          130 ; 0x001: 0x82
kfbh.type:                            1 ; 0x002: KFBTYP_DISKHEAD
kfbh.datfmt:                          1 ; 0x003: 0x01
kfbh.block.blk:                       0 ; 0x004: blk=0
kfbh.block.obj:              2147483648 ; 0x008: disk=0
kfbh.check:                   804703913 ; 0x00c: 0x2ff6cea9
kfbh.fcn.base:                     7030 ; 0x010: 0x00001b76
kfbh.fcn.wrap:                        0 ; 0x014: 0x00000000
kfbh.spare1:                          0 ; 0x018: 0x00000000
kfbh.spare2:                          0 ; 0x01c: 0x00000000
kfdhdb.driver.provstr:    ORCLDISKDATA1 ; 0x000: length=13
kfdhdb.driver.reserved[0]:   1096040772 ; 0x008: 0x41544144
kfdhdb.driver.reserved[1]:           49 ; 0x00c: 0x00000031
kfdhdb.driver.reserved[2]:            0 ; 0x010: 0x00000000
kfdhdb.driver.reserved[3]:            0 ; 0x014: 0x00000000
kfdhdb.driver.reserved[4]:            0 ; 0x018: 0x00000000
kfdhdb.driver.reserved[5]:            0 ; 0x01c: 0x00000000
kfdhdb.compat:                186646528 ; 0x020: 0x0b200000
kfdhdb.dsknum:                        0 ; 0x024: 0x0000
kfdhdb.grptyp:                        2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts:                        3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname:               DATA_0000 ; 0x028: length=9
kfdhdb.grpname:                    DATA ; 0x048: length=4
kfdhdb.fgname:                DATA_0000 ; 0x068: length=9
kfdhdb.capname:                         ; 0x088: length=0
kfdhdb.crestmp.hi:             32991850 ; 0x0a8: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.crestmp.lo:           3718423552 ; 0x0ac: USEC=0x0 MSEC=0xa9 SECS=0x1a MINS=0x37
kfdhdb.mntstmp.hi:             32992197 ; 0x0b0: HOUR=0x5 DAYS=0x1e MNTH=0xa YEAR=0x7dd
kfdhdb.mntstmp.lo:           3175588864 ; 0x0b4: USEC=0x0 MSEC=0x1e9 SECS=0x14 MINS=0x2f
kfdhdb.secsize:                     512 ; 0x0b8: 0x0200
kfdhdb.blksize:                    4096 ; 0x0ba: 0x1000
kfdhdb.ausize:                  1048576 ; 0x0bc: 0x00100000
kfdhdb.mfact:                    113792 ; 0x0c0: 0x0001bc80
kfdhdb.dsksize:                    1910 ; 0x0c4: 0x00000776
kfdhdb.pmcnt:                         2 ; 0x0c8: 0x00000002
kfdhdb.fstlocn:                       1 ; 0x0cc: 0x00000001
kfdhdb.altlocn:                       2 ; 0x0d0: 0x00000002
kfdhdb.f1b1locn:                      2 ; 0x0d4: 0x00000002
kfdhdb.redomirrors[0]:                0 ; 0x0d8: 0x0000
kfdhdb.redomirrors[1]:                1 ; 0x0da: 0x0001
kfdhdb.redomirrors[2]:            65535 ; 0x0dc: 0xffff
kfdhdb.redomirrors[3]:            65535 ; 0x0de: 0xffff
kfdhdb.dbcompat:              168820736 ; 0x0e0: 0x0a100000
kfdhdb.grpstmp.hi:             32991850 ; 0x0e4: HOUR=0xa DAYS=0x13 MNTH=0xa YEAR=0x7dd
kfdhdb.grpstmp.lo:           3718281216 ; 0x0e8: USEC=0x0 MSEC=0x1e SECS=0x1a MINS=0x37
kfdhdb.vfstart:                       0 ; 0x0ec: 0x00000000
kfdhdb.vfend:                         0 ; 0x0f0: 0x00000000
kfdhdb.spfile:                        0 ; 0x0f4: 0x00000000
kfdhdb.spfflg:                        0 ; 0x0f8: 0x00000000
kfdhdb.ub4spare[0]:                   0 ; 0x0fc: 0x00000000
kfdhdb.ub4spare[1]:                   0 ; 0x100: 0x00000000
kfdhdb.ub4spare[2]:                   0 ; 0x104: 0x00000000
kfdhdb.ub4spare[3]:                   0 ; 0x108: 0x00000000
kfdhdb.ub4spare[4]:                   0 ; 0x10c: 0x00000000
kfdhdb.ub4spare[5]:                   0 ; 0x110: 0x00000000
kfdhdb.ub4spare[6]:                   0 ; 0x114: 0x00000000
kfdhdb.ub4spare[7]:                   0 ; 0x118: 0x00000000
kfdhdb.ub4spare[8]:                   0 ; 0x11c: 0x00000000
kfdhdb.ub4spare[9]:                   0 ; 0x120: 0x00000000
kfdhdb.ub4spare[10]:                  0 ; 0x124: 0x00000000
kfdhdb.ub4spare[11]:                  0 ; 0x128: 0x00000000
kfdhdb.ub4spare[12]:                  0 ; 0x12c: 0x00000000
kfdhdb.ub4spare[13]:                  0 ; 0x130: 0x00000000
kfdhdb.ub4spare[14]:                  0 ; 0x134: 0x00000000
kfdhdb.ub4spare[15]:                  0 ; 0x138: 0x00000000
kfdhdb.ub4spare[16]:                  0 ; 0x13c: 0x00000000
kfdhdb.ub4spare[17]:                  0 ; 0x140: 0x00000000
kfdhdb.ub4spare[18]:                  0 ; 0x144: 0x00000000
kfdhdb.ub4spare[19]:                  0 ; 0x148: 0x00000000
kfdhdb.ub4spare[20]:                  0 ; 0x14c: 0x00000000
kfdhdb.ub4spare[21]:                  0 ; 0x150: 0x00000000
kfdhdb.ub4spare[22]:                  0 ; 0x154: 0x00000000
kfdhdb.ub4spare[23]:                  0 ; 0x158: 0x00000000
kfdhdb.ub4spare[24]:                  0 ; 0x15c: 0x00000000
kfdhdb.ub4spare[25]:                  0 ; 0x160: 0x00000000
kfdhdb.ub4spare[26]:                  0 ; 0x164: 0x00000000
kfdhdb.ub4spare[27]:                  0 ; 0x168: 0x00000000
kfdhdb.ub4spare[28]:                  0 ; 0x16c: 0x00000000
kfdhdb.ub4spare[29]:                  0 ; 0x170: 0x00000000
kfdhdb.ub4spare[30]:                  0 ; 0x174: 0x00000000
kfdhdb.ub4spare[31]:                  0 ; 0x178: 0x00000000
kfdhdb.ub4spare[32]:                  0 ; 0x17c: 0x00000000
kfdhdb.ub4spare[33]:                  0 ; 0x180: 0x00000000
kfdhdb.ub4spare[34]:                  0 ; 0x184: 0x00000000
kfdhdb.ub4spare[35]:                  0 ; 0x188: 0x00000000
kfdhdb.ub4spare[36]:                  0 ; 0x18c: 0x00000000
kfdhdb.ub4spare[37]:                  0 ; 0x190: 0x00000000
kfdhdb.ub4spare[38]:                  0 ; 0x194: 0x00000000
kfdhdb.ub4spare[39]:                  0 ; 0x198: 0x00000000
kfdhdb.ub4spare[40]:                  0 ; 0x19c: 0x00000000
kfdhdb.ub4spare[41]:                  0 ; 0x1a0: 0x00000000
kfdhdb.ub4spare[42]:                  0 ; 0x1a4: 0x00000000
kfdhdb.ub4spare[43]:                  0 ; 0x1a8: 0x00000000
kfdhdb.ub4spare[44]:                  0 ; 0x1ac: 0x00000000
kfdhdb.ub4spare[45]:                  0 ; 0x1b0: 0x00000000
kfdhdb.ub4spare[46]:                  0 ; 0x1b4: 0x00000000
kfdhdb.ub4spare[47]:                  0 ; 0x1b8: 0x00000000
kfdhdb.ub4spare[48]:                  0 ; 0x1bc: 0x00000000
kfdhdb.ub4spare[49]:                  0 ; 0x1c0: 0x00000000
kfdhdb.ub4spare[50]:                  0 ; 0x1c4: 0x00000000
kfdhdb.ub4spare[51]:                  0 ; 0x1c8: 0x00000000
kfdhdb.ub4spare[52]:                  0 ; 0x1cc: 0x00000000
kfdhdb.ub4spare[53]:                  0 ; 0x1d0: 0x00000000
kfdhdb.acdb.aba.seq:                  0 ; 0x1d4: 0x00000000
kfdhdb.acdb.aba.blk:                  0 ; 0x1d8: 0x00000000
kfdhdb.acdb.ents:                     0 ; 0x1dc: 0x0000
kfdhdb.acdb.ub2spare:                 0 ; 0x1de: 0x0000


Its header has been restored properly.


Main things to see..

kfbh.type:                            1 ; 0x002: KFBTYP_DISKHEAD
kfdhdb.driver.provstr:    ORCLDISKDATA1 ; 0x000: length=13
kfdhdb.grptyp:                        2 ; 0x026: KFDGTP_NORMAL
kfdhdb.hdrsts:                        3 ; 0x027: KFDHDR_MEMBER
kfdhdb.dskname:               DATA_0000 ; 0x028: length=9
kfdhdb.grpname:                    DATA ; 0x048: length=4
kfdhdb.fgname:                DATA_0000 ; 0x068: length=9




KFBTYP_DISKHEAD - Heade is valid.
ORCLDISKDATA1   - This is the disk name which been created using oracleasm [DATA1]
DATA_0000       - Disk name and fail group name in asm instance.
DATA  - Is the Diskgroup name.


Now we have the proper header details on the device. Now do the scandisk to list out this disk.


---------------------------------------------------------------------------------------------------------------
Note ---- If kfdhdb.driver.provstr:    shows only ORCLDISK and not the full disk name i.e. ORCLDISKDATA1 then
you can run the below force-rename to rename the disk.. in our case it is listing properly.

[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm force-renamedisk /dev/sdi1 DATA1
---------------------------------------------------------------------------------------------------------------




[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm scandisks
Scanning the system for Oracle ASMLib disks:               [  OK  ]

[root@rhel11gr2rac1 ~]# /etc/init.d/oracleasm listdisks
ACFSDISK
DATA1
DATA2
FRA1
FRA2
OCR_VOTE_DISK


Now the DATA1 disk is getting visible.

Below from the asm instance.


SQL> select path, name, state, failgroup, mode_status, total_mb, label , header_status from V$asm_disk;

PATH                 NAME                           STATE    FAILGROUP                      MODE_ST   TOTAL_MB LABEL                           HEADER_STATU
-------------------- ------------------------------ -------- ------------------------------ ------- ---------- ------------------------------- ------------
ORCL:DATA1                                          NORMAL                                  ONLINE           0 DATA1                           MEMBER
                     DATA_0000                      NORMAL   DATA_0000                      OFFLINE          0                                 UNKNOWN
ORCL:ACFSDISK        ACFSDISK                       NORMAL   ACFSDISK                       ONLINE        9759 ACFSDISK                        MEMBER
ORCL:DATA2           DATA_0001                      NORMAL   DATA_0001                      ONLINE        1911 DATA2                           MEMBER
ORCL:FRA1            FRA_0000                       NORMAL   FRA_0000                       ONLINE         955 FRA1                            MEMBER
ORCL:FRA2            FRA_0001                       NORMAL   FRA_0001                       ONLINE         921 FRA2                            MEMBER
ORCL:OCR_VOTE_DISK   OCR_VOTING_0000                NORMAL   OCR_VOTING_0000                ONLINE         959 OCR_VOTE_DISK                   MEMBER


Now add this disk to the diskgroup data using force option.

Note- To add the existing disk which has been droped we can use the force option, but not for adding a new disk.


SQL> alter diskgroup DATA add disk 'ORCL:DATA1' force;

Diskgroup altered.

SQL> select path, name, state, failgroup, mode_status, total_mb, label , header_status from V$asm_disk;

PATH                 NAME                           STATE    FAILGROUP                      MODE_ST   TOTAL_MB LABEL                           HEADER_STATU
-------------------- ------------------------------ -------- ------------------------------ ------- ---------- ------------------------------- ------------
                     DATA_0000                      NORMAL   DATA_0000                      OFFLINE          0                                 UNKNOWN
ORCL:ACFSDISK        ACFSDISK                       NORMAL   ACFSDISK                       ONLINE        9759 ACFSDISK                        MEMBER
ORCL:DATA1           DATA1                          NORMAL   DATA1                          ONLINE        1910 DATA1                           MEMBER
ORCL:DATA2           DATA_0001                      NORMAL   DATA_0001                      ONLINE        1911 DATA2                           MEMBER
ORCL:FRA1            FRA_0000                       NORMAL   FRA_0000                       ONLINE         955 FRA1                            MEMBER
ORCL:FRA2            FRA_0001                       NORMAL   FRA_0001                       ONLINE         921 FRA2                            MEMBER
ORCL:OCR_VOTE_DISK   OCR_VOTING_0000                NORMAL   OCR_VOTING_0000                ONLINE         959 OCR_VOTE_DISK                   MEMBER

7 rows selected.

SQL> alter diskgroup data drop disk 'ORCL:DATA1';
alter diskgroup data drop disk 'ORCL:DATA1'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15054: disk "ORCL:DATA1" does not exist in diskgroup "DATA"


SQL> alter diskgroup data drop disk 'DATA1';

Diskgroup altered.








Data 1 is showing as member.




Using this data we can create the oraleasm disk again.




From the asm instance lets check what are the disks are listing.

SQL> select path, name, header_status from V$asm_disk;

PATH                                     NAME                           HEADER_STATU
---------------------------------------- ------------------------------ ------------
                                         DATA_0000                      UNKNOWN
ORCL:ACFSDISK                            ACFSDISK                       MEMBER
ORCL:DATA2                               DATA_0001                      MEMBER
ORCL:FRA1                                FRA_0000                       MEMBER
ORCL:FRA2                                FRA_0001                       MEMBER
ORCL:OCR_VOTE_DISK                       OCR_VOTING_0000                MEMBER



--- Here i have changed the asm_diskstring from default value null to "ORCL:*" and once this is done
my asm instance keep rebooting for every one sec...


-- Here just stop the instance and recreate the spfile from a pfile with modified parameter values and restart the instance.


http://www.usn-it.de/index.php/2011/07/12/oracle-grid-infrastructure-how-to-recover-from-a-messed-up-asmcssd-diskstring/


evice "/dev/sdi1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdi2" is marked an ASM disk with the label "DATA2"
Device "/dev/sdj" is not marked as an ASM disk
Device "/dev/sdj1" is marked an ASM disk with the label "DATA1"
Device "/dev/sdj2" is marked an ASM disk with the label "DATA2"

Device "/dev/sdm1" is marked an ASM disk with the label "ACFSDISK"




Test scenario.


---

OCR_VOTING DISK is external redundancy.


[oracle@rhel11gr2rac1 bin]$ ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.DATA.dg
               ONLINE  OFFLINE      rhel11gr2rac1
               ONLINE  OFFLINE      rhel11gr2rac2
ora.FRA.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER.lsnr
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.OCR_VOTING.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
               ONLINE  ONLINE       rhel11gr2rac1            mounted on /golden_
                                                             gate
               OFFLINE OFFLINE      rhel11gr2rac2            volume /golden_gate
                                                             offline
ora.asm
               ONLINE  ONLINE       rhel11gr2rac1            Started
               ONLINE  ONLINE       rhel11gr2rac2            Started
ora.gsd
               OFFLINE OFFLINE      rhel11gr2rac1
               OFFLINE OFFLINE      rhel11gr2rac2
ora.net1.network
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.ons
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.registry.acfs
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
      1        ONLINE  UNKNOWN      rhel11gr2rac2
goldengate-vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cvu
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ggdb.db
      1        OFFLINE OFFLINE
ora.layatest.db
      1        ONLINE  OFFLINE                               Instance Shutdown
      2        ONLINE  OFFLINE
ora.oc4j
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac2.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan1.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan2.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan3.vip
      1        ONLINE  ONLINE       rhel11gr2rac1




[oracle@rhel11gr2rac1 bin]$ ./ocrconfig -showbackup

rhel11gr2rac2     2013/10/01 22:56:35     /grid/11.2/cdata/urac1rac2/backup00.ocr

rhel11gr2rac1     2013/07/13 21:00:51     /grid/11.2/cdata/urac1rac2/backup01.ocr

rhel11gr2rac2     2013/10/01 22:56:35     /grid/11.2/cdata/urac1rac2/day.ocr

rhel11gr2rac2     2013/10/01 22:56:35     /grid/11.2/cdata/urac1rac2/week.ocr
PROT-25: Manual backups for the Oracle Cluster Registry are not available


[oracle@rhel11gr2rac1 bin]$ ./crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   7c8085fce8f24fb3bff9beb4ff52deb3 (ORCL:OCT_VOTE_DISK_1) [OCR_VOTING]
Located 1 voting disk(s).




Node 1

gpnptool output.


[oracle@rhel11gr2rac1 bin]$ ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

rNRZ4TPVhG5kUWAMBhieKUD4opQ=L9c9t1Z9ksM7cJDvfKvUrmOWFGbpuUaBMe7+tFdtXgV0ZNgzriu4BV9PBjqPYai9ZIavHYbpIaJiO4RYAIq0/GBJ5EGyMtTMMIFwpAINWMM92kPjCxsB9e81hbXJrdwzLXEWW+iwzT00KEkz+HH4MbzZAj0eEssZoNP939pwv1I=
Success.


Node 2

gpnptool output.


[root@rhel11gr2rac2 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

rNRZ4TPVhG5kUWAMBhieKUD4opQ=L9c9t1Z9ksM7cJDvfKvUrmOWFGbpuUaBMe7+tFdtXgV0ZNgzriu4BV9PBjqPYai9ZIavHYbpIaJiO4RYAIq0/GBJ5EGyMtTMMIFwpAINWMM92kPjCxsB9e81hbXJrdwzLXEWW+iwzT00KEkz+HH4MbzZAj0eEssZoNP939pwv1I=
Success.


SQL> select name, state , type from V$asm_diskgroup;

NAME                           STATE       TYPE
------------------------------ ----------- ------
ACFDG                          MOUNTED     EXTERN
FRA                            MOUNTED     NORMAL
OCR_VOTING                     MOUNTED     EXTERN

SQL> select name, group_number from V$asm_diskgroup;

NAME                           GROUP_NUMBER
------------------------------ ------------
ACFDG                                     1
FRA                                       3
OCR_VOTING                                4

SQL> col path form a40;
SQL> select path, header_status, group_number from V$asm_disk;

PATH                                     HEADER_STATU GROUP_NUMBER
---------------------------------------- ------------ ------------
ORCL:ACFSDISK                            MEMBER                  1
ORCL:FRA1                                MEMBER                  3
ORCL:FRA2                                MEMBER                  3
ORCL:OCT_VOTE_DISK_1                     MEMBER                  4


/dev/sdg1 -- OCT_VOTE_DISK_1

-- Stopped crs on both the nodes.



sde1



kfed read /dev/sde1 | grep kfdhdb.dskname



[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdc1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCR_VOTING_0000 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdd1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCR_VOTING_0000 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sde1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCR_VOTING_0001 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdf1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCR_VOTING_0001 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdg1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCT_VOTE_DISK_1 ; 0x028: length=15
[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdh1 | grep kfdhdb.dskname
kfdhdb.dskname:         OCT_VOTE_DISK_1 ; 0x028: length=15


[root@rhel11gr2rac1 bin]# ./kfed read /dev/sdi1 | grep kfdhdb.dskname
kfdhdb.dskname:                   DATA1 ; 0x028: length=5


Device "/dev/sdk1" is marked an ASM disk with the label "FRA1"
Device "/dev/sdk2" is marked an ASM disk with the label "FRA2"

[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk FRA1
Removing ASM disk "FRA1":                                  [  OK  ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm deletedisk FRA2
Removing ASM disk "FRA2":                                  [  OK  ]


create new ocr disk using /dev/sdc1 /dev/sde1 /dev/sdk1

format the above files...

[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_1 /dev/sdc1
Marking disk "OCR_VOTING_DISK_1" as an ASM disk:           [  OK  ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_2 /dev/sde1
Marking disk "OCR_VOTING_DISK_2" as an ASM disk:           [  OK  ]
[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm createdisk OCR_VOTING_DISK_3 /dev/sdk1
Marking disk "OCR_VOTING_DISK_3" as an ASM disk:           [  OK  ]


[root@rhel11gr2rac1 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
OCR_VOTING_DISK_1
OCR_VOTING_DISK_2
OCR_VOTING_DISK_3


[root@rhel11gr2rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.diskmon' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rhel11gr2rac1'
CRS-2676: Start of 'ora.asm' on 'rhel11gr2rac1' succeeded

[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  INTERMEDIATE rhel11gr2rac1            OCR not started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        OFFLINE OFFLINE
ora.crsd
      1        OFFLINE OFFLINE
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        OFFLINE OFFLINE
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1


[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
Located 0 voting disk(s).


SQL> select name , state from V$asm_diskgroup;


NAME                           STATE
------------------------------ -----------
ACFDG                          DISMOUNTED


SQL> select path, header_status, name from V$asm_disk;

PATH                                     HEADER_STATU NAME
---------------------------------------- ------------ ------------------------------
ORCL:ACFSDISK                            MEMBER
ORCL:OCR_VOTING_DISK_3                   PROVISIONED
ORCL:OCR_VOTING_DISK_2                   PROVISIONED
ORCL:OCR_VOTING_DISK_1                   PROVISIONED


SQL> create diskgroup OCR_VOTING normal redundancy disk 'ORCL:OCR_VOTING_DISK_1','ORCL:OCR_VOTING_DISK_2','ORCL:OCR_VOTING_DISK_3'
  2  attribute 'compatible.asm' = '11.2';

Diskgroup created.

SQL> select path, header_status, name from V$asm_disk;

PATH                                     HEADER_STATU NAME
---------------------------------------- ------------ ------------------------------
ORCL:ACFSDISK                            MEMBER
ORCL:OCR_VOTING_DISK_1                   MEMBER       OCR_VOTING_DISK_1
ORCL:OCR_VOTING_DISK_2                   MEMBER       OCR_VOTING_DISK_2
ORCL:OCR_VOTING_DISK_3                   MEMBER       OCR_VOTING_DISK_3


[root@rhel11gr2rac1 bin]# ./ocrconfig -restore /grid/11.2/cdata/urac1rac2/backup00.ocr


[root@rhel11gr2rac1 bin]# ./ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          3
         Total space (kbytes)     :     262120
         Used space (kbytes)      :       3520
         Available space (kbytes) :     258600
         ID                       :  845960811
         Device/File Name         : +OCR_VOTING
                                    Device/File integrity check succeeded

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

         Cluster registry integrity check succeeded

         Logical corruption check succeeded


[root@rhel11gr2rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  ONLINE       rhel11gr2rac1            Started
ora.cluster_interconnect.haip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.crf
      1        OFFLINE OFFLINE
ora.crsd
      1        OFFLINE OFFLINE
ora.cssd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cssdmonitor
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ctssd
      1        ONLINE  ONLINE       rhel11gr2rac1            ACTIVE:0
ora.diskmon
      1        OFFLINE OFFLINE
ora.drivers.acfs
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.evmd
      1        OFFLINE OFFLINE
ora.gipcd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.gpnpd
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.mdnsd
      1        ONLINE  ONLINE       rhel11gr2rac1


[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
Located 0 voting disk(s).

[root@rhel11gr2rac1 bin]# ./crsctl replace votedisk +OCR_VOTING
Successful addition of voting disk 9e87f77bd3064f87bf36380306d013ed.
Successful addition of voting disk b0de29805c974f12bffbf5f3a2564e99.
Successful addition of voting disk 007d0a5205bd4f0ebf97a2098200ba01.
Successfully replaced voting disk group with +OCR_VOTING.
CRS-4266: Voting file(s) successfully replaced

[root@rhel11gr2rac1 bin]# ./crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   9e87f77bd3064f87bf36380306d013ed (ORCL:OCR_VOTING_DISK_1) [OCR_VOTING]
 2. ONLINE   b0de29805c974f12bffbf5f3a2564e99 (ORCL:OCR_VOTING_DISK_2) [OCR_VOTING]
 3. ONLINE   007d0a5205bd4f0ebf97a2098200ba01 (ORCL:OCR_VOTING_DISK_3) [OCR_VOTING]
Located 3 voting disk(s).


[root@rhel11gr2rac1 bin]# ./crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rhel11gr2rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.mdnsd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.ctssd' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.cssd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rhel11gr2rac1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rhel11gr2rac1'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rhel11gr2rac1' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rhel11gr2rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rhel11gr2rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.

[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
FRA1
FRA2
OCT_VOTE_DISK_1
You have new mail in /var/spool/mail/root
[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm scandisks
Scanning the system for Oracle ASMLib disks:               [  OK  ]
[root@rhel11gr2rac2 bin]# /etc/init.d/oracleasm listdisks
ACFSDISK
OCR_VOTING_DISK_1
OCR_VOTING_DISK_2
OCR_VOTING_DISK_3

Now start the crs on both the nodes.

[root@rhel11gr2rac1 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.



[root@rhel11gr2rac2 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.

[root@rhel11gr2rac1 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ACFDG.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.DATA.dg
               ONLINE  OFFLINE      rhel11gr2rac1
               ONLINE  OFFLINE      rhel11gr2rac2
ora.FRA.dg
               ONLINE  OFFLINE      rhel11gr2rac1
               ONLINE  OFFLINE      rhel11gr2rac2
ora.LISTENER.lsnr
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.OCR_VOTING.dg
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.acfdg.acfsvolume.acfs
               ONLINE  ONLINE       rhel11gr2rac1            mounted on /golden_
                                                             gate
               ONLINE  ONLINE       rhel11gr2rac2            mounted on /golden_
                                                             gate
ora.asm
               ONLINE  ONLINE       rhel11gr2rac1            Started
               ONLINE  ONLINE       rhel11gr2rac2            Started
ora.gsd
               OFFLINE OFFLINE      rhel11gr2rac1
               OFFLINE OFFLINE      rhel11gr2rac2
ora.net1.network
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.ons
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
ora.registry.acfs
               ONLINE  ONLINE       rhel11gr2rac1
               ONLINE  ONLINE       rhel11gr2rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ggateapp
      1        ONLINE  UNKNOWN      rhel11gr2rac1
goldengate-vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.cvu
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.ggdb.db
      1        OFFLINE OFFLINE
ora.layatest.db
      1        ONLINE  OFFLINE                               Instance Shutdown
      2        ONLINE  OFFLINE
ora.oc4j
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac1.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.rhel11gr2rac2.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan1.vip
      1        ONLINE  ONLINE       rhel11gr2rac2
ora.scan2.vip
      1        ONLINE  ONLINE       rhel11gr2rac1
ora.scan3.vip
      1        ONLINE  ONLINE       rhel11gr2rac1




[root@rhel11gr2rac1 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

FOnQAojiSKzhlhGxaunsL5wQzAE=TW4UgXJH9g1xmQDSc/RKse9OulVpQl/94qp9Toy9HwtWhJCAdgId2V5JTdyxaRHmUi0YCEYOnmRA0K8wj8him6RrWwm1J+U6o85JyIVQSb76tkKqV+4AxrPVkVpmXRqroKS+Z6fTNLmR35nwoRXAzU2bx4j2zyVVXrSHIvpOHKg=
Success.


[root@rhel11gr2rac2 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

FOnQAojiSKzhlhGxaunsL5wQzAE=TW4UgXJH9g1xmQDSc/RKse9OulVpQl/94qp9Toy9HwtWhJCAdgId2V5JTdyxaRHmUi0YCEYOnmRA0K8wj8him6RrWwm1J+U6o85JyIVQSb76tkKqV+4AxrPVkVpmXRqroKS+Z6fTNLmR35nwoRXAzU2bx4j2zyVVXrSHIvpOHKg=
Success.


























No comments:

Post a Comment