#!/usr/local/bin/perl
# 
# $Header: has/install/crsconfig/crsconfig_lib.pm /st_has_11.2.0/163 2013/07/22 04:23:08 sidshank Exp $
#
# crsconfig_lib.pm
# 
# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. 
#
#    NAME
#      crsconfig_lib.pm - Library module for root scripts
#
#    DESCRIPTION
#      crsconfig_lib.pm - Library module containing functions for Oracle
#                         Grid Infrastructure and SI-HAS configurations
#
#    NOTES
#
#    MODIFIED   (MM/DD/YY)
#    sidshank    07/19/13 - backport 14196853.
#    xyuan       06/27/13 - Fix bug 17026238 - backport xyuan_bug-14603264
#    rdasari     06/26/13 - backport bug 14296767
#    sidshank    06/21/13 - backport 16932128.
#    xyuan       06/19/13 - Fix bug 16981100
#    xyuan       06/06/13 - Back out xyuan_rfi_backport_16591778_11.2.0.4.0
#    jgrout      06/04/13 - Backport fix for bug 14615073 from 12.1.0.1
#    xyuan       05/30/13 - Fix bug 16882382
#    xyuan       05/29/13 - Fix bug 16879664 - backport xyuan_bug-16591778
#    xyuan       05/24/13 - Fix bug 16854157
#    gadiga      05/23/13 - setup TFA
#    xyuan       05/22/13 - Fix bug 16846764
#    xyuan       05/22/13 - Fix bug 16833151
#    shullur     05/20/13 - For removing hard exit while moving directories.
#                           Bug 16836877
#    minzhu      05/20/13 - 16833857, reset to weak start_dep for asm with HAIP
#    rtamezd     05/13/13 - Fix bug 13621968
#    ksviswan    05/12/13 - Backport ksviswan_bug-13425727 from
#                           st_has_11.2.0.3.0gipsu
#    gmaldona    04/17/13 - bug 16679789
#    papatel     03/25/13 - Fix Bug 16444600
#    rdasari     02/25/13 - XbranchMerge
#                           rdasari_ci_backport_14407395_11.2.0.3.5gipsu from
#                           st_has_11.2.0.3.0gipsu
#    xyuan       02/21/13 - Fix bug 16367315
#    fjlee       02/11/13 - bug 16307750
#    gmaldona    01/31/13 - bug-14190671 oc4j should be disabled for hp-ux
#    shullur     01/23/13 - Backport
#                           arpshukl_ci_backport_14168708_11.2.0.3.5gipsu from
#                           st_has_11.2.0.3.0gipsu
#    xyuan       01/18/13 - BACKPORT OF BUG 14535011
#    vdandu      01/15/13 - Backport vdandu_bug-14513204 from main
#    takiba      12/25/12 - Backport takiba_bug-13868167 from main
#    rdasari     12/20/12 - Backport rdasari_14407395_112034 from
#    jmunozn     11/22/12 - Use qosctl to add QoS users
#    arpshukl    10/18/12 - Backport arpshukl_bug-14168708 from main
#    rdasari     08/17/12 - do not stop_diskmon after CSS stop
#    xyuan       08/15/12 - Fix bug 14149052
#    xyuan       07/19/12 - Fix bug 14331566 & 14296767
#    rvadraha    07/18/12 - Bug14324057:Fix regexp in instantiate_config_params 
#    tvbabu      07/17/12 - bug 14310221
#    xyuan       07/05/12 - Fix bug 13569454
#    xyuan       07/03/12 - Fix bug 13541062
#    shullur     06/20/12 - Backport shullur_bug_13905229 from Main
#    sidshank    06/19/12 - fix for bug 14168051.
#    sidshank    02/26/12 - Fixing the path issue during pattern match on
#                           windows. Bug 13724808.
#    rtamezd     02/23/12 - XbranchMerge rtamezd_bug-13652088 from
#    rtamezd     02/07/12 - Export func. s_register_service
#    ksviswan    01/27/12 - Backport ksviswan_bug-13425727 from
#                           st_has_11.2.0.3.0gipsu
#    ksviswan    11/23/11 - Fix out of place patch Bug 13425727
#    xyuan       10/11/11 - Fix bug 13082238
#    shullur     09/29/11 - For fixing rerun issue of root script. Bug
#                           12976590.
#    rdasari     09/22/11 - remove OCR on DG in deconfigure_ASM if DG is reused 
#    sidshank    09/06/11 - changes related to 12947871
#    rdasari     08/25/11 - export stopOracleRestart
#    ksviswan    08/19/11 - Fix Bug 12886285
#    xesquive    08/19/11 - add functions set_bold and reset_bold
#    xyuan       08/16/11 - Fix bug 12876957
#    xesquive    08/16/11 - Add two function set_bold and reset_bold, refer to
#                           bug-12765467
#    nvira       08/10/11 - skip the check for patches in development env
#    rdasari     08/08/11 - add cluvfy check to preUpgradeChecks
#    xyuan       08/04/11 - Fix bug 12529687
#    xyuan       07/27/11 - Fix bugs 12795595,12802995,12734710
#    xyuan       07/26/11 - Fix bug 12752359
#    shullur     07/25/11 - For Fixing bug 12794367.
#    gmaldona    07/20/11 - setup OC4J wallet in upgrades
#    nkorehis    07/19/11 - fix bug 12710656
#    xyuan       07/18/11 - Fix bug 12758739
#    xyuan       07/18/11 - Fix bug 12727247
#    jgrout      07/15/11 - Fix bugs 9900672 and 12648797
#    xyuan       07/15/11 - Fix bug 12746415
#    rdasari     07/15/11 - do not use avlookup for siha
#    xyuan       07/14/11 - Fix bug 12698968,12728939
#    xyuan       07/12/11 - Fix bug 12585291 & add -keepdg option
#    xyuan       07/11/11 - Fix bug 12701521
#    xyuan       07/08/11 - Fix bug 12695852
#    ahabbas     06/30/11 - addition of broadcast support
#    shullur     06/28/11 - XbranchMerge shullur_bug-12639016 from main
#    shullur     06/22/11 - For fixing bug 12673460. Remove exits from CHM
#                           upgrade code.
#    shullur     06/20/11 - For fixing upgrade issue for NT/AIX in CHM
#    xyuan       06/15/11 - Fix Bug 12640884
#    xyuan       06/04/11 - Fix bugs 12630162 and 12640551
#    ysharoni    06/02/11 - bug12605752 incomplete gpnp setup requires reconfig
#    xyuan       06/01/11 - Fix bug 12606663
#    shullur     06/01/11 - For fixing Bug 12614795. Handle 11.2.0.1 to 
#                           11.2.0.3 upgrade case.
#    rdasari     05/31/11 - fix set_file_perms
#    xyuan       05/26/11 - Add syncToPhysicalDisk,syncCkptFile,isResTypeRegistered-Bug 12580496
#    ksviswan    05/25/11 - Fix Bugs 12551138,12575120
#    shulur      05/19/11 - For fixing CHM upgrade bug. Bug 11852891.
#    rdasari     05/19/11 - fix olr backup
#    ksviswan    05/17/11 - forceupgrade impl and fix Bug 12550897
#    ksviswan    05/16/11 - Fix Bug 12547838,12537036
#    ysharoni    05/12/11 - Backport ysharoni_rsgetpval11203 from st_has_11.2.0
#    sidshank    05/10/11 - Backport sidshank_bug-12393931 from main
#    rdasari     05/10/11 - do not export checkOCR
#    gmaldona    05/10/11 - add functionality to update OCR wallet
#    jleys       05/04/11 - Backport jleys_bug-12407128 from main
#    minzhu      05/02/11 - Backport minzhu_bug-11881389 from st_has_11.2.0
#    rdasari     04/29/11 - downgrade implementation
#    ysharoni    04/26/11 - bug 12344495, change gpnptool getpval to use
#                           tmpfile (thus ensuring gpnptool-only output)
#    ksviswan    04/26/11 - Additional upgrade error checks. Fix Bugs 12344535,12349600
#    josmith     04/20/11 - Add ACFS replication upgrade callouts
#    ksviswan    04/14/11 - Backport ksviswan_marchbugs2 from main
#    ksviswan    04/13/11 - XbranchMerge ksviswan_opauto_segregate from main
#    jleys       04/11/11 - Backport jleys_bug-11821497 from main
#    shullur     04/10/11 - Backport shullur_bug-12327712 from main
#    dpham       04/06/11 - Backport dpham_bug-9842464 from main
#    agraves     04/05/11 - Backport agraves_bug-11731099 from main
#    rdasari     04/04/11 - add preUpgradeChecks
#    smatpadi    04/04/11 - Backport smatpadi_bug-11071429 from main
#    ksviswan    03/30/11 - Forward merge fix for bug 11670716
#    ksviswan    03/30/11 - Backport ksviswan_marchbugs1 from main
#    gnagiah     03/09/11 - Backport gnagiah_bug-10357090 from main
#    ksviswan    03/08/11 - Backport ksviswan_febbugs2 from main
#    anutripa    03/07/11 - Backport anutripa_bug-10338550 from main
#    sbezawad    03/06/11 - Backport sbezawad_bug-10420872 from main
#    ksviswan    02/03/11 - Backport ksviswan_janbugs4 from main
#    ahabbas     01/28/11 - Backport ahabbas_bug-11077756 from main
#    akhaladk    01/27/11 - Backport akhaladk_bug-11664015 from main
#    ksviswan    01/20/11 - Backport ksviswan_janbugs2 from main
#    jleys       01/19/11 - Backport jleys_bug-10358654 from main
#    dpham       01/19/11 - Backport dpham_bug-9842464 from main
#    dpham       01/19/11 - Backport dpham_bug-10192027 from st_has_11.2.0
#    ksviswan    01/18/11 - Backport ksviswan_janbugs1 from main
#    shullur     01/12/11 - For Enabling CHM on HPUX and AIX.
#    dpham       01/11/11 - Backport dpham_bug-10044370 from st_has_11.2.0
#    dpham       01/06/11 - Backport dpham_bug-10073372 from main
#                         - Backport dpham_bug-10011084 from main
#                         - Backport dpham_bug-10069489 from main
#    dpham       11/29/10 - Fix bug 10252691
#    dpham       11/23/10 - Fix bug 10327539
#    ksviswan    01/14/11 - Fix Bug 9916435,10285176,10036190,10012452
#    dpham       01/13/11 - XbranchMerge dpham_bug-10250703 from st_has_11.2.0
#                         - XbranchMerge dpham_bug10143663_nt from st_has_11.2.0
#    jleys       01/10/11 - Add more comments
#    dpham       01/10/11 - Fix start_GNS() function
#    jleys       01/06/11 - Add comments
#    jleys       01/04/11 - Consolidate old config storage
#    shullur     11/15/10 - For enabling CHM on HPUX and AIX.
#    dpham       12/08/10 - Fix bug 10161844 & 10044370
#    dpham       12/01/10 - fix bug 10119861
#                         - XbranchMerge dpham_bug-10073372 from st_has_11.2.0
#    shullur     11/15/10 - For enabling CHM on HPUX and AIX.
#    dpham       10/27/10 - Rollback dpham_bug-9879090_main txn
#    ggomezb     10/27/10 - XbranchMerge ggomezb_bug-10115294 from
#                           st_has_11.2.0
#    ksviswan    10/24/10 - Fix Bug 10215901
#    ggomezb     10/22/10 - Bug 10115294
#    dpham       09/29/10 - Fix bug 10143662
#    dpham       09/29/10 - Fix bug 10056987
#    dpham       09/21/10 - Fix bug 10133658
#    dpham       09/12/10 - Fix bug 10069852
#    dpham       09/12/10 - Fix bug 10097307
#    ksviswan    08/20/10 - remove clusterguid ckpt property write(lrgisu6d).
#    dpham       08/19/10 - Fix bug 10036834
#    dpham       08/16/10 - Fix bug 10028875
#    ksviswan    08/13/10 - Fix Bug 10027089
#    dpham       08/10/10 - Fix bug 10006758
#    dpham       08/09/10 - Fix bug 9980155
#    ahabbas     08/06/10 - bug#9930739 - disable zeroconf on linux
#    ksviswan    08/04/10 - Fix Bug 9974887,10005400
#    jchandar    08/04/10 - remove oclumon and ologdbg from OTN
#    ksviswan    08/03/10 - Fix Bugs 9962375,9935761
#    gsanders    07/22/10 - change acfs registry group perms to rwx
#    samjo       07/07/10 - Disable CHM/OS in Windows 11.2.0.2.0
#    ksviswan    07/27/10 - Fix Bug 9945857
#    dpham       07/21/10 - Fix bug 9931050
#    dpham       07/21/10 - Fix bug 9926238
#    ksviswan    07/20/10 - Fix Bug 9723529
#    dpham       07/16/10 - Backport dpham_bug-9859193 from main
#    dpham       07/15/10 - Fix bug 9879090
#    dpham       07/14/10 - Fix bug 9900630
#    ysharoni    07/12/10 - bug9857108 check env/prf cname for cfgclone
#    ksviswan    07/12/10 - Fix Bugs 9847255,9876334,9897671
#    dpham       07/08/10 - Add waitForVipRes() function (9888648)
#    dpham       07/07/10 - Bypass writeCkpt() function if it's dev
#    ksviswan    07/07/10 - XbranchMerge ksviswan_11202_bugs12 from main
#    cwang       07/07/10 - XbranchMerge cwang_bug-9799693 from main
#    dpham       07/06/10 - Fix bug 9879012 and 9886787
#                         - Disable oc4j on NT (9886644)
#    dpham       07/05/10 - Fix bug 9859193
#    ksviswan    07/01/10 - Fix Bugs 9710203,9847267,9867442,9847492, 9847519,9831218                  #                           9831218,9851558,9843244.
#    samjo       06/30/10 - Bug 9799693. Add check in configure_ASM
#    dpham       06/30/10 - Add add_local_olr() and set_perms_ocr_vdisk() 
#                           functions (9850696)
#    samjo       06/25/10 - Remove intermediate modifier in ora.asm startup dep
#                           for CRSD
#    minzhu      06/23/10 - not install HAIP if sun cluster
#    dpham       06/22/10 - Add quotes to -diskList (9847217)
#    jleys       06/23/10 - Pass $CFG to CSS_add_vfs
#    ksviswan    06/22/10 - Fix Bug 9839024
#    dpham       06/22/10 - Fix bug 9492881
#    dpham       06/21/10 - Enable/start OC4J resource when upgrade from 
#                           11201 to 11202 (9812188
#    dpham       06/15/10 - Move setperm_vdisks() to rootcrs.pl (9776829)
#    fjlee       06/14/10 - bug9391709 add /etc/rc.d/rc3.d/K15ohasd and change
#                           K19ohasd to K15ohasd
#    tvbabu      06/10/10 - BUG 9776162 deleting the crf ora files from proper
#                           directory
#    ksviswan    06/03/10 - Fix 9773975,9790543,9732945,9785215,9723851,9754914 and 9737629 
#    khsingh     05/21/10 - checkpoint changes
#    minzhu      06/04/10 - add force option for gipcd/gpnpd
#    dpham       06/01/10 - Fix bug 9648820
#    dpham       06/01/10 - Fix bug 9760673
#    spavan      05/29/10 - fix bug9742132 - add cvu resource in upgrade
#    ksviswan    05/27/10 - Fix bugs 9558452 and 9751119
#    ksviswan    05/26/10 - Fix bug 9740970
#    ysharoni    05/26/10 - add get_ocr_getif_info
#    dpham       05/20/10 - Add setperm_vdisks (9705190
#    dpham       05/20/10 - XbranchMerge dpham_bug-9342385 from st_has_11.2.0.1.0
#    papatel     05/20/10 - Bug 9663639 - modify interface string from srvctl
#                           config nodeapps output to have "/" inplace of ":"
#    dpham       05/20/10 - Fix create_dir issue in isFilesystemSupported()
#    dpham       05/19/10 - XbranchMerge dpham_bug9282344 from st_has_11.2.0.1.0
#    ksviswan    05/14/10 - Fix Bugs 9554029,9577992 and 9696538
#    ahabbas     05/13/10 - cleanup haip resource on exclusive startup
#    ksviswan    05/12/10 - Implement logic for 11201 to 11202 SIHA upgrade - Bug 9647774
#    dpham       05/11/10 - Fix bug 9697491, 9697577, and 9684049
#    ahabbas     05/11/10 - bug#9700454 - reorder HAIP startup
#    ahabbas     05/05/10 - addition of HAIP dependencies for ASM
#    dpham       05/05/10 - Fix bug 9652409
#    ysharoni    05/03/10 - bug 9653395 add gpnpd start check
#    akhaladk    05/03/10 - Add evmd dependency on gipcd.
#    dpham       04/28/10 - Fix bug 9415301
#    ksviswan    04/28/10 - Fix for bugs 9561129,9573957,9657576, and 9649100
#    dpham       04/28/10 - Fix bug 9665937
#    dpham       04/27/10 - Add UNLOCK (9446443)
#			  - Add is112ASMExists function (9554590)
#    minzhu      04/26/10 - change gipcd dependency
#    dpham       04/19/10 - Change filter type in ModActionScript() to 
#			    application (9257105)
#    ksviswan    04/22/10 - Fix bugs 9587846,9484786 and 9563139
#    samjo       04/21/10 - Bug 9590202 . Add ora.a.gipcd dep to ora.crsd
#    dpham       04/20/10 - Fix bug 9586429, 9489026
#    dpham       04/19/10 - Change filter type in ModActionScript() to 
#			    application (9257105)
#    dpham       04/19/10 - Fix bug 9471229
#    dpham       04/16/10 - Fix bug 9468735,9586969,9433749,9553888
#    ahabbas     04/14/10 - change default haip type to cluster
#    samjo       01/20/10 - Bug 9129708. Change configure_OCR in SIHA.
#    sukumar     04/12/10 - Fix the startup order, again.
#    dpham       04/02/10 - Add isFilesystemSupported()
#    dpham       03/29/10 - XbranchMerge dpham_bug-9436711 from st_has_11.2.0.1.0
#			  - XbranchMerge dpham_bug-9445489 from st_has_11.2.0.1.0
#    dpham       03/24/10 - Fix bug 9458482, 9481431, 9463649, 9490063
#    dpham       03/24/10 - Fix bug 9481431
#    dpham       03/21/10 - Fix deconfig_ASM() not to return FAILED if
#                           fail to start ora.asm resource (9492994)
#    dpham       03/18/10 - Add ModActionScript() function (9257105) 
#    dpham       03/17/10 - Fix bug 9524959
#    dpham       03/17/10 - Fix RemoveScan()
#    sukumar     03/17/10 - Enable Solaris.
#    dpham       03/11/10 - Add installUSMDriver() function
#    dpham       03/09/10 - Fix bug 9205270
#    ksviswan    03/09/10 - upgrade fixes for ocr/vd on ASM. 11201 to 11202
#                           Fix bugs 9447348, 9454098
#    ksviswan    03/09/10 - Fix bug 9457558
#    sukumar     03/08/10 - Fix double startup of CRF.
#    dpham       03/02/10 - Fix isCRSAlreadyConfigured (9436711)
#    dpham       02/17/10 - Not to append subnet in getHostVIP() when add 
#			    newnode (9375427)
#    ahabbas     02/23/10 - conditional installation of HAIP
#    ahabbas     02/23/10 - ensure cleanup of haip type file
#    sukumar     02/19/10 - Remove START_DEPENDENCIES for ora.crf, move it up.
#    spavan      02/18/10 - fix case of start_CVU
#    dpham       02/17/10 - Fix owner of OLR_LOCATION (9259332)
#    dpham       02/12/10 - Fix add_ons()
#    ksviswan    02/11/10 - Fix Bug 9243302
#    dpham       02/10/10 - Fix check_service() funtion to use 'crsctl stat res'
#    ksviswan    02/08/10 - Fix Bug 8678092
#    jleys       02/01/10 - Remove test code
#    jleys       11/16/09 - Reroute all system cmds through system_cmd_capture
#    dpham       11/05/09 - Increase wait time for crsd and evmd to start
#                           from 24 to 48 (9062113)
#    ksviswan    01/26/09 - Fix bug 8563905
#    dpham       01/25/09 - Fix configNewNode() not to fail if listener already running
#    dpham       02/01/10 - Add add_ons() function (9129615)
#    dpham       01/28/10 - XbranchMerge dpham_bug-9268527 from st_has_11.2.0.1.0
#    agusev      01/22/10 - fix for 9254569
#    dpham       01/15/10 - Fix stop_local_only_stack() to work w/ Windows
#    dpham       01/08/09 - Add isOwnerGroupValid()
#    agusev      01/22/10 - fix for 9254569
#    ksviswan    01/18/10 - Implemented logic for 11.2 patchset upgrade
#    dpham       01/08/10 - Add isOwnerGroupValid()
#			  - Remove unnecessary messages
#    dpham       01/05/09 - Capture outputs in check_OldCrsStack() function
#    sujkumar    01/17/10 - Do not set permissions on symlink in dev env
#    dpham       01/05/10 - Capture outputs in check_OldCrsStack() function
#    ksviswan    01/15/10 - fix bug 8525773
#    dpham       12/21/09 - Fix acfs on windows
#    ksviswan    12/18/09 - export osd s_remove_file
#    dpham       12/15/09 - perform_init_config() function needs to return to caller
#			  - Trace srvctl command during add/start OC4J
#    dpham       12/10/09 - Fix bug 9229309
#    ksviswan    12/09/09 - Fix checkpoint testenv issues
#    sujkumar    12/03/09 - Log CRF failure message only in logs
#    dpham       12/03/09 - XbranchMerge dpham_bug-9146276 from
#                           st_has_11.2.0.1.0
#    dpham       11/30/09 - Change 'root.sh' error msg (9146276
#    dpham       11/29/09 - Fix bug 8873683
#    dpham       11/24/09 - XbranchMerge dpham_bug-8769081 from st_has_11.2.0.1.0
#			  - XbranchMerge dpham_bug9048549 from st_has_11.2.0.1.0
#    ahabbas     11/23/09 - XbranchMerge ahabbas_acfs1 from main
#    ksviswan    11/12/09 - Fix Bug 8525773
#    dpham       11/11/09 - Return a return code from s_configureAllRemoteNodes
#                           to caller
#    sbasu       11/09/09 - autostart OC4J resource for 11.2.0.2
#    diguma      11/04/09 - do not issue stop if resource is already stopped
#    sukumar     11/04/09 - Enable CRF
#    ksviswan    11/04/09 - XbranchMerge ksviswan_autopatch_impl1 from main
#    samjo       11/03/09 - Move IPD OS after CSS
#    samjo       11/03/09 - Disable CRF
#    ahabbas     11/02/09 - addition of haip
#    ksviswan    10/28/09 - Patching Support
#    dpham       10/27/09 - Fix bug 9048549 and bug 9054343
#    dpham       10/15/09 - Add REMOTENODE for Windows
#    diguma      10/15/09 - 8770139: check_service grep
#    dpham       10/12/09 - Call s_getAbsLink to check for OCR symlink (9004597)
#    mimili      10/11/09 - enable evmd in SIHA to replace eonsd 
#                         - add dependencies for cluster mode
#    ahabbas     10/07/09 - only enable ACFS for those tests that use it
#    dpham       10/05/09 - Fix bugs: 8932218,8580937,8892375,8553637,8913585 
#    ksviswan    10/01/09 - CheckPoint Implementation for fresh  Grid Infrastructure install
#    dpham       09/30/09 - Change owner of registry.acfs to "NT AUTHORITY\\SYSTEM" on NT
#    jachang     09/24/09 - Omit setting permissions on voting files
#    ysharoni    09/15/09 - bug 8739811 - gpnp config manifest owner
#    sujkumar    09/22/09 - Add IPD/OS related changes
#    dpham       09/22/09 - XbranchMerge dpham_bug-8889417 from
#                           st_has_11.2.0.1.0
#    ysharoni    09/15/09 - bug 8739811 - gpnp config manifest owner  
#    dpham       09/15/09 - Change OCR file owner to root in an upgrade (8889417)
#    dpham       09/14/09 - XbranchMerge dpham_bug-8761196 from
#    dpham       09/14/09 - XbranchMerge dpham_bug-8484319 from
#                           st_has_11.2.0.1.0
#    dpham       09/02/09 - add add_ITDIR_to_dirs() function (bug 8484319)
#    ysharoni    09/01/09 - remove sticky bit from wallets (bug 8821492)
#    dpham       08/18/09 - Add double-quotes for NT
#    jleys       07/29/09 - XbranchMerge jleys_bug-8667932_11.2.0.1.0_pl from
#    jleys       07/23/09 - Fix for bug 8667932; check CSSD can run realtime
#    ksviswan    07/21/09 - add unlock for Oracle Restart Home
#    dpham       08/21/09 - XbranchMerge dpham_bug-8776078 from
#                           st_has_11.2.0.1.0
#    dpham       08/17/09 - ACFS driver should start as "NT AUTHORITY\\SYSTEM" on NT
#    dpham       08/02/09 - Fix bug 8761196
#    ksviswan    07/30/09 - XbranchMerge ksviswan_bug-8693624 from
#    dpham       07/29/09 - XbranchMerge dpham_bug-8727340 from
#                           st_has_11.2.0.1.0
#    dpham       07/28/09 - Fix split function to work for other languages
#    rwessman    07/27/09 - Bug 8705125
#    ksviswan    07/21/09 - add unlock for Oracle Restart Home
#    dpham       07/20/09 - XbranchMerge dpham_bug8655947 from
#    dpham       07/15/09 - XbranchMerge dpham_bug-8416640_2 from
#                           st_has_11.2.0.1.0
#    dpham       07/15/09 - XbranchMerge dpham_bug-8664938 from main
#    dpham       07/12/09 - fix bug 8416640
#    dpham       07/12/09 - Set owner of ohasd.bin and crsd.bin to root and set its
#                           permissions to '0741'
#    dpham       07/08/09 - Add isPathonASM
#			  - Fix getHostVIP() function (bug 8664269,8659066)
#    sunsridh    07/07/09 - bug 8658959 Let ora.diskmon get pulled up always
#    dpham       07/03/09 - 8656033
#    ksviswan    06/24/09 - Fix Bug 8627151
#    dpham       06/24/09 - Fix logic error in isLastNodeToStart() function
#    dpham       06/17/09 - Always call ASMCA during deconfigure_ASM
#			  - Fix get OCR key in get_OldVipInfoFromOCRDump()
#			  - Fix isCRSAlreadyConfigured
#    dpham       06/10/09 - Set perm '0600' on olr location for SIHA
#			  - Use single-quotes on diskgroup if it contains '$'
#			  - Pass nodes_to_start list for DHCP
#			  - Check proper return code in wait_for_stack_start()
#    dpham       06/05/09 - Allow asm diskgroup contains '$'
#    ysharoni    05/31/09 - bug 8557547 - no empty asm disco str
#    dpham       05/31/09 - Check for "10.1" in getUpgradeConfig
#			  - Add isNodeappsExists()  
#    dpham       05/29/09 - Call get_oifcfg_iflist() from new crshome if unable
#			    to get from old crshome 
#			  - Fix configNode & start_Nodeapps for DHCP (8556760)
#    spavan      05/28/09 - fix bug7562166 - add cvu resource
#    jleys       05/27/09 - Clarify call to prep voting files
#    dpham       05/25/09 - Fix configNewNode for DHCP (bug 8541115)
#    dpham       05/20/09 - Add s_copyOCRLoc
#			  - Export PARAM_FILE_NAME for asmca
#    jleys       05/19/09 - Prepare voting files in upgrade
#    dpham       05/12/09 - Add getUpgradeConfig, configNode, configFirstNode, 
#			    validateNetmask, backupOLR
#    hchau       05/12/09 - Start ctssd resource with env var CTSS_REBOOT=TRUE
#                           when starting the stack
#    dpham       05/11/09 - Add get_OldVipInfoFromOCRDump and export stop_resource
#    vmanivel    05/08/09 - Bug 8258489, removing language references
#    garnaiz     05/05/09 - add new shutdown dependency to cssd
#    dpham       05/05/09 - Convert hostname to lowercase (8489146)
#    dpham       05/04/09 - Fix automerge issue on $$upgrade_ref
#    ksviswan    05/04/09 - Implement downgrade
#    garnaiz     05/01/09 - add subroutine to stop diskmon and capture output
#                           and use it in perform_initial_config
#    samjo       04/30/09 - Bug 8447184. Tolerate ora.asm in INTERMEDIATE state
#                           for ora.crsd
#    dpham       04/29/09 - Add double-quotes on -attr for windows (bug 8339645)
#                         - Separate add/start resources (bug 8462980)
#    ysharoni    05/01/09 - removing incorrect , in optional prf_cif gpnptool
#                           par
#    ysharoni    04/29/09 - bug8466476 fix - no implicit same-mask interconn
#    ysharoni    04/28/09 - replacing get_clusterguid to use std crs ver
#    dpham       04/27/09 - Export s_houseCleaning
#    ksviswan    04/25/09 - Add Patching support
#    hchau       04/22/09 - Bug 8265795. Make crsd, evmd, and asm depends on
#                           ctss
#    dpham       04/22/09 - Add get_OldVipInfoFromOCRDump & isLastNodeToUpgrade
#    dpham       04/21/09 - XbranchMerge dpham_bug-8249129 from
#                           st_has_11.2beta2
#    sbasu       04/20/09 - #8447374:disable OC4J resource
#    dpham       04/20/09 - Remove single quote from LANGUAGE_ID
#    garnaiz     04/16/09 - bug 8438116: explicitly stop diskmon when going
#                           from exclusive to clustered
#    dpham       04/15/09 - Call s_createLocalOnlyOCR for OS-specific
#    ksviswan    04/15/09 - XbranchMerge ksviswan_rootmisc_fixes from
#                           st_has_11.2beta2
#    ksviswan    04/13/09 - Exit if active version change fails during upgrade.
#    ksviswan    04/09/09 - XbranchMerge ksviswan_bug-8408487 from
#                           st_has_11.2beta2
#    garnaiz     04/09/09 - XbranchMerge garnaiz_bug-8413328 from
#                           st_has_11.2beta2
#    agusev      04/09/09 - XbranchMerge agusev_bug-8323709 from main
#    dpham       04/08/09 - XbranchMerge dpham_bug-8412144 from main
#    agusev      04/07/09 - Fix for 8323709
#    ksviswan    04/07/09 - Fix bug 8408487. check if VOTING_DISKS defined prior to use
#    dpham       04/07/09 - Add isASMExists to check if ASM exists during upgrade
#			  - Move checkServiceDown function from crsdelete.pm module
#    dpham       04/02/09 - get OldVipInfo for 10.1
#    samjo       04/01/09 - Bug 7394469. Update ora.crsd dep
#    jleys       04/01/09 - Create lastgasp directory
#    dpham       03/31/09 - ACFS is not supported in SIHA in 11gR2
#    dpham       03/30/09 - Obsolete GetLocalNode. Use $CFG->HOST instead
#    spavan      03/27/09 - install cvuqdisk rpm as part of root
#    dpham       03/27/09 - Ensure ASM_DISKSTRING is defined
#    ksviswan    03/26/09 - get ONS port info for upgrade - Bug 8373077
#    ksviswan    03/24/09 - Add special start nodepps for upgrade 
#    dpham       03/20/09 - Remove OCFS_CONFIG var. It's used only for NT
#    dpham       03/17/09 - Add createLocalOnlyOCR (bug 8353813)
#    seviswan    03/18/09 - Add -oratabLocation switch to configure_ASM
#    dpham       03/17/09 - Remove 'ora.daemon.type' from log directory
#                         - Fix invalid return code check from CSS_start_exclusive
#    agusev      03/12/09 - Fix for 7605771 (cssd perms in SIHA)
#    samjo       03/04/09 - Bug 8218839. Add pullup dep bet CRSD and ASM
#    agraves     03/12/09 - Move usm_root to acfsroot.
#    dpham       03/11/09 - Remove srvctl trace in get_OldVipInfo
#    ksviswan    03/11/09 - Fix configNodeapps for upgrade - Bug 8329144
#    dpham       03/10/09 - Export OCFS_CONFIG variable
#                         - olrlocation permission should be "0600".
#                         - Set OCRCONFIG & OLRCONFIG to null in 
#                           add_olr_ocr_vdisk_locs on Windows
#    ksviswan    03/10/09 - Fix migrate local ocr logic - Bug 8322300
#    dpham       03/09/09 - Set traces in get_oldconfig_info
#    agraves     03/06/09 - Add option for ASMADMIN group name to clscfg
#                           install and upgrade.
#    ysharoni    03/06/09 - temp fix for bug 8258942
#    dpham       03/06/09 - Add ora.registry.acfs resource for non-ASM OCR/VD
#    dpham       03/05/09 - Remove "clscfg -local -l $LANGUAGE_ID" for SIHA.
#     			  - Change ASM_DISCOVER_STRING to ASM_DISKSTRING in deconfigure_ASM
#    jleys       02/06/09 - Remove expansion of VF discovery string
#    ysharoni    03/02/09 - Add get_crs_version,get_ocr_privatenames_info
#    dpham       03/02/09 - Set ora.registry.acfs owner to root
#     			  - Start ora.asm resource during deconfigure ASM
#    jleys       02/06/09 - Remove expansion of VF discovery string
#    ksviswan    03/01/09 - Fix Old VIP info logic for multinode - Bug 8287142
#    sravindh    03/01/09 - Fix logic in validating interface list
#    sunsridh    02/26/09 - Disable ora.diskmon on Win32
#    dpham       02/26/09 - usm_root and ora.drivers.acfs get installed on all nodes
#     			          - ora.registry.acfs resource gets added and started 
#                           on the last node
#    dpham       02/24/09 - Add isInterfaceValid
#    dpham       02/23/09 - Add $OPROCDDIR
#    ksviswan    02/23/09 - Restore ocr.loc and local.ocr - Bug 8280425
#    dpham       02/22/09 - Add isCRSAlreadyConfigured
#    diguma      02/21/09 - Support more post clscfg commands during config
#    ysharoni    02/23/09 - adding prdr wallet
#    dpham       02/20/09 - Set SRVM_TRACE after getHostVIP
#             			  - Fix getHostVIP to check for 'VIP exist' 
#    samjo       02/16/09 - Change 'crsctl changeav' to 'crsctl set crs
#                           activeversion'
#    dpham       02/16/09 - Set ownership of OLRCONFIG to $SUPERUSER
#    dpham       02/14/09 - Rollback ASM_DISKS & ASM_DISCOVER_STRING
#    jleys       02/08/09 - Do not show exclusive failure in msgs
#    sravindh    02/16/09 - Review comments
#    dpham       02/11/09 - Call usm_root on every node
#    dpham       02/11/09 - Validate usm_root to ensure it exists
#    sravindh    02/10/09 - Fix bug 7714358
#    dpham       02/09/09 - Add deconfigure_ASM
#    priagraw    02/06/09 - reorder startup of gipcd
#    dpham       02/05/09 - Fix ORA_CRS_HOME error 
#    dpham       01/29/09 - ocrconfig_loc is not populate properly in 
#                           local_only_config_exists
#    dpham       02/05/09 - Revert setting 'root' ownership on crshome and OCR
#    hchau       02/04/09 - Add RC_START RC_KILL to global vars
#    dpham       02/04/09 - Add isRAC_appropriate to check for rac_on/rac_off
#    ysharoni    02/03/09 - Change cluutil output handling.
#    dpham       01/29/09 - ocrconfig_loc is not populate properly in i
#                           local_only_config_exists
#    dpham       01/29/09 - Export s_set_ownergroup & s_set_perms
#    dpham       01/27/09 - Add unlockCRSHOME
#    dpham       01/22/09 - Skip validate VOTING_DISKS for SIHA
#    dpham       01/21/09 - Add "start ora.registry.acfs"
#                         - Set owner group/permission on cfgtoollogs/crsconfig
#                         - Set owner group/permission on srvmcfg*.log
#    diguma      01/16/09 - move cssdmonitor dependencies to type file
#    ysharoni    01/16/09 - Add nodelst to push gpnp conf in upgrade
#    rsreekum    01/16/09 - export configNewNode
#    jleys       01/08/09 - Use null string for ASM discover string
#    diguma      01/08/09 - changing the auto_start for css in cluster mode
#    sbasu       12/31/08 - fix srvctl tracing for oc4j srvctl commands
#    dpham       01/13/08 - Fix "FAILED TO START NODEAPPS" (7563279
#                         - "add nodeapps" should be called only on 1st node, 
#                           "add vip" should be called on other nodes (7449794
#    dpham       01/13/08 - Change file test to "-e $ocrfile" in validate_SICSS
#    dpham       01/09/08 - Set owner/group of OCR path to root/dba
#    ysharoni    12/25/08 - add netinfo to profile for upgrade
#    dpham       01/07/08 - Fix "FAILED TO FIND EARLIER VERSION DBHOME" 
#    dpham       12/29/08 - Fix CRS_NODEVIP double quotes issue
#    ysharoni    12/25/08 - add netinfo to profile for upgrade
#    ksviswan    12/23/08 - Fix Bug 7561694
#    agraves     12/22/08 - Change usmfs to registry.acfs
#                           Change usm to drivers.acfs.
#    dpham       12/22/08 - Remove double quotes from $nodevip 
#                           when add nodeapps
#    jleys       12/19/08 - Add INIT to list of global variables
#    jleys       12/18/08 - Fix typo in get_oldconfig_info
#    garnaiz     12/16/08 - set diskmon user env variable
#    dpham       01/07/08 - Fix "FAILED TO FIND EARLIER VERSION DBHOME" 
#    jleys       12/10/08 - Remove commented out code and some trace statements
#    jleys       12/10/08 - Use system_cmd for olr_initial_config
#    jleys       12/01/08 - Fix ASM_DISKS quotes problem
#    jleys       11/29/08 - Set delete flag in $CFG
#    jleys       11/26/08 - VAlidate Oracle Home as part of config init
#    jleys       11/25/08 - Fix SIHA validation
#    jleys       11/14/08 - Packagize perl scripts
#    dpham       12/11/08 - Fix ocfs, permissions, start services issues
#    rwessman    12/02/08 - Bug 7587535.
#    rwessman    12/02/08 - Bug 7609364.
#    dpham       12/02/08 - Remove ORA_DBA_GROUP validation
#    sravindh    11/25/08 - Fix from siyarlag for bug 7597160
#    sravindh    11/25/08 - Bugfix 7597160
#    dpham       11/21/08 - Add single quote on ASM_DISCOVERY_STRING
#    dpham       11/20/08 - Return FALSE if unable to exit exclusive mode
#    ksviswan    11/20/08 - fix ExtractVotedisk
#    dpham       11/13/08 - Add createConfigEnvFile for Time Zone  
#    jleys       11/12/08 - Use error instead of trace to write to both stdout
#                           and log
#    dpham       11/10/08 - Create ASM diskgroup as install user
#                         - Start diskgroup on all nodes
#    dpham       11/07/08 - Change usmca to asmca.
#    yizhang     11/07/08 - Fix bug 7539974
#    sbasu       11/05/08 - Add configOC4JContainer() to config/start OC4J containter
#    dpham       11/05/08 - Check for ASM disk group in validate_SICSS
#    jleys       10/31/08 - Remove setting of ORA_CRS_HOME in
#                           olr_initial_config
#    dpham       10/30/08 - Create CRS resource for OCR/Voting disk group.
#    jleys       10/30/08 - Add diagnostic output
#    jleys       10/28/08 - Use system_cmd for start_resource
#    jleys       10/26/08 - Cleanup compile warnings
#    diguma      10/17/08 - 7492916: wrong attributes for SIHA
#    dphami      10/16/08 - On windows, OCR and voting disks should be files not dirs.
#    yizhang     10/27/08 - Fix bug 7509687
#    rwessman    10/27/08 - Bug 7512890.
#    dpham       10/27/08 - configure_ASM should returns $success not $sucess
#                         - remove "!start_resource("ora.asm", "-init")"
#    dpham       10/22/08 - Add call to s_start_ocfs_driver
#    lmortime    10/20/08 - Bug 7279735 - Making "cluster" primary and
#                           "clusterware" an alias
#    ysharoni    10/20/08 - fail in case if wallet creation failed.
#    dpham       10/20/08 - Remove quotes from -diskGroupName and -redundancy of usmca.
#    diguma      10/17/08 - 7492916: wrong attributes for SIHA
#    dpham       10/16/08 - R:\ocr or /ocr should be file not dir.
#    dpham       10/14/08 - Add ExtractVotedisk function
#    rwessman    10/15/08 - Bug 7482219.
#    sunsridh    10/03/08 - Adding diskmon agent
#    jleys       08/19/08 - Convert all use of s_run_as_user to run_as_user
#    jleys       09/24/08 - Do not complete initial config if VF add not
#                           successful
#    dpham       09/30/08 - Add CRS resources for OCR/Voting disk group (bug 6665952).
#                         - Fix 'Resource ora.asm is already running' issue (bug 7423931).
#    jleys       09/29/08 - Remove the CSSD monitor from the SIHA install
#    ppallapo    09/26/08 - Add ocrid, cluster_guid to gpnp profile
#    dpham       09/25/08 - Add new code for DHCP
#    rwessman    09/25/08 - Bug 7428250.
#    dpham       09/22/08 - Set OLR file permission to ORACLE_OWNER/ORA_DBA_GROUP (bug 7411347). 
#    diguma      09/20/08 - 
#    akhaladk    09/17/08 - 
#    rsreekum    09/16/08 - Fix issue with check_service 
#    rwessman    09/16/08 - Bug 7392881.
#    ppallapo    09/15/08 - Change ORACLE_DBA_GROUP to ORA_DB_GROUP
#    dpham       09/09/08 - Always delete olr file if it exists. 
#    akhaladk    09/04/08 - Fix acls
#    yizhang     09/01/08 - 
#    lmortime    08/29/08 - Bug 7279735 - Making "cluster" primary and
#                           "clusterware" an alias
#    ysharoni    08/26/08 - remove Public attr in profile net defn
#    dpham       08/26/08 - add is_dev_env check before calling setParentDir2Root 
#    rxkumar     08/21/08 - fix bug7309465
#    hkanchar    08/14/08 - 
#    dpham       08/13/08 - Set owner/group of ORA_CRS_HOME and its parent dir to root/dba
#    khsingh     08/07/08 - ,
#    dpham       08/07/08 - Add trim function.
#                         - Check for valid ASM_DISCOVERY_STRING 
#    khsingh     08/06/08 - add crs_init_scripts
#    khsingh     08/04/08 - add File::Find
#    hkanchar    07/30/08 - Updage clscfg usage to include language id for OLR
#    dpham       07/28/08 - '-configureLocalASM' for usmca should be used only on the 1st node.
#    dpham       07/23/08 - Fix "NO_VAL" of OCR/VF when ASM is used. 
#    rwessman    07/22/08 - Incorporated review comments from jcreight.
#    ysharoni    07/17/08 - networks list became comma-delimited
#    dpham       07/16/08 - 
#    agusev      07/11/08 - Changed the way crsd being running is determined
#                           for exclusive start
#    jleys       07/11/08 - Put fix fo bug 7159411 back in
#    dpham       07/10/08 - Fix listener on a new node bug 7169845
#                         - Fix netmask/if for new node
#    rwessman    07/01/08 - Added support for GNS.
#    dpham       06/30/08 - add usmfs resource
#    ysharoni    06/30/08 - fix gpnp global/local home recognition pbm
#    hqian       06/20/08 - Files: change owner before changing permissions
#    jleys       06/13/08 - Add hosts to clscfg for Sameer
#    jleys       04/21/08 - Correct merge errors
#    jleys       04/19/08 - Merge updates
#    jleys       04/15/08 - Add new sub for olr in SIHA
#    jleys       04/06/08 - Add diagnostics
#    ysharoni    03/05/08 - fixes for undefined gpnp packg vars
#    jleys       03/01/08 - Add initial config function
#    dpham       06/20/08 - create ocr and olr parent directory
#    gdbhat      06/20/08 - Bug 6054661
#    dpham       06/17/08 - fix date issue (bug 7010382)
#    dpham       06/16/08 - add new node logic
#    jgrout      05/19/08 - realign crsctl commands, fix check_service
#    srisanka    05/12/08 - validate SIHA params
#    ysharoni    05/06/08 - Network info format change
#    srisanka    04/30/08 - Bug 7010382: fix month representation
#    dpham       04/28/08 - Add new subroutines for root deconfig  
#    jleys       04/25/08 - Review comments
#    ysharoni    04/22/08 - internal subroutines converted to normal
#    jleys       04/21/08 - Add comments
#    jleys       04/21/08 - Add function to determine if this is the last node
#    ysharoni    03/31/08 - bug 6895319
#    srisanka    03/19/08 - use trace for all verbose messages
#    ysharoni    03/05/08 - fixes for underfined gpnp packg vars
#    srisanka    02/11/08 - new APIs and fixes for output redirection
#    jgrout      02/12/08 - Fix bug 6607370
#    skakarla    02/07/08 - quoting discovery 
#    srisanka    01/09/08 - separate generic and OSD code
#    ysharoni    12/27/07 - root wallet created by orapki, not mkwallet
#    jgrout      12/20/07 - Fix copy_to_initdir, copy_to_rcdirs
#                            for bug 6678133
#    ysharoni    12/09/07 - add gpnp code
#    srisanka    08/01/07 - Creation
# 

package crsconfig_lib;

use strict;

use vars qw($VERSION @ISA @EXPORT @EXPORT_OK);

$VERSION = '1';

use English;
use Exporter;
use File::Copy;
use File::Path;
use File::Find;
use File::Basename;
use File::Spec::Functions;
use File::Temp qw/ tempfile/;
use Net::Ping;
use Term::ANSIColor;
use Sys::Hostname;
use IO::Handle;
use POSIX qw(tmpnam);
use Carp;
use Socket;
use Env qw(SRVM_TRACE);
use Term::ANSIColor;

use constant ERROR                     => "-1";
use constant FAILED                    =>  "0";
use constant SUCCESS                   =>  "1";
use constant TRUE                      =>  "1";
use constant FALSE                     =>  "0";
use constant WARNING                   =>  "2";
use constant GPNP_SETUP_BAD            => "-1";  # invalid/error
use constant GPNP_SETUP_NONE           =>  '0';  # none
use constant GPNP_SETUP_LOCAL          =>  '1';  # good local setup
use constant GPNP_SETUP_GOTCLUSTERWIDE =>  '2';  # good clusterwide; that just made local
use constant GPNP_SETUP_CLUSTERWIDE    =>  '3';  # good local same as clusterwide
use constant CKPTSUC                   =>  'SUCCESS';
use constant CKPTFAIL                  =>  'FAIL';
use constant CKPTSTART                 =>  'START';
use constant CMDSUC                    =>  '0';
use constant CMDFAIL                   =>  '1';

# How much of the stack do we want to start?
use constant START_STACK_NONE          => 0;
use constant START_STACK_MDNSD         => 1; # Start MDNSD
use constant START_STACK_GPNPD         => 2; # Start GPNPD
use constant START_STACK_GIPCD         => 3; # Start GIPCD
use constant START_STACK_CRF           => 4; # Start CHM/OS
use constant START_STACK_CTSSD         => 5; # Start CTSSD
use constant START_STACK_CSSD          => 6; # Start CSSD
use constant START_STACK_ASM           => 7; # Start ASM 
# Start the rest, EVMD & CRSD cannot be started independtly
use constant START_STACK_ALL           => 8;

# --- gpnp string constants:
use constant GPNP_DIRNAME              => 'gpnp';

use constant GPNP_W_DIRNAME            => 'wallets';
use constant GPNP_W_ROOT_DIRNAME       => 'root';
use constant GPNP_W_PRDR_DIRNAME       => 'prdr';
use constant GPNP_W_PEER_DIRNAME       => 'peer';
use constant GPNP_W_PA_DIRNAME         => 'pa';

use constant GPNP_P_DIRNAME            => 'profiles';
use constant GPNP_P_PEER_DIRNAME       => 'peer';

use constant GPNP_PROFILE_NAME         => 'profile.xml';
use constant GPNP_PROFSAV_NAME         => 'profile_orig.xml';

use constant GPNP_WRL_FILE_PFX         => 'file:';
use constant GPNP_WALLET_NAME          => 'ewallet.p12';
use constant GPNP_SSOWAL_NAME          => 'cwallet.sso';

use constant GPNP_CERT_NAME            => 'cert.txt';
use constant GPNP_CERTRQ_NAME          => 'certreq.txt';
use constant GPNP_RTCERT_NAME          => 'b64certificate.txt';
use constant GPNP_PDUMMY               => 'gpnp_wallet1';

use constant GPNP_W_ROOT_DN            => '"CN=GPnP_root"';
use constant GPNP_W_PA_DN              => '"CN=GPnP_pa"';
use constant GPNP_W_PEER_DN            => '"CN=GPnP_peer"';
use constant GPNP_W_KEYSZ              => '1024';
use constant GPNP_W_EXPDT              => '"01/01/2099"';
use constant GPNP_W_CVALID             => '9999';

our @exp_vars;
our $CFG;

# Becase oracss.pm uses the constants defined here, export in a BEGIN
# block so that they will not cause a compilation failure
BEGIN {
  @ISA = qw(Exporter);

  my @exp_const = qw(TRUE FALSE ERROR FAILED SUCCESS CKPTSUC CKPTFAIL CKPTSTART
                     GPNP_SETUP_BAD GPNP_SETUP_NONE GPNP_SETUP_LOCAL
                     GPNP_SETUP_GOTCLUSTERWIDE GPNP_SETUP_CLUSTERWIDE

                     START_STACK_NONE START_STACK_MDNSD
                     START_STACK_GPNPD START_STACK_GIPCD
                     START_STACK_CTSSD START_STACK_CSSD
                     START_STACK_ALL START_STACK_ASM
                     START_STACK_CRF
                    );

  # The @exp_vars is an array of variables that must be exported, i.e.
  # are global variables.  Do not add to this list
  # The only reason for this list is that there were global variables
  # in OSDs that were difficult to convert to using the object.  No global
  # variables should be added; all new variables should be referenced via
  # the $CFG object

  # temporarely export $g_force, $g_delete, $g_lastnode, $g_downgrade,
  # and $g_version to OSD modules until they are packaged
  our @exp_vars = qw ($CRSCTL
                      $CRS_NODEVIPS
                      $CRS_STORAGE_OPTION
                      $DEBUG
                      $FAILED
                      $g_force
                      $g_deinstall
                      $g_keepdg
                      $g_delete
                      $g_lastnode
                      $g_downgrade
                      $g_version
                      $GNS_ADDR_LIST
                      $GNS_CONF
                      $GNS_DOMAIN_LIST
                      $HAS_GROUP
                      $HAS_USER
                      $HOST
                      $ID
                      $INIT
                      $IT
                      $LANGUAGE_ID
                      $NETWORKS
                      $NODE_NAME_LIST
                      $OCRCONFIG
                      $OCRCONFIGDIR
                      $OCRLOC
                      $OCR_LOCATIONS
                      $OLRCONFIG
                      $OLRCONFIGDIR
                      $OLRLOC
                      $OLR_LOCATION
		      $OPROCDDIR
                      $ORACLE_HOME
                      $ORACLE_OWNER
                      $ORA_CRS_HOME
                      $ORA_DBA_GROUP
                      $PARAM_FILE_PATH
                      $RCALLDIR
                      $RCKDIR
                      $RCSDIR
                      $RC_START
                      $RC_KILL
                      $RC_KILL_OLD
                      $RC_KILL_OLD2
                      $REMOTENODE
                      $SCAN_NAME
                      $SCAN_PORT
                      $SCRBASE
                      $SRVCONFIG
                      $SRVCTL
                      $SRVLOC
                      $SUCCESS
                      $SUPERUSER
                      $UPGRADE
                      $DOWNGRADE
                      $VOTING_DISKS
                      $wipCkptName
                      $isNodeCrashed
                      $isRerun
                      $unlock_crshome
                      $destcrshome
                      $NOCRSSTOP
                     );

  my @exp_osd  = qw(s_redirect_souterr      s_osd_setup s_init_scr
                    s_restore_souterr       s_get_config_key
                    s_check_SuperUser       s_get_platform_family
                    s_ResetOCR              s_CleanTempFiles 
                    s_reset_crshome         s_ResetVotedisks
                    s_set_ownergroup        s_set_perms
                    s_get_olr_file          s_removeCvuRpm
                    s_houseCleaning         s_ResetOLR
                    s_is92ConfigExists      s_removeGPnPprofile
                    s_copyOCRLoc            s_RemoveInitResources
                    s_crf_check_bdbloc      s_crf_remove_itab
                    s_getAbsLink            s_remove_file
                    s_resetParentDirOwner   s_deltOldServ
                    s_removeSCR             s_removeFenceServ
                    s_copyRegKey            s_stopDeltOldASM 
                    s_is_HAIP_supported     s_is_HAIP_NonFatal
                    s_install_initd         s_remove_initd
                    s_CheckNetworkConfig    s_upgrade_services
                    s_unregister_service    s_isSUSELinux
                    s_add_upstart_conf      s_remove_upstart_conf
                    s_is_Linux_Upstart      s_validate_olrconfig
                    s_restoreInitScripts    s_restoreolrloc
                    s_checkolrbackup        s_checkOldCrsOwner
                    s_restoreASMFiles       s_cleanASMFiles
                    s_register_service      s_get_qosctl_path
		    s_deltService
		    s_delete_ASM_Services_downgrade
                    );

  my @exp_func = qw(check_CRSConfig validate_olrconfig validateOCR
                    is_dev_env validate_ocrconfig olr_initial_config
                    copy_file check_SuperUser configLastNode
                    initial_cluster_validation upgrade_OCR ValidateCommand
                    start_resource push_clusterwide_gpnp_setup
                    ExtractVotedisks get_ocrdisk get_ocrmirrordisk 
                    get_ocrloc3disk get_ocrloc4disk get_ocrloc5disk 
                    get_ocrlocaldisk initialize_local_gpnp configure_hasd
                    configure_OCR create_dir create_dirs crs_exec_path
                    perform_initial_config perform_upgrade_config
                    instlststr_to_gpnptoolargs oifcfgiflst_to_instlststr
                    register_service start_service stopOracleRestart check_service
                    tolower_host export_vars setup_param_vars 
                    instantiate_scripts copy_wrapper_scripts set_file_perms
                    trace error backtrace dietrap get_oldconfig_info 
                    get_OldVipInfo get_OldVipInfoFromOCRDump stop_OldCrsStack
                    run_env_setup_modules first_node_tasks check_OldCrsStack
                    isAddNode isLastNodeToStart isLastNodeToUpgrade
                    local_only_config_exists migrate_dbhome_to_SIHA
                    local_only_stack_active stop_local_only_stack
                    start_clusterware run_crs_cmd system_cmd system_cmd_capture
                    wait_for_stack_start configNewNode configureAllRemoteNodes
                    unlockCRSHome unlockHAHome isCkptexist get_crs_version 
                    isRAC_appropriate deconfigure_ASM check_NewCrsStack 
                    isCRSAlreadyConfigured isInterfaceValid
                    configureCvuRpm checkServiceDown update_ons_config
                    getUpgradeConfig configNode backupOLR createLocalOnlyOCR
                    getCurrentNodenameList isNodeappsExists quoteDiskGroup
                    isPathonASM stop_resource getCkptStatus writeCkpt
                    perform_start_cluster perform_initialize_local_gpnp
                    perform_register_service perform_start_service 
                    perform_init_config perform_olr_config
                    perform_configure_hasd perform_configNode
                    remove_checkpoints RemoveScan RemoveNodeApps 
                    crf_config_generate crf_delete_bdb isCRFSupported 
                    removeCHMDB perform_CHM_upgrade getCHMAttrib movedir
                    isCurrentVersion112 isVersionLT11202 get_olsnodes_info
                    crf_do_delete run_as_user run_as_user2 run_as_user3   
                    isOwnerGroupValid system add_ons isVersion112 isVersion111
                    isVersion10 isVersionLT11203 ModActionScript 
                    isFilesystemSupported wait_for_gpnpd_start
                    performpost112upg checkOLR upgsihamodel 
                    queryClusterConfig setperm_vdisks 
                    isONSexist isCkptSuccess writeCkptPropertyFile
                    set_perms_ocr_vdisk read_file isHAIPNonFatal
                    isHAIPsupported  checkPatchforupg
                    isOCRonASM isRolling read_file modifyClusterName
                    isASMRunning start_asm stop_asm
                    unlockCRSHomeforpatch unlockHAHomeforpatch
                    isVersionMatch getcrsrelver isCkptPropertyExists
                    getCkptPropertyValue writeCkptProperty getcrsrelver1
                    isReusedg isFirstNodeToStart isPrereqIgnored
                    system_cmd_capture_noprint checkOCR
                    unlockPatchHome stopClusterware preUpgradeChecks
                    RemoveVotingfiles AddVotingFiles ExtractDiskgroups
                    RemoveVotingDiks AddVotingDisks start_excl_crs
                    stop_crs_check start_crs_check start_nocrs_stack
                    start_crsd_check stop_crsd_check removeCkptFile
                    trim add_localOlr_OlrConfig_OcrConfig
                    preForceUpgradeChecks rscPreChecks
                    set_bold reset_bold InitEnv setup_tfa remove_tfa
                   );

  my @exp_arrays = qw(@crs_init_resources @ns_files @ns_files);

  @EXPORT  = qw($CFG);
  push @EXPORT, @exp_const, @exp_func, @exp_osd, @exp_vars, @exp_arrays;
}

use oracss;
use oraacfs;

# FIXME: These should be moved to crsdelete.pm, which is the place
# they are referenced
our @crs_init_resources = ("ora.evmd","ora.crsd","ora.cssd",
                           "ora.cssdmonitor","ora.gpnpd","ora.gipcd","ora.mdnsd");
our @ns_files = ("CSS","CRS","EVM","PROC","css","crs","evm","proc");
our @crs_nodevip_list_old;
our $srvctl_trc_dir;
our $srvctl_trc_suff = 0;

my %stack_start_levels =
  (
   START_STACK_MDNSD    => 'Oracle clusterware daemons up to MDNSD',
   START_STACK_GPNPD    => 'Oracle clusterware daemons up to GPNPD',
   START_STACK_GIPCD    => 'Oracle clusterware daemons up to GIPCD',
   START_STACK_CRF      => 'Oracle clusterware daemons up to CHM/OS',
   START_STACK_CTSSD    => 'Oracle clusterware daemons up to CTSSD',
   START_STACK_CSSD     => 'Oracle clusterware daemons up to CSSD',
   START_STACK_ASM      => 'Oracle clusterware daemons up to ASM',
   START_STACK_ALL      => 'the Oracle clusterware stack'
   );

# The exported varables are required until the osd layer can adopt
# a package approach
# Do not add to this list, see the comment for @exp_vars
our ($CRSCTL,
     $CRS_NODEVIPS,
     $CRS_STORAGE_OPTION,
     $DEBUG,
     $GNS_ADDR_LIST,
     $GNS_CONF,
     $GNS_DOMAIN_LIST,
     $GPNP_ORIGIN_FILE,
     $HAS_GROUP,
     $HAS_USER,
     $HOST,
     $HOSTNAME,
     $ID,
     $INIT,
     $IT,
     $LANGUAGE_ID,
     $NETWORKS,
     $NODE_NAME_LIST,
     $OCRCONFIG,
     $OCRCONFIGDIR,
     $OCRLOC,
     $OCR_LOCATIONS,
     $OLRCONFIG,
     $OLRCONFIGDIR,
     $OLRLOC,
     $OLR_LOCATION,
     $OPROCDDIR,
     $ORACLE_HOME,
     $ORACLE_OWNER,
     $ORA_CRS_HOME,
     $ORA_DBA_GROUP,
     $RCALLDIR,
     $RCKDIR,
     $RCSDIR,
     $RC_START,
     $RC_KILL,
     $RC_KILL_OLD,
     $REMOTENODE,
     $SCAN_NAME,
     $SCAN_PORT,
     $SCRBASE,
     $SRVCONFIG,
     $SRVCTL,
     $SRVLOC,
     $SUPERUSER,
     $UPGRADE,
     $DOWNGRADE,
     $VOTING_DISKS,
     $wipCkptName,
     $isNodeCrashed,
     $isRerun,
     $unlock_crshome,
     $destcrshome,
     $NOCRSSTOP
    );

my %elements = ( 'SUPERUSER'           => 'SCALAR',
                 'IS_SIHA'             => 'SCALAR',
                 'UPGRADE'             => 'SCALAR',
                 'paramfile'           => 'SCALAR',
                 'osdfile'             => 'SCALAR',
                 'addfile'             => 'SCALAR',
                 'crscfg_trace_file'   => 'SCALAR',
                 'crscfg_trace'        => 'SCALAR',
                 'UNLOCK'              => 'SCALAR',
                 'unlock_crshome'      => 'SCALAR',
                 'destcrshome'         => 'SCALAR',
                 'hahome'              => 'SCALAR',
                 'OLD_CRS_HOME'        => 'INDIRECT', # see %indirect
                 'oldcrshome'          => 'INDIRECT', # see %indirect
                 'oldcrsver'           => 'INDIRECT', # see %indirect
                 'CLSCFG_EXTRA_PARMS'  => 'ARRAY',
                 'oldconfig'           => 'HASH',
                 'params'              => 'HASH',
                 'hosts'               => 'ARRAY',
                 'srvctl_trc_suff'     => 'COUNTER',
                 'DOWNGRADE'           => 'SCALAR',
                 'NOCRSSTOP'           => 'SCALAR',
                 'REMOTENODE'          => 'SCALAR'
               );

# The %indirect hash allows the proper methods to be called from 'new',
# when the initializing hash that is passed to 'new' is not properly
# done.
# The correct way of specifying, for example, 'oldcrsver' and
# 'oldcrshome' in the initializing hash is not:
#    oldcrsver  => $oldcrsver
#    oldcrshome => $oldcrshome
# but is:
#    my @oldcrsversion = split('\.', $oldcrsver);
#    oldconfig  => 
#      {ORA_CRS_VERSION => \@oldcrsversion,
#       ORA_CRS_HOME    => $oldcrshome
#      }
my %indirect = ( 'oldcrsver'    => \&oldcrsver,
                 'oldcrshome'   => \&oldcrshome,
                 'OLD_CRS_HOME' => \&OLD_CRS_HOME
               );

our $TRUE = TRUE;
our $FALSE = FALSE;
our $ERROR = FAILED;
our $SUCCESS = SUCCESS;
our $FAILED = FAILED;
our $FIRSTNODE = "first";
our $LASTNODE = "last";

our $wrapdir_crs; # this var will be used across multiple functions

# OSD API definitions
use s_crsconfig_lib;

# Currently the OSDs are not packaged, so no version will show up
# Once they are packaged, the version can be queried to allow osds
# to be updated on different platforms in separate txns and behavior
# adjusted according to capabilities of the version
our $OSD_VERSION = $s_crsconfig_lib::VERSION;

my @force_upgrade_nodes;

sub export_vars {
  for my $var (@exp_vars) {
    $var =~ s/^\$//;

    # CRSCTL is for OSDs & delete functions
    if ($var eq "CRSCTL") {
      $CRSCTL = crs_exec_path('crsctl');
    }
    elsif ($var eq "SRVCTL") {
      $SRVCTL = crs_exec_path('srvctl');
    }
    elsif ($CFG->defined_param($var)) {
      my $val = $CFG->params($var);
      $val =~ s!\\!\\\\!g; # for Windows
      $val =~ s!\"!\\\"!g; # for Windows
      eval("\$$var = \"$val\"");
    }
    elsif ($CFG->config_value($var)) {
      my $val = $CFG->config_value($var);
      $val =~ s!\\!\\\\!g; # for Windows
      $val =~ s!\"!\\\"!g; # for Windows
      eval("\$$var = \"$val\"");
    }
  }
}

# Define our version of system to ensure proper diagnostics are obtained
# always
sub system {
  return system_cmd(@_);
}

sub run_crs_cmd {
  my $exec = shift;
  my @cmd = (crs_exec_path($exec), @_);

  return system_cmd(@cmd);
}

sub crs_exec_path {
  my ($cfg, $name);
  $cfg = $name = shift;
  if (@_ > 0) { $name = shift; } # called as a method
  else { $cfg = $CFG; }
  return catfile($cfg->ORA_CRS_HOME, 'bin', $name);
}

# If an element is defined as a counter, return incremented value
# Basically, this is the equivalent of ++counter
sub increment_counter {
  my $cfg  = shift;
  my $name = shift;
  my $incr = 1;
  my $ret;
  if (@_) { 
    $incr = shift;
    if (@_) { croak "Too many args to increment_counter $name"; }
  }
  else {
    $ret = $cfg->{$name} + $incr;
    $cfg->{$name} = $ret;
  }

  return $ret;
}

sub start_clusterware {
  my $level = $_[0];
  my $status = SUCCESS;

  trace ("Starting", $stack_start_levels{$level});
  if (
      ($level < START_STACK_MDNSD ||
            start_resource("ora.mdnsd", "-init"))  &&
      ($level < START_STACK_GPNPD ||
            (start_resource("ora.gpnpd", "-init") &&
             wait_for_gpnpd_start()))  &&
      ($level < START_STACK_GIPCD ||
            start_resource("ora.gipcd", "-init"))  &&
      ($level < START_STACK_CRF || !isCRFSupported() ||
            start_resource("ora.crf", "-init"))    &&
      ($level < START_STACK_CSSD || CSS_start_clustered()) &&
      ($level < START_STACK_CTSSD ||
            start_resource("ora.ctssd", "-init", 
                      "-env", "USR_ORA_ENV=CTSS_REBOOT=TRUE"))  &&
      ($level < START_STACK_ASM ||
       !$CFG->ASM_STORAGE_USED  || # if ASM used, start it
            start_resource("ora.asm", "-init"))  &&
      ($level < START_STACK_ALL ||
            (start_resource("ora.crsd", "-init")   &&
             start_resource("ora.evmd", "-init")))
     )
  {
    trace ("Successfully started requested Oracle stack daemons");
  }
  else
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    $status = FAILED;
  }

  return $status;
}

sub isHAIPsupported
{
  if((! $CFG->IS_SIHA) && s_is_HAIP_supported() &&
     (!is_dev_env() || ($ENV{'INSTALL_HAIP'} eq "true"))
    )
  {
    return TRUE;
  }
  else {
    return FALSE;
  }
}

sub isHAIPNonFatal
{
  if( ($ENV{'HAIP_NON_FATAL'} eq "true") || s_is_HAIP_NonFatal() )
  {
    return TRUE;
  }
  else 
  {
    return FALSE;
  }
}


sub perform_start_cluster
{
  start_cluster();
}

sub start_cluster
{
  # Start gpnpd
  if ( ! start_clusterware(START_STACK_GPNPD) )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start Oracle Grid Plug and Play \(GPnP\) component");
  }

  # Upgrade the CSS voting disks. Only valid for upgrades from pre112
  # Note: cssvfupgd can only be executed once during an upgrade, hence only
  # call it if this is the first node to upgrade
  if ($UPGRADE && (! isVersion112()) && (isFirstNodeToUpgrade())) {
      if (!isCkptexist("ROOTCRS_CSSUPG"))
      {
         trace("Writing checkpoint for CSS voting file upgrade");
         writeCkpt("ROOTCRS_CSSUPG", CKPTSTART);
         $wipCkptName = "ROOTCRS_CSSUPG";
      }

      if (!isCkptSuccess("ROOTCRS_CSSUPG"))
      {
         $wipCkptName = "ROOTCRS_CSSUPG";

         if (! CSS_upgrade($CFG) ) {
            writeCkpt("ROOTCRS_CSSUPG", CKPTFAIL);
            die ("CSS voting file upgrade failed");
         } else {
            trace ("CSS voting file upgrade completed");
            writeCkpt("ROOTCRS_CSSUPG", CKPTSUC);
            $wipCkptName = "ROOTCRS_STACK";
         }
      }
  }
  
  # Start CSS in clustered mode
  if ( ! CSS_start_clustered() )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start Cluster Synchorinisation Service in clustered mode");
  }

  # Start CTSS with reboot option to signal step sync
  # Note: Before migrating stack startup to 'crsctl start crs',
  #       'CTSS_REBOOT=TRUE' is a workaround to signal step sync.
  if ( ! start_resource("ora.ctssd", "-init",
			"-env", "USR_ORA_ENV=CTSS_REBOOT=TRUE") )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start Cluster Time Synchronisation Service - CTSS");
  }

  # Start CRF
  if (! isCRFSupported() || ! start_resource("ora.crf", "-init"))
  {
    trace ("start_clster: Failed to start CRF");
    # We don't want CHM/OS failure to stop rest of the stack
    # from coming up. So, no exit here!
  }

  # startup the HAIP if it has been configured 
  if(isHAIPsupported())
    {
      my $crsctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
      my @cmd    = ($crsctl, 'modify', 'res', 
                    'ora.cluster_interconnect.haip', '-attr', 
                    "\"ENABLED=1\"", '-init');
      my $status;
      $status = system_cmd(@cmd);
      if ( 0 != $status ) {
        trace("Failed to enable HAIP statrtup: $status");
      }

      if ( ! start_resource("ora.cluster_interconnect.haip", "-init") ) {
        trace("Failed to startup HAIP");

        # if HAIP is not considered fatal on the platform then disable
      # further startup attempts
      if( isHAIPNonFatal() )
      {
        my @cmd    = ($crsctl, 'modify', 'res', 
                      'ora.cluster_interconnect.haip', '-attr', 
                      "\"ENABLED=0\"", '-init');
        $status = system_cmd(@cmd);
        if( 0 != $status )
        {
          trace("Failed to disable HAIP statrtup: $status");
        }
      }
      else
      {
        die("HAIP startup failure considered fatal, terminating");
      }
    }
  }
  
  # Start ASM if needed. Start if old version is 11.2 and ocr/vd is on ASM
  if ( (($CRS_STORAGE_OPTION == 1) || ($UPGRADE && isVersion112() && isOCRonASM())) && (! start_resource("ora.asm", "-init")) )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start ASM");
  }
  
  if ($DEBUG) { trace("Upgrading OCR..."); }
  if ( ($UPGRADE) && (!upgrade_OCR()) )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to upgrade Oracle Grid Infrastructure Repository");
  }

  # Start CRS
  if ( ! start_resource("ora.crsd", "-init") )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start Cluster Ready Services");
  }

  # Start EVM
  if ( ! start_resource("ora.evmd", "-init") )
  {
    error ("Failed to start Oracle Grid Infrastructure stack");
    die("Failed to start Event Manager");
  }

  if (!wait_for_stack_start(360)) 
  { 
    die("Oracle Grid Infrastructure stack start initiated but failed to complete"); 
  }

  # wait for cluster to come up
  check_service("cluster", 60);
}

###---------------------------------------------------------
#### Function for tracing logging messages for root scripts
# ARGS : 0
sub trace
{
    my ($sec, $min, $hour, $day, $month, $year) =
        (localtime) [0, 1, 2, 3, 4, 5];
    $month = $month + 1;
    $year = $year + 1900;

    if ($CFG && $CFG->crscfg_trace) {
      my $CRSCFG_TRACE_FILE = $CFG->crscfg_trace_file;
      if ($CRSCFG_TRACE_FILE) {
        open (TRCFILE, ">>$CRSCFG_TRACE_FILE")
          or die "trace(): Can't open $CRSCFG_TRACE_FILE for append: $!";
      }
      printf TRCFILE  "%04d-%02d-%02d %02d:%02d:%02d: @_\n",
        $year, $month, $day, $hour, $min, $sec;
      close (TRCFILE);
    } else {
      printf "%04d-%02d-%02d %02d:%02d:%02d: @_\n",
        $year, $month, $day, $hour, $min, $sec;
    }
}

####---------------------------------------------------------
#### Function for dumping errors on STDOUT
# ARGS : 0 
sub error
{
    print "@_\n";

    if ($CFG && $CFG->crscfg_trace && $CFG->crscfg_trace_file) {
        trace (@_);
    }
    if ($DEBUG) {
      trace("###### Begin Error Stack Trace ######");
      backtrace();
      trace("####### End Error Stack Trace #######\n");
    }
}

sub backtrace {
  my $levels = $_[0];
  my $done = FALSE;

  trace(sprintf("    %-15s %-20s %-4s %-10s", "Package", "File",
                "Line", "Calling"));
  trace(sprintf("    %-15s %-20s %-4s %-10s", "-" x 15, "-" x 20,
                "-" x 4, "-" x 10));
  for (my $bt = 1; ((!$levels && !$done) || $bt <= $levels); $bt++) {
    my @caller = caller($bt);
    if (scalar(@caller) == 0) { $done = TRUE; }
    else {
      my $pkg = $caller[0];
      my $file = basename($caller[1]);
      my $line = $caller[2];
      my $sub  = $caller[3];
      trace(sprintf("%2d: %-15s %-20s %4d %s", $bt, $pkg, $file,
                    $line, $sub));
    }
  }
}

sub dietrap {
  trace("###### Begin DIE Stack Trace ######");
  backtrace(0);
  trace("####### End DIE Stack Trace #######\n");
  #wipCkptName is the global variable to store active checkpoint in progress.
  #Needed to handle ctrl-c cases.
  trace ("'$wipCkptName' checkpoint has failed");
  if ($wipCkptName !~ m/^null/i && $wipCkptName ne '') {
     writeCkpt($wipCkptName, CKPTFAIL); 
  }
  die @_;
};

sub print_config
{
  my $cfg = shift;

  my @cfgfiles = ($cfg->paramfile);

  if ($cfg->osdfile && -e $cfg->osdfile) { push @cfgfiles, $cfg->osdfile; }
  if ($cfg->addfile && -e $cfg->addfile) { push @cfgfiles, $cfg->addfile; }

  trace ("### Printing the configuration values from files:");
  for my $file (@cfgfiles) { trace("   $file"); }

  # validates if any value is assigned to the script variables
  for my $key (sort(keys %{$cfg->params})) {
    my $val = $cfg->params($key);
    trace("$key=$val");
  }

  trace ("### Printing other configuration values ###");
  my %cfgh = %{($cfg)};
  for my $key (sort(keys %cfgh)) {
    my $ref = ref($cfg->$key);
    my $val = $cfgh{$key};

    if (!$ref) { trace("$key=$val"); } # scalar
    elsif ($ref eq "ARRAY") { trace("$key=" . join(' ', @{($val)})); }
    elsif ($ref eq "HASH" && $key ne "params" &&
           scalar(keys(%{($val)}))) {
      trace("Printing values from hash $key");
      my %subh = %{($val)};
      for my $hkey (sort(keys(%subh))) {
        trace("  $key key $hkey=$subh{$hkey}");
      }
    }
  }

  trace ("### Printing of configuration values complete ###");

  return;
}

####---------------------------------------------------------
#### Function for checking and returning Super User name
# ARGS : 0
sub check_SuperUser
{
    my $superuser = s_check_SuperUser ()
        or trace("Not running as authorized user");
    return $superuser;
}

####---------------------------------------------------------
#### Function for getting this host name in lower case with no domain name
# ARGS : 0
sub tolower_host
{
    my $host = hostname () or return "";

    # If the hostname is an IP address, let hostname remain as IP address
    # Else, strip off domain name in case /bin/hostname returns FQDN
    # hostname
    my $shorthost;
    if ($host =~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) {
        $shorthost = $host;
    } else {
        ($shorthost,) = split (/\./, $host);
    }

    # convert to lower case
    $shorthost =~ tr/A-Z/a-z/;

    die "Failed to get non-FQDN host name for " if ($shorthost eq "");

    return $shorthost;
}

sub trim
################################################################################
# Function: Remove leading and trailing blanks.
#
# Arg     : string
#
# Return  : trimmed string
################################################################################
{
   my $str = $_;
   $str = shift;
   $str =~ s/^\s+//;
   $str =~ s/\s+$//;
   return $str ;
}

####---------------------------------------------------------
#### Function for validating SIHA installer variables list
# ARGS: 1
# ARG1: Filename in which these parameters are set
sub validateSIHAVarList
{
    my $all_found = SUCCESS;
    my $cfg = shift;
    my $paramfile = $cfg->paramfile;

    # list of params that MUST be specified in the param file
    my @required = ("ORACLE_HOME", "ORACLE_OWNER", "ASM_UPGRADE");

    if ($CFG->platform_family eq 'unix') {
      push @required, "ORA_DBA_GROUP";
    }

    for my $param (@required) {
      if (!$cfg->params($param) || $cfg->params($param) =~ /^%/) {
        error("Required parameter $param not found in $paramfile");
        $all_found = FAILED;
      }
    }

    return $all_found;
}

####---------------------------------------------------------
#### Function for validating installer variables list
# ARGS: 0
#
# This validates parameters by checking to see that all have had proper
# substituion.
# There is probably a list of required parameters that should be
# verified to be specified and reasonable
sub validateCRSVarList
{
  my %params = %{($CFG->params)};
  my $all_set = SUCCESS;
  my @keys   = keys(%params);

  trace ("Checking parameters from paramfile " . $CFG->paramfile .
         " to validate installer variables");

  # validates if any value is assigned to the script variables
  for my $key (sort(@keys)) {
    my $val = $CFG->params($key);
    if ($val =~ /^%/) {
      error ("No value set for the parameter $key. ",
             "Use parameter file ", $CFG->paramfile, " to set values");
      $all_set = FAILED;
    }
  }

  return $all_set;
}

####-----------------------------------------------------------------------
#### Function for performing one-time clusterwide setup
# ARGS: 0
sub first_node_tasks
{
    # if in ADE env, skip these steps and return success
    if (is_dev_env()) {
        return SUCCESS;
    }

    return s_first_node_tasks ();
}

####---------------------------------------------------------
#### Validating OCR locations based on existing ocr settings
# ARGS: 3
# ARG1 : Path for Oracle CRS home
# ARG2 : Cluster name
# ARG3 : Comma separated OCR locations
sub validateOCR
{
    my $crshome = $_[0];
    my $clustername = $_[1];
    my $ocrlocations = $_[2];

    my $status = SUCCESS;

    if (!$crshome) {
        error ("Null value passed as Oracle CRS home");
        return FAILED;
    }

    if (!(-d $crshome)) {
        error ("The file \"$crshome\" does not exist");
        return FAILED;
    }

    trace ("Oracle CRS home = $crshome");

    if (!($clustername)) {
        error ("Null value passed as Oracle Grid Infrastructure name");
        return FAILED;
    }

    trace ("Oracle cluster name = $clustername");

    if (isAddNode($HOST, $CFG->params('NODE_NAME_LIST'))) {
       if (! s_copyOCRLoc()) {
          error ("Unable to copy OCR locations");
          return FAILED;
       }
    } elsif (! $ocrlocations) {
       error ("Null value passed as OCR locations");
       return FAILED;
    }

    trace ("OCR locations = $ocrlocations");
    trace ("Validating OCR");

    # call OSD API
    return s_validateOCR ($crshome, $clustername, $ocrlocations);
}

####---------------------------------------------------------
#### Function for 'TOUCH'ing local.olr file if it does not exist
#    It also validates/sets up OLR config if does not exist
# ARGS: 2
# ARG1 : Complete path of OLR location
# ARG2 : CRS Home
sub validate_olrconfig
{
    my $olrlocation = $_[0];
    my $crshome     = $_[1];
    my $IS_SIHA     = $CFG->IS_SIHA;
    my $ORACLE_OWNER = $CFG->params('ORACLE_OWNER');
    my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');

    if (!$olrlocation) {
        error ("Null value passed for olr location");
        return FAILED;
    }

    if (-e $olrlocation )
    {
       if ((!$CFG->UPGRADE) || ($CFG->UPGRADE && !isVersion112()))
       {
          # delete olr file if exists
          if ($DEBUG) { trace ("unlink ($olrlocation)");}
          unlink ($olrlocation) or error ("Can't delete $olrlocation: $!");
       }
    }

    if (!(-f $olrlocation)) 
    {
       # create an empty file and reset permission
       if ($DEBUG) { trace ("create $olrlocation");}
       open (FILEHDL, ">$olrlocation") or return FAILED;
       close (FILEHDL);

       if ($IS_SIHA) {
          s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $olrlocation)
               or die "Can't change ownership on $olrlocation";
          s_set_perms ("0600", $olrlocation) 
               or die "Can't set permissions on $$olrlocation";
       } else {
          s_set_ownergroup ($CFG->SUPERUSER, $ORA_DBA_GROUP, $olrlocation)
               or die "Can't change ownership on $olrlocation";
          s_set_perms ("0600", $olrlocation) 
               or die "Can't set permissions on $$olrlocation";

       }
    }
    trace ("OLR location = " . $olrlocation);

    if (!$crshome) {
        error ("Null value passed for CRS Home");
        return FAILED;
    }

    if (!(-d $crshome)) {
        error ("The Oracle CRS home path \"$crshome\" does not exist");
    }
    trace ("Oracle CRS Home = " . $crshome);

    # OSD to validate OLR config
    my $rc = s_validate_olrconfig ($olrlocation, $crshome);

    return $rc;
}

####---------------------------------------------------------
#### Function for returning OLR location
# ARGS: 0
sub get_olrdisk
{
    trace ("Retrieving OLR file location");
    return s_get_olrdisk ();
}

####---------------------------------------------------------
#### Function for validating OCR config
# ARGS: 2
# ARG1 : ocrlocations
# ARG2 : isHas
sub validate_ocrconfig
{
    my $ocrlocations = $_[0];
    my $isHas        = $_[1];

    if (!$ocrlocations) {
        error ("Null value passed for olr locations");
        return FAILED;
    }

    trace ("OCR locations = " . $ocrlocations);

    # OSD to validate OCR config
    s_validate_ocrconfig ($ocrlocations, $isHas) or return FAILED;

    return SUCCESS;
}

sub get_ocrdisk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR location
# Args    : none
#-------------------------------------------------------------------------------
{
   trace ("Retrieving OCR main disk location");

   if ($CFG->platform_family ne "windows") {
      my $OCRCONFIG = $CFG->params('OCRCONFIG');
      if (!(-r $OCRCONFIG)) {
         error ("Either " . $OCRCONFIG . " does not exist or is not readable");
         error ("Make sure the file exists and it has read and execute access");
         return $OCRCONFIG;
      }
   }

   return s_get_config_key("ocr", "ocrconfig_loc");
}

sub get_ocrmirrordisk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR mirror location
# Args    : none
#-------------------------------------------------------------------------------
{
   trace ("Retrieving OCR mirror disk location");

   if ($CFG->platform_family ne "windows") {
      my $OCRCONFIG = $CFG->params('OCRCONFIG');
      if (!(-r $OCRCONFIG)) {
         error ("Either " . $OCRCONFIG . " does not exist or is not readable");
         error ("Make sure the file exists and it has read and execute access");
         return $OCRCONFIG;
      }
   }

   return s_get_config_key("ocr", "ocrmirrorconfig_loc");
}

sub get_ocrloc3disk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR location 3
# Args    : none
#-------------------------------------------------------------------------------
{
   trace ("Retrieving OCR loc3 disk location");
   return s_get_config_key("ocr", "ocrconfig_loc3");
}

sub get_ocrloc4disk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR location 4
# Args    : none
#-------------------------------------------------------------------------------
{
   trace ("Retrieving OCR loc4 disk location");
   return s_get_config_key("ocr", "ocrconfig_loc4");
}

sub get_ocrloc5disk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR location 5
# Args    : none
#-------------------------------------------------------------------------------
{
   trace ("Retrieving OCR loc5 disk location");
   return s_get_config_key("ocr", "ocrconfig_loc5");
}

sub get_ocrlocaldisk
#-------------------------------------------------------------------------------
# Function: Retrieving OCR local_only location
# Args    : none
#-------------------------------------------------------------------------------
{
    trace ("Retrieving OCR local_only location");
    return s_get_config_key("ocr", "local_only");
}

sub get_srvdisk
#-------------------------------------------------------------------------------
# Function: Retrieving SRV location
# Args    : none
#-------------------------------------------------------------------------------
{
    trace ("Retrieving SRV location");
    return s_get_config_key("srv", "srvconfig_loc");
}

####---------------------------------------------------------
#### Check if this is a SI CSS configuration
sub validate_SICSS
{
    trace ("Validating for SI-CSS configuration");

    my $ocrfile = get_ocrdisk();

    if (!$ocrfile) {
        trace ("Unable to retrieve ocr disk info");
        return SUCCESS;
    }

    if ($ocrfile =~ /\+/) {
       # return if ocrfile is ASM disk group
       return SUCCESS;
    }
    else {
       if (! (-e $ocrfile)) {
          error ("The file " . $ocrfile . " does not exist");
          return SUCCESS;
       }
    }

    # OCR location already specified. Check if it is used for
    # single instance CSS/ASM
    my $local_flag = s_get_config_key("ocr", "local_only");
    if (!$local_flag) {
        return FAILED;
    }

    # convert to upper-case
    $local_flag =~ tr/a-z/A-Z/;

    trace ("LOCAL_FLAG = " . $local_flag);

    # Previous installation of 10g single instance
    if ($local_flag eq "TRUE") {
        error ("CSS is configured for single instance Oracle databases.");
        error ("Delete this configuration using the command " .
               "'localconfig delete' before proceeding with RAC " .
               "configuration.");
        return FAILED;
    }

    return SUCCESS;
}

####---------------------------------------------------------
#### Check if this is a SI HAS configuration
sub validate_SIHAS
{
    trace ("Validing for SI-HAS configuration");

    my $olrfile = get_olrdisk();

    if (!$olrfile) {
        trace ("olr.loc file does not exist");
        return FALSE;
    }

    if (!(-f $olrfile)) {
        error ("The file " . $olrfile . " does not exist");
        return FALSE;
    }

    # ocr.loc already has a location specified. Check if it is used for
    # single instance CSS/ASM

    my $local_flag = s_get_config_key ("ocr", "local_only");
    if (!$local_flag) {
        return FALSE;
    }

    # convert to upper-case
    $local_flag =~ tr/a-z/A-Z/;

    # Previous installation of 10g single instance
    if ($local_flag eq "TRUE") {
        return TRUE;
    } else {
        return FALSE;
    }
}

####---------------------------------------------------------
#### Function to check if OCR is on ASM
sub isOCRonASM
{
    trace ("Checking if OCR is on ASM");

    my $ocrfile   = get_ocrdisk();
    my $ocrmirror = get_ocrmirrordisk();
    my $ocrloc3   = get_ocrloc3disk();
    my $ocrloc4   = get_ocrloc4disk();
    my $ocrloc5   = get_ocrloc5disk();


    if (!$ocrfile) {
        trace ("OCR config does not exist");
        return FALSE;
    }

    if (($ocrfile =~ /\+/) || ($ocrmirror =~ /\+/) || ($ocrloc3 =~ /\+/) || ($ocrloc4 =~ /\+/) || ($ocrloc5 =~ /\+/)) {
        return TRUE;
    } else {
        return FALSE;
    }
}

####---------------------------------------------------------
#### Function to check if OCR is on ASM
sub isPathonASM
{
   trace ("Checking if given path is on ASM");

   my $diskpath = $_[0];

   if (!$diskpath) {
      trace ("Device path is not specified");
      return FALSE;
   }

   if ($diskpath =~ /\+/) {
      return TRUE;
   } 
   else {
      return FALSE;
   }
}

####---------------------------------------------------------
#### Function for checking if CRS is already configured
# ARGS : 1
# ARG1 : CRS home
# ARG2 : Host name
# ARG3 : CRS user
# ARG4 : isCrsConfigured? (OUT var)
sub check_CRSConfig
{
    my $crshome   = $_[0];
    my $hostname  = $_[1];
    my $crsuser   = $_[2];
    my $dbagroup  = $_[3];
    my $gpnpghome = $_[4];
    my $gpnplhome = $_[5];

    my $crsconfigok = FALSE;
    my $gpnp_setup_type = GPNP_SETUP_BAD;

    # init outers
    $_[6] = $crsconfigok; 
    $_[7] = $gpnp_setup_type; 

    trace ("Oracle CRS home = " . $crshome);

    if (!$hostname) {
        error ("Null value passed for host name");
        return FAILED;
    }

    trace ("Host name = " . $hostname);

    if (!$crsuser) {
        error ("Null value passed for Oracle crs user");
        return FAILED;
    }

    trace ("CRS user = " . $crsuser);

    ## Define gpnp globals and validate gpnp directories.
    # Note: This step must be performed unconditionally,
    #       because successfull script use gpnp globals.
    #
    if (! verify_gpnp_dirs( $crshome, $gpnpghome, $gpnplhome,
                            $hostname, $crsuser, $dbagroup ) ) {
        trace("GPnP cluster-wide dir: $gpnpghome, local dir: $gpnplhome.");
        error ("Bad GPnP setup. Check log; GPnP directories must exist.");
        return FAILED;
    }

    ##Checking if CRS has already been configured
    trace ("Checking to see if Oracle CRS stack is already configured");

    # call OSD API
    if (s_check_CRSConfig ($hostname, $crsuser)) {
        $crsconfigok = TRUE;
    }

    ## GPnP validate existing setup, if any
    #  If a cluster-wide setup found, it will be promoted to local 
    $gpnp_setup_type = check_gpnp_setup( $crshome, 
                                         $gpnpghome, $gpnplhome, $hostname,
                                         $crsuser, $dbagroup ); 

    if ($gpnp_setup_type != GPNP_SETUP_GOTCLUSTERWIDE &&
        $gpnp_setup_type != GPNP_SETUP_CLUSTERWIDE)
    {
      trace ("GPNP configuration required");
      $crsconfigok = FALSE;  # gpnp setup is not ok or not finalized
    }
    $CFG->gpnp_setup_type($gpnp_setup_type);

    # reinit outers
    $_[6] = $crsconfigok; 
    $_[7] = $gpnp_setup_type; 
   return SUCCESS;
}

sub validate_9iGSD
#---------------------------------------------------------------------
# Function: Validating if 9iGSD is up
#
# Args    : none
#---------------------------------------------------------------------
{
   trace ("Checking to see if any 9i GSD is up");

   my $exists = s_checkOracleCM ();

   if (! $exists) {
      return $SUCCESS;
   }

   my $lsdb = catfile($ORA_CRS_HOME, 'bin', 'lsdb');

    open (LSDB, "$lsdb -g|")
        or error ("Can't execute \"" . $lsdb . " -g\" and read output: " . $!);
    my @GSDNODE = <LSDB>;
    close (LSDB);

    my $GSDCHK_STATUS = $?;
    # if GSD running, lsdb will print GSD's node name

    if (($GSDCHK_STATUS != 0) && (@GSDNODE)) {
       my $rootscript = "root.sh";
       if ($CFG->platform_family eq "windows") {
          $rootscript = "gridconfig.bat";
       }

        error ("9i GSD is running on node " . @GSDNODE);
        error ("Stop the GSD and rerun $rootscript");
        return FAILED;
    }

    return SUCCESS;
}

####---------------------------------------------------------
#### Function for setting permissions on CRS home files and directories
#
# ::::::::::TBD:::::::::
# How do we incorporate this function under the new scheme?
# ::::::::::TBD:::::::::
#
# ARGS : 3
# ARG1 : crs home path
# ARG2 : The Oracle owner
# ARG3 : The Oracle DBA group
#
#sub setperm_crshome

####---------------------------------------------------------
# 
sub strrchr { substr($_[0], rindex($_[0], $_[1]) + 1) }

####------ [ GPNP

####---------------------------------------------------------

=head2 get_oifcfg_iflist

  Gets "oifcfg iflist" networks interface info, e.g.
  ("eth0  10.0.0.0  PRIVATE 255.255.252.0", 
   "eth1  140.87.4.0  UNKNOWN 255.255.252.0") 
  Note that adapter name (e.g. eth1) can be quoted and contain spaces
  on some platforms, and ip net addr can be ipv6. 

=head3 Parameters

  string with oifcfg home location. If undef, then current home is used.

=head3 Returns

  =head4 returns a list of strings-net intf defs
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_oifcfg_iflist
{
   return get_oifcfg_info(($_[0], 'iflist','-p','-n'));
}

=head2 get_oifcfg_getif

  Gets "oifcfg getif" networks interface info, e.g.
  ("eth0  10.0.0.0  global  public", 
   "eth1  140.87.4.0  global  cluster_interconnect") 
  Note that adapter name (e.g. eth1) can be quoted and contain spaces
  on some platforms, and ip net addr can be ipv6. 

=head3 Parameters

  string with oifcfg home location. If undef, then current home is used.

=head3 Returns

  =head4 returns a list of strings-net intf defs
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_oifcfg_getif
{
   return get_oifcfg_info(($_[0], 'getif'));
}

=head2 get_oifcfg_info

  Gets oifcfg networks interface info for specified command line params.

=head3 Parameters

  =head4 string with oifcfg home location. If undef, then current home is used.
  =head4 Rest of arguments passed to oifcfg cmdline. 

=head3 Returns

  =head4 returns a list of strings-net intf defs; Warning messages, if any, 
         are filtered out.
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_oifcfg_info
{
   my @intfs = ();

   my ($home, @args) = @_;
   my $cmd;
   if (! defined $home) {
     $cmd = crs_exec_path('oifcfg');
   } else {
     $cmd = catfile( $home, 'bin', 'oifcfg' );
   }
   # run oifcfg asking for intf, net, type and mask
   my @out = system_cmd_capture(($cmd, @args));
   my $rc  = shift @out;

   # read-in interface list 
   if (0 == $rc) {
      trace "---Got oifcfg out ($cmd ".join(' ',@args)."):";
      foreach (0..$#out) {
         my $intf = $out[$_];
         trace $intf ;
         # total failure should return rc, else filter out non-fatal
         # error messages, if any, e.g. "PRIF-nn: error....."
         if ($intf !~ /^PR[A-Z]+-[0-9]+: /) {
           push @intfs, $intf ;
         } else {
           error( $intf );
         }
      }
   } else {
      push @intfs, "$cmd ".join(' ',@args)." failed."; 
   }
   return ($rc, @intfs);
}

=head2 get_olsnodes_info

  Gets olsnodes output for given command line params. 

=head3 Parameters

  string with olsnodes home location. If undef, then current home is used.

=head3 Returns

  =head4 returns a list of strings with node names. Warning messages, if any,
         are filtered out.
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_olsnodes_info
{
   my @nodes = ();

   my ($home, @args) = @_;
   my $cmd;
   if (! defined $home) {
     $cmd = crs_exec_path('olsnodes');
   } else {
     $cmd = catfile( $home, 'bin', 'olsnodes' );
   }

   # run olsnodes w/given pars
   my @out = system_cmd_capture(($cmd, @args));
   my $rc  = shift @out;

   # read-in interface list 
   if (0 == $rc) {
      trace "---Got olsnodes out ($cmd ".join(' ',@args)."):";
      foreach (0..$#out) {
         my $node = $out[$_];
         trace $node ;
         # total failure should return rc, else filter out non-fatal
         # error messages, if any, e.g. "PRCO-nn: error....."
         if ($node !~ /^PR[A-Z]+-[0-9]+: /) {
           push @nodes, $node ;
         } else {
           error( $node );
         }
      }
   } else { 
      push @nodes, "$cmd ".join(' ',@args)." failed.";
   }
   return ($rc, @nodes);
}


=head2 get_ocr_getif_info

  Gets OCR information about configured interfaces in form 
  indentical to "oifcfg getif" output, e.g. line 
   "Local Area Connection 4  140.87.136.0  global  cluster_interconnect,public"
  see get_oifcfg_getif()

  This is used to replace "oifcfg getif" when stack is not running during
  rolling upgrade from older pcw versions.
  OCR access performed with ORA_CRS_HOME ocrdump utility by dumping
  SYSTEM.css.interfaces.global keys.

=head3 Parameters

  string with ocrdump home location. If undef, then current home is used.

=head3 Returns

  =head4 returns a list of strings with "oifcfg getif"-like output.
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_ocr_getif_info
{
   my @intfs = ();

   my $home = $_[0];
   my $cmd;
   if (! defined $home) {
     $cmd = crs_exec_path('ocrdump');
   } else {
     $cmd = catfile( $home, 'bin', 'ocrdump' );
   }
   # run "ocrdump -stdout -keyname SYSTEM.css.interfaces.global"
   my @args = ($cmd, '-stdout', '-keyname', 'SYSTEM.css.interfaces.global');
   my @out = system_cmd_capture(@args);
   my $rc  = shift @out;

   if ($DEBUG) {
     trace "---SYSTEM.css.interfaces.global OCR dump:\n".
           join(' ',@args)."\nout: \n".join("\n",@out)."\n";
   }
   # read-in dumped css keys 
   if (0 == $rc) {
      foreach (0..$#out) {
         my $curidx = $_;
         my $key;
         my $itf;
         my $ada;
         my $net;
         my $typ;

         # e.g. 
         # [SYSTEM.css.interfaces.global.eth0.192|d168|d1|d0.1]
         # ORATEXT : cluster_interconnect,public
         $key = $out[$_];
         $ada = $net = $typ = $itf = undef;
         if ($key =~ m/^\[SYSTEM\.css\.interfaces\.global\.([^\]^\.]+)\.([^\]^\.]+)\.1\]$/) { # take last key - nodename
           $ada = $1;
           $net = $2;
           if (defined $ada && defined $net) {
             $net =~ s/\|d/\./g;  # for ipv4, make ip notation
             $key = $out[$_+1];
             if ( $key =~ m/^ORATEXT : (.+)$/) { # take type classification for current network
               $typ = $1;
               $itf = "$ada  $net  global  $typ";
             }
             if ($DEBUG) {
               trace ("ocr intf parsed: [$curidx] -$ada-$net-$typ-=");
             }
           }
         }
         if (defined $itf) {
           push @intfs, $itf;
         }

      }
      #if ($DEBUG)
      {
        trace ("---ocr iflist:");
        foreach (0..$#intfs) {
          trace ($intfs[$_]);
        }
        trace ("---");
      }
      if (scalar(@intfs) == 0) {
         $rc = 2;
         error "Failed to get a list of interfaces from OCR. ".
               "Setup way not work properly."; 
      }
   } else { 
      push @intfs, "".join(' ', @args)." failed.";
   }
   return ($rc, @intfs);
}


=head2 get_ocr_privatenames_info

  Gets OCR information about configured private nodenames in form 
  indentical to "olsnodes -p" output (nodename private_names...).

  This is used to replace olsnodes-p where not available, e.g. on 10.1.
  OCR access performed with ORA_CRS_HOME ocrdump utility by dumping
  SYSTEM.css keys.
  If all fails, will try "olsnodes" (node names only) in old home as 
  a last resort.

=head3 Parameters

  string with olsnodes home location. If undef, then current home is used.

=head3 Returns

  =head4 returns a list of strings with node names.
  =head4 result code (0 for success) as a first member of array.

=cut

sub get_ocr_privatenames_info
{
   my @nodes = ();

   my $home = $_[0];
   my $cmd;
   if (! defined $home) {
     $cmd = crs_exec_path('ocrdump');
   } else {
     $cmd = catfile( $home, 'bin', 'ocrdump' );
   }

   # run "ocrdump -stdout -keyname SYSTEM.css"
   my @args = ($cmd, '-stdout', '-keyname', 'SYSTEM.css');
   my @out = system_cmd_capture(@args);
   my $rc  = shift @out;

   if ($DEBUG) {
     trace "---SYSTEM.css OCR dump:\n".
           join(' ',@args)."\nout: \n".join("\n",@out)."\n";
   }
   # read-in dumped css keys 
   if (0 == $rc) {
      my @nodnames = grep(/^\[SYSTEM\.css\.node_names\.[^\]\.]+\]$/,   @out);
      my @pvtnames = grep(/^\[SYSTEM\.css\.privatenames\.[^\]\.]+\]$/, @out);

      if (!(@nodnames) || !(@pvtnames)) {
         trace "Warning: OCR has no css public or private node names. ";
      }
      if ($DEBUG) {
         trace "---OCR node_names: ".  join(' ',@nodnames).
              "\n---OCR pvt_names: ".  join(' ',@pvtnames);
      }
      # to keep it simple, we do not do any matching by nodenum between 
      # node_names and privatenames, since they are not used together anyways;
      # assuming same order, same number.
 
      foreach (0..$#nodnames) {
         my $curidx = $_;
         my $nodname = $nodnames[$_];
         if (defined $nodname) {
           $nodname =~ m/^.+\.([^\]\.]+)\]$/; # take last key - nodename
           $nodname = $1;
         }
         # normally both arrays will be paired
         my $pvtname = $pvtnames[$_];
         if (defined $pvtname) {
           $pvtname =~ m/^.+\.([^\]\.]+)\]$/; # take last key - nodename
           $pvtname = $1;
         }
         if (defined $nodname) {
            push @nodes, "$nodname $pvtname";
         }
         trace ("ocr node parsed: -$nodname-$pvtname-="); 
      }
      if (scalar(@nodes) == 0) {
         error "Failed to get a list of CSS nodes from OCR. ".
               "Setup way not work properly."; 

         # return at least a list of nodes (no -p), so setup can propagate
         # properly - run in old home.
         return get_olsnodes_info($CFG->OLD_CRS_HOME);
      }
   } else { 
      push @nodes, "".join(' ', @args)." failed.";
   }
   return ($rc, @nodes);
}


=head2 get_upgrade_netinfo

  This is a top-level call to get a string with network information for 
  upgrade config.
  (This is an analog of NETOWORKS installer interview parameter.)
  Note: OLD_CRS_HOME must be set to use this function.

=head3 Parameters

  None.

=head3 Returns

  returns a string - a comma-separated list of net intf defs, installer-style

=cut

sub get_upgrade_netinfo
{
   my @iflist_out;    # oifcfg iflist results
   my @getif_out;     # oifcfg getif results
   my @olsnodes_out;  # olsnodes -p results
   my @iflist_info;   # parsed oifcfg iflist results
   my @getif_info;    # parsed oifcfg getif results
   my @ols_info;      # parsed olsnodes results
   my @olsif_info;    # parsed olsnodes results matched against oifcfg iflist
   my @net_info;      # consolidated parsed interfaces to use in prf net cfg
   my $rc;
   my $generrmsg = "Cannot get node network interfaces";

   my $OLD_CRS_HOME = $CFG->OLD_CRS_HOME;
   check_dir( $OLD_CRS_HOME ) or die "Old CRS Home directory is invalid.";

   my @OLD_CRS_VERSION = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if (! @OLD_CRS_VERSION) {
     @OLD_CRS_VERSION = get_crs_version($CFG->ORA_CRS_HOME);
   }

   # oifcfg iflist
   ($rc,@iflist_out) = get_oifcfg_iflist($CFG->ORA_CRS_HOME);
   die "$generrmsg (".join(' ',@iflist_out).")" if ($rc!=0);

   # check if old crs is up, if so, do oifcfg getif 
   my $crs_is_up = check_OldCrsStack();
   if ($crs_is_up) {
      # getif must be invoked from old crshome
      ($rc,@getif_out) = get_oifcfg_getif($OLD_CRS_HOME);
      die "$generrmsg (".join(' ',@getif_out).")" if ($rc!=0);
   } else {
     # get getif data from OCR if the stack is down
      ($rc,@getif_out) = get_ocr_getif_info($ORA_CRS_HOME);
      die "$generrmsg (".join(' ',@getif_out).")" if ($rc!=0);
   }

   # olsnodes private addrs
   if ($crs_is_up && (! ($OLD_CRS_VERSION[0] == 10 && $OLD_CRS_VERSION[1] == 1))) {
   
      ($rc,@olsnodes_out) = get_olsnodes_info($OLD_CRS_HOME, '-p');
      die "$generrmsg (".join(' ',@olsnodes_out).")" if ($rc!=0); 
   
   } else {
      # CRS <10.2 does not support olsnodes -p

      if (! $crs_is_up ) {
         trace("old version Grid Infrastructure not running -- will try OCR interconnects instead ");
      } else {
         trace( "\"olsnodes -p\" unavailable in ".
             join('.', @OLD_CRS_VERSION). 
             " -- will try OCR interconnects instead ");
      }

      ($rc,@olsnodes_out) = get_ocr_privatenames_info($ORA_CRS_HOME);
      die "$generrmsg (".join(' ',@olsnodes_out).")" if ($rc!=0); 
   } 

   @iflist_info = parse_netinfo( \@iflist_out ); 
   $rc = shift @iflist_info; 
   die "$generrmsg (failed to parse oifcfg iflist output)" if ($rc!=0); 

   @getif_info = parse_netinfo( \@getif_out ); 
   $rc = shift @getif_info; 
   die "$generrmsg (failed to parse oifcfg getif output)" if ($rc!=0); 

   @ols_info = parse_olsnodesp_netinfo( \@olsnodes_out ); 
   $rc = shift @ols_info; 
   die "$generrmsg (failed to parse olsnodes -p output)" if ($rc!=0); 

   # validate olsnodes info against getif/iflist data
   # first, match to node interfaces 
   @olsif_info = match_node_netintfs( \@ols_info, \@iflist_info );
   $rc = shift @olsif_info; 
   die "$generrmsg (failed to get olsnodes networks)" if ($rc!=0); 

   # then, consolidate oifcfg-getif and olsnodes-p results
   @net_info = match_getif_netintfs( \@getif_info, \@olsif_info );
   $rc = shift @net_info; 
   die "$generrmsg (failed to consolidate network info)" if ($rc!=0); 

   my $s_instiflist = '';
   foreach (0..$#net_info) {
      my $intfref = $net_info[$_];
      $s_instiflist = oifcfg_intf_to_instlststr( $intfref, $s_instiflist );
   }
   trace ("upgrade netlst: \"".$s_instiflist."\""); 

   # convert netinfo into cmdline pars 
   if ($DEBUG) {
     my @netprogram =  instlststr_to_gpnptoolargs( $s_instiflist );
     trace ("upgrade netcmd: \"".join(' ',@netprogram)."\""); 
   }

   # get a comma-separated list of cluster nodes to push cluster-wide
   # gpnp setup to
   my $s_cluster_nodes = get_upgrade_node_list( \@ols_info );
   trace ("upgrade node list: \"".$s_cluster_nodes."\""); 

   return ($s_instiflist, $s_cluster_nodes);
}

=head2 get_upgrade_node_list

  Create a comma-seaparated list of cluster nodes based on parsed oldnodes 
  info. 

=head3 Parameters

  =head4 an array reference of parsed olsnodes-p info (refs to parsed lines).

=head3 Returns

  =head4 returns a comma-separated list of cluster nodes.

=cut

sub get_upgrade_node_list
{
   my $olsref = $_[0];    #ref
   my @olshs = @{$olsref};     # array of parsed olsnodes host/pvthost arrays
   my $s_nodes_list = "";

   foreach (0..$#olshs) {
      my $olsintfref = $olshs[$_];
      my ($node, $pvtnode) = @{$olsintfref};

      if (defined $node) {
         if (! ($s_nodes_list eq "")) {
            $s_nodes_list .= ",";
         }
         $s_nodes_list .= $node;
      }
   }
   if ($DEBUG) {
      trace "Cluster node list, per olsnodes: \"$s_nodes_list\"";
   }
   return $s_nodes_list;
}

=head2 match_node_netintfs

  Create a table of oifcfg-style cluster-interconnect interfaces based on 
  oldnodes -p private node names, resolved and matched against available 
  node interfaces. 

=head3 Parameters

  =head4 an array reference of parsed olsnodes-p info (refs to parsed lines).
  =head4 an array reference of parsed oifcfg-iflist info (refs to parsed lines)

=head3 Returns

  =head4 returns an array of parsed oifcfg-iflist style info (array of refs) 
         for cluster-interconnect interfaces inferred from olsnodes-p output.
  =head4 result code (0 for success) as a first member of array.

=cut

sub match_node_netintfs
{
   my $olsref = $_[0];    #ref
   my $oiflstref = $_[1]; #ref
   my @olshs = @{$olsref};     # array of parsed olsnodes host/pvthost arrays
   my @intfs = @{$oiflstref};  # array of parsed oifcfg iflist arrays
   my @netinfo = ();
   my $rc = 0;

   trace "Processing ".scalar(@olshs)." olsnodes:";
   foreach (0..$#olshs) {
      my $olsintfref = $olshs[$_];
      my $node;
      my $pvtnode;
      my $iaddr;
      my $saddr;
      ($node, $pvtnode) = @{$olsintfref};
      trace "   node $_ pub:$node pvt:$pvtnode=";

      # [ resolve private node into addr 
      if (defined $pvtnode) {
         my $name;
         my $aliases;
         my $addrtype;
         my $length;
         my @addrs;
         ($name, $aliases, $addrtype, $length, @addrs) = 
              gethostbyname $pvtnode
              or die "Can't gethostbyname on $pvtnode: $!";

         (($addrtype == AF_INET) && (scalar(@addrs) > 0)) or next;
         ($length == 4) or error "IPv6 is currently not supported"; # 16

         trace "     $pvtnode addrs: ";
         for my $iiaddr (@addrs) {
            $saddr = undef;
            $iaddr = undef;
            if (defined $iiaddr) {
               $saddr = inet_ntoa( $iiaddr );
               trace "       $saddr ";

               # toberevised: +ipv6 - inet_pton
               $iaddr = ipv4_atol($saddr); 
            }
            if (defined $saddr) {
               # Now loop through the iflist interfaces and see if 
               # node private addr matched for known adapter
               foreach (0..$#intfs) {
                  my $intfref = $intfs[$_];
                  my $ada; 
                  my $net;
                  my $nod;
                  my $typ;
                  my $msk;
                  ($ada, $net, $nod, $typ, $msk ) = @{$intfref};
                  if ((defined $msk) && (defined $net) && (defined $ada)) {

                     # toberevised: +ipv6 - inet_pton
                     my $imask = ipv4_atol($msk); 
                     my $inet  = ipv4_atol($net); 

                     my $match = FALSE;
                     $match = TRUE if (($imask & $iaddr) == $inet);
                     my $unique = TRUE;
                     if ($match) {
                        foreach (0..$#netinfo) {
                           my  $intfref1 = $netinfo[$_];
                           my  $ada1 = @{$intfref1}[0];
                           if ($ada1 eq $ada) {
                              $unique = FALSE; 
                              last;
                           }
                        }
                        if ($unique) { 
                           # make a new cluster_interconnect intf
                           my @intf = ($ada, $net, 'global', 
                                       'cluster_interconnect', $msk );
                           push @netinfo, \@intf; 
                        }
                     }
                     trace("        matching olsnodes/iflist ".
                           "(net $net == $saddr & $msk) match=$match ".
                           "unique=$unique");
                  }
               }
            }
         }
      }
      # ] 
   }
   return ($rc, @netinfo);
}


=head2 match_getif_netintfs

  Create a consolidated table of all used oifcfg-style interfaces based on 
  oifcfg getif info, ammended with resolved oldnodes -p info, if any.
  Resulting netinfo will be used as a NETWORKS info in the gpnp profile.

=head3 Parameters

  =head4 an array reference of parsed oifcfg-getif info (refs to parsed lines).
  =head4 an array reference of parsed oifcfg-olsnodes info from 
         match_node_netintfs (refs to parsed lines)

=head3 Returns

  =head4 returns a merged array of parsed oifcfg-getif style info (array of 
         refs) for available network interfaces.
  =head4 result code (0 for success) as a first member of array.

=cut

sub match_getif_netintfs

{
   my $getifref = $_[0]; #ref
   my $olsifref = $_[1]; #ref
   my @getifs = @{$getifref};  # array of parsed oifcfg getifs arrays
   my @olsifs = @{$olsifref};  # array of parsed olsnodes matched to iflist 
   my @netinfo = ();
   my $rc = 0;

   my $ada; 
   my $net;
   my $nod;
   my $typ;
   my $msk;
   my $getif_pvt_ifs = 0;
   my $getif_pub_ifs = 0;

   # copy getif array into resulting netinfo as base, to be ammended with 
   # olsnodes info
   foreach (0..$#getifs) {
      my $gifref = $getifs[$_];
      ($ada, $net, $nod, $typ, $msk ) = @{$gifref};
      if (defined $ada) {
         my @netif = ($ada, $net, $nod, $typ, $msk );
         push @netinfo, \@netif;  # add unconditionally
 
         # count if types
         if ($typ =~ m/cluster_interconnect/i) {
           $getif_pvt_ifs++;
         }
         if ($typ =~ m/public/i) {
           $getif_pub_ifs++;
         }
      }
   }
   # If getif has no interconnects, try to derive them from 
   # olsnodes/ocr-private-name info:
   # loop through olsnodes interfaces and see if there is something not
   # yet present in results
   #
   # Note: ALL olsifs are cluster_interconnects
   #
   if ( $getif_pvt_ifs == 0 ) {
     foreach (0..$#olsifs) {
        my $olsifref = $olsifs[$_];
        ($ada, $net, $nod, $typ, $msk ) = @{$olsifref};

        my $match = FALSE;
        my $typealt = FALSE;
        if (defined $ada) {

          foreach (0..$#netinfo) {
            my $netifref = $netinfo[$_];
            my $ada_if; 
            my $net_if;
            my $nod_if;
            my $typ_if;
            my $msk_if;
            ($ada_if, $net_if, $nod_if, $typ_if, $msk_if ) = @{$netifref};
            #   0        1         2        3       4
            
            # if a different interface with the same subnet found in results
            # do not add it, just update the type. 
            # (Loose match on subnet, not adapter name if ($ada eq $ada_if))
            #
            if ($net eq $net_if) {
               $match = TRUE;

               # make sure olsnodes type is included in the results type
               # - If there are multiple public types defined, override 
               #   matching interfaces to be cluster_interconnects
               # - If there is only single public, make it dual-purpose
               #
               if ($typ_if !~ m/$typ/i) { 
                 $typealt = TRUE;
                 if ($getif_pub_ifs > 1) {
                   $getif_pub_ifs--;
                   ${$netifref}[3] = $typ; # replace
                 } else {
                   ${$netifref}[3] = $typ_if .= ",$typ"; # append
                 }
               }
               last;
            }
         }
         # if olsnodes interface is not int the results yet, add a copy 
         if (! $match) {
            my @olsif = @{$olsifref};
            push @netinfo, \@olsif;
         }
         trace(" matching olsnodes/getif $ada-$net match=$match ".
               "typealt=$typealt ");
       }
     } # for olsifs
   }
   trace "---resulting upgrade iflist:";
   foreach (0..$#netinfo) {
      my $netifref = $netinfo[$_];
      ($ada, $net, $nod, $typ, $msk ) = @{$netifref};
      trace ("intf $_: -$ada-$net-$nod-$typ-$msk-");  
   }
   trace "---";
   return ($rc, @netinfo);
}

=head2 parse_netinfo

  Parse oifcfg-style netinfo (iflist/getif) into array of refs to array
  for each interface, containing interface definition elements.
  See oifcfg_intf_parse.

=head3 Parameters

  An array reference of strings with oifcfg output.

=head3 Returns

  =head4 returns a resulting array of parsed interfaces, each represented as
         an array containing string components of interface definition 
         (interface name (unquoted), masked addr, 
         scope (global/node), type (public,cluter_interconnect), mask).
  =head4 result code (0 for success) as a first member of array.

=cut

sub parse_netinfo
{
   my $netoutref = $_[0]; #ref
   my @intfs = @{$netoutref};
   my @netinfo = ();
   my $rc = 0;

   foreach (0..$#intfs) {
      my $idef = $intfs[$_];
      my $ada; 
      my $net;
      my $nod;
      my $typ;
      my $msk;
      my @net = oifcfg_intf_parse( $idef );
      ($ada, $net, $nod, $typ, $msk ) = @net;
      if ((defined $typ) && (defined $net) && (defined $ada)) {
        push @netinfo, \@net;
      }
   }
   return ($rc, @netinfo);
}

=head2 parse_olsnodesp_netinfo

  Parse olsnodes-style netinfo into array of refs to array
  for each node info. Each array contains public and private node name.

=head3 Parameters

  An array reference of strings with olsnodes output.

=head3 Returns

  =head4 returns an array of parsed olsnodes node info arrays (public/private
         node name).
  =head4 result code (0 for success) as a first member of array.

=cut

sub parse_olsnodesp_netinfo
{
   my $netoutref = $_[0]; #ref
   my @intfs = @{$netoutref};
   my @netinfo = ();
   my $rc = 0;

   foreach (0..$#intfs) {
      my $idef = $intfs[$_];
      my $n;
      my $host;
      my $pvthost;

      if ($DEBUG)
        { trace ("intf: $idef"); }

      # olsnodes -p will give output in form "<hostname> <privatehonm>" lines.
      $idef =~ s/^\s+|\s+$//g;
      $idef =~ s/\s+/ /g;
      $n    = rindex( $idef, ' ' );
      $host = substr( $idef, 0, $n );

      $pvthost = substr( $idef, $n+1 );

      my @net = ($host, $pvthost);
      if (defined $host) {
        push @netinfo, \@net;
      }
      if ($DEBUG)
        { trace ("node parsed: -$host-$pvthost-="); }
   }
   return ($rc, @netinfo);
}

####---------------------------------------------------------

=head2 ipv4_atol
  Convert a string with decimal dotted ipv4 to network-ordered long
  Note: this is quite similar to inet_aton(); however, it does not tries
  to resolve hostnames as inet_aton does.

=head3 Parameters

  String containing decimal-dotted ipv4 address value.

=head3 Returns

 @returns a network-ordered long ipv4 address value.

=cut

sub ipv4_atol
{
  return unpack('N',pack('C4',split(/\./,shift)));
}

=head2 ipv4_ltoa
  Convert a network-ordered long ipv4 address value to a 
  string decimal dotted ipv4 notation.

=head3 Parameters

  Long containing network-ordered ipv4 address value.

=head3 Returns

 @returns a string dotted-decimal ipv4 address value.

=cut

sub ipv4_ltoa
{
  return inet_ntoa(pack('N',shift));
}

=head2 oifcfg_intf_parse

  Parse a single net interface description produced by oifcfg cmd.
  For example:
  a) oifcfg iflist output:
   "eth0  10.0.0.0  PRIVATE  255.255.252.0", 
   "eth1  140.87.4.0  UNKNOWN  255.255.252.0",
   "Local Area Connection 3  140.87.128.0  PRIVATE"
  b) oifcfg getif output:
   "Local Area Connection 4  140.87.136.0  global  cluster_interconnect,public"

=head3 Parameters

  A string containing oifcfg interface definition, see examples above. 
  (Other strings, such as warnings, etc., must be filtered out.)

=head3 Returns

 @returns an array of interface parameters:
   ($adapter_name, $network, $node_name, $type_list, $mask)
   Where:
   adapter_name is the name of net adapter, unquoted;
   network is a network bits of adapter address;
   node_name is 'global' for cluster-wide config, or node name if node-specific;
   type_list is a comma-separated list of network types (valid values are
             unknown|local|public|private|cluster_interconnect);
   mask is a mask bits;
   If any of the parameters was not defined, undef returned in its place.

=cut

sub oifcfg_intf_parse 
{
   my $idef = $_[0]; 
   my $valid_types = "(local|public|private|unknown|cluster_interconnect)([,](local|public|private|unkown|cluster_interconnect))*";
   my $an_; 
   my $ada; 
   my $net;
   my $nod;
   my $typ;
   my $msk;
   my $iaddr; 
   my $n; 
   if ($DEBUG) 
      { trace ("intf: $idef"); } 
   $idef =~ s/^\s+|\s+$//g; 
   $n   = rindex( $idef, ' ' );
   $an_ = substr( $idef, 0, $n );
   $an_ =~ s/\s+$//; 
 
   $typ = substr( $idef, $n+1 );
   if ($typ !~ m/$valid_types/i) { # if cannot be type, check if mask
      $msk = $typ;
      $typ = undef;

      # validate mask
      $iaddr = ipv4_atol($msk); # toberevised: +ipv6 - inet_pton
      if (! defined $iaddr) {
         $msk = undef;
      } else {
         $msk = ipv4_ltoa($iaddr); # toberevised: +ipv6 - inet_ntop
      }
      $iaddr = undef;
      if (defined $msk)
      {
         $n   = rindex( $an_, ' ' );
         $typ = substr( $an_, $n+1 );
         $an_ = substr( $an_, 0, $n );
         $an_ =~ s/\s+$//; 
      }
   }
   if ($typ !~ m/$valid_types/i) {
      $typ = undef;
   }
   if (defined $typ) {
      $n = rindex( $an_, ' ' );
      if (1 <= $n) {
         $ada = substr( $an_, 0, $n );
         $ada =~ s/\s+$//;
         $net = substr( $an_, ($n+1) );

         # validate address, if not addr, must be scope (nodename/global) 
         $iaddr = ipv4_atol($net); # toberevised: +ipv6 - inet_pton
         if ((! defined $iaddr) || ($iaddr == 0)) {
           $nod = $net;
           $net = undef;
           $n = rindex( $ada, ' ' );
           if (1 <= $n) {
             $net = substr( $ada, ($n+1) );
             $ada = substr( $ada, 0, $n );
             $ada =~ s/\s+$//;
           }
           # validate address
           $iaddr = ipv4_atol($net); #toberevised: +ipv6 -inet_pton
         }
         if ((! defined $iaddr) || ($iaddr == 0)) {
           $net = undef;
         } else {
           $net = ipv4_ltoa($iaddr); # toberevised: +ipv6 - inet_ntop
         }
         $iaddr = undef;
      }
   }
   if ($DEBUG) 
      { trace ("intf parsed: -$ada-$net-$nod-$typ-$msk-="); } 
   return ($ada, $net, $nod, lc($typ), $msk );
}

=head2 oifcfgiflst_to_instlststr

 Create GPnP networks list based on oifcfg info
 Example of output:
 "Local Area Connection 3"/140.87.128.0:public,"Local Area Connection 4"/140.87.136.0:cluster_interconnect|public
 or 
 eth0/10.0.100.0:cluster_interconnect,eth1/140.87.4.0.0:public
 Adaptor name can be quoted. /\<>|"*? are not legal for in adapter name (note,
 spaces or commas can appear). List is space-separated.
 For the sake of "oifcfg iflist" compatibility, UNKNOWN/PRIVATE/LOCAL types
 recognized (UNKNOWN mapped to public, PRIVATE mapped to cluster_interconnect,
 and LOCAL skipped. oifcfg types can be combined (comma-separated) - inst
 list uses | separator, though installer never produces interfaces with
 multiple types.

=head3 Parameters

  Reference to array of oifcfg-style output, e.g.
   ("eth0  10.0.0.0  PRIVATE", "eth1  140.87.4.0  UNKNOWN") 
   ("Local Area Connection 3 140.87.128.0 PUBLIC", i
    "Local Area Connection 4 140.87.136.0 CLUSTER_INTERCONNECT,PUBLIC" )
   
=head3 Returns 

  returns a string with installer-style net info.

=cut

sub oifcfgiflst_to_instlststr
{
   my $intfsref  = $_[0]; # ref
   my @intfs     = @{$intfsref};
   my $s_instiflist = '';
   foreach (0..$#intfs) {
      my $idef = $intfs[$_];
      my @intf = oifcfg_intf_parse( $idef );

      $s_instiflist = oifcfg_intf_to_instlststr( \@intf, $s_instiflist );
   }
   trace ("inst netlst:\"".$s_instiflist."\""); 
   return $s_instiflist;
}

=head2 oifcfgiflst_to_instlststr

 Create GPnP network string based oifcfgi-style info for a single net intf.
 Used by oifcfgiflst_to_instlststr

=head3 Parameters

  =head4 Reference to array of parsed oifcfg-style interface, see 
         oifcfg_intf_parse.
  =head4 A ref to a string containing resulting net info string to append.
   
=head3 Returns 

  returns a string with installer-style net info

=cut

sub oifcfg_intf_to_instlststr
{
   my $idefref  = $_[0]; # ref # ref to parsed interface definition
   my $s_instiflist = $_[1];   # list of NETWORKS, installer-style

   my $ada; 
   my $net;
   my $nod;
   my $typ;
   my $msk;
   ($ada, $net, $nod, $typ, $msk ) = @{$idefref};
   $s_instiflist = '' if (! defined $s_instiflist);

   if ((! defined $typ) || (! defined $net) || (! defined $ada)) {
      return; # bad intf definition
   }
   # For the sake of "oifcfg iflist" compatibility, UNKNOWN/PRIVATE/LOCAL types
   # recognized (UNKNOWN mapped to public, 
   # PRIVATE mapped to cluster_interconnect,
   # and LOCAL skipped. oifcfg types can be combined (comma-separated) - inst
   # list uses | separator, though installer never produces interfaces with
   # multiple types.
   #
   if ($typ =~ m/LOCAL/i) {
      return; # skip "do_not_use" itf
   }
   $typ =~ s/(UNKNOWN|unknown)/public/g; 
   $typ =~ s/(PRIVATE|private)/cluster_interconnect/g; 
   $typ =~ s/,/|/g;  # replace separator

   # tabs, \/|"'*:<>? are normally illegal in adapter names
   if ($ada =~ /[ ,:<>\t\^\(\)\\\/\*\?\|\[\]\+]/) { 
      $ada = '"'.$ada.'"';
   }
   if (!($s_instiflist eq '')) {
      $s_instiflist .= ',';
   }
   $s_instiflist .= $ada.'/'.$net.':'.lc($typ);
   if ($DEBUG) 
      { trace ("inst netlst:\"".$s_instiflist."\""); }
   return $s_instiflist;
}

=head2 instlststr_to_gpnptoolargs
   
 Get list of gpnptool net info params based on current list of
 networks in installer-style network list, see oifcfgiflst_to_instlststr.

=head3 Parameters

  =head4 A string with installer-style networks list.
   
=head3 Returns 

  returns a string with gpnptool profile create/edit net generating params.

=cut

sub instlststr_to_gpnptoolargs
{
   my $networks = $_[0];

   my @intfs = ();
   my @program = ( '-hnet=gen', '-gen:hnet_nm="*"' );

   if ($DEBUG) 
      { trace ("iflist: '".$networks."'"); }

   if( is_dev_env() )
   {
     push( @program, '-haip_bm=multicast');
   }

   #$networks =~ s/^{|}$//g;
   push(@intfs, $+) while $networks =~ m{
       ("[^\"\\]*(?:\\.[^\"\\]*)*"[^,]+)[,]?  # groups def inside quotes
           | ([^,]+)[,]?
           | [,\s]
         }gx;
   push(@intfs, undef) if substr($networks,-1,1) eq '\s';
   if ($DEBUG)
      { trace ("iflist: ".join("\n", @intfs)); }

   my $i=0;
   foreach (0..$#intfs) {
      my $idef = $intfs[$_];
      my $an_;
      my $ada;
      my $net;
      my $typ;
      my $styp;
      my $n;
      if ($DEBUG) 
         { trace ($idef); }
      if ($idef !~ m/(^.+)[:]((public|cluster_interconnect)([|](public|cluster_interconnect))?)$/) {
         error ("Ivalid network type in \"$idef\" - skipped; ".
                "only \"public\" and \"cluster_interconnect\" are allowed.");
         next;
      } else {
         $an_ = $1;
         $typ = $2;
         $typ =~ s/[|]/,/g; # make std list
         $styp = $typ;
         $n = rindex( $an_, '/' );
         if (1 <= $n)
         {
            $ada = substr( $an_, 0, $n );
            $net = substr( $an_, ($n+1) );
         }
         if ($idef =~ m/"([^\"]+)"/) {
            $ada = $1;
         }
         $i++;
         if ($DEBUG) 
            { trace ("$i => '".$ada."','".$net."','".$styp."'"); }
         push( @program, '-gen:net=net'.$i );
         push( @program, '-net'.$i.':net_ip="'.  $net           .'"' );
         push( @program, '-net'.$i.':net_ada="'. $ada           .'"' );
         push( @program, '-net'.$i.':net_use="'. lc($styp)      .'"' );
      }
   }
   if ($DEBUG)
      { trace ("gpnptool pars: ".join(' ', @program)); }
   return @program;
}

####---------------------------------------------------------
#### Package-wide GPnP constants. 
#
# --- constant result codes:
# gpnp setup result

# gpnp global pars
  our $GPNP_CRSHOME_DIR         ;
  our $GPNP_HOST                ;
  our $GPNP_ORAUSER             ;     
  our $GPNP_ORAGROUP            ;

# gpnp directories
  our $GPNP_GPNPHOME_DIR        ;

  our $GPNP_WALLETS_DIR         ;
  our $GPNP_W_ROOT_DIR          ;
  our $GPNP_W_PRDR_DIR        ;
  our $GPNP_W_PEER_DIR          ;
  our $GPNP_W_PA_DIR            ;

  our $GPNP_PROFILES_DIR        ;
  our $GPNP_P_PEER_DIR          ;

  our $GPNP_GPNPLOCALHOME_DIR   ;

  our $GPNP_L_WALLETS_DIR       ;
  our $GPNP_L_W_ROOT_DIR        ;
  our $GPNP_L_W_PRDR_DIR      ;
  our $GPNP_L_W_PEER_DIR        ;
  our $GPNP_L_W_PA_DIR          ;

  our $GPNP_L_PROFILES_DIR      ;
  our $GPNP_L_P_PEER_DIR        ;

# gpnp files
  our $GPNP_W_ROOT_FILE   ;
  our $GPNP_WS_PA_FILE    ;
  our $GPNP_WS_PEER_FILE  ;
  our $GPNP_WS_PRDR_FILE  ;

  our $GPNP_C_ROOT_FILE   ;
  our $GPNP_C_PA_FILE     ;
  our $GPNP_C_PEER_FILE   ;

  our $GPNP_P_PEER_FILE   ;
  our $GPNP_P_SAVE_FILE   ;

  our $GPNP_L_W_ROOT_FILE   ;
  our $GPNP_L_W_PA_FILE     ;
  our $GPNP_L_WS_PA_FILE    ;
  our $GPNP_L_W_PEER_FILE   ;
  our $GPNP_L_WS_PEER_FILE  ;
  our $GPNP_L_WS_PRDR_FILE  ;

  our $GPNP_L_CRQ_PA_FILE   ;
  our $GPNP_L_CRQ_PEER_FILE ;

  our $GPNP_L_C_ROOT_FILE   ;
  our $GPNP_L_C_PA_FILE     ;
  our $GPNP_L_C_PEER_FILE   ;

  our $GPNP_L_P_PEER_FILE   ;
  our $GPNP_L_P_SAVE_FILE   ;

# gpnp peer wrls
  our $GPNP_W_PEER_WRL      ;
  our $GPNP_L_W_PEER_WRL    ;

# gpnp prdr wrls
  our $GPNP_W_PRDR_WRL    ;
  our $GPNP_L_W_PRDR_WRL  ;

# package tools
  our $GPNP_E_GPNPTOOL      ;
  our $GPNP_E_GPNPSETUP     ;

####---------------------------------------------------------
#### Define package-wide GPnP constants. Values validated separately. 
#### This sub MUST be called before any gpnp setup handling takes place.
# ARGS: 6
# ARG1 : Path for Oracle CRS home
# ARG2 : Path for directory containing gpnp dir with a cluster-wide setup
# ARG3 : Path for directory containing gpnp dir with a local setup
# ARG4 : Current Hostname
# ARG5 : OracleOwner user
# ARG6 : OracleDBA group
# @returns SUCCESS or $FAILURE
#
#static
sub define_gpnp_consts
{
    my $crshome    = $_[0];
    my $gpnpdir    = $_[1];
    my $gpnplocdir = $_[2];
    my $host       = $_[3];
    my $orauser    = $_[4];
    my $oragroup   = $_[5];

    # gpnp directories:
    $GPNP_CRSHOME_DIR     = $crshome;
    $GPNP_HOST            = $host;
    $GPNP_ORAUSER         = $orauser;
    $GPNP_ORAGROUP        = $oragroup;

    # -- cluster-wide
    $GPNP_GPNPHOME_DIR    = catdir( $gpnpdir, GPNP_DIRNAME );
    $GPNP_WALLETS_DIR     = catdir( $GPNP_GPNPHOME_DIR, 
                                    GPNP_W_DIRNAME );
    $GPNP_W_ROOT_DIR      = catdir( $GPNP_WALLETS_DIR, 
                                    GPNP_W_ROOT_DIRNAME );
    $GPNP_W_PRDR_DIR    = catdir( $GPNP_WALLETS_DIR, 
                                    GPNP_W_PRDR_DIRNAME );
    $GPNP_W_PEER_DIR      = catdir( $GPNP_WALLETS_DIR, 
                                    GPNP_W_PEER_DIRNAME );
    $GPNP_W_PA_DIR        = catdir( $GPNP_WALLETS_DIR, 
                                    GPNP_W_PA_DIRNAME );
    $GPNP_PROFILES_DIR    = catdir( $GPNP_GPNPHOME_DIR, 
                                    GPNP_P_DIRNAME );
    $GPNP_P_PEER_DIR      = catdir( $GPNP_PROFILES_DIR, 
                                    GPNP_P_PEER_DIRNAME );
    # -- local
    $GPNP_GPNPLOCALHOME_DIR  = catdir( $gpnplocdir, GPNP_DIRNAME, $host );
    $GPNP_L_WALLETS_DIR   = catdir( $GPNP_GPNPLOCALHOME_DIR, 
                                    GPNP_W_DIRNAME );
    $GPNP_L_W_ROOT_DIR    = catdir( $GPNP_L_WALLETS_DIR, 
                                    GPNP_W_ROOT_DIRNAME );
    $GPNP_L_W_PRDR_DIR    = catdir( $GPNP_L_WALLETS_DIR, 
                                    GPNP_W_PRDR_DIRNAME );
    $GPNP_L_W_PEER_DIR    = catdir( $GPNP_L_WALLETS_DIR, 
                                    GPNP_W_PEER_DIRNAME );
    $GPNP_L_W_PA_DIR      = catdir( $GPNP_L_WALLETS_DIR, 
                                    GPNP_W_PA_DIRNAME );
    $GPNP_L_PROFILES_DIR  = catdir( $GPNP_GPNPLOCALHOME_DIR, 
                                    GPNP_P_DIRNAME );
    $GPNP_L_P_PEER_DIR    = catdir( $GPNP_L_PROFILES_DIR, 
                                    GPNP_P_PEER_DIRNAME );
    # gpnp files:

    # -- cluster-wide
    $GPNP_ORIGIN_FILE     = catfile( $GPNP_GPNPHOME_DIR, 'manifest.txt' );
    $GPNP_W_ROOT_FILE     = catfile( $GPNP_W_ROOT_DIR, GPNP_WALLET_NAME );
    $GPNP_WS_PA_FILE      = catfile( $GPNP_W_PA_DIR,   GPNP_SSOWAL_NAME );
    $GPNP_WS_PEER_FILE    = catfile( $GPNP_W_PEER_DIR, GPNP_SSOWAL_NAME );
    $GPNP_WS_PRDR_FILE    = catfile( $GPNP_W_PRDR_DIR, GPNP_SSOWAL_NAME );
    $GPNP_C_ROOT_FILE     = catfile( $GPNP_W_ROOT_DIR, GPNP_RTCERT_NAME );
    $GPNP_C_PA_FILE       = catfile( $GPNP_W_PA_DIR,   GPNP_CERT_NAME );
    $GPNP_C_PEER_FILE     = catfile( $GPNP_W_PEER_DIR, GPNP_CERT_NAME );
    $GPNP_P_PEER_FILE     = catfile( $GPNP_P_PEER_DIR, GPNP_PROFILE_NAME );
    $GPNP_P_SAVE_FILE     = catfile( $GPNP_P_PEER_DIR, GPNP_PROFSAV_NAME );

    # -- local
    $GPNP_L_W_ROOT_FILE   = catfile( $GPNP_L_W_ROOT_DIR, GPNP_WALLET_NAME );
    $GPNP_L_W_PA_FILE     = catfile( $GPNP_L_W_PA_DIR,   GPNP_WALLET_NAME );
    $GPNP_L_WS_PA_FILE    = catfile( $GPNP_L_W_PA_DIR,   GPNP_SSOWAL_NAME );
    $GPNP_L_W_PEER_FILE   = catfile( $GPNP_L_W_PEER_DIR, GPNP_WALLET_NAME );
    $GPNP_L_WS_PEER_FILE  = catfile( $GPNP_L_W_PEER_DIR, GPNP_SSOWAL_NAME );
    $GPNP_L_WS_PRDR_FILE  = catfile( $GPNP_L_W_PRDR_DIR, GPNP_SSOWAL_NAME );
    $GPNP_L_CRQ_PA_FILE   = catfile( $GPNP_L_W_PA_DIR,   GPNP_CERTRQ_NAME );
    $GPNP_L_CRQ_PEER_FILE = catfile( $GPNP_L_W_PEER_DIR, GPNP_CERTRQ_NAME );
    $GPNP_L_C_ROOT_FILE   = catfile( $GPNP_L_W_ROOT_DIR, GPNP_RTCERT_NAME );
    $GPNP_L_C_PA_FILE     = catfile( $GPNP_L_W_PA_DIR,   GPNP_CERT_NAME );
    $GPNP_L_C_PEER_FILE   = catfile( $GPNP_L_W_PEER_DIR, GPNP_CERT_NAME );
    $GPNP_L_P_PEER_FILE   = catfile( $GPNP_L_P_PEER_DIR, GPNP_PROFILE_NAME );
    $GPNP_L_P_SAVE_FILE   = catfile( $GPNP_L_P_PEER_DIR, GPNP_PROFSAV_NAME );

    # gpnp peer wrls
    $GPNP_W_PEER_WRL      =  "".GPNP_WRL_FILE_PFX.$GPNP_W_PEER_DIR;
    $GPNP_L_W_PEER_WRL    =  "".GPNP_WRL_FILE_PFX.$GPNP_L_W_PEER_DIR;

    # gpnp prdr wrls
    $GPNP_W_PRDR_WRL      =  "".GPNP_WRL_FILE_PFX.$GPNP_W_PRDR_DIR;
    $GPNP_L_W_PRDR_WRL    =  "".GPNP_WRL_FILE_PFX.$GPNP_L_W_PRDR_DIR;

    # package tools
    $GPNP_E_GPNPTOOL      = catfile( $crshome, 'bin', 'gpnptool' );
    $GPNP_E_GPNPSETUP     = catfile( $crshome, 'bin', 'cluutil' );

    return SUCCESS;
}

####---------------------------------------------------------
#### Verify directory exists
# ARGS: 1
# ARG1 : Path to check
# @returns SUCCESS or $FAILURE
# static
sub check_dir { 
  my $chkdirnm  = $_[0];
  if (!(defined($chkdirnm ))) {
    error ("Null dirname in setup $chkdirnm");
    return FAILED;
  } 
  if (!(-d $chkdirnm)) {
    error ("The setup directory \"$chkdirnm\" does not exist");
    return FAILED;
  }
  # not checking perms, since they may not be valid for root
  return SUCCESS;
}

####---------------------------------------------------------
#### Verify file exists
# ARGS: 1
# ARG1 : Path to check
# @returns SUCCESS or $FAILURE
# static
sub check_file { 
  my $chkfilenm  = $_[0];
  if (!(defined($chkfilenm))) {
    error ("Null filename in setup");
    return FAILED;
  } 
  if (!(-f $chkfilenm)) {
    trace ("The setup file \"$chkfilenm\" does not exist");
    return FAILED;
  }
  # not checking perms, since they may not be valid for root
  return SUCCESS;
}

####---------------------------------------------------------
#### Copy file from one local location to another
#
# Copies file from one local location to another
# if user/group given, will chown copied file as it
# ARGS: 4
# ARG1 : Source file path
# ARG2 : Destination file path
# ARG3 : User owner to set (or undef)
# ARG4 : Group owner to set (or undef)
# @returns SUCCESS or $FAILURE
#
sub copy_file { 
  my $src = $_[0]; 
  my $dst = $_[1]; 
  my $usr = $_[2]; 
  my $grp = $_[3]; 

  if (! (-f $src)) {
     trace("  $src ? -f failed" );
     return FAILED;
  }
  trace("  copy \"$src\" => \"$dst\"" );
  if (! copy( $src, $dst ))
  {
    error( "Failed to copy \"$src\" to \"$dst\": $!" ); 
    return FAILED;
  }
  # chown to specific user if requested
  if (defined( $usr ) && defined( $grp )) 
  {
    trace("  set ownership on \"$dst\" => ($usr,$grp)" );
    if (FAILED == s_set_ownergroup ($usr, $grp, $dst)) 
    { 
      error( "Failed to set ownership on $dst" ); 
      return FAILED;
    }
  }
  return SUCCESS;
}

####---------------------------------------------------------
#### Check gpnp setup in given home is complete and valid
# Note: osd-type failure (perms, etc.) will cause invalid
#       setup and attempt to recreate local setup later
# static
sub check_gpnp_home_setup { 
  my $islocal = $_[0]; # boolean (TRUE  - local home (node-specific), 
                       #          FALSE - global home (seed)
  my $gpnphome;
  my $gpnp_p_peer;  
  my $gpnp_w_peer;  
  my $gpnp_w_prdr;  
  my $gpnp_wrl_peer;  
  my $gpnp_wrl_prdr;  
  my $orauser = $GPNP_ORAUSER;

  # assign appropriate gpnp home
  if ($islocal) {
    $gpnphome = $GPNP_GPNPLOCALHOME_DIR; # validated
    $gpnp_p_peer = $GPNP_L_P_PEER_FILE;  
    $gpnp_w_peer = $GPNP_L_WS_PEER_FILE;
    $gpnp_wrl_peer = $GPNP_L_W_PEER_WRL;  
    $gpnp_w_prdr = $GPNP_L_WS_PRDR_FILE;
    $gpnp_wrl_prdr = $GPNP_L_W_PRDR_WRL;  
  } else {
    $gpnphome = $GPNP_GPNPHOME_DIR; # validated
    $gpnp_p_peer = $GPNP_P_PEER_FILE;
    $gpnp_w_peer = $GPNP_WS_PEER_FILE;
    $gpnp_wrl_peer = $GPNP_W_PEER_WRL;  
    $gpnp_w_prdr = $GPNP_WS_PRDR_FILE;
    $gpnp_wrl_prdr = $GPNP_W_PRDR_WRL;  
  }

  # check for mandatory peer profile and wallet, 
  my $profile_ok = check_file( $gpnp_p_peer );
  my $wallet_ok  = check_file( $gpnp_w_peer );
  my $rwallet_ok = check_file( $gpnp_w_prdr );

  trace( "chk gpnphome $gpnphome: profile_ok $profile_ok ".
         "wallet_ok $wallet_ok r/o_wallet_ok $rwallet_ok" );

  if (! $profile_ok || ! $wallet_ok ) {
     trace("chk gpnphome $gpnphome: INVALID (bad profile/wallet)");
     return FAILED;
  }
  # now check profile sig against wallet (wallet owner or peer)
  my $rc = run_gpnptool_verifysig( $gpnp_p_peer, $gpnp_wrl_peer, $orauser );
  if ($rc <= 0) {
     trace("chk gpnphome $gpnphome: INVALID (bad profile signature)");
     return FAILED;
  }
  # now check profile sig against r/o wallet
  if (! $rwallet_ok ) {
     trace("chk gpnphome $gpnphome: INCOMPLETE (base gpnp config is ok, but ".
           "gpnp config reader wallet is missing)");
     return FAILED;
  } else {
     # Note: prdr wallet does not have an owner, must be validated against peer
     my $rc = run_gpnptool_verifysig( $gpnp_p_peer, $gpnp_wrl_prdr, 
                                      $orauser );
     if ($rc <= 0) {
        $rwallet_ok = FAILED;
        trace("chk gpnphome $gpnphome: INVALID (base gpnp config is ok, but ".
              "gpnp config reader wallet is invalid - ".
              "does not verify peer profile signature)");
        return FAILED;
     }
  }
  if ( $rwallet_ok ) { # if no errors noticed on r/o wallet 
    trace("chk gpnphome $gpnphome: OK");
  }

  # Now that gpnp setup is verified as valid,
  # get and log profile vitals - cname, cguid, sequence, PA
  # Note: order is significant and used in @pvals array below

  my @ppars = ( '-prf_cn', '-prf_cid', '-prf_sq', '-prf_pa' );
  my @pvals = run_gpnptool_getpval( $gpnp_p_peer, \@ppars, $orauser );
  $rc = shift @pvals;
  if ($rc < 0) {
     trace("chk gpnphome $gpnphome: INVALID ".
           "(failed to get vital profile parameters)");
     return FAILED;
  }
  # verify profile cname with cname set in script environment, 
  # make sure they are the same.
  my $p_cname = shift @pvals;  # 1st par in @ppars array above
  my $t_cname  = $CFG->params('CLUSTER_NAME');
  if (! ($p_cname eq $t_cname)) {
     trace("chk gpnphome $gpnphome: INVALID ".
           "(target clustername \"$t_cname\" is different from ".
           "\"$p_cname\" in existing gpnp profile \"$gpnp_p_peer\")");
     return FAILED;
  }

  # make sure profile permissions are correct, ignore res
  gpnp_wallets_set_ownerperm($islocal);  # error(s) logged 

  return SUCCESS;
}

####---------------------------------------------------------
#### Define and verify GPnP local/cluster-wide gpnp directories 
#### This sub MUST be called before any gpnp setup handling takes place.
# ARGS: 6
# ARG1 : Path for Oracle CRS home
# ARG2 : Path for directory containing gpnp dir with a cluster-wide setup
# ARG3 : Path for directory containing gpnp dir with a local setup
# ARG4 : Hostname, must be given
# ARG5 : OracleOwner user
# ARG6 : OracleDBA group
# @returns SUCCESS or $FAILURE
#
#static
sub verify_gpnp_dirs
{
    my $crshome    = $_[0];
    my $gpnpdir    = $_[1];
    my $gpnplocdir = $_[2];
    my $host       = $_[3];
    my $orauser    = $_[4];
    my $oragroup   = $_[5];

    #-------------
    # Check pars

    if (!$crshome) {
        error ("Empty path specified for Oracle CRS home");
        return FAILED;
    }
    if (!(-d $crshome)) {
        error ("The Oracle CRS home path \"" . $crshome . "\" does not exist");
        return FAILED;
    }
    trace ("Oracle CRS home = " . $crshome);
    if (!$host) {
        error ("Hostname is required for GPnP setup");
        return FAILED;
    }
    trace ("GPnP host = " . $host);

    check_dir( $gpnpdir ) or return FAILED;
    check_dir( $gpnplocdir ) or return FAILED;

    # define package-wide dir names on validated params
    define_gpnp_consts( $crshome, $gpnpdir, $gpnplocdir, $host,
                        $orauser, $oragroup ) 
      or return FAILED;

    trace ("Oracle GPnP home = $GPNP_GPNPHOME_DIR");
    trace ("Oracle GPnP local home = $GPNP_GPNPLOCALHOME_DIR");

    # Check defined const dirs:
    # 1) mandatory
    check_dir( $GPNP_GPNPHOME_DIR ) or return FAILED;
    check_dir( $GPNP_W_PEER_DIR ) or return FAILED;
    check_dir( $GPNP_P_PEER_DIR ) or return FAILED;

    check_dir( $GPNP_GPNPLOCALHOME_DIR ) or return FAILED;
    check_dir( $GPNP_L_W_PEER_DIR ) or return FAILED;
    check_dir( $GPNP_L_P_PEER_DIR ) or return FAILED;

    # 2) optional
    check_dir( $GPNP_W_ROOT_DIR );
    check_dir( $GPNP_W_PA_DIR );
    check_dir( $GPNP_L_W_ROOT_DIR );
    check_dir( $GPNP_L_W_PA_DIR );

    trace("GPnP directories verified. ");
    return SUCCESS;
}

####---------------------------------------------------------
#### Verify GPnP local/cluster-wide file setup (wallet(s)/profiles)
#### Note: verify_gpnp_dirs must be called prior calling this function
# All parameters validated elsewhere
# ARGS: 6
# ARG1 : Path for Oracle CRS home
# ARG2 : Path for directory containing gpnp dir with a cluster-wide setup
# ARG3 : Path for directory containing gpnp dir with a local setup
# ARG4 : Current Hostname
# ARG5 : OracleOwner user
# ARG6 : OracleDBA group
# @returns 
#  GPNP_SETUP_BAD   - if local setup is bad/inconsistent, or error of some
#                      kind occured - local setup must be created
#  GPNP_SETUP_NONE  - if local setup must be created
#  GPNP_SETUP_LOCAL - if local setup already valid, but not cluster-wide
#                      (i.e. there is no cluster-wide setup found;
#                      -x, if succeeded, must push the setup)
#  GPNP_SETUP_GOTCLUSTERWIDE
#                      if local setup is valid, and was just promoted
#                      from a valid cluster-wide setup
#  GPNP_SETUP_CLUSTERWIDE
#                      if local setup is valid, and matches cluster-wide
# 
sub check_gpnp_setup
{
    my $crshome    = $GPNP_CRSHOME_DIR; # validated
    my $gpnpdir    = $GPNP_GPNPHOME_DIR; # validated
    my $gpnplocdir = $GPNP_GPNPLOCALHOME_DIR; # validated
    my $host       = $GPNP_HOST;
    my $orauser    = $GPNP_ORAUSER;
    my $oragroup   = $GPNP_ORAGROUP;

    my $rc      = 0;
    my @program ;

    # 1) Make sure global (seed) and local (node-specific) gpnp dirs
    #    are distinct
    if ($GPNP_GPNPLOCALHOME_DIR eq $GPNP_GPNPHOME_DIR)
    {
      error( "Invalid GPnP home locations: "
            ."cluster-wide \"$GPNP_GPNPHOME_DIR\", "
            ."node-specific \"$GPNP_GPNPLOCALHOME_DIR\". "
            ."Must be different." );
    }
    # 2) check local setup exists and valid
    trace( "---Checking local gpnp setup...");
    my $gpnploc_valid = check_gpnp_home_setup( TRUE );

    # 3) check cluster-wide setup exists and valid
    trace( "---Checking cluster-wide gpnp setup...");
    my $gpnp_valid = check_gpnp_home_setup( FALSE );

    trace( "gpnp setup checked: local valid? $gpnploc_valid ".
           "cluster-wide valid? $gpnp_valid" );

    # 4) see if we can assume cluster-wide setup, return current 
    #    type of gpnpsetup accordingly
    #
    if ( $gpnp_valid && $gpnploc_valid) {

      # if both setups valid, check local setup verifies against
      # cluster-wide wallet (wallet owner or peer)
      $rc = run_gpnptool_verifysig( $GPNP_L_P_PEER_FILE, 
                                    $GPNP_W_PEER_WRL, $orauser );
      if ($rc <= 0) {
        trace("Failed to veirfy a local peer profile \"$GPNP_L_P_PEER_FILE\" ".
              "against cluster-wide wallet \"$GPNP_W_PEER_WRL\" ".
              "rc=$rc (0==invalid,<0==error).\n".
              "Will try to take a cluster-wide setup." );

        # promote cluster-wide setup 
        if (take_clusterwide_gpnp_setup())
        {
          trace( "gpnp setup: GOTCLUSTERWIDE" );
          return GPNP_SETUP_GOTCLUSTERWIDE;
        } 
        else # copy was not successfull - stick with local setup
        { 
          trace( "Failed to copy cluster-wide setup.\n".
                 "gpnp setup: LOCAL" );
          return GPNP_SETUP_LOCAL;
        }
      } else {
          trace( "Local and Cluster-wide setups signed with same wallet.\n".
                 "gpnp setup: CLUSTERWIDE" );
        return GPNP_SETUP_CLUSTERWIDE; # identical setups
      }
    } elsif ( $gpnp_valid ) { # cluster-wide setup only, just try to take that
      if (take_clusterwide_gpnp_setup())
      {
        trace( "gpnp setup: GOTCLUSTERWIDE" );
        return GPNP_SETUP_GOTCLUSTERWIDE;
      } 
      else # copy was not successfull - no good setup or no setup
      { 
        trace( "Failed to copy cluster-wide setup.\n".
               "gpnp setup: BAD" );
        return GPNP_SETUP_BAD;
      }
    } elsif ( $gpnploc_valid ) { # local setup only
      trace( "gpnp setup: LOCAL" );
      return GPNP_SETUP_LOCAL;
    } else {
      trace( "gpnp setup: NONE" );
      return GPNP_SETUP_NONE;
    }  
    return GPNP_SETUP_BAD; # neverreached
}


####---------------------------------------------------------
#### Copy cluster-wide GPnP file setup to be node-local
#### (Copy local wallet(s)/profiles from global stage area on current node)
#
# NOTE:  for use by check_gpnp_setup() 
#
# @returns SUCCESS or $FAILURE
#
#static
sub take_clusterwide_gpnp_setup
{
    my $crshome    = $GPNP_CRSHOME_DIR; # validated
    my $gpnpdir    = $GPNP_GPNPHOME_DIR; # validated
    my $gpnplocdir = $GPNP_GPNPLOCALHOME_DIR; # validated
    my $usr        = $GPNP_ORAUSER;
    my $grp        = $GPNP_ORAGROUP;

    # copy cluster-wide setup files
    trace("Taking cluster-wide setup as local");

    # mandatory
    my $status = 
    copy_file( $GPNP_P_PEER_FILE,  # peer profile
               $GPNP_L_P_PEER_FILE, 
               $usr, $grp );
    if ($status == SUCCESS) { $status = 
    copy_file( $GPNP_WS_PEER_FILE, # peer wallet
               $GPNP_L_WS_PEER_FILE, 
               $usr, $grp ); }

    # optional
    if ($status == SUCCESS) { 
      copy_file( $GPNP_WS_PRDR_FILE, # prdr wallet
                 $GPNP_L_WS_PRDR_FILE, 
                 $usr, $grp ); 

      copy_file( $GPNP_P_SAVE_FILE,  # saved profile
                 $GPNP_L_P_SAVE_FILE, 
                 $usr, $grp ); 

      copy_file( $GPNP_W_ROOT_FILE,  # root wallet
                 $GPNP_L_W_ROOT_FILE, 
                 $usr, $grp );
      copy_file( $GPNP_WS_PA_FILE,   # pa wallet
                 $GPNP_L_WS_PA_FILE, 
                 $usr, $grp );

      copy_file( $GPNP_C_ROOT_FILE,  # root cert
                 $GPNP_L_C_ROOT_FILE, 
                 $usr, $grp );
      copy_file( $GPNP_C_PEER_FILE,  # peer cert
                 $GPNP_L_C_PEER_FILE, 
                 $usr, $grp );
      copy_file( $GPNP_C_PA_FILE,    # pa cert
                 $GPNP_L_C_PA_FILE, 
                 $usr, $grp );

      # Make sure copied local wallet permissions changed, ignore res
      my $islocal = TRUE;
      gpnp_wallets_set_ownerperm( $islocal );  # error(s) logged 

    }    
    unless ($status == SUCCESS) { 
      error( "Failed to take cluster-wide GPnP setup as local" );
    }
    return $status;
}

####---------------------------------------------------------
#### Copy file from local path to remote path for given list of nodes
#### if user given, will run copy as it
#### This routine is gpnp-setup specific.
# ARGS: 4
# ARG1 : Source file name
# ARG2 : Destination remote path
# ARG3 : User-owner
# ARG4 : List of nodes to copy
# @returns SUCCESS or $FAILURE
#
# static
sub copy_gpnpsetup_to_nodes  {
  my $src  = $_[0]; 
  my $dst  = $_[1]; 
  my $user = $_[2]; 
  my $nodelist = $_[3];  # comma-separated scalar list

  my $rc      = 0;
  my @capout  = ();

  if (! (-f $src)) {
     trace("  $src ? -f failed" );
     return FAILED;
  }
  trace("  $src =>  $dst" );
  my @program = ($GPNP_E_GPNPSETUP,
                 '-sourcefile', $src, 
                 '-destfile',   $dst, 
                 '-nodelist',   $nodelist ); 

  # run as specific user, if requested
  trace( '     rmtcpy: '.join(' ', @program) );
  $rc = run_as_user2($user, \@capout, @program);

  # cluutil return 0 err code and errors, if any, on stdout
  if (scalar(@capout) > 0)
  {
    trace( "---rmtcopy { $nodelist } output---\n".join('', @capout));
    trace( "---rmtcopy---." );
  }
  if (0 != $rc) 
  {
     error("Failed to rmtcopy \"$src\" to \"$dst\" ".
           "for nodes {$nodelist}, rc=$rc" ); 
     return FAILED;
  }
  return SUCCESS;
}

####---------------------------------------------------------
#### Push GPnP local file setup to be cluster-wide 
#### (Copy local wallet(s)/profiles to global stage area on current node as well
#### as list of cluster nodes)
#
# NOTE:  check_gpnp_setup() MUST be called prior calling this sub
#
# ARGS: 1
# ARG1 : List of comma-separated cluster node names to push gpnp file setup to
#        (inclusion of current node is ok)
# @returns SUCCESS or $FAILURE
#
sub push_clusterwide_gpnp_setup
{
    my $nodelist   = $_[0];
    my $crshome    = $GPNP_CRSHOME_DIR; # validated
    my $gpnpdir    = $GPNP_GPNPHOME_DIR; # validated
    my $gpnplocdir = $GPNP_GPNPLOCALHOME_DIR; # validated
    my $host       = $GPNP_HOST;
    my $orauser    = $GPNP_ORAUSER;
    my $oragroup   = $GPNP_ORAGROUP;

    # TOBEREVISED - normally, current node is a part of a node list
    #               and cluster-wide setup pushed through localhost rmtcopy
    #               Perhaps current node can be treated specially (order,local)

    $nodelist =~ s/ //g;
    trace("Pushing local gpnpsetup to cluster nodes: {$nodelist}");

    # opt manifest 1st
    my $origout = tmpnam(); # concurrency not an issue here
    open( MFT, ">$origout" ) or # non-fatal
       error "Can't open \"$origout\": $!";

    print MFT "---GPnP cluster-wide configuration---\n";
    print MFT "origin: $host\n";
    print MFT "push_list: {$nodelist}\n";
    print MFT "owner: $GPNP_ORAUSER,".
                         "$GPNP_ORAGROUP\n";
    print MFT "TS: ".gmtime()." UTC (".localtime()." local)\n";
    close( MFT );

    # set MFT owner to orauser, to make sure rmt copy succeeds
    s_set_ownergroup ($orauser, $oragroup, $origout) or # non-fatal
      error( "Can't change ownership on $origout" );

    s_set_perms ("0640", $origout) or # non-fatal
      error( "Can't set permissions on $origout" );

    copy_gpnpsetup_to_nodes( $origout,             # push config manifest
                      $GPNP_ORIGIN_FILE, 
                      $orauser, $nodelist ); 
    unlink($origout);

    # mandatory
    my $status = 
    copy_gpnpsetup_to_nodes( $GPNP_L_P_PEER_FILE,  # peer profile
                      $GPNP_P_PEER_FILE, 
                      $orauser, $nodelist );
    if ($status == SUCCESS) { $status = 
    copy_gpnpsetup_to_nodes( $GPNP_L_WS_PEER_FILE, # peer wallet
                      $GPNP_WS_PEER_FILE, 
                      $orauser, $nodelist ); }

    # optional
    if ($status == SUCCESS) { 
    copy_gpnpsetup_to_nodes( $GPNP_L_P_SAVE_FILE,  # saved profile
                      $GPNP_P_SAVE_FILE, 
                      $orauser, $nodelist ); 

    copy_gpnpsetup_to_nodes( $GPNP_L_W_ROOT_FILE,  # root wallet
                      $GPNP_W_ROOT_FILE, 
                      $orauser, $nodelist );
    copy_gpnpsetup_to_nodes( $GPNP_L_WS_PRDR_FILE, # prdr (r/o) wallet
                      $GPNP_WS_PRDR_FILE, 
                      $orauser, $nodelist ); 
    copy_gpnpsetup_to_nodes( $GPNP_L_WS_PA_FILE,   # pa wallet
                      $GPNP_WS_PA_FILE, 
                      $orauser, $nodelist );

    copy_gpnpsetup_to_nodes( $GPNP_L_C_ROOT_FILE,  # root cert
                      $GPNP_C_ROOT_FILE, 
                      $orauser, $nodelist );
    copy_gpnpsetup_to_nodes( $GPNP_L_C_PEER_FILE,  # peer cert
                      $GPNP_C_PEER_FILE, 
                      $orauser, $nodelist );
    copy_gpnpsetup_to_nodes( $GPNP_L_C_PA_FILE,    # pa cert
                      $GPNP_C_PA_FILE, 
                      $orauser, $nodelist );
    }
    unless ($status == SUCCESS) { 
       print STDERR "rmtcopy aborted\n";
    }
    return $status;
}

####---------------------------------------------------------
#### Create GPnP wallet(s)
# ARGS: 6
# ARG1 : Parameter hash
# ARG2 : Hostname, can be null for non-host specific setup
# ARG3 : Force wallet creation (if FALSE, wallet won't be created if exists)
# @returns SUCCESS or $FAILURE
#
sub create_gpnp_wallets
{
    my $host     = $_[0];
    my $force    = $_[1];

    my $crshome  = $CFG->ORA_CRS_HOME; # validated
    my $gpnpdir  = $CFG->params('GPNPCONFIGDIR'); # validated
    my $orauser  = $CFG->params('ORACLE_OWNER');
    my $oragroup = $CFG->params('ORA_DBA_GROUP');
    my $islocal  = FALSE;

    my $status  = SUCCESS;
    my $rc      = 0;
    my @program ;

    #-------------
    # Check existing setup, if any

    my $GPNPHOME_DIR = catdir( $gpnpdir, 'gpnp' );
    my $WALLETS_DIR =  catdir( $GPNPHOME_DIR, 'wallets' );
    if ($host) {
        $WALLETS_DIR = catdir( $GPNPHOME_DIR, $host, 'wallets' );
        $islocal = TRUE;
    }
    trace ("Oracle CRS home = " . $crshome);
    trace ("Oracle GPnP wallets home = $WALLETS_DIR");

    my $W_ROOT_DIR  = catdir( $WALLETS_DIR, 'root' );
    my $W_PA_DIR    = catdir( $WALLETS_DIR, 'pa' );
    my $W_PEER_DIR  = catdir( $WALLETS_DIR, 'peer' );
    my $W_PRDR_DIR  = catdir( $WALLETS_DIR, 'prdr' );

    my $WALLET_NAME = 'ewallet.p12';
    my $SSOWAL_NAME = 'cwallet.sso';

    my $W_ROOT_FILE = catfile( $W_ROOT_DIR, $WALLET_NAME );
    my $W_PEER_FILE = catfile( $W_PEER_DIR, $SSOWAL_NAME );
    my $W_PRDR_FILE = catfile( $W_PRDR_DIR, $SSOWAL_NAME );
    my $W_PA_FILE   = catfile( $W_PA_DIR,   $SSOWAL_NAME );

    trace ("Checking if GPnP setup exists");
    if (!(-d $W_ROOT_DIR)) {
        error ("The directory \"$W_ROOT_DIR\" does not exist");
        return FAILED;
    }
    if (!(-d $W_PEER_DIR)) {
        error ("The directory \"$W_PEER_DIR\" does not exist");
        return FAILED;
    }
    if (!(-d $W_PRDR_DIR)) {
        error ("The directory \"$W_PRDR_DIR\" does not exist");
        return FAILED;
    }
    if (!(-d $W_PA_DIR)) {
        error ("The directory \"$W_PA_DIR\" does not exist");
        return FAILED;
    }
    if (-f $W_PEER_FILE) {
        trace ("$W_PEER_FILE wallet exists");
        if (! $force)
        {
           trace ("$W_PEER_FILE exists and force is not requested. Done.");
           return SUCCESS;
        }
    }
    trace ("$W_PEER_FILE wallet must be created");

    if (-f $W_PRDR_FILE) {
        trace ("Warning: existing $W_PRDR_FILE wallet will be deleted.");
    }
    if (-f $W_PA_FILE) {
        trace ("Warning: existing $W_PA_FILE wallet will be deleted.");
    }


    #-------------
    # Create wallet(s)

    my $E_ORAPKI    = catfile( $crshome, 'bin', 'orapki' );

    my $CERT_NAME   = 'cert.txt';
    my $CERTRQ_NAME = 'certreq.txt';
    my $RTCERT_NAME = 'b64certificate.txt';
    my $PDUMMY      = 'gpnp_wallet1';

    my $W_ROOT_DN   = '"CN=GPnP_root"';
    my $W_PA_DN     = '"CN=GPnP_pa"';
    my $W_PEER_DN   = '"CN=GPnP_peer"';
    my $W_KEYSZ     = '1024';
    my $W_EXPDT     = '"01/01/2099"';
    my $W_CVALID    = '9999';

    my $CRQ_PA_FILE = catfile( $W_PA_DIR, $CERTRQ_NAME );
    my $CRQ_PEER_FILE = catfile( $W_PEER_DIR, $CERTRQ_NAME );
    my $C_ROOT_FILE = catfile( $W_ROOT_DIR, $RTCERT_NAME );
    my $C_PA_FILE   = catfile( $W_PA_DIR, $CERT_NAME );
    my $C_PEER_FILE = catfile( $W_PEER_DIR, $CERT_NAME );

    # Delete old wallets/certificats, if they exist.
    {
      trace ("Removing old wallets/certificates, if any");
      $status &= s_remove_file( $W_ROOT_FILE );
      $status &= s_remove_file( $W_PA_FILE );
      $status &= s_remove_file( $W_PEER_FILE );
      $status &= s_remove_file( $W_PRDR_FILE );

      $status &= s_remove_file( catfile( $W_PEER_DIR, $WALLET_NAME ) );
      $status &= s_remove_file( catfile( $W_PRDR_DIR, $WALLET_NAME ) );
      $status &= s_remove_file( catfile( $W_PA_DIR,   $WALLET_NAME ) );

      $status &= s_remove_file( $CRQ_PA_FILE );
      $status &= s_remove_file( $CRQ_PEER_FILE );
      $status &= s_remove_file( $C_ROOT_FILE );
      $status &= s_remove_file( $C_PA_FILE );
      $status &= s_remove_file( $C_PEER_FILE );
    }
    #-------------
    # 1.a Create root wallet 
    if (SUCCESS == $status) 
    {
        print( "  root wallet\n" ); #FIXME output from lib breaks conv
        trace( "Creating GPnP Root Wallet..." );
        @program = ( $E_ORAPKI, 'wallet', 'create', 
                     '-wallet', "\"$W_ROOT_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-nologo' ); 
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to create a root wallet for Oracle Cluster GPnP. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # 1.b Create self-signed root wallet certificate 
    if (SUCCESS == $status) 
    {
        print( "  root wallet cert\n" ); #FIXME output from lib breaks conv
        trace( "Creating GPnP Root Certificate..." );
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_ROOT_DIR\"", 
                     '-pwd', $PDUMMY,
                     '-self_signed',
                     '-dn', $W_ROOT_DN,
                     '-keysize', $W_KEYSZ,
                     '-validity', $W_CVALID, 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to create a root certificate for Oracle Cluster GPnP.".
                 " orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # 1.c Export root wallet certificate 
    if (SUCCESS == $status) 
    {
        print( "  root cert export\n" ); #FIXME output from lib breaks conv
        trace( "Exporting GPnP Root Certificate..." );
        @program = ( $E_ORAPKI, 'wallet', 'export', 
                     '-wallet', "\"$W_ROOT_DIR\"", 
                     '-pwd', $PDUMMY,
                     '-dn', $W_ROOT_DN,
                     '-cert', "\"$C_ROOT_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to export root certificate for Oracle Cluster GPnP. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 2. Create empty wallets for peer, prdr & pa (cwallet.sso  ewallet.p12)
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer wallet\n" ); #FIXME output from lib breaks conv
        trace( "Creating GPnP Peer Wallet..." );
        @program = ( $E_ORAPKI, 'wallet', 'create', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-auto_login', 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to create a peer wallet for Oracle Cluster GPnP. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) prdr
    if (SUCCESS == $status) 
    {
        print( "  profile reader wallet\n" ); #FIXME output from lib breaks conv
        trace( "Creating GPnP Profile Reader Wallet..." );
        @program = ( $E_ORAPKI, 'wallet', 'create', 
                     '-wallet', "\"$W_PRDR_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-auto_login', 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to create a profile reader wallet for ".
                 "Oracle Cluster GPnP. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # c) pa
    if (SUCCESS == $status) 
    {
        print( "  pa wallet\n" ); #FIXME output from lib breaks conv
        trace( "Creating GPnP PA Wallet..." );
        @program = ( $E_ORAPKI, 'wallet', 'create', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-auto_login', 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to create a PA wallet for Oracle Cluster GPnP. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 3. Add private key to a wallet
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer wallet keys\n" ); #FIXME output from lib breaks conv
        trace("Adding private key to GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-dn', $W_PEER_DN,
                     '-keysize', $W_KEYSZ, 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot add private key to a wallet. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) pa
    if (SUCCESS == $status) 
    {
        print( "  pa wallet keys\n" ); #FIXME output from lib breaks conv
        trace("Adding private key to GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-dn', $W_PA_DN,
                     '-keysize', $W_KEYSZ, 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot add private key to a wallet. orapki rc=$rc" );
           $status = FAILED;
        }
    }

    #-------------
    # 4. Create cert request (B64) for each (certreq.txt)
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer cert request\n" ); #FIXME output from lib breaks conv
        trace("Creating certificate request for GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'export', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-dn', $W_PEER_DN,
                     '-request', "\"$CRQ_PEER_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot export a certificate request from a wallet. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) pa
    if (SUCCESS == $status) 
    {
        print( "  pa cert request\n" ); #FIXME output from lib breaks conv
        trace("Creating certificate request for GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'export', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-dn', $W_PA_DN,
                     '-request', "\"$CRQ_PA_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot export a certificate request from a wallet. ".
                 "orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 5. Create certificate files (B64) for each 
    #    (cert.txt signed with same root wallet (valid 27yrs))
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer cert\n" ); #FIXME output from lib breaks conv
        trace("Creating certificate for GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'cert', 'create', 
                     '-wallet', "\"$W_ROOT_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-request', "\"$CRQ_PEER_FILE\"",
                     '-cert', "\"$C_PEER_FILE\"",
                     '-validity', $W_CVALID, 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot create a peer certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) pa
    if (SUCCESS == $status) 
    {
        print( "  pa cert\n" ); #FIXME output from lib breaks conv
        trace("Creating certificate for GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'cert', 'create', 
                     '-wallet', "\"$W_ROOT_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-request', "\"$CRQ_PA_FILE\"",
                     '-cert', "\"$C_PA_FILE\"",
                     '-validity', $W_CVALID, 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot create a PA certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 6. Add root certificate as trusted cert to all user wallets 
    #    (to allow import certificates not only as self-signed)
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer root cert TP\n" ); #FIXME output from lib breaks conv
        trace("Adding Root Certificate TP to GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_ROOT_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot add a root TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) prdr
    if (SUCCESS == $status) 
    {
        print( "  profile reader root cert TP\n" ); #FIXME output from lib 
        trace("Adding Root Certificate TP to GPnP Profile Reader Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PRDR_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_ROOT_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a Profile Reader Wallet ".
                 "for Oracle Cluster GPnP. ".
                 "Cannot add a root TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # c) pa
    if (SUCCESS == $status) 
    {
        print( "  pa root cert TP\n" ); #FIXME output from lib breaks conv
        trace("Adding Root Certificate TP to GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_ROOT_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot add a root TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 7. Add cross certificates as trust points
    # a) peer - add pa
    if (SUCCESS == $status) 
    {
        print( "  peer pa cert TP\n" ); #FIXME output from lib breaks conv
        trace("Adding PA Certificate as a TP into a GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_PA_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot add a PA TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) pa - add peer
    if (SUCCESS == $status) 
    {
        print( "  pa peer cert TP\n" ); #FIXME output from lib breaks conv
        trace("Adding peer Certificate as a TP into a GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_PEER_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot add a peer TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # c) prdr - add peer
    if (SUCCESS == $status) 
    {
        print( "  profile reader pa cert TP\n" ); #FIXME output from lib 
        trace("Adding PA Certificate as a TP into a GPnP ".
              "Profile Reader Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PRDR_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_PA_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a Profile Reader Wallet ".
                 "for Oracle Cluster GPnP. ".
                 "Cannot add a PA TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # c) prdr - add pa
    if (SUCCESS == $status) 
    {
        print( "  profile reader peer cert TP\n" ); #FIXME output from lib
        trace("Adding peer Certificate as a TP into a GPnP ".
              "Profile Reader Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PRDR_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-trusted_cert', '-cert', "\"$C_PEER_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a Profile Reader Wallet ".
                 "for Oracle Cluster GPnP. ".
                 "Cannot add a peer TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    #-------------
    # 8. Finally, add user certificate to user wallets (to add public key cert)
    # a) peer
    if (SUCCESS == $status) 
    {
        print( "  peer user cert\n" ); #FIXME output from lib breaks conv
        trace("Adding PA Certificate as a TP into a GPnP Peer Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PEER_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-user_cert', '-cert', "\"$C_PEER_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a peer wallet for Oracle Cluster GPnP. ".
                 "Cannot add a PA TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # b) pa
    if (SUCCESS == $status) 
    {
        print( "  pa user cert\n" ); #FIXME output from lib breaks conv
        trace("Adding peer Certificate as a TP into a GPnP PA Wallet...");
        @program = ( $E_ORAPKI, 'wallet', 'add', 
                     '-wallet', "\"$W_PA_DIR\"", 
                     '-pwd', $PDUMMY, 
                     '-user_cert', '-cert', "\"$C_PA_FILE\"", 
                     '-nologo' );
        trace( join(' ', @program) );
        $rc = system( "@program" );
        if (0 != $rc) {
           error("Failed to make a PA wallet for Oracle Cluster GPnP. ".
                 "Cannot add a peer TP certificate. orapki rc=$rc" );
           $status = FAILED;
        }
    }
    # Delete intermediate files and non-sso wallets, if they exist.
    {
      $status &= s_remove_file( catfile( $W_PEER_DIR, $WALLET_NAME ) );
      $status &= s_remove_file( catfile( $W_PRDR_DIR, $WALLET_NAME ) );
      $status &= s_remove_file( catfile( $W_PA_DIR,   $WALLET_NAME ) );
      $status &= s_remove_file( $CRQ_PA_FILE );
      $status &= s_remove_file( $CRQ_PEER_FILE );
      $status &= s_remove_file( $C_ROOT_FILE );
      $status &= s_remove_file( $C_PA_FILE );
      $status &= s_remove_file( $C_PEER_FILE );
    } 
    if (SUCCESS == $status) {

        # Change file ownership to non-root
        $status = gpnp_wallets_set_ownerperm( $islocal );  # error(s) logged 

    }
    if (SUCCESS == $status) {
        trace ("GPnP Wallets successfully created.");    
    }
    return $status;
}

# set gpnp wallet/certs ownership and permissions
sub gpnp_wallets_set_ownerperm()
{
    my $islocal = $_[0]; # boolean (TRUE  - local home (node-specific), 
                         #          FALSE - global home (seed)
    my $status  = SUCCESS;
    my $haderr  = FALSE;

    my $orauser  = $CFG->params('ORACLE_OWNER');
    my $oragroup = $CFG->params('ORA_DBA_GROUP');

    # file paths are validated
    my $gpnp_c_root;
    my $gpnp_w_root;  
    my $gpnp_c_peer;
    my $gpnp_w_peer;  
    my $gpnp_c_pa;
    my $gpnp_w_pa;  
    my $gpnp_w_prdr;  

    # assign appropriate gpnp home
    if ($islocal) {
       $gpnp_c_root = $GPNP_L_C_ROOT_FILE;
       $gpnp_w_root = $GPNP_L_W_ROOT_FILE;
       $gpnp_c_peer = $GPNP_L_C_PEER_FILE;
       $gpnp_w_peer = $GPNP_L_WS_PEER_FILE;
       $gpnp_c_pa   = $GPNP_L_C_PA_FILE;
       $gpnp_w_pa   = $GPNP_L_WS_PA_FILE;
       $gpnp_w_prdr = $GPNP_L_WS_PRDR_FILE;
    } else {
       $gpnp_c_root = $GPNP_C_ROOT_FILE;
       $gpnp_w_root = $GPNP_W_ROOT_FILE;
       $gpnp_c_peer = $GPNP_C_PEER_FILE;
       $gpnp_w_peer = $GPNP_WS_PEER_FILE;
       $gpnp_c_pa   = $GPNP_C_PA_FILE;
       $gpnp_w_pa   = $GPNP_WS_PA_FILE;
       $gpnp_w_prdr = $GPNP_WS_PRDR_FILE;
    }

    # Change file ownership to non-root
    # Cert files are now not kept: $gpnp_c_root, $gpnp_c_peer, $gpnp_c_pa,
    my @resfiles = ( $gpnp_w_root,
                     $gpnp_w_peer, $gpnp_w_prdr,
                     $gpnp_w_pa   );
    trace("resfiles are @resfiles");

    foreach (@resfiles) {
       if ($CFG->platform_family eq "windows") {
          my $nt_authority = "NT AUTHORITY\\SYSTEM";
          my $admin        = s_getGroupName();
          if ($DEBUG) {
             trace ("s_set_ownergroup_win ($nt_authority, $admin, $_)");
          }
          if (($status = s_set_ownergroup_win ($nt_authority,
                                               $admin, $_))  != $SUCCESS) {
             error ("Can't set ownership on $_");
             $haderr = TRUE;
          }
       } else {
          # set permissions/owner on wallets/certs
          if (($status = s_set_perms ("700", $_)) != $SUCCESS) {
             error "Can't set permissions on $_";
             $haderr = TRUE;
          }
          if (($status = s_set_ownergroup ($orauser,
                                           $oragroup, $_)) != $SUCCESS) {
             error ("Can't set ownership on $_");
             $haderr = TRUE;
          }
       }
    }
    if ($CFG->platform_family eq "windows") {
    } else {
       # ease permissions/owner on config reader wallet
       if (($status = s_set_perms ("750", $gpnp_w_prdr)) != $SUCCESS) {
          error "Can't set permissions on $gpnp_w_prdr";
          $haderr = TRUE;
       }
    }
    # head out
    if ( $haderr ) {
        error ("Error(s) occurred while setting GPnP Wallets ".
               "ownership/permissions.");    
    } else {
        trace ("GPnP Wallets ownership/permissions successfully set.");    
    }
    return $status;
}

####---------------------------------------------------------
#### Function for returning OCRID 
# ARGS: 1
# ARG1: ORA_CRS_HOME
# @returns ID fetched from  ocrcheck or -1 in case of error
sub get_ocrid
{
   my $crshome = $_[0];
   my $id      = -1;
   my $ocrcheck;

   if ($CFG->platform_family eq "windows") {
      $ocrcheck = catfile($crshome, 'bin', 'ocrcheck.exe');
   }
   else {
      $ocrcheck = catfile($crshome, 'bin', 'ocrcheck');
   }

   trace("Executing ocrcheck to get ocrid");
   open(OCRCHECK, "$ocrcheck |" );
   my @output = <OCRCHECK>;
   close(OCRCHECK);

   my @txt = grep (/ ID /, @output);
   foreach my $line (@txt) {
      my ($idstring, $oid)  = split(/:/, $line);
      $id = trim($oid);
   }

   $CFG->OCR_ID($id);

   if ($id == -1) {
      trace("get_ocrid : Failed to get ocrid ");
   }

   return $id;
}

####---------------------------------------------------------
#### Function for returning CLUSTER_GUID. 
# 1) First , checks for Grid Infrastructure active version 
#    NOTE: CFG->oldconfig('ORA_CRS_VERSION') must be set.
# 2) If, version < 11.1.0.7 , returns -1.
# 3) Else, get the clusterguid from "crsctl get css clusterguid".
# ARGS: 1
# ARG1: ORA_CRS_HOME
# @returns ID fetched from  crsctl or -1 in case of error

sub get_clusterguid
{
   my $home = $_[0];
   my $id = -1;

   ## If here , must be 11.1.0.7 and higher
   my @OLD_CRS_VERSION = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if (($OLD_CRS_VERSION[0]   < 11) ||
       (($OLD_CRS_VERSION[0] == 11) && ($OLD_CRS_VERSION[1] == 1) &&
        ($OLD_CRS_VERSION[2] ==  0) && ($OLD_CRS_VERSION[3]  < 7)))
   {
      trace("Skipping clusterguid fetch for ".join('.',@OLD_CRS_VERSION));
      return -1;
   }
   trace("Fetching clusterguid from ".join('.',@OLD_CRS_VERSION));

   my $cmd;
   if (! defined $home) {
     $cmd = crs_exec_path('crsctl');
   } else {
     $cmd = catfile( $home, 'bin', 'crsctl' );
   }

   # run "crsctl get css clusterguid"
   my @out = system_cmd_capture(($cmd, "get", "css", "clusterguid"));
   my $rc  = shift @out;

   # if succeeded, get the guid, output must be a single line
   if ($rc == 0) {
      my $outid = $out[0];
      $id = trim($outid);
      trace( "Got CSS GUID: $id (".join(' ',@out).")" );
   }
   else
   {
      trace ("Retrieval of CSS GUID failed (rc=$rc), ".
             "with the message:\n".join("\n", @out)."\n");
   }

   if ($id == -1) {
      trace("get_clusterguid : Failed to get clusterguid");
   }
   return $id;
}



####---------------------------------------------------------
#### Run gpnptool with options
# ARGS: 3
# ARG1 : ref to array of gpnptool arguments (verb + switches)
# ARG2 : user to run gpnptool as (or undef if don't care)
# ARG3 : if reference to an array var passed, return captured output 
#        (stderr in case of gpnptool) value (strings are "as is", not chomped);
#        if undefined, no capture takes place.
# @returns numeric exit code from gpnptool (0 on success)
#
sub run_gpnptool
{
    my $argsref   = $_[0]; #ref
    my $user      = $_[1];
    my $capoutref = $_[2]; #ref

    my $rc = -1;

    my @program = ($GPNP_E_GPNPTOOL, @{$argsref});

    # run as specific user, if requested
    trace ('gpnptool: run '.join(' ', @program));
    $rc = run_as_user2($user, $capoutref, @program);
    trace ("gpnptool: rc=$rc");
    if (defined($capoutref))
    {
      trace ("gpnptool output:\n".  join('', @{$capoutref}) );
    }
    return $rc;
}

####---------------------------------------------------------
#### Run gpnptool to get param value(s) for given profile
# ARGS: 3
# ARG1 : profile filepath (verified)
# ARG2 : reference to an array of gpnptool cmdline switches of
#        profile params to get their value.
#        e.g. -prf_sq (for ProfileSequence, see "gpnptool help getpval")
#        Invalid switch will result in general gpnptool error.
#        Passing a valid switch that has no representation in the profile
#        will result in special "<n/a>" value, but no cmd error.
# ARG3 : user to run gpnptool as (or undef if don't care)
# @returns array of elements, 
#        first being a numeric result code: 
#          <0   error occured
#          ==0  values successfully obtained from a specified profile
#        rest of elements are requested string parameter values,
#        or an error message, if error returned.
#
sub run_gpnptool_getpval
{
    my $gpnp_p    = $_[0]; # validated
    my $gpnp_pr   = $_[1]; # ref 
    my $orauser   = $_[2];

    my @gpnp_pars = @{$gpnp_pr};

    if ((! defined $gpnp_pr) || (0 == $#gpnp_pars)) {
      error( "No parameters specified to query from \"$gpnp_p\" profile." );
      return ( -2, "No parameters" ); 
    }

    # capture gpnptool getpval output into a temp file, to ensure
    # no shell errors get into the results
    my $gpnptool_res_file = tmpnam(); # concurrency not an issue here

    my @gpnptool_args = ( 'getpval',
                          "-p=\"$gpnp_p\"",
                          "-o=\"$gpnptool_res_file\"",
                          @gpnp_pars
                        );
    my @gpnptool_out  = ();  # stdout/stderr out discarded by this fn
    my @gpnptool_getpval_res = ();

    # run gpnptool getpval as orauser, capturing stdout/err
    my $rc = run_gpnptool( \@gpnptool_args, $orauser, \@gpnptool_out );
    if (0 != $rc) {
      error("Failed to get parameter value(s) from a \"$gpnp_p\" profile.");
    } else {

      # read getpval results from temp file
      @gpnptool_getpval_res = read_file( $gpnptool_res_file );

      # delete results temp file
      s_remove_file($gpnptool_res_file);

      chomp(@gpnptool_getpval_res); # remove newlines (output one par per line)
    }
    return ($rc, @gpnptool_getpval_res);
}

####---------------------------------------------------------
#### Run gpnptool verify for given profile and wallet loc (WRL)
# ARGS: 3
# ARG1 : profile filepath (verified)
# ARG2 : WRL (gpnptool-recognized wallet locator string, e.g. 
#        'file:/mypath/') 
# ARG3 : user to run gpnptool as (or undef if don't care)
# @returns numeric result:
#          <0   error occured
#          ==0  profile signature does not matches against given wallet
#          ==1  profile signature matches against given wallet
#
sub run_gpnptool_verifysig
{
    my $gpnp_p   = $_[0]; # validated
    my $gpnp_wrl = $_[1]; # validated
    my $orauser  = $_[2];

    my @gpnptool_args = ( 'verify', 
                          "-p=\"$gpnp_p\"", 
                          "-w=\"$gpnp_wrl\"",  
                          '-wu=peer'
                        );
    my @gpnptool_out  = ();

    # run gpnptool verify as orauser, capturing stdout/err
    my $rc = run_gpnptool( \@gpnptool_args, $orauser, \@gpnptool_out );
    if (0 != $rc) {
        trace("Failed to verify a \"$gpnp_p\" profile against ".
              "cluster-wide wallet \"$gpnp_wrl\" gpnptool rc=$rc" );
        return -1;
    }
    # TOBEREVISED - gpnptool error code - now suc on invalid sig 
    my $gpnptool_res = join('', @gpnptool_out );
    if ($gpnptool_res =~ m/signature is valid/i) {
        trace("Profile \"$gpnp_p\" signature is VALID ".
              "for wallet \"$gpnp_wrl\"" );
        return 1;
    }
    elsif ($gpnptool_res =~ m/signature is not valid/i) {
        trace("Profile \"$gpnp_p\" signature is INVALID".
              " for wallet \"$gpnp_wrl\"" );
        return 0;
    }
    else {
        error("Profile \"$gpnp_p\" signature verified, ".
              "but no signature status string ".
              "found in \"\n$gpnptool_res\n\"");
        return -2;
    }
    return -1;
}

sub initialize_local_gpnp
{
  my $hostname =   $_[0];
  my $gpnp_setup_type = $_[1];
  my $gpnp_descr = "unknown";
  my $status;
  my $ckptgpnp;
  my $ckptName = "ROOTCRS_GPNPSETUP";

  if (isCkptexist($ckptName))
  {
    $ckptgpnp = getCkptStatus($ckptName);

    trace("'$ckptName' state is $ckptgpnp");

    if ($ckptgpnp eq CKPTSUC) {
      trace("Local GPNP is already initialized");
      $wipCkptName = "ROOTCRS_STACK";
      return SUCCESS;
    }
  }
 
  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = "ROOTCRS_GPNPSETUP";

 SWITCH: {
    $gpnp_setup_type == GPNP_SETUP_BAD  and 
      $gpnp_descr = "dirty", last;
    $gpnp_setup_type == GPNP_SETUP_NONE and
      $gpnp_descr = "none", last;
    $gpnp_setup_type == GPNP_SETUP_LOCAL and 
      $gpnp_descr = "local", last;
    $gpnp_setup_type == GPNP_SETUP_GOTCLUSTERWIDE and 
      $gpnp_descr = "new-cluster-wide", last;
    $gpnp_setup_type == GPNP_SETUP_CLUSTERWIDE and 
      $gpnp_descr = "cluster-wide", last;
    $gpnp_descr = "unknown";        # default case
  }

  trace ("GPnP setup state: $gpnp_descr");
  if ($gpnp_setup_type == GPNP_SETUP_BAD) {
    trace("Forcing re-creation of gpnp setup.");
    $gpnp_setup_type = GPNP_SETUP_NONE;
  }

  # unless GPnP configuration we running is cluster-wide, or on good local 
  # gpnp profile/wallet config, create local setup
  if ($gpnp_setup_type == GPNP_SETUP_GOTCLUSTERWIDE ||
      $gpnp_setup_type == GPNP_SETUP_CLUSTERWIDE) {
    trace("GPnP cluster configuration already performed");
  }
  elsif ($gpnp_setup_type == GPNP_SETUP_LOCAL) {
    trace("GPnP cluster configuration not required for non-clustered ",
          "config");
  }
  elsif ($gpnp_setup_type == GPNP_SETUP_NONE) {
    trace ("Creating local GPnP setup for clustered node...");

   $status =  create_gpnp_wallets($hostname, TRUE );
   if ($status != SUCCESS) {
      error ("Creation of Oracle GPnP wallets failed for $hostname");
      writeCkpt($ckptName, CKPTFAIL);
      die("Failed to create GPnP wallets for $hostname");
   }

    trace ("<--- GPnP wallets successfully created");

    # gpnp: Create gpnp peer profile for host (force) with given pars
    trace ("Creating GPnP peer profile --->");

    $status = create_gpnp_peer_profile($hostname,
                             TRUE, # force (create, not edit)
                             TRUE  # sign with peer wallet
                            );
    if ($status != SUCCESS) {
      error ("Creation of Oracle GPnP peer profile failed for $hostname");
      writeCkpt($ckptName, CKPTFAIL);
      die("Failed to create GPnP peer profile for $hostname");
    }
    trace ("<--- GPnP peer profile successfully created");

    trace ("GPnP local setup successfully created\n");
  }

  writeCkpt($ckptName, CKPTSUC);
  $wipCkptName = "ROOTCRS_STACK";
  return;
}
  ####---------------------------------------------------------
#### Create GPnP peer profile
# ARG1 : Parameter hash
# ARG1 : Hostname, can be null for non-host specific setup
# ARG2 : Force profile creation (if FALSE, won't be created if exists)
# ARG3 : If 1, attempt to sign a profile with a peer wallet
# @returns SUCCESS or $FAILURE
#
sub create_gpnp_peer_profile
{
    my $host     = $_[0];
    my $force    = $_[1];
    my $sign     = $_[2];

    my $status   = SUCCESS;
    my $rc       = 0;
    my @gpnptool_args;
    my @gpnptool_out ;
    my $edit     = FALSE;
    my $verb     = 'create'; 

    my $crshome  = $CFG->ORA_CRS_HOME; # validated
    my $gpnpdir  = $CFG->params('GPNPCONFIGDIR'); # validated
    my $orauser  = $CFG->params('ORACLE_OWNER');
    my $oragroup = $CFG->params('ORA_DBA_GROUP');
    my $p_paloc  = $CFG->params('GPNP_PA');
    my $p_cname  = $CFG->params('CLUSTER_NAME');
    my $p_cssdis = $CFG->VF_DISCOVERY_STRING;
    my $p_cssld  = $CFG->params('CSS_LEASEDURATION');
    my $p_asmdis = $CFG->params('ASM_DISCOVERY_STRING');
    my $p_asmspf = $CFG->params('ASM_SPFILE');
    my $p_ocrid  = $CFG->oldconfig('OCRID');
    my $p_nets   = $CFG->params('NETWORKS');
    my $p_clstid = $CFG->oldconfig('CLUSTER_GUID');

    # if old set of networks defined, use them instead
    my $p_oldnets = $CFG->oldconfig('NETWORKS');
    if ((defined $p_oldnets) && !($p_nets eq $p_oldnets)) {
       $p_nets = $p_oldnets;
    }

    #-------------
    # Check existing setup, if any

    my $GPNPHOME_DIR = catdir( $gpnpdir, 'gpnp' );
    my $PROFILES_DIR;
    if ($host) {
        $PROFILES_DIR = catdir( $GPNPHOME_DIR, $host, 'profiles' );
    } else {
        $PROFILES_DIR = catdir( $GPNPHOME_DIR, 'profiles' );
    }
    trace ("Oracle CRS home = " . $crshome);
    trace ("Oracle GPnP profiles home = $PROFILES_DIR");
    trace ("Oracle GPnP profiles parameters: ");
    trace ("   paloc=$p_paloc=");
    trace ("   cname=$p_cname=");
    trace ("   cssdisco=$p_cssdis=");
    trace ("   cssld=$p_cssld=");
    trace ("   asmdisco=$p_asmdis=");
    trace ("   asmspf=$p_asmspf=");
    trace ("   netlst=$p_nets=");
    trace ("   ocrid=$p_ocrid=");
    trace ("   clusterguid=$p_clstid=");

    my $P_PEER_DIR  = catdir( $PROFILES_DIR, 'peer' );
    my $P_PEER_FILE = catfile( $P_PEER_DIR, 'profile.xml' );
    my $P_SAVE_FILE = catfile( $P_PEER_DIR, 'profile_orig.xml' );

    my $SSOWAL_NAME ;
    my $WALLETS_DIR ;
    my $W_PEER_DIR  ;
    my $W_PEER_FILE ;
    my $W_PEER_WRL  ;

    if (0 != $sign) {
        if ($host) {
           $WALLETS_DIR = catdir( $GPNPHOME_DIR, $host, 'wallets' );
        } else {
           $WALLETS_DIR = catdir( $GPNPHOME_DIR, 'wallets' );
        }
        $SSOWAL_NAME = 'cwallet.sso';
        $W_PEER_DIR  = catdir( $WALLETS_DIR, 'peer' );
        $W_PEER_FILE = catfile( $W_PEER_DIR, $SSOWAL_NAME );
        $W_PEER_WRL  = 'file:'.$W_PEER_DIR;
    }
    trace ("Checking if GPnP setup exists");
    if (0 != $sign) {
        if (!(-d $W_PEER_DIR)) {
           error ("The directory \"$W_PEER_DIR\" does not exist");
           return FAILED;
        }
        if (!(-r $W_PEER_FILE)) {
           error ("The GPnP peer wallet file \"" . $W_PEER_FILE .
               "\" does not exist or is not readable");
           return FAILED;
        }
    }
    if (!(-d $P_PEER_DIR)) {
        error ("The directory \"$P_PEER_DIR\" does not exist");
        return FAILED;
    }
    if (-f $P_PEER_FILE) {
        trace ("$P_PEER_FILE wallet exists");
        if (! $edit) {
           if (! $force)
           {
              trace ("GPnP peer profile \"".$P_PEER_FILE.
                     "\" exists and force is not requested. Done.");
              return SUCCESS;
           }
           unlink ($P_PEER_FILE, $P_SAVE_FILE); 
        }
    } else {
        $edit = FALSE;    
        trace ("$P_PEER_FILE profile must be created");
    }

    #-------------
    # Create/edit profile(s)
    {
      # make sure asmdis is not empty (replace empty value with 
      # a predefined value (see bug 8557547)
      if (!$p_asmdis || $p_asmdis eq "") {
         $p_asmdis = 
            "++no-value-at-profile-creation--never-updated-through-ASM++";
      }

      # convert netinfo into cmdline pars
      my @netprogram =  instlststr_to_gpnptoolargs( $p_nets );

      my @ocridparam;
      if (!$p_ocrid || $p_ocrid == -1) {
         trace("OCRID is not available, hence not set in GPnP Profile");
      }
      else {
         @ocridparam = ('-ocr=ocr', "-ocr:ocr_oid=\"$p_ocrid\"" );
      }
      my @clstidparam;
      if (!$p_clstid || $p_clstid == -1) {
         trace("ClusterGUID is not available, hence not set in GPnP Profile");
      }
      else {
         @clstidparam = ("-prf_cid=\"$p_clstid\"");
      } 
      # cmdline
      $verb = 'edit' if $edit;
      @gpnptool_args = ( $verb, 
                 "-o=\"$P_PEER_FILE\"", '-ovr', 
                 '-prf', 
                      "-prf_sq=1", "-prf_cn=$p_cname", "-prf_pa=\"$p_paloc\"",
                 @netprogram,
                 '-css=css', 
                      "-css:css_dis=\"$p_cssdis\"", "-css:css_ld=$p_cssld",
                 '-asm=asm', 
                      "-asm:asm_dis=\"$p_asmdis\"", "-asm:asm_spf=\"$p_asmspf\""
                ); 

      if ($edit) {
         push(@gpnptool_args, "-p=\"$P_PEER_FILE\"");
      }

      if (@ocridparam) {
         push(@gpnptool_args, @ocridparam);
      }

      if (@clstidparam) {
         push(@gpnptool_args, @clstidparam);
      }

      @gpnptool_out  = () ;

      $rc = run_gpnptool( \@gpnptool_args, $orauser, \@gpnptool_out);
      if (0 != $rc) {
        error("Failed to $verb a peer profile for Oracle Cluster GPnP. ".
              "gpnptool rc=$rc" );
        $status = FAILED;
      } 
    }
    # sign profile if req 
    if ((SUCCESS == $status) && (0 != $sign))
    {
      @gpnptool_args = ( 'sign',
                 "-p=\"$P_PEER_FILE\"", 
                 "-o=\"$P_PEER_FILE\"", '-ovr', 
                 "-w=\"$W_PEER_WRL\"",          
                 '-rmws' # compact; or format, e.g. '-fmt=0,2'
                 );
      @gpnptool_out  = () ;
      $rc = run_gpnptool( \@gpnptool_args, $orauser, \@gpnptool_out);
      if (0 != $rc) {
        error("Failed to sign a peer profile for Oracle Cluster GPnP. ".
              "gpnptool rc=$rc" );
        $status = FAILED;
      } 
    } 
    # save created profile on success
    if (SUCCESS == $status) 
    {
        copy( $P_PEER_FILE, $P_SAVE_FILE ) or # non-fatal
          error("Failed to copy \"$P_PEER_FILE\" to \"$P_SAVE_FILE\": $!");
    }
    # change file ownership to non-root
    if (SUCCESS == $status)
    {
      if ( -f $P_PEER_FILE )
      {
        if ($DEBUG) 
          { trace ("  s_set_ownergroup($orauser, $oragroup, $P_PEER_FILE)");}
        if (FAILED ==
           ($status = s_set_ownergroup ($orauser, $oragroup, $P_PEER_FILE))) {
          error( "Can't change ownership on $P_PEER_FILE" ); 
        }  
      }
      if ( -f $P_SAVE_FILE )
      {
        if ($DEBUG) 
          { trace ("  s_set_ownergroup($orauser, $oragroup, $P_SAVE_FILE)");}

        s_set_ownergroup ($orauser, $oragroup, $P_SAVE_FILE) or # non-fatal
        error( "Can't change ownership on $P_SAVE_FILE" ); 
      }
    }
    # for extra check, verify created peer profile against wallet 
    # after chown
    if (SUCCESS == $status)
    {
      if ($DEBUG)
      {
        $rc = run_gpnptool_verifysig( $P_PEER_FILE, 
                                      $W_PEER_WRL, $orauser );
        if ($rc <= 0) {
          trace("Failed to verify a peer profile \"$P_PEER_FILE\"".
                " with WRL=$W_PEER_WRL. rc=$rc");
          error("Failed to verify a peer profile \"$P_PEER_FILE\"".
              " with WRL=$W_PEER_WRL. rc=$rc");
          $status = FAILED;
        } 
      }
    }
    if (SUCCESS == $status) {
        trace("GPnP peer profile $verb successfully completed.");        
    }
    return $status;
}
####------ ] GPNP

####---------------------------------------------------------
#### Function for copying onc.config to Oracle 10g home
# ARGS : 1
# ARG1 : Oracle CRS home
sub copyONSConfig
{
    my $crshome = $_[0];

    if (!$crshome) {
        error ("Empty path specified for Oracle CRS home");
        return FAILED;
    }

    if (!(-d $crshome)) {
        error ("The Oracle CRS home path \"" . $crshome . "\" does not exist");
        return FAILED;
    }

    trace ("Oracle CRS home = " . $crshome);
    trace ("Copying ONS config file to 10.2 CRS home");

    my $OLSNODESBIN = catfile ($crshome, "bin", "olsnodes");
    if (-x $OLSNODESBIN) {
        open (OLSNODES, "$OLSNODESBIN -l|");
        my $NODE_NAME = <OLSNODES>;
        close (OLSNODES);
        my $OCRDUMPBIN = catfile ($crshome, "bin", "ocrdump");
        if (-x $OCRDUMPBIN) {
            if ($CFG->platform_family eq "windows") {
                open (OCRDUMP, "$OCRDUMPBIN -stdout -keyname
                      CRS.CUR.ora!$NODE_NAME!ons.ACTION_SCRIPT |");
            }
            else {
                open (OCRDUMP, "$OCRDUMPBIN -stdout -keyname
                      'CRS.CUR.ora!$NODE_NAME!ons.ACTION_SCRIPT'|");
            }

            my @output = <OCRDUMP>;
            close (OCRDUMP);
            my $txt = grep (/ORATEXT/, @output);
            my ($key, $ONS_OH) = split (/:/, $txt);
            $ONS_OH =~ s!/bin/racgwrap!!g;
            $ONS_OH =~ s/^ //g;
            ## checking if ONS resource is configured
            if ($ONS_OH) { 
                ##ONS resource is configured
                my $ONSCONFIG = catfile($ONS_OH, "opmn", "conf", "ons.config");
                my $ONSCONFIG_CH =
                    catfile($ONS_OH, "opmn", "conf", "ons.config");
                if (-f $ONSCONFIG) {
                    ##The ons.config file exists at source location
                    copy ($ONSCONFIG_CH, "$ONSCONFIG_CH.orig");
                    copy ($ONSCONFIG, $ONSCONFIG_CH); 
                    trace ("$ONSCONFIG was copied successfully to " .
                           $ONSCONFIG_CH);
                }
            }
        }
    }

    return SUCCESS;
}

####---------------------------------------------------------
#### Function for creating a directory
# ARGS:
# arg 0 -- directory path to be created
#
# Returns: list of directories -- including intermediate directories -- created
sub create_dir
{
    my $dir_path = $_[0];

    # convert '\' to '/' (for NT)
    $dir_path =~ s!\\!/!g;

    # If dir_path doesn't already exist, create it.
    #
    # If dir_path exists as a symlink, then if the target of the symlink
    # doesn't exist, create the target path.  This is applicable especially
    # to ADE environments where we might already have a symlink pointing to
    # some directory in the has_work/ tree.
    my $link_path;
    if ($link_path = s_isLink ($dir_path)) {
        if ($CFG->DEBUG) {
            trace ("  $dir_path is a SYMLINK to $link_path; changing cwd to " .
                   dirname ($dir_path) . " and resetting DIR_PATH");
        }
        chdir (dirname ($dir_path));
        $dir_path = $link_path;
    }

    my @dirs;

    if (!(-e $dir_path)) {
        if ($CFG->DEBUG) { trace ("  mkpath ($dir_path)");}
        @dirs = mkpath ($dir_path)
                  or die "Can't create $dir_path: $!";
    }

    return @dirs;
}

sub perform_register_service
{
  my $srv = $_[0];
  my $ckptStatus;
  my $ckptName = "ROOTCRS_OHASD";

  if (isCkptexist($ckptName))
  {
    $ckptStatus = getCkptStatus($ckptName);

    trace("'$ckptName' state is $ckptStatus");

    if ((($ckptStatus eq CKPTSTART) && $isNodeCrashed) ||
        ($ckptStatus eq CKPTFAIL))
    {
      trace("Unregistering OHASD service configirations");
      unregister_service($srv);
      $isNodeCrashed = FALSE;
    } elsif ($ckptStatus eq CKPTSUC) {
       trace("'$srv' already registered");
       $wipCkptName = "ROOTCRS_STACK";
       return SUCCESS;
    }
  }

  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = $ckptName;

  register_service($srv);

   if ($srv eq "ohasd") {
      # install_initd if it's SuSE Linux
      s_install_initd();
   }

  trace("'$srv' is now registered");
}

####---------------------------------------------------------
#### Function for registering daemon/service with init
# ARGS: 1
# ARG1: daemon to be registered
sub register_service
{
    my $srv = $_[0];
    my $status;
    my $ckptName = "ROOTCRS_OHASD";
    
    # call OSD API
    $status = s_register_service ($srv);
    if ($status == SUCCESS)
    {
         if ($srv eq "ohasd") {
         # install_initd if it's SuSE Linux
            s_install_initd() or die "Can't install $srv service: $!";
         }
      writeCkpt($ckptName,CKPTSUC);
    } else {
      writeCkpt($ckptName,CKPTFAIL);
    }

    $wipCkptName = "ROOTCRS_STACK";
    return $status;

}

####---------------------------------------------------------
#### Function for unregistering daemon/service
# ARGS: 1
# ARG1: daemon to be registered
sub unregister_service
{
    my $srv = $_[0];

    # call OSD API
    return s_unregister_service ($srv);
}


sub perform_start_service
{
  my $srv = $_[0];

  # Check if the service/daemon has started
  trace ("Checking the status of $srv");
  my $ohasdStatus = check_service ($srv, 3, FALSE);

  if ($ohasdStatus) {
    trace ("$srv is already running");
  } else {
    trace ("$srv is not already running.. will start it now");
    start_service($srv);
  }


  # Check if the service/daemon has started
  trace ("Checking the status of $srv");
  $ohasdStatus = check_service ($srv, 360);

  if ($ohasdStatus) {
    trace ("$srv started successfully");
  } else {
    trace ("$srv has failed to start");
    exit 1;
  }
}

####---------------------------------------------------------
#### Function for starting daemon/service
# ARGS: 2
# ARG1: daemon to be started
# ARG2: user as whom daemon/service needs to be started
sub start_service
{
    my $srv  = $_[0];
    my $user = $_[1];
    my $status;

    # call OSD API
    $status = s_start_service ($srv);

    if ($status == SUCCESS) {
       trace("Started service '$srv'");
    } else {
       trace("Failed to start  service '$srv'");
    }

    return $status;
}

####---------------------------------------------------------
#### Function for stopping daemon/service
# ARGS: 2
# ARG1: daemon to be stopped
# ARG2: user as whom daemon/service needs to be stopped
sub stop_service
{
    my $srv = $_[0];
    my $user = $_[1];

    return s_stop_service ($srv, $user);
}

####---------------------------------------------------------
#### Function for checking daemon/service
# ARGS: 2
# ARG1: daemon to be checked
# ARG2: num retries
# ARG3: boolean to enable error output or not.
sub check_service
{
    my $srv = $_[0];
    my $retries = $_[1];
    my $showtrace = $_[2];

    my $srv_running = FALSE;
    my $CRSCTL = crs_exec_path("crsctl");
    my $cmd = "$CRSCTL check $srv";
    my $grep_val;
    my @chk;
    my @cmdout;

    # for OHASD, we need to grep for CRS-4638
    # cannot use grep on Windows, customers are unlikely to have grep
    # on their systems

    # for CRS, we need to grep for CRS-4537
    if ($srv eq "ohasd") {
      $grep_val = "4638";
      $cmd = "$CRSCTL check has";
    } 
    elsif ($srv eq "cluster") {
      my $node  = $CFG->HOST;
      $cmd      = "$CRSCTL check $srv -n $node";
      $grep_val = "4537";
      trace("Running $cmd");
    }
    elsif ($srv eq "css") {
      $grep_val = "4529";
    } 
    else {
      # Add a check before the stat
      $cmd      = "$CRSCTL check resource $srv -init";
      @chk = system_cmd_capture1($showtrace, $cmd);
      # Return code of command is set on close, so capture now
      my $rc0 = shift @chk;
      if ($rc0 != 0) {
        trace("Check of service \"$_[0]\" failed\n".join("\n", @chk));
      }

      $cmd      = "$CRSCTL status resource $srv -init";
      $grep_val = "STATE=ONLINE";
    }

    # Wait for srv to start up
    while ($retries && ! $srv_running) {
      @chk = system_cmd_capture1($showtrace, $cmd);
      # Return code of command is set on close, so capture now
      my $rc = shift @chk;

      if ($grep_val) { @cmdout = grep(/$grep_val/, @chk); } # for OHASD

      # if scalar(@cmdout) > 0, we found the msg we were looking for
      if (($grep_val && scalar(@cmdout) > 0) ||
          (!$grep_val && $rc == 0)) {
        $srv_running = TRUE;
      }
      else {
        trace ("Checking the status of $srv");
        sleep (5);
        $retries--;
      }
    }

    # perform OSD actions
    s_check_service ($srv, $srv_running);

    return $srv_running;
}

sub start_resource
{
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $CRSCTL = catfile ($ORA_CRS_HOME, "bin", "crsctl");
  my @cmd = ($CRSCTL, 'start', 'resource', (@_));
  my $success = TRUE;
  my @out;

  @out = system_cmd_capture_noprint(@cmd);
  my $status = shift @out;
  if ($status == 0) {
    if (scalar(@out) >0 ) {
      trace(join("\n>  ", ("Command output:", @out)),
            "\n>End Command output");
    }

    trace("Start of resource \"$_[0]\" Succeeded");
  }
  elsif (scalar(grep(/CRS-0*5702/i, @out)) != 0) { # resource already up
    trace("The resource \"$_[0]\" was already running");
  }
  elsif ($status != 0) 
  {
    if (scalar(@out) >0 ) {
      trace(join("\n>  ", ("Command output:", @out)),
            "\n>End Command output");
    }

    if (!check_service($_[0], 10))
    {
      error("Start of resource \"$_[0]\" failed\n".join("\n", @out));
      $success = FALSE;
    }
    else
    {
      trace("The resource \"$_[0]\" is already running");
    }
  }

  return $success;
}

sub stop_resource
{
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $CRSCTL = catfile ($ORA_CRS_HOME, "bin", "crsctl");
  my @cmd = ($CRSCTL, 'stop', 'resource', (@_));
  my $success = TRUE;
  my @out;

  @out = system_cmd_capture_noprint(@cmd);
  my $status = shift @out;

  if ($status == 0) { # stop successful
    if (scalar(@out) >0 ) {
      trace(join("\n>  ", ("Command output:", @out)),
            "\n>End Command output");
    }
  }
  elsif (scalar(grep(/CRS-0*2500/i, @out)) != 0) { # resource not up
    trace("Resource '@_' was not online but was successfully stopped");
  }
  elsif (check_service($_[0], 1)) { # check if service is still running
    trace("Stop of resource \"@_\" failed\n".join("\n", @out));
    $success = FALSE;
  }

  return $success;
}

sub stop_diskmon
{
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $CRSCTL = catfile ($ORA_CRS_HOME, "bin", "crsctl");
  my $success = TRUE;

  # no diskmon in windows
  if ($CFG->platform_family eq "windows")
  {
    return TRUE;
  }

  if (check_service("ora.diskmon", 2)) {

    my @output = system_cmd_capture($CRSCTL,
                                    "stop",
                                    "resource",
                                    "ora.diskmon",
                                    "-init");
    my $status = shift @output;

    if ($status != 0 && !scalar(grep(/CRS\-2500/, @output)))
    {
        error("Stop of resource \"ora.diskmon\" failed\n".join("\n", @output));
        $success = FALSE;
    }
  }
  return $success;
}

sub local_only_config_exists {
  my $found      = FALSE;
  my $local_only = s_get_config_key("ocr", "local_only");
  my $ocrcfg_loc = s_get_config_key("ocr", "ocrconfig_loc");
  my $db_home    = "";

  if ($local_only =~ m/true/i) {
     $CFG->oldconfig('OCRCONFIG', $ocrcfg_loc);

     # get older version DBHOME path
    if ($ocrcfg_loc =~ m/(.+).cdata.localhost.local.ocr/) {
       $db_home = $1;
       if ($db_home) {
          $CFG->oldconfig('DB_HOME', $db_home);
          $found = TRUE;
          trace ("local_only config exists");
          #get old  releaseversion of SIHA/SI-CSS.
          my @oldCrsVer = get_has_version($db_home);
          $CFG->oldconfig('ORA_CRS_VERSION', \@oldCrsVer);
       }
       else {
          error ("Failed to find earlier version DBHOME");
       }
    }
    else {
       error ("OCR location file /etc/oracle/ocr.loc is corrupted.\n" .
              "If this is a fresh install, ensure that /etc/oracle is empty");
    }
  }

  return $found;
}

####---------------------------------------------------------
#### Function name : migrate_dbhome_to_SIHA
# ARGS 0:
# This routine does the operations in the following sequence.
# 1) Take a backup copy of older ocr file.
# 2) Update the location of ocr.loc
# 3) touch and change file permissions.
# 4) Create necessary configuration with 'crsctl pin css' command.

sub migrate_dbhome_to_SIHA {
  my $db_home;
  my $ret = FAILED;
  my $status;
  my $OCRCONFIGBIN = crs_exec_path("ocrconfig");
  my $CRSCTLBIN = crs_exec_path("crsctl");
  my $HOST = tolower_host();
  my $ORACLE_OWNER = $CFG->params('ORACLE_OWNER');
  my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
  $ENV{'NLS_LANG'} = $CFG->params('LANGUAGE_ID');

  my $ocrcfg_loc = $CFG->oldconfig('OCRCONFIG');
  my $copy_lococr = "$ORACLE_HOME/cdata/localhost/localsiasm.ocr";
  my $new_lococr = "$ORACLE_HOME/cdata/localhost/local.ocr";

  # copy over older local-only OCR to SIHA home
  if (defined $ocrcfg_loc) {
     if (copy_file ($ocrcfg_loc, $copy_lococr) != SUCCESS) {
        error ("Copy of older local-only OCR failed");
     }
  }

  # Now touch, set owner and perm
  if (!(-e $new_lococr)) {
    if ($CFG->DEBUG) { trace ("Creating empty file $new_lococr");}
    # create an empty file
    open (FILEHDL, ">$new_lococr") or die "Can't create $new_lococr: $!";
    close (FILEHDL);
  }
  # Set ownership/group
  if ($CFG->DEBUG) {
    trace ("s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $new_lococr)");
  }
  s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $new_lococr)
    or die "Can't set ownership on $new_lococr";

  # Set permissions, if specified
  s_set_perms ("0640", $new_lococr)
    or die "Can't set permissions on $new_lococr";

  # update ocr.loc
  if (defined $ocrcfg_loc) {
     if (0 !=  system_cmd($OCRCONFIGBIN, '-repair',
                                         '-replace', $ocrcfg_loc,
                                         '-replacement', $new_lococr)) {
        error("Replace of older local-only OCR failed");
     }
  }

  # Create configuration needed for compatibility with older DBs (10.x, 11.1).
  $ret = configure_OCR();
  if (!$ret)
  {
    error("Error createing local-only OCR ");
    trace ("Creating local-only OCR ... failed");
  }
  else
  {
    trace ("Creating local-only OCR ... succeeded");
  }

  return $ret;

}

sub local_only_stack_active {
  $ENV{'ORACLE_HOME'} = $CFG->oldconfig('DB_HOME');

  my $restart_css = FALSE;
    # check if older version SI CSS is running
  my $OLD_CRSCTL = catfile ($CFG->oldconfig('DB_HOME'), "bin", "crsctl");
  my $status = system ("$OLD_CRSCTL check css");

  if (0 == $status) {
    # set flag to restart SIHA CSS before we're done
    $restart_css = TRUE;
  }

  $ENV{'ORACLE_HOME'} = $ORACLE_HOME;
  return $restart_css;
}

sub stop_local_only_stack {
   my $OLD_SIHOME    = $CFG->oldconfig('DB_HOME');
   my $stack_stopped = SUCCESS;
   my $status;

   #Bug 8280425. Take a backup of ocr.loc and local.ocr
   #before invoking localconfig -delete
   my $local_ocr      = catfile($OLD_SIHOME, 'cdata',
				'localhost', 'local.ocr');
   my $local_ocr_save = catfile($OLD_SIHOME, 'cdata', 
				'localhost', 'local.ocr.save');
   my ($ocr_loc, $ocr_loc_save);
   
   if ($CFG->platform_family eq "windows") {
      $ocr_loc      = $OCRLOC;
      $ocr_loc_save = $OCRLOC . '.save';
      trace("backing up $ocr_loc registry");
      s_copyRegKey($ocr_loc, $ocr_loc_save);
   }
   else {
      $ocr_loc      = catfile ($OCRCONFIGDIR, 'ocr.loc');
      $ocr_loc_save = catfile ($OCRCONFIGDIR, 'ocr.loc.save');
      trace("backing up $ocr_loc");
      if (copy_file ($ocr_loc, $ocr_loc_save) != SUCCESS) {
         error ("backup of $ocr_loc failed");
      }
   }

   # backing up local_ocr
   trace("backing up $local_ocr");
   if (copy_file ($local_ocr, $local_ocr_save) != SUCCESS) {
      error ("backup of $local_ocr failed");
   }

   if ($CFG->platform_family eq "windows") {
      s_stopDeltOldASM();
      s_stopService("OracleCSService");
      if (s_isServiceRunning("OracleCSService")) {
         s_stopService("OracleCSService");
      }

      s_deltService("OracleCSService")
   }

  # stop old SI CSS
   trace ("Stopping older version SI CSS");
   my $OLD_LOCALCONFIGBIN = catfile ($CFG->oldconfig('DB_HOME'),
                                     "bin", "localconfig");
   $status = system ("$OLD_LOCALCONFIGBIN delete");
   if ($status == 0) {
      trace ("Older version SI CSS successfully stopped/deconfigured");
   }
   else {
      $stack_stopped = FAILED;
      error ("Failed to stop/deconfigure older version SI CSS");
   }

   # Bug 8280425 'localconfig -delete' removes the ocr.loc and local.ocr
   # restore the same.
   if ($CFG->platform_family eq "windows") {
      trace("restoring $ocr_loc registry");
      s_copyRegKey($ocr_loc_save, $ocr_loc);
   }
   else {
      trace("restoring $ocr_loc");
      if (copy_file ($ocr_loc_save, $ocr_loc) != SUCCESS) {
         error ("Restore of older $ocr_loc failed");
      }
   }

  # restoring local_ocr
   trace("restoring $local_ocr");
   if (copy_file ($local_ocr_save, $local_ocr ) != SUCCESS) {
      error ("Restore of $local_ocr failed");
   }


   if ($CFG->platform_family eq "unix") {
      s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $ocr_loc)
               or die "Can't change ownership on $ocr_loc";

      s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $local_ocr)
               or die "Can't change ownership on $local_ocr";
   }

   return $stack_stopped;
}

#
## The following APIs have been pulled in from crsconfig_util.pm
#

sub source_file
{
    my $file = $_[0];

    open (SRCFILE, $file) or die "Couldn't open $file: $!";
    my $contents = join "", <SRCFILE>;
    close SRCFILE;

    eval $contents;
    die "Couldn't eval $file: $@\n" if $@;
}

sub read_file
{
    my $file = $_[0];
    open (FILE, "<$file") or die "Can't open $file: $!";
    my @contents = (<FILE>);
    close (FILE);

    return @contents;
}

# ARGS
# ARG 0: paramfile
sub setup_param_vars
{
    my $paramfile = $_[0];
    my $platform  = s_get_platform_family();

    # To support the use of 'strict', it is necessary to create a small
    # program to be executed via 'eval'
    # Because 'strict' requires all variables to be declared, the scope
    # of 'my' variables is the program, and variables that are defined
    # by the parameter file are used in the definition of subsequent
    # parameter file entries, e.g. DIRPREFIX, to get the scoping right
    # it is necessary to create a program that will allow previously
    # defined values
    my @epgm;
    open(PARAMFILE, $paramfile) or die "Cannot open $paramfile: $!";

    while (<PARAMFILE>) {
        if ($_ !~ /^#|^\s*$/) {
            # The magic below takes params of the form KEY=VAL and sets them as
            # variables in the perl context
            chomp;
            $_ = trim ($_);
            my ($key, $val) = split ('=');

            # store this in a hash that is returned
            if ((0 > index($val,'"')) && $key ne 'ASM_DISK_GROUP') {
               if (! is_dev_env()) {
                  # get ORACLE_OWNER from 'USERNAME' env on NT
                  if ($key eq 'ORACLE_OWNER' && $platform eq 'windows') {
                     $val             = $ENV{'USERNAME'};
                     my $userDomain   = $ENV{'USERDOMAIN'};
                     $userDomain      =~ tr/A-Z/a-z/;
                     my $computerName = $ENV{'COMPUTERNAME'};
                     $computerName    =~ tr/A-Z/a-z/;
                     # if $userDomain is the same as $computerName, it means
                     # the user is in the workgroup.  Domain is not needed.
                     if ($userDomain && ($userDomain ne $computerName)) {
                        $val = $userDomain . '\\' . $val;
                     }
                  }
               }

              # escape \ (for NT)
              $val =~ s!\\!\\\\!g;
              push @epgm, "my \$$key=\"$val\";";
            } else { # won't allow perl var subst
              # escape "'"
              $val =~ s!\'!\\'!g;
              push @epgm, "my \$$key='$val';";
            }
            push @epgm, '$CFG->params(', "'$key',\$$key);";
        }
    }

    close (PARAMFILE);
    eval("@epgm");

    # if there was an error log it
    if ($@) { trace($@); }

    return;
}

# ARGS
# none
sub instantiate_config_params
{
    # If it contains a pattern of the form '%foo%' AND a mapping exists
    # for 'foo', replace '%foo%' with the corresponding value.
    my $rexp="[a-zA-Z_%]+";
    foreach (@_) {
      my @matchlist = $_ =~ /%(${rexp})%/g;
      foreach my $match (@matchlist) {
        if (defined($CFG->config_value($match))) {
          my $sub = $CFG->config_value($match);
          $_ =~ s/%(${match})%/$sub/g;
        }
        elsif ($CFG->defined_param($match)) {
          my $sub = $CFG->params($match);
          $_ =~ s/%(${match})%/$sub/g;
        }
        elsif ($match eq "%%") {
          $_ =~ s/%(${match})%/$match/;
        }
      }
      @matchlist = $_ =~ /\$(${rexp})/g;
      foreach my $match (@matchlist) {
        if ($CFG->config_value($match)) {
          my $sub = $CFG->config_value($match);
          $_ =~ s/\$(${match})/$sub/g;
        }
        elsif ($CFG->defined_param($match)) {
          my $sub = $CFG->params($match);
          $_ =~ s/\$(${match})/$sub/g;
        }
      }
    }
}

# Module for script instantiation
#
# Instantiate all files in $CH/crs/sbs/ directory -- replace %FOO%
# with value for FOO (obtained from crsconfig_params) -- and place
# this in $CH/crs/utl/ directory
#
# ARGS
# none
sub instantiate_scripts
{
    my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
    my $sbsdir   = catfile ($ORA_CRS_HOME, "crs", "sbs");
    my @sbsfiles = glob (catfile ($sbsdir, "*.sbs"));

    $wrapdir_crs = catfile ($ORA_CRS_HOME, "crs", "utl");

    # create $wrapdir_crs if it doesn't exist already
    create_dir ($wrapdir_crs);

    foreach my $srcfile (@sbsfiles) {
        my @sbsfile = read_file ($srcfile);

        # strip off .sbs suffix
        (my $dstfile = basename ($srcfile)) =~ s/\.sbs//g;
        my $dstpath = catfile ($wrapdir_crs, $dstfile);
        if ($DEBUG) { trace ("SRC FILE: $srcfile; DST PATH: $dstpath");}

        open (DSTPATH, ">${dstpath}")
            or die "Can't open $dstpath: $!";

        foreach my $line (@sbsfile) {
            # skip blanks and comments
            if ( ($line !~ /^#|^\s*$/) && ($line !~ /\s*case/i)){
                instantiate_config_params ($line);
            }
            print DSTPATH "$line";
        }

        close (DSTPATH);
    }
}

# ARGS
# none
sub create_dirs
{
    # Directories Creation module
    #
    # Create directories with ownership/permissions as specified in
    # crs/utl/crsconfig_dirs

    my $dirs = shift;

    foreach my $line (@$dirs) {
        chomp ($line);
        next if ($line =~ /^#|\$|^\s*$/);  # skip blanks, comments, and lines that 
                                           # unable to instantiate
        # replace variables in input line
        my @matches = $line =~ /(\$\w+)/g;
        for my $match (@matches) {
          if (defined($CFG->config_value($match))) {
            my $sub = $CFG->config_value($match);
            $line =~ s/${match}/$sub/g;
          } elsif ($CFG->defined_param($match)) {
            my $sub = $CFG->params($match);
            $line =~ s/${match}/$sub/g;
          }
        }

        if ($DEBUG) { trace ("crsconfig_dirs: LINE is $line");}
        my ($platform, $dir_path, $owner, $grp, $perms) = split (/ /, $line);

        my $myplatformfamily = s_get_platform_family ();
        $myplatformfamily =~ tr/A-Z/a-z/;

        if (($platform eq "all") || ($platform =~ m/$myplatformfamily/)) {

            my @dirs_created = create_dir ($dir_path);

            # if no dir was created, add dir_path to list to set ownership/perms
            # below
            if (!@dirs_created)
            {
                if ($DEBUG) {
                    trace ("  no dir created; adding $dir_path to list");
                }
                push (@dirs_created, $dir_path);
            }

            # Setting same ownership/permissions for all intermediate dirs
            # as well
            if (@dirs_created) {
                foreach my $dir (@dirs_created) {
                    # Skip setting ownership and permissions of the /etc/init.d directory on AIX
                    # because this directory doesn't exist on AIX
                    # hence: on NT, s_set_ownergroup and s_set_perms have no op
   		    if ($CFG->platform_family ne "windows") {
                       next if (($OSNAME eq 'aix') && (trim($dir) eq trim($ID))); 
                       # Change permissions on /etc/init.d and /etc/inittab only for the test environment
                       next if ((! is_dev_env()) && ((trim($dir) eq trim($ID)) || 
                                                     (trim($dir) eq trim($IT))) );
                       # Change permissions on RCALLDIR dir's only for the
                       # test environment
                       my $set_perms;
                       $set_perms = check_dir_in_RCALLDIR($dir) ;
                       next if ((! is_dev_env()) && ($set_perms == TRUE)) ;
                       if ($DEBUG) {
                          trace ("  s_set_ownergroup ($owner, $grp, $dir)");
                       }
                       s_set_ownergroup ($owner, $grp, $dir)
                          or die "Can't change ownership on $dir";
                    
                       if ($perms) {
                          if ($DEBUG) {trace ("  s_set_perms ($perms, $dir)");}
                          s_set_perms ($perms, $dir)
                             or die "Can't set permissions on $dir";
                       }
                    }
                }
            }

        }
    }
}

#
# Check whether given directory ( from crsconfig_dirs ) present
# in RCALLDIR dir's. 
#
# ARGS
# arg 0 -- dir 
sub check_dir_in_RCALLDIR
{ 
    my $dir = $_[0];
    my @RCALLDIRLIST = split ( /\s+/, $CFG->params('RCALLDIR'));
    foreach my $rc (@RCALLDIRLIST) {
        if ( trim($rc) eq trim($dir) ) {
            return TRUE ; 
        }
    }
    return FALSE ; 
}

# ARGS
# none
sub copy_wrapper_scripts
{
    #
    # Wrapper copy module
    #
    # Copy files from SOURCE to DEST as specified in
    # crs/utl/crsconfig_files
    #
    my @wcfile = read_file (catfile ($wrapdir_crs, "crsconfig_files"));
    foreach my $line (@wcfile) {
        chomp ($line);
        next if ($line =~ /^#|^\s*$/);  # skip blanks and comments
        my ($platform, $src, $dst) = split (/ /, $line);

        my $myplatformfamily = s_get_platform_family ();
        $myplatformfamily =~ tr/A-Z/a-z/;

        if (($platform eq "all") || ($platform =~ m/$myplatformfamily/)) {
            # If the dest file already exists, first remove it
            if (-e $dst) {
                if ($DEBUG) { trace ("unlink ($dst)");}
                unlink ($dst) or error ("Can't delete $dst: $!");
            }
            if ($DEBUG) { trace ("copy ($src, $dst)");}
            copy($src, $dst) or error ("Can't copy $src to $dst: $!");
        }
    }
}

#
# File permissions module
#
# Set ownership/permissions as specified in
# crs/utl/crsconfig_fileperms (after touching the file, if
# required)
#
# ARGS
# arg 0 -- param hash
sub set_file_perms
{
    my $fpfile     = shift;
    my $SUPERUSER  = $CFG->SUPERUSER;
    my $myplatform = $CFG->platform_family;
    my ($file_name, $bin_file);

    foreach my $line (@$fpfile) {
        chomp ($line);
        next if ($line =~ /^#|^\s*$/);  # skip blanks and comments

        # replace variables in input line
        $line =~ s/(\$\w+)/$1/eeg;
        if ($DEBUG) { trace ("crsconfig_fileperms: LINE is $line");}

        my ($platform, $file_path, $owner, $grp, $perms) = split (/\s+/, $line);
        if (($platform eq "all") || ($platform =~ m/$myplatform/)) {
            if (!(-e $file_path) &&  ( $file_path !~ m/.*acfs.*/ )) {
                if ($CFG->DEBUG) { trace ("Creating empty file $file_path");}
                # create an empty file
                open (FILEHDL, ">$file_path") or die(dieformat(255, $file_path, $!));
                close (FILEHDL);
            } elsif (!(-e $file_path) &&  ( $file_path =~ m/.*acfs.*/ )) {
                trace ("skipping  set file perm for acfs file $file_path. File does not exist");
                next;
            }

            # Set ownership/group
            if ($CFG->DEBUG) {
              trace ("s_set_ownergroup ($owner, $grp, $file_path)");
            }

            $file_name = basename($file_path);

            # in dev env, perms should not be set on symlinks
            my $link_path;
            if ($link_path = s_isLink ($file_path)) {
              $bin_file = TRUE;
            }
            elsif (($file_name =~ /.bin/) && ($owner eq $CFG->HAS_USER)) {
               $bin_file = TRUE;
            }
            else {
               $bin_file = FALSE;
            }

            if ($bin_file && is_dev_env()) {
               trace("Development env... Not setting permissions on $file_name");
            }
            else {
               if (($file_name =~ /skgxn2/) && ($link_path))
               {
                 trace("Skip changing the ownership and ".
                       "permissions of $file_name");
                 next;
               }

               # Set ownership/group
               s_set_ownergroup ($owner, $grp, $file_path)
                    or die "Can't set ownership on $file_path";

               # Set permissions, if specified
               if ($perms) {
                  if ($CFG->DEBUG) { trace ("s_set_perms ($perms, $file_path)");}
                  s_set_perms ($perms, $file_path)
                        or die "Can't set permissions on $file_path";
               }
            }
        }
    }

   if (! is_dev_env() && (! $CFG->IS_SIHA)) {
      if ($myplatform eq "unix") {
         if ((!$CFG->ASM_STORAGE_USED) && (! isOCRonASM())) {
	    # in an upgrade, OCR_LOCATIONS would be empty.  
	    if ($CFG->UPGRADE) {
	       my $ocrconfig_loc = s_get_config_key('ocr', 'ocrconfig_loc');
	       my $ocrmirror_loc = s_get_config_key('ocr', 'ocrmirrorconfig_loc');
	       trace ("ocrconfig_loc=$ocrconfig_loc");
	       trace ("ocrmirror_loc=$ocrmirror_loc");
	       if ($ocrconfig_loc) {
	          # check if it's a symbolic link
	          if (-l $ocrconfig_loc) {
		     my $abs_ocr = s_getAbsLink($ocrconfig_loc);
		     s_setParentDirOwner ($SUPERUSER, $abs_ocr);
		  }
		  else {
                     if (-e $ocrconfig_loc) {
		        s_setParentDirOwner ($SUPERUSER, $ocrconfig_loc);
                     }
	          }
	       }

	       if ($ocrmirror_loc) {
	          if (-l $ocrmirror_loc) {
		     my $abs_ocrmirror = s_getAbsLink($ocrmirror_loc);
		     s_setParentDirOwner ($SUPERUSER, $abs_ocrmirror);
		  }
		  else {
                     if (-e $ocrmirror_loc) {
		        s_setParentDirOwner ($SUPERUSER, $ocrmirror_loc);
                     }
	          }
	       }
	    }
	    else {
	       my @ocr_locs = split (/\s*,\s*/, $CFG->params('OCR_LOCATIONS'));
               foreach my $loc (@ocr_locs) {
		  # Set owner/group of OCR path to root/dba
		  trace ("set owner/group of OCR path");
                  if (-e $loc) {
		     s_setParentDirOwner ($SUPERUSER, $loc);
                  }
               }
            }
         }
      }
   }
}


# ARGS
# none
sub add_RCALLDIR_to_dirs
{
    my $SUPERUSER = $CFG->SUPERUSER;
    my $dirsfile = catfile ($wrapdir_crs, "crsconfig_dirs");
    open (DIRSFILE, ">>$dirsfile")
        or die "Can't open $dirsfile for append: $!";

    my $myplatformfamily = s_get_platform_family ();
    $myplatformfamily =~ tr/A-Z/a-z/;

    # add RCALLDIR locations to crsconfig_dirs
    my @RCALLDIRLIST = split (/ /, $RCALLDIR);
    foreach my $rc (@RCALLDIRLIST) {
        print DIRSFILE "$myplatformfamily $rc $SUPERUSER $SUPERUSER 0755\n";
    }

    close (DIRSFILE);
}

sub add_ITDIR_to_dirs
#---------------------------------------------------------------------
# Function: add IT_DIR directory to crsconfig_dirs
# Args    : none
#---------------------------------------------------------------------
{
   if ($CFG->defined_param('IT_DIR')) {
      if (is_dev_env()) {
	 my $itdir     = $CFG->params('IT_DIR');
         my $SUPERUSER = $CFG->SUPERUSER;
         my $dirsfile  = catfile ($wrapdir_crs, 'crsconfig_dirs');
         my $platform  = s_get_platform_family();

         open (DIRSFILE, ">>$dirsfile")
              or die "Can't open $dirsfile for append: $!";
         print DIRSFILE "$platform $itdir $SUPERUSER $SUPERUSER 0755\n";
         close (DIRSFILE);
      }
   }
}

sub add_localOlr_OlrConfig_OcrConfig
#-------------------------------------------------------------------------------
# Function: add local olr to crsconfig_fileperms file
# Args    : none
#-------------------------------------------------------------------------------
{
   my $SUPERUSER     = $CFG->SUPERUSER;
   my $ORACLE_OWNER  = $CFG->params('ORACLE_OWNER');
   my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
   my $platform      = $CFG->platform_family;
   my ($OCRCONFIG, $OLRCONFIG);

   # open crsconfig_fileperms
   my $permsfile = catfile ($wrapdir_crs, "crsconfig_fileperms");
   open (FPFILE, ">>$permsfile")
        or die "Can't open $permsfile for append: $!";

   # add OLR_LOCATION
   if ($CFG->IS_SIHA) {
      print FPFILE
            "$platform $OLR_LOCATION $ORACLE_OWNER $ORA_DBA_GROUP 0600\n";
   }
   else {
      print FPFILE
            "$platform $OLR_LOCATION $SUPERUSER $ORA_DBA_GROUP 0600\n";
   }

   if ($platform eq "unix") {
      $OCRCONFIG = $CFG->params('OCRCONFIG');
      $OLRCONFIG = $CFG->params('OLRCONFIG');

      # add OLRCONFIG
      if ($OLRCONFIG) {
         if (is_dev_env()) {
            print FPFILE "$platform $OLRCONFIG " .
                         "$ORACLE_OWNER $ORA_DBA_GROUP 0644\n";
         } else {
            print FPFILE "$platform $OLRCONFIG " .
                         "$SUPERUSER $ORA_DBA_GROUP 0644\n";
         }
      }

      # add OCRCONFIG
      if (! $CFG->IS_SIHA) {
         if ($OCRCONFIG) {
            print FPFILE
                  "$platform $OCRCONFIG $SUPERUSER " .
                   "$ORA_DBA_GROUP 0644\n";
         }
      }
   }

   close (FPFILE);
}

sub set_perms_ocr_vdisk
{
   if ($CFG->UPGRADE) {
      return SUCCESS;
   }

   my $IS_SIHA       = $CFG->IS_SIHA;
   my $SUPERUSER     = $CFG->SUPERUSER;
   my $ORACLE_OWNER  = $CFG->params('ORACLE_OWNER');
   my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
   my $platform      = $CFG->platform_family;
   my ($line, @dirs, @files);

   # add OCR_LOCATION and OCR_MIRROR_LOCATION
   if (! $IS_SIHA) {
      # OCR permissions need to change to 0600 when CSSD dependency on OCR
      # goes away. Bypass if ASM is used.

      if (!$CFG->ASM_STORAGE_USED) {
         my @ocr_locs = split (/\s*,\s*/, $OCR_LOCATIONS);
         my $loc;
         foreach $loc (@ocr_locs) {
            $line = join (' ', $platform, $loc,
                          $SUPERUSER, $ORA_DBA_GROUP, '0640');
            push @files, $line;

            # set owner and permission of OCR directory
            my @loc_dirs;
            if ($platform eq "windows") {
               @loc_dirs = split (/\\/, $loc);
            }
            else {
               @loc_dirs = split (/\//, $loc);
            }

            my $nbr_of_levels = scalar (@loc_dirs);
            # $nbr_of_levels = 2 means it's at the root directory (exp: R:\ocr).
            # Therefore, no need to add to crsconfig_dirs
            if ($nbr_of_levels > 2 ) {
               my ($dir) = split ($loc_dirs[$nbr_of_levels-1], $loc);
               $line     = join (' ', $platform, $dir,
                                 $ORACLE_OWNER, $ORA_DBA_GROUP, '0755');
               push @dirs, $line;
           }
         }
      }
   }

   # add all voting disks.  Bypass if ASM is used.
   # XXX: is this step required? Existing shell scripts don't seem to be
   # using validate_VDisks() function
   if (!$CFG->ASM_STORAGE_USED) {
      my @votingdisks = split (/\s*,\s*/, $VOTING_DISKS);

      my $vdisk;
      foreach $vdisk (@votingdisks) {
         # voting disk should not be precreated since the startup script may
         # be run as a different user than crs user. Precreating/touching
         # a voting disk prematurely will cause later I/Os to fail, such as
         # voting file upgrade/create
         #
         # set owner and permission of votind disks directory by adding to
         # crsconfig_dirs
         my @vdisk_dirs;
         if ($platform eq "windows") {
            @vdisk_dirs = split (/\\/, $vdisk);
         }
         else {
            # other platforms
            @vdisk_dirs = split (/\//, $vdisk);
         }

         my $nbr_of_levels = scalar (@vdisk_dirs);
         # $nbr_of_levels = 2 means it's at the root directory (exp: R:\vdisk).
         # Therefore, no need to add to crsconfig_dirs
         if ($nbr_of_levels > 2 ) {
            my ($dir) = split ($vdisk_dirs[$nbr_of_levels-1], $vdisk);
            $line     = join (' ', $platform, $dir,
                              $ORACLE_OWNER, $ORA_DBA_GROUP, '0755');
            push @dirs, $line;
         }
      }
   }

   # add OCRCONFIGDIR and OLRCONFIGDIR to crsconfig_dirs
   my $owner;
   if (is_dev_env()) {
      $owner = $ORACLE_OWNER;
   } else {
      $owner = $SUPERUSER;
   }

   if ($OCRCONFIGDIR) {
      $line = join (' ', $platform, $OCRCONFIGDIR, $owner, $ORA_DBA_GROUP, '0755');
      push @dirs, $line;
   }

   if (($OLRCONFIGDIR) && ($OLRCONFIGDIR ne $OCRCONFIGDIR)) {
      $line = join (' ', $platform, $OLRCONFIGDIR, $owner, $ORA_DBA_GROUP, '0755');
      push @dirs, $line;
   }

   create_dirs (\@dirs);
   set_file_perms (\@files);
}

sub isFirstNodeToStart 
######################################################################
# Returns:
#   FALSE   if node is not first to start
#   TRUE    if node is     first to start
######################################################################
{
   my $isFirst = FALSE;

   # Get the list of nodes that have started
   my $olsnodes = catfile($CFG->ORA_CRS_HOME, 'bin', 'olsnodes');
   open ON, "$olsnodes |";
   my @olsnodes = (<ON>);
   close ON;

   chomp @olsnodes;

   if (scalar(@olsnodes) == 1)
   {
      $isFirst = TRUE;
   }

   return $isFirst;
}

# Arguments:
#   0. Name of host to check
#   1. List of nodes in config
#
# Returns:
#   FALSE   if node is not last to start
#   TRUE    if node is     last to start
sub isLastNodeToStart {
  my $nodelst   = $_[1];
  my $hostname  = $_[0];
  my $isLast = FALSE;
  my $lastnode = 0;
  my %nodes;

  my @nodelist = split(',', $nodelst);

  # Get the list of nodes that have started
  my $olsnodes = catfile($CFG->ORA_CRS_HOME, 'bin', 'olsnodes');
  open ON, "$olsnodes -n|";
  my @olsnodes = (<ON>);
  close ON;

  chomp @olsnodes;

  # If all of the nodes in the configuration are up, find out if
  # we are the last node in the list
  #
  # There are 2 'special' cases to consider.
  #  Node numbers starting from 0 (numbers hould start from 1 soon
  #  Leases are not sequential (some lease slots not taken)
  # For these cases, we want to identify the highest node number
  # in use and select that node as the last node to start
  if (scalar(@nodelist) == scalar(@olsnodes)) {
    my ($nodename, $nodenum);

    # Create a hash with key of node number, value of hostname
    %nodes = map { ($nodename, $nodenum) = (split(' ', $_));
                   if ($nodenum > $lastnode) {
                     $lastnode = $nodenum;
                   }
                   # Get the highest node number of nodes started
                   $nodenum => $nodename;
                 } @olsnodes;
    if ($hostname =~ /$nodes{$lastnode}/i) {
       $isLast = TRUE;
       trace "Host $hostname is the last node to start";
    }
  }

  return $isLast;
}

sub isLastNodeToUpgrade
{
   trace ("isLastNodeToUpgrade...");

   if ($CFG->force) {
      trace("force upgrade invoked");
      return TRUE;
   }

   if (defined $CFG->isLstNodeToUpgrade)
   {
     my $cachedValue = $CFG->isLstNodeToUpgrade;
     trace("isLastNodeToUpgrade: $cachedValue");
     return $CFG->isLstNodeToUpgrade;
   }

   my $lastnode_to_upgrade = TRUE;
   my $crsctl = catfile ($CFG->params('ORACLE_HOME'), "bin", "crsctl");

   # get current releaseversion
   open (QUERYCRS, "$crsctl query crs releaseversion |");
   my $output = <QUERYCRS>;
   close (QUERYCRS);

   my $release_version = getVerInfo($output);

   trace ("release_version=$release_version");

   # get current softwareversion
   my @nodes = split (/,/, $CFG->params('NODE_NAME_LIST'));
   my $software_version;
   my $host = tolower_host();

   foreach my $nodename (@nodes) {
      if (lc($nodename) =~ /\b$host\b/i) { next; }
      open (QUERYCRS, "$crsctl query crs softwareversion $nodename |");
      $output = <QUERYCRS>;
      close (QUERYCRS);

      $software_version = getVerInfo($output);

      trace ("software_version on $nodename=$software_version");

      # compare version
      if ($software_version ne $release_version) {
         $lastnode_to_upgrade = FALSE;
      }
   }

   trace("last node to upgrade is $lastnode_to_upgrade");
   $CFG->isLstNodeToUpgrade($lastnode_to_upgrade);
   return $lastnode_to_upgrade;
}

#This function is currently called only for upgrade from 
#11201 to 11202. This will not work for pre11.2 to 11.2 upgrade.
sub isFirstNodeToUpgrade
{
   trace ("isFirstNodeToUpgrade...");

   # This function is frequently invoked on each node during upgrade, and
   # it could take a long time especially when stack is down.
   # So we should not do the same query every time, instead we should
   # cache the value of $firstnode_to_upgrade the first time this function
   # is executed.
   if (defined $CFG->isFstNodeToUpgrade)
   {
     my $cachedValue = $CFG->isFstNodeToUpgrade;
     trace ("isFirstNodeToUpgrade: $cachedValue");
     return $CFG->isFstNodeToUpgrade;
   }

   my $firstnode_to_upgrade = TRUE;
   my $crsctl = catfile ($CFG->params('ORACLE_HOME'), "bin", "crsctl");
   my $host = tolower_host ();
   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   my $oldcrsver  = join('.',@oldcrs_ver);
   chomp $oldcrsver;

   # get current softwareversion
   my @nodes = split (/,/, $CFG->params('HOST_NAME_LIST'));
   my $software_version;
   my $mysoftver;

   $mysoftver = getcursoftversion($host);
   trace("software version on local host $host is $mysoftver");
   foreach my $nodename (@nodes) {
    
      if (lc($nodename) !~ /\b$host\b/i)
      {
         $software_version = getcursoftversion($nodename);

         trace ("software_version on $nodename=$software_version");
         trace ("oldcrs ver is $oldcrsver");
    
         # compare version with software version on the current node
         if (("$mysoftver" =~ "$oldcrsver") && ($software_version !~ $mysoftver)) {
            $firstnode_to_upgrade = FALSE;
            trace("first check for firstnodetoupgrade failed");
         }
         if (("$mysoftver" !~ "$oldcrsver") && ($software_version =~ $mysoftver)) {
            $firstnode_to_upgrade = FALSE;
            trace("second check for firstnodetoupgrade failed");
         }
      }
   }

   trace("First node to upgrade is $firstnode_to_upgrade");
   $CFG->isFstNodeToUpgrade($firstnode_to_upgrade);
   return $firstnode_to_upgrade;
}

sub InitEnv
{
   # create dirs & set permissions
   my @crsconfig_dirs = read_file(catfile($CFG->ORA_CRS_HOME, 'crs',
                                          'utl', 'crsconfig_dirs'));
   create_dirs(\@crsconfig_dirs);

   my @crsconfig_fileperms = read_file(catfile($CFG->ORA_CRS_HOME, 'crs',
                                               'utl', 'crsconfig_fileperms'));
   set_file_perms(\@crsconfig_fileperms);

   # Set owner/group of ORA_CRS_HOME and its parent dir to root/dba
   if (! is_dev_env() && (! $CFG->IS_SIHA) &&
      ($CFG->platform_family eq "unix"))
   {
      s_setParentDirOwner($CFG->SUPERUSER, $CFG->ORA_CRS_HOME);
   }
}

# ARGS
# none
sub run_env_setup_modules
{
    instantiate_scripts ();

    #
    # Before Directories Creation module is invoked, we need to add entries
    # for RCALLDIR locations to crsconfig_dirs
    # Note: this is done only on platforms where RCALLDIR is defined.
    #
    if ($RCALLDIR) {
        add_RCALLDIR_to_dirs ();
    }

    add_ITDIR_to_dirs();

    # Before File Permissions module is invoked, we need to add entries for
    # OCR and Voting Disk locations to crsconfig_fileperms. This is not
    # required to be done for upgrade scenarios. Bug 8236090.

    if (! $CFG->UPGRADE)
    {
       add_localOlr_OlrConfig_OcrConfig();
    }

    my @crsconfig_dirs = read_file (catfile ($wrapdir_crs, "crsconfig_dirs"));
    create_dirs (\@crsconfig_dirs);

    copy_wrapper_scripts ();

    my @crsconfig_fileperms = read_file (catfile ($wrapdir_crs,
                                                  "crsconfig_fileperms"));
    set_file_perms (\@crsconfig_fileperms);

    # Set owner/group of ORA_CRS_HOME and its parent dir to root/dba
    if (! is_dev_env() && (! $CFG->IS_SIHA) && 
       ($CFG->platform_family eq "unix"))
    {
       my $SUPERUSER  = $CFG->SUPERUSER;
       s_setParentDirOwner ($SUPERUSER, $ORA_CRS_HOME);
    }

    # create s_crsconfig_$HOST_env.txt file
    s_createConfigEnvFile ();
}

#checkpoint wrapper function for perform_initial_config
sub perform_init_config
{
  my $ckptStatus;
  my $ckptName = "ROOTCRS_BOOTCFG";

  if (isCkptexist($ckptName)) 
  {
    $ckptStatus = getCkptStatus($ckptName);
    trace("'$ckptName' state is $ckptStatus");

    if ((($ckptStatus eq CKPTSTART) && $isNodeCrashed) || 
	($ckptStatus eq CKPTFAIL))
    {
      clean_perform_initial_config();
      $isNodeCrashed=FALSE;
    } 
    elsif ($ckptStatus eq CKPTSUC) 
    {
      trace("Node specific initial boot configuration already completed");
      $wipCkptName = "ROOTCRS_STACK";
      return SUCCESS;
    }
  }

  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = $ckptName;
  return perform_initial_config();
}

=head2 perform_initial_config

   Checks for existing CSS configuration and creates initial
   configuration if no configuration found

=head3 Parameters

   The parameter hash

=head3 Returns

  TRUE  - A  CSS configuration was found or created
  FALSE - No CSS configuration was found and none created

=cut

sub perform_initial_config {
  my $rc;
  my $success = TRUE;
  my $ASM_DISK_GROUP = $CFG->params('ASM_DISK_GROUP');
  my $ckptName = "ROOTCRS_BOOTCFG";
  my $excl_ret;
  my $css_up_normal = FALSE;

  ## Enter exclusive mode to setup the environment
  trace ("Checking if initial configuration has been performed");

  writeCkpt($ckptName, CKPTSTART);

  if (checkServiceDown("css")) {
    $excl_ret = CSS_start_exclusive();
  } elsif (!CSS_is_configured()) {
    $excl_ret = CSS_EXCL_SUCCESS;
  } else {$css_up_normal = TRUE;}


  if ((!$css_up_normal) && $excl_ret != CSS_EXCL_SUCCESS)
  {
    # These resources may have been started as part of CSS
    # startup, so stop them now
    stop_resource("ora.gipcd", "-f", "-init");
    stop_resource("ora.gpnpd", "-f", "-init");
    stop_resource("ora.mdnsd", "-init");
    stop_diskmon();

    if ($excl_ret == CSS_EXCL_FAIL_CLUSTER_ACTIVE) 
    {
      error("An active cluster was found during exclusive startup,",
            "restarting to join the cluster");
    }
    else {
      error("The exlusive mode cluster start failed, see Grid Infrastructure alert log",
            "for more information");
      $success = FALSE;
    }
  }
  # in business
  # Need to find out whether we should be doing something as
  # exclusive node or not. Use CSS voting files as a way of
  # checking cluster initialization status
  elsif (CSS_is_configured()) {
    trace("Existing configuration setup found");
  }
  else {
    trace("Performing initial configuration for cluster");

    if (!start_resource("ora.ctssd", "-init")) 
    {
      $success = FALSE;
      writeCkpt($ckptName, CKPTFAIL);
      die("Cluster Time Synchronisation Service  start in exclusive mode failed");

    }
    # If ASM diskgroup is defined, need to configure and start ASM
    # ASM is started as part of the config
    elsif ($CFG->ASM_STORAGE_USED)
    {
      if (isODA())
      {
        $success = configure_ASM_oda();
      }
      else
      {
        $success = configure_ASM();
      }
      if (!$success)
      {
       writeCkpt($ckptName, CKPTFAIL);
       die("Did not succssfully configure and start ASM");
     }
    }
    # ocrconfig - Create OCR keys
    if (! configure_OCR()) {
      $success = FALSE;
      writeCkpt($ckptName, CKPTFAIL);
      die("Oracle Grid Infrastructure Repository configuration failed");
    }
    elsif (!start_resource("ora.crsd", "-init")) {
      $success = FALSE;
      writeCkpt($ckptName, CKPTFAIL);
      die("Grid Infrastructure exclusive mode start of Cluster Ready Services failed");
    } else {
      trace ("Creating voting files");

      if (!add_voting_disks()) {
          writeCkpt($ckptName, CKPTFAIL);
          die("Failed to add voting disks");
      }

      my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
      my $CRSCTL = catfile ($ORA_CRS_HOME, "bin", "crsctl");
      system("$CRSCTL query css votedisk");
    }

    trace("The node:$HOST is the first node to configure");

    # Allow additional commands to be executed if set
    if ($success && $CFG->CRSCFG_POST_CMD) {
      my @cmdl = @{$CFG->CRSCFG_POST_CMD};
      for my $cmd (@cmdl) { my @cmd = @{$cmd}; system_cmd(@cmd); }
    }
  }

  if ($excl_ret == CSS_EXCL_SUCCESS)
  {
     # Push local gpnp setup to be cluster-wide.
     # This will copy local gpnp file profile/wallet setup to a
     # list of cluster nodes, including current node.
     # This promotes a node-local gpnp setup to be
     # "cluster-wide"
     my $node_name_list = $CFG->params('NODE_NAME_LIST');
     trace ("Promoting local gpnp setup to cluster-wide. " .
           "Nodes {$node_name_list}");

     if (! push_clusterwide_gpnp_setup( $CFG->params('NODE_NAME_LIST')))
     {
        $success = FALSE;
        writeCkpt($ckptName, CKPTFAIL);
        die ("Failed to promote local gpnp setup to other " .
            "cluster nodes");
     }

     trace("Stopping CSS which is running exclusive mode");
     stop_css_exclusive();
  }

  if ($success != TRUE) {
     writeCkpt($ckptName, CKPTFAIL);
  } else {
     writeCkpt($ckptName, CKPTSUC);
     $wipCkptName = "ROOTCRS_STACK";
  }
  return $success;
}


sub add_voting_disks
{
      my $ASM_DISK_GROUP = $CFG->params('ASM_DISK_GROUP');
      my $success;
      # Depending on whether using ASM or not, create
      # accordingly
      if ($CFG->ASM_STORAGE_USED)
      {
        $success = CSS_add_vfs($CFG, "+$ASM_DISK_GROUP");
      } else {
        $success = CSS_add_vfs($CFG,
                               split(',', $CFG->params('VOTING_DISKS')));
      }
      return $success;
}

sub delete_voting_disks
{
      my $ASM_DISK_GROUP = $CFG->params('ASM_DISK_GROUP');
      my $success;
      # Depending on whether using ASM or not, delete
      # accordingly
      if ($CFG->ASM_STORAGE_USED)
      {
        $success = CSS_delete_vfs($CFG, "+$ASM_DISK_GROUP");
      } else {
        $success = CSS_delete_vfs($CFG,
                               split(',', $CFG->params('VOTING_DISKS')));
      }
}

sub RemoveVotingfiles
{
  my $crshome      = $_[0];
  my $VOTING_FILES = $_[1];
  my $crsctl;
  my $delvfcmd;

  if (! $crshome)
  {
    error("Invalid CRS home");
    return FAILED;
  }

  $crsctl = catfile($crshome, 'bin', 'crsctl');
  $delvfcmd = "$crsctl delete css votedisk @$VOTING_FILES";

  trace("Executing '$delvfcmd'");
  my @output = system_cmd_capture($delvfcmd);
  my $rc = shift @output;
  if (0 != $rc)
  {
    error("Voting files deletion failed");
    return FAILED;
  }
  
  trace("Voting files deletion succeeded");
  return SUCCESS;
}

sub AddVotingFiles
{
  my $crshome      = $_[0];
  my $VOTING_FILES = $_[1];
  my $crsctl;
  my $addvfcmd;

  if (!$crshome)
  {
    error("Invalid CRS home");
    return FAILED;
  }

  $crsctl = catfile($crshome, 'bin', 'crsctl');
  $addvfcmd = "$crsctl add css votedisk @$VOTING_FILES";

  trace("Executing \'$addvfcmd\'");
  my @output = system_cmd_capture($addvfcmd);
  my $rc = shift @output;
  if ($rc != 0)
  {
    error("Voting files add failed");
    return FAILED;
  }

  return SUCCESS;
}

sub RemoveVotingDiks
{
  my $crshome = $_[0];
  my $ASM_DISK_GROUP = $_[1];
  my $crsctl;
  my $cmd;
  my @output;
  my $rc;
  my $dgname;

  if (!$crshome)
  {
    error("Invalid CRS home");
    return FAILED;
  }

  $crsctl = catfile($crshome, "bin", "crsctl");
  $dgname = $ASM_DISK_GROUP;
  if ($CFG->platform_family ne "windows")
  {
    $ASM_DISK_GROUP = "'+$ASM_DISK_GROUP'";
  }
  else
  {
    $ASM_DISK_GROUP = "+$ASM_DISK_GROUP";
  }

  $cmd = "$crsctl delete css votedisk $ASM_DISK_GROUP";
  trace("Executing \'$cmd\'");
  @output = system_cmd_capture($cmd);
  $rc = shift @output;

  if (0 == $rc)
  {
    trace("crsctl delete for vds in $dgname ... success");
  }
  else
  {
    error ("crsctl delete for vds in $dgname ... failed");
    return FAILED;
  }

  return SUCCESS;
}

sub AddVotingDisks
{
  my $crshome        = $_[0];
  my $ASM_DISK_GROUP = $_[1];
  my $crsctl;
  my $addvfcmd;
  
  if (!$crshome)
  {
    error("Invalid CRS home");
    return FAILED;
  }

  $crsctl = catfile($crshome, 'bin', 'crsctl');

  my $dgname = $ASM_DISK_GROUP;

  if ($CFG->platform_family ne "windows")
  { 
    $ASM_DISK_GROUP = "'+$ASM_DISK_GROUP'";
  }
  else
  {
    $ASM_DISK_GROUP = "+$ASM_DISK_GROUP";
  }

  trace("Creating voting files in ASM diskgroup $dgname");
  $addvfcmd = "$crsctl replace votedisk $ASM_DISK_GROUP";

  trace("Executing \'$addvfcmd\'");

  my @output = system_cmd_capture($addvfcmd);
  my $rc = shift @output;
  if ($rc != 0)
  {
    error("Voting disks add failed");
    return FAILED;
  }

  return SUCCESS;
}

sub start_nocrs_stack
{
  my $crshome = shift;

  my $cmd;
  my @chk;
  my $stop_rc = -1; 
  my $rc = -1;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');

  $cmd = "$crsctl start crs -excl -nocrs";
  trace("Executing \'$cmd\'");
  $stop_rc = system_cmd("$cmd");

  $cmd = "$crsctl check css";
  @chk = system_cmd_capture($cmd);
  $rc = shift @chk;
  if ((0 == $stop_rc) && (0 == $rc) && (scalar(grep(/4529/, @chk)) > 0))
  {
    return SUCCESS;
  }
  else
  {
    return FAILED;
  }
}

sub start_excl_crs
{
  my $crshome = shift;

  my $cmd;
  my @chk;
  my $stop_rc = -1;
  my $rc = -1;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');

  $cmd = "$crsctl start crs -excl";
  trace("Executing \'$cmd\'");
  $stop_rc = system_cmd("$cmd");

  $cmd = "$crsctl check crs";
  @chk = system_cmd_capture($cmd);
  $rc = shift @chk;
  if ((0 == $stop_rc) && (0 == $rc) && (scalar(grep(/4529/, @chk)) > 0))
  {
    trace("Cluster Synchronization Services is running");
    if (scalar(grep(/4692/, @chk) > 0))
    {
      trace("Cluster Ready Services is running in exclusive mode");
    }
    return SUCCESS;
  }
  else
  {
    return FAILED;
  }
}

sub stop_crsd_check
{
  my $crshome = shift;

  my $cmd;
  my @chk;
  my $stop_rc = -1;
  my $rc = -1;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');
  
  $cmd = "$crsctl stop resource ora.crsd -init -f";
  trace("Executing \'$cmd\'");
  $stop_rc = system_cmd("$cmd");
  
  $cmd = "$crsctl check crs";
  @chk = system_cmd_capture($cmd);
  $rc = shift @chk;
  if ((0 == $rc) && (scalar(grep(/4535/, @chk)) > 0))
  {
    return SUCCESS;
  }
  else
  {
    return FAILED;
  }
}

sub start_crsd_check
{
  my $crshome = shift;

  my $cmd;
  my @chk;
  my $rc = -1;
  my $retries = 120;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');

  $cmd = "$crsctl start resource ora.crsd -init";
  trace("Executing \'$cmd\'");
  system_cmd("$cmd");

  $cmd = "$crsctl check crs";
  while ($retries--)
  {
    @chk = system_cmd_capture($cmd);
    $rc = shift @chk;

    if ((0 == $rc) && 
         ((scalar(grep(/4537/, @chk)) > 0) || 
          (scalar(grep(/4692/, @chk)) > 0)))
    {
      trace("The resource ora.crsd is online");
      last;
    }

    sleep(5);
  }

  if ($retries > 0)
  {
    return SUCCESS;
  }
  else
  {
    return FAILED;
  }
}

sub stop_crs_check
{
  my ($crshome, $force) = @_;

  my $cmd;
  my @chk;
  my $rc;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');

  if (uc($force) eq "Y")
  {
    $cmd = "$crsctl stop crs -f";
  }
  else
  {
    $cmd = "$crsctl stop crs";
  }

  trace("Executing \'$cmd\'");
  system_cmd("$cmd");

  $cmd = "$crsctl check crs";
  @chk = system_cmd_capture($cmd);
  $rc = shift @chk;
  if ((0 == $rc) && (scalar(grep(/4639/, @chk)) > 0))
  {
    return SUCCESS;
  }
  else
  {
    return FAILED;
  }
}

sub start_crs_check
{
  my ($crshome, $service, $retries) = @_;

  my $cmd;
  my @chk;
  my $rc;
  my $is_up = FALSE;
  my $crsctl = catfile($crshome, 'bin', 'crsctl');

  $cmd = "$crsctl start $service";
  trace("Executing \'$cmd\'");
  system_cmd("$cmd");

  $cmd = "$crsctl check crs";
  while ($retries)
  {
    @chk = system_cmd_capture($cmd);
    $rc = shift @chk;
    if ($rc == 0)
    {
      $is_up = TRUE;
      last;
    } 

    trace ("Waiting for Oracle CRSD and EVMD to start");
    sleep (5);
    $retries--;  
  }

  if ($is_up)
  {
    # Wait for cluster to start up
    my $node = $CFG->HOST;
    my $cluretries = 120;
    my $clurunning = FALSE;
    my $cmd  = "$crsctl check cluster -n $node";
    my @chk;
    my $rc = -1;
    trace("Executing '$cmd'");

    while($cluretries && !$clurunning)
    {
      @chk = system_cmd_capture1(TRUE, $cmd);
      $rc = shift @chk;
      my @output = grep(/4537/, @chk);
      if (scalar(@output) > 0)
      {
        $clurunning = TRUE;
      }
      else
      {
        trace("Checking the status of Clusterware every few minutes");
        sleep(5);
        $cluretries--;
      }
    }

    if ($clurunning)
    {
      trace ("Oracle CRS stack is running");
    }
    else
    {
      trace("Oracle Clusterware stack start initiated but failed to complete\n");
      $is_up = FALSE;
    }
  }
  else
  {
    trace("Timed out waiting for the CRS stack to start\n");
  }
 
  return $is_up;
}

sub removeCkptFile
{
  my $host = tolower_host();
  my $filename = "ckptGridHA_$host.xml";
  my $ckptfile = catfile($CFG->params('ORACLE_BASE'), 'Clusterware', $filename);

  trace("CkptFile: $ckptfile");
  if (-e $ckptfile)
  {
    trace("Remove the checkpoint file '$ckptfile'");
    s_remove_file($ckptfile);
  }
}

sub stop_css_exclusive()
{
    my $ckptName = "ROOTCRS_BOOTCFG";
    my $success;

    trace ("Exiting exclusive mode");
    if (! stop_resource("ora.crsd", "-init"))
    {
       $success = FALSE;
       writeCkpt($ckptName, CKPTFAIL);
       die("Failed to stop Cluster Ready Services");
    }

    if ($CFG->ASM_STORAGE_USED &&
        !stop_resource('ora.asm', '-init')) {
      $success = FALSE;
      writeCkpt($ckptName, CKPTFAIL);
      die("Failed to stop ASM");
    }

    if (! stop_resource("ora.ctssd", "-init")) {
       $success = FALSE;
       writeCkpt($ckptName, CKPTFAIL);
       die("Grid Infrastructure Time Synchronisation service stop in exclusive mode failed");
    }

    if( $CFG->ASM_STORAGE_USED && isHAIPsupported() && 
        !stop_resource("ora.cluster_interconnect.haip", "-init") )
    {
       $success = FALSE;
       writeCkpt($ckptName, CKPTFAIL);
       die("High Availability IP stop in exclusive mode failed");
     }

    if (!CSS_stop() ||
        !stop_resource("ora.gipcd", "-f", "-init") ||
        !stop_resource("ora.gpnpd", "-f", "-init") ||
        !stop_resource("ora.mdnsd", "-init"))
    {
      $success = FALSE;
      writeCkpt($ckptName, CKPTFAIL);
      die("Failed to exit exclusive mode");
    }
}



=head2 perform_upgrade_config

   Upgrades configuration and pushes cluster-wide gpnp setup 

=head3 Parameters

   The parameter hash

=head3 Returns

  TRUE  - A  configuration was found and upgraded
  FALSE - No configuration was found or upgraded

=cut

sub perform_upgrade_config {
  my $rc;
  my $success = TRUE;
  my $gpnp_setup_type = $_[0];


    if ($DEBUG) { trace("OCR upgraded; gpnp setup type: $gpnp_setup_type"); }

    if ((! isVersion112()) && ($gpnp_setup_type != GPNP_SETUP_GOTCLUSTERWIDE) &&
        ($gpnp_setup_type != GPNP_SETUP_CLUSTERWIDE)) 
    {
      my ($nodelist, $skippednodelist) = getLiveNodeList();

      if ($skippednodelist)
      {
        set_bold();
        print "The following nodes:\n";
        print "    $skippednodelist\n";
        print "are unreachable, and will be skipped during this upgrade. "
             ."Use 'rootupgrade.sh -force' after upgrading all reachable "
             ."nodes, to complete the upgrade\n";
        reset_bold();
      }

      if (is_dev_env())
      {
        $nodelist = $CFG->params('NODE_NAME_LIST');
      }

      # Push local gpnp setup to be cluster-wide.
      # This will copy local gpnp file profile/wallet setup to a
      # list of cluster nodes, including current node.
      # This promotes a node-local gpnp setup to be
      # "cluster-wide"
      trace ("Promoting local GPnP setup to cluster-wide. " .
             "Nodes {$nodelist}");

      if ((! $nodelist) || (! push_clusterwide_gpnp_setup($nodelist))) {
         error ("Failed to promote local GPnP setup to other " .
                "cluster nodes");
         $success = FALSE;
      }
    } else {
      trace ("Skipping push GPnP configuration cluster-wide");
    }
  return $success;
}


=head2 olr_initial_config

   Creates or updates OLR

=head3 Parameters

   parameters hash

=head3 Returns

  TRUE  - OLR configuration was     created or updated
  FALSE - OLR configuration was not created or updated

=cut


sub perform_olr_config
{
  my $ckptName = "ROOTCRS_OLR";

  if (isCkptexist($ckptName))
  {
    my $ckptStatus = getCkptStatus($ckptName);
    trace("'$ckptName' state is '$ckptStatus'");

    if (isCkptSuccess($ckptName)) 
    {
      trace("OLR is already initialized");
      $wipCkptName = "ROOTCRS_STACK";
      return SUCCESS;
    }

  }

  trace("Initializing OLR now..");
  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = $ckptName;

  if (! $CFG->IS_SIHA) 
  {
    initial_cluster_validation();
  }

  olr_initial_config();

  writeCkpt($ckptName, CKPTSUC);
  print("OLR initialization - successful\n");
  
}

sub olr_initial_config {
  my $status;
  my $rc = FALSE;
  my @cmd;
  my $ckptName = "ROOTCRS_OLR";
  my @out = ();

  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $OCRCONFIGBIN = catfile ($ORA_CRS_HOME, "bin", "ocrconfig");
  my $CLSCFGBIN = catfile ($ORA_CRS_HOME, "bin", "clscfg");
  my $ORACLE_OWNER = $CFG->params('ORACLE_OWNER');
  my $ORACLE_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
  my $asmgrp       = $CFG->params('ORA_ASM_GROUP');


  trace ("Creating or upgrading Oracle Local Registry (OLR)");
  if ($CFG->IS_SIHA) {
    $status = run_as_user($ORACLE_OWNER,
                          "$OCRCONFIGBIN -local -upgrade");
  }
  else {
    @cmd = ($OCRCONFIGBIN, '-local', '-upgrade', $ORACLE_OWNER, 
            $ORACLE_DBA_GROUP);
    $status = system_cmd(@cmd);
  }

  if (0 == $status) {
    trace ("OLR successfully created or upgraded");
  } else {
    trace("$OCRCONFIGBIN -local -upgrade failed with error: $status");
    error ("Failed to create or upgrade OLR");
    writeCkpt($ckptName, CKPTFAIL);
    die("Failed to create or upgrade OLR");
  }

  my $lang_id = $CFG->params('LANGUAGE_ID');
  $lang_id =~ s/'//g; # remove single quotes;

  # OLR exists, and upgrade OLR
  if (($UPGRADE) && isVersion112())
  {
     #upgrade keys in OLR
     trace ("$CLSCFGBIN -localupgrade");

     if ($CFG->IS_SIHA) {
       $status = run_as_user($ORACLE_OWNER,
                         "$CLSCFGBIN -localupgrade");
     }
     else {
       @out = system_cmd_capture("$CLSCFGBIN -localupgrade");
       $status = shift @out;
     }
     trace ("'$CLSCFGBIN -localupgrade' - successful");
  }
  # Upgrading from pre 11.2. OLR does not exist. Use info from existing OCR
  # to populate OLR. Create OLR from scratch.
  elsif (($UPGRADE) && !isVersion112())
  {
     #upgrade keys in OLR
     if ($CFG->IS_SIHA) {
       $status = run_as_user($ORACLE_OWNER,"$CLSCFGBIN -localadd");
     }
     else {
       @out = system_cmd_capture("$CLSCFGBIN -localadd -avlookup");
       $status = shift @out;
     }
  }
  # Fresh install. Create OLR from scratch.
  else 
  {
     ## create keys in OLR
     trace ("$CLSCFGBIN -localadd");
  
     if ($CFG->IS_SIHA) {
       $status = run_as_user($ORACLE_OWNER,
                         "$CLSCFGBIN -localadd");
     }
     else {
       @out = system_cmd_capture("$CLSCFGBIN -localadd");
       $status = shift @out;
     }
     trace ("'$CLSCFGBIN -localadd' - successful");
  }

  if (0 == $status) {
    trace ("Keys created in the OLR successfully");
    $rc = TRUE;
    writeCkpt($ckptName, CKPTSUC);
    $wipCkptName = "ROOTCRS_STACK";
  } else {
    error ("Failed to create keys in the OLR, rc = $status, Message:\n ",
           "@out \n"); 
    writeCkpt($ckptName, CKPTFAIL);
    die("Failed to create keys in the OLR");
  }

  return $rc;
}

sub is_dev_env
{
    my $isDevEnv = uc($ENV{'_SCLS_DEVELOPMENT'});
    if ($isDevEnv eq "TRUE") {
        return TRUE;
    } else {
        return FALSE;
    }
}

# Thin wrapper of OSD function to run a command as a specified user
# Parameters:
#   1. user to run command as
#   remaining arguments are the command to run
sub run_as_user
{
  my $user = shift;
  trace("Running as user $user: @_");
  return $CFG->s_run_as_usere($user, @_);
}

# Thin wrapper of OSD function to run a command as a specified user
# Parameters:
#   1. user to run command as
#   2. array reference for output capture
#   remaining arguments are the command to run
sub run_as_user2
{
  my $user = shift;
  my $aref = shift;
  trace("Running as user $user: @_");
  return $CFG->s_run_as_user2e($user, $aref, @_);
}

# Thin wrapper of OSD function to run a command as a specified user
# Parameters:
#   1. user to run command as
#   2. command to run
#   3. value to be injected
sub run_as_user3
{
  my $user  = shift;
  my $cmd   = shift;
  my $param = shift;
  my @out   = ();
  my $cmd2  = join (@{$cmd});
  trace("Running as user $user: $cmd2");
  my $rc = $CFG->s_run_as_user3e($user, \@{$cmd}, $param, \@out);
  return $rc;
}

sub perform_configure_hasd
{
  my $mode     = $_[0];
  my $hostname = $_[1];
  my $owner    = $_[2]; # the owner of the software bits aka CRS USER
  my $pusr     = $_[3]; # the privileged user 
  my $grp      = $_[4];

  my $ckptStatus;
  my $ckptName = "ROOTCRS_INITRES";

  if (isCkptexist($ckptName))
  {
    $ckptStatus = getCkptStatus($ckptName);

    trace("'$ckptName' state is $ckptStatus");

    if ((($ckptStatus eq CKPTSTART) && $isNodeCrashed) ||
        ($ckptStatus eq CKPTFAIL))
    {
       trace("Removing OHASD init resources and types");
       configure_hasd('crs', $hostname, $owner, $pusr, $grp, "delete");
       $isNodeCrashed = FALSE;
    } elsif ($ckptStatus eq CKPTSUC) {
       trace("OHASD Resources are already configured");
       $wipCkptName = "ROOTCRS_STACK";
       return SUCCESS;
    }
  }
  else
  {
    trace("The checkpoint '$ckptName' does not exist");
    if ($isRerun && isResTypeRegistered())
    {
      trace("Removing OHASD init resources and types");
      configure_hasd('crs', $hostname, $owner, $pusr, $grp, "delete");
    }
  }

  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = $ckptName;

  configure_hasd('crs', $hostname, $owner, $pusr, $grp);
}

sub isResTypeRegistered
{
  my @output;
  my $rc = -1;
  my $ckptName = "ROOTCRS_INITRES";
  my $crsctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
  
  my %resTypes = ("ora.daemon.type"         => 1,
                  "ora.haip.type"           => 1,
                  "ora.mdns.type"           => 1,
                  "ora.gpnp.type"           => 1,
                  "ora.gipc.type"           => 1,
                  "ora.cssd.type"           => 1,
                  "ora.cssdmonitor.type"    => 1,
                  "ora.crs.type"            => 1,
                  "ora.evm.type"            => 1,
                  "ora.ctss.type"           => 1,
                  "ora.crf.type"            => 1,
                  "ora.asm.type"            => 1,
                  "ora.drivers.acfs.type"   => 1,
                 );

  trace("Check if OHASD resource types have been registered");
  @output = system_cmd_capture($crsctl, "stat", "type", "-init");
  $rc = shift @output;

  if ($rc != 0)
  {
    error("Failed to check status of init resource types");
    return FAILED;
  }

  my @typeNm = grep (/TYPE_NAME/, @output);
  foreach my $type (@typeNm)
  {
    my @val = split(/=/, $type); 
    if (1 == $resTypes{$val[1]})
    {
      my $res = $val[1];
      trace("A resource type with the name '$res' is already registered");
      return TRUE; 
    }
  }
   
  trace("Not found any registered resource type");
  return FALSE;
}

sub configure_hasd {
  my $mode     = $_[0];
  my $hostname = $_[1];
  my $owner    = $_[2]; # the owner of the software bits aka CRS USER
  my $pusr     = $_[3]; # the privileged user 
  my $grp      = $_[4];
  my $action   = $_[5];
  my @out      = ();
  my $status;
  my $ckptName = "ROOTCRS_INITRES";

  # Register these resources for SIHA only
  my @registerTypesHAS = ("cssd", "crs", "evm", "ctss");

  # Register these resources for Grid Infrastructure only
  my @registerTypesCRS = ("mdns", "gpnp", "gipc", "cssd", "cssdmonitor",
                          "crs", "evm", "ctss", "crf", "asm", "drivers.acfs");
 
  if ($CFG->platform_family eq "windows") {
     if (! $owner) {
        $pusr = '';
        $grp  = '';
     }
  } else {
     push(@registerTypesHAS, "diskmon");
     push(@registerTypesCRS, "diskmon");
  }
      
  my $ORACLE_HOME = $ENV{'ORACLE_HOME'};

  # 
  # Make sure CRS_HOME is set
  # 
  if ( ! $ORACLE_HOME ) {
    writeCkpt($ckptName, CKPTFAIL);
    die "ERROR: ORACLE_HOME is not set in the environment";
  }

  # Set Homes
  my $CRS_HOME_BIN      = catdir($ORACLE_HOME,"bin");
  my $CRS_HOME_SCRIPT   = catdir($ORACLE_HOME,"crs","profile");
  my $CRS_HOME_TEMPLATE = catdir($ORACLE_HOME,"crs","template");

  my $crsctl = catfile( $CRS_HOME_BIN, "crsctl");

  ## set the owners : user IDs to spawn agents as
  my $MDNSOWNER = $owner;
  my $GPNPOWNER = $owner;
  my $GIPCOWNER = $owner;
  my $CSSOWNER = $pusr;
  my $EVMOWNER = $owner;
  my $CRSOWNER = $pusr;

  # set the users : explit execution rules (may or may not be equal to owner)
  my $MDNSUSER = $owner;
  my $GPNPUSER = $owner;
  my $GIPCUSER = $owner;
  my $CSSUSER = $owner;
  my $EVMUSER = $owner;
  my $CRSDUSER = $owner;

  my @types;
  if ($mode eq "has") {
    @types = @registerTypesHAS;
  } elsif ($mode eq "crs") {
    @types = @registerTypesCRS;
  }

  my $baseType = 'ora.daemon.type';
  my $file     = 'daemon.type';
  my $infile = catfile($CRS_HOME_TEMPLATE, $file);
  my $logdir = catdir($ORA_CRS_HOME, "log", $hostname);
  my $ohasdlog = catdir($logdir, "ohasd");
  my $outfile = catfile($ohasdlog, $baseType);
  my $haipType = 'ora.haip.type';
  my $name;

  #set default action to add
  if (! $action) {
     $action = "add";
  }

  instantiateTemplate($infile, $outfile);
  if ($action eq "add") {
     trace("Registering type $baseType");
     system_cmd($crsctl, "add","type", $baseType, 
		"-basetype", "cluster_resource", "-file", "$outfile", "-init");
     if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add base type cluster_resource");
     }

     unlink($outfile);
  }  

  if (($action eq "add") && isHAIPsupported())
  {
    trace("Registering type $haipType");
    $infile  = catfile($CRS_HOME_TEMPLATE, 'haip.type');
    $outfile = catfile($ohasdlog, $haipType);
    instantiateTemplate($infile, $outfile);
    system_cmd($crsctl, "add","type", $haipType, "-basetype", 
               "cluster_resource", "-file", "$outfile", "-init");
    s_remove_file($outfile);

    # as part of the resource addition, also fixup any system config files
    s_CheckNetworkConfig();
  }

  # register the infrastructure resources
  foreach my $type (@types)
  {
    $file = $type . '.type';
    $name = 'ora.' . $type . '.type';
    $infile = catfile($CRS_HOME_TEMPLATE, $file);
    $logdir = catdir($ORA_CRS_HOME, "log", $hostname);
    $ohasdlog = catdir($logdir, "ohasd");
    $outfile = catfile($ohasdlog, $file);

    instantiateTemplate($infile, $outfile);

    if ($action eq "add") {
       trace("Registering type '$name'");
       $status =  system_cmd($crsctl, "add","type", $name, "-basetype",
                  $baseType, "-file", "$outfile", "-init");

       if ($status != CMDSUC) 
       {
	 trace("Failed to add type '$name'");
	 writeCkpt($ckptName,CKPTFAIL);
	 die("Failed to register Grid Infrastructure type $name");
       }
       else {
	 trace("Type '$name' added successfully");
       }

       # remove outfile, not fatal if it fails
       unlink($outfile);
     }
  }

  my @evm_attr = ("ACL='owner:$EVMOWNER:rw-,pgrp:$grp:rw-," .
                       "other::r--,user:$EVMUSER:rwx'"); 
  my @asm_attr    = ("ACL='owner:$CSSUSER:rw-,pgrp:$grp:rw-," .
                     "other::r--,user:$CSSUSER:rwx'");
  my @css_attr    = ("CSS_USER=$CSSUSER");
  my @crsd_attr   = ("ACL='owner:$CRSOWNER:rw-,pgrp:$grp:rw-," .
                     "other::r--,user:$CRSDUSER:r-x'");
  my @css_attrmon = ("CSS_USER=$CSSUSER",
                     "ACL='owner:$CSSOWNER:rw-,pgrp:$grp:rw-," .
                     "other::r--,user:$CSSUSER:r-x'");
  my @diskmon_attr = ("USR_ORA_ENV=ORACLE_USER=$CSSUSER");

  if ($ENV{'CSSDAGENT_ATTR'}) { push @css_attr, $ENV{'CSSDAGENT_ATTR'}; }
  if ($ENV{'CSSDAGENT_ATTR'}) { push @css_attrmon, $ENV{'CSSDAGENT_ATTR'}; }

  if ($mode eq "crs")
  {
    push @css_attr, "ACL='owner:$CSSOWNER:rw-,pgrp:$grp:rw-," .
      "other::r--,user:$CSSUSER:r-x'"; 
    push @css_attr, "AUTO_START=always";

    if ($CFG->platform_family eq "windows")
    {
      push @css_attr,
	"START_DEPENDENCIES='hard(ora.gpnpd,ora.gipcd,ora.cssdmonitor)" . "pullup(ora.gpnpd,ora.gipcd)'";
      
      push @css_attr, "STOP_DEPENDENCIES='hard(intermediate:ora.gipcd,intermediate:ora.cssdmonitor)'";
    }
    else
    {
      push @css_attr, "START_DEPENDENCIES='weak(concurrent:ora.diskmon)" . "hard(ora.cssdmonitor,ora.gpnpd,ora.gipcd)" . "pullup(ora.gpnpd,ora.gipcd)'";

      push @css_attr, "STOP_DEPENDENCIES='hard(intermediate:ora.gipcd,shutdown:ora.diskmon,intermediate:ora.cssdmonitor)'";

      push @diskmon_attr, "START_DEPENDENCIES='weak(concurrent:ora.cssd)" .
                          "pullup:always(ora.cssd)'";
    }

    push @diskmon_attr, "ACL='owner:$CSSOWNER:rw-,pgrp:$grp:rw-," .
                        "other::r--,user:$CSSUSER:r-x'";
   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.mdnsd", "-attr", 
                 "\"ACL='owner:$MDNSOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$MDNSUSER:rwx'\"", 
                 '-type', 'ora.mdns.type', '-init');
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.mdnsd");
      }
   } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.mdnsd", "-f", "-init");
   }

   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.gpnpd", "-attr", 
                 "\"ACL='owner:$GPNPOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$GPNPUSER:rwx',START_DEPENDENCIES='weak(ora.mdnsd)'\"", 
                 "-type", "ora.gpnp.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.gpnpd");
      }
    } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.gpnpd", "-f", "-init");
   }

   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.gipcd", "-attr", 
                 "\"ACL='owner:$GIPCOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$GIPCUSER:rwx',START_DEPENDENCIES='hard(ora.gpnpd)',STOP_DEPENDENCIES=hard(intermediate:ora.gpnpd)\"", 
                 "-type", "ora.gipc.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
	writeCkpt($ckptName,CKPTFAIL);
	die("Failed to add  resource ora.gipcd");
      }
    } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.gipcd", "-f", "-init");
   }

    if ($CFG->platform_family ne "windows") 
    {
       if ($action eq "add") 
       {
	 $status = system_cmd($crsctl, "add", "resource", "ora.diskmon",
                  "-attr", '"', join(',', @diskmon_attr), '"',
                  "-type", "ora.diskmon.type", "-init");

         if ($status != CMDSUC) {
	   writeCkpt($ckptName, CKPTFAIL);
	   die("Failed to add  resource ora.diskmon");
         }
       } else {
         @out = system_cmd_capture($crsctl, "delete", "resource", "ora.diskmon", "-f", "-init");
       }
    }
    
   if ($action eq "add") {
      my @cmd =  ($crsctl, "add", "resource", "ora.cssdmonitor",
                  "-attr", '"', join(',', @css_attrmon), '"',
                  "-type", "ora.cssdmonitor.type", "-init", "-f");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.cssdmonitor");
      }
   } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.cssdmonitor", "-f", "-init");
   }
   
   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.cssd",
                 "-attr", '"', join(',', @css_attr), '"',
                 "-type", "ora.cssd.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.cssd");
     }
   } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.cssd", "-f", "-init");
   }

   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.ctssd", "-attr", 
                 "\"ACL='owner:$CRSOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$CRSDUSER:r-x'\"", 
                 "-type", "ora.ctss.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.ctssd");
     }
   } else {
     @out = system_cmd_capture($crsctl, "delete", "resource", "ora.ctssd", "-f", "-init");
   }

   if ( $action eq "add" ) {
      if (isHAIPsupported() ) {
         my @cmd = ($crsctl, "add", "resource", "ora.cluster_interconnect.haip", "-attr", 
                    "\"ACL='owner:$CRSOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$CRSDUSER:r-x'\"", 
                    "-type", "ora.haip.type", "-init");
         if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
            push @cmd, '-buildowner';
         }

         my $status = system_cmd(@cmd);
         if ($status != CMDSUC) {
            writeCkpt($ckptName,CKPTFAIL);
            die("Failed to add  resource ora.cluster_interconnect.haip");
         }
      }
   } else {
     @out = system_cmd_capture($crsctl, "delete", "resource", "ora.cluster_interconnect.haip", "-f", "-init");
   }
   
   # add startup and stop dependencies for evmd in cluster mode; 
   # those dependencies not used in SIHA
   push @evm_attr, "START_DEPENDENCIES='hard(ora.cssd, ora.ctssd, ora.gipcd)" .
	                                "pullup(ora.cssd, ora.ctssd, ora.gipcd)'";

   push @evm_attr, "STOP_DEPENDENCIES='hard(intermediate:ora.cssd,intermediate:ora.gipcd)'";
   
   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.evmd",
                 "-attr", '"', join(',', @evm_attr), '"',
                 "-type", "ora.evm.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.evmd");
     }
   } else {
     @out = system_cmd_capture($crsctl, "delete", "resource",
                                "ora.evmd", "-f", "-init");
   }

   if (isCRFSupported()) {
      if ($action eq "add") {
         my @cmd = ($crsctl, "add", "resource", "ora.crf", "-attr", 
                    "\"ACL='owner:$CRSOWNER:rw-,pgrp:$grp:rw-,other::r--,user:$CRSDUSER:r-x'\"",
                    "-type", "ora.crf.type", "-init");

         if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
            push @cmd, '-buildowner';
         }

         my $status = system_cmd(@cmd);
         if ($status != CMDSUC) {
            writeCkpt($ckptName,CKPTFAIL);
            die("Failed to add  resource ora.crf");
         }
      } else {
         @out = system_cmd_capture($crsctl, "delete", "resource", "ora.crf", "-f", "-init");
      }
   }

    # ora.ctssd dependency only needed for cluster and not for siha
    if (isHAIPsupported() )
    {
      if( ($OSNAME eq "hpux") || ($OSNAME eq "aix") )
      {
        push @asm_attr, "START_DEPENDENCIES='hard(ora.cssd,ora.ctssd)" .
            "pullup(ora.cssd,ora.cluster_interconnect.haip,ora.ctssd)" .
            "weak(ora.cluster_interconnect.haip,ora.drivers.acfs)'";
      }
      else
      {
        push @asm_attr, "START_DEPENDENCIES='hard(ora.cssd,ora.cluster_interconnect.haip,ora.ctssd)" .
             "pullup(ora.cssd,ora.cluster_interconnect.haip,ora.ctssd)" .
             "weak(ora.drivers.acfs)'";
      }

      push @asm_attr, "STOP_DEPENDENCIES='hard(intermediate:ora.cssd," .
                                    "shutdown:ora.cluster_interconnect.haip)'";
    }
    else
    {
      push @asm_attr, "START_DEPENDENCIES='hard(ora.cssd,ora.ctssd)" .
                                          "pullup(ora.cssd,ora.ctssd)" .
                                          "weak(ora.drivers.acfs)'";
    }
    
    # When OCR is on ASM, add ora.asm as a HARD and PULLUP start dependency
    # These need to be consistent with :
    # has/crs/template/crs.type
    # prou.c
    if (($CFG->ASM_STORAGE_USED) || ( $CFG->UPGRADE && isOCRonASM() )) 
    {
       push @crsd_attr, "START_DEPENDENCIES='hard(ora.asm,ora.cssd,ora.ctssd,ora.gipcd)" .
	 "pullup(ora.asm,ora.cssd,ora.ctssd,ora.gipcd)'";

       push @crsd_attr, "STOP_DEPENDENCIES='hard(shutdown:ora.asm,intermediate:ora.cssd,intermediate:ora.gipcd)'";
     }

   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.asm",
                 "-attr", '"', join(',', @asm_attr), '"',
                 "-type", "ora.asm.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.asm");
      }
   } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.asm", "-f", "-init");
   }

   if ($action eq "add") {
      my @cmd = ($crsctl, "add", "resource", "ora.crsd",
                 "-attr", '"', join(',', @crsd_attr), '"',
                 "-type", "ora.crs.type", "-init");
      if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
         push @cmd, '-buildowner';
      }

      my $status = system_cmd(@cmd);
      if ($status != CMDSUC) {
       writeCkpt($ckptName,CKPTFAIL);
       die("Failed to add  resource ora.crsd");
      }
   } else {
      @out = system_cmd_capture($crsctl, "delete", "resource", "ora.crsd", "-f", "-init");
   }

   if (isACFSSupported()) {
      my $asmgrp = $CFG->params('ORA_ASM_GROUP');
      if ($CFG->platform_family ne "windows") {
         $owner  = $CFG->SUPERUSER;
      }

      if ($action eq "add") {
         my @cmd = ($crsctl, "add", "resource", "ora.drivers.acfs", "-attr", 
                    "\"ACL='owner:$owner:rwx,pgrp:$asmgrp:r-x,other::r--,user:$CRSDUSER:r-x'\"", 
                    "-type", "ora.drivers.acfs.type","-init");
         if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
            push @cmd, '-buildowner';
         }

         my $status = system_cmd(@cmd);
         if ($status != CMDSUC) {
            writeCkpt($ckptName,CKPTFAIL);
            die("Failed to add  resource ora.drivers.acfs");
         }
      } else {
          @out = system_cmd_capture($crsctl, "delete", "resource", 
                                    "ora.drivers.acfs", "-f", "-init");
      }
   }
  } 
  elsif ($mode eq "has") {
    # SI-HA cssd does not depend on mdnsd/gpnpd
    push @css_attr, "ACL='owner:$CSSOWNER:rwx,pgrp:$grp:rwx,other::r--'"; 
    push @css_attr, "RESTART_ATTEMPTS=5";
    push @diskmon_attr, "ACL='owner:$CSSOWNER:rwx,pgrp:$grp:rwx,other::r--'";

    if ($CFG->platform_family ne "windows")
    {
      push @css_attr, "START_DEPENDENCIES='weak(concurrent:ora.diskmon)'";
      push @css_attr, "STOP_DEPENDENCIES='hard(shutdown:ora.diskmon)'";
      push @diskmon_attr, "START_DEPENDENCIES='weak(concurrent:ora.cssd)" .
                          "pullup:always(ora.cssd)'";

      if ($action eq "add") {
         my @cmd = ($crsctl, "add", "resource", "ora.diskmon",
                    "-attr", '"', join(',', @diskmon_attr), '"',
                    "-type", "ora.diskmon.type", "-init");
         if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
            push @cmd, '-buildowner';
         }

         my $status = system_cmd(@cmd);
         if ($status != CMDSUC) {
          writeCkpt($ckptName,CKPTFAIL);
          die("Failed to add  resource ora.diskmon");
         }
       } else {
	 @out = system_cmd_capture($crsctl, "delete", "resource", "ora.diskmon", "-f", "-init");
       }
    }

    if ($action eq "add") {
       my @cmd = ($crsctl, "add", "resource", "ora.cssd",
                  "-attr", '"', join(',', @css_attr), '"',
                  "-type", "ora.cssd.type", "-init");
       if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
          push @cmd, '-buildowner';
       }

       my $status = system_cmd(@cmd);
       if ($status != CMDSUC) {
         writeCkpt($ckptName,CKPTFAIL);
         die("Failed to add  resource ora.cssd");
       }
    } else {
          @out = system_cmd_capture($crsctl, "delete", "resource", "ora.cssd", "-f", "-init");
    }
     
    if ($action eq "add") {
       my @cmd = ($crsctl, "add", "resource", "ora.evmd",
                  "-attr", '"', join(',', @evm_attr), '"',
                  "-type", "ora.evm.type", "-init");
       if (! is_dev_env() && ($CFG->platform_family eq "windows")) {
          push @cmd, '-buildowner';
       }

       my $status = system_cmd(@cmd);
       if ($status != CMDSUC) {
         writeCkpt($ckptName,CKPTFAIL);
         die("Failed to add  resource ora.evmd");
       }
     } else {
          @out = system_cmd_capture($crsctl, "delete", "resource", "ora.evmd", "-f", "-init");
	}
  }

  if (($action eq "delete") && isHAIPsupported())
  {
    trace("Unregistering type $haipType");
    @out = system_cmd_capture($crsctl, "delete","type", $haipType, "-init");
  }

  if ($action eq "delete")
  {
     trace("Deleting base types");

     foreach my $type (@types)
     {
       $file = $type . '.type';
       $name = 'ora.' . $type . '.type';
       $logdir = catdir($ORA_CRS_HOME, "log", $hostname);
       $ohasdlog = catdir($logdir, "ohasd");
       @out =  system_cmd_capture($crsctl, "delete","type", $name, "-init");
     }

     @out = system_cmd_capture($crsctl, "delete","type", $baseType, "-init");
  } 

  if ($action eq "add")
  {
    writeCkpt($ckptName,CKPTSUC);
    $wipCkptName = "ROOTCRS_STACK";
  }

  return TRUE;
}

#----------------------( instantiateTemplate )--------------------------#
#                                                                       #
#                                                                       #
#  FUNCTION: instantiateTemplate                                        #
#                                                                       #
#  PURPOSE: Instantiates the cap file with the CRS HOME location        #
#                                                                       #
#-----------------------------------------------------------------------#
sub instantiateTemplate
{
  my $ORACLE_HOME = $CFG->params('ORACLE_HOME');

  my ($inFile, $outFile) = @_;

  #TODO Define this based on platforms
  my $FSEP = '/';

  # If I can read the template or cap, instantiate the file replacing any 
  # special values
  if ( -r $inFile) 
  {
    open (INF, "<", "$inFile") or
        fatal("Unable to open $inFile, $!, ");

    # Make sure to open output file safely
    if ( -r $outFile ) 
    {
      trace("Removing pre existing $outFile from a previous run.");
      unlink($outFile) or 
          die("ERROR: Unable to remove $outFile,  $!,");
    }
    open (OUTF, ">", "$outFile") or
        die("ERROR: Unable to open file for writing $outFile,  $!,");

    # 
    # Filter Transformations
    #
    while (<INF>) 
    {
      # Modify CRS Home
      s/%ORA_CRS_HOME%/$ORACLE_HOME/;

      # Modify File Separators for NT
      s/\/\//$FSEP/g;
      print OUTF $_;
    }
    close(INF) or
        die("ERROR: Unable to close $inFile, $!");

    close(OUTF) or
        die("ERROR: Unable to close $outFile, $!");

  } 
  else 
  {
    error("$inFile is not readable.",
          "Verify the value of ORACLE_HOME, ");
    die("ERROR: Failed to register $inFile with the OHASD,");
  }

}

sub ValidateCommand
#---------------------------------------------------------------------
# Function: Validate system command to ensure command exists and
#           exececutable.
# Args    : 1
#---------------------------------------------------------------------
{
   my $cmd = $_[0];

   trace("Validating $cmd");
   if (-x $cmd) {
      return (TRUE);
   } else {
      return (FALSE);
   }
}

sub isOwnerGroupValid
#-------------------------------------------------------------------------------
# Function: Validate owner and group
# Args    : none
# Returns : TRUE - if owner and group is     valid
#	    FALSE- if owner and group is NOT valid
#-------------------------------------------------------------------------------
{
   # validate owner
   my $owner = $CFG->params('ORACLE_OWNER');
   if (($CFG->force) && ($owner =~ "%")) {
      return FALSE;
   }

   # validate group
   my $group = $CFG->params('ORA_DBA_GROUP');
   if (($CFG->force) && ($group =~ "%")) {
      return FALSE;
   }

   return TRUE;
}

sub getHostVIP
#---------------------------------------------------------------------
# Function: Get Host's VIP from CLUSTER_NEW_VIPS
# Args    : [0] Hostname
# Returns : Host's VIP
#---------------------------------------------------------------------
{
   my $hostname     = $_[0];
   my @new_hosts    = split (/,/, $CFG->params('CLUSTER_NEW_HOST_NAMES'));
   my @new_vips     = split (/,/, $CFG->params('CLUSTER_NEW_VIPS'));
   my $nbr_of_hosts = scalar(@new_hosts);
   my $nbr_of_vips  = scalar(@new_vips);
   my $srvctl       = catfile ($CFG->ORA_CRS_HOME, 'bin', 'srvctl');
   my $ix           = 0;
   my ($hostVIP, $netmask, $if);

   if (($CFG->params('CRS_DHCP_ENABLED') ne 'true') &&
       ($nbr_of_hosts != $nbr_of_vips)) 
   {
      die("Mismatching Host to VIP IPs specified");
   }

   # check version number
   # use '-S 1' if version is 11.2. Otherwise use '-a'
   my @version = get_crs_version($CFG->ORA_CRS_HOME);

   if ($version[0] > '10' && $version[1] > '1') {
      # use '-S 1'
      open OPUT, "$srvctl config nodeapps -S 1|";
      my @viplist = grep(/netmask=/, <OPUT>);
      close OPUT;

      trace("viplist = '@viplist'");

      if (scalar(@viplist) > 0) {
         trace ("viplist='$viplist[0]'");

         # get new VIP
         $ix = 0;
         foreach my $host (@new_hosts) {
            chomp $host;
            if ($hostname =~ /$host/i) {
               last;
            }
            $ix++;
         }

         # append netmask/if to new vip
         my @list = split(/} +/, $viplist[0]);
         foreach (@list) 
	 {
	   trace("IP list=$_");

            if ($_ =~ /netmask=/) {
               $netmask = parseText ($_);
            }
            elsif ($_ =~ /interfaces=/) {
               $if = parseText ($_);
            }
         }

	 trace("netmask = $netmask");
	 trace("interface = $if");
      }
   }
   else {
      # use '-a'
      open OUTPUT, "$srvctl config nodeapps -a|";
      my @viplist = (<OUTPUT>);
      chomp @viplist;

      trace ("VIPList=$viplist[1]");
      close OUTPUT;

      if ($viplist[1] =~ /:/) {
         # get new VIP
         my @VIPs = split (/\//, $viplist[1]);

         if (scalar(@VIPs) >= 5) 
	 {
	   $if      = $VIPs[5];
	   trace("interface = $if");
	 }

         if (scalar(@VIPs) >= 4) 
	 {
	   $netmask = $VIPs[4];
	   trace("netmask = $netmask");
	 }

         $ix = 0;
         foreach my $host (@new_hosts) {
            chomp $host;
            if ($hostname =~ /$host/i) {
               last;
            }

            $ix++;
         }
      }
   }

   # append netmask/if to new vip
   if ($netmask) {
      $hostVIP = $new_vips[$ix] . "/" . $netmask;
      if ($if) {
         # translate / replace ":" with "|" in the interface string
         $if =~ tr/[:]/[|]/;
         $hostVIP = $hostVIP . "/" . $if;
      }
   }

   sub parseText {
      # extract netmask and interface
      my $line = $_[0];
      $line =~ s/{//g;
      $line =~ s/}//g;
      my ($dummy, $text) = split (/=/, $line);
      chomp $text;

      return $text;
   }

   trace("hostVIP = $hostVIP");

   return $hostVIP;
}

sub isAddNode
#---------------------------------------------------------------------
# Function: Check if hostname is a new node.
#
# Args    : [0] Name of host to check
#           [1] List of nodes in config
#
# Returns : TRUE  if     new node
#           FALSE if not new node
#---------------------------------------------------------------------
{
   my $hostname = $_[0];
   my $nodelist = $_[1];

   if ($CFG->defined_param('CRS_ADDNODE') && $CFG->params('CRS_ADDNODE') eq "true")
   {
     if ($DEBUG) { trace("CRS_ADDNODE is defined to 'true'"); }
     return TRUE;
   } elsif ($nodelist !~ /\b$hostname\b/i) 
   {
     if ($DEBUG) { trace("'$hostname' is not part of existing nodelist:$nodelist"); }
     return TRUE;
   }

   if ($DEBUG) { trace("Not an add node scenario"); }

   return FALSE;
}

sub srvctl
#-------------------------------------------------------------------------------
# Function: Run srvctl with the given arguments.
# Args    : [0] - TRUE  if run as ORACLE_OWNER
#               - FALSE if run as root
#           [1] - srvctl arguments
# Returns : TRUE  if successful
#           FALSE if failed
#-------------------------------------------------------------------------------
{
   my $run_as_oracle_owner = $_[0];
   my $srvctl_args         = $_[1];
   my $ORA_CRS_HOME        = $CFG->ORA_CRS_HOME;
   my $ORACLE_OWNER        = $CFG->params('ORACLE_OWNER');
   my $ORA_DBA_GROUP       = $CFG->params('ORA_DBA_GROUP');
   my $srvctlbin           = catfile ($ORA_CRS_HOME, "bin", "srvctl");
   my $status;
 
   # set trace file 
   my $srvctl_trc_file = catfile ($ORA_CRS_HOME, 'cfgtoollogs', 'crsconfig', 
                                  "srvmcfg" . $srvctl_trc_suff++ . ".log");

   $ENV{SRVM_TRACE}       = "TRUE";
   $ENV{SRVCTL_TRACEFILE} = $srvctl_trc_file;

   my $cmd = "${srvctlbin} $srvctl_args";
   trace ("Invoking \"${cmd}\"");
   trace ("trace file=$srvctl_trc_file");

   if ($run_as_oracle_owner) {
      $status = run_as_user ($ORACLE_OWNER, ${cmd});
   }
   else {
      $status = system (${cmd});
   }

   # set owner & permission of trace file
   s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $srvctl_trc_file);
   s_set_perms ("0775", $srvctl_trc_file);

   #Bug 9718507 . srvctl return code 2 is now only for warnings that can
   #be ignored
   if ((0 == $status) || (2 == $status)) {
      return TRUE;
   } else {
      trace ("  \"${cmd}\" failed with status ${status}.");
      return FALSE;
   }
}
sub getSubnet
#---------------------------------------------------------------------
# Function: Get subnet from $NETWORKS
#
# Args    : [0] $NETWORKS
#
# Returns : subnet
#---------------------------------------------------------------------
{
   my $networks = $_[0];
   my $subnet;

   if ($networks =~ /\bpublic\b/) {
      my @network_ifs = split (/,/, $networks);

      foreach my $network_if (@network_ifs) {
         if ($network_if =~ /\bpublic\b/) {
            # strip out "eth*" and ":public"
            my ($eth, $txt) = split (/\//, $network_if);
            ($subnet, $txt) = split (/:public/, $txt);
            last;
         }
      }
   }

   return $subnet;
}

sub add_Nodeapps
#-------------------------------------------------------------------------------
# Function: Add nodeapps for static IP & DHCP
# Args    : [0] upgrade_opt
#           [1] nodevip
#           [2] DHCP_flag
#           [3] nodes_to_add
#           [4] nodes_to_start
# Returns : TRUE  if success
#           FALSE if failed
#           nodes_to_start - list of nodes to start
#-------------------------------------------------------------------------------
{
   my $upgrade_opt        = shift;
   my $nodevip_ref        = shift;
   my $isDHCP             = shift;
   my $nodes_to_add_ref   = shift;
   my $nodes_to_start_ref = shift;

   trace ("adding nodeapps...");
   trace ("upgrade_opt=$upgrade_opt");
   trace ("nodevip=@$nodevip_ref");
   trace ("DHCP_flag=$isDHCP");
   trace ("nodes_to_add=@$nodes_to_add_ref");

   my $config_nodeapps  = catfile ($CFG->ORA_CRS_HOME, "bin",
                                   "srvctl config nodeapps");
   my $vip_exists       = FALSE;
   my $success          = TRUE;
   my $run_as_owner     = FALSE;
   my @output;

   if ($isDHCP) {
      trace ("add nodeapps for DHCP");
      my $node    = $$nodes_to_add_ref[0];
      push @$nodes_to_start_ref, $node;

      # Currently we don't support multiple public subnets. The installer should
      # be smart enough not to allow user to select more than 1 public subnet.
      my $subnet  = getSubnet ($CFG->params('NETWORKS'));
      my $nodevip = shift (@$nodevip_ref);
      $nodevip    =~ s/AUTO/$subnet/;  # substitute AUTO w/ subnet

      my $status = srvctl($run_as_owner,
                          "add nodeapps -S \"$nodevip\" $upgrade_opt");
      if (${status}) {
         trace ("add nodeapps -S $nodevip on node=$node ... passed");
      } else {
         error ("add nodeapps -S $nodevip on node=$node ... failed");
         $success = FALSE;
      }

      return $success;
   }

   # add nodeapps for STATIC IP
   trace("add nodeapps for static IP");

   # The following  check is only valid in case of an addnode scenario
   if (! $CFG->UPGRADE) {
      trace("Running srvctl config nodeapps to detect if VIP exists");
      open OPUT, "$config_nodeapps |";
      @output = grep(/(^PRKO-2312|^PRKO-2331|^PRKO-2339)/, <OPUT>);
      close OPUT;
      if (scalar(@output) == 0) {
         trace ("vip exists");
         $vip_exists = TRUE;
      } else {
         trace ("output=@output");
      }
   }

   foreach my $node (@$nodes_to_add_ref) {
      $node =~ tr/A-Z/a-z/; #convert to lowercase
      my $nodevip = shift (@$nodevip_ref)
                       or die "ERROR: No more elements in crs_nodevip_list";
      my $status;
      my $cmd;
      my @txt = grep (/$node/, @output);

      if (scalar(@txt) == 0) {   # nodeapps is not yet added on this node
         if ($vip_exists) {
            $cmd = "add vip -n $node -k 1 -A $nodevip";
            $status = srvctl($run_as_owner,
                             "add vip -n $node -k 1 -A \"$nodevip\" " .
                             "$upgrade_opt");
         } else {
            $vip_exists = TRUE;
            if ($CFG->UPGRADE){
               my($onslocport, $onsremport) = get_ons_port($node);
               $cmd = "add nodeapps -n $node -l $onslocport " .
                      "-r $onsremport -A $nodevip";
               $status = srvctl($run_as_owner,
                         "add nodeapps -n $node -l $onslocport -r $onsremport " .
                         "-A \"$nodevip\" $upgrade_opt");
            } else {
               $cmd = "add nodeapps -n $node -A $nodevip";
               $status = srvctl($run_as_owner,
                                "add nodeapps -n $node -A \"$nodevip\" " .
                                "$upgrade_opt");
            }
         }

         if (${status}) {
            push @$nodes_to_start_ref, $node;
            trace ("$cmd on node=$node ... passed");
         } else {
            error ("$cmd on node=$node ... failed");
            $success = FALSE;
         }
      }
   }

   trace ("nodes_to_start=@$nodes_to_start_ref");
   return $success;
}

sub start_Nodeapps
#-------------------------------------------------------------------------------
# Function: Start nodeapps for static IP & DHCP
# Args    : [0] - DHCP_flag - TRUE if it's DHCP
#           [1] - nodes_to_start - list of nodes to be started
# Returns : TRUE  if success
#           FALSE if failed
#-------------------------------------------------------------------------------
{
   my $isDHCP             = shift;
   my $nodes_to_start_ref = shift;
   trace ("starting nodeapps...");
   trace ("DHCP_flag=$isDHCP");
   trace ("nodes_to_start=@$nodes_to_start_ref");

   my $srvctl  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'srvctl');
   my $success = TRUE;
   my $exit_value;
   my @output;
   my $cmd;
   my $rc;
   my $status;

   foreach my $node (@$nodes_to_start_ref) {
      if (($isDHCP) && (! isFirstNodeToStart())) {
         # wait for vip resource to exist before start vip
         my $vip_exists = waitForVipRes();
         if (! $vip_exists) {
            error ("VIP resource does not exist");
            return FALSE;
         }

         $cmd = "$srvctl start vip -i $node";
      } 
      else {
         $cmd = "$srvctl start nodeapps -n $node";
      }

      $rc = `$cmd`;
      $exit_value=$?>>8;

      trace("exit value of start nodeapps/vip is $exit_value");
      if ( $exit_value != 0) {
         my @lines = split("\n",$rc);
         trace("output for start nodeapps is  @lines");
         @output=grep(!/(^PRKO-2419|^PRKO-242[0-3])/,@lines);
         trace("output of startnodeapp after removing already started mesgs is @output");

         if (scalar(@output) >= 1) {
            error ("$cmd ... failed");
            $success = FALSE;
         } else {
            trace ("$cmd ... passed");
         }
      }
   }

   return $success;

}

sub add_GNS
#---------------------------------------------------------------------
# Function: Add GNS
# Args    : [0] list of addresses on which GNS is to listen
#           [1] domain(s) which GNS is to service.
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   if ($CFG->params('GNS_CONF') ne "true") {
      trace ("GNS is not to be configured - skipping");
      return TRUE;
   }

   my ($address_list, $domain_list) = @_;
   my $run_as_owner                 = FALSE;
   my $status = srvctl($run_as_owner,
                       "add gns -i ${address_list} -d ${domain_list}");

   if (TRUE == ${status}) {
      trace ("add gns -i $address_list -d $domain_list ... passed");
   } else {
      error ("add gns -i $address_list -d $domain_list ... failed");
      return FALSE;
   }

   return TRUE;
}

sub start_GNS
#---------------------------------------------------------------------
# Function: Start GNS
# Args    : nonde
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   if ($CFG->params('GNS_CONF') ne "true") {
      trace ("GNS is not to be configured - skipping");
      return TRUE;
   }

   # check if gns is running
   my $srvctl    = catfile ($CFG->ORA_CRS_HOME, 'bin', 'srvctl');
   my $save_lang = $ENV{LANG};
   $ENV{LANG}    = "";

   open OPUT, "$srvctl status gns|";
   my @output = grep(/not running/, <OPUT>);
   close OPUT;
   $ENV{LANG} = $save_lang;

   if (scalar(@output) > 0) {
      trace ("gns is not running");
   }
   else {
      trace ("gns is running");
      return TRUE;
   }

   # start gns
   my $run_as_owner = FALSE;
   my $status       = srvctl($run_as_owner, "start gns");

   if (${status}) {
      trace ("start gns ... passed");
   } else {
      error ("start gns ... failed");
      return FALSE;
   }

   return TRUE;
}

sub enable_GSD
{
   # enable GSD
   my $success 	    = TRUE;
   my $run_as_owner = TRUE;
   my $status       = srvctl($run_as_owner, "enable nodeapps -g");

   if (${status}) {
      trace ("enable nodeapps -g ... passed");
   } else {
      error ("enable nodeapps -g ... failed");
      $success = FALSE;
   }

   if ($success) {
      # start nodeapps
      $run_as_owner = FALSE;
      my $status = srvctl($run_as_owner, "start nodeapps");

      if (${status}) {
         trace ("start nodeapps ... passed");
      } else {
         # At this point Network resource is already started.
         # Therefore it's OK to ignore return code from "start nodeapps".
         trace ("start nodeapps ... failed. It's OK!!!");
      }
   }

   return $success;
}

sub configNewNode
#---------------------------------------------------------------------
# Function: Configure nodeapps for new node
# Args    : [0] new node
#           [1] DHCP tag to indicate if DHCP is used
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   my $newnode             = $_[0];
   my $srvctl 		   = catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl');
   my $run_as_oracle_owner = FALSE;
   my (@capout, @cmd, $status);

   trace ("Configure Nodeapps for new node=$newnode");

   if ($CFG->params('CRS_DHCP_ENABLED') eq 'false') 
   {
      # get VIP
      my $hostvip = getHostVIP($newnode);
      if (! $hostvip) {
         die("Unable to get VIP info for new node");
      }
      # add vip
	$status = srvctl($run_as_oracle_owner,
                       "add vip -n $newnode -k 1 -A \"$hostvip\" ");

	if (${status}) {
	  trace ("add vip on node=$newnode ... success");
	} else {
	  error ("add vip on node=$newnode ... failed");
	  return FALSE;
	}

   }
     
     # wait for vip resource to exist before start vip
     if ($CFG->params('CRS_DHCP_ENABLED') eq 'true') {
        my $vip_exists = waitForVipRes();
        if (! $vip_exists) {
           error ("VIP resource does not exist");
           return FALSE;
        }
     }

     # start vip
     $status = srvctl($run_as_oracle_owner, "start vip -i $newnode");

     if (${status}) {
       trace ("start vip on node:$newnode ... success");
     } else {
       error ("start VIP on node:$newnode ... failed");
       return FALSE;
     }

     # start listener
     @cmd    = ($srvctl, 'start', 'listener', '-n', $newnode);
     $status = run_as_user2($CFG->params('ORACLE_OWNER'), \@capout, @cmd);

     if ($status == 0) {
       trace ("@cmd ... success");
     } # check if listener already running
     elsif (scalar(grep(/PRCC-1015/, @capout)) > 0) {
       trace ("listener was already running on $newnode");
     }
     else {
       error ("@cmd ... failed");
       return FALSE;
     }

   return TRUE;
}

sub createDiskgroupRes
#---------------------------------------------------------------------
# Function: Create and start ASM diskgroup resource on all nodes
# Args    : none
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   my $ORA_CRS_HOME     = $CFG->ORA_CRS_HOME;
   my $ORACLE_OWNER     = $CFG->params('ORACLE_OWNER');
   my $ORACLE_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
   my $success          = TRUE;
   my $cmd;

   trace ("Adding ASM diskgroup resource");

   # convert ASM_DISK_GROUP to upper-case
   my $crsctl	      = catfile($ORA_CRS_HOME, 'bin', 'crsctl');
   my $ASM_DISK_GROUP = uc($CFG->params('ASM_DISK_GROUP'));
   if ($ASM_DISK_GROUP =~ /\$/) {
      # if diskgroup contains '$', put single-quotes around it
      quoteDiskGroup($ASM_DISK_GROUP);
      $cmd = "$crsctl create diskgroup '$ASM_DISK_GROUP'";
   }
   else {
      $cmd = "$crsctl create diskgroup $ASM_DISK_GROUP";
   }

   my $status = run_as_user ($ORACLE_OWNER, $cmd);

   if ($status == 0) {
      trace ("create diskgroup $ASM_DISK_GROUP ... success");
   } else {
      error ("create diskgroup $ASM_DISK_GROUP ... failed");
      return FALSE;
   }

   trace ("Successfully created disk group resource");

   # since diskgroup rescource is successfully added on the lastnode
   # we need to start diskgroup rescource on other nodes
   # get the local node
   my $olsnodes = catfile($ORA_CRS_HOME, 'bin', 'olsnodes -l');
   open (OLSNODES, "$olsnodes |") or die "olsnodes failed: $!";
   my @output = (<OLSNODES>);
   close OLSNODES;
   chomp @output;
   my $local_node = $output[0];

   # get the list of all nodes
   $olsnodes = catfile($ORA_CRS_HOME, 'bin', 'olsnodes');
   open (OLSNODES, "$olsnodes |") or die "olsnodes failed: $!";
   my @nodes = (<OLSNODES>);
   close OLSNODES;

   # build node_list from olsnodes, except for local node
   my $node_list = "";
   foreach my $node (@nodes) {
      chomp $node;
      if ($node ne $local_node) {
         if ($node_list ne "") {
            $node_list = $node_list . ",";
         }

         $node_list = $node_list . $node;
      }
   }

   # node_list eq "" means 1-node install
   if ($node_list ne "") {
      # start diskgroup on all nodes
      my $run_as_oracle_owner = FALSE;
      $status = srvctl($run_as_oracle_owner,
                       "start diskgroup -g $ASM_DISK_GROUP -n \"$node_list\" ");

      if (${status}) {
         trace ("start diskgroup resource ... success");
      } else {
         error ("start diskgroup resource ... failed");
         return FALSE;
      }
   }

   return $success;
}


sub perform_configNode
{
  my $ckptStatus;
  my $ckptName = "ROOTCRS_NODECONFIG";

  if (isCkptexist($ckptName))
  {
    $ckptStatus = getCkptStatus($ckptName);
    trace("'$ckptName' state is $ckptStatus");

    if (($ckptStatus eq CKPTSUC) && (! ($UPGRADE && $CFG->force))) {
       trace("CRS node configuration resources are already configured");
       $wipCkptName = "ROOTCRS_STACK";
       return SUCCESS;
    }
  }

  writeCkpt($ckptName, CKPTSTART);
  $wipCkptName = $ckptName;

  configNode();
}

sub configNode
#---------------------------------------------------------------------
# Function: Configure node
# Args    : none
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   trace ("Configuring node");
   my $DHCP_flag = FALSE;
   my $success   = TRUE;
   my $ckptName = "ROOTCRS_NODECONFIG";
   my $asmcainitflag = FALSE;

   

   # set DHCP_flag to TRUE if it's DHCP
   my $crs_nodevips = $CFG->params('CRS_NODEVIPS');
   $crs_nodevips    =~ s/'//g; # ' in comment to avoid confusion of editors.
   $crs_nodevips    =~ s/"//g; # remove " on Windows
   $crs_nodevips    =~ s/\/\*/\//g; # handle different interface case

   my @crs_nodevip_list = split (/\s*,\s*/, $crs_nodevips);

   if ($crs_nodevip_list[0] =~ /\bAUTO/) {
      $DHCP_flag = TRUE;
   }

   # configure new node
   if (isAddNode($HOST, $CFG->params('NODE_NAME_LIST'))) 
   {
     trace("Performing new node configurations");
     $success = configNewNode($HOST);
     if ($success) 
     {
       writeCkptProperty($ckptName, $HOST, "ADDED_NODE");
       writeCkpt($ckptName, CKPTSUC);
     } else {
       writeCkpt($ckptName, CKPTFAIL);
       die("Failed to perform new node configuration");
     }
 
     return $success;
   }

   # configure upgraded node 
   if ($CFG->UPGRADE)
   {
     trace("Performing upgraded node configurations");
     my $lastNode = FALSE;

     if (! isVersion112()) 
     {
       if (isLastNodeToUpgrade ($HOST, $CFG->params('NODE_NAME_LIST'))) {
	 $lastNode = TRUE;
	 $success = configLastNode(@crs_nodevip_list_old);
       }

       # clean-up from upgrade
       if ($CFG->platform_family eq 'unix') {
	 s_houseCleaning();
       }
       
       if ($success) {
	 writeCkpt($ckptName, CKPTSUC);
       } else {
	 writeCkpt($ckptName, CKPTFAIL);
         die("Failed to perform last node configuration for upgrade");
       } 
       
       return $success;
     } else {

        # Update the replication resources on all nodes 
        acfsrepl_updateResources();
        if (isFirstNodeToUpgrade ($HOST, $CFG->params('NODE_NAME_LIST'))) {
	        $lastNode = FALSE;
          
          if (! isRolling())
          {
             $success = perform_nonrollingASM_upg($FIRSTNODE, $asmcainitflag);
             if (!$success) {
              writeCkpt($ckptName, CKPTFAIL);
              die("Failed to perform first node non rolling asm upgrade tasks for cluster ");
             }
          }

          $success = upgrademodel($FIRSTNODE);
          if (!$success) {
              writeCkpt($ckptName, CKPTFAIL);
              die("Failed to perform first node tasks for cluster modeling upgrade");
          }
          
          # Stop and disable OC4J since credentials will be changed.
          stop_J2EEContainer();
          #FIXME: move line above inside validation below once OC4J is expanded in 
          #old CRS home. Bug 16075979.
          if (disable_J2EEContainer() == SUCCESS) {
              $success = TRUE;
              trace("J2EE (OC4J) Container Resource stopped and disabled");
          } else {
              $success = FALSE;
              trace("Failed to perform J2EE (OC4J) Container Resource stop/disabled");
              writeCkpt($ckptName, CKPTFAIL);
              die("Failed to perform J2EE (OC4J) Container Resource stop/disabled");
          }

        }

        if (isLastNodeToUpgrade ($HOST, $CFG->params('NODE_NAME_LIST'))) {
            
            $success = setActiveversion();
            if (!$success) {
              writeCkpt($ckptName, CKPTFAIL);
              die("Failed to set active version of the Grid Infrastructure");
            }
            if (isRolling()) {
               $success = perform_rollingASM_upg($LASTNODE);
               if (!$success) {
                 writeCkpt($ckptName, CKPTFAIL);
                 die("Failed to perform last node tasks for ASM rolling upgrade");
               }
               trace("end rolling ASM upgrade in $LASTNODE node");
            }

            if (! isRolling()) {
               $success = perform_nonrollingASM_upg($LASTNODE, $asmcainitflag);
               if (!$success) {
                 writeCkpt($ckptName, CKPTFAIL);
                 die("Failed to perform last node tasks for ASM nonrolling upgrade");
               }
               trace("end nonrolling ASM upgrade in $LASTNODE node");
            }

            $success = upgrademodel($LASTNODE);
            if (!$success) {
              writeCkpt($ckptName, CKPTFAIL);
              die("Failed to perform last node tasks for cluster modeling upgrade");
            }

            upgrade_acfs_registry();
            acfsrepl_lastNode();

            # Add Wallet, enable and start OC4J resource
            if (addWallet_J2EEContainer() == SUCCESS &&
                start_J2EEContainer() == SUCCESS) 
            {
                $success = TRUE;
                trace("J2EE (OC4J) Container Resource resource successfully upgraded");
            } else {
                $success = FALSE;
                trace("Failed to perform J2EE (OC4J) Container Resource upgrade");
                writeCkpt($ckptName, CKPTFAIL);
                die("Failed to perform J2EE (OC4J) Container Resource upgrade");
            }

            # add cvu resource
            if (!isCVUConfigured ()) {
               if (add_CVU ()) {
                 start_CVU();
                 trace ("cvu resource successfully upgraded");
               }
            }
        }

        if ($success) 
        {
	   writeCkpt($ckptName, CKPTSUC);
        } else {
           writeCkpt($ckptName, CKPTFAIL);
           die("Failed to perform upgrade configuration from old 11.2 install");
        }

        return $success;
     }
   }

   # configure fresh install node
   my $upgrade_option;
   my @nodevip;
   my @nodes_to_add;
   my @nodes_to_start;

   # add nodeapps
   my @node_list = split (',', $CFG->params('NODE_NAME_LIST'));
   my $ix        = 0;

   foreach my $node (@node_list) {
      if ($CFG->HOST =~ /$node$/i) {
         push @nodevip, $crs_nodevip_list[$ix];
         push @nodes_to_add, $node;
         last; # done for this node
      } else {
         $ix++;
      }
   }

   if (isFirstNodeToStart()) 
   {
     $success = add_Nodeapps($upgrade_option, \@nodevip, $DHCP_flag, 
			     \@nodes_to_add, \@nodes_to_start);
     if ($success != TRUE) {
       writeCkpt($ckptName, CKPTFAIL);
       die("Failed to add  Nodeapps");
     }

     $success = configFirstNode($DHCP_flag, \@nodes_to_start);
     if ($success != SUCCESS) { 
       writeCkpt($ckptName, CKPTFAIL);
       die("FirstNode configuration failed"); 
     }

   } else {
      if ($DHCP_flag) {
         push @nodes_to_start, $nodes_to_add[0];
      }
      else {
         $success = add_Nodeapps($upgrade_option, \@nodevip, $DHCP_flag, 
				 \@nodes_to_add, \@nodes_to_start);
         if ($success != SUCCESS) {
            writeCkpt($ckptName, CKPTFAIL);
            die("Failed to add Nodeapps");
         }
      }

      if ($success) {
         $success = start_Nodeapps($DHCP_flag, \@nodes_to_start);
      }
         if ($success != SUCCESS) {
            writeCkpt($ckptName, CKPTFAIL);
            die("Failed to start Nodeapps");
         }

      if (! s_is92ConfigExists()) 
      {
	if ($DEBUG) { trace("Removing GSD PID file"); }
	remove_gsd_file();
      }

      # wait for cluster to come up
      check_service("cluster", 60);
   }

   writeCkpt($ckptName, CKPTSUC);
   $wipCkptName = "ROOTCRS_STACK";
   return $success;
}

sub configFirstNode
#---------------------------------------------------------------------
# Function: Configure first node
# Args    : [0] DHCP_flag
#           [1] nodes_to_start
# Returns : TRUE  if success
#           FALSE if failed
#---------------------------------------------------------------------
{
   my $DHCP_flag          = shift;
   my $nodes_to_start_ref = shift;

   trace ("Configuring first node");
   trace ("DHCP_flag=$DHCP_flag");
   trace ("nodes_to_start=@$nodes_to_start_ref");

   my $success = SUCCESS;

   #set the network interface - Bug 9243302
   setNetworkInterface();
   

   if (($CFG->params('ASM_UPGRADE') =~ m/false/i) && (! isASMExists())) {
      trace("Prior version ASM does not exist , Invoking add asm");
      add_ASM();  # add ora.asm
      if ($CFG->ASM_STORAGE_USED) {
         createDiskgroupRes();  # add disk group resource, if necessary
      }
    }

   add_acfs_registry();

   if ($success &&
       add_GNS($CFG->params('GNS_ADDR_LIST'),
               $CFG->params('GNS_DOMAIN_LIST')) &&
       add_scan() &&
       add_scan_listener() &&
       add_J2EEContainer() == SUCCESS) {
       $success = SUCCESS;
   } else {
       $success = FAILED;
   }
   
   if ($success) {
    add_CVU();
   }

   if ($success &&
       start_Nodeapps($DHCP_flag, \@$nodes_to_start_ref) &&
       start_GNS() &&
       start_scan() &&
       start_scan_listener() && 
       start_J2EEContainer() == SUCCESS)
   {
       $success = SUCCESS;
       if (($CFG->params('ASM_UPGRADE') =~ m/false/i) && (isASMExists())) {
          $success = start_acfs_registry(\@$nodes_to_start_ref);
       }

       if (s_is92ConfigExists()) {
          $success = enable_GSD();
       }
       else {
	  remove_gsd_file();
       }
   } else {
       $success = FAILED;
   }

   if ($success) {
      start_CVU();
   }

   return $success;
}

sub configLastNode
#---------------------------------------------------------------------
# Function: Configure last node (for upgrade only)
# Args    : [0] crs_nodevip_list - contains viplist
# Returns : TRUE  if success
#           FALSE if failed
#
# This routine is also invoked during FORCEUPGRADE 
#---------------------------------------------------------------------
{
   my @crs_nodevip_list = @_;
   trace ("Configuring last node");
   trace("Old nodeapps list is  @crs_nodevip_list");
   my $success        = TRUE;
   my $crsctlbin      = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");
   my $DHCP_flag      = FALSE;
   my $upgrade_option = "-u";
   my $status;
   my @nodes_to_start; #nodes where the nodeapps needs to be started.
   my @nodes_to_add;   #nodes where the nodeapps needs to be configured.

   # set DHCP_flag to TRUE if it's DHCP
   if ($crs_nodevip_list[0] =~ /\bAUTO/) {
      $DHCP_flag = TRUE;
   }

   # for upgrade VIP information would be read from existing OCR.
   upgrade_config();

   # Add nodeapps:
   # The force upgrade node list contains only the nodes that
   # have been upgraded. Otherwise, that's a normal upgrade and
   # all nodes should be involved.
   if (($CFG->force) && scalar(@force_upgrade_nodes) > 0)
   {
     trace("add nodeapps on force upgrade nodes");
     @nodes_to_add = @force_upgrade_nodes;
   }
   else
   {
     @nodes_to_add = split (',', $CFG->params('NODE_NAME_LIST'));
   }
   $success = add_Nodeapps($upgrade_option, \@crs_nodevip_list,
                           $DHCP_flag, \@nodes_to_add, \@nodes_to_start);

   # Trigger the active version change.
   if ($success && (! setActiveversion())) {
      $success = FAILED;
   }

   #Check for old ASM having a different owner than the grid owner
   # and change it to grid owner.

   if (isASMExists() && !isVersion112()) {
       update_ASMowner();
   }

   if ($success &&
      ($CFG->params('ASM_UPGRADE') =~ m/false/i) &&
      (! isASMExists())) {
      trace("Prior version ASM does not exist , Invoking add asm");
      add_ASM();  # add ora.asm
      if ($CFG->ASM_STORAGE_USED) {
         createDiskgroupRes();  # add disk group resource, if necessary
      }

   }


   add_acfs_registry();

   if ($success &&
       add_GNS($CFG->params('GNS_ADDR_LIST'),
               $CFG->params('GNS_DOMAIN_LIST')) &&
       add_scan() &&
       add_scan_listener() &&
       add_J2EEContainer()) {
       $success = SUCCESS;
   } else {
       $success = FAILED;
   }

   if ($success) {
      add_CVU ();
   }

   trace("Nodes to start is @nodes_to_start");

   my @alive_nodes_to_start;
   my %nodeStatus = getNodeStatus($CFG->ORA_CRS_HOME);
   foreach my $clunode (@nodes_to_start)
   {
     if (1 == $nodeStatus{$clunode}) # active node
     {
       push(@alive_nodes_to_start, $clunode);
     }
   }
   trace("Start nodeapps on active nodes: @alive_nodes_to_start");

   if ($success &&
       start_Nodeapps($DHCP_flag, \@alive_nodes_to_start) &&
       start_GNS() &&
       start_scan() &&
       start_scan_listener() &&
       start_J2EEContainer())
   {
       $success = SUCCESS;
       if (($CFG->params('ASM_UPGRADE') =~ m/false/i) && (is112ASMExists())) {
          $success = start_acfs_registry(\@nodes_to_start);
       }

       if (s_is92ConfigExists()) {
          $success = enable_GSD();
       }
       else {
	  remove_gsd_file();
       }
   } else {
       $success = FAILED;
   }

   if ($success) {
     start_CVU();
   }

   # wait for cluster to come up
   check_service("cluster", 60);

   return $success;
}

sub rscPreChecks
#---------------------------------------------------------------------
# Function: Pre-checks for running root script 
# Args    : None
# Returns : None
#---------------------------------------------------------------------
{
  trace("Performing few checks before running scripts");

  if ($CFG->platform_family eq "unix")
  {
    my @capout = ();
    my $rc;
    my $user = $CFG->params('ORACLE_OWNER');

    trace("Attempt to get current working directory");
    $rc = run_as_user2($user, \@capout, 'pwd');
    if (0 != $rc)
    {
      trace("Failed to get current working directory");
      for my $line (@capout)
      {
        chomp($line);
        print "$line\n";
      } 

      set_bold();
      print "Run root.sh from a directory that has read/execute access to the grid owner '$user'\n";
      reset_bold();

      exit(1);
    }
  }

  trace("Pre-checkes for running scripts passed");
}

sub getNodeStatus
#---------------------------------------------------------------------
# Function: Gets the status of each node in the cluster
# Args    : [0] CRS home (input)
# Returns : A hash array containing active/inactive nodes
#           The active nodes are marked with '1'.
#---------------------------------------------------------------------
{
  my $crshome = $_[0];
  my $rc = 0;
  my @olsnodes_out;
  my %nodeStatus;

  ($rc, @olsnodes_out) = get_olsnodes_info($crshome, '-s');
  if (0 != $rc)
  {
    die "Cannot get active nodes (" . join(' ',@olsnodes_out). ")";
  }

  foreach my $nodeinfo (@olsnodes_out)
  {
    if ($DEBUG) { trace("nodeinfo: $nodeinfo"); }

    my @nodeinf = split(/\s+/, $nodeinfo);
    if ($nodeinf[1] =~ /Active/)
    {
      $nodeStatus{${nodeinf[0]}} = 1;
    }
    else
    {
      $nodeStatus{${nodeinf[0]}} = 0;
    }
  }

  return %nodeStatus;
}

sub preForceUpgradeChecks
#---------------------------------------------------------------------
# Function: Pre-checks for force upgrade
# Args    : [0] The nodes that must be excluded from the cluster (output)
# Returns : SUCCESS if successful
#           FAILED if failed
# Does the following checks.
# 1. Check if the local node is upgraded
# 2. Check for inactive nodes and exclude the ones that are not upgraded
#    from the upgraded cluster
#---------------------------------------------------------------------
{
  my $nodes_to_del = $_[0];
  my @upgradedNodes;
  my @notupgradedNodes;

  my @clunodes = split (',', $CFG->params('NODE_NAME_LIST'));

  trace("Performing force upgrade pre checks");

  my $relver = getcrsrelver($CFG->ORA_CRS_HOME);

  my @activever = get_crs_version($CFG->oldconfig('ORA_CRS_HOME'));
  # This checks that the cluster has not already been upgraded
  if (isVersionMatch($relver, join('.', @activever)))
  {
    error("Force upgrade pre-checks: ".
          "The Oracle Grid Infrastructure has been already ".
          "upgraded to [$relver]");
    return FAILED;
  }
    
  my $softver = getcursoftversion($CFG->HOST, $CFG->oldconfig('ORA_CRS_HOME'));
  # This checks that the current node has been upgraded
  if (!(isVersionMatch($relver, $softver)))
  {
    error("Force upgrade pre-checks: ".
          "The upgrade has not yet been performed on this node. ".
          "Re-run rootupgrade.sh without the force option");
    return FAILED;
  } # This checks that the current node has been completely upgraded
  elsif (CKPTSUC ne getCkptStatus("ROOTCRS_STACK"))
  {
    error("Force upgrade pre-checks: ".
          "The upgrade didn't complete on this node. ".
          "Re-run rootupgrade.sh without the force option");
    return FAILED; 
  }

  my %nodeStatus = getNodeStatus($CFG->ORA_CRS_HOME);
  foreach my $clunode (@clunodes)
  {
    if (1 == $nodeStatus{$clunode}) # active node
    {
      push(@upgradedNodes, $clunode);
    }
    else
    {
      $softver = getcursoftversion($clunode);
      # The inactive node list should only contain un-upgraded nodes
      if (!isVersionMatch($relver, $softver))
      {
        push(@notupgradedNodes, $clunode);
      }
      else
      {
        trace("The node $clunode may have been upgraded ".
              "but is inactive right now");
        push(@upgradedNodes, $clunode);
      }
    }
  }

  trace("Cluster nodes : @clunodes");
  trace("Upgraded nodes  : @upgradedNodes");
  trace("Not upgraded nodes: @notupgradedNodes");

  my $forceupg = 1;
  # We must make sure that the force upgrade is applicable to
  # all active nodes 
  foreach my $clunode (@upgradedNodes)
  {
    if ($CFG->HOST =~ /^$clunode$/i) { next; }
    $softver = getcursoftversion($clunode);
    if (!isVersionMatch($relver, $softver))
    {
      error("Force upgrade pre-checks: ".
            "The upgrade has not yet been performed on $clunode. ".
            "Re-run rootupgrade.sh without force option on $clunode");
      $forceupg = 0;
    }
  }

  if (0 == $forceupg)
  {
    return FAILED;
  }

  @force_upgrade_nodes = @upgradedNodes;
  @{$nodes_to_del} = @notupgradedNodes;
  
  return SUCCESS; 
}

# Get CRS active version major number (i.e. 10, 11, etc.) Stack must be up.
sub getCRSMajorVersion {
  my $crsctlbin = catfile ($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
  my $ver 	= 0;
  my @cmd 	= ($crsctlbin, 'query', 'crs', 'activeversion');
  my @out 	= system_cmd_capture(@cmd);
  my $rc  	= shift @out;

  if ($rc == 0) {
     my $verinfo    = getVerInfo($out[0]);
     my @versionarr = split(/\./, $verinfo);
     $ver	    = $versionarr[0];
     trace("crs major version=$ver");
  }
  else {
     error ("@cmd ... failed rc=$rc with message:\n @out \n");
  }

  return $ver;
}

=head2 get_crs_version

  Gets parsed version numbers of active CRS version.
  Version is a result of "crsctl query crs activeversion" command.
  Stack (CRS) must be up for this to succeed.

=head3 Parameters

  string with crsctl home location. If undef, then current home is used.
   
=head3 Returns

=head4 returns an array of version numbers major to minor. 
       If error occurred, all numbers will be 0. Error will be printed.       

=cut

sub get_crs_version 
{
   my $home = $_[0];
   my @ver  = (0, 0, 0, 0, 0);
   my ($crsctl);
   my $verstring;

   if (! defined $home) {
      $crsctl = crs_exec_path('crsctl');
   } else {
      $crsctl = catfile($home, 'bin', 'crsctl' );
   }

   if (isOCRonASM()) {
      trace("setting ORAASM_UPGRADE to 1");
      $ENV{'ORAASM_UPGRADE'} = "1";
   }

   # run "crsctl query crs activeversion" -- stack must be up
   # Example output:
   # Oracle Grid Infrastructure active version on the cluster is [11.2.0.0.2]
   my @cmd = ($crsctl, 'query', 'crs', 'activeversion');
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;

   # if succeeded, parse to ver numbers, output must be a single line,
   # version is 5 numbers, major to minor (see above)
   if ($rc == 0) {
      $verstring = getVerInfo($out[0]);
      trace( "Got CRS active version: ".join('.', $verstring) );
   }
   else {
      trace ("@cmd ... failed rc=$rc with message:\n @out \n");
      trace ("Getting active version from OCR");
      $verstring = get_OldCrsVersionfromOCR();
      trace("old crs active version retrieved from OCR is $verstring");
   }
   @ver = split(/\./, $verstring);
 
   #check version validity
   if (scalar(@ver) != 5) {
      die("Not a valid Grid Infrastructure version. The version obtained is @ver");
   }

   trace( "Got CRS active version: ".join('.', @ver) );
   return @ver;

}

sub get_has_version
{
   my $home = $_[0];
   my @ver  = (10, 1, 0, 0, 0);
   my ($crsctl);

   if (! defined $home) {
      $crsctl = crs_exec_path('crsctl');
   } else {
      $crsctl = catfile($home, 'bin', 'crsctl' );
   }

   # run "crsctl query crs releaseversion" -- local CSS must be up
   # Example output:
   # Oracle Grid Infrastructure  version on the local node is [11.2.0.0.2]
   my @cmd = ($crsctl, 'query', 'has', 'releaseversion');
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;
   my $verstring;

   # if succeeded, parse to ver numbers, output must be a single line,
   # version is 5 numbers, major to minor (see above)
   if ($rc == 0) {
      $verstring = getVerInfo($out[0]);
      @ver          = split(/\./, $verstring);
      trace( "Got High Availability release version: ".join('.', @ver) );
   }
   elsif ($rc != 0) {
       @cmd = ($crsctl, 'query', 'crs', 'releaseversion');
       @out = system_cmd_capture(@cmd);
       $rc  = shift @out;
       if ($rc == 0) {
         $verstring = getVerInfo($out[0]);
         @ver          = split(/\./, $verstring);
         trace( "Got Single Instance CSS release version: ".join('.', @ver));
       }
       elsif ($rc != 0) {
         @cmd = ($crsctl, 'query', 'crs', 'activeversion');
         @out = system_cmd_capture(@cmd);
         $rc  = shift @out;
         if ($rc == 0) {
           $verstring = getVerInfo($out[0]);
           @ver          = split(/\./, $verstring);
           trace( "Got Single Instance CSS release version: ".join('.', @ver));
         }
         else {
           error ("@cmd ... failed rc=$rc with message:\n @out \n");  
         }
      }
   }
   else {
     error ("@cmd ... failed rc=$rc with message:\n @out \n");  
   }

   #check version validity
   if (scalar(@ver) != 5) {
      die("Not a  valid Oracle Restart version. The version obtained is @ver");
   }

   return @ver;
}


sub upgrade_config {
  # On the last node create a OCR backup.
  my $ORA_CRS_HOME    = $CFG->ORA_CRS_HOME;
  my $crsctlbin       = catfile ($ORA_CRS_HOME, "bin", "crsctl");
  my $ocrconfigbin = catfile ($ORA_CRS_HOME, "bin", "ocrconfig");
  my $success = FAILED;
  my $status;
  my $cmd;

  if (getCRSMajorVersion() > 10)
  {
    $cmd = "$ocrconfigbin -manualbackup";
    trace ("Invoking \"$cmd\"");
    my @out = system_cmd_capture("$cmd");
    $status = shift @out;
    if (0 == $status) {
      trace ("$cmd completed  successfully");
    } else {
      error ("OCR manual backup operation failed with $status");
    }
  }

  # Should we proceed with the upgrade if the OCR backup fails?

  # Tell crs subsystem to copy the old resource profiles to new
  # engine. Which does not copy the nodeapps.
  $cmd = "$crsctlbin startupgrade";
  trace ("Invoking \"$cmd\"");
  $status = system_cmd ("$cmd");

  if ($status == 0) { $success = SUCCESS; }

  return $success;
}

sub add_scan 
{
   my $run_as_oracle_owner = FALSE;
   my $status = srvctl($run_as_oracle_owner, "add scan -n $SCAN_NAME");

   if (${status}) {
      trace ("add scan=$SCAN_NAME ... success");
   } else {
      error ("add scan=$SCAN_NAME ... failed");
      return FALSE;
   }

   return TRUE;
}

sub start_scan 
{
   my $run_as_oracle_owner = FALSE;
   my $status = srvctl($run_as_oracle_owner, "start scan");

   if (${status}) {
      trace ("start scan ... success");
   } else {
      error ("start scan ... failed");
      return FALSE;
   }

   return TRUE;
}

sub add_scan_listener 
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "add scan_listener -p $SCAN_PORT -s");

   if (${status}) {
      trace ("add scan listener ... success");
   } else {
      error ("add scan listener ... failed");
      return FALSE;
  }

  return TRUE;
}

sub start_scan_listener 
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "start scan_listener");

   if (${status}) {
      trace ("start scan listener ... success");
   } else {
      error ("start scan listener ... failed");
      return FALSE;
   }

   return TRUE;
}

sub add_ASM 
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "add asm");

   if (${status}) {
      trace ("add asm ... success");
   } else {
      error ("add asm ... failed");
      return FALSE;
   }

   return TRUE;
}

sub isODA
{
  my $status = FALSE;

  my $OAKLIB = catfile("/opt", "oracle", "extapi", "64", "oak", "liboak.2.so");
  if (-f $OAKLIB)
  {
    $status = TRUE;	
    return $status;
  }
  else
  {
    return $status;
  }
}

=head2 configure_ASM_oda

   Creates or updates ASM for ODA

=head3 Parameters

   None

=head3 Returns

  TRUE  - ASM configuration was     created or updated
  FALSE - ASM configuration was not created or updated

=head3 Notes

  This will start ASM as part of the configuration if it is successful

=cut
sub configure_ASM_oda
{
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $status;
  my $ASMDISKS = $CFG->params('ASM_DISKS');
  my $ASM_DISCOVERY_STRING = $CFG->params('ASM_DISCOVERY_STRING');

  trace ("Configuring ASM for ODA");

  my @runodaDg = ("/opt/oracle/oak/onecmd/runodaDg.pl");
  if ($CFG->params('ASM_DISK_GROUP') ){
     my $diskgroup = $CFG->params('ASM_DISK_GROUP');
     if ($diskgroup =~ /\$/) {
        # if diskgroup contains '$', put single-quotes around it
        quoteDiskGroup($diskgroup);
        push @runodaDg, '-d', "'$diskgroup'";
     }
     else {
        push @runodaDg, '-d', $diskgroup;
     }
  }

  # When this is run as superuser
  if ($CFG->params('ASM_DISKS')) {
    push @runodaDg, '-l', "'$ASMDISKS'";
  }

  if ($CFG->params('ASM_REDUNDANCY')) {
    push @runodaDg, '-r', $CFG->params('ASM_REDUNDANCY');
  }

  if ($CFG->params('ASM_DISCOVERY_STRING')) {
    push @runodaDg, '-s', "'$ASM_DISCOVERY_STRING'";
  }
  
   push @runodaDg, '-o', $ORA_CRS_HOME;


  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runodaDg");
  $status       = run_as_user($CFG->params('ORACLE_OWNER'), @runodaDg);

  if ($status != 0) {
     $success = FALSE;
     error ("Configuration of ODA ASM ... failed");
  }
  else
  {
    #Bug - 10417856
    if (check_service("ora.asm", 20)) {
      $success = TRUE;
    }
    else {
      error ("The ora.asm resource is not ONLINE");     
      $success = FALSE;
    }
  }

  return $success;
}

=head2 configure_ASM

   Creates or updates ASM

=head3 Parameters

   None

=head3 Returns

  TRUE  - ASM configuration was     created or updated
  FALSE - ASM configuration was not created or updated

=head3 Notes

  This will start ASM as part of the configuration if it is successful

=cut

sub configure_ASM {
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $status;
  my $ASMDISKS = $CFG->params('ASM_DISKS');
  my $ASM_DISCOVERY_STRING = $CFG->params('ASM_DISCOVERY_STRING');

  trace ("Configuring ASM via ASMCA");

  # Do not change the order of these parameters as asmca requires the
  # parameters to be in a specific order or it will fail
  my @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"), '-silent');
  if ($CFG->params('ASM_DISK_GROUP') ){
     my $diskgroup = $CFG->params('ASM_DISK_GROUP');
     if ($diskgroup =~ /\$/) {
        # if diskgroup contains '$', put single-quotes around it
        quoteDiskGroup($diskgroup);
        push @runasmca, '-diskGroupName', "'$diskgroup'";
     }
     else {
        push @runasmca, '-diskGroupName', $diskgroup;
     }
  }

  # When this is run as superuser
  if ($CFG->params('ASM_DISKS')) {
    push @runasmca, '-diskList', "'$ASMDISKS'";
  }

  if ($CFG->params('ASM_REDUNDANCY')) {
    push @runasmca, '-redundancy', $CFG->params('ASM_REDUNDANCY');
  }

  if ($CFG->params('ASM_DISCOVERY_STRING')) {
    push @runasmca, '-diskString', "'$ASM_DISCOVERY_STRING'";
  }

  if (isFirstNodeToStart()) {
    push (@runasmca, ('-configureLocalASM'));
  }

  if (isReusedg()) {
     push (@runasmca, ('-reuseDiskGROUP'));
  }

  if ($CFG->params('ASM_AU_SIZE')) {
    push @runasmca, '-au_size', $CFG->params('ASM_AU_SIZE');
  }

  if ($CFG->defined_param('ORATAB_LOC')) {
      push (@runasmca, ('-oratabLocation'), $CFG->params('ORATAB_LOC'));
  }

  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca");
  $status       = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca);
  my $asmca_log = catdir($CFG->params('ORACLE_BASE'), 'cfgtoollogs', 'asmca');

  if ($status != 0) {
     $success = FALSE;
     error ("Configuration of ASM ... failed");
     error ("see asmca logs at $asmca_log for details");
  }
  else
  {
    #Bug - 10417856
    if (check_service("ora.asm", 20)) {
      $success = TRUE;
    }
    else {
      error ("The ora.asm resource is not ONLINE");     
      $success = FALSE;
    }
  }

  return $success;
}

=head2 configure_OCR

   Creates or updates OCR

=head3 Parameters

   None

=head3 Returns

  TRUE  - OCR configuration was     created or updated
  FALSE - OCR configuration was not created or updated

=cut

sub configure_OCR {
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $OCRCONFIGBIN = catfile ($ORA_CRS_HOME, "bin", "ocrconfig");
  my $status;
  my $lang_id = $CFG->params('LANGUAGE_ID');
  my $asmgrp       = $CFG->params('ORA_ASM_GROUP');
  my $IS_SIHA = $CFG->IS_SIHA;
  my $host = $CFG->HOST;

  # In case of cluster, 
  # Format/upgrade OCR using 'ocrconfig -upgrade ..' 
  # Populate keys using 'clscfg -install ...'
  if (!$IS_SIHA)
  {
    my @runocrconfig = ("$OCRCONFIGBIN", "-upgrade",
			$CFG->params('ORACLE_OWNER'),
			$CFG->params('ORA_DBA_GROUP'));

    my $CLSCFGBIN = catfile ($ORA_CRS_HOME, "bin", "clscfg");
    my @runclscfg = ("$CLSCFGBIN", "-install",
		     "-h", $CFG->params('HOST_NAME_LIST'),
		     '-o', $ORA_CRS_HOME, 
		     '-g', $asmgrp);

    if ($CFG->CLSCFG_EXTRA_PARMS) {
      push @runclscfg, @{$CFG->CLSCFG_EXTRA_PARMS};
    }

    # abort if OCR is already configured
    if (!$isRerun && checkOCR($ORA_CRS_HOME))
    {
      trace("Existing OCR configuration found, aborting the configuration. Rerun configuration setup after deinstall");
      die "Existing OCR configuration found, aborting the configuration. Rerun configuration setup after deinstall";
    }

    trace ("Creating or upgrading OCR keys");
    $status = system_cmd("@runocrconfig");
    if (0 != $status) {
      error ("Failed to create Oracle Cluster Registry configuration,",
	     "rc $status");
      $success = FALSE;
    }
    else {
      trace ("OCR keys are successfully populated");

      if (!s_reset_srvconfig()) {
	error("Reset of OCR location in srvconfig failed");
	$success = FALSE;
      }
      else {
	#
	# clscfg - Initialize the Oracle Cluster Registry for the
	#          cluster. Should be done once per cluster install.
	#          Overwriting a configuration while any CRS daemon is
	#          running can cause serious issues.
	#
	trace("Executing clscfg");
	$status = system_cmd("@runclscfg");
	
	# Get true return value of clscfg (i.e. rc for spawned
	# process)
        if ((0 == $status) || ($status == 105 && (isReusedg() || $isRerun))) {
           trace ("Oracle Cluster Registry initialization completed");

          if ($CFG->CLSCFG_POST_CMD) {
            my @cmd = @{$CFG->CLSCFG_POST_CMD};
            system_cmd(@cmd);
          }
	} 
	else {	     
              error("Failed to initialize Oracle Cluster Registry for cluster,",
                "rc $status"); 
              $success = FALSE;
	  
	}
      }
    }
  }
  # In case of SIHA, 
  # Format OCR using 'clscfg -local'
  # Pin the node using 'crsctl pin css ...'
  else {
    # Create necessary configuration with 'clscfg -local'. Needed for 
    # compatibility with older DBs (10.x, 11.1). 
    my $clscfg = catfile ($ORA_CRS_HOME, "bin", "clscfg");
    my $cmdclscfg = "$clscfg -local -g $asmgrp";
    my $rc = system_cmd ($cmdclscfg);
    if ($rc != 0) {
       error("Creating local-only OCR ($cmdclscfg) ... failed");
       exit;
    } else {
       trace ("Creating local-only OCR ($cmdclscfg) ... succeeded");
    }

    my $crsctl = catfile ($ORA_CRS_HOME, "bin", "crsctl");
    my $cmdcrsctl = "$crsctl pin css -n $host";
    $rc = system_cmd ($cmdcrsctl);
    if ($rc != 0) {
       error("Pin node ($cmdcrsctl) ... failed");
       exit;
    } else {
       trace ("Pin node ($cmdcrsctl) ... succeeded");
    }
  }

  return $success;
}

# Execute a system command and analyze the return codes
sub system_cmd {
  my $rc = 0;

  trace("Executing @_");
  my @output = system_cmd_capture("@_");

  $rc = shift @output;

  for my $line (@output) { print "$line\n"; }

  return $rc;
}

=head2 system_cmd_capture1

  Capture the output from a system command and analyze the return codes

=head3 Parameters

   trace flag to control error output and Command to be executed

=head3 Returns

  Array containing both the return code and the captured output 
  The command output is chomped

=head3 Usage

  To capture the data of command foo:
    my @out = system_cmd_capture('foo')
    my $rc = shift @out;

  The @out now contains only the output of the command 'foo'

=cut

sub system_cmd_capture1 {
  my $trcflag = shift;

  if ($trcflag) { return system_cmd_capture(@_); }
  else { return system_cmd_capture_noprint(@_); }
}

#Capture the output from a system command
sub system_cmd_capture {
  my @output = system_cmd_capture_noprint(@_);
  my $rc     = shift @output;

  if (scalar(@output) > 0) {
    trace(join("\n>  ", ("Command output:", @output)),
          "\n>End Command output");
  }

  return ($rc, @output)
}



=head2 system_cmd_capture_noprint

  Capture the output from a system command and analyze the return codes

=head3 Parameters

   Command to be executed

=head3 Returns

  Array containing both the return code and the captured output 
  The command output is chomped

=head3 Usage

  To capture the data of command foo:
    my @out = system_cmd_capture('foo')
    my $rc = shift @out;

  The @out now contains only the output of the command 'foo'

=cut

sub system_cmd_capture_noprint {
  my $rc  = 0;
  my $prc = 0;
  my @output;

  trace("Executing cmd: @_");

  if (!open(CMD, "@_ 2>&1 |")) { $rc = -1; }
  else {
    @output = (<CMD>);
    close CMD;
    # the code return must be after the close
    $prc = $CHILD_ERROR >> 8; # get program return code right away
    chomp(@output);
  }

  if ($prc != 0) {
    # program returned error code
    # error("Command return code of $prc from command: @_");
    $rc = $prc;
  }
  elsif ($rc < 0 || ($rc = $CHILD_ERROR) < 0)  {
    # It is almost certainly wrong to keep this output from going
    # directly to the trace file, which will occur if not careful,
    # because it will be difficult to determine the cause of serious
    # problems, but this is by request
    push @output,
    "Failure in execution (rc=$rc, $CHILD_ERROR, $!) for command @_";
  }
  elsif ($rc & 127) {
    # program returned error code
    my $sig = $rc & 127;
    # It is almost certainly wrong to keep this output from going
    # directly to the trace file, which will occur if not careful,
    # because it will be difficult to determine the cause of serious
    # problems, but this is by request
    push @output,
    "Failure with signal $sig from command: @_";
  }
  elsif ($rc)  { trace("Failure with return code $rc from command @_"); }

  return ($rc, @output);
}

sub ExtractDiskgroups
#---------------------------------------------------------------------
# Function: Extract diskgrops
# Args    : none
#---------------------------------------------------------------------
{
   # Check if CRS is up
   my $crsctl    = catfile ($ORA_CRS_HOME, "bin", "crsctl");
   my $cluster_is_up = check_service ("cluster", 20);
   my $crs_is_up = check_service ("ohasd", 10);
   my $start_exclusive = FALSE;
   my @dglist;

   if (!$crs_is_up) {
     trace("OHASD is not up. So starting CRS exclusive");
     start_service("crsexcl"); 
     $crs_is_up = TRUE;
   }
   else {
     trace("OHASD is already up.");

     if (!$cluster_is_up) {
       trace("Starting CSS exclusive");
       $start_exclusive = TRUE;
       my $css_rc = CSS_start_exclusive();
       if ($css_rc != CSS_EXCL_SUCCESS) {
         $start_exclusive = FALSE;
         trace ("CSS failed to enter exclusive mode to extract votedisk");
       }
     }
   }

   if (($crs_is_up) || ($start_exclusive)) {
      trace("Querying CSS vote disks for getting DGs");
      open (QUERY_VOTEDISK, "$crsctl query css votedisk|");
      my @css_votedisk = (<QUERY_VOTEDISK>);
      chomp @css_votedisk;
      close QUERY_VOTEDISK;

      if ($start_exclusive) {
         CSS_stop();
      }

      foreach my $votedisk (@css_votedisk) {
         trace("Voting disk is : $votedisk");
         if ($votedisk =~ / \[(.+)\]/) {
            # $votedisk contains '1.  2 282bf2a833f54f02bf4befd002fa90d6
            # (/dev/raw/raw1) [OCRDG]'
            push (@dglist, $1);
         }
      }
   }

   trace ("Diskgroups found: @dglist");
   return @dglist;
}

sub ExtractVotedisks
#---------------------------------------------------------------------
# Function: Extract Voting disks
#
# Args    : none
#---------------------------------------------------------------------
{
   # Check if CRS is up
   my $crsctl    = catfile ($ORA_CRS_HOME, "bin", "crsctl");
   my $cluster_is_up = check_service ("cluster", 20);
   my $crs_is_up = check_service ("ohasd", 10);
   my $start_exclusive = FALSE;
   my @votedisk_list;

   if (!$crs_is_up) {
     trace("OHASD is not up. So starting CRS exclusive");
     start_service("crsexcl"); 
     $crs_is_up = TRUE;
   }
   else {
     trace("OHASD is already up.");

     if (!$cluster_is_up) {
       trace("Starting CSS exclusive");
       $start_exclusive = TRUE;
       my $css_rc = CSS_start_exclusive();
       if ($css_rc != CSS_EXCL_SUCCESS) {
         $start_exclusive = FALSE;
         trace ("CSS failed to enter exclusive mode to extract votedisk");
       }
     }
   }

   if (($crs_is_up) || ($start_exclusive)) {
      trace("Querying CSS vote disks");
      open (QUERY_VOTEDISK, "$crsctl query css votedisk|");
      my @css_votedisk = (<QUERY_VOTEDISK>);
      chomp @css_votedisk;
      close QUERY_VOTEDISK;

      if ($start_exclusive) {
         CSS_stop();
      }

      foreach my $votedisk (@css_votedisk) {
         trace("Voting disk is : $votedisk");
         # get line contains ' (/'
         if ($votedisk =~ / \(/) {
            # $votedisk contains '1.  2 282bf2a833f54f02bf4befd002fa90d6
            # (/dev/raw/raw1) [OCRDG]'
            # parse $votedisk to get '/dev/raw/raw1'
            my $vdisk;
            my ($dummy, $text) = split (/\(/, $votedisk);
            ($vdisk, $dummy) = split (/\)/, $text);
            push (@votedisk_list, $vdisk);
         }
      }
   }

   trace ("Vote disks found: @votedisk_list");
   return @votedisk_list;
}

sub add_J2EEContainer
#---------------------------------------------------------------------
# Function: Add the OC4J Container
# Args    : none
#---------------------------------------------------------------------
{
   my $oc4j_status   = FAILED;
   my $run_as_oracle = TRUE;
   my $add_oc4j      = 'add oc4j';

   # Add the OC4J resource
   my $status = srvctl($run_as_oracle, $add_oc4j);

   if (${status}) {
      trace ("J2EE (OC4J) Container Resource Add ... passed ...");
      $oc4j_status = addWallet_J2EEContainer(); 
   } else {
      trace ("J2EE (OC4J) Container Resource Add ... failed ...");
   }

   return $oc4j_status;
}

sub addWallet_J2EEContainer
#---------------------------------------------------------------------
# Function: Add the wallet for OC4J Container
# Args    : none
#---------------------------------------------------------------------
{
   # Status variables
   my $oc4j_addwallet_status = FAILED;
   my $wallet_exist          = FAILED;
   my $wallet_not_exist      = FAILED;
   # Verify the existence of the wallet
   my $user   = $CFG->params('ORACLE_OWNER');
   my $crsctl = crs_exec_path('crsctl');
   my @cmd    = ($crsctl, 'query', 'wallet', '-type',' APPQOSADMIN');
   my @out    = ();
   my $rc     = run_as_user2($user, \@out, @cmd);
   # search in the out if the wallet not exists in output query   
   foreach(@out){
      if ($_ =~ m/CRS-4030/i) {
         # This code means the wallet exists
         $wallet_exist = SUCCESS;
         last;
      } elsif ($_ =~ m/CRS-4031/i) {
         # This code means the wallet not exists
         $wallet_not_exist = SUCCESS;
         last;
      }
   }
   
   if ($wallet_not_exist == SUCCESS) {
      # The wallet does not exist, we must now add the wallet
      my @cmd2 = ($crsctl, 'add', 'wallet', '-type',' APPQOSADMIN');
      my @out2 = ();
      my $rc2  = run_as_user2($user, \@out2, @cmd2);

      if ($rc2 == 0) {
         trace ("J2EE (OC4J) Container Resource Add Wallet ... passed ...");
         $oc4j_addwallet_status = addQosUsers();
      } else {
         trace ("J2EE (OC4J) Container Resource Add Wallet ... failed ...");
         error ("crsctl failed, the messages are @out2");  
      }
   } elsif ($wallet_exist == SUCCESS) {
      # the wallet exists
      trace ("J2EE (OC4J) Container Resource Add Wallet ... already exists ...");
      $oc4j_addwallet_status = addQosUsers();
   } else {
      trace ("J2EE (OC4J) Container Resource Query Wallet ... failed ...");
      error ("crsctl failed, the messages are @out");
   }

   return $oc4j_addwallet_status;
}

sub addQosUsers
#------------------------------------------------------------------------------
# Function: Adds the QoS users (oc4jadmin and qosadmin).
# Args    : none.
# Returns : SUCCESS in case the users are added succesfully (or already exist).
#           FAILED in case of any error.
#------------------------------------------------------------------------------
{
   my $rc = FAILED;
   my $user = $CFG->params('ORACLE_OWNER');
   my @cmd = (s_get_qosctl_path(), '-autogenerate');
   my @out = ();
   my $rf = run_as_user2($user, \@out, @cmd);

   if ( $rf == 0 ) {
      trace("qosctl output: " . join("\n", @out));
      if ( verifyQosUser("oc4jadmin") == SUCCESS &&
           verifyQosUser("qosadmin") == SUCCESS ) {
         $rc = SUCCESS;
      }
   } else {
      trace(join(">  ", ("Error encountered in the command @cmd \n",
            @out, "End Command output")));
   }
   return $rc;
}

sub verifyQosUser
#------------------------------------------------------------------------------
# Function: Verifies existence of a QoS users.
# Args    : [0] user name to be verified.
# Returns : SUCCESS in case the user exists, FAILED otherwise
#------------------------------------------------------------------------------
{
   my $rc = FAILED;
   my $user = $CFG->params('ORACLE_OWNER');
   my $userName = $_[0];
   my @cmd = (crs_exec_path('crsctl'), 'query', 'wallet', '-type',
              'APPQOSADMIN',  '-user', $userName);
   my @out = ();
   my $rf = run_as_user2($user, \@out, @cmd);

   if (scalar(grep(m/CRS-4028/i, @out)) > 0) {
      $rc = SUCCESS;
   } else {
      trace(join(">  ", ("Error encountered in the command @cmd \n",
            @out, "End Command output")));
      trace("Verify QoS user $userName failed");
   }
   return $rc;
}

sub start_J2EEContainer
#---------------------------------------------------------------------
# Function: Start the OC4J Container
# Args    : none
#---------------------------------------------------------------------
{
   my $oc4j_status = FAILED;
   
   # If J2EE Container is to be enabled, start the OC4J resource
   if (get_J2EEContainerToBeEnabled()) {
      # Enable oc4j resource
      $oc4j_status = enable_J2EEContainer();
      if ($oc4j_status == SUCCESS) {
         # Start oc4j resource
         my $user = $CFG->params('ORACLE_OWNER');
         my $srvctl = crs_exec_path('srvctl');
         my @cmd = ($srvctl, 'start', 'oc4j');
         my @out  = ();
         my $status = run_as_user2($user, \@out, @cmd);
      
         #We have success with these conditions:
         # - ($status == 0) : OC4J started 
         # - PRCC-1014 : OC4J is already running
         if (($status == 0) || (scalar(grep(m/PRCC-1014/i, @out)) > 0)) {
            $oc4j_status = SUCCESS;
            trace ("J2EE (OC4J) Container Resource Start ... passed ...");
         } else {
            $oc4j_status = FAILED;
            trace(join(">  ", ("Error encountered in the command @cmd \n", 
                  @out, "End Command output")));
            trace("J2EE (OC4J) Container Resource Start ... failed ...");
         }
      }
   }
   else {
      # If J2EE Container is not to be enabled, disable it
      $oc4j_status = disable_J2EEContainer();
   }
   
   return $oc4j_status;
}

sub stop_J2EEContainer
#---------------------------------------------------------------------
# Function: Stop the OC4J Container
# Args    : none
#---------------------------------------------------------------------
{  
   my $oc4j_status = FAILED;

   # Stop oc4j resource
   my $user = $CFG->params('ORACLE_OWNER');
   my $srvctl = crs_exec_path('srvctl');
   my @cmd = ($srvctl, 'stop', 'oc4j');
   my @out  = ();
   my $status = run_as_user2($user, \@out, @cmd);
      
   #We have success with these conditions:
   # - ($status == 0) : OC4J stopped
   # - PRCC-1016 : OC4J is already stopped
   if (($status == 0) || (scalar(grep(m/PRCC-1016/i, @out)) > 0)) {
      $oc4j_status = SUCCESS;
      trace ("J2EE (OC4J) Container Resource Stop ... passed ...");
   } else {
      $oc4j_status = FAILED;
      trace(join(">  ", ("Error encountered in the command @cmd \n",
            @out, "End Command output")));
      trace("J2EE (OC4J) Container Resource Stop ... failed ...");
   }

   return $oc4j_status;
}

sub enable_J2EEContainer
#---------------------------------------------------------------------
# Function: Enable OC4J Container
# Args    : none
#---------------------------------------------------------------------
{
   my $oc4j_status   = FAILED;

   # Enable OC4J resource during upgrade
   # - From 11.2.0.1 or 11.2.0.2.x to 11.2.0.3
   # - From 11.2.0.3.x to 11.2.0.4
   my $user = $CFG->params('ORACLE_OWNER');
   my @cmd = (catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl'), 'enable', 'oc4j');
   my @out = ();
   my $rc = run_as_user2($user, \@out, @cmd);

   # We have success with these conditions:
   # - ($rc == 0) : OC4J was successfully enabled
   # - PRKO-2116  : OC4J is already enabled
   if ($rc == 0 || scalar(grep(m/PRKO-2116/i, @out)) > 0) {
      $oc4j_status = SUCCESS;
      trace ("J2EE (OC4J) Container Resource enable ... passed");
   }
   else {
      trace ("J2EE (OC4J) Container Resource enable ... failed");
   }

   return $oc4j_status;
}

sub disable_J2EEContainer
#---------------------------------------------------------------------
# Function: Disable OC4J Container
# Args    : none
#---------------------------------------------------------------------
{
   my $oc4j_status   = FAILED;
   #  Disable OC4J resource.
   my $srvctl = crs_exec_path('srvctl');
   my $user = $CFG->params('ORACLE_OWNER');
   my @cmd  = ($srvctl, 'disable', 'oc4j');
   my @out  = ();
   my $rf   = run_as_user2($user, \@out, @cmd);

   # We have success with these conditions:
   # - ($rf == 0) : This means the OC4J was successfully disabled.
   # - PRKO-2115 : OC4J is already disabled
   if (($rf == 0) || (scalar(grep(m/PRKO-2115/i, @out)) > 0)) {
      $oc4j_status = SUCCESS;
      trace("J2EE (OC4J) Container Resource Disable ... passed");
   } else {
      trace(join(">  ", ("Error encountered in the command @cmd \n",
            @out, "End Command output")));
      trace("J2EE (OC4J) Container Resource Disable ... failed ...");
   }

   return $oc4j_status;
}


sub configureAllRemoteNodes
#---------------------------------------------------------------------
# Function: Configure all remote nodes for Windows
# Args    : none
#---------------------------------------------------------------------
{
   if (($CFG->platform_family eq "windows") &&
       (! $REMOTENODE) && (! $CFG->addnode))
   {
      return s_configureAllRemoteNodes();
   }

   return SUCCESS;
}

#For 10.1, get the oracle home location where the VIP
#resources are configured.
sub get101viphome
{
  my $host    = $CFG->HOST;
  my $ocrdump = catfile ($ORACLE_HOME, 'bin', 'ocrdump');

  # get ons.ACTION_SCRIPT keyname
  if ($CFG->platform_family eq "windows") {
     open (OCRDUMP, "$ocrdump -stdout -keyname " .
           "CRS.CUR.ora!$host!ons.ACTION_SCRIPT |");
  }
  else {
     open (OCRDUMP, "$ocrdump -stdout -keyname " .
           "'CRS.CUR.ora!$host!ons.ACTION_SCRIPT' |");
  }

  my @output = <OCRDUMP>;
  close (OCRDUMP);

  if ($DEBUG) { trace("get101viphome:ocrdump = @output"); }

  # get vip home
  my @txt = grep (/ORATEXT/, @output);
  my ($key, $vip_home) = split (/: /, $txt[0]);
  $vip_home =~ s!/bin/racgwrap!!g;
  $vip_home =~ s/^ //g;
  chomp($vip_home);

  if ($DEBUG) { trace("get101viphome:ons home = $vip_home"); }

  return $vip_home;
}

# gets the VIp information from the OCR. CRS stack needs to be up
# before calling this sub routine.
sub get_OldVipInfo
{
  my @CRS_NODEVIP_LIST;
  my $vip_index     = 0;
  my $OLD_CRS_HOME  = $CFG->OLD_CRS_HOME;
  my @SNODES        = split (/,/, $NODE_NAME_LIST);
  my $ORACLE_OWNER  = $CFG->params('ORACLE_OWNER');
  my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP');
  my ($srvctlbin, $vip_name, $ip, $old_netmask, $intif);
  my ($new_netmask, $vip);

  if (($CFG->force) && scalar(@force_upgrade_nodes) > 0)
  {
    trace("Force upgrade nodes: @force_upgrade_nodes");
    @SNODES = @force_upgrade_nodes;
  }

  # if version is 10.1, use dbhome. Otherwise, use OLD_CRS_HOME.
  my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};

  if ($old_version[0] eq "10" &&
      $old_version[1] eq "1") {

     my $ons_home = get101viphome();
     $srvctlbin = catfile ($ons_home, 'bin', 'srvctl');
     $ENV{'ORACLE_HOME'}  = $ons_home;
  } else {
     $srvctlbin = catfile ($OLD_CRS_HOME, 'bin', 'srvctl');
  }

  foreach my $nodename (@SNODES) {
     if ($old_version[0] > '10' && $old_version[1] > '1') {
        open(SRVCMDF, "$srvctlbin config nodeapps -n $nodename -S 1|")
             || die "Could not get existing VIP information\n";
        my @buffer = grep(/ip=/, <SRVCMDF>);
        close SRVCMDF;

     if ($DEBUG) { trace("output(srvctl config nodeapps -S): @buffer"); }

        if (scalar(@buffer) > 0) {
           trace ("buffer=$buffer[0]");
           my @list = split(/ +/, $buffer[0]);
           foreach (@list) {
              if ($_ =~ /name=/) {
                 $vip_name = parseText ($_);
              }
              elsif ($_ =~ /ip=/) {
                 $ip = parseText ($_);
              }
              elsif ($_ =~ /netmask=/) {
                 $old_netmask = parseText ($_);
              }
              elsif ($_ =~ /interfaces=/) {
                 $intif = parseText ($_);
              }
           }

       }  else {
    trace("Could not find IP details from running the command 'srvctl config nodeapps -n $nodename -S 1");
      }
     }
     else {
        open(SRVCMDF, "$srvctlbin config nodeapps -n $nodename -a|")
             || die "Could not get existing VIP information\n";
        my @buffer = <SRVCMDF>;
        close SRVCMDF;

    if ($DEBUG) { trace("output(srvctl config nodeapps): @buffer"); }

        my $VipValue = $buffer[0];
        chomp($VipValue);
        trace("VIpValue =  $VipValue");
        my ($Name, $Value) = split(/:/, $VipValue, 2);
        my $val1;
        ($val1, $vip_name, $ip, $old_netmask, $intif)
                           = split(/\//, $Value);
     }

     chomp $vip_name;
     chomp $ip;
     chomp $old_netmask;
     chomp $intif;

     $intif =~ s/\:/\|/g;

     trace("Older VIP IP address (vip_name) = $vip_name");
     trace("Older VIP IP name(ip) = $ip");
     trace("Older VIP IP netmask (old_netmask) = $old_netmask");
     trace("Older VIP IP interface (intif) =$intif");

     # use vip_name if it exists, otherwise use ip
     if (! $vip_name) {
        $vip = $ip;
     }
     else {
        $vip = $vip_name;
     }

     if (validateNetmask($old_netmask, $intif, \$new_netmask)) {
        if ($intif) {
           $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask/$intif";
        } 
        else {
           $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask";
        } 
     } 
     else {
        if ($intif) {
           $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask/$intif";
        } 
        else {
           $CRS_NODEVIP_LIST[$vip_index] = "$vip/$old_netmask";
        }
     }

     trace ("vip on $nodename = $CRS_NODEVIP_LIST[$vip_index]");
     $vip_index++;
  }

   sub parseText {
      # extract netmask and interface
      my $line = $_[0];
      $line =~ s/{//g;
      $line =~ s/}//g;
      my ($dummy, $text) = split (/=/, $line);
      chomp $text;

      return $text;
   }

  $ENV{'ORACLE_HOME'} = $ORACLE_HOME;
  return @CRS_NODEVIP_LIST;
}

sub get_OldVipInfoFromOCRDump
#-------------------------------------------------------------------------------
# Function:  Get old VIP info from ocrdump
# Args    :  none
# Returns :  @vip_list
#-------------------------------------------------------------------------------
{
   trace("Getting old VIP details from ocrdump");
   my $ocrdump  = catfile ($CFG->params('ORACLE_HOME'), 'bin', 'ocrdump');
   my @nodes    = split (/,/, $CFG->params('NODE_NAME_LIST'));
   my $ix       = 0;
   my @vip_list;

   if (($CFG->force) && scalar(@force_upgrade_nodes) > 0)
   {
     trace("Force upgrade nodes: @force_upgrade_nodes");
     @nodes = @force_upgrade_nodes;
   }

   foreach my $nodename (@nodes) {
      $nodename = lc($nodename);
      # get IP from DATABASE.NODEAPPS.$nodename.VIP.IP
      if ($CFG->platform_family eq "windows") {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "DATABASE.NODEAPPS.$nodename.VIP.IP |");
      }
      else {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "'DATABASE.NODEAPPS.$nodename.VIP.IP' |");
      }

      my @output = <OCRDUMP>;
      close (OCRDUMP);

      trace("Dump from OCR: @output");

      my @txt = grep (/ORATEXT/, @output);
      my ($key, $ip) = split (/: /, $txt[0]);
      chomp($ip);
      trace("IP address (found from dump) = $ip");

      # get NETMASK from DATABASE.NODEAPPS.$nodename.VIP.NETMASK
      if ($CFG->platform_family eq "windows") {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "DATABASE.NODEAPPS.$nodename.VIP.NETMASK |");
      }
      else {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "'DATABASE.NODEAPPS.$nodename.VIP.NETMASK' |");
      }

      @output = <OCRDUMP>;
      close (OCRDUMP);

      trace("Dump from OCR: @output");

      @txt = grep (/ORATEXT/, @output);
      my $old_netmask;
      ($key, $old_netmask) = split (/: /, $txt[0]);
      chomp($old_netmask);

      trace("IP netmask (found from dump) = $old_netmask");

      # get network interface name
      if ($CFG->platform_family eq "windows") {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "CRS.CUR.ora!$nodename!vip.USR_ORA_IF |");
      }
      else {
         open (OCRDUMP, "$ocrdump -stdout -keyname " .
               "'CRS.CUR.ora!$nodename!vip.USR_ORA_IF' |");
      }

      @output = <OCRDUMP>;
      close (OCRDUMP);

      trace("Dump from OCR: @output");

      @txt = grep (/ORATEXT/, @output);
      my $intif;
      ($key, $intif) = split (/: /, $txt[0]);
      my $new_netmask;
      chomp($intif);
      trace("Network interface (found from dump)=$intif");

      if ($ip ne "" && $old_netmask ne "") {
         if (validateNetmask($old_netmask, $intif, \$new_netmask)) {
            if ($intif) {
               $vip_list[$ix] = "$ip/$new_netmask/$intif";
            }
            else {
               $vip_list[$ix] = "$ip/$new_netmask";
            }
         }
         else {
            if ($intif) {
               $vip_list[$ix] = "$ip/$new_netmask/$intif";
            }
            else {
               $vip_list[$ix] = "$ip/$old_netmask";
            }
         }

         $ix++;
      }
   }

   if ($DEBUG) { trace("vip_list = @vip_list"); }

   return @vip_list;
}

#Gets the OLD Grid Infrastructure ONS port information
sub get_ons_port
{
#---------------------------------------------------------------------
# Function: Get the ONS port used by the old version crs
#---------------------------------------------------------------------
   my $node = $_[0];
   my $home;
   my $ONSCONFFILE;
   my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
   my @buf2;
   my $Name;
   my $portnum;
   my $useocr;
   my $localport;
   my $remoteport;
   my $locport;
   my $remport;
   my $cmd;
   my $ocrkey;
   my $line;
   my $idx = 0;
 
   # if version is 10.1, use dbhome where ons is configured. Otherwise, use OLD_CRS_HOME.
   my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if ($old_version[0] eq "10" &&
      $old_version[1] eq "1") {
      $home = get101viphome();
   }
   else
   {
      $home = $CFG->OLD_CRS_HOME;
   }
 
   $ONSCONFFILE = catfile( $home, 'opmn' , 'conf', 'ons.config');
   trace("The ons conf file location is: $ONSCONFFILE");
   open(FONS, $ONSCONFFILE) or
        trace("Could not  open \"$ONSCONFFILE\": $!");

   while(<FONS>) {
     if(/^useocr\=on\b/i) { $useocr=$_;  }
     if(/^remoteport\b/i) { $remoteport=$_;  }
     if(/^localport\b/i)  { $localport=$_;  }
   }

   close (FONS);
   #get the remote port
   if (defined($useocr))
   {
      trace("useocr is on. get the remote port from OCR");
      $cmd = catfile( $ORA_CRS_HOME, 'bin', 'ocrdump' );
      $ocrkey = "DATABASE.ONS_HOSTS";

      my @args = ($cmd, '-stdout', '-keyname', $ocrkey);
      my @out = system_cmd_capture(@args);

      if ($DEBUG) { trace("ocrdump output: @out"); }

      my $rc  = shift @out;
      foreach $line (@out)
      {
       if($line =~ m/DATABASE\.ONS_HOSTS\.$node.*\.PORT\]/i)
       {
          @buf2 = $out[$idx+1];
          last;
       }
       $idx++;
      }

      if ($DEBUG) { trace("ocrdump output for port information is: @buf2"); }

      if (scalar(@buf2) != 0) {
        ($Name, $portnum) = split(/:/, $buf2[0]);
      }
         $remport = trim($portnum);
   }
   else 
   {
    if (defined($remoteport)) {
        ($Name, $portnum) = split(/=/, $remoteport);
        $remport = trim($portnum);
    }
   }

   #always get the localport from ons.config if present
   if (defined($localport)) {
     ($Name, $portnum) = split(/=/, $localport);
     $locport = trim($portnum);
   }

   #set 11.2 default values for ons port
   if (! $locport) {
       trace("setting default port  for ons localport");
       $locport = "6100";
   }
   if (! $remport) {
       trace("setting default port  for ons remoteport");
       $remport = "6200";
   }

   trace("Local port=$locport");
   trace("Remote port=$remport");

   return ($locport, $remport);
}



sub getONSnodelist()
#---------------------------------------------------------------------
# Function: Get the list of nodes where ONS is configured. The nodes could
# be outside of the cluster nodes as well(Bug 8563905).
#---------------------------------------------------------------------
{

   my $cmd;
   my @nodelist;
   my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME; 
   my $ONSCONFFILE;
   my $line;
   my $e1;
   my $e2;
   my $e3;
   my $e4;
   my $home;
   my $onsnodes;
   my $node;
   my $port;
   my $Name;
   my $list;
   my @tmparr;
   my @nodelistocr;
   my @nodelistons;
   my %union = ();
   my $e;


   #Get the onsnodelist from ons.config file

   # if version is 10.1, use dbhome where ons is configured. Otherwise, use OLD_CRS_HOME.
   my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if ($old_version[0] eq "10" &&
      $old_version[1] eq "1") {
      $home = get101viphome();
   }
   else
   {
      $home = $CFG->OLD_CRS_HOME;
   }

   $ONSCONFFILE = catfile( $home, 'opmn' , 'conf', 'ons.config');
   trace("ons conf file is $ONSCONFFILE");
   open(FONS, $ONSCONFFILE) or
        trace("Could not  open \"$ONSCONFFILE\": $!");

   while(<FONS>) {
      if(/^nodes\b/i) { $onsnodes=$_;  }
   }
   close (FONS);        

   ($Name, $list) = split(/=/, $onsnodes);
   @tmparr = split(/,/, $list);
   foreach my $elem (@tmparr)
   {
      ($node, $port) = split(/:/, $elem);
      push @nodelistons, $node;
   }

   trace("ons node list from ons.config is @nodelistons\n");

   #Get the ons nodelist from OCR

   $cmd = catfile( $ORA_CRS_HOME, 'bin', 'ocrdump' );
   my $ocrkey = "DATABASE.ONS_HOSTS";
   my @args = ($cmd, '-stdout', '-keyname', $ocrkey);
   my @out = system_cmd_capture(@args);
   my $rc  = shift @out;

   foreach $line (@out)
   {
      if($line =~ m/PORT\]/i)
      {
          ($e1, $e2, $e3, $e4) = split(/\./, $line);
          trace("ONS is configured on node $e3. Adding to onsnodelist");
          $e3 =~ s/!/./g;
          push @nodelistocr, $e3;
      }
   }

   trace("ons nodelist from OCR is @nodelistocr\n");

   foreach $e (@nodelistons) {$union{$e} = 1 }
   foreach $e (@nodelistocr) {$union{$e} = 1 }
   @nodelist = keys %union;

   trace("union of on nodelist from ons.config and OCR is @nodelist\n");

   return @nodelist;

}

#update ons.config - Bug 8424681
 
sub update_ons_config
{
   my @nodelist = getONSnodelist();
   my $node;
   my $host = $CFG->HOST;
   my $str = "nodes="; 
   my $ONSCONFFILE = catfile( $ORACLE_HOME, 'opmn' , 'conf', 'ons.config');

   foreach $node (@nodelist)
   {
    my($onslocport, $onsremport) = get_ons_port($node);
    trace ("ons remoteport for $node is $onsremport");
    if ($node ne $nodelist[-1])
    {
       $str = $str . "$node:$onsremport" . ",";
    }
    else
    {
       $str = $str . "$node:$onsremport";
    }
   }

   trace("ons nodes string is $str");

   my($onslocport, $onsremport) = get_ons_port($host);
   trace("ons conf file is $ONSCONFFILE");
   open(FONS, ">>$ONSCONFFILE") or
        error ("Could not  open \"$ONSCONFFILE\": $!");
   print FONS "localport=$onslocport\n";
   print FONS "remoteport=$onsremport\n";
   print FONS "$str\n";
   close FONS;
}

# Stops the old running crs stack.
sub stop_OldCrsStack
{
  my $OLD_CRS_HOME = $CFG->OLD_CRS_HOME;
  my $status;
  if (check_OldCrsStack()) {
     $status = s_stop_OldCrsStack($OLD_CRS_HOME);
     if (0 == $status) {
      trace ("Old CRS stack stopped successfully");
     } else {
      die("Unable to stop Old CRS stack");
     }
  } else {
    #ignore the status. Force stop
     s_stop_OldCrsStack($OLD_CRS_HOME);
  }
  sleep(60);
}


# Start the old crs stack.
sub start_OldCrsStack
{
  my $old_crshome = $CFG->OLD_CRS_HOME;
  my $crsctl      = catfile ($old_crshome, 'bin', 'crsctl');
  my $rc;
  my @output;
  my $retries = 25;
  my $is_up = FALSE;

  if ( isVersion112()) {

      @output = system_cmd_capture($crsctl, 'start', 'crs');
      $rc = shift (@output);
  }

  if (0 != $rc) {
      die ("Unable to start Old CRS stack");
  }

   while ($retries) {
     if (check_OldCrsStack()) {
        $is_up = TRUE;
        last;
     }
     trace ("Waiting for old Grid Infrastructure stack to start");
     sleep (5);
     $retries--;
   }

  if ($is_up) {
     trace ("old Grid Infrastructure stack started successfully");
  } else {
     die("Timed out waiting for the old Grid Infrastructure stack to start.");
  }

}


sub check_OldCrsStack
#-------------------------------------------------------------------------------
# Function: Check if the stack is up from old  CrsHome
# Args    : none
#-------------------------------------------------------------------------------
{
   trace("check old crs stack");
   my $old_crshome = $CFG->OLD_CRS_HOME;
   my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   my $crsctl      = catfile ($old_crshome, 'bin', 'crsctl');
   my $crs_stat    = catfile ($old_crshome, 'bin', 'crs_stat');
   my $status      = FAILED;

   if (isVersion112())
   {
     trace("11gR2 check routine");
     my @output = system_cmd_capture($crsctl, 'check', 'crs');
     my $rc = shift @output;
     if ((0 == $rc) &&
      (scalar(grep(/4537/, @output)) > 0) &&
      (scalar(grep(/4529/, @output)) > 0) &&
      (scalar(grep(/4533/, @output)) > 0))
     {
       $status = SUCCESS;
     }
   }
   elsif ($old_version[0] eq "10" &&
       $old_version[1] eq "1")
   {
      my @output = system_cmd_capture($crs_stat);
      my $rc     = shift @output;
      my @cmdout = grep(/CRS-0184/, @output);

      trace("rc=$rc output=@output");
      if ($rc == 0 && scalar(@cmdout) == 0) {
         $status = SUCCESS;
      }
   }
   else {
      my @output;
      @output         = system_cmd_capture($crsctl, 'check', 'cssd');
      my $status_cssd = shift @output;
      trace("cssd=@output");
      
      @output         = system_cmd_capture($crsctl, 'check', 'crsd');
      my $status_crsd = shift @output;
      trace("crsd=@output");

      my $status_evmd = FALSE;
      if (($CFG->UPGRADE) &&
         ($CFG->platform_family eq "windows") &&
         ($old_version[0] eq "10" && $old_version[1] eq "2"))
      {
         trace("Ignore evmd status");
         $status_evmd =FALSE;
      }
      else {
         @output = system_cmd_capture($crsctl, 'check', 'evmd');
         $status_evmd = shift @output;
         trace("evmd=@output");
      }

      #All the three daemons need to be up.
      if ((! $status_crsd) &&
          (! $status_evmd) &&
          (! $status_cssd))
      {
         $status = SUCCESS;
      }
   }

   if ($status == SUCCESS) {
      trace ("Earlier version Oracle Grid Infrastructure is running");
   }
   else {
      trace ("Earlier version Oracle Grid Infrastructure is not running");
   }

   return $status;
}

sub check_NewCrsStack
#-------------------------------------------------------------------------------
# Function: Check if the stack is up from current CrsHome
# Args    : none
#-------------------------------------------------------------------------------
{
   trace("check new crs stack");
   my $status = TRUE;
   my $new_crshome = $ORACLE_HOME;

   if ((!check_service("ora.cssd", 3)) &&
        (!check_service("ora.crsd", 3)) &&
        (!check_service("ora.evmd", 3)))
   {
      $status = FALSE;
   }

   if ($status)
   {
      trace ("Oracle Grid Infrastructure is running from $new_crshome");
   }
   else {
      trace ("Oracle Grid Infrastructure is not running from $new_crshome");
   }

   return $status;
}

sub get_oldconfig_info 
{
  trace ("Retrieving older cluster configuration data");
  my $oldCrsHome;
  my $ckptName = "ROOTCRS_OLDHOMEINFO";
  my $ckptStatus;
  my @oldCrsVer;
  my $verstring;
  my $errmsg;

  $wipCkptName = $ckptName;

  if (isCkptexist($ckptName)) {
     $ckptStatus = getCkptStatus($ckptName);
  } else {
     writeCkpt($ckptName, CKPTSTART);
  }


  #Get Old CRS Home
 
  if ($ckptStatus eq CKPTSUC) {
     $oldCrsHome = getCkptPropertyValue($ckptName, "OLD_CRS_HOME");
     $oldCrsHome = trim($oldCrsHome);
     trace("old crs home from ckpt property is $oldCrsHome");
  } else {
     if (-e $OLRCONFIG) 
     {
        # Get old CRS home
        $oldCrsHome = s_get_olr_file ("crs_home");
        $errmsg = "Failed to retrieve old Grid Infrastructure home from OLR file $OLRCONFIG";
     } else {
        $oldCrsHome = s_getOldCrsHome();
        $errmsg = "Failed to retrieve old Grid Infrastructure home location";
     }

     if (! $oldCrsHome)
     {
       writeCkpt($ckptName, CKPTFAIL);
       $wipCkptName = "ROOTCRS_STACK";
       die ($errmsg);
     }
  }

  trace("Old CRS Home = $oldCrsHome");

  $CFG->oldconfig('ORA_CRS_HOME', $oldCrsHome);
  $CFG->OLD_CRS_HOME($oldCrsHome);


  if (isOCRonASM()) {
      trace("copying gpnp files to use new home binaries");
      copy_gpnpfiles($CFG->OLD_CRS_HOME, $CFG->ORA_CRS_HOME);
  }

  # Get old CRS version, use new stack binaries
  if ($ckptStatus eq CKPTSUC) {
     $verstring = getCkptPropertyValue($ckptName, "OLD_CRS_VERSION");
     @oldCrsVer = split(/\./, $verstring);
     trace("old crs version from ckpt property is @oldCrsVer");
     
  }
  else
  {
     # Get old CRS version, use new version binary 
     @oldCrsVer = get_crs_version($CFG->ORA_CRS_HOME);
  }

  


  $CFG->oldconfig('ORA_CRS_VERSION', \@oldCrsVer);

  trace("The active version of the Oracle Clustereware is '@oldCrsVer'");

  # Get cluster GUID/OCRID, use new stack binaries
  my $oldClusterID = get_clusterguid($oldCrsHome);
  my $oldOCRID     = get_ocrid($CFG->ORA_CRS_HOME);
  $CFG->oldconfig('CLUSTER_GUID', $oldClusterID);
  $CFG->oldconfig('OCRID', $oldOCRID);

  if (!isVersion112())
  {
    # populate $NETWORK/NODE_NAME_LIST info for upgrade
    my ($networks, $nodes) = get_upgrade_netinfo();

    #check validity
    if (!$networks || !$nodes) {
       writeCkpt($ckptName, CKPTFAIL);
       $wipCkptName = "ROOTCRS_STACK";
       die("Failed to get networks and nodes information");
       
    } 
    $CFG->oldconfig('NETWORKS', $networks);
    $CFG->params('NETWORKS', $networks);

    $CFG->oldconfig('NODE_NAME_LIST', $nodes);
    # fix bug 9048549
    ##  $CFG->params('NODE_NAME_LIST', $nodes);

    trace ("  old networks =$networks");
    trace ("  old nodes    =$nodes");
  } 

  trace ("  old CrsHome  =$oldCrsHome");
  trace ("  old CrsVer   =@oldCrsVer");
  trace ("  old ClusterID=$oldClusterID");
  trace ("  old OCRID    =$oldOCRID");

   
  my $verinfo = join('.',@oldCrsVer);
  trace("Version Info retrieved is : $verinfo");

  if ($ckptStatus ne CKPTSUC)
  {
     writeCkptProperty($wipCkptName, "OLD_CRS_HOME", $oldCrsHome);
     writeCkptProperty($wipCkptName, "OLD_CRS_VERSION", $verinfo);
  }

  writeCkpt($ckptName, CKPTSUC);
  $wipCkptName = "ROOTCRS_STACK";
  return;
}

=head2 initial_cluster_validation

  Perform validations for the cluster installation as well as
  initializes some component files

=head3 Parameters

  None

=head3 Returns

  None, errors result in termination of the script

=cut

sub initial_cluster_validation {
  my $OLR_LOCATION = $CFG->OLR_LOCATION;
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $CLUSTER_NAME = $CFG->params('CLUSTER_NAME');
  my $OCR;

  validate_SICSS () or die "validate_SICSS failed";
  validate_9iGSD () or die "validate_9iGSD failed";
  validate_olrconfig ($OLR_LOCATION, $ORA_CRS_HOME)
    or die "Error in validate_olrconfig";

  if ($CFG->ASM_STORAGE_USED) {
     my $diskgroup = $CFG->params('ASM_DISK_GROUP');
     $OCR = "+" . $diskgroup;
  }
  else { $OCR = $CFG->params('OCR_LOCATIONS'); }

  if (! $CFG->UPGRADE)
  {
   validateOCR ($ORA_CRS_HOME, $CLUSTER_NAME, $OCR)
    or die "validateOCR failed for $OCR";
  }

  if (!CSS_CanRunRealtime($CFG)) {
    die("CSS cannot be run in realtime mode");
  }

  if (isAddNode($HOST, $CFG->params('NODE_NAME_LIST'))) {
     if ((isOCRonASM()) && ($CFG->params('CRS_STORAGE_OPTION') != 1))
     {
        $CFG->params('CRS_STORAGE_OPTION', 1);
     }
     elsif ((! isOCRonASM()) && ($CFG->params('CRS_STORAGE_OPTION') == 1))
     {
        $CFG->params('CRS_STORAGE_OPTION', 2);
     }
  }

  return;
}

=head2 wait_for_stack_start

  Wait for the stack to start up

=head3 Parameters

  Number of chcks to see if the stack is up, made every 5 seconds

=head3 Returns

  SUCCESS  Stack is up
  FAILED   Stack is not up

=head3 Usage


=cut

sub wait_for_stack_start {
  # Wait until the daemons actually start up
  my $is_up = FALSE;
  my $retries = shift;
  my $crsctl = crs_exec_path('crsctl');
  my @output;
  my $rc;

  # Complete success. This is the last node of the install.
  # Wait for CRSD and EVMD to start up
  while ($retries) {
    @output = system_cmd_capture($crsctl, 'check', 'crs');
    $rc	    = shift @output;

    if ($rc == 0) {
      $is_up = TRUE;
      last;
    }

    trace ("Waiting for Oracle CRSD and EVMD to start");
    sleep (5);
    $retries--;
  }

  if ($is_up) {
    #Also wait for check cluster to work.
    if (!check_service("cluster", 120))
    {
       die("Oracle Grid Infrastructure stack start initiated but failed to complete");
    }
    trace ("Oracle CRS stack installed and running");
   
  } else {
    error ("Timed out waiting for the CRS stack to start.");
    exit 1;
  }

  return $is_up;
}


=head2 wait_for_gpnpd_start

  Wait for the gpnpd to open local endpoint
  This will be revised after start_cluster_resource allow PENDING handling.

=head3 Returns

  SUCCESS  Gpnpd is up
  FAILED   Gpnpd is not up

=head3 Usage


=cut

sub wait_for_gpnpd_start {
  my $is_up = FALSE;
  my $retries = 10;
  my $gpnptool = crs_exec_path('gpnptool');
  my @output;
  my $rc;

  # Wait until the gpnpd start and open local endpoint
  while ($retries) {
    @output = system_cmd_capture($gpnptool, 'lfind');
    $rc     = shift @output;

    if ($rc == 0) {
      $is_up = TRUE;
      last;
    }
    trace ("Waiting for GPNPD to start");
    sleep (5);
    $retries--;
  }

  if ($is_up) {
  } else {
    error ("Timed out waiting for the GPNPD to start.");
    exit 1;
  }

  return $is_up;
}

sub upgrade_OCR {
  my $ret = TRUE;

  my @runocrconfig = ("ocrconfig", "-upgrade",
                      $CFG->params('ORACLE_OWNER'),
                      $CFG->params("ORA_DBA_GROUP"));

  # ocrconfig - Create OCR keys
  trace ("Creating or upgrading OCR keys");
  my $status = run_crs_cmd(@runocrconfig);

  if ($status == 0) 
  {
    trace ("OCR keys are successfully populated");
    s_reset_srvconfig () or die "reset srvconfig failed";
  } else {
    error ("Failed to create Oracle Cluster Registry configuration");
    $ret = FALSE;
  }

  return $ret;
}

=head2 new

  This is the class constructor method for this class

=head3 Parameters

  A hash containing values for any key that is listed in the accessor
  methods section

=head3 Returns

  A blessed class

=head3 Usage

  my $cfg = crsconfig_lib->new(
                paramfile           => $PARAM_FILE_PATH,
                osdfile             => $defsfile,
                crscfg_trace        => TRUE,
                HOST                => $HOST
                )

  This creates an object with parameters built from $PARAM_FILE_PATH
  and $defsfile, for HOST $HOST with tracing turned on.  The values
  specified may be retrieved via the standard access methods, e.g.
    my $host = $cfg->HOST;
  will set $host to the $HOST value set in the hash passed to 'new'.

  While it is possible to pass any key/value pair to 'new', even ones
  for which there are no access methods, the values cannot be easily
  used without an access method.  For the list of access methods, see
  the 'Access Methods Section' below

=cut

# Class constructor and methods
sub new {
  my ($class, %init) = @_;
  $CFG = {};
  for my $element (keys %init) {
    if (defined($init{$element})) {
      my $type = $elements{$element};
      if (($type eq 'ARRAY' || $type eq 'HASH') &&
             ref($init{$element}) ne $type) {
        croak "Initializer for $element must be $type reference";
      }
      if ($type ne 'INDIRECT') { $CFG->{$element} = $init{$element}; }
    }
  }

  # Initialize stuff not in the initializer
  for my $element (keys %elements) {
    if (!defined($CFG->{$element})) {
      my $type = $elements{$element};
      if ($type eq 'ARRAY') { $CFG->{$element} = []; }
      elsif ($type eq 'HASH') { $CFG->{$element} = {}; }
      elsif ($type eq 'COUNTER') { $CFG->{$element} = 0; }
    }
  }

  bless $CFG, $class;

  for my $element (keys %init) {
    if (defined($init{$element})) {
      my $type = $elements{$element};
      if ($type eq 'INDIRECT') { $indirect{$element}($CFG, $init{$element}); }
    }
  }

  if (! $CFG->paramfile) {
    die("No configuration parameter file was specified");
  }

  if (! -e $CFG->paramfile) {
    die("Configuration parameter file", $CFG->paramfile,
        "cannot be found");
  }

  print ("Using configuration parameter file: ", $CFG->paramfile, "\n");

  # Set up the parameters
  setup_param_vars($CFG->paramfile);

  # Now set various defaults/values based on various input
  my $OH = $CFG->params('ORACLE_HOME'); # for convenience

  if ($OH) { trace("Using Oracle CRS home $OH"); }
  else {
    die("The Oracle CRS home path not found in the configuration",
        "parameters");
  }

  if (!(-d $OH)) {
    die("The Oracle CRS home path \"$OH\" does not exist");
  }

  my $default_trc_dir = catfile($OH, 'cfgtoollogs', 'crsconfig');
  my $default_olr_dir = catfile($OH, 'cdata');

  if ($CFG->IS_SIHA) { # Define stuff for SIHA
    $CFG->parameters_valid($CFG->validateSIHAVarList);


    # trace file
    if (!$CFG->crscfg_trace_file) {
       my $file = "roothas.log";
       if ($CFG->CRSDelete) { $file = "hadelete.log"; }
       if ($CFG->HAPatch)   { $file = "hapatch.log"; }

       $CFG->crscfg_trace_file(catfile($default_trc_dir, $file));
    }

    if (!$CFG->OLR_DIRECTORY) {
      $CFG->OLR_DIRECTORY(catfile($default_olr_dir, 'localhost'));
    }
  }
  else { # Define stuff for clustered mode
    $CFG->parameters_valid(validateCRSVarList());
    if (!$CFG->crscfg_trace_file) {
      my $host = $CFG->HOST;
      my $file = "rootcrs_$host.log";
      if ($CFG->CRSDelete) { $file = "crsdelete_$host.log"; }
      if ($CFG->CRSPatch)  { $file = "crspatch_$host.log"; }
      if ($CFG->DOWNGRADE) { $file = "crsdowngrade_$host.log"; }

      $CFG->crscfg_trace_file(catfile($default_trc_dir, $file));
    }

    if (!$CFG->OLR_DIRECTORY) {
      $CFG->OLR_DIRECTORY($default_olr_dir);
    }
  }

  # We really should destroy $CFG here; this will be impelmented later
  if ($CFG->parameters_valid) {
    trace("The configuration parameter file", $CFG->paramfile,
          "is valid");
  }
  else {
    die("The configuration parameter file", $CFG->paramfile,
        "is not valid");
  }

  if ($CFG->osdfile && -e $CFG->osdfile) {
    setup_param_vars($CFG->osdfile);
  }

  if ($CFG->addfile && -e $CFG->addfile) {
     setup_param_vars($CFG->addfile);

     if ($CFG->defined_param('CRS_ADDNODE') &&
        $CFG->params('CRS_ADDNODE') eq "true")
     {
        $CFG->addnode(TRUE);
     }
     else {
        $CFG->addnode(FALSE);
     }
  }

  $CFG->SUPERUSER(check_SuperUser());

  if ($CFG->SUPERUSER) { $CFG->user_is_superuser(TRUE); }
  else {
    # If we are not SUPERUSER, indicate this and set SUPERUSER to
    # ORACLE_OWNER
    $CFG->user_is_superuser(FALSE);
    $CFG->SUPERUSER($CFG->params('ORACLE_OWNER'));
  }

  # Set some default values, if necessary
  if (!$CFG->ORA_CRS_HOME) {
    $CFG->ORA_CRS_HOME($CFG->params('ORACLE_HOME'));
  }

  if (!$CFG->HOST) { $CFG->HOST(tolower_host()); }

  $CFG->OLR_LOCATION(catfile($CFG->OLR_DIRECTORY, $CFG->HOST . '.olr'));

  if ($CFG->SUPERUSER &&
      $CFG->defined_param('OLASTGASPDIR') &&
      ! -e $CFG->params('OLASTGASPDIR')) {
    mkpath($CFG->params('OLASTGASPDIR'));
  }

  if (!$CFG->HAS_USER) {
    $CFG->HAS_USER($CFG->params('ORACLE_OWNER'));
  }

  if (!$CFG->HAS_GROUP) {
    $CFG->HAS_GROUP($CFG->params('ORA_DBA_GROUP'));
  }

  if (!$CFG->s_run_as_user2p) {
    $CFG->s_run_as_user2p(\&crsconfig_lib::s_run_as_user2);
  }

  if (!$CFG->s_run_as_userp) {
    $CFG->s_run_as_userp(\&crsconfig_lib::s_run_as_user);
  }

  if (!$CFG->s_run_as_user3p) {
    $CFG->s_run_as_user3p(\&crsconfig_lib::s_run_as_user3);
  }

  # If the versions of the 'run_as user' commands with parm order
  # reversed exist, as inidicated in the sybol table, set them now
  if (!$CFG->s_run_as_user2_v2p &&
      defined($s_crsconfig_lib::{'s_run_as_user2_v2'})) {
    $CFG->s_run_as_user2p_v2(\&crsconfig_lib::s_run_as_user2_v2);
  }

  if (!$CFG->s_run_as_user_v2p && 
      defined($s_crsconfig_lib::{'s_run_as_user_v2'})) {
    $CFG->s_run_as_userp_v2(\&crsconfig_lib::s_run_as_user_v2);
  }

  if (! $CFG->UPGRADE) {
    if ($CFG->ASM_STORAGE_USED) {
      # Put a null string in for VF discover string so that the change
      # will not be rejected (bug 7694835)
      $CFG->VF_DISCOVERY_STRING('');
    }
    else {
      if (! $CFG->IS_SIHA) {
        if ($CFG->defined_param('VOTING_DISKS'))
        {
           $CFG->VF_DISCOVERY_STRING($CFG->params('VOTING_DISKS'));
        }
        else
        {
           $CFG->VF_DISCOVERY_STRING('');
        }
      }
    }
  }

  $CFG->platform_family(lc(s_get_platform_family()));

  # To allow s_crsconfig_lib functions to work until they have been
  # properly packaged, put some variables into the global domain
  export_vars();

  $CFG->print_config;

  # set owner & permission of trace file
  s_set_ownergroup ($CFG->params('ORACLE_OWNER'),
                    $CFG->params('ORA_DBA_GROUP'),
                    $CFG->crscfg_trace_file);
  s_set_perms ("0775", $CFG->crscfg_trace_file);

  return $CFG;
}

# Access Methods Section
#   This section contains the accessor method used in this class
#
# Adding a new accessor method:
#   Unless the accessor method must do something special, use the
#   standard access methods:
#     access_array  - get/set a value from/in an array
#     access_hash   - get/set a value from/in an hash
#     access_scalar - get/set a scalar value
#
# Examples:
#   (scalar FOOS)
#     (method definition) sub FOOS {return access_scalar(@_);}
#     (value set)   $CFG->FOOS('BARS'); (sets $CFG->FOOS to 'BARS')
#     (value get)   my $foos = $CFG->FOOS;
#
#   (array FOOA)
#     (method definition) sub FOOA {return access_array(@_);}
#     (value set)   $CFG->FOOA(0, 'BARA'); (sets FOOA[0])
#     (value set)   $CFG->FOOA(\@BARA); (sets FOOA array to @BARA)
#     (value get)   my $fooa0 = $CFG->FOOA(0); (gets FOOA[0])
#     (value get)   my @fooa = @{$CFG->FOOA}; (gets @FOOA)
#
#   (hash FOOH)
#     (method definition) sub FOOH {return access_hash(@_);}
#     (value set)   $CFG->FOOH('BARk', 'BARv'); (sets FOOH key BARk to BARv)
#     (value set)   $CFG->FOOH(\%BARH); (sets $CFG->FOOH to %BARH)
#     (value get)   my $barv = $CFG->FOOH('BARk'); (gets FOOH{'BARk'})
#     (value get)   my %fooh = %{$CFG->FOOH}; (gets %FOOH)
#
#   (counter FOOC) initial value preset to 0
#     (method definitions)
#        sub FOOC {return access_counter(@_);}
#        sub pp_FOOC {return access_counter(@_);}
#        sub FOOC_pp {return access_counter(@_);}
#     (value get - assuming a current value of 3)
#        (no value change)
#          my $cval = $CFG->FOOC; (returns 3, current value still 3)
#        (increment before returning ++FOOC)
#          my $cval = $CFG->pp_FOOC; (returns 4, current value now 4)
#        (increment after returning FOOC++)
#          my $cval = $CFG->FOOC_pp; (returns 3, current value now 4)


# Accessor methods for class elements
sub addnode             { return access_scalar(@_); }
sub CLSCFG_EXTRA_PARMS  { return access_array(@_); }
sub CLSCFG_POST_CMD     { return access_array(@_); }
sub CRSCFG_POST_CMD     { return access_array(@_); }
sub CRSDelete           { return access_scalar(@_); }
sub DEBUG               { return access_scalar(@_); $DEBUG = $CFG->DEBUG; }
sub HAS_GROUP           { return access_scalar(@_); }
sub HAS_USER            { return access_scalar(@_); }
sub HOST                { return access_scalar(@_); }
sub IS_SIHA             { return access_scalar(@_); }
sub OCR_ID              { return access_scalar(@_); }
sub ORA_CRS_HOME        { return access_scalar(@_); }
sub SUPERUSER           { return access_scalar(@_); }
sub VF_DISCOVERY_STRING { return access_scalar(@_); }
sub OLR_DIRECTORY       { return access_scalar(@_); }
sub OLR_LOCATION        { return access_scalar(@_); }
sub UPGRADE             { return access_scalar(@_); }
sub DOWNGRADE           { return access_scalar(@_); }
sub NETWORKS            { return access_scalar(@_); }
sub addfile             { return access_scalar(@_); }
sub gpnp_setup_type     { return access_scalar(@_); }
sub hosts               { return access_array(@_); }
sub osdfile             { return access_scalar(@_); }
sub paramfile           { return access_scalar(@_); }
sub parameters_valid    { return access_scalar(@_); }
sub platform_family     { return access_scalar(@_); }
sub user_is_superuser   { return access_scalar(@_); }
sub oldconfig           { return access_hash(@_); }
sub s_run_as_user2p     { return access_scalar(@_); }
sub s_run_as_user2_v2p  { return access_scalar(@_); }
sub s_run_as_userp      { return access_scalar(@_); }
sub s_run_as_user3p     { return access_scalar(@_); }
sub s_run_as_user_v2p   { return access_scalar(@_); }
sub unlock_crshome      { return access_scalar(@_); }
sub destcrshome         { return access_scalar(@_); }
sub hahome              { return access_scalar(@_); }
sub CRSPatch            { return access_scalar(@_); }
sub HAPatch             { return access_scalar(@_); }
sub force               { return access_scalar(@_); }
sub deinstall           { return access_scalar(@_); }
sub keepdg              { return access_scalar(@_); }
sub lastnode            { return access_scalar(@_); }
sub REMOTENODE          { return access_scalar(@_); }
sub UNLOCK              { return access_scalar(@_); }
sub NOCRSSTOP           { return access_scalar(@_); }
sub isFstNodeToUpgrade  { return access_scalar(@_); }
sub isLstNodeToUpgrade  { return access_scalar(@_); }

# Counters
# pp_ for increment before, eg pp_foo same as ++foo
# _pp for increment after, eg foo_pp same as foo++
# both take an argument for increment amount (default increment is 1)
sub srvctl_trc_suff     { return access_counter(@_); }
sub pp_srvctl_trc_suff  { return access_counter(@_); } # ++srvctl_trc_suff
sub srvctl_trc_suff_pp  { return access_counter(@_); } # srvctl_trc_suff++

sub GPNP_GPNPHOME_DIR   { return access_scalar(@_); }
sub GPNP_WALLETS_DIR    { return access_scalar(@_); }
sub GPNP_W_ROOT_DIR     { return access_scalar(@_); }
sub GPNP_W_PEER_DIR     { return access_scalar(@_); }
sub GPNP_W_PRDR_DIR     { return access_scalar(@_); }
sub GPNP_W_PA_DIR       { return access_scalar(@_); }
sub GPNP_PROFILES_DIR   { return access_scalar(@_); }
sub GPNP_P_PEER_DIR     { return access_scalar(@_); }

# -- local
sub GPNP_GPNPLOCALHOME_DIR { return access_scalar(@_); }
sub GPNP_L_WALLETS_DIR  { return access_scalar(@_); }
sub GPNP_L_W_ROOT_DIR   { return access_scalar(@_); }
sub GPNP_L_W_PEER_DIR   { return access_scalar(@_); }
sub GPNP_L_W_PRDR_DIR   { return access_scalar(@_); }
sub GPNP_L_W_PA_DIR     { return access_scalar(@_); }
sub GPNP_L_PROFILES_DIR { return access_scalar(@_); }
sub GPNP_L_P_PEER_DIR   { return access_scalar(@_); }

# gpnp files:

# -- cluster-wide
sub GPNP_ORIGIN_FILE   { return access_scalar(@_); }
sub GPNP_W_ROOT_FILE   { return access_scalar(@_); }
sub GPNP_WS_PA_FILE    { return access_scalar(@_); }
sub GPNP_WS_PEER_FILE  { return access_scalar(@_); }
sub GPNP_WS_PRDR_FILE  { return access_scalar(@_); }
sub GPNP_C_ROOT_FILE   { return access_scalar(@_); }
sub GPNP_C_PA_FILE     { return access_scalar(@_); }
sub GPNP_C_PEER_FILE   { return access_scalar(@_); }
sub GPNP_P_PEER_FILE   { return access_scalar(@_); }
sub GPNP_P_SAVE_FILE   { return access_scalar(@_); }

# -- local
sub GPNP_L_W_ROOT_FILE { return access_scalar(@_); }
sub GPNP_L_W_PA_FILE   { return access_scalar(@_); }
sub GPNP_L_WS_PA_FILE  { return access_scalar(@_); }
sub GPNP_L_W_PEER_FILE { return access_scalar(@_); }
sub GPNP_L_WS_PEER_FILE { return access_scalar(@_); }
sub GPNP_L_WS_PRDR_FILE { return access_scalar(@_); }
sub GPNP_L_CRQ_PA_FILE { return access_scalar(@_); }
sub GPNP_L_CRQ_PEER_FILE { return access_scalar(@_); }
sub GPNP_L_C_ROOT_FILE { return access_scalar(@_); }
sub GPNP_L_C_PA_FILE   { return access_scalar(@_); }
sub GPNP_L_C_PEER_FILE { return access_scalar(@_); }
sub GPNP_L_P_PEER_FILE { return access_scalar(@_); }
sub GPNP_L_P_SAVE_FILE { return access_scalar(@_); }

# gpnp peer wrls
sub GPNP_W_PEER_WRL { return access_scalar(@_); }
sub GPNP_L_W_PEER_WRL { return access_scalar(@_); }
sub GPNP_W_PRDR_WRL { return access_scalar(@_); }
sub GPNP_L_W_PRDR_WRL { return access_scalar(@_); }

# package tools
sub GPNP_E_GPNPTOOL { return access_scalar(@_); }
sub GPNP_E_GPNPSETUP { return access_scalar(@_); }

sub config_value { return $_[0]->{$_[1]}; }

# The following methods are required to translate values associated
# with the old CRS installation that were passed as scalars into
# key/value pairs in the oldconfig hash
# These should not be required, but because code was built around them
# after an incorrect implementation, it was less destabilizing to
# provide translation methods than to correct the code
sub oldcrshome          {
  my $class = shift;
  return oldconfig($class, 'ORA_CRS_HOME', @_);
}

sub OLD_CRS_HOME        { return oldcrshome(@_); }

sub oldcrsver           {
  my $class = shift;
  my $rv;
  my @ver;
  if (scalar(@_) == 0) {
    @ver = @{oldconfig($class, 'ORA_CRS_VERSION')};
    $rv = join('.', @ver);
  }
  else {
    @ver =(split('\.', $_[0]));
    $rv = oldconfig($class, 'ORA_CRS_VERSION', \@ver);
  }
  return $rv;
}

sub params {
  my $cfg = shift;

  # If the parameter has not been defined, error.  This prevents
  # typos from going unnoticed
  if (scalar(@_) == 1 && !defined($cfg->{'params'}->{$_[0]})) {
    die ("Parameter $_[0] not defined");
  }
  else { return access_hash($cfg, @_); }
}

sub defined_param {
  my $cfg = shift;
  my $defd = FALSE;
  if (scalar(@_) > 1) { croak "Only 1 parameter allowed: @_"; }
  else { $defd = defined($cfg->{'params'}->{$_[0]}); }

  return $defd;
}

sub defined_value {
  my $cfg = shift;
  my $defd = FALSE;
  if (scalar(@_) > 1) { croak "Only 1 parameter allowed: @_"; }
  else { $defd = defined($cfg->{$_[0]}); }

  return $defd;
}

sub ASM_STORAGE_USED {
  my $cfg = shift;
  my $val;
  my $ret;
  if (@_) { # Setting the value, so keep params in sync
    $ret = shift;
    if ($ret) { $cfg->params('CRS_STORAGE_OPTION', 1); }
    else { $cfg->params('CRS_STORAGE_OPTION', 2); }
  }
  elsif ($cfg->params('CRS_STORAGE_OPTION') == 1) { $ret = TRUE; }
  else { $ret = FALSE; }

  return $ret;
}

# If tracing is turned on after initialization, make sure directory is
# created
sub crscfg_trace {
  my $ret = access_scalar(@_);
  # if the call was to turn tracing on, make sure trace dir created
  if ($ret && scalar(@_) > 1) { setup_trace_dir(); }
  return $ret;
}

sub crscfg_trace_file {
  my $ret = access_scalar(@_);
  # if the call was set the file name, make sure trace dir created
  if (scalar(@_) > 1) { setup_trace_dir(); }
  return $ret;
}

###### End Access Methods Section #######

# Set up the trace directory
sub setup_trace_dir {
  if ($CFG->crscfg_trace &&
      $CFG->crscfg_trace_file &&
      ! -e $CFG->crscfg_trace_file) {
    my $trace_dir = dirname($CFG->crscfg_trace_file);
    if (! -e $trace_dir) {
      my $tracing = $CFG->crscfg_trace;

      print "Creating trace directory\n";
      # temporarily turn off tracing to avoid recursing in create_dir
      $CFG->crscfg_trace(0);
      create_dir($trace_dir);
      $CFG->crscfg_trace($tracing);
    }
  }
}

# Execute a command as a user (do not invoke directly, use run_as_user)
sub s_run_as_usere {
  my $cfg = shift;
  my $user = shift;
  my $pgm = $cfg->s_run_as_userp;
  my @args = ("@_", $user);
  if ($cfg->s_run_as_user_v2p) {
    $pgm = $cfg->s_run_as_user_v2p;
    @args = ($user, @_);
  }
  return &$pgm(@args);
}

# Execute a command as a user (do not invoke directly, use run_as_user3)
sub s_run_as_user3e {
  my $cfg   = shift;
  my $user  = shift;
  my $cmd   = shift;
  my $param = shift;
  my $ref   = shift;
  my $pgm  = $cfg->s_run_as_user3p;
  my @args = ($user, \@{$cmd}, $param);
  my @out  = &$pgm(@args);
  my $rc   = shift @out;
  @{$ref}  = @out;
  return $rc;
}

# Execute a command as a user, returning the output
# (do not invoke directly, use run_as_user2)
sub s_run_as_user2e {
  my $cfg = shift;
  my $user = shift;
  my $aref = shift;
  my $rc;
  my @args = (\@_, $user, $aref);
  if (!$cfg->s_run_as_user2_v2p) {
    my $pgm  = $cfg->s_run_as_user2p;;
    $rc = &$pgm(\@_, $user, $aref);
  }
  else {
    my $pgm = $cfg->s_run_as_user2_v2p;
    my @out = &$pgm($user, @_);
    $rc = shift @out;
    @{$aref} = @out;
  }
  return $rc;
}

# Low level access methods
sub access_scalar {
  my $class  = shift;

  # find where we were called from so that we know what element
  # Get callers name
  my @caller = caller(1);

  my $name = $caller[3];
  # strip class name to get element name
  my $class_name = ref($class);
  $name =~ s/$class_name\:\://;

  my $ret;
  if (@_ > 1) { croak "Too many args to $name"; }
  if (@_) {$ret = shift; $class->{$name} = $ret; }
  else { $ret = $class->{$name}; }

  return $ret;
}

sub access_array {
  my $class  = shift;

  # find where we were called from so that we know what element
  # Get callers name
  my @caller = caller(1);

  my $name = $caller[3];
  # strip class name to get element name
  my $class_name = ref($class);
  $name =~ s/$class_name\:\://;

  my $init;
  my $ret;

  if (! @_) { $ret = $class->{$name}; }
  else {
    $init = shift;
    if (ref($init) eq 'ARRAY' && !@_) {
      $class->{$name} = $init;
      $ret = $class->{$name};
    }
    elsif (@_ > 1) { croak "Too many args to $name"; }
    elsif (@_) { $class->{$name}->[$init] = $ret = shift; }
    else { $ret = $class->{$name}->[$init]; }
  }

  return $ret;
}

sub access_hash {
  my $class  = shift;

  # find where we were called from so that we know what element
  # Get callers name
  my @caller = caller(1);

  my $name = $caller[3];
  # strip class name to get element name
  my $class_name = ref($class);
  $name =~ s/$class_name\:\://;

  my $init;
  my $ret;
  if (! @_) { $ret = $class->{$name}; }
  else {
    $init = shift;
    if (ref($init) eq 'HASH' && !@_) {
      $class->{$name} = $init;
      $ret = $class;
    }
    elsif (@_ > 1) { croak "Too many args to $name"; }
    elsif (@_) { $class->{$name}->{$init} = $ret = shift; }
    else { $ret = $class->{$name}->{$init}; }
  }

  return $ret;
}

sub access_counter {
  my $class  = shift;

  # find where we were called from so that we know what element
  # Get callers name
  my @caller = caller(1);

  my $name = $caller[3];
  # strip class name to get element name
  my $class_name = ref($class);
  $name =~ s/$class_name\:\://;

  my $elt_name = $name;
  my $pre  = $elt_name =~ s/^pp_//;
  my $post = $elt_name =~ s/_pp$//;

  my $ret = $class->{$elt_name};
  my $incr = 1;
  if (@_ > 1) { croak "Too many args to $name: @_"; }

  if (@_) { $incr = shift; }
  if ($pre) { $class->$elt_name(($ret += $incr)); }
  elsif ($post) { $class->$elt_name($ret + $incr); }

  return $ret;
}

sub stopClusterware
#-------------------------------------------------------------------------------
# Function: Stop crs / cluster 
# Args    : oracle home, Resource (crs/cluster)
# Returns : TRUE  if success
#           FALSE if failed
#-------------------------------------------------------------------------------
{
   my ($home, $res) = @_;
   my $crsctl;

   if (! defined $home) {
     $crsctl = crs_exec_path('crsctl');
   } else {
     $crsctl = catfile( $home, 'bin', 'crsctl' );
   }

   my $success 	= TRUE;

   if ($CFG->platform_family eq "windows") {
      if (! defined $home) {
        $crsctl = crs_exec_path("crsctl.exe");
      } else {
        $crsctl = catfile( $home, 'bin', 'crsctl.exe' );
      }
   } else {
      if (! defined $home) {
        $crsctl = crs_exec_path('crsctl');
      }  else {
        $crsctl = catfile( $home, 'bin', 'crsctl' );
      }
   }

   if (! -e $crsctl) {
      error ("$crsctl does not exist to proceed stop Grid Infrastructure");
      return FALSE;
   }

   # stop resource
   my @cmd;
   if ($res eq 'crs') {
      @cmd = ($crsctl, 'stop', 'crs', '-f');
   } 
   else {
      @cmd = ($crsctl, 'stop', 'resource', '-all', '-init');
   } 
  
   system (@cmd);
   trace ("@cmd");

   if (! checkServiceDown("cluster")) {
      print "################################################################\n";
      print "#You must kill processes or reboot the system to properly      #\n";
      print "#cleanup the processes started by Oracle Grid Infrastructure   #\n";
      print "################################################################\n";
      return FALSE;
   }

   # check if ohasd & crs are still up
   if ($res eq 'crs') {
      if (! checkServiceDown("ohasd")) {
         error "Unable to stop CRS\n";
         $success = FALSE;
      }
   }

   return $success;
}

sub stopOracleRestart
#-------------------------------------------------------------------------------
# Function: Stop Oracle Restart Stack
# Returns : TRUE  if success
#           FALSE if failed
#-------------------------------------------------------------------------------
{
   my $success  = TRUE;
   my $crsctl;

   if ($CFG->platform_family eq "windows") {
      $crsctl = crs_exec_path("crsctl.exe");
   } else {
      $crsctl = crs_exec_path("crsctl");
   }

   if (! -e $crsctl) {
      error ("$crsctl does not exist to proceed stop Grid Infrastructure");
      return FALSE;
   }

   # stop the SIHA stack
  
   trace("Stopping the Oracle Restart stack");
   my  @cmd = ($crsctl, 'stop', 'has', '-f');
   trace("Executing @cmd");
   my @out       = system_cmd_capture(@cmd);
   my $rc        = shift @out;

   trace("stop of Oracle Restart output is @out");

   # check if ohasd is  still up
   if (! checkServiceDown("ohasd")) {
         error "Unable to stop Oracle Restart\n";
         $success = FALSE;
   }

   return $success;
}

sub unlockHAHomefordeinstall
{
   trace ("Unlock Oracle Restart home...");
   my $exclfile = $_[0];
   my $unlock_hahome;
   my $status = SUCCESS;

   trace("Exclude file used is $exclfile");

   if (!$CFG->UNLOCK) { $CFG->UNLOCK(TRUE); }

   #Try to get the home path from olr.loc
   $unlock_hahome = s_get_olr_file ("crs_home");
   trace ("Home location in olr.loc is $unlock_hahome");

   if (! $unlock_hahome)
   {
      $unlock_hahome = $CFG->hahome;
   }

   if ($CFG->hahome)
   {
     if ($NOCRSSTOP && ($unlock_hahome eq $CFG->hahome))
     {
       trace("Attempting to unlock an active Oracle HA home");
     }
     elsif ($NOCRSSTOP)
     {
       $unlock_hahome = $CFG->hahome;
     }
   }

   trace("Oracle HA home to unlock is $unlock_hahome");

   # validate if crshome exists
   if (! -e $unlock_hahome) {
      error  "Oracle Restart home: $unlock_hahome not found\n";
      return FALSE;
   }

   if (! $NOCRSSTOP)
   {
     $status = stopOracleRestart();
   }

   # check the status of HA stack
   if ($status) {
      s_reset_crshome1($ORACLE_OWNER, $ORA_DBA_GROUP, 755, $unlock_hahome,$exclfile);
      print "Successfully unlock $unlock_hahome\n";
   }
   else {
      print "The Oracle Restart stack failed to stop.\n";
      print "You should stop the stack with 'crsctl stop has' and rerun the command\n";
   }
}

sub unlockCRSHomefordeinstall
{
   trace ("Unlock crshome...");
   my $exclfile = $_[0];
   my $statusCRS;

   trace ("crshome passed is $unlock_crshome");
   trace("Exclude file used is $exclfile");
   my $unlock_crshome1;

   if (!$CFG->UNLOCK) { $CFG->UNLOCK(TRUE); }

   #Try to get the home path from olr.loc
   $unlock_crshome1 = s_get_olr_file ("crs_home");
   trace ("Home location in olr.loc is $unlock_crshome1");

   if (! $unlock_crshome1)
   {
      $unlock_crshome1 = $unlock_crshome;
   }

   #NOCRSSTOP option to be passed to unlock an inactive grid home
   if ($unlock_crshome) {
        if ($NOCRSSTOP && ($unlock_crshome eq $unlock_crshome1)) {
           error("You can't unlock an active CRS home");
        }
        elsif ($NOCRSSTOP) {
           $unlock_crshome1 = $unlock_crshome;
        }
   }

   trace ("unlock crs home is $unlock_crshome1");

   # validate if crshome exists
   if (! -e $unlock_crshome1) {
      error  "crshome: $unlock_crshome not found\n";
      return FALSE;
   }

   if (! $NOCRSSTOP) {
      $statusCRS = stopClusterware($unlock_crshome1, "crs");
   } else { $statusCRS = SUCCESS; }

   if ($statusCRS) {
      unlockHome($ORACLE_OWNER, $ORA_DBA_GROUP, 755, $unlock_crshome1);
      print "Successfully unlock $unlock_crshome1\n";
   } else {
      print "The Oracle Grid Infrastructure stack failed to stop.\n";
      print "You should stop the stack with 'crsctl stop crs' and rerun the command\n";
   }
}

sub unlockCRSHome
{
  my $exclfile = catfile($CFG->ORA_CRS_HOME, 'crs', 'install', 'install.excl');
  unlockCRSHomefordeinstall( $exclfile);
}

sub unlockCRSHomeforpatch
{
  my $exclfile = catfile($CFG->ORA_CRS_HOME, 'OPatch', 'crs', 'installPatch.excl');
  unlockCRSHomefordeinstall( $exclfile);
}

sub unlockPatchHome
{
  my $home = $_[0];
  my $exclfile = catfile($home, 'crs', 'install', 'installPatch.excl');
  modifyparamfile("ORACLE_HOME", $home);
  modifyparamfile("CRFHOME", $home);
  modifyparamfile("JREDIR", "$home/jdk/jre/");
  modifyparamfile("JLIBDIR", "$home/jlib");
  $NOCRSSTOP = TRUE;
  $unlock_crshome=$home;
  #set new home instantiate prior to unlock
  $ORACLE_HOME  = $home;
  $ORA_CRS_HOME = $ORACLE_HOME;
  $ENV{'ORACLE_HOME'}  = $ORACLE_HOME;
  $CFG->ORA_CRS_HOME($home);
  my $patchhome = $CFG->ORA_CRS_HOME;
  error("unlock home is  set to $patchhome");
  instantiate_scripts(); 
  unlockCRSHomefordeinstall( $exclfile);
}

sub unlockHAHome
{
  my $exclfile = catfile($CFG->ORA_CRS_HOME, 'crs', 'install', 'install.excl');
  unlockHAHomefordeinstall( $exclfile);
}

sub unlockHAHomeforpatch
{
  my $exclfile = catfile($CFG->ORA_CRS_HOME, 'OPatch', 'crs', 'installPatch.excl');
  unlockHAHomefordeinstall( $exclfile);
}


sub isRAC_appropriate
#-------------------------------------------------------------------------------
# Function:  Check if rac_on/rac_off on Unix
# Args    :  none
# Returns :  TRUE  if rac_on/rac_off     needs to be set
#            FALSE if rac_on/rac_off not needs to be set
#-------------------------------------------------------------------------------
{
   my $myplatformfamily = s_get_platform_family ();
   $myplatformfamily =~ tr/A-Z/a-z/;

   if ($myplatformfamily eq "unix") {
      return s_isRAC_appropriate ();
   } 
   else {
      return TRUE
   }
}

sub deconfigure_ASM {
   trace ("De-configuring ASM...");

   my $crsctl	       = catfile ($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
   my $owner 	       = $CFG->params('ORACLE_OWNER');
   my $start_exclusive = FALSE;
   my $rc              = FALSE;
   my $status;
   my @dglist;

   # Check if CRS is up
   my $crs_is_up  = check_service ("ohasd", 2);
   my $cluster_is_up = check_service ("cluster", 2);

   if (!$crs_is_up) {
     trace("OHASD is not up. So starting Grid Infrastructure exclusive");
     start_service("crsexcl"); 
     $crs_is_up = TRUE;
   }
   else {
     trace("OHASD is already up");

     if (!$cluster_is_up){
       trace("Starting CSS exclusive");
       $start_exclusive = TRUE;
       my $css_rc = CSS_start_exclusive();
       if ($css_rc != CSS_EXCL_SUCCESS) {
         $start_exclusive = FALSE;
         trace ("CSS failed to enter exclusive mode to de-configure ASM");
       }
     }
   }
   if (($crs_is_up) || ($start_exclusive)) {
      open(QUERY_VOTEDISK, "$crsctl query css votedisk |");
      my @css_votedisk = (<QUERY_VOTEDISK>);
      chomp @css_votedisk;
      close QUERY_VOTEDISK;
      foreach my $votedisk (@css_votedisk)
      {
        trace("Voting disk is : $votedisk");
        if ($votedisk =~ / \[(.+)\]/)
        {
          # E.g., $votedisk is:
          #   '1.  2 282bf2a833f54f02bf4befd002fa90d6(/dev/raw/raw1) [OCRDG]'
          # parse $votedisk to get 'OCRDG'
          push(@dglist, $1);
        }
      }
      trace("Diskgroups found: @dglist");
      if ($dglist[0])
      {
        $CFG->params('ASM_DISK_GROUP', $dglist[0]);
      }

      # delete voting disks on ASM
      if (! CSS_delete_vfs ()) {
 	 trace ("Unable to delete voting files in exclusive mode");
         return FALSE;
      }

      # start ora.asm resource
      if (! start_resource("ora.asm", "-init")) {
         trace ("Unable to start ora.asm resource to deconfigure ASM");
      }
   }
    
   # call asmca -deleteLocalASM to delete diskgroup
   # Do not change the order of these parameters as asmca requires the
   # parameters to be in a specific order or it will fail
   my @runasmca = (catfile ($CFG->ORA_CRS_HOME, "bin", "asmca"),
                   '-silent', '-deleteLocalASM');

   if ($CFG->params('ASM_DISK_GROUP') ){
      my $diskgroup = $CFG->params('ASM_DISK_GROUP');
      if ($diskgroup =~ /\$/) {
         # if diskgroup contains '$', put single-quotes around it
         quoteDiskGroup($diskgroup);
         push @runasmca, '-diskGroups', "'$diskgroup'";
      }
      else {
         push @runasmca, '-diskGroups', $diskgroup;
      }
   }

   if (($CFG->defined_param('ASM_DISKSTRING')) && ($CFG->params('ASM_DISKSTRING'))) {
      my $disktring = $CFG->params('ASM_DISKSTRING');
      push @runasmca, '-diskString', "'$disktring'";
   }

   if ($CFG->params('NODE_NAME_LIST')) {
      push @runasmca, '-nodeList', $CFG->params('NODE_NAME_LIST');
   }

   # ASM_DROP_DISKGROUPS is valid only if the root script is invoked
   # from the deinstall tool, and we pass -reuseDiskgroup to asmca if
   # ASM_DROP_DISKGROUPS is set to false.
   #
   # $CFG->keepdg can be set along with -deconfig
   if ((($CFG->defined_param('ASM_DROP_DISKGROUPS')) &&
        ('false' eq $CFG->params('ASM_DROP_DISKGROUPS'))) || $CFG->keepdg)
   {
     trace("Re-use existing diskgroups");
     push @runasmca, '-reuseDiskgroup';

     removeOCROnDG();
   }

   $ENV{'PARAM_FILE_NAME'} = $CFG->paramfile;
   $status = run_as_user($owner, @runasmca);
   my $ASMCA_log = catdir($CFG->params('ORACLE_BASE'), 'cfgtoollogs', 'asmca');

   if ($status == 0) {
      trace ("de-configuration of ASM ... success");
      trace ("see asmca logs at $ASMCA_log for details");
      $rc = TRUE;
   }
   else {
      error ("de-configuration of ASM ... failed with error $status");
      error ("see asmca logs at $ASMCA_log for details");
   }

   return $rc;
}

# Function: Remove the OCR file in the diskgroup
sub removeOCROnDG()
{
  my $crsctl = catfile($ORA_CRS_HOME, "bin", "crsctl");
  my @statcmd = ($crsctl, "stat", "res", "ora.asm", "-init", "-p");
  
  open STATOUT, "@statcmd |";
  my @output = <STATOUT>;
  close (STATOUT);

  my @txt = grep (/GEN_USR_ORA_INST_NAME/, @output);
  trace("ASM SID attribute: @txt");
  my ($key, $val) = split(/=/, $txt[0]);

  chomp($val);
  trace("ORACLE_SID = $val");
  
  $ENV{'ORACLE_SID'} = $val;
  $ENV{'ORACLE_HOME'} = $ORA_CRS_HOME;

  # remove the OCR file
  my $dgname = $CFG->params('ASM_DISK_GROUP');
  my $dg = "+".$dgname;
  my $asmcmd = catfile($ORA_CRS_HOME, "bin", "asmcmd");
  my $ocrfile = "\%REGISTRY.255.\%";
  my @cmd1 = ($asmcmd, "find", $dg, $ocrfile);
  my @capout1;
  my $rc = run_as_user2($CFG->params('ORACLE_OWNER'), \@capout1, @cmd1);
  
  my $ocrdgfile = $capout1[0];
  chomp($ocrdgfile);
  trace("OCR file on the diskgroup: $ocrdgfile");

  my @cmd2 = ($asmcmd, "rm", "-f",  $ocrdgfile);
  my @capout2;
  $rc = run_as_user2($CFG->params('ORACLE_OWNER'), \@capout2, @cmd2) 
    || die("Failed to remove OCR file $ocrdgfile on ASM using command @cmd2 ... output: @capout2");
}

sub isCRSAlreadyConfigured
#-------------------------------------------------------------------------------
# Function: Check if CRS is already configured on this node
# Args    : none
# Return  : TRUE  if CRS is     already configured
# 	    FALSE if CRS is not already configured
#-------------------------------------------------------------------------------
{
   my $olr_exists      = FALSE;
   my $localOCR_exists = FALSE;
   my $crshome;
   my $crs_exists = s_check_CRSConfig($CFG->HOST, 
				      $CFG->params('ORACLE_OWNER'));

   if ($CFG->platform_family eq "windows") {
      $crshome = s_get_olr_file ("crs_home");
      if ($crshome) {
         $olr_exists = TRUE;
      }
   }
   else {
      if (-e $OLRCONFIG) {
         $crshome = s_get_olr_file ("crs_home");
         if ($crshome) {
            $olr_exists = TRUE;
         }
      }
   }

   if ($CFG->platform_family eq "windows") {
      $localOCR_exists = local_only_config_exists();
   }
   else {
      if (-e $OCRCONFIG) {
         $localOCR_exists = local_only_config_exists();
      }
   }

   if ($olr_exists && $crs_exists) {
      print "CRS is already configured on this node for crshome=$crshome\n";
      print "Cannot configure two CRS instances on the same cluster.\n";
      print "Please deconfigure before proceeding with the " .
            "configuration of new home. \n";
      trace ("CRS is already configured on this node for crshome=$crshome");
      trace ("Cannot configure two CRS instances on the same cluster.");
      trace ("Please deconfigure before proceeding with the " .
             "configuration of new home. ");
      return TRUE;
   }
   elsif ((! $olr_exists) && (! $crs_exists)) {
      trace ("CRS is not yet configured. Hence, will proceed to configure CRS");
      return FALSE;
   }
   elsif ($CFG->UPGRADE && (! $olr_exists)) {
      return FALSE;
   }
   elsif (!$CFG->UPGRADE && (! $olr_exists) && ($localOCR_exists)) {
      return FALSE;
   }
   else {
      my $rootscript = "root.sh";
      my $rootdeconfig = "rootcrs.pl";
      if ($CFG->platform_family eq "windows") {
         $rootscript = "gridconfig.bat";
      }

      if ($CFG->IS_SIHA) {
         $rootdeconfig = "roothas.pl";
      }

      print "Improper Oracle Grid Infrastructure configuration found on this host\n";
      print "Deconfigure the existing cluster configuration before starting\n";
      print "to configure a new Grid Infrastructure \n";
      print "run \'$ORACLE_HOME/crs/install/$rootdeconfig -deconfig\' \n";
      print "to configure existing failed configuration and then rerun $rootscript\n";
      trace ("Improper Oracle Grid Infrastructure configuration found on this host");
      trace ("Deconfigure the existing cluster configuration before starting");
      trace ("to configure a new Grid Infrastructure");
      trace ("run \'$ORACLE_HOME/crs/install/$rootdeconfig -deconfig\' ");
      trace ("to deconfigure existing failed configuration and then rerun $rootscript");
      return TRUE;
   }
}

sub isInterfaceValid
#-------------------------------------------------------------------------------
# Function:  Check if interface is valid
# Args    :  none
#-------------------------------------------------------------------------------
{
   my $networks       = $CFG->params('NETWORKS');
   my $rc             = FALSE;
   my @interface_list = split (/,/, $networks);

   my $pi_count = 0;
   foreach my $interface (@interface_list) {
      if ($interface =~ /\bcluster_interconnect\b/) {
         $pi_count++;
      }
   }

   # if more than 1 interface, at least 1 private interface
   # otherwise, it's invalid
   if (scalar(@interface_list) == 1 ||
      (scalar(@interface_list) > 1 && $pi_count >= 1)) {
      $rc = TRUE;
   } else {
      print  "Invalid interface. There are more than one interface,\n";
      print  "but there is no private interface specified\n";
      trace ("Invalid interface. There are more than one interface,");
      trace ("but there is no private interface specified");
   }

   return $rc;
}

sub configureCvuRpm
#------------------------------------------------------------------------------
# Function:  Install cvuqdisk rpm on Linux 
# Args    :  none
#-------------------------------------------------------------------------------
{
   my $platform_family = s_get_platform_family ();

   if ($platform_family eq "unix")
   {
      s_configureCvuRpm();
   }
}

sub add_CVU
#------------------------------------------------------------------------------
# Function:  Create the cvu resource
# Args    :  none
#-------------------------------------------------------------------------------
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "add cvu");

   if (${status}) {
      trace ("add cvu ... success");
   } else {
      error ("add cvu ... failed");
      return FALSE;
   }

   return TRUE;
}

sub start_CVU
#------------------------------------------------------------------------------
# Function:  Start the cvu resource
# Args    :  none
#-------------------------------------------------------------------------------
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "start cvu");

   if (${status}) {
      trace ("start cvu ... success");
   } else {
      error ("start cvu ... failed");
      return FALSE;
   }

   return TRUE;
}

sub createLocalOnlyOCR
#-------------------------------------------------------------------------------
# Function:  Create local-only OCR
# Args    :  none
#-------------------------------------------------------------------------------
{
   trace ("create Local Only OCR...");

   my $IS_SIHA    = $CFG->IS_SIHA;
   my $owner      = $CFG->params('ORACLE_OWNER');
   my $dba_group  = $CFG->params('ORA_DBA_GROUP');
   my $local_ocr  = catfile ($ORACLE_HOME, "cdata", "localhost", "local.ocr");

   s_createLocalOnlyOCR();
   
   # create local.ocr and set ownergroup
   open (FILEHDL, ">$local_ocr") or die "Unable to open $local_ocr: $!";
   close (FILEHDL);
   s_set_ownergroup ($owner, $dba_group, $local_ocr)
                or die "Can't change ownership on $local_ocr";
   s_set_perms ("0640", $local_ocr)
                or die "Can't set permissions on $local_ocr";


   # validate local.ocr and update ocr.loc
   validate_ocrconfig ($local_ocr, $IS_SIHA)
                or die "Error in validate_ocrconfig";

}
sub checkServiceDown
#---------------------------------------------------------------------
# Function: Check if service is down
# Args    : 1 - service
# Returns : TRUE  if service is down
#           FALSE if service is up
#---------------------------------------------------------------------
{
   my $srv      = $_[0];
   my $crsctl   = catfile ($ORACLE_HOME, "bin", "crsctl");
   my $srv_down = FALSE;
   my $grep_val;
   my $cmd;
   my $node;
   my @cmdout;

   # for OHASD, we need to grep for CRS-4639
   if ($srv eq "ohasd") {
      $grep_val = "4639";
      $cmd      = "$crsctl check has";
   } elsif ($srv eq "cluster") {
      $grep_val = "4639";
      $cmd      = "$crsctl check cluster -n $HOST";
   } elsif ($srv eq "css") {
      $grep_val = "4639|4530";
      $cmd      = "$crsctl check css";
   }
   my @chk = system_cmd_capture($cmd);
   my $rc  = shift @chk;

   if ($grep_val) {
      @cmdout = grep(/$grep_val/, @chk);
   }

   # if scalar(@cmdout) > 0, we found the msg we were looking for
   if (($grep_val && scalar(@cmdout) > 0) ||
       (!$grep_val && $rc == 0)) {
      $srv_down = TRUE;
   }

   return $srv_down;
}

sub is112ASMExists
#-------------------------------------------------------------------------------
# Function:  Check if 11.2 ASM exists
# Args    :  none
# Returns :  TRUE  if     exists
#            FALSE if not exists
#-------------------------------------------------------------------------------
{
   if (check_service("ora.asm", 1)) {
      return TRUE;
   }
   else {
      return FALSE;
   }
}

sub isASMExists
#-------------------------------------------------------------------------------
# Function:  Check if ASM exists
# Args    :  none
# Returns :  TRUE  if     exists
#            FALSE if not exists
#-------------------------------------------------------------------------------
{
   my $crs_home = $CFG->ORA_CRS_HOME;
   my $host     = $CFG->HOST;
   my $crs_stat = catfile ($crs_home, 'bin', 'crs_stat');

   open (CRSSTAT, "$crs_stat |");

   # temporarely using crs_stat to find pre 11.2 ASM
   # grep "ora.$host*asm"
   my @txt = grep /ora.$host.*asm/, <CRSSTAT>;

   close (CRSSTAT);

   if (scalar(@txt) == 0) {
      trace ("check ASM exists done and ASM does not exist");
      return FALSE;
   }

   return TRUE;
}

sub isASMRunning
#-------------------------------------------------------------------------------
# Function:  Check if ASM running
# Args    :  none
# Returns :  TRUE  if     running
#            FALSE if not running
#-------------------------------------------------------------------------------
{
   my $crsctl   = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");
   my @cmd      = ($crsctl, 'status', 'resource', 'ora.asm');
   my $grep_val = 'ONLINE';
   my @out      = system_cmd_capture(@cmd);

   my $rc     = shift @out;
   my @cmdout = grep(/$grep_val/, @out);

   # if scalar(@cmdout) > 0, we found the msg we were looking for
   if (scalar(@cmdout) > 0) {
      trace("ora.asm is online");
      return TRUE;
   }
   else {
      trace("ora.asm is not online");
      return FALSE;
   }
}
      
sub setActiveversion
{
   my $crsctl  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
   my @cmd;

   if ($CFG->force) {
     @cmd = ($crsctl, 'set', 'crs', 'activeversion', '-force');
   } else {
     @cmd = ($crsctl, 'set', 'crs', 'activeversion');
   }

   my $status  = system (@cmd);

   if (0 == $status) {
      trace ("@cmd ... passed");
      sleep(60);  # Wait until CRS changes to new engine
   } else {
      error ("@cmd ... failed");
      return FAILED;
   }

   return SUCCESS;
}



sub queryClusterConfig
{
  
  my @crs_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};
  trace ("Old CRS version=@crs_version");

  if(isVersion112())
  {
    #Invoke the 11.2patchupg module. 
    #TODO - need to see if this will also be applicable for upgrade to 12g 
    performpost112upg();
    return;
  }

  my $new_crs_running = FALSE;
  my $old_crs_running = FALSE;
  #Check if Old Grid Infrastructure is running
  if ($isRerun)
  {
    trace("Trying to check if new CRS stack is partially up");
    $new_crs_running = check_NewCrsStack();
  }

  if (!$new_crs_running)
  {
    $old_crs_running = check_OldCrsStack();
  }
  else {
    $old_crs_running = FALSE;
  }
  
  if (($crs_version[0] eq '10' && $crs_version[1] eq '1') &&
      (! isNodeappsExists())) 
  {
    my $crs_nodevips = $CFG->params('CRS_NODEVIPS');
    if ($crs_nodevips eq "") 
    {
      error ("Set CRS_NODEVIPS parameter in crsconfig_params and rerun rootupgrade.sh");
      error ("The format is  as follows: ");
      error ("     CRS_NODEVIPS='node1-vip/node1-netmask/node1-interface," .
	     "		    node2-vip/node2-netmask/node2-interface'");
      exit 1;
    } else {
      $crs_nodevips    =~ s/'//g; # ' in comment to avoid confusion of editors.
      $crs_nodevips    =~ s/"//g; # remove " on Windows
      $crs_nodevips    =~ s/\/\*/\//g; # handle different interface case

      @crs_nodevip_list_old = split (/\s*,\s*/, $crs_nodevips);
    }
  } 
  else {
    if ($old_crs_running) {  #stack is running
      # Retrieve VIP config info from old stack
      if ($CFG->platform_family eq "windows") {
	@crs_nodevip_list_old = get_OldVipInfoFromOCRDump();
      }
      else {
	@crs_nodevip_list_old = get_OldVipInfo();
      }
    }
    else {
      @crs_nodevip_list_old = get_OldVipInfoFromOCRDump();
    }
  }
 
  if (scalar(@crs_nodevip_list_old) == 0) {
      die("Failed to retrieve old Virtual Interface information");
  } 

  trace("VIP details previous cluster(crs_nodevip_list_old)=@crs_nodevip_list_old");

  trace("new crs running is $new_crs_running");

  if (! $new_crs_running) {
     stop_OldCrsStack();
  }

  if ($CFG->platform_family eq "windows") {
      if (! s_deltOldServ()) {
            die ("Unable to delete old Windows Grid Infrastructure services");
      }
  }

  #Still doesn't handle if one of the deamons is hanging around. But does
  #exit if the old stack is fully up even after the stop. We now do a force stop
  #as part of the stopping the old crs stack to handle a case when just one of the deamons is up

  if ((! $new_crs_running) && (check_OldCrsStack())) {
    die("Failed to stop the old Grid Infrastructure Stack");
  }

  # update ons.config file
  update_ons_config();

  my $vfds = CSS_get_old_VF_string();
  $CFG->VF_DISCOVERY_STRING($vfds);
  if (! $CFG->VF_DISCOVERY_STRING) {
    die("Cannot complete the upgrade without the voting file list");
  }

  trace ("Voting file discovery string:", $CFG->VF_DISCOVERY_STRING);

  
  # Prepare the voting files for upgrade; if there is an error here
  # it should not be considered fatal, since this is only an attempt
  # to correct a case where a voting file's skgfr block 0 was
  # overwritten, which is not likely to have happened
  $CFG->CSS_prep_old_VFs();

}

sub validateNetmask
#-------------------------------------------------------------------------------
# Function: Validate netmask 
# Args    : [0] - old netmask
#	    [1] - network interface 
#	    [1] - new netmask
# Returns : TRUE  if success
#	    FALSE if failed
#	    new netmask
#-------------------------------------------------------------------------------
{
   my $old_netmask 	= $_[0];
   my $netif		= $_[1];
   my $new_netmask_ref 	= $_[2];
   my $oifcfg 		= catfile($CFG->OLD_CRS_HOME, 'bin', 'oifcfg');
   my $success		= TRUE;
   $$new_netmask_ref 	= $old_netmask;

   # if version is 10.1, use new GRID home for oifcfg
   # 10.1 oifcfg does not support -n option.
   my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};

   if ($old_version[0] eq "10" &&
      $old_version[1] eq "1") 
   {
      $oifcfg = catfile($CFG->ORA_CRS_HOME, 'bin', 'oifcfg');
   }

   open OPUT, "$oifcfg iflist -p -n |";

   if ($netif) { #not null
      my @output = grep { /\b$netif\b/i } (<OPUT>);

      trace("OIFCFG_IFLIST output : @output");

      if (scalar(@output) == 0) { #not found
	 trace("Unable to find netmask for network interface=$netif");
   	 $success = FALSE;
      }
      else {
	 if ($DEBUG) { trace("Processing OIFCFG_IFLIST output: $output[0]"); }

         my @netmask_list  = split (/ +/, $output[0]);
         if (scalar(@netmask_list) > 2) {
             $$new_netmask_ref = $netmask_list[-1]; #last entry
             chomp $$new_netmask_ref;
             if ($old_netmask ne $$new_netmask_ref) {
                trace("old_netmask=$old_netmask does NOT match " .
                      "new_netmask=$$new_netmask_ref");
                $success = FALSE;
             }
         } else {
            $success = FALSE;
         }
      }
   } else {
      my @output = grep { /\b$old_netmask\b/i } (<OPUT>);

      trace("Processing OIFCFG IFLIST output: @output");

      if (scalar(@output) == 0) { #not found
	 trace("Unable to find netmask=$old_netmask");
   	 $success = FALSE;
      }
   }

   close OPUT;
   return $success;
}

sub backupOLR
#-------------------------------------------------------------------------------
# Function: Backup OLR 
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
   my $ocrconfig = catfile($CFG->params('ORACLE_HOME'), 'bin', 'ocrconfig');
   my $cmd = "$ocrconfig -local -manualbackup";
   my @out = system_cmd_capture($cmd);
   my $rc = shift @out;

   if ($rc == 0) {
      trace ("Successfully generated OLR backup");
   } else {
      trace ("Failed to generate OLR backup");
      return FAILED;
   }

   return SUCCESS;
}

sub getCurrentNodenameList
#-------------------------------------------------------------------------------
# Function: Get current NODE_NAME_LIST
# Args    : none
# Returns : @node_list - list of nodes
#-------------------------------------------------------------------------------
{
   my @node_list = split (/,/, $CFG->params('NODE_NAME_LIST'));

   return @node_list;
}

sub isNodeappsExists
#-------------------------------------------------------------------------------
# Function: Check if nodeapps exists 
# Args    : none
# Returns : TRUE  if     exists
#	    FALSE if not exists
#-------------------------------------------------------------------------------
{
   @crs_nodevip_list_old = get_OldVipInfoFromOCRDump();
   if (scalar(@crs_nodevip_list_old) == 0) {
      return FALSE;
   }

   return TRUE;
}

sub quoteDiskGroup
#-------------------------------------------------------------------------------
# Function: Check if asm disk group contains '$'
# Args    : diskgroup
# Returns : diskgroup w/ '\' character
#-------------------------------------------------------------------------------
{
   if ($_[0]) {
      $_[0] =~ s/\$/\\\$/g;
   }
}

sub getVerInfo
#-------------------------------------------------------------------------------
# Function: Get the the Version from the String Passed
# Args    : VerString
# Returns : VerInfo 
#-------------------------------------------------------------------------------
{
   my $verstring = $_[0];
   my @verarray = (0, 0, 0, 0, 0);
   my $verinfo;
   trace("Version String passed is: $verstring");
   if ($verstring)
   {
      $verstring =~ m/\[?(\d*)\.(\d*)\.(\d*)\.(\d*)\.(\d*)\]?.*$/;
      @verarray = ($1, $2, $3, $4, $5);
      $verinfo = join('.',@verarray);
      trace("Version Info returned is : $verinfo");
   }
   else
   {
      trace("Null Version String is Passed to getVerInfo");
   }

   return $verinfo;
}

sub getCkptStatus
{
   my $ckptName = $_[0];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-chkckpt', '-name', $ckptName, '-status');

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   # cluutil return 0 err code and errors, if any, on stdout
   if (scalar(grep(/START/, @capout)) > 0)
   {
     trace("The '$ckptName' status is START");
     return CKPTSTART;
   }
   elsif (scalar(grep(/SUCCESS/, @capout)) > 0)
   {
     trace("The '$ckptName' status is SUCCESS");
     return CKPTSUC;
   }
   elsif (scalar(grep(/FAIL/, @capout)) > 0)
   {
     trace("The '$ckptName' status is FAILED");
     return CKPTFAIL;
   }
}

sub isCkptSuccess
{
   my $ckptName = $_[0];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-chkckpt', '-name', $ckptName, '-status');

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   # cluutil return 0 err code and errors, if any, on stdout
   if (scalar(grep(/SUCCESS/, @capout)) > 0)
   {
     trace("The '$ckptName' status is SUCCESS");
     return TRUE;
   }
   else
   {
     trace("The '$ckptName' is either in START/FAILED state");
     return FALSE;
   }
}

sub crf_config_generate
{
  my $mynameEntry = $_[0];
  my $hlist = "";
  my $nodelist = "";
  my $master = "";
  my $replica = "";
  my $masterpub = "";
  my $bdbloc = $_[1]; 
  my $usernm = $_[2];
  my $clustnm = $CFG->params('CLUSTER_NAME');
  my $crfhome = $ORA_CRS_HOME;
  my $configfile;

  (undef, $configfile) = tempfile();

  $hlist=$_[3];
  $hlist =~ s/ //g;
  $nodelist=$_[3];
  chomp($nodelist);
  my @hosts = split(/[,]+/, $nodelist);
  $master = $hosts[0];

  if ($CFG->platform_family eq "windows")
  {
    $usernm = "";
  }
  
  # no replica if less than 2 nodes
  if (scalar(@hosts) >= 2) { $replica = $hosts[1]; }

  if ($mynameEntry eq $master)
  {
    $masterpub=$HOST;
  }

  my $orafile = catfile ($crfhome, "crf", "admin", "crf${HOST}.ora");
  if (!-e $orafile)
  {
    open CONFIG_FILE,'>',$configfile or die $!;
    print CONFIG_FILE  "BDBLOC=$bdbloc\n" ;
    trace ("Bdb location updated again with $bdbloc");
  }
  else
  {
    copy_file($orafile, $configfile);
    trace ("Copied existing orafile");
    open CONFIG_FILE,'>>',$configfile or die $!;
    if (getCHMAttrib("BDBLOC", $configfile) eq "")
    {
      print CONFIG_FILE  "BDBLOC=$bdbloc\n" ;
      trace("BDB location copied if not present");
    }
  } 
 
  if (getCHMAttrib("HOSTS", $configfile) eq "")
  {
    print CONFIG_FILE  "HOSTS=$hlist\n" ;
  }
  if (getCHMAttrib("MASTER", $configfile) eq "")
  {
    print CONFIG_FILE  "MASTER=$master\n" ;
  }
  if (getCHMAttrib("REPLICA", $configfile) eq "")
  {
    print CONFIG_FILE  "REPLICA=$replica\n" ;
  }
  if (getCHMAttrib("MYNAME", $configfile) eq "")
  {
    print CONFIG_FILE  "MYNAME=$mynameEntry\n" ;
  }
  if (getCHMAttrib("MASTERPUB", $configfile) eq "")
  {
    print CONFIG_FILE  "MASTERPUB=$masterpub\n" ;
  }
  if (getCHMAttrib("CLUSTERNAME", $configfile) eq "")
  {
    print CONFIG_FILE  "CLUSTERNAME=$clustnm\n" ;
  }
  if (getCHMAttrib("USERNAME", $configfile) eq "")
  {
    print CONFIG_FILE  "USERNAME=$usernm\n";
  }
  if (getCHMAttrib("CRFHOME", $configfile) eq "")
  {
    print CONFIG_FILE  "CRFHOME=$crfhome\n" ;
  }
  close CONFIG_FILE ;

  return $configfile;
}

sub isCkptexist
#-------------------------------------------------------------------------------
# Function: Verify if checkpoint exist
# Args    : Check point Name
# Returns : boolean
#-------------------------------------------------------------------------------
{
   my $ckptName = $_[0];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-chkckpt', '-name', $ckptName);
   # run as specific user, if requested
   trace( '     ckpt: '.join(' ', @program) );

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   # cluutil return 0 err code and errors, if any, on stdout
   if (scalar(grep(/TRUE/, @capout)) > 0)
   {
     
     return SUCCESS;
   }
   if ((0 != $rc) || (scalar(grep(/FALSE/, @capout))) > 0)
   {
     trace("checkpoint $ckptName does not exist");
     return FAILED;
   }
}

sub syncCkptFile
{
  my $host = tolower_host();
  my $filename = "ckptGridHA_$host.xml";
  my $ckptfile = catfile($CFG->params('ORACLE_BASE'), 'Clusterware', $filename);

  trace("CkptFile: $ckptfile");
  if (-e $ckptfile)
  {
    trace("Sync the checkpoint file '$ckptfile'");
    s_syncToPhysicalDisk($ckptfile); 
  }
}

sub writeCkpt
{
   if (is_dev_env()) {
      return SUCCESS;
   }

   my $ckptName =  $_[0];
   my $ckptState = $_[1];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-writeckpt', '-name', $ckptName, '-state', $ckptState);

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   if (0 != $rc)
   {
     error("Failed to write the checkpoint:'$ckptName' with status:$ckptState.Error code is $rc");
     return FAILED;
   }
   else
   {
     trace("Succeeded in writing the checkpoint:'$ckptName' with status:$ckptState");
     # We don't error out here if sync fails, just try best
     syncCkptFile();
     return SUCCESS;
   }
}

sub writeCkptPropertyFile
{
   my $ckptName      = $_[0];
   my $ckptPropFile  = $_[1];

   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-writeckpt', '-name', $ckptName, '-pfile', $ckptPropFile);

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   if (0 != $rc)
   {
     error("Failed to write contents of pfile :$ckptPropFile for checkpoint:$ckptName. Error code is $rc");
     return FAILED;
   }
   else
   {
     trace("write contents of pfile :$ckptPropFile for checkpoint:$ckptName succeeded");
     return SUCCESS;
   }
}

sub writeCkptProperty
{
   my $ckptName      = $_[0];
   my $ckptPropName  = $_[1];
   my $ckptPropValue = $_[2];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-writeckpt', '-name', $ckptName, '-pname', $ckptPropName, '-pvalue', $ckptPropValue);

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   if (0 != $rc)
   {
     error("Failed to add (property/value):('$ckptPropName/'$ckptPropValue') for checkpoint:$ckptName.Error code is $rc");
     return FAILED;
   }
   else
   {
     trace("Succeeded to add (property/value):('$ckptPropName/'$ckptPropValue') for checkpoint:$ckptName");
     # We don't error out here if sync fails, just try best
     syncCkptFile();
     return SUCCESS;
   }
}

sub getCkptPropertyValue
{
   my $ckptName      = $_[0];
   my $ckptPropName  = $_[1];
   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my $pname;
   my $pval;
   my @out = ();

   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-listckpt', '-name', $ckptName, '-pname', $ckptPropName);

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;
   if (0 != $rc)
   {
     error("Failed to get property value for property:$ckptPropName for checkpoint:$ckptName. Error code is $rc");
     return FAILED;
   }
   else
   {
     trace("Succeeded to getproperty value:@capout");
     @out = grep(/=/, @capout);
     ($pname, $pval) = split(/=/,$out[0]); 
   }

  return $pval;
}

sub isCkptPropertyExists
{
   my $ckptName      = $_[0];
   my $ckptPropName  = $_[1];

   my $crshome  = $CFG->ORA_CRS_HOME;
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $ORACLE_BASE = $CFG->params('ORACLE_BASE');
   my $CKPTTOOL = catfile( $crshome, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-ckpt', '-oraclebase', $ORACLE_BASE, '-chkckpt', '-name', $ckptName, '-pname', $ckptPropName);

   my $env = $ENV{'SRVM_TRACE'};
   undef $SRVM_TRACE;

   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   $ENV{'SRVM_TRACE'} = $env;

   if (scalar(grep(/TRUE/, @capout)) > 0)
   {
     return SUCCESS;
   }

   if ((0 != $rc) || (scalar(grep(/FALSE/, @capout))) > 0)
   {
     trace("The property $ckptPropName for checkpoint $ckptName does not exist");
     return FAILED;
   }
}

sub clean_start_cluster
{
  my $ckpt = "ROOTCRS_STRTSTACK";
  trace("Cleaning Grid Infrastructure stack startup failure");
  my $CRSCTL = catfile ($ORA_CRS_HOME, "bin", "crsctl");
  my $rc;
  my @out = system_cmd_capture($CRSCTL, "stop", "resource", "-all", "-init");
  $rc = shift @out;
  if ($rc !=0) {
     trace("clean_start_cluster output is @out");
  }
  writeCkpt($ckpt, CKPTSTART);
}

sub clean_olr_initial_config
{
   my $ckpt = "ROOTCRS_OLRSETUP";
   trace("Deleting the failed configuration of OLR");
   #olr_initial_config is idempotent. No cleanup needed
   #s_ResetOLR();
   writeCkpt($ckpt, CKPTSTART);
}

sub clean_perform_initial_config
{
   my $crsctl = catfile ($CFG->params('ORACLE_HOME'), "bin", "crsctl");
   my @out = ();
   my $rc;

   trace("Executing crsctl stop cluster");

   @out = system_cmd_capture($crsctl, "stop", "cluster", "-f");
   $rc = shift @out;
   if ($rc !=0) {
     trace("clean_perform_initial_config output is @out");
   }
}


sub clean_configNode
{
   trace("Deleting the failed Node configuration");
   #Need to resue Remove Resources in crsdelete.pm ?
   if (isFirstNodeToStart()) {
      RemoveOC4J();
      RemoveScan();
      #Need to remove asm/asm diskgroup.
      RemoveASM();
   }
   RemoveNodeApps();
}


sub remove_checkpoints
{
   my $host = tolower_host ();
   my @dirs = ('Clusterware', "$host");

   foreach (@dirs) { 
      my $dir = catdir ($CFG->params('ORACLE_BASE'), $_);
      if (-e $dir) {
         trace ("Removing checkpoints dir: $dir");
         rmtree ($dir);
      }
   }
}

sub RemoveNodeApps
#---------------------------------------------------------------------
# Function: Remove nodeapps
# Args    : 0
#---------------------------------------------------------------------
{
   trace ("Removing nodeapps...");

   # check if nodeapps is configured
   my $srvctl = catfile ($CFG->ORA_CRS_HOME, "bin", "srvctl");
   system ("$srvctl config nodeapps");

   if ($CHILD_ERROR != 0) {
      return $SUCCESS;
   }

   my $cmd;
   my $status;
   my $node  = $CFG->HOST;
   my $force = '';

   if ($CFG->force) {
      $force = '-f';
   }

   # stop nodeapps
   $cmd = "$srvctl stop nodeapps -n $node $force";
   $status = system ($cmd);

   if ($status == 0) {
      trace ("$cmd ... success");
   } else {
      trace ("$cmd ... failed");
   }

   # remove nodeapps if lastnode, otherwise remove VIP
   if ($CFG->lastnode) {
      $cmd    = "$srvctl remove nodeapps -y $force";
      $status = system ($cmd);

      if ($status == 0) {
         trace ("$cmd ... success");
      } else {
         trace ("$cmd ... failed");
      }
   }
   else {
      $cmd    = "$srvctl remove vip -i $node -y $force";
      $status = system ($cmd);

      if ($status == 0) {
         trace ("$cmd ... success");
      } else {
         trace ("$cmd ... failed");
      }
   }
}

sub RemoveScan
#-------------------------------------------------------------------------------
# Function: Remove scan and scan listener
# Args    : 0
#-------------------------------------------------------------------------------
{
   trace("Removing scan....");

   my $owner = $CFG->params('ORACLE_OWNER');
   my (@cmd, $force);

   if ($CFG->force) {
      $force = '-f';
   }

   my $srvctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl');

   # stop/remove scan listener
   @cmd = ($srvctl, 'stop', 'scan_listener', "$force");
   trace ("@cmd");
   run_as_user($owner, @cmd);

   @cmd = ($srvctl, 'remove', 'scan_listener', '-y', "$force");
   trace ("@cmd");
   run_as_user($owner, @cmd);

   # stop/remove scan vip
   @cmd = ($srvctl, 'stop', 'scan', "$force");
   trace ("@cmd");
   system (@cmd);

   # remove scan vip
   @cmd = ($srvctl, 'remove', 'scan', '-y', "$force");
   trace ("@cmd");
   system(@cmd);
}

sub RemoveASM
{
    my $crsctl = catfile ($CFG->params('ORACLE_HOME'), "bin", "crsctl");
    my $srvctl = catfile ($CFG->ORA_CRS_HOME, "bin", "srvctl");
    my @out = ();

   @out = system_cmd_capture($crsctl, "delete", "resource", "ora.asm", "-f");
}

sub RemoveOC4J
{
   my $crsctl = catfile ($CFG->params('ORACLE_HOME'), "bin", "crsctl");
   my @out = ();

   @out = system_cmd_capture($crsctl, "delete", "resource", "ora.oc4j", "-f");
}

sub crf_kill_for_sure
{
  kill(15, $_[0]);

  # if that didn't work, use force
  if (kill(0, $_[0]))
  {
    kill(9, $_[0]);
  }
}

# delete the bdb files in bdbloc.
sub crf_delete_bdb
{
  my $bdbloc = $_[0];

  if ($bdbloc ne "" and -d $bdbloc)
  {
    # remove files which we created.
    chdir $bdbloc or print "ERROR:Cannot chdir to $bdbloc, invalid BDB path.";
    opendir(DIR, "$bdbloc") || print "Error in opening dir $bdbloc\n";
    trace("Removing contents of CHM/OS repository directory $bdbloc\n");
    my $crfbdbfile;
    my @ldbfiles = grep(/\.ldb$/,readdir(DIR));
    foreach $crfbdbfile (@ldbfiles)
    {
      unlink ($crfbdbfile);
    }

    # database files
    rewinddir (DIR);
    my @bdbfiles = grep(/\.bdb$/,readdir(DIR));
    foreach $crfbdbfile (@bdbfiles)
    {
      unlink ($crfbdbfile);
    }

    # env files
    rewinddir (DIR);
    my @dbfiles = grep(/__db.*$/,readdir(DIR));
    foreach $crfbdbfile (@dbfiles)
    {
      unlink ($crfbdbfile);
    }

    # archive log
    rewinddir (DIR);
    my @bdblogfiles = grep(/log.*$/,readdir(DIR));
    foreach $crfbdbfile (@bdblogfiles)
    {
      unlink ($crfbdbfile);
    }
    closedir (DIR);
  }
}

sub isCRFSupported
{
  my $osysmond         = catfile ($CFG->ORA_CRS_HOME, "bin", "osysmond");
  my $osysmond_exe     = catfile ($CFG->ORA_CRS_HOME, "bin", "osysmond.exe");

  if ($^O =~ /win/i)
  {
    if (! (-e $osysmond_exe))
    {
       trace ("CHM/OS osysmond not found");
       return FALSE;
    }
    return TRUE;
  }
  elsif ($^O =~ /linux|solaris|aix/i)
  {
     my $mach =`uname -m`;
     chomp($mach);

     if (! (-e $osysmond))
     {
        trace ("CHM/OS osysmond not found");
        return FALSE;
     }
     elsif ($mach eq "s390x" || $mach eq "s390")
     {
  	trace ("CHM/OS not supported on this platform");
        return FALSE;
     }

     return TRUE;
  }

  trace ("CHM/OS is not supported on this platform");
  return FALSE;
}

# delete a node from the CRF install
sub crf_do_delete
{
  # shutdown the sysmond, ologgerd, oproxyd if they are running

  my $cmd;
  my $instdir;
  my $defpath;
  my $rootpath;
  my $configfile;
  my $line;
  my $bdbloc;
  my $admindir;
  my $admin;
  my $runpth;
  trace("Check and delete older CHM/OS installation");

  if ($CFG->platform_family eq "windows")
  {
    $instdir = "C:\\Program Files\\oracrf";
    $defpath = "$ENV{SYSTEMROOT}"."\\system32\\";
    $rootpath = "$ENV{SYSTEMROOT}";
  }
  else
  {
    $instdir = "/usr/lib/oracrf";
    $defpath = "/usr/bin/";
    $rootpath = "/";
  }

  my $instfile = catfile("$instdir","install","installed");
  if (! -f $instfile)
  {
    trace("INFO: The OS Tool is not installed at $instdir.");
  }
  else
  {
    trace("Older CHM/OS installation detected ... Stopping and removing it ... ");
    $cmd = "oclumon"." ". "stop"." ". "all";
    my $crfhome = $instdir;
    $admindir = catfile("$instdir", "crf");
    if ( -d $admindir)
    {
      $configfile = catfile("$instdir", "crf", "admin", "crf${HOST}.ora");
      $runpth = catfile("$crfhome", "crf", "admin", "run");
      $admin = 'crf';
    }
    else
    {
      $configfile = catfile("$instdir", "admin", "crf${HOST}.ora");
      $runpth = catfile("$crfhome", "admin", "run");
      $admin = 'admin';
    }

    system("$cmd");
    sleep(5);

    # read config file to find older BDB loc
    $bdbloc = getCHMAttrib("BDBLOC", $configfile);
    if ($bdbloc eq "default")
    {
      $bdbloc = catfile($instdir, "crf", "db", "$CFG->HOST");
    }
    
    my $pidf=catfile("$runpth","crfmond","s${HOST}.pid");
    if (-f $pidf)
    {
      open(PID_FILE, $pidf);
      while (<PID_FILE>)
      {
        crf_kill_for_sure($_);
      }
      close(PID_FILE);
      unlink($pidf);
    }
    my $dir;
    $dir = catfile("$runpth", "crfmond");
    rmdir("$dir");

    # ologgerd now
    $pidf=catfile("$runpth","crflogd","l${HOST}.pid");
    if (-f $pidf)
    {
      open(PID_FILE, $pidf);
      while (<PID_FILE>)
      {
        crf_kill_for_sure($_);
      }
      close(PID_FILE);
      unlink($pidf);
    }
    $dir = catfile("$runpth", "crflogd");
    rmdir("$dir");

    # proxy next
    $pidf=catfile("$runpth","crfproxy","p${HOST}.pid");
    if (-f $pidf)
    {
      open(PID_FILE, $pidf);
      while (<PID_FILE>)
      {
        crf_kill_for_sure($_);

        # give some time to oproxy to react.
        sleep 2;
      }
      close(PID_FILE);
      unlink($pidf);
    }
    $dir = catfile("$runpth", "crfproxy");
    rmdir("$dir");

    # ask crfcheck to shutdown cleanly
    $pidf=catfile("$crfhome","log","${HOST}","crfcheck/crfcheck.lck");
    if (-f $pidf)
    {
      open(PID_FILE, $pidf);
      while (<PID_FILE>)
      {
        kill 15, $_;
      }
      close(PID_FILE);
    }

    my $rootpath;
    s_crf_remove_itab();

    # remove the tree
    trace("Removing install dirs from $instdir ...\n");
    my $filed;
    my $file;
    foreach $filed ('bin', 'lib', $admin, 'jlib', 'mesg', 'log',
                     'install', 'jdk', 'db')
    {
      $file = catfile("$instdir", "$filed"); 
      rmtree("$file", 0, 0);
    }

    # delete old bdb files.
    trace("Deleting older CHM/OS repository files at: ", $bdbloc);
    crf_delete_bdb($bdbloc);

    unlink("$defpath"."crfgui");
    unlink("$defpath"."oclumon");
    unlink("$defpath"."ologdbg");

    # change dir to a safer place
    chdir $rootpath;
    trace("Removing CRFHOME path $instdir...\n");
    rmdir $instdir;
    trace("Old CHM/OS install removal operation completed.\n");
  }
}

sub add_ons
{
   my $run_as_owner = TRUE;
   my $status       = srvctl($run_as_owner, 'add ons');

   if ($status) {
      trace ("srvctl add ons ... success");
   }
   else {
      error ("srvctl add ons ... failed");
      return FALSE;
   }

   return TRUE;
}

sub setNetworkInterface
{
  my $success = TRUE;
  my $oifcfg = catfile($CFG->params('ORACLE_HOME'), 'bin', 'oifcfg');
  my $networks = $CFG->params('NETWORKS');
  $networks =~ s/,/ /g;
  my $ohown = $CFG->params('ORACLE_OWNER');
  my @cmd = ($oifcfg, 'setif -global', $networks);
  my $rc;
  my @out;
 
  $rc = run_as_user2($ohown, \@out, @cmd);

  if ($rc == 0)
  {
     trace("$oifcfg setif -global $networks successful");
  } else {
     error("$oifcfg setif -global $networks failed rc = $rc");
     $success = FALSE;
  }

  return $success;

}



sub performpost112upg
{

   my $oldcrshome;
   my $oldolrlocation;
   my $crshome = $CFG->ORA_CRS_HOME;
   my $olrlocation = $CFG->OLR_LOCATION;
   my $host = $CFG->HOST;
   my $success;
   my @old_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   my $oldcrsver  = join('.',@old_ver);
   my $crsctl; 
   my $old_crs_running;
   my $asmcainitflag;

   #Find old home
   $oldcrshome = s_get_olr_file ("crs_home");

   if ($oldcrshome eq $crshome) {
      trace("11.2 upgrade actions are already performed");
      return;
   }

   #Check if Old Grid Infrastructure is running
   if (! $CFG->IS_SIHA) {
      $old_crs_running = check_OldCrsStack();
   }
 
   $oldolrlocation = s_get_olr_file ("olrconfig_loc");

   #check validity of olr file before upgrade. Part fix for bug 9466585
   if (checkOLR () != SUCCESS) {
      error("The OLR from version $oldcrsver is missing or unusable for upgrade");
      error("Correct this issue and rerun the command.");
      error("If you cannot correct this, contact Oracle Support Services.");
      die("Invalid OLR during upgrade");
   }
  
   trace("The old crs home = $oldcrshome");
   trace("The old OLR location = $oldolrlocation");

   if ((! $CFG->IS_SIHA) && (! $old_crs_running))
   {
     trace("Make sure the older stack is completely down");
     stopClusterware($oldcrshome, "crs") ||
       die("Failed to stop the lower version of Grid Infrastructure stack");
   }

   if ($CFG->IS_SIHA) {
      if (!perform_sihaASM_upg())
      {
        die("ASMCA failed to upgrade ASM for Oracle Restart configuration");
      }
   }
   
   if ((isRolling()) && (!$CFG->IS_SIHA) && (isFirstNodeToUpgrade ())) {

         if (! $old_crs_running) {
            trace("Old stack is down. Attempting to start for rolling migration");
            start_OldCrsStack();
            $old_crs_running = SUCCESS;
         }

         # A sleep of 31 seconds before invoking asmca is a temporary fix
         # for bug 12721330, and only in 11.2.0.3.0
         sleep(31);

         trace("start ASM rolling upgrade on first node");
         if (!perform_rollingASM_upg($FIRSTNODE))
         {
           die("ASMCA failed to upgrade ASM in a rolling fashion for Oracle Grid Infrastructure configuration");
         }
   }

   if ((! isRolling()) && (!$CFG->IS_SIHA) && (isFirstNodeToUpgrade ())) {
         $asmcainitflag = TRUE;
         if (!perform_nonrollingASM_upg($FIRSTNODE, $asmcainitflag))
         {
           die("ASMCA failed to upgrade ASM in a nonrolling fashion for Oracle Grid Infrastructure configuration");
         }
   }

   if ((! $CFG->IS_SIHA) && $old_crs_running) {
     # stop the stack if it is already running.
     if (! stopClusterware($oldcrshome, "crs")) {
        die("Failed to stop the lower version of Grid Infrastructure stack");
     }
   }

   if ($CFG->IS_SIHA) {
      # Disable ora.cssd
      my $crsctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
      my @cmd = ($crsctl, 'modify', 'res', 'ora.cssd', '-attr',
                  "\"ENABLED=0\"", '-init');
      my $status = system_cmd(@cmd);
      if (0 != $status)
      {
        die("Failed to disable CSSD startup");
      } 

      if ($CFG->platform_family ne "windows") {
        # Disable ora.diskmon
        @cmd = ($crsctl, 'modify', 'res', 'ora.diskmon', '-attr',
                 "\"ENABLED=0\"", '-init');
        $status = system_cmd(@cmd);
        if (0 != $status)
        {
          die("Failed to disable DISKMON startup");
        }
      } 
      if (! stopOracleRestart()) {
         die("Failed to stop old Oracle Restart stack");
      }
   }

   if ($CFG->platform_family eq "windows") {
      s_unregister_service("ohasd");
      if (! $CFG->IS_SIHA) {
         s_upgrade_services();
      }
      if ((! is_dev_env()) && (isRolling())) {
         if (! upgrade_asm_service()) {
            die("ASMCA failed to upgrade ASM service in a rolling fashion for Oracle Grid Infrastructure configuration");
         }
      }
   }

   #take old olr.loc backup
   if ($CFG->platform_family eq "windows") {
      my $olr_loc     = $OLRLOC;
      my $olr_loc_bkp = $OLRLOC . '.bkp';
      trace("backing up olr registry");
      s_copyRegKey($olr_loc, $olr_loc_bkp);
   }
   else {
      my $oldolr = catfile ($OCRCONFIGDIR, 'olr.loc');
      my $oldolrbkp = catfile ($OCRCONFIGDIR, 'olr.loc.bkp');
      trace("backing up old olr.loc file $oldolr");
      copy_file($oldolr, $oldolrbkp);
   }

   if (! $CFG->IS_SIHA)
   {
      if (isFirstNodeToUpgrade ())
      {
         #copy necessary files to new home
          copyfiles($oldcrshome, $crshome);

         #copy olr file to new home and set permissions
         copy_file($oldolrlocation, $olrlocation);
      }
      elsif (!isHomeShared())
      {
         #copy gpnp global files to new home
         copyfiles($oldcrshome, $crshome);

         #copy olr file to new home and set permissions
         copy_file($oldolrlocation, $olrlocation);
      }

      copy_gpnpnodefiles($oldcrshome, $crshome);
   } else {
          #copy necessary files to new home
          copyfiles($oldcrshome, $crshome);
          copy_gpnpnodefiles($oldcrshome, $crshome);

          #copy olr file to new home and set permissions
          copy_file($oldolrlocation, $olrlocation);
   }

   if ($CFG->IS_SIHA) {
     trace("setting olr ownership to $ORACLE_OWNER after upgrade");
     s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $olrlocation);
   } else {
     trace("setting olr ownership to superuser after upgrade");
     s_set_ownergroup ($CFG->SUPERUSER, $ORA_DBA_GROUP, $olrlocation);
   }

   s_set_perms ("0600", $olrlocation);

   #For handling CHM Upgrade. Bug 11852891. 
   if (!($CFG->IS_SIHA) && !(isVersionLT11202())) {
     if (!perform_CHM_upgrade()) {
       trace("Couldn't perform CHM upgrade");
       die("Failed to upgrade Cluster Health Monitor (CHM) service for Oracle Clusterware configuration.");
     }
   }

}

sub upgrade_asm_service
{
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;

  # Do not change the order of these parameters as asmca requires the
  # parameters to be in a specific order or it will fail

  my @asmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"),
                  '-silent', '-upgradeLocalASM', '-winSvc');

  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @asmca");
  my $status = run_as_user($CFG->params('ORACLE_OWNER'), @asmca);

  if ($status != 0) {
    $success = FALSE;
    error("ASMCA: Failed to upgrade ASM service");
  }

  return $success;
}

sub copyfiles
{
   my $sourcedir = $_[0];
   my $destdir   = $_[1];
   my $host = tolower_host();

   #copy gpnp, network/admin/, cdata and opmn/conf files
   if (! $CFG->IS_SIHA) {
      copy_gpnpglobalfiles($sourcedir, $destdir); 
      copydir(catdir($sourcedir,'network','admin'), catdir($destdir,'network','admin'));
      copydir(catdir($sourcedir,'cdata'), catdir($destdir,'cdata'));

      # for opmn/conf, we only need to copy ons.config file
      my $sourcefile = catfile ($sourcedir, 'opmn', 'conf', 'ons.config');
      my $destfile   = catfile ($destdir, 'opmn', 'conf', 'ons.config');
      copy_file($sourcefile, $destfile, $ORACLE_OWNER, $ORA_DBA_GROUP);
   } 
   else {
      copydir(catdir($sourcedir,'network','admin'), catdir($destdir,'network','admin'));
   }
}

sub copy_gpnpfiles
{
   my $sourcedir = $_[0];
   my $destdir   = $_[1];

   copy_gpnpglobalfiles($sourcedir, $destdir);
   copy_gpnpnodefiles($sourcedir, $destdir);
}

sub copy_gpnpglobalfiles
{
   my $sourcedir = $_[0];
   my $destdir   = $_[1];

   copydir(catdir($sourcedir,'gpnp','profiles'), catdir($destdir,'gpnp','profiles'));
   copydir(catdir($sourcedir,'gpnp','wallets'), catdir($destdir,'gpnp','wallets'));
}

sub copy_gpnpnodefiles
{
   my $sourcedir = $_[0];
   my $destdir   = $_[1];
   my $host = tolower_host();

   copydir(catdir($sourcedir,'gpnp',$host), catdir($destdir,'gpnp',$host));
}

sub copydir
{
   my $sourcedir = $_[0];
   my $destdir   = $_[1];
   my ($filepath, $filename, $destfilepath, $dirpath);
   my @filelist;
   trace("source dir=$sourcedir");
   trace("dest   dir=$destdir");

   find(sub { push @filelist, $File::Find::name }, $sourcedir);

   if ($CFG->platform_family eq "windows") {
      # escape char '\' on Windows
      $destdir =~ s!\\!\\\\!g;
      $sourcedir =~ s!\\!\\\\\\\\!g;
   }

   foreach $filepath (@filelist) {
     if ( -f $filepath) {
        $filename = basename($filepath);
        # Do not copy OCR backup files
        if ($filename =~ /\.ocr$/)
        {
          trace("Skipping the file '$filename'");
          next; 
        }
 
        $dirpath = dirname($filepath);

        if ($CFG->platform_family eq "windows") {
           # escape char '\' on Windows
           $dirpath =~ s!\\!\\\\!g;
        }

        $dirpath =~ s/$sourcedir/$destdir/g;

        if (! -e $dirpath) {
          trace("creating directory $dirpath");
           mkpath( $dirpath);
           s_set_ownergroup($ORACLE_OWNER, $ORA_DBA_GROUP, $dirpath);
        }

        $destfilepath = catfile($dirpath, $filename);
        trace ("copying file=$filename");
        trace ("source file path=$filepath");
        trace ("dest file path=$destfilepath");
        copy_file($filepath, $destfilepath,$ORACLE_OWNER, $ORA_DBA_GROUP);
     }
   }
}

sub isVersion10
{
   my $is10ver;
   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if ($oldcrs_ver[0] eq '10') {
       $is10ver = TRUE;
   } else {
       $is10ver = FALSE;
   }
   return $is10ver;
}

sub isVersion111
{
   my $is111ver;
   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if ($oldcrs_ver[0] eq '11' && $oldcrs_ver[1] eq '1') {
       $is111ver = TRUE;
   } else {
       $is111ver = FALSE;
   }
   return $is111ver;
}

sub isVersion112
{
   my $is112ver;
   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if ($oldcrs_ver[0] eq '11' && $oldcrs_ver[1] eq '2') {
       $is112ver = TRUE;
   } else {
       $is112ver = FALSE;
   }
   return $is112ver;
}

sub isVersionLT11202
{
   my $is112ver;
   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if (int($oldcrs_ver[0]) <= int('10')) {
      $is112ver = TRUE;
   }elsif (int($oldcrs_ver[0]) == int('11') && int($oldcrs_ver[1]) <= int('2') 
           && int($oldcrs_ver[3]) < int('2')) 
   {
       $is112ver = TRUE;
   } else {
       $is112ver = FALSE;
       if ($CFG->platform_family eq "windows" || 
	   $^O =~ /aix/i) {
           if (int($oldcrs_ver[3]) < int('3')) {
               $is112ver = TRUE;
           }
       }
   }
   return $is112ver;
}

sub isVersionLT11203
{
  my $res;
 
  if (ref($_[0]) ne 'ARRAY')
  {
    error("Invalid argument");
    return FALSE;
  }

  my @ver = @{$_[0]};
  if ($ver[0] < '11')
  {
    $res = TRUE;
  }
  elsif (($ver[0] eq '11') && ($ver[1] < '2'))
  {
    $res = TRUE;
  }
  elsif (($ver[0] eq '11') && ($ver[1] eq '2') &&
          ($ver[2] eq '0') && ($ver[3] < '3'))
  {
    $res = TRUE;
  }
  else
  {
    $res = FALSE;
  }

  return $res;
}

sub isCurrentVersion112
{
  my @crsver = get_crs_version($CFG->ORA_CRS_HOME);
  
  if ($crsver[0] eq '11' && $crsver[1] eq '2')
  {
    return TRUE;
  }
  else
  {
    return FALSE;
  }  
}

sub upgrademodel
{
   my $nodeinfo = $_[0];
   my $host = tolower_host ();
   my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   my $srcver = join('.',@old_version);
   my $destver = getcursoftversion($host);
   my $success = TRUE;
   my $run_as_owner = FALSE;

   chomp $srcver;

   my $status = srvctl($run_as_owner,
                          "upgrade model  -s $srcver -d $destver -p $nodeinfo");
      if (${status}) {
         trace ("srvctl upgrade model -$nodeinfo  ... passed");
      } else {
         error ("srvctl upgrade model -$nodeinfo ... failed");
         $success = FALSE;
      }
   return $success;
}

# crsctl query software version from a 11.2 home against pre11.2 stack hangs. 
# The following logic is incorporated to get the pre11.2 software version. 
# 1. use the new home crsctl if the new home stack is running.
# 2. If the new home is not up, check if the old home stack is running 
# and use the crsctl from the old home.
# 3. If the new home stack and the old home stack is not running, first try 
# with new home crsctl and then from the old home crsctl.This ordering is because
# the ocr key location to get this information is different between pre11.2 and 11.2
# and we don't want to use the old crsctl with the 11.2 configuration in place.

 
sub getcursoftverpre112
{
  my $host = $_[0];
  my $cmd;
  my @out;
  my $rc;
  my $verstring;
  my $crsctl;
  my $oldcrshome = $CFG->OLD_CRS_HOME;
  my $newcrshome = $CFG->params('ORACLE_HOME');
  my $old_crs_up;
  my $new_crs_up; 
 
  $new_crs_up = check_NewCrsStack();

  if ($new_crs_up) {
     $crsctl = catfile ($newcrshome, "bin", "crsctl");  
     $cmd = "$crsctl query crs softwareversion $host";
     @out = system_cmd_capture("$cmd");
     $rc = shift @out;  
  } else {
     $old_crs_up = check_OldCrsStack();
     if ($old_crs_up) {
       $crsctl = catfile ($oldcrshome, "bin", "crsctl");
       $cmd = "$crsctl query crs softwareversion $host";
       @out = system_cmd_capture("$cmd");
       $rc = shift @out;
     } else {
       $crsctl = catfile ($newcrshome, "bin", "crsctl");
       $cmd = "$crsctl query crs softwareversion $host";
       @out = system_cmd_capture("$cmd");
       $rc = shift @out;

       if ($rc == 0) {
         $verstring = getVerInfo($out[0]);
         trace( "Got CRS softwareversion for $host: ".join('.', $verstring) );
       } else { 
         $crsctl = catfile ($oldcrshome, "bin", "crsctl");
         $cmd = "$crsctl query crs softwareversion $host";
         @out = system_cmd_capture("$cmd");
         $rc = shift @out;
           
       }
     }
  }

    if ($rc == 0) {
      $verstring = getVerInfo($out[0]);
      trace( "Got CRS softwareversion for $host: ".join('.', $verstring) );
    } else {
      trace("Failed to get pre11.2 software version for $host");
    }

    return $verstring;

}

sub getcursoftversion
{
    my $host = $_[0];
    my $crshome = $_[1];
    my $cmd;
    my @out;
    my $rc;
    my $verstring;
    my $crshomeArg = 0;

    if (!defined $crshome)
    {
      $crshome = $CFG->params('ORACLE_HOME');
    }
    else
    {
      $crshomeArg = 1;
    }

    if ((0 == $crshomeArg) && (! isVersion112())) {
       $verstring = getcursoftverpre112($host); 
    }
    else {

    if (isOCRonASM()) {
      trace("setting ORAASM_UPGRADE to 1");
      $ENV{'ORAASM_UPGRADE'} = "1";
    }
    
    trace("Getting current software version on $host");
    trace("CRS home is $crshome");
    my $crsctl = catfile ($crshome, "bin", "crsctl");
    if ($CFG->IS_SIHA) {
      $cmd = "$crsctl query has softwareversion";
    } else {
	$cmd = "$crsctl query crs softwareversion $host";
    }

    @out = system_cmd_capture("$cmd");
    $rc = shift @out;

    if ($rc == 0) {
      $verstring = getVerInfo($out[0]);
      trace( "Got CRS softwareversion for $host: ".join('.', $verstring) );
    }
    else {
      trace ("$cmd ... failed rc=$rc with message:\n @out \n");
      trace ("Getting software version for $host from OCR");
      $verstring = get_softVersionfromOCR($host);
      trace("software version retrieved from OCR is $verstring");
    }
    
    }

    chomp $verstring;
    trace ("The software version on $host is $verstring");
    return $verstring;
}

sub perform_CHM_upgrade {
  my $crfhome = $_[0];
  my $patching = 1;
  my $oldcrfhome;
  my $oldorafile;
  my $oldcfgfile;
  my $bdblocbkp;
  my $oldorafilebkp;
  my $oldcfgfilebkp;
  my $neworafile;
  my $newcfgfile;
  my $oldbdbloc;
  my $oldbdbsize;
  my $HOST = tolower_host();

  if ($crfhome eq "")
  {
    $crfhome = $CFG->ORA_CRS_HOME;
    $patching = 0; #It says it is rolling upgrade
  }

  #Find old home
  $oldcrfhome = s_get_olr_file ("crs_home");

  if ($oldcrfhome eq $crfhome) {
     trace("11.2 CHM upgrade actions are already performed");
     return TRUE;
  }
  trace("The old crf home = $oldcrfhome");

  #Get the bdb location from the old environment
  $oldbdbloc = getCHMAttrib("BDBLOC");
  if ($oldbdbloc eq "default")
  {
    $oldbdbloc = catfile ($oldcrfhome, "crf", "db", "${HOST}");
  }
  $oldbdbsize = getCHMAttrib("BDBSIZE");

  #  Get the old ora file and cfg file full path
  $oldorafile = catfile ($oldcrfhome, "crf", "admin", "crf${HOST}.ora");
  if (!-e $oldorafile)
  {
    trace("Info: No old ora file present at ", $oldorafile); 
    return TRUE;
  }

  $oldcfgfile = catfile ($oldcrfhome, "crf", "admin", "crf${HOST}.cfg");
  if (!-e $oldcfgfile)
  {
    trace("Info: No old cfg file present at ", $oldcfgfile); 
    return TRUE;
  }

  # Check if the new admin directory path exists.
  my $newadmindir = catdir ($crfhome, "crf", "admin");
  if (!-d $newadmindir)
  {
    trace("Info: New admin directory doesn't exist at", $newadmindir); 
    die("Failed to find new administrative directory ($newadmindir) of Cluster Health Monitor (CHM).");
  }
 
  #  Get the new ora file and cfg file full path
  $neworafile = catfile ($crfhome, "crf", "admin", "crf${HOST}.ora");
  $oldorafilebkp = "$neworafile"."bkp";
  $newcfgfile = catfile ($crfhome, "crf", "admin", "crf${HOST}.cfg");
  $oldcfgfilebkp = "$newcfgfile"."bkp";

  # Check if it is a rolling upgrade or patching
  if (((isRolling()) && (!$CFG->IS_SIHA)) || $patching == 1)
  {
    # Create backup of the old configuration file to hande in case of failure.
    if (!copy_file($oldcfgfile, $oldcfgfilebkp))
    {
      trace("Info: Failed to copy from $oldcfgfile to $oldcfgfilebkp");
    }

    #copy the old configuration file.
    if (!copy_file($oldcfgfile, $newcfgfile))
    {
      trace("Info: Failed to copy from $oldcfgfile to $newcfgfile");
    }

    # Create a bakup of ora file to revert back in case of failure.
    if (!copy_file($oldorafile, $oldorafilebkp))
    {
      trace("Info: Failed to copy from $oldorafile to $oldorafilebkp");
    }

    # Just retain BDB Location and BDB Size from the old ora file.
    open (FP, ">$neworafile") || return TRUE;

    if (index($oldbdbloc, $oldcrfhome) == -1)
    {
      print FP "BDBLOC=$oldbdbloc\n";
    }
    elsif ($patching == 1)
    {
      #Create a backup of oldbdb directory to handle in case of failure.
      $bdblocbkp = catdir("$crfhome", "crf", "db", "$HOST");
      movedir($oldbdbloc, $bdblocbkp);
    }
      
    # Don't move the bdb files in case of patching.
    if ($patching == 0)
    {
      my $hostbk = "$HOST"."bkp";
      #Create a backup of oldbdb directory to handle in case of failure.
      $bdblocbkp = catdir("$crfhome", "crf", "db", "$hostbk");
      movedir($oldbdbloc, $bdblocbkp);
    }
    print FP "BDBSIZE=$oldbdbsize\n";
    close (FP);
  }
  
  return TRUE;
}

sub movedir {
  my $srcdir = $_[0];
  my $destdir = $_[1];

  if (! -d $destdir) {
    mkdir($destdir);
  }
  
  opendir(DIR, $srcdir) or trace "Can't open $srcdir: $!";
  my @files = grep {!/^\.+$/ } readdir(DIR);
  
  foreach my $file (@files) {
    my $old = "$srcdir/$file";
    move($old, $destdir) or trace "Move $old -> $destdir failed: $!";
  }
  close(DIR);
}

sub perform_rollingASM_upg {
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $status;
  my $nodeinfo = $_[0];
  my @runasmca;

  # Do not change the order of these parameters as asmca requires the
  # parameters to be in a specific order or it will fail

  if ($nodeinfo eq $FIRSTNODE) {
     trace("Starting ASM rolling upgrade");
     @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"), 
		  '-silent', '-upgradeLocalASM', '-firstNode');
  } else {
     trace("End ASM rolling upgrade");
     @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"), 
		  '-silent', '-upgradeLocalASM', '-lastNode', $CFG->OLD_CRS_HOME);
  }

  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca");
  $status = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca);

  if ($status != 0) {
    $success = FALSE;
    error("ASM rolling upgrade action failed, see logs for details");
  }

  return $success;
}

sub perform_nonrollingASM_upg {
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $status;
  my $nodeinfo = $_[0];
  my $initflag = $_[1];
  my @runasmca;

  # Do not change the order of these parameters as asmca requires the
  # parameters to be in a specific order or it will fail

  if (($nodeinfo eq $FIRSTNODE) && ($initflag)) {
     trace("Starting ASM nonrolling upgrade with -init");
     @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"),
                  '-silent', '-upgradeLocalASM', '-init');
  }  elsif ($nodeinfo eq $FIRSTNODE) {
     trace("Invoking  asmca non-rolling command after first node Grid Infrastructure upgrade");
     @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"),
                  '-silent', '-upgradeLocalASM', '-firstNode', '-nonRolling' , $CFG->OLD_CRS_HOME);

  }  else {
     trace("End ASM non-rolling upgrade");
     @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"),
                  '-silent', '-upgradeLocalASM', '-lastNode', '-nonRolling' , $CFG->OLD_CRS_HOME);
  }

  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca");
  $status = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca);

  if ($status != 0) {
    $success = FALSE;
    error("ASM rolling upgrade action failed, see logs for details");
  }

  return $success;
}


sub perform_sihaASM_upg {
  my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME;
  my $success = TRUE;
  my $status;
  my @runasmca;

  trace("Start ASM  upgrade for Oracle Restart");
  @runasmca = (catfile ($ORA_CRS_HOME, "bin", "asmca"),
                  '-silent', '-upgradeLocalASM');

  trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca");
  $status = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca);

  if ($status != 0) {
    $success = FALSE;
    error("ASM upgrade failed, see logs for details");
  }

  return $success;
}



sub ModActionScript 
#-------------------------------------------------------------------------------
# Function:  Modify all db resources ACTION_SCRIPT and set it to %crs_home%
# Args    :  none
#-------------------------------------------------------------------------------
{
   trace("Modify ACTION_SCRIPT");
   # get db resources
   my $crsctl = catfile ($CFG->ORA_CRS_HOME, 'bin', 'crsctl');
   my @cmd    = ($crsctl, 'stat', 'res', '-w',
                 '"((TYPE = application) AND (NAME en .db) AND (NAME st ora.))"');
   my @output = system_cmd_capture(@cmd);
   my $rc     = shift @output;
   my @resname = grep(/NAME=/, @output);

   trace("output=@output");
   trace("resname=@resname");

   # modify ACTION_SCRIPT
   my ($racgwrap, $dummy, $res);

   if ($CFG->platform_family eq "windows") {
      $racgwrap = catfile('%CRS_HOME%', 'bin', 'racgwrap.bat');
   }
   else {
      $racgwrap = catfile('%CRS_HOME%', 'bin', 'racgwrap');
   }

   foreach (@resname) {
      ($dummy, $res) = split ('NAME=');
      @cmd = ($crsctl, 'modify', 'res', $res, '-attr',
	      "\"ACTION_SCRIPT=$racgwrap\"");
      $rc  = system_cmd(@cmd);

      if ($rc == 0) {
         trace("@cmd ... success");
      }
      else {
         trace("@cmd ... failed");
      }
   }
}

sub isFilesystemSupported
{
   if (is_dev_env()) {
      return SUCCESS;
   }

   my $success = SUCCESS;
   my $msg1    = "Root is not able to perform superuser operations " . 
	         "on this filesystem";
   my $msg2    = "Check export options on NAS filer";

   if (! s_set_ownergroup ($ORACLE_OWNER, $ORA_DBA_GROUP, $ORACLE_HOME)) {
      error ("Filesystem of $ORACLE_HOME is not supported");
      error ($msg1);
      error ($msg2);
      trace ("Filesystem of $ORACLE_HOME is not supported");
      trace ($msg1);
      trace ($msg2);
      return FAILED;
   } else {
      #Reset the ownership of grid home directory to root. Bug 9974887 
      s_set_ownergroup ($CFG->SUPERUSER, $ORA_DBA_GROUP, $ORACLE_HOME);
   }

   if (($CFG->platform_family eq 'unix') && ($CRS_STORAGE_OPTION == 2)) {
      my @ocr_locs = split (/\s*,\s*/, $CFG->params('OCR_LOCATIONS'));
      foreach my $loc (@ocr_locs) {
         create_dir (dirname($loc));
         $success = s_set_ownergroup($ORACLE_OWNER, $ORA_DBA_GROUP, 
                                     dirname($loc));

         if (! $success) {
            error ("Filesystem of $loc is not supported");
            error ($msg1);
            error ($msg2);
            trace ("Filesystem of $loc is not supported");
            trace ($msg1);
            trace ($msg2);
         }
      }
   }

   return $success;
}

sub remove_gsd_file
{
   trace ("Removing the gsd.pid file");
   my $gsd_file = catfile ($CFG->ORA_CRS_HOME, "srvm", "admin", "gsd.pid");

   s_remove_file ($gsd_file);
}

sub checkOLR
#-------------------------------------------------------------------------------
# Function: check if  OLR file exists and is configured.
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
   my $cmd = catfile($CFG->params('ORACLE_HOME'), 'bin', 'ocrcheck');
   my @out = system_cmd_capture($cmd, "-local", "-debug");
   my $rc  = shift @out;
   my $check_passed = FAILED;

   trace("checkOLR rc=$rc");
   my @cmdout1 = grep(/(^PROTL-708)/, @out);
   my @cmdout2 = grep(/(^PROTL-715)/, @out);
   my @cmdout3 = grep(/(^PROTL-721)/, @out);
   
   if ($rc == 0 && scalar(@cmdout1) == 0 &&
       scalar(@cmdout2) == 0 && scalar(@cmdout3) == 0)
   {
     $check_passed = SUCCESS;
     trace ("OLR check: passed");
   } 
   else 
   {
     trace ("OLR check: failed");
   }

   return $check_passed;
}

sub checkOCR
#-------------------------------------------------------------------------------
# Function: checks if the OCR is configured properly and not corrupted.
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
   my $crs_home = $_[0];
   my $cmd = catfile($crs_home, 'bin', 'ocrcheck');
   my @out = system_cmd_capture($cmd, "-debug");
   my $rc  = shift @out;
   my $check_passed = FAILED;

   trace("checkOCR rc=$rc");
   my @cmdout1 = grep(/(^PROT-708)/, @out);
   my @cmdout2 = grep(/(^PROT-715)/, @out);
   my @cmdout3 = grep(/(^PROT-721)/, @out);
   
   if ($rc == 0 && scalar(@cmdout1) == 0 &&
       scalar(@cmdout2) == 0 && scalar(@cmdout3) == 0)
   {
     $check_passed = SUCCESS;
     trace ("OCR check: passed");
   } 
   else 
   {
     trace ("OCR check: failed");
   }

   return $check_passed;
}

#------------------------------------------------------------------------------
# Function: Does checks before proceeding with the upgrade.
# Args    : none.
# Returns : TRUE or FALSE
#-------------------------------------------------------------------------------
sub preUpgradeChecks
{
  my $rel_ver = $_[0];

  # check if the crs owner of the new home is same as the crs owner of the old home
  if (!s_checkOldCrsOwner())
  {
    return FALSE;
  }

  if ((! is_dev_env()) && isFirstNodeToUpgrade())
  {
    my ($activeNodes, $inactiveNodes) = getLiveNodeList();

    if (isRolling() && isVersion112())
    {
      trace("Pre-checks for post-11.2 rolling upgrade");

      if ((! $isRerun) && ($inactiveNodes))
      {
        print color 'bold';
        print  "The following nodes:\n";
        print  "    $inactiveNodes\n";
        print  "are inactive, and must be brought back up or deleted, then re-run rootupgrade.sh\n";
        print color 'reset';

        return FALSE;
      }
      trace("Rerun: $isRerun; inactive nodes: $inactiveNodes");
    }

    trace("CRS release version $rel_ver for verifying patches");
    if (!verifyPatches($activeNodes, $rel_ver))
    {
      return FALSE;
    }
  }

  # none of the pre upgrade checks failed.
  trace ("all pre-upgrade checks passed");
  return TRUE;
}

sub verifyPatches
{

  my $nodelist = $_[0];
  my $rel_ver  = $_[1];

  trace("Verifying patches with cluvfy, node list = $nodelist; ".
        "release version = $rel_ver");

  my $cluvfy =  catfile($CFG->ORA_CRS_HOME, 'bin', 'cluvfy');
  my $oldhome = $CFG->OLD_CRS_HOME;
  my $newhome = $CFG->ORA_CRS_HOME;
  my $user = $CFG->params('ORACLE_OWNER');
  my @capout = ();
  my $old_crs_up =  check_OldCrsStack();
  my @cmd = ();

     @cmd  = ($cluvfy, 'stage', '-pre', 'crsinst', '-n', $nodelist, '-upgrade', 
              '-src_crshome',$oldhome, '-dest_crshome', $newhome, '-dest_version',
              $rel_ver, '-_patch_only');

  my $rc =  run_as_user2($user, \@capout, @cmd);  

  if ($rc != 0) {
    error("cluvfy patch check failed, output is @capout");
    error("The cluvfy tool found some mandatory patches are not installed.");
    error("These patches need to be installed before the upgrade can proceed.");
    return FAILED;
  }
  else  
  {
    trace("cluvfy patch check passed, output is @capout");
    return SUCCESS;
  }
}

sub upgsihamodel
#-------------------------------------------------------------------------------
# Function: call srvctl upgrade model for SIHA
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
  my $success;
 
  trace("Invoking srvctl upgrade model for SIHA"); 
  $success = upgrademodel($FIRSTNODE);
  $success = upgrademodel($LASTNODE);
}

sub setperm_vdisks
#-------------------------------------------------------------------------------
# Function: Set voting disks permission
#-------------------------------------------------------------------------------
{
   if (($CFG->platform_family eq 'windows') || 
       ($CFG->ASM_STORAGE_USED) ||
       (isOCRonASM()))
   {
      return SUCCESS;
   }

   my $owner  = $CFG->params('ORACLE_OWNER');
   my $group  = $CFG->params('ORA_DBA_GROUP');
   my $rc     = SUCCESS;
   my @vdisks = ExtractVotedisks();

   foreach (@vdisks) {
      if ($DEBUG) { trace ("set permission on $_"); }
      if (! s_set_ownergroup ($owner, $group, $_)) {
         $rc = FAILED;
         error ("Failed to change ownership of $_");
         last;
      }

      if (! s_set_perms("0640", $_)) {
         $rc = FAILED;
         error ("Failed to set permissions on $_");
         last;
      }
   }

   return $rc;
}

sub isONSexist

#-------------------------------------------------------------------------------
# Function: Find if ONS is already configured
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
  my $status = SUCCESS;
  my @out;
  my $rc;

  trace("Invoking srvctl config ons");

  my $srvctl =  catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl');

  @out = system_cmd_capture($srvctl, "config", "ons");
  $rc = shift @out;  

  if ($rc != 0) {
    trace("ONS resource does not exist. output is @out");
    $status = FAILED;
  } else  {
    trace("ONS resource  exist. output is @out");
  }

  return $status;

}

sub isCVUConfigured
#-------------------------------------------------------------------------------
# Function: Find if CVU resource is already configured
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
  my $status = SUCCESS;
  my @out;
  my $rc;

  trace("Invoking srvctl config cvu");

  my $srvctl =  catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl');

  @out = system_cmd_capture($srvctl, "config", "cvu");
  $rc = shift @out;  

  if ($rc != 0) {
    trace("CVU resource does not exist. output is @out");
    $status = FAILED;
  } else  {
    trace("CVU resource  exist. output is @out");
  }

  return $status;
}

sub waitForVipRes 
#-------------------------------------------------------------------------------
# Function: Wait for VIP resource to exist
# Args    : none
# Returns : SUCCESS or FAILED
#-------------------------------------------------------------------------------
{
   my $host       = $CFG->HOST;
   my $srvctl     = catfile($CFG->ORA_CRS_HOME, 'bin', 'srvctl');
   my $retries    = 12;
   my $vip_exists = FALSE;
   my ($rc, @output);

   while ($retries) {
      @output = system_cmd_capture($srvctl, 'status', 'vip', '-n', "$host");
      $rc     = shift @output;

      trace("rc=$rc output=@output");
      my @cmdout = grep(/(^PRKO-2165)/, @output);
      if ($rc == 0 && scalar(@cmdout) == 0) {
         $vip_exists = TRUE;
         last;
      }

      trace ("Waiting for VIP resource");
      sleep (5);
      $retries--;
   }

   if ($vip_exists) {
      trace ("VIP resource exists");
   } 
   else {
      trace ("Timed out waiting for VIP resource");
   }

   return $vip_exists;
}

sub checkPatchforupg
{
   my $home = $_[0];
   my $rc;
   my @cmdout;
   my @output;
   my $cmd;
   my $opatch = catfile ($ORACLE_HOME, "OPatch", "opatch");
   my $user = $CFG->params('ORACLE_OWNER');
   my $status = FAILED;

   #Check if the patchid is present in the  11201 crshome
   $cmd = "$opatch lsinventory -oh $home";

   $rc = run_as_user2($user, \@output, $cmd);

   # grep for bugs 9655006 or 9413827
   @cmdout = grep(/(9655006|9413827)/, @output);

   if ((scalar(@cmdout) > 0) && ($rc == 0)) {
      $status = SUCCESS;
   } else {
      trace("opatch lsinventory didn't find fix for bug 9413827 in $home");
   }

   return $status;

}

sub isHomeShared
{
   #use a path that's always writable in the grid home to do sharedness check.
   my $pathtochk  = catdir( $ORACLE_HOME, 'crs', 'install');
   my @capout = ();
   my $rc;
   my $user = $CFG->params('ORACLE_OWNER');
   my $nodelist = $CFG->params('NODE_NAME_LIST');
   my $CKPTTOOL = catfile( $ORACLE_HOME, 'bin', 'cluutil');
   my @program = ($CKPTTOOL, '-chkshare', -oh , $pathtochk , '-localnode', $HOST, '-nodelist', $nodelist);
   my $status;

   trace("checking if path $pathtochk is shared");
   # run as specific user, if requested
   $rc = run_as_user2($user, \@capout, @program);
   # cluutil return 0 err code and errors, if any, on stdout
   if (scalar(grep(/TRUE/, @capout)) > 0)
   {
     trace("The oracle home $ORACLE_HOME is shared ");
     $status = TRUE;
   }
   else
   {
     trace("The oracle home $ORACLE_HOME is not shared ");
     $status = FALSE;
   }
   
   return $status;
}

sub get_OldCrsVersionfromOCR
{
   trace("Getting old crs active verison  from ocrdump");

   my $OCRDUMPBIN  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'ocrdump');
   if ($CFG->platform_family eq "windows") {
      open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
              "SYSTEM.version.activeverison |");
   }
   else {
      open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
                  "'SYSTEM.version.activeverison'|");
   }
   my @output = <OCRDUMP>;
   close (OCRDUMP);

   trace("ocrdump output for active version is @output");
   my @txt = grep (/ORATEXT/, @output);
   my ($key, $actver) = split (/:/, $txt[0]);

   trace ("key is $key");
   trace ("activeversion is $actver");
   return $actver;
                         
}

sub get_softVersionfromOCR
{
   my $nodename = $_[0];

   trace("Getting crs sotware verison  from ocrdump");

   my $OCRDUMPBIN  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'ocrdump');
   if ($CFG->platform_family eq "windows") {
      open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
              "SYSTEM.version.hostnames.$nodename |");
   }
   else {
      open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
                  "'SYSTEM.version.hostnames.$nodename'|");
   }
   my @output = <OCRDUMP>;
   close (OCRDUMP);

   trace("ocrdump output for software version on $nodename is @output");
   my @txt = grep (/ORATEXT/, @output);
   my ($key, $softver) = split (/:/, $txt[0]);
   trace ("key is $key");
   trace ("softver is $softver");    
   return $softver;

}

sub isRolling
{
  my $isrolling = TRUE;
  if ($CFG->params('ISROLLING') =~ m/false/i) {
      $isrolling = FALSE;
   }
  trace("Rolling upgrade is set to $isrolling");
  return $isrolling;
}

sub isReusedg
{
  my $isreusedg = FALSE;
  if ($CFG->params('REUSEDG') =~ m/true/i) {
      $isreusedg = TRUE;
   }
  trace("Reuse Disk Group  is set to $isreusedg");
  return $isreusedg;
}

sub isPrereqIgnored
{
  my $isprereqignored = FALSE;
  if ($CFG->params('USER_IGNORED_PREREQ') =~ m/true/i) {
      $isprereqignored = TRUE;
   }
  trace("USER_IGNORED_PREREQ  is set to $isprereqignored");
  return $isprereqignored;
}

sub start_asm
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "start asm");

   if ($status) {
      trace ("start asm ... success");
   } 
   else {
      error ("start asm ... failed");
      return FALSE;
   }

   return TRUE;
}

sub stop_asm
{
   my $run_as_oracle_owner = TRUE;
   my $status = srvctl($run_as_oracle_owner, "stop asm -f");

   if ($status) {
      trace ("stop asm ... success");
   }
   else {
      error ("stop asm ... failed");
      return FALSE;
   }

   return TRUE;
}

sub modifyClusterName
{
   trace("modify CLUSTER_NAME in crsconfig_params");

   my $params_file = $CFG->paramfile;
   my @params_table = read_file ($params_file);
   my $updateParamsFile = FALSE;
   my $ix = 0;
   my $cluster_name;

   foreach my $rec (@params_table) {
      chomp($rec);
      if ($rec =~ m/^CLUSTER_NAME=/) {
         my ($key, $value) = split (/=/, $rec);
         if (! $value) {
            $cluster_name = getClusterName();
            if ($cluster_name) {
               $params_table[$ix] = 'CLUSTER_NAME=' . "$cluster_name";
               $updateParamsFile = TRUE;
               last;
            }
         }
      }

      $ix++;
   }

   if ($updateParamsFile) {
      # save original params file
      my $save_file = catfile (dirname($params_file), 'crsconfig_params.saved');
      copy_file ($params_file, $save_file);

      # delete old file and create new file
      if (s_remove_file($params_file)) {
         open (SFILE, ">$params_file")
            || die ("Can't open $params_file to write: $!");

         foreach my $rec (@params_table) {
            chomp($rec);
            print SFILE "$rec\n";
         }

         close SFILE;
      }
   }

   if ($updateParamsFile)
   {
     # Fix bug 14296767
     trace("Create the ORA_CRS_HOME/cdata/CLUSTER_NAME directory");
     my ($line, $dir, @dirs);

     $dir = catdir($CFG->ORA_CRS_HOME, 'cdata', $cluster_name);
     trace("The missing directory is $dir");
     $line = join(' ', 'all', $dir, $CFG->params('ORACLE_OWNER'),
                   $CFG->params('ORA_DBA_GROUP'), '0775');
     push(@dirs, $line);
     create_dirs(\@dirs);
   }
}

sub getClusterName
{
   my $olsnodes = catfile($CFG->ORA_CRS_HOME, 'bin', 'olsnodes');
   my @cmd = ($olsnodes, '-c');
   my @out = system_cmd_capture(@cmd);
   my $rc = shift @out;
   my $cluster_name;

   if ($rc == 0) {
      $cluster_name = $out[0];
   }

   chomp($cluster_name);
   trace("cluster_name=$cluster_name");
   return $cluster_name;
}

sub getcrsrelver
{
   my $home = $_[0];
   my ($crsctl);
   my $verstring;

   if (! defined $home) {
      $crsctl = crs_exec_path('crsctl');
   } else {
      $crsctl = catfile($home, 'bin', 'crsctl' );
   }

   my @cmd = ($crsctl, 'query', 'crs', 'releaseversion');
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;

   # if succeeded, parse to ver numbers, output must be a single line,
   # version is 5 numbers, major to minor (see above)
   if ($rc == 0) {
      $verstring = getVerInfo($out[0]);
      trace( "Got CRS release version: ".join('.', $verstring) );
   }
   else {
      
      error ("@cmd ... failed rc=$rc with message:\n @out \n");
   }
   return $verstring;

}
sub isVersionMatch
{
   my $ver1 = $_[0];
   my $ver2 = $_[1];
   my $status;

   chomp $ver1;
   chomp $ver2;

   trace("version1  is $ver1");
   trace("version2  is $ver2");

   if ($ver1 eq $ver2) {
      $status = TRUE;
   } else {
      $status = FALSE;
   }

   trace("Version match status is $status");
   return $status;
}

# This subroutine takes argument as attribute of ora file and returns 
# the corresponding value of that attribute in the ora file.
# For no we allow only BDBSIZE and BDBLOC attribute as parameter.
# Eg: If ora file has a line with "BDBSIZE = 25632", 
# this subroutine returns 25632.
sub getCHMAttrib
{
  my $arg = $_[0];
  my $orafile = $_[1];
  my $loc;
  my $home = s_get_olr_file ("crs_home");
  $HOST = tolower_host();

  if ($arg ne "" && ($arg eq "BDBSIZE" || $arg eq "BDBLOC"))
  {
    # Read the ora file to get the BDB path
    if ($orafile eq "")
    {
      $orafile = catfile ($home, "crf", "admin", "crf${HOST}.ora");
    }
    if (!-f $orafile)
    {
      trace("Info: No ora file present at ", $orafile); 
      return "";
    }

    my @filecontent = read_file ($orafile);
    foreach my $line (@filecontent)
    {
      # skip blanks and comments
      if ($line !~ /^#|^\s*$/)
      {
        if ($line =~ /$arg=(.*)/) { $loc = $1; last;}
      }
    }
    return $loc;
  }
  return "";
}

sub removeCHMDB
{
  my $bdbloc = getCHMAttrib("BDBLOC");
    if ($bdbloc eq "default")
  {
    $bdbloc = catfile($CFG->ORA_CRS_HOME, "crf", "db", $CFG->HOST);
  }
  if (-d $bdbloc)
  {
    # delete old bdb files.
    trace("Deleting CHM repository at files at: ", $bdbloc);
    crf_delete_bdb($bdbloc);
  }
  else
  {
    trace("CHM repository path not found");
  }
}

sub getcrsrelver1
{   
   my $home = $_[0];
   my ($sqlplus);
   my $verstring;

   if (! defined $home) {
      $sqlplus = crs_exec_path('sqlplus');
   } else {
      $sqlplus = catfile($home, 'bin', 'sqlplus' );
   }

   my @cmd = ($sqlplus, '-V');
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;
   # if succeeded, parse to ver numbers, output must be a single line,
   # version is 5 numbers, major to minor (see above)
   if ($rc == 0) {
      my ($s1, $s2, $s3, $s4) = split(/ /, $out[1]);
      chomp $s3;
      $verstring = $s3;
      trace( "Got CRS release version: $verstring ");
   }
   else {

      error ("@cmd ... failed rc=$rc with message:\n @out \n");
   }
   return $verstring;
}

sub update_ASMowner
#-------------------------------------------------------------------------------
# Function:  update ASM owner to grid owner 
# Args    :  none
# Returns :  none
#-------------------------------------------------------------------------------
{
   trace("Updating ASM owner if required");
   my $CRSCTL  = catfile ($CFG->params('ORACLE_HOME'), 'bin', 'crsctl');
   my $rc;
   my @out;
   my $ohown = $CFG->params('ORACLE_OWNER');
   my $cmd;

   $cmd = "$CRSCTL stat resource -w 'NAME en .asm'";
   @out = system_cmd_capture("$cmd");
   $rc = shift @out;

   my @cmdout = grep(/NAME/, @out);

   trace("ASM resource list is @cmdout");

   foreach my $str (@cmdout) {
   
     my ($tag, $resname) = split(/=/, $str);
     chomp ($resname);
     if ($resname =~ "ora.asm") { 
       trace("skip update owner for ora.asm resource");
       next;
     }
     $cmd = "$CRSCTL getperm resource $resname";
     @out = system_cmd_capture("$cmd");
     $rc = shift @out;

     my @cmdout = grep(/^owner:/, @out);
     
     trace("$resname ACL is @cmdout");
     my @val = split(/:/, $cmdout[0]);

     my $asmown = $val[1];
     chomp ($asmown);

     trace("asm owner is $asmown");

     if ($asmown =~ $ohown ) {
         trace("ASM resource owner is the same as grid owner.Nothing to be done");
         next;
     }

     $cmd = "$CRSCTL setperm resource $resname -o $ohown";
     @out = system_cmd_capture("$cmd");
     $rc = shift @out;


     if ($rc == 0) {
        trace ("Successfully updated owner for resource $resname");
     } else {
        error ("Failed to update owner for resource $resname");
     }
   }
}

sub unlockHome
#-------------------------------------------------------------------------------
# Function:  unlocks the GI Home for patching. sets the owner of all the files/dirs
#            in crsconfig_fileperms and crsconfig_dirs to GI Home owner. This is 
#            sufficient for patching GI Home.
# Args    :  none
# Returns :  none
#-------------------------------------------------------------------------------
{
 my ($owner, $group, $perms, $basedir) = @_;
 my %dirown;
 my %fileown;
 my ($a, $dirname, $own, $grp, $perm, $fname);

 trace("The CRS home passed in is $basedir");
 if (! $basedir)
 {
   $basedir = $ORA_CRS_HOME;
 }

 my $platform = s_get_platform_family();
 if ($platform eq 'windows') {
    $basedir =~ s/\\/\\\\/g;
 }

 my $wrapdir_crs_utl = catdir($basedir, "crs", "utl");

 my @crsconfig_dirs = read_file (catfile ($wrapdir_crs_utl, "crsconfig_dirs"));
 my @crsconfig_fileperms = read_file (catfile ($wrapdir_crs_utl,
                                                  "crsconfig_fileperms"));
 foreach my $line (@crsconfig_dirs) {
    chomp ($line);
    next if ($line =~ /^#|\$|^\s*$/);
    ($a, $dirname, $own, $grp, $perm) = split(/ /, $line);
    if ($own =~ $SUPERUSER) {
      $dirown{$dirname} = $own;
    } else { next ;}
 }

 foreach my $line (@crsconfig_fileperms) {
    chomp ($line);
    next if ($line =~ /^#|\$|^\s*$/);
    ($a, $fname, $own, $grp, $perm) = split(/ /, $line);
    if ($own =~ $SUPERUSER) {
      $fileown{$fname} = $own;
    } else { next ;}
 }

 foreach my $dir (keys%dirown) {

   next if ($dir !~ m/^$basedir.*/);
   s_set_ownergroup ($owner, $group, $dir);
 }

 foreach my $file (keys%fileown) {

   next if ($file !~ m/^$basedir.*/);
   s_set_ownergroup ($owner, $group, $file);
 }
}


sub modifyparamfile
{

   my $param = $_[0];
   my $paramval = $_[1];

   trace("modify $param $CFG->paramfile");

   my $params_file = $CFG->paramfile;
   my @params_table = read_file ($params_file);
   my $updateParamsFile = FALSE;
   my $ix = 0;

   foreach my $rec (@params_table) {
      chomp($rec);
      if ($rec =~ m/^$param=/) {
         my ($key, $value) = split (/=/, $rec);
         $params_table[$ix] = $param . "=" . "$paramval";
         $updateParamsFile = TRUE;
         last;
      }

      $ix++;
   }
   if ($updateParamsFile) {
      # save original params file
      my $save_file = catfile (dirname($params_file), 'crsconfig_params.saved');
      copy_file ($params_file, $save_file);

      # delete old file and create new file
      if (s_remove_file($params_file)) {
         open (SFILE, ">$params_file")
            || die ("Can't open $params_file to write: $!");

         foreach my $rec (@params_table) {
            chomp($rec);
            print SFILE "$rec\n";
         }

         close SFILE;
      }
   }
}


sub getLiveNodeList
#-------------------------------------------------------------------------------
# Function:  Acquire the node list where all live nodes in cluster are involved
# Args    :  none
# Returns :  The live node list separated by commas 
#            &
#            the down node list separated by commas
#-------------------------------------------------------------------------------
{
  my $liveNodeList;
  my $downNodeList;
  my @cluNodes = split(/,/, $CFG->params('NODE_NAME_LIST')); 

  foreach my $node (@cluNodes)
  {
    my $ret = pingWithICMP($node, 30); 
    if (1 == $ret)
    {
      $liveNodeList = "$liveNodeList"."$node,";
    }
    else
    {
      set_bold();
      print "The node '$node' cannot be reached using ICMP ping\n";
      reset_bold();
      $downNodeList = "$downNodeList"."$node,";
    }
  }

  if ($liveNodeList)
  {
    # Chop the last comma off
    chop($liveNodeList);
  }

  trace("Live node list: {$liveNodeList}");

  if ($downNodeList)
  {
    chop($downNodeList);
  }

  trace("Down node list: {$downNodeList}");
  
  return ($liveNodeList, $downNodeList);
}

####-----------------------------------------------------------------------
#### Check a remote host for reachability using ICMP ping
# ARGS: 2 
# ARG1: hostname or IP address
# ARG2: timeout value
# @returns 
#   $ret: 1 - reachable; 0 - unreachable; undef - invalid hostname/IP address
#
# Note: The caller must run as root or be setuid to root
sub pingWithICMP
{
  my ($host, $timeout) = @_;
  my $pObj;

  $pObj = Net::Ping->new("icmp", $timeout, 64);
  my $ret = $pObj->ping($host);

  return $ret;
}

#-------------------------------------------------------------------------------
# Function: Enables bold print if platform is not windows.
# Args    : none
# Returns : none
#-------------------------------------------------------------------------------
sub set_bold
{
  my $platform = s_get_platform_family();

  if ($platform ne 'windows') {
    print color 'bold';
  }
}

#-------------------------------------------------------------------------------
# Function: Disabled bold print if platform is not windows.
# Args    : none
# Returns : none
#-------------------------------------------------------------------------------
sub reset_bold
{
  my $platform = s_get_platform_family();

  if ($platform ne 'windows') {
    print color 'reset';
  }
}

#-------------------------------------------------------------------------------
# Function: Calls the TFA setup script
# Args    : none
# Returns : none
#-------------------------------------------------------------------------------
sub setup_tfa
{
  my $TFASCRIPT = catfile($CFG->ORA_CRS_HOME, "crs", "install", "tfa_setup.sh");
  if ( -f "$TFASCRIPT" )
  {
    my $cmd = "$TFASCRIPT -silent -crshome ".$CFG->ORA_CRS_HOME;
    print "Installing Trace File Analyzer\n";
    trace ("Installing Trace File Analyzer\n");
    my @output = system_cmd_capture( "$cmd" );
  }
}

#-------------------------------------------------------------------------------
# Function: Remove TFA
# Args    : none
# Returns : none
#-------------------------------------------------------------------------------
sub remove_tfa
{
  my $HOST = tolower_host();
  # check if the TFA is running from GIHOME
  my $TFAHOME = catfile($CFG->ORA_CRS_HOME, "tfa", "$HOST", "tfa_home");
  if ( -d "$TFAHOME" ) 
  {
    my $TFA_UNINSTALL_SCRIPT = catfile($TFAHOME, "bin", "uninstalltfa.sh");
    if ( -f "$TFA_UNINSTALL_SCRIPT" )
    {
      my $cmd = "$TFA_UNINSTALL_SCRIPT -silent -local -crshome ".$CFG->ORA_CRS_HOME;
      print "Removing Trace File Analyzer\n";
      trace ("Removing Trace File Analyzer\n");
      my @output = system_cmd_capture( "$cmd" );
    }
  }
}

sub get_J2EEContainerToBeEnabled
#---------------------------------------------------------------------
# Function: Determine whether or not the J2EE Container should be enabled
# Args    : none 
#---------------------------------------------------------------------
{
   # If Unix, call Unix OSD
   my $platform = s_get_platform_family();
   if ($platform eq "unix") {
      return s_get_J2EEContainerToBeEnabled();
   }

   # If Windows, enable OC4J (bug 9900672)
   return TRUE;
}

sub s_get_J2EEContainerToBeEnabled
#---------------------------------------------------------------------
# Function: Determine whether or not the J2EE Container should be enabled
# Args    : none 
#---------------------------------------------------------------------
{
  # Disable OC4J on HP-UX (bug 12648797)
  if ($OSNAME eq 'hpux')
  {
    return FALSE;
  }
  
  # Disable OC4J on z/Linux (bug 14615073)
  my $mach =`uname -m`;
  chomp($mach);

  if ($mach eq "s390x" || $mach eq "s390")
  {
    return FALSE;
  }

  return TRUE;
}

11;
__END__

=head2 <sub-name>


=head3 Parameters


=head3 Returns


=head3 Usage


=cut


