support/oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ea39d398ae2f10c155203ef0a025c362021efb55
oracle/vg/Exadata_commands.txt

2479 lines
106 KiB
Plaintext
Raw Normal View History

2026-03-12 20:23:15
2026-03-12 21:23:47 +01:00
Exadata Commands
useradd
dcli -l oracle -g dbs_group_pr01pimi --serial " echo '@/export/ora_stage/vishal/sqlscripts/flush_sql 1j6sgrbmy4592'|sqlplus -s / as sysdba" |grep 'successfully completed'
dcli -l oracle -g dbs_group_bt01pimi --serial " echo '@/export/ora_stage/vishal/sqlscripts/flush_sql 1j6sgrbmy4592'|sqlplus -s / as sysdba" |grep 'successfully completed'
dcli -l oracle -c dm02db04,dm02db05,dm02db06 --serial " echo '@/export/ora_stage/vishal/sqlscripts/flush_sql 5f4d1xd2uftj1'|sqlplus -s / as sysdba" |grep 'successfully completed'
dcli -l oracle -c dm02db04,dm02db05,dm02db06 --serial " echo 'alter system flush shared_pool;'|sqlplus -s as sysdba"|grep 'successfully completed'
#######################################
# Exadata Cell Serice Startup Time
#######################################
# CELLSRVC Cell Server process
dcli -l root -g ~/cell_group "ps -ef|grep 'cellsrv 100' |grep -v grep"
# MS (Monitoring Server) process
dcli -l root -g ~/cell_group "ps -ef|grep 'cellrsomt' |grep -v grep"
dcli -l root -g ~/cell_group "ps -ef|grep 'cellrsmmt' |grep -v grep"
dcli -l root -g ~/cell_group "ps -ef|grep 'cellrsbkm' |grep -v grep"
dcli -l root -g ~/cell_group "ps -ef|grep 'oc4j.jar' |grep -v grep"
# RS (Restart Server) process
dcli -l root -g ~/cell_group "ps -ef|grep 'cellrssrm' |grep -v grep"
# OSWatcher
dcli -l root -g ~/cell_group "ps -ef|grep 'OSWatcher.sh' |grep -v grep"
#############################
# Exadata Hardware Checks
#############################
#-- Get LED Light Status
dcli -l root -g all_group '/usr/bin/ipmitool sunoem led get all |egrep -v "OFF|na|OK.*ON" '
#-- Cell level checks
dcli -l root -g cell_group " cellcli -e LIST CELL ATTRIBUTES name,cellNumber,status,fanStatus,powerStatus,temperatureStatus,cellsrvStatus,msStatus,rsStatus,releaseVersion,releaseTrackingBug"
#-- 11.2.3.2.0 onwards
dcli -l root -g cell_group " cellcli -e LIST CELL ATTRIBUTES name,cellNumber,status,flashCacheMode,fanStatus,powerStatus,temperatureStatus,cellsrvStatus,msStatus,rsStatus,releaseVersion,releaseTrackingBug"
#-- Physical HardDisks
dcli -l root -g cell_group " cellcli -e \"LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,physicalRPM,physicalInsertTime,errorcount,errHardReadCount,errHardWriteCount,errMediaCount,errOtherCount,errSeekCount,lastFailureReason where disktype=harddisk AND status != 'normal' \" "
#-- Physical HardDisks LUNs
dcli -l root -g cell_group " cellcli -e \"LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=harddisk AND status != 'normal' \" "
#-- FlashDisks
dcli -l root -g cell_group " cellcli -e \"LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,lastFailureReason where disktype=flashdisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=flashdisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST lun ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=flashdisk and physicalDrives='' \" "
#-- Cell Disk Checks
dcli -l root -g cell_group " cellcli -e \"LIST CELLDISK ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where disktype=harddisk and status !='normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST CELLDISK ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where disktype=flashdisk and status !='normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST CELLDISK ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where status !='normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST CELLDISK ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where disktype=harddisk and freespace !=0 \" "
dcli -l root -g cell_group " cellcli -e \"LIST CELLDISK ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where disktype=flashdisk and freespace !=0 \" "
#-- Grid Disk Checks
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,cachingPolicy,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where status != 'active' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,cachingPolicy,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where asmmodestatus != 'ONLINE' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,cachingPolicy,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where asmdeactivationoutcome != 'Yes' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,cachingPolicy,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where cachingPolicy != 'default' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,cachingPolicy,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where name like '.*RECO.*' \" and cachingPolicy != 'default' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where asmdeactivationoutcome != 'Yes' \" "
#-- FlashCache
dcli -l root -g cell_group " cellcli -e 'LIST flashcache attributes name,status,size,creationTime,degradedCelldisks,effectiveCacheSize ' "
#-- FlashLog
dcli -l root -g cell_group " cellcli -e 'LIST flashlog attributes name,status,size,creationTime,degradedCelldisks,effectiveSize,efficiency ' "
#-- Battery Checks
#-- Check current battery temperature (Should be < 60 C)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep Temperature:'
#-- Check current Battery Charge Capacity (Should be > 800 mAh)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuCapacityInfo -a0 | grep "Full Charge" '
#-- Check current Battery erors (Should be < 10%)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | grep "Max Error"'
#-- FlashCache
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,cachingPolicy where name like '.*RECO.*' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,cachingPolicy where cachingPolicy != 'default' \" "
dcli -l root -g cell_group " cellcli -e \"LIST METRICCURRENT WHERE name = 'GD_BY_FC_DIRTY' and metricObjectName like '.*RECO.*' \" "
dcli -l root -g cell_group " cellcli -e \"LIST METRICCURRENT WHERE name = 'GD_IO_BY_R_LG' and metricObjectName like '.*RECO.*' \" "
dcli -l root -g cell_group " cellcli -e \"LIST METRICCURRENT WHERE name = 'GD_IO_BY_R_SM' and metricObjectName like '.*RECO.*' \" "
#############################
# Exadata Version Checks
#############################
#-- Exadata Software
dcli -l root -g all_group imageinfo --image-version
dcli -l root -g all_group imageinfo --image-status
dcli -l root -g all_group imageinfo --kernel-version
dcli -l root -g cell_group imageinfo --cell-version
dcli -l root -g cell_group imageinfo --cell-rpm-version
#-- Kernel Version
dcli -l root -g all_group 'uname -r'
dcli -l root -g all_group 'rpm -q kernel'
dcli -l root -g all_group 'rpm -q kernel-uek'
#-- Check Firmwares
dcli -l root -g all_group '/opt/oracle.SupportTools/CheckHWnFWProfile -d ' > /tmp/CheckHWnFWProfile.txt
for i in SystemModel BIOSVersion ILOMVersion LightsOutFirmwareVersion \
DiskControllerModel DiskControllerFirmwareRevision DiskControllerBatteryBackupUnitVersion DiskControllerPCIeSlotWidth \
MinimumSASExpanderFirmware SASBackplaneFirmwareVersion \
BackplaneFirmwareVersion InfinibandHCAChipRevision InfinibandHCAFirmwareVersion InfinibandHCAHardwareRev InfinibandHCAPCIeSlotWidth \
MinimumCPLDFirmware \
FansAndFanCount PowersuppliesAndCount \
PhysicalMemoryGB ProcessorInformation
do
echo "-------------- $i ---------------------"
grep -A1 $i /tmp/CheckHWnFWProfile.txt |egrep -v "^--|$i"
done |less
# For 11.2.3.3.0 and above
for i in Server_Model BIOSVendor BIOSVersion ProcessorModel SocketCount ILOMVersion PhysicalMemoryGB \
DiskControllerModel DiskControllerPCIeSlotWidth DiskControllerSilicon DiskControllerFirmwareRevision \
DiskControllerPCIeSlotNumber DiskControllerBatteryBackupUnitVersion FanCount Powersupply_Info \
InfinibandHCA InfinibandHCAImageType InfinibandHCAPCIeSlotSpeed InfinibandHCAPCIeSlotWidth InfinibandHCAChipRevision \
InfinibandHCADeviceID InfinibandHCAFirmwareVersion InfinibandHCAPCIeSlotNumber InfinibandHCAHardwareRev InfinibandHCAPSID \
Ether10GCount Ether10GModel
do
echo "-------------- $i ---------------------"
grep -h $i /tmp/CheckHWnFWProfile.txt
done |less
#-- Check FDOM version
grep MARVELL /tmp/CheckHWnFWProfile.txt
grep /dev/ /tmp/CheckHWnFWProfile.txt
#-- Check Disks firmware
egrep 'SAS|SATA' /tmp/CheckHWnFWProfile.txt
#############################
# Exadata Patching Checks
#############################
#-- Check startup check failures
dcli -l root -g all_group "cat /var/log/cellos/validations.log |egrep 'FAILED|ERROR'"
# Alert Log
#####################
less $ORACLE_BASE/diag/rdbms/`echo $ORACLE_SID|cut -b 1-8`/$ORACLE_SID/trace/alert_$ORACLE_SID.log
dcli -l oracle -g dbs_group 'egrep -A10 "Aug 01.*2012" $ORACLE_BASE/diag/rdbms/`echo $ORACLE_SID|cut -b 1-8`/$ORACLE_SID/trace/alert_$ORACLE_SID.log |egrep "2012|ORA-00600|trc" | grep -B1 ORA- |egrep -v "^--" '
# Search listener Log on Exadata
#####################################
less $ORACLE_BASE/diag/tnslsnr/`hostname -s`/listener/trace/listener.log
dcli -l oracle -g dbs_group "grep 31-JUL-2012 /u01/app/oracle/diag/tnslsnr/\`hostname -s\`/listener/trace/listener.log |egrep 'ORA-00060'"
########################
# Compute node details
########################
DBMCLI> describe dbserver
name modifiable
bbuStatus
comment modifiable
coreCount
cpuCount
diagHistoryDays modifiable
emailFormat modifiable
emailSubscriber modifiable
fanCount
fanStatus
iaasIdleInUse modifiable
iaasMode modifiable
iaasReason modifiable
id
interconnectCount
interconnect1 modifiable
interconnect2 modifiable
interconnect3 modifiable
interconnect4 modifiable
interconnect5 modifiable
interconnect6 modifiable
interconnect7 modifiable
interconnect8 modifiable
ipaddress1
ipaddress2
ipaddress3
ipaddress4
ipaddress5
ipaddress6
ipaddress7
ipaddress8
kernelVersion
locatorLEDStatus
location modifiable
makeModel
metricCollection modifiable
metricHistoryDays modifiable
msVersion
notificationMethod modifiable
notificationPolicy modifiable
pendingCoreCount modifiable
powerCount
powerStatus
realmName modifiable
releaseImageStatus
releaseVersion
releaseTrackingBug
smtpFrom modifiable
smtpFromAddr modifiable
smtpPort modifiable
smtpPwd modifiable
smtpServer modifiable
smtpToAddr modifiable
smtpUser modifiable
smtpUseSSL modifiable
snmpEngineID modifiable
snmpSubscriber modifiable
snmpUser modifiable
status
syslogConf modifiable
temperatureReading
temperatureStatus
traceLevel modifiable
upTime
msStatus
rsStatus
dcli -l root -g dbs_group " dbmcli -e 'list dbserver attributes name,coreCount,cpuCount,pendingCoreCount' "
########################
# Cell Details
########################
dcli -l root -g cell_group " cellcli -e ' LIST CELL ATTRIBUTES name,cellNumber,status,flashCacheMode,flashCacheCompress,fanStatus,powerStatus,temperatureStatus,cellsrvStatus,msStatus,rsStatus,releaseVersion,releaseTrackingBug ' "
# FlashCache Compression
##########################
#If disabled, then value is FALSE or NULL
dcli -l root -g cell_group "cellcli -e LIST CELL attributes name,flashCacheCompress "
dcli -l root -g cell_group 'grep flashCacheCompress $OSSCONF/cell_disk_config.xml'
# Physical Disk Details
########################
cellcli -e ' DESCRIBE PHYSICALDISK '
dcli -l root -g cell_group " cellcli -e \"LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,lastFailureReason where disktype=harddisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,lastFailureReason where disktype=flashdisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e ' LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,physicalSize,errCmdTimeoutCount,errHardReadCount,errHardWriteCount,errMediaCount,errOtherCount,errSeekCount,errorCount,notPresentSince,lastFailureReason where disktype=harddisk ' "
dcli -l root -g cell_group " cellcli -e ' LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,lastFailureReason where disktype=flashdisk ' "
# Should display count=0
dcli -l root -g dbs_group /opt/MegaRAID/MegaCli/MegaCli64 -PdList -a0 | grep "Predictive Failure Count"
# LUN Details
#####################
cellcli -e ' DESCRIBE LUN '
dcli -l root -g cell_group " cellcli -e \"LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=harddisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e \"LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=flashdisk AND status != 'normal' \" "
dcli -l root -g cell_group " cellcli -e LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=harddisk "
dcli -l root -g cell_group " cellcli -e LIST LUN ATTRIBUTES name,status,diskType,deviceName,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=flashdisk "
# CELL DISK Details
####################
cellcli -e ' DESCRIBE CELLDISK '
dcli -l root -g cell_group " cellcli -e \"LIST celldisk ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where status !='normal' \" "
dcli -l root -g cell_group " cellcli -e ' LIST celldisk ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel ' "
# GridDisk Details
##################
cellcli -e ' DESCRIBE griddisk '
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where status != 'active' \" "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,status,diskType,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment where asmmodestatus != 'ONLINE' \" "
dcli -l root -g cell_group " cellcli -e ' LIST GRIDDISK ATTRIBUTES name,status,diskType,size,offset,errorcount,creationTime,asmmodestatus,asmdeactivationoutcome,comment ' "
dcli -l root -g cell_group " cellcli -e \"LIST GRIDDISK ATTRIBUTES name,cachingPolicy where cachingPolicy = 'none' \" "
# FlashDisk Details
###################
#If disabled, then value is FALSE or NULL
dcli -l root -g cell_group "cellcli -e LIST CELL attributes name,flashCacheCompress "
dcli -l root -g cell_group 'grep flashCacheCompress $OSSCONF/cell_disk_config.xml'
dcli -l root -g cell_group " cellcli -e ' LIST PHYSICALDISK ATTRIBUTES name,status,diskType,physicalSize,errorcount,physicalInsertTime,lastFailureReason where disktype=flashdisk ' "
dcli -l root -g cell_group " cellcli -e ' LIST lun ATTRIBUTES name,status,diskType,raidLevel,isSystemLun,lunSize,physicalDrives,lunWriteCacheMode,errorCount where disktype=flashdisk ' "
dcli -l root -g cell_group " cellcli -e ' LIST celldisk ATTRIBUTES name,status,diskType,lun,size,freespace,errorcount,creationTime,interleaving,raidlevel where disktype=flashdisk ' "
dcli -l root -g cell_group " cellcli -e ' LIST flashcache attributes name,status,size,creationTime,degradedCelldisks,effectiveCacheSize ' "
dcli -l root -g cell_group " cellcli -e ' LIST flashlog attributes name,status,size,creationTime,degradedCelldisks,effectiveSize,efficiency ' "
# FlashCache Contents
#######################
dcli -l root -g cell_group " cellcli -e ' LIST FLASHCACHECONTENT ATTRIBUTES dbUniqueName,dbID,objectNumber,tableSpaceNumber,cachedSize,cachedKeepSize,hitCount,missCount,hoursToExpiration ' "
# Reset FlashCache
alter cell events = "immediate cellsrv.cellsrv_flashcache(Reset,0,0,0)"
# For Exadata Server Software 11.2.3.2.0 onwards
################################################
#-- Flush FlashCache
CELLCLI> alter flashcache all fush
#-- Convert FlashCache Mode
CELLCLI> alter flashcache all fush
CELLCLI> drop flashcache
CELLCLI> list flashcache
CELLCLI> alter cell flashCacheMode=WriteThrough
or
CELLCLI> alter cell flashCacheMode=WriteBack
CELLCLI> create flashcache all
CELLCLI> list flashcache
# How to find which cell node is a SQL or session is running on and what it is doing?
#######################################################################################
cellcli -e ' LIST ACTIVEREQUEST where dbName = mydb and instanceNumber = 1 '
cellcli -e ' LIST ACTIVEREQUEST where sessionID=1271 and sessionSerNumber=31026 detail '
##########################
# Serial Numbers
##########################
#-- Exadata Rack serial number
/usr/bin/ipmitool sunoem cli "show /SP system_identifier"
#-- Exadata Compute/Cell node - serial number
/usr/bin/ipmitool sunoem cli "show /SYS product_serial_number"
#-- Rack level and Individual Serial Numbers
/opt/oracle.SupportTools/CheckHwnFWProfile -S
#################################################################
# IPMI Tool (ILOM) Commands
#################################################################
###############################
#-- ILOM (SP - Service Processor) Commands
###############################
/usr/bin/ipmitool sunoem cli "reset -script /SP"
#-- Restart ILOM
cellcli -e ' alter cell restart bmc '
or
/usr/bin/ipmitool sunoem cli "reset -script /SP"
#-- Force restarting ILOM
/usr/bin/ipmitool bmc reset cold
#-- If still unable to restart ILOM, try SSH to ILOM and issue follwoing.
reset -script /SP
###############################
# ILOM Server Commands
###############################
/usr/bin/ipmitool sunoem cli "reset -script /SYS"
/usr/bin/ipmitool sunoem cli "stop -script /SYS"
/usr/bin/ipmitool sunoem cli "start -script /SYS"
#-- Server Power (chassis power Commands: status, on, off, cycle, reset, diag, soft)
/usr/bin/ipmitool power status
/usr/bin/ipmitool power on
/usr/bin/ipmitool power soft
/usr/bin/ipmitool power off
/usr/bin/ipmitool power cycle
/usr/bin/ipmitool power reset
/usr/bin/ipmitool power diag
/usr/bin/ipmitool chassis power off
###############################
# ILOM Console
###############################
#-- Start Console
start -script /SP/Console
#-- Stop Console
Prese ESC and (
#-- Show ILOM Version & Firmware
/usr/bin/ipmitool sunoem cli "version"
#-- ILOM Cli
/usr/bin/ipmitool sunoem cli
#-- Show System properties
show -d properties -level 1 /SYS
show -d properties -level 1 /SP
show -d properties -level 2 /SP
show -d properties -o table -level all /SYS
#-- Show Targets
show -d targets /SP
show -d targets /SYS
#-- Show system faults
#
/usr/bin/ipmitool sunoem cli " show -level all /SP/faultmgmt"
# Show as table
/usr/bin/ipmitool sunoem cli "show -o table -level all /SP/faultmgmt"
/usr/bin/ipmitool sunoem cli "show -o table -level all /SP/faultmgmt"
/usr/bin/ipmitool sunoem cli "start -script /SP/faultmgmt/shell
fmadm faulty -a
"
/usr/bin/ipmitool sunoem cli "show -d properties -o table -level all /SYS/FAN_FAULT"
/usr/bin/ipmitool sunoem cli "show -d properties -o table -level all /SYS/PS_FAULT"
/usr/bin/ipmitool sunoem cli "show -d properties -o table -level all /SYS/TEMP_FAULT"
/usr/bin/ipmitool sunoem cli "show -d properties -o table -level all /SYS Target Type==(Indicator) Target==(*FAULT)"
#-- Show event log
/usr/bin/ipmitool sel list
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class<>(Audit)"
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==(ASR)"
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Severity==(critical)"
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Severity==(critical) -level all -output table"
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==(value) Type==(value) Severity==(value)"
/usr/bin/ipmitool sunoem cli "show /SP/session"
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==Sensor "
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==Fault "
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==Power "
/usr/bin/ipmitool sunoem cli "show /SP/logs/event/list Class==IPMI "
#-- Show Console history
/usr/bin/ipmitool sunoem cli "show /SP/console/history"
#-- Clear event log (when it gets full)
/usr/bin/ipmitool sunoem cli
--> set /SP/logs/event clear=true
#Get LED Light Status
dcli -l root -g all_group '/usr/bin/ipmitool sunoem led get all |egrep -v "OFF|na|OK.*ON" '
dcli -l root -g all_group '/usr/bin/ipmitool sunoem led get SERVICE'
dcli -l root -g all_group '/usr/bin/ipmitool sunoem led get /SP/SERVICE'
##########################
# Boot Order
##########################
#-- Get current boot order
/usr/bin/biosconfig -get_boot_order
#-- Change next boot device
#-- (Boot Order file can be obtained by -get_boot_order, and then change FIRST tag under BOOt_ORDER_OVERRISDE
biosconfig -set_boot_override <boot_order_filename>
##################################
# Disk Controller Commmands
##################################
#-- Help
/opt/MegaRAID/MegaCli/MegaCli64 -help
#-- Get all the information
/opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -a0
#-- Get disk controller log and event
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetEvents -info -aALL
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetEvents -info -aALL
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetEvents -info -aALL
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetEvents -info -aALL
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetEvents -info -aALL
/opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetLatest 100 -aALL
#-- Check Disk Controller Cache Policy (Should be Disabled)
dcli -l root -g all_group "/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -a0 |grep 'Disk Cache Policy'"
#-- Check cache policy at Individual Disks level (Should be WriteBack)
dcli -l root -g all_group "/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -a0 |grep 'Cache Policy:'"
dcli -l root -g all_group "/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -a0 |grep 'Cache Policy:' |grep Through"
#-- Check Battery Type
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | grep BatteryType'
#-- Check Battery Charging status
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | grep "Charging Status" '
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | grep "Discharging" '
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | while read line ; do echo '
Upgrade Disk Firmware
#######################
/opt/oracle.SupportTools/CheckHWnFWProfile -action updatefw -component HardDisk -attribute all_fw -slot 8:1 -caller MS
/opt/MegaRAID/MegaCli/MegaCli64 -PdFwDownload -PhysDrv[8:1] -f /var/log/exadatatmp/firmware/ActualFirmwareFiles/H7240AS60.A2D2.fw -a0
################################
#-- Check Battery Learn Cycle
###############################
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep "Auto-Learn Mode" '
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuProperties -a0'|grep 'Auto Learn Period'
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep "Learn Cycle Active" '
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep "Learn Cycle Requested" '
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep "Learn Cycle Status" '
#-- Initiate Battery Learn Cycle
/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -BbuLearn -a0
#-- Check auto learn mode, Should be enabled on compute node and disabled on cell node.
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuProperties -a0'|grep 'Auto-Learn Mode'
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuProperties -a0'|grep 'Auto Learn Period'
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuProperties -a0'
#-- Check current battery temperature (Should be < 60 C)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -a0 | grep Temperature:'
#-- Check current Battery Charge Capacity (Should be > 800 mAh)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuCapacityInfo -a0 | grep "Full Charge" '
#-- Check current Battery erors (Should be < 10%)
dcli -l root -g all_group '/opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus -a0 | grep "Max Error"'
#-- Display all information on Disk Controller
/opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL
#-- Display Summary
/opt/MegaRAID/MegaCli/MegaCli64 -ShowSummary -aALL
##########################
# Recreate the Cell
##########################
#-- Drop disks from ASM diskgroup
sqlplus / as sysasm
alter diskgroup DATA drop disks in failgroup LABCELL03 force;
alter diskgroup RECO drop disks in failgroup LABCELL03 force;
alter diskgroup DBFS_DG drop disks in failgroup LABCELL03 force;
#-- Create celldisk and griddisks
cellcli -e ' create celldisk all '
cellcli -e ' CREATE GRIDDISK ALL HARDDISK PREFIX=DATA , size=212G '
cellcli -e ' CREATE GRIDDISK ALL HARDDISK PREFIX=RECO , size=29.125G '
cellcli -e ' CREATE GRIDDISK ALL HARDDISK PREFIX=DBFS_DG , size=29.125G '
cellcli -e ' CREATE GRIDDISK ALL FlashDisk PREFIX=FLASH_DM01 , size=212G '
cellcli -e ' CREATE GRIDDISK ALL HARDDISK PREFIX=RECO , size=29.125G '
#-- Add disk to ASM diskgroup
sqlplus / as sysasm
alter diskgroup DATA add failgroup LABCELL03 disk 'o/192.168.10.5/DATA*' rebalance power 11 nowait;
alter diskgroup RECO add failgroup LABCELL03 disk 'o/192.168.10.5/RECO*' rebalance power 11 nowait;
alter diskgroup DBFS_DG add failgroup LABCELL03 disk 'o/192.168.10.5/DBFS_DG*' rebalance power 11 nowait;
##########################
# Exadata Cell Metrics
##########################
# Metric definition
cellcli -e describe METRICDEFINITION
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like 'FC.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like 'DB.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where description like '.*flash.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where description like '.*read.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where description like '.*write.*per.*sec.*' "
Cluster wide metrics
#---------------------------
# Smart IO Metrics
#---------------------------
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like 'SIO.*' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_EL_OF' " #The cumulative number of megabytes eligible for offload by smart I/O.
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_OF_RE' " #The cumulative number of interconnect megabytes returned by smart I/O.
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_EL_OF_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_OF_RE_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_RD_FC_SEC' " #Number of megabytes per second read from flash cache by smart IO
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_RD_HD_SEC' " #Number of megabytes per second read from hard disk by smart IO
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'SIO_IO_SI_SV_SEC' " #Number of megabytes per second saved by storage index
#---------------------------
# FlashCache Metrics
#---------------------------
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like '.*FC.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where description like '.*flash.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like '.*FC.*' and description like '.*read.*' "
cellcli -e "list METRICDEFINITION attributes name,metricType,objectType,persistencePolicy,unit,description where name like '.*FC.*' and description like '.*writ.*' "
FC_BY_ALLOCATED Instantaneous FLASHCACHE MB "Number of megabytes allocated in flash cache"
FC_BY_DIRTY Instantaneous FLASHCACHE MB "Number of unflushed megabytes in FlashCache"
FC_BY_STALE_DIRTY Instantaneous FLASHCACHE MB "Number of unflushed megabytes in FlashCache which cannot be flushed because cached disks are not accessible"
FC_BY_USED Instantaneous FLASHCACHE MB "Number of megabytes used on FlashCache"
FC_IO_BY_R Cumulative FLASHCACHE MB "Number of megabytes read from FlashCache"
FC_IO_BY_R_SEC Rate FLASHCACHE MB/sec "Number of megabytes read per second from FlashCache"
FC_IO_RQ_R Cumulative FLASHCACHE "IO requests" "Number of requests read from FlashCache"
FC_IO_RQ_R_SEC Rate FLASHCACHE IO/sec "Number of requests read per second from FlashCache"
FC_IO_BY_W Cumulative FLASHCACHE MB "Number of megabytes written to FlashCache"
FC_IO_BY_W_SEC Rate FLASHCACHE MB/sec "Number of megabytes per second written to FlashCache"
FC_IO_BY_W_POPULATE_SEC Rate FLASHCACHE MB/sec "Number of megabytes per second that are population writes into flash cache due to read miss"
# FlashCache Bytes Used Across the cluster
dcli -l root -g cell_group "cellcli -e list metriccurrent where name LIKE 'FC_BY_.*' " |sort -k2,2
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_BY_ALLOCATED' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_BY_DIRTY' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_BY_STALE_DIRTY' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_BY_USED' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_IO_BY_R_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_IO_BY_W_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_IO_RQ_R_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_IO_RQ_W_SEC' "
dcli -l root -g cell_group "cellcli -e list metriccurrent where name = 'FC_IO_BY_W_SEC' "
#---------------------------
# Database IO per second
#---------------------------
# Current Metric
cellcli -e "describe metriccurrent"
cellcli -e list metriccurrent where name = 'DB_IO_BY_SEC'
cellcli -e list metriccurrent where name = 'DB_FC_IO_BY_SEC'
cellcli -e list metriccurrent where name = 'FC_BY_USED'
######################
#-- Metric History
######################
cellcli -e "describe metrichistory"
#-------------------
# Last 10 minutes
#-------------------
cellcli -e "list metrichistory where name = 'DB_IO_BY_SEC' and collectionTime > '`date --date=\" 10 min ago \" +%Y-%m-%dT%H:%M:%S%z`' "
#----------------------------
# Last 1 hour for a database
#----------------------------
cellcli -e "list metrichistory attributes collectionTime,metricObjectName,name,metricValue,metricValueAvg,metricValueMax,metricValueMin where name = 'DB_IO_BY_SEC' \
and collectionTime > '`date --date=\" 1 hour ago \" +%Y-%m-%dT%H:%M:%S%z`' "
and metricObjectName = 'STBY52_PR01HSTI'
cellcli -e "list metrichistory attributes collectionTime,name,metricValue,metricValueAvg,metricValueMax,metricValueMin \
where name = 'FC_BY_USED' and collectionTime > '`date --date=\" 24 hours ago \" +%Y-%m-%dT%H:%M:%S%z`' "
#----------------------------
# Specific Time Window
#----------------------------
cellcli -e "list metrichistory where name = 'DB_IO_BY_SEC' and metricObjectName = 'STBY52_PR01PIMI' and collectionTime > '2015-04-16T14:00:00-08:00' and collectionTime < '2015-04-16T18:00:00-08:00' "
##########################################
# Storage Server Metric By database
###########################################
DB_FC_IO_BY_SEC The number of megabytes of I/O per second for this database to flash cache.
DB_FC_IO_RQ The number of I/O requests issued by a database to flash cache.
DB_FC_IO_RQ_SEC The number of I/O requests issued by a database to flash cache per second.
DB_FD_IO_BY_SEC The number of megabytes of I/O per second for this database to flash disks.
DB_FD_IO_LOAD The average I/O load from this database for flash disks.
DB_FD_IO_RQ_LG The number of large I/O requests issued by a database to flash disks.
DB_FD_IO_RQ_LG_SEC The number of large I/O requests issued by a database to flash disks per second.
DB_FD_IO_RQ_SM The number of small I/O requests issued by a database to flash disks.
DB_FD_IO_RQ_SM_SEC The number of small I/O requests issued by a database to flash disks per second.
DB_FD_IO_TM The cumulative latency of reading blocks by a database from flash disks.
DB_FD_IO_TM_RQ The rate which is the average latency of reading blocks per request by a database from flash disks.
DB_FD_IO_UTIL The percentage of flash resources utilized from this database.
DB_FL_IO_BY The number of megabytes written to Oracle Exadata Smart Flash Log.
DB_FL_IO_BY_SEC The number of megabytes written per second to Oracle Exadata Smart Flash Log.
DB_FL_IO_RQ The number of I/O requests issued to Oracle Exadata Smart Flash Log.
DB_FL_IO_RQ_SEC The number of I/O requests per second issued to Oracle Exadata Smart Flash Log.
FL_IO_DB_BY_W The number of megabytes written to hard disk by Oracle Exadata Smart Flash Log.
FL_IO_DB_BY_W_SEC The number of megabytes written per second were written to hard disk by Oracle Exadata Smart Flash Log.
DB_IO_BY_SEC The number of megabytes of I/O per second for this database to hard disks.
DB_IO_LOAD The average I/O load from this database for hard disks.
DB_IO_RQ_LG The cumulative number of large I/O requests issued by the database. A large value indicates a heavy large I/O workload from this database.
DB_IO_RQ_LG_SEC The rate of large I/O requests issued by a consumer group per second over the past minute. A large value indicates a heavy large I/O workload from this database in the past minute.
DB_IO_RQ_SM The cumulative number of small I/O requests issued by the database. A large value indicates a heavy small I/O workload from this database.
DB_IO_RQ_SM_SEC The rate of small I/O requests issued by a consumer group per second over the past minute. A large value indicates a heavy small I/O workload issued by this database in the past minute.
DB_IO_TM_LG The cumulative latency of reading large blocks by a database from hard disks.
DB_IO_TM_LG_RQ The rate which is the average latency of reading large blocks per request by a database from hard disks.
DB_IO_TM_SM The cumulative latency of reading small blocks by a database from hard disks.
DB_IO_TM_SM_RQ The rate which is the average latency of reading small blocks per request by a database from hard disks.
DB_IO_UTIL_LG The percentage of disk resources utilized by large requests from this database.
DB_IO_UTIL_SM The percentage of disk resources utilized by small requests from this database.
DB_IO_WT_LG The cumulative number of milliseconds that large I/O requests issued by the database have waited to be scheduled by IORM. A large value indicates that the I/O workload from this database is exceeding the allocation specified for it in the interdatabase plan.
DB_IO_WT_LG_RQ The average number of milliseconds that large I/O requests issued by the database have waited to be scheduled by IORM in the past minute. A large value indicates that the I/O workload from this database is exceeding the allocation specified for it in the interdatabase plan.
DB_IO_WT_SM The cumulative number of milliseconds that small I/O requests issued by the database have waited to be scheduled by IORM. A large value indicates that the I/O workload from this database is exceeding the allocation specified for it in the interdatabase plan.
DB_IO_WT_SM_RQ The average number of milliseconds that small I/O requests issued by the database have waited to be scheduled by IORM in the past minute. A large value indicates that the I/O workload from this database is exceeding the allocation specified for it in the interdatabase plan.
###############################################
# Tabular IO per database (MB/sec) on a cell
################################################
#fromtime="`date --date=\" 30 min ago \" +%Y-%m-%dT%H:%M:%S%z`"
#totime="`date +%Y-%m-%dT%H:%M:%S%z`"
fromtime='2016-02-16T07:30:00-08:00'
totime='2016-02-16T10:30:00-08:00'
# DB_IO_BY_SEC
# DB_IO_RQ_SM_SEC
# DB_IO_RQ_LG_SEC
# DB_IO_UTIL_SM
# DB_IO_UTIL_LG
# DB_FD_IO_RQ_SM_SEC
# DB_FD_IO_RQ_LG_SEC
declare -a DB=(`cellcli -e "list metriccurrent attributes metricObjectName where name = 'DB_IO_LOAD' " |awk '{print $1} ' |xargs echo`)
for ((i=-1; i<${#DB[@]}; i++))
do
if [ "$i" == -1 ]; then
printf "\n";
printf "%25s" "collectionTime"
else
printf "%$(expr ${#DB[$i]} + 1)s" "${DB[$i]}"
fi
if [ "$i" == `expr ${#DB[@]} - 1` ]; then printf " Total \n"; fi;
done; \
i=0; \
total=0; \
cellcli -e "list metrichistory attributes collectionTime,metricObjectName,metricValue where name = 'DB_IO_LOAD'
and collectionTime > '$fromtime' and collectionTime < '$totime' and metricObjectName like '.*.*' " |
while read LINE
do
collectionTime=`echo $LINE |awk '{print $1}'`
metricObjectName=`echo $LINE |awk '{print $2}'`
metricValue=`echo $LINE |awk '{print $3}'| tr -d ','` # Remove commas
#if [ "$metricObjectName" == "${DB[$(expr ${#DB[@]} - 1 )]}" ]
if [ "$i" -eq 0 ]
then
printf "%25s " "$collectionTime"
fi;
#Round off decimals
printf "%$(expr ${#DB[$i]} + 1)s" $(printf '%.*f' 0 "$metricValue")
total=$(awk "BEGIN {print $total + $metricValue; exit}")
#printf "%6s" "$metricValue"
i=$(expr $i + 1)
if [ $i -eq ${#DB[@]} ] ; then
i=0;
printf "%10s" $(printf '%.*f' 0 "$total");
printf "\n";
total=0;
fi
done; printf "\n";
cellcli -e "list metrichistory attributes collectionTime,metricObjectName,name,metricValue where collectionTime > '2015-05-21T05:36:46-0700' and collectionTime < '2015-05-21T05:40:46-0700' and name like '.*DB.*' and metricObjectName = 'STBY02_PR01MTGI' " |sort -k1,2,3
##########################################
# Tabular database IO on a cell
# (Multiple metrics for a database)
###########################################
# cellcli -e "list metriccurrent attributes metricObjectName where name = 'DB_IO_BY_SEC' " |awk '{print $1} '
#fromtime="`date --date=\" 30 min ago \" +%Y-%m-%dT%H:%M:%S%z`"
#totime="`date --date=\" 1 min ago \" +%Y-%m-%dT%H:%M:%S%z`"
fromtime='2015-11-08T00:00:00-08:00'
totime='2015-11-08T03:00:00-08:00'
DBNAME=ST01PIMI
#Order the metriclist in Alphabetical order
declare -a METRICLIST="DB_FC_IO_BY_SEC DB_FC_IO_RQ_SEC DB_FD_IO_UTIL DB_IO_BY_SEC DB_IO_LOAD DB_IO_RQ_LG_SEC DB_IO_RQ_SM_SEC DB_IO_WT_LG_RQ DB_IO_WT_SM_RQ " ; \
declare -a METRICLIST=(${METRICLIST}) ; \
echo "" ; \
echo "#############################################" ; \
echo "#### Database - $DBNAME " ; \
echo "#############################################" ; \
echo "" ; \
PRINT_FORMAT="%25s %12s %12s %12s %12s %12s %12s %12s %12s %12s %12s %12s" ; \
printf "${PRINT_FORMAT}" "--------------" "------" "-------" "---------" "------" "-------" "--------" "--------" "--------" "--------" ; printf "\n" ; \
printf "${PRINT_FORMAT}" "collectionTime" "Flash" "Flash" "FlashDisk" "Total" "" "" "" "Large IO" "Small IO" ; printf "\n" ; \
printf "${PRINT_FORMAT}" " " "Cache" "Cache" "Load" "IO" "IO Load" "Large IO" "Small IO" "AvgWait" "AvgWait" ; printf "\n" ; \
printf "${PRINT_FORMAT}" " " "MB/Sec" "Req/Sec" "(%)" "MB/Sec" "(%)" "Req/Sec" "Req/Sec" "(ms)" "(ms)" ; printf "\n" ; \
printf "${PRINT_FORMAT}" "--------------" "------" "-------" "---------" "------" "-------" "--------" "--------" "--------" "--------" ; printf "\n" ; \
printf "\n" ; \
i=0; \
cellcli -e "list metrichistory attributes collectionTime,metricObjectName,name,metricValue where
collectionTime > '$fromtime' and collectionTime < '$totime'
and metricObjectName like '$DBNAME'
and name like '$(echo ${METRICLIST[*]} | tr " " "|" )'
" | sort -k1,3 | while read LINE
do
collectionTime=`echo $LINE |awk '{print $1}'`
metricObjectName=`echo $LINE |awk '{print $2}'`
metricName=`echo $LINE |awk '{print $3}'`
if [ "$i" -eq 0 ]
then
printf "%25s" "$collectionTime"
fi
while [ $i -lt ${#METRICLIST[@]} ]
do
if [ "${METRICLIST[$i]}" == "$metricName" ]
then
metricValue=`echo $LINE |awk '{print $4}'`
break
else
metricValue=""
printf "%13s" "$metricValue"
i=$(expr $i + 1)
continue
fi
done;
# Remove commas
metricValue=$(echo $metricValue | tr -d ',' )
# Round off decimals
if [ "$metricName" != "DB_IO_LOAD" ] && [[ "$metricName" != DB_IO_WT* ]]
then
metricValue=$(printf '%.*f' 0 "$metricValue"|awk '{print $1}')
fi
printf "%13s" "$metricValue"
#printf "%30s" "$metricName $metricValue"
i=$(expr $i + 1)
if [ $i -ge ${#METRICLIST[@]} ] ; then
i=0;
printf "\n"
fi
done; printf "\n" ;
##########################
# colleclt Commands
##########################
collectl -scdmnfx -o T -i 2
##########################
# Colmux Commands
##########################
colmux -command -scdmnf -reverse -column 0 -addr <comma-separated-server-LIST or filename>
#-- For Exadata
colmux -command -scdmnf -reverse -column 0 -port 64000 -addr all_group
#---------------------------------------------
# Collectl top or Colmux top
# Show cluster wide top (order by CPU Pct)
#---------------------------------------------
colmux -command "-sZ -i:2 " -column 12 -lines 50 -port 64123 -addr dbs_group
colmux -command "-sZ -i:2 --procfilt=fora_smon " -column 12 -lines 50 -port 64123 -addr dbs_group
colmux -command "-sZ -i:2 --procfilt=fbt01pimi " -column 12 -lines 50 -port 64123 -addr dbs_group_bt01pimi
#-------------------------
# Hard Disk Statistics
#-------------------------
sudo colmux -command '-sD ' -lines 100 -reverse -column 1 -port 64002 -addr dm02cel01
# Show individual disk details
# Exadata spinning disks - sd[a-l]
# Exadata Flashdisks - sd[m-z] & sda[a-c]
colmux -command '-sD --dskfilt sd[a-b]$' -reverse -column 0 -port 64123 -addr cell_group
##########################
# Colplot commands
##########################
colplot -dir /var/log/collectl/consolidated -contains dm01db01 -date "20120909" -time "00:00-24:00" -plots cpu,mem,disk,net,nfssum,ib,paging,swap,inode,proc,tcp -filetype pdf -email emailaddress -subject "DM01 Plots"
colplot -dir /var/log/collectl/consolidated -contains dm01db01 -date "20120909-20120910" -time "00:00-24:00" -plots cpu,mem,disk,net,nfssum,ib,paging,swap,inode,proc,tcp -filetype pdf -email vishal@vishalgupta.com -subject "DM01 Plots"
colplot -dir /export/ora_stage/colplot -contains dm51cel01 -date "20130303" -time "00:00-24:00" -plots disk -filetype pdf -email vishal@vishalgupta.com -subject "DM51 Plots"
colplot -dir /usr/share/collectl/plotfiles -plots cpu -filetype png -email vishal@vishalgupta.com -email "/export/ora_stage/colplot_graphs/dm02db01.png"
colplot -dir /var/log/collectl -contains dm02db01 -date "20130325-20130325" -time "18:19-19:19" -plots cpu -width 0.7 -height 0.3 -filetype png -filedir "/export/ora_stage/colplot_graphs/hourly"
for i in {1..9}; do mkdir dm01db0$i; done
for i in {0..6}; do mkdir dm01db1$i; done
for i in {1..9}; do mkdir dm01db0$i; done
for i in {0..6}; do mkdir dm01db1$i; done
for i in {1..9}; do mkdir dm01cel0$i; done
for i in {0..9}; do mkdir dm01cel1$i; done
for i in {0..8}; do mkdir dm01cel2$i; done
##########################
# Install Collectl
##########################
#-- ColPlot require the http to installed on the server.
#-- Follwoign RPMs are needed for colplot to work
- httpd
- mailcap (For emails)
- sharutils (For /usr/bin/uuencode for emails)
- gnuplot
- ghostscript (For PDFs)
- cups-libs
- cairo
- libtiff
- urw-fonts
- chkfontpath
- xorg-x11-font-utils
- ghostscript-fonts (For PDFs)
dcli -l root -g ~/all_group -f /export/ora_stage/vishal/collectl/collectl-3.6.1-4.noarch.rpm -d /var/tmp/
dcli -l root -g ~/all_group -f /export/ora_stage/vishal/collectl/collectl-utils-3.1.0-1.noarch.rpm -d /var/tmp/
dcli -l root -g ~/all_group rpm -Uvh /var/tmp/collectl-3.6.1-4.noarch.rpm /var/tmp/collectl-utils-3.1.0-1.noarch.rpm
dcli -l root -g ~/all_group service collectl start
dcli -l root -g ~/all_group service collectl status
dcli -l root -g ~/dbs_group mkdir /var/www/html/colplot
dcli -l root -g ~/dbs_group ln -s /usr/share/doc/collectl-utils-3.2.1/FAQ-colplot.html /var/www/html/colplot/FAQ-colplot.html
dcli -l root -g ~/dbs_group ln -s /usr/share/doc/collectl-utils-3.2.1/FAQ-colgui.html /var/www/html/colplot/FAQ-colgui.html
dcli -l root -g ~/dbs_group ln -s /usr/share/doc/collectl-utils-3.2.1/colplot-help.html /var/www/html/colplot/colplot-help.html
##########################
# Setup Colplot
##########################
#-- Add "-P -oz" to following line in /etc/collectl.conf
DaemonCommands = -f /var/log/collectl -r00:00,7 -m -F60 -s+YZ -P -oz
dcli -l root -g all_group grep ^DaemonCommands /etc/collectl.conf
dcli -l root -g all_group -f /etc/collectl.conf -d /etc
dcli -l root -g all_group grep ^DaemonCommands /etc/collectl.conf
#-- Restart collectl
dcli -l root -g all_group service collectl status
dcli -l root -g all_group service collectl stop
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group 'ls /var/log/collectl/*tab*'
#-- Remove older file stored in compressed format.
dcli -l root -g all_group 'rm /var/log/collectl/*raw*'
dcli -l root -g all_group 'rm /var/log/collectl/*log*'
#-- Transfer existing files
for i in `cat all_group`
do
scp -p $i:/var/log/collectl/*tab /export/ora_stage/colplot/
scp -p $i:/var/log/collectl/*prc /export/ora_stage/colplot/
scp -p $i:/var/log/collectl/*slb /export/ora_stage/colplot/
done
#-- Setup collectl consolidation
# Added by Vishal Gupta for Ron Reimer
# It consolidates today's collectl statistics files from all nodes to single node
# so that graphs could be generated in a single browser
* * * * * /export/u02/dba/monitoring/scripts/collectl_consolidation.sh > /tmp/collectl_consolidation.log 2>&1
5 0 * * * /export/u02/dba/monitoring/scripts/collectl_consolidation_daily.sh > /tmp/collectl_consolidation_daily.log 2>&1
#-- Check Colplot default directory
dcli -l root -g dbs_group ' grep PlotDir /etc/colplot.conf'
###########################################
# Collectl Changes/Customizatons
###########################################
######### collectl ###########
- /etc/collectl.conf
- Gather configuration to store data in plottable format
# START - Changed by Vishal
#DaemonCommands = -f /var/log/collectl -r00:00,7 -m -F60 -sYZ
DaemonCommands = -f /var/log/collectl -r00:00,7 -m -F60 -P -oz -sbcdfijmnstxDN
# END - Changed by Vishal
######### colplot ###########
- /usr/bin/colplot
- Change colplot home page cgi script to stop it from trying to find earlier and latest file.
my $pparams= {fdate=>20010101, tdate=>29991231, contains=>'', unique=>$uniqueFlag };
#Changed by Vishal
#findFiles(3, $mycfg, $pparams, "$dir$sep*", undef) ||
# error("No plottable files match your selection criteria. Are your dir and/or file protections right?");
- Default the output radio button to last 30 mintues.
# START - Changed by Vishal
#print "<input type=radio name=timeframe value=fixed checked>\n";
#print "<font=5><i>OR</i>\n";
#print "<b>Last: <input type=text name=winsize size=1 value=$winsize> Minutes\n";
#print "<input type=radio name=timeframe value=float>\n";
print "<input type=radio name=timeframe value=fixed>\n";
print "<font=5><i>OR</i>\n";
print "<b>Last: <input type=text name=winsize size=1 value=$winsize> Minutes\n";
print "<input type=radio name=timeframe value=float checked>\n";
# END - Changed by Vishal
- Change Display to PlotSys (Plot-System).
#Start of Changes by Vishal
<option value=sp >SysPlot</option>
<option value=ps selected>PlotSys</option>
#End Changes by Vishal
dcli -l root -g ~/all_group -d /usr/bin -f /usr/bin/colplot
- /etc/colplot.conf
- Change default PlotDir
PlotDir = /export/ora_stage/colplot
- /usr/share/collectl/colplotlib.defs
- Define DiskRead and DiskWrite plots
diskread { title=DiskReadMB yname=[DSK]ReadKBTot }
diskwrite { title=DiskWriteMB yname=[DSK]WriteKBTot }
diskread { cat=disk type=s desc=Disk Read (MB) }
diskwrite { cat=disk type=s desc=Disk Write (MB) }
dcli -l root -g ~/all_group -d /usr/share/collectl/ -f /usr/share/collectl/colplotlib.defs
######### colmux ###########
- Changes in /usr/bin/colmux
- Add "-o ServerAliveInterval=3600". This helps to remove the cell servers dropping off from colmux output.
# START - Changed by Vishal
#my $Ssh='/usr/bin/ssh -o StrictHostKeyChecking=no -o BatchMode=yes';
my $Ssh='/usr/bin/ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ServerAliveInterval=60';
# END - Changed by Vishal
%s/ssh \-n/ssh \-n \-o ServerAliveInterval=3600/g
grep ssh /usr/bin/colmux
dcli -l root -g ~/all_group -d /usr/bin/ -f /usr/bin/colmux
grep ssh /usr/bin/colmux
ps -ef|grep collectl
- Change default age from 2 to 10
# START - Changed by Vishal
#my $age=2;
my $age=10;
# END - Changed by Vishal
- Change the interface to use for communication.
# Modified by Vishal
#$myaddr=`$Ifconfig $interface | grep addr:`;
$myaddr=`$Ifconfig eth0| grep addr:`;
dcli -l root -g ~/all_group -d /usr/bin -f /usr/bin/colmux
##########################
# Disable Collectl
##########################
dcli -l root -g ~/all_group service collectl stop
dcli -l root -g ~/all_group chkconfig collectl off
dcli -l root -g ~/all_group chkconfig --list collectl
##########################
# Enable Collectl
##########################
dcli -l root -g ~/all_group service collectl start
dcli -l root -g ~/all_group chkconfig collectl on
dcli -l root -g ~/all_group chkconfig --list collectl
##########################
# De-Install Collectl
##########################
dcli -l root -g ~/all_group service collectl stop
dcli -l root -g ~/all_group rpm -e collectl collectl-utils
#####################################
# Install collectl on all nodes
######################################
dcli -l root -g all_group cp -p /etc/collectl.conf /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.6.5-2.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#############################################
# Install collectl-utils on all nodes
############################################
dcli -l root -g dbs_group cp -p /etc/colplot.conf /tmp/
dcli -l root -g all_group '/usr/bin/collectl -v |head -1'
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.7.4.src.tar.gz -d /var/tmp/
dcli -l root -g all_group "cd /var/tmp/ ; gunzip -c collectl-3.7.4.src.tar.gz |tar xvf - > /dev/null"
dcli -l root -g all_group service collectl status
dcli -l root -g all_group service collectl stop
dcli -l root -g all_group chkconfig collectl off
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rpm -e collectl
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group "cd /var/tmp/collectl-3.7.4; sh INSTALL"
dcli -l root -g all_group '/usr/bin/collectl -v |head -1'
dcli -l root -g all_group -f /tmp/collectl.conf -d /tmp/
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm /var/tmp/collectl-3.7.4.src.tar.gz
dcli -l root -g all_group rm -fR /var/tmp/collectl-3.7.4
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#####################################
# Upgrade collectl to 3.6.5-2
######################################
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group cp -p /etc/collectl.conf /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.6.5-2.noarch.rpm -d /var/tmp/
dcli -l root -g all_group service collectl stop
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#####################################
# Downgrade collectl to 3.6.5-2
######################################
dcli -l root -g all_group cp -p /etc/collectl.conf /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.6.5-2.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-3.6.5-2.noarch.rpm
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#####################################
# Upgrade collectl to 3.6.9-1
######################################
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group cp -p /etc/collectl.conf /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.6.9-1.noarch.rpm -d /var/tmp/
dcli -l root -g all_group service collectl stop
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-3.6.9-1.noarch.rpm
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm /var/tmp/collectl-3.6.9-1.noarch.rpm
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#####################################
# Upgrade collectl to 3.7.4-1
######################################
dcli -l root -g all_group cp -p /etc/collectl.conf /tmp/
dcli -l root -g all_group '/usr/bin/collectl -v |head -1'
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-3.7.4.src.tar.gz -d /var/tmp/
dcli -l root -g all_group "cd /var/tmp/ ; gunzip -c collectl-3.7.4.src.tar.gz |tar xvf - > /dev/null"
dcli -l root -g all_group service collectl status
dcli -l root -g all_group service collectl stop
dcli -l root -g all_group chkconfig collectl off
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rpm -e collectl
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group "cd /var/tmp/collectl-3.7.4; sh INSTALL"
dcli -l root -g all_group '/usr/bin/collectl -v |head -1'
dcli -l root -g all_group cp -p /tmp/collectl.conf /etc/
dcli -l root -g all_group -d /etc/ -f /etc/collectl.conf
dcli -l root -g all_group service collectl start
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig collectl on
dcli -l root -g all_group chkconfig --list collectl
dcli -l root -g all_group rm /var/tmp/collectl-3.7.4.src.tar.gz
dcli -l root -g all_group rm -fR /var/tmp/collectl-3.7.4
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave
#####################################
# Upgrade collectl-utils to 3.1.0-1
######################################
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group cp -p /etc/colplot.conf /usr/bin/colplot /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-utils-3.1.0-1.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl-utils
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-utils-3.1.0-1.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-utils-3.1.0-1.noarch.rpm
dcli -l root -g all_group cp -p /tmp/colplot.conf /etc/
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave /etc/colplot.conf.rpmsave
dcli -l root -g all_group rpm -q collectl-utils
#dcli -l root -g all_group cp -p /tmp/colplot /usr/bin/colplot
#dcli -l root -g all_group cp -p /tmp/colmux /usr/bin/colmux
#####################################
# Upgrade collectl-utils to 3.2.1-1
######################################
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group cp -p /etc/colplot.conf /usr/bin/colplot /usr/bin/colmux /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-utils-3.2.1-1.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl-utils
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-utils-3.2.1-1.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-utils-3.2.1-1.noarch.rpm
dcli -l root -g all_group cp -p /tmp/colplot.conf /etc/
dcli -l root -g all_group cp -p /tmp/colplot /usr/bin/colplot
dcli -l root -g all_group cp -p /tmp/colmux /usr/bin/colmux
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave /etc/colplot.conf.rpmsave
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group service httpd restart
########################################
# Downgrade collectl-utils to 3.2.1-1
########################################
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group cp -p /etc/colplot.conf /usr/bin/colplot /usr/bin/colmux /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-utils-3.2.1-1.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl-utils
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-utils-3.2.1-1.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-utils-3.2.1-1.noarch.rpm
dcli -l root -g all_group cp -p /tmp/colplot.conf /etc/
dcli -l root -g all_group cp -p /tmp/colplot /usr/bin/colplot
dcli -l root -g all_group cp -p /tmp/colmux /usr/bin/colmux
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave /etc/colplot.conf.rpmsave
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group service httpd restart
######################################
# Upgrade collectl-utils to 4.7.1-1
######################################
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group cp -p /etc/colplot.conf /usr/bin/colplot /usr/bin/colmux /tmp/
dcli -l root -g all_group -f /export/ora_stage/vishal/collectl/collectl-utils-4.7.1-1.noarch.rpm -d /var/tmp/
dcli -l root -g all_group rpm -e collectl-utils
dcli -l root -g all_group rpm -Uvh /var/tmp/collectl-utils-4.7.1-1.noarch.rpm
dcli -l root -g all_group rm /var/tmp/collectl-utils-4.7.1-1.noarch.rpm
dcli -l root -g all_group -f /tmp/colplot.conf -d /etc/
dcli -l root -g all_group -f /tmp/colmux -d /usr/bin/
dcli -l root -g all_group rm -f /etc/collectl.conf.rpmsave /etc/colplot.conf.rpmsave
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group service httpd restart
##############################
# Install after cell patching (Collectl 3.7.4, collectl-utils 4.7.1-1 )
###############################
dcli -l root -g cell_group '/usr/bin/collectl -v |head -1'
dcli -l root -g cell_group -f /export/ora_stage/vishal/collectl/collectl-3.7.4.src.tar.gz -d /var/tmp/
dcli -l root -g cell_group "cd /var/tmp/ ; gunzip -c collectl-3.7.4.src.tar.gz |tar xvf - > /dev/null"
dcli -l root -g cell_group "cd /var/tmp/collectl-3.7.4; sh INSTALL"
dcli -l root -g cell_group '/usr/bin/collectl -v |head -1'
dcli -l root -g cell_group -f /export/ora_stage/vishal/collectl/collectl-utils-4.7.1-1.noarch.rpm -d /var/tmp/
dcli -l root -g cell_group rpm -Uvh /var/tmp/collectl-utils-4.7.1-1.noarch.rpm
dcli -l root -g cell_group rpm -q collectl-utils
dcli -l root -g cell_group rm /var/tmp/collectl-3.7.4.src.tar.gz
dcli -l root -g cell_group rm -fR /var/tmp/collectl-3.7.4
dcli -l root -g cell_group rm -f /etc/collectl.conf.rpmsave
dcli -l root -g cell_group rm /var/tmp/collectl-utils-4.7.1-1.noarch.rpm
dcli -l root -g cell_group -d /etc -f /etc/collectl.conf
dcli -l root -g cell_group -d /etc -f /etc/colplot.conf
dcli -l root -g cell_group -d /usr/bin/ -f /usr/bin/colmux
dcli -l root -g cell_group service collectl start
dcli -l root -g cell_group service collectl status
dcli -l root -g cell_group chkconfig collectl on
dcli -l root -g cell_group chkconfig --list collectl
##########################################
# Collect & Collectl-utils Verification
##########################################
dcli -l root -g all_group rpm -q collectl
dcli -l root -g all_group rpm -q collectl-utils
dcli -l root -g all_group service collectl status
dcli -l root -g all_group chkconfig --list collectl
#Customization Verification
# (Needs to have -P -oz at the end of the line)
dcli -l root -g all_group 'grep ^DaemonCommands /etc/collectl.conf'
# (PlotDir needs to be /export/ora_stage/colplot)
dcli -l root -g dbs_group 'grep PlotDir /etc/colplot.conf'
# Should be 10
dcli -l root -g all_group "grep '^my \$age' /usr/bin/colmux"
# Should have ServerAliveInterval=60 in it
dcli -l root -g all_group "grep '^my \$Ssh' /usr/bin/colmux"
# Should be commented
dcli -l root -g all_group "egrep 'findFiles.*undef' /var/www/html/colplot/index.cgi |head -1"
- /var/www/html/colplot/index.cgi
- Change colplot home page cgi script to stop it from trying to find earlier and latest file.
my $pparams= {fdate=>20010101, tdate=>29991231, contains=>'', unique=>$uniqueFlag };
#Changed by Vishal
#findFiles(3, $mycfg, $pparams, "$dir$sep*", undef) ||
# error("No plottable files match your selection criteria. Are your dir and/or file protections right?");
- Default the output radio button to last 30 mintues.
#Changed by Vishal
#print "<input type=radio name=timeframe value=fixed checked>\n";
print "<input type=radio name=timeframe value=fixed>\n";
print "<font=5><i>OR</i>\n";
print "<b>Last: <input type=text name=winsize size=1 value=$winsize> Minutes\n";
#Changed by Vishal
#print "<input type=radio name=timeframe value=float>\n";
print "<input type=radio name=timeframe value=float checked>\n";
###################################
# Troubleshooting
###################################
###################################
# Infiniband Troubleshooting
###################################
# Infinicheck
###################
/opt/oracle.SupportTools/ibdiagtools/infinicheck -g ~/dbs_ib_group
# infiniband commands
######################
ibswitches
ibhosts
ibnodes
ibstatus
iblinkinfo
# From first node
###################
uname -a
cat /etc/hosts
dcli -g all_group -l root 'imageinfo;rds-info -I|head -40;ibstat;ibstatus;ibnetdiscover;ibclearcounters;ibcheckstate -v;rpm -qa|grep -i ofa;'
# From ALL IB Switches
##########################
uname -a
version
env_test
listlinkup
getmaster
ibnetdiscover
#ibclearcounters
ibcheckstate -v
# Check whether link is physically present or not
cat /sys/class/net/ib0/carrier
cat /sys/class/net/ib1/carrier
##
export SUBNET_MGR_GID=`sminfo | cut -d" " -f7 | cut -c3-16`;
export SUBNET_MGR_LOC="OTHER";
for IB_NODE_GID in `ibswitches | cut -c14-27`;
do
if [ $SUBNET_MGR_GID = $IB_NODE_GID ];
then
export SUBNET_MGR_LOC="IB_SWITCH";
fi;
done;
echo $SUBNET_MGR_LOC;
###################################
# OS Watcher Commands
###################################
#############################################################
#-- Get IO Statistics on a hardisk from OS Watcher
#############################################################
File Format
###############
zzz ***Tue Feb 11 09:01:54 PST 2014 Sample interval: 5 secconds
Linux 2.6.32-400.11.1.el5uek (servername) 02/11/14
Time: 09:01:54
avg-cpu: %user %nice %system %iowait %steal %idle
1.56 0.00 1.46 1.43 0.00 95.55
Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util
sda 2.64 13.87 7.62 37.58 5994.46 4488.26 231.91 2.53 55.87 1.75 7.89
sda1 0.00 0.00 0.01 0.00 1.21 0.00 125.98 0.00 65.33 59.17 0.06
cd /opt/oracle.oswatcher/osw/archive/oswiostat
bunzip2 -cf *iostat_13.10.01.0400*.dat* | egrep -v Device | awk '
BEGIN {
date="";
time="";
}
{
if ($1 == "zzz")
{
date=$4 "-" $3 "-" $7
}
if ($1 == "Time:")
{
time=$2
}
# print entire line if svctm ($11) is greater than 50ms
#if ( $11 > 50) {
# print date " Time: " time " " $0 ;
# }
#
# print entire line if %util ($12) is greater than 50%
if ( $12 > 50) {
print date " Time: " time " " $0 ;
}
}
'
###################################################################################
#-- Get IO Statistics on a hardisk from OS Watcher (TABULAR for each disk device)
###################################################################################
### HardDisks ##########
cd /opt/oracle.oswatcher/osw/archive/oswiostat
bunzip2 -cf *iostat_14.02.22*.dat* | egrep -v Device | awk '
BEGIN {
date=""; time="";
printf "Date\t Time \tsda\tsdb\tsdc\tsdd\tsde\tsdf\tsdg\tsdh\tsdi\tsdj\tsdk\tsdl\n" ;
sda_val=""; sdb_val=""; sdc_val=""; sdd_val="";
sde_val=""; sdf_val=""; sdg_val=""; sdh_val="";
sdi_val=""; sdj_val=""; sdk_val=""; sdl_val="";
}
{
if ($1 == "zzz") {date=$4 "-" $3 "-" $7}
if ($1 == "Time:") {time=$2 }
# %util ($12)
if ( $1 == "sda") { sda_val= $12 };
if ( $1 == "sdb") { sdb_val= $12 };
if ( $1 == "sdc") { sdc_val= $12 };
if ( $1 == "sdd") { sdd_val= $12 };
if ( $1 == "sde") { sde_val= $12 };
if ( $1 == "sdf") { sdf_val= $12 };
if ( $1 == "sdg") { sdg_val= $12 };
if ( $1 == "sdh") { sdh_val= $12 };
if ( $1 == "sdi") { sdi_val= $12 };
if ( $1 == "sdj") { sdj_val= $12 };
if ( $1 == "sdk") { sdk_val= $12 };
if ( $1 == "sdl") { sdl_val= $12 };
if ($1 == "Time:") {
printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" ,date,time,sda_val,sdb_val,sdc_val,sdd_val,sde_val,sdf_val,sdg_val,sdh_val,sdi_val,sdj_val,sdk_val,sdl_val;
}
}
'
### Flash Disks ##########
cd /opt/oracle.oswatcher/osw/archive/oswiostat
bunzip2 -cf *iostat_13.10.01.0400*.dat* | egrep -v Device | awk '
BEGIN {
date=""; time="";
printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n","Date ","Time ","sdn","sdo","sdp","sdq","sdr","sds","sdt","sdu","sdv","sdw","sdx","sdy","sdz","sdaa","sdab","sdac" ;
sdn_val=""; sdo_val=""; sdp_val=""; sdq_val="";
sdr_val=""; sds_val=""; sdt_val=""; sdu_val="";
sdv_val=""; sdw_val=""; sdx_val=""; sdy_val="";
sdz_val=""; sdaa_val=""; sdab_val=""; sdac_val="";
}
{
if ($1 == "zzz") {date=$4 "-" $3 "-" $7}
if ($1 == "Time:") {time=$2 }
# %util ($12)
if ( $1 == "sdn") { sdn_val= $12 };
if ( $1 == "sdo") { sdo_val= $12 };
if ( $1 == "sdp") { sdp_val= $12 };
if ( $1 == "sdq") { sdq_val= $12 };
if ( $1 == "sdr") { sdr_val= $12 };
if ( $1 == "sds") { sds_val= $12 };
if ( $1 == "sdt") { sdt_val= $12 };
if ( $1 == "sdu") { sdu_val= $12 };
if ( $1 == "sdv") { sdv_val= $12 };
if ( $1 == "sdw") { sdw_val= $12 };
if ( $1 == "sdx") { sdx_val= $12 };
if ( $1 == "sdy") { sdy_val= $12 };
if ( $1 == "sdz") { sdz_val= $12 };
if ( $1 == "sdaa") { sdaa_val= $12 };
if ( $1 == "sdab") { sdab_val= $12 };
if ( $1 == "sdac") { sdac_val= $12 };
if ($1 == "Time:") {
printf ("%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\t%-s\n" ,date,time,sdn_val,sdo_val,sdp_val,sdq_val,sdr_val,sds_val,sdt_val,sdu_val,sdv_val,sdw_val,sdx_val,sdy_val,sdz_val,sdaa_val,sdab_val,sdac_val);
}
}
'
#############################################################
#-- Filter the OS Watcher Top output for a particular process
#############################################################
cd /opt/oracle.oswatcher/osw/archive/oswtop/
for i in `ls *14.04.24.0500.dat*`;
do
bunzip2 -c $i |egrep -h '^zzz|^top|tnslsnr LISTENER -inherit' | awk '
BEGIN { date="";time="";}
{
if ($1 == "zzz")
{
date=$4 "-" $3 "-" $7
}
else {
if ($1 == "top")
{
time=$3
}
else {
print date " " time " " $0 ;
}
}
}
'
done
#######################################################################
#-- Filter the OS Watcher Top output for high CPU usage processes
#######################################################################
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 - PID
2 - USER
3 - PRIORITY
4 - Nice
5 - Virtual Memory
6 - Resident Memory
7 - Shared Memory
8 - Status
9 - %CPU
10 - %Memory
11 - Time
12 - Command
cd /opt/oracle.oswatcher/osw/archive/oswtop/
for i in `ls *14.04.29.2000*.dat.bz2`
do
bunzip2 -c $i | egrep -v '^Tasks|^Mem|^Swap' | awk '
BEGIN { date=""; time=""; }
{
if ($1 == "zzz") { date=$4 "-" $3 "-" $7 }
else { if ($1 == "top") { time=$3 }
else { if ($9 >= 80) #Resource Usage
{ print date " " time " " $0 ; }
}
}
}
'
done
#######################################################################
#-- Filter the OS Watcher Top output for high CPU usage processes
#######################################################################
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 - PID
2 - USER
3 - PRIORITY
4 - Nice
5 - Virtual Memory
6 - Resident Memory
7 - Shared Memory
8 - Status
9 - %CPU
10 - %Memory
11 - Time
12 - Command
cd /opt/oracle.oswatcher/osw/archive/oswtop/
for i in `ls *14.04.29.2000*.dat.bz2`
do
bunzip2 -c $i | egrep -v '^Tasks|^Mem|^Swap' | awk '
BEGIN { date=""; time=""; }
{
if ($1 == "zzz") { date=$4 "-" $3 "-" $7 }
else { if ($1 == "top") { time=$3 }
else { if ($9 >= 80) #Resource Usage
{ print date " " time " " $0 ; }
}
}
}
'
done
#######################################################################
#-- Expect script
#######################################################################
# Add ssh keys between nodes
expect -c '
spawn dcli -l root -g all_group -k
while {1} {
expect "(yes/no)?" { send "yes\n"}
expect "password:" { send "welcome1\n"}
}
'
dcli -l root -g all_group hostname
dcli -l root -g dbs_group "dcli -l root -g all_group hostname |wc -l"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e 'LIST flashcache attributes name,status,size,creationTime,degradedCelldisks,effectiveCacheSize '"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e LIST FLASHLOG"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e DROP FLASHCACHE"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e ' CREATE GRIDDISK ALL FlashDisk PREFIX=FLASH_DM01 , size=20G '"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e ' CREATE FLASHLOG ALL FLASHDISK '"
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e ' CREATE flashcache ALL '"
dcli -l root -g /tmp/vishal/newgroups/cell_group " cellcli -e 'LIST flashcache attributes name,status,size,creationTime,degradedCelldisks,effectiveCacheSize ' "
dcli -l root -g /tmp/vishal/newgroups/cell_group "cellcli -e LIST FLASHLOG"
create diskgroup DATA_DM02_EXT NORMAL REDUNDANCY
disk 'o/*/DATA_DM02_EXT*'
ATTRIBUTE 'COMPATIBLE.ASM'='11.2.0.2.0'
, 'COMPATIBLE.RDBMS'='11.2.0.2'
, 'AU_SIZE'='4194304'
,'cell.smart_scan_capable'='TRUE'
;
##########################################
# Exadata Cell Parameters & Events
##########################################
Exadata Cell Events
######################
immediate cellsrv.cellsrv_statedump
immediate cellsrv.cellsrv_dump
immediate cellsrv.cellsrv_flashcache
immediate cellsrv.cellsrv_resetstats
immediate cellsrv.cellsrv_setparam
immediate cellsrv.cellsrv_storidx
trace[cellsrv.cellsrv_events_layer]
# Cell server system state dump
alter cell events = "immediate cellsrv.cellsrv_statedump(2,0)";
or issue kill -12 on cellsrv OS pid.
alter cell events = "immediate cellsrv.cellsrv_statedump(0,0)";
alter cell events = "immediate cellsrv.cellsrv_statedump(1,0)";
alter cell events = "immediate cellsrv.cellsrv_statedump(2,0)";
Reference - http://progeeking.com/2013/10/28/exadata-io-statistics/
#####################################################################
# I/O latency statistics (Note the IO L stats)
alter cell events="immediate cellsrv.cellsrv_dump('iolstats',0)";
alter cell events="immediate cellsrv.cellsrv_resetstats('iolstats')";
alter cell events="immediate cellsrv.cellsrv_dump('iolhiststats',0)";
# I/O reason statistics:
alter cell events="immediate cellsrv.cellsrv_dump('ioreasons',0)";
alter cell events="immediate cellsrv.cellsrv_resetstats('ioreasons')";
# Basic I/O statistics:
alter cell events="immediate cellsrv.cellsrv_dump('devio_stats',0)";
# Predicate I/O statistics:
alter cell events="immediate cellsrv.cellsrv_dump('predicateio',0)";
# Reset FlashCache Contents
alter cell events = "immediate cellsrv.cellsrv_flashcache(Reset,0,0,0)";
# Clear cellsrv statistics
alter cell events = "immediate cellsrv.cellsrv_resetstats()";
#
alter cell events = "immediate cellsrv.cellsrv_setparam('_cell_gen_time_stats_level','1')";
alter cell events = "immediate cellsrv.cellsrv_setparam('_cell_gen_time_stats_level','0')";
# To enable tracing of the auto disk management modules on a storage cell, run cellcli and enter the following:
alter cell events='trace[cellsrv.cellsrv_events_layer] memory=highest,disk=highest'
# To disable tracing of the auto disk management modules on a storage cell, run cellcli and enter the following:
alter cell events='trace[cellsrv.cellsrv_events_layer] off';
# Storage index - Diagnose Storage index id in Exadata cell server.
alter cell events = "immediate cellsrv.cellsrv_storidx(dumpridx, all, 0, 0, 0);
alter cell events="immediate cellsrv.cellsrv_storidx('dumpridx|purge|disable|enable','all|GridDiskName', objd, tsn, dbid)";
dumpridx - dump storage indexes
purge - purge storage indexes for specified disk/db/object
disable - disable storage indexes for specified disk/db/object
enable - disable storage indexes for specified disk/db/object
all/griddiskname - dump for all griddisk/dump for specific griddisk
objd - data_object_id from all_objects
tsn - tablespace number <20> ts# from ts$
dbid - ksqdngunid from x$ksqdn
# Dump cell memory (http://progeeking.com/2013/10/28/exadata-cellsrv-memory-usage/)
# Memory usage summary
alter cell events="immediate cellsrv.cellsrv_dump('memsummary',0)"
# SGA Memory Summary
alter cell events="immediate cellsrv.cellsrv_dump('sgaheapsummary',0)";
# PGA Memory Details
alter cell events="immediate cellsrv.cellsrv_dump('pgaheap',0)";
# Dump cell parameters
alter cell events="immediate cellsrv.cellsrv_dump('cellparams',0)";
# Setting $OSSCONF/cellinit.ora parameters dynamically.
alter cell events="immediate cellsrv.cellsrv_setparam('_cell_io_hang_time','90')"
alter cell events="immediate cellsrv.cellsrv_setparam('_cell_io_hang_kill_time','95')"
alter cell events="immediate cellsrv.cellsrv_setparam('_cellrsdef_heartbeat_timeout','10')" (default 6s)
Possible parameter
_cell_1mb_buffers_hugepage_support=false
_cell_disable_ant_check_reid=true
_cell_io_hang_reboot=false
_cell_io_hang_time=60
_cell_num_16k_buffers=2000
_cell_num_1mb_brr_buffers=5
_cell_num_1mb_buffers=200
_cell_num_1mb_bwr_buffers=5
_cell_num_32k_buffers=1000
_cell_num_64k_buffers=1000
_cell_num_8k_buffers=5000
_cell_num_buffers=1200
_cell_print_all_params=true
_cellrsbkp_poll_invl=15
_cellrsdef_fast_restart=0
_cellrsdef_heartbeat_timeout=6
_cellrsdef_srvc_cleanup_time=5
_cellrsdef_srvc_dump=30
_cellrsms_poll_invl=60
_cellrsos_poll_invl=15
_cellrssrv_poll_invl=15
_ms_cell_ioctl_timeout=60000
_reconnect_to_cell_attempts=4
_reonnect_to_cell_freq_in_sec=4
_skgxp_gen_ant_off_rpc_timeout_in_sec=300
_skgxp_gen_rpc_timeout_in_sec=90
_skgxp_udp_use_tcb=false
_skgxp_udp_use_tcb=false
# Storage Index parameters
_cell_enable_storage_index_for_loads=TRUE
_cell_enable_storage_index_for_writes=TRUE
_cell_si_max_num_diag_mode_dumps=20
_cell_storage_index_columns=0
_cell_storage_index_diag_mode=false
_cell_storage_index_partial_rd_sectors=512
_cell_storage_index_partial_reads_threshold_percent=85
_cell_storage_index_sizing_factor=2
alter cell events="immediate cellsrv.cellsrv_dump('cellparams',0)";
Dumping configuration parameter values
version = 0.0 (default = )
_cell_max_pll_pred_writes = 36
_cell_pred_writes_autotune_enabled = TRUE
_cell_max_pll_pred_reads = 36
_cell_pred_reads_autotune_enabled = TRUE
_cell_max_cellsup_time_in_sec = 60
_cell_max_flash_largeios = 48
_cell_num_threads_in_short_wait = 40
_cell_max_pll_pred_filters = 24 (default = 0)
_cell_pred_filters_autotune_enabled = TRUE
_cell_num_threads = 100
_cell_num_buffers = 5000
_cell_num_1mb_buffers = 5000 (default = 0)
_cell_num_1mb_bwr_buffers = 180
_cell_num_1mb_brr_buffers = 180
_cell_max_dynbufs_memsize = 3072 (default = 0)
_cell_listener_port = 5042
_cell_listener_backlog = 1000
_cell_listener_pll_jobs = 23
_cell_listener_req_batch = 100
_cell_num_0_byte_recv_ports = 4
_cell_num_512_byte_recv_ports = 1
_cell_num_2_kb_recv_ports = 1
_cell_num_4_kb_recv_ports = 2
_cell_num_8_kb_recv_ports = 4
_cell_num_16_kb_recv_ports = 2
_cell_num_32_kb_recv_ports = 2
_cell_num_64_kb_recv_ports = 1
_cell_num_1_mb_recv_ports = 2 (default = 1)
_ms_cell_ioctl_timeout = 600000
_cell_cd_metadata_slow_list =
_cell_cd_metadata_prepare_timeout_in_sec = 5
_cell_iorm_test_mode = FALSE
_cell_iorm_perf_stats = FALSE
_cell_iorm_wl_mode = 0
_cell_iorm_hipri_alloc = 0
_cell_iorm_medpri_alloc = 0
_cell_iorm_lowpri_alloc = 0
_cell_iorm_asm_alloc = 0
_cell_iorm_lutil_limit = 0
_cell_iorm_hints_enabled = FALSE
_iorm_hint0 = -1
_iorm_priority0 = -1
_iorm_hint1 = -1
_iorm_priority1 = -1
_iorm_hint2 = -1
_iorm_priority2 = -1
_iorm_hint3 = -1
_iorm_priority3 = -1
_iorm_hint4 = -1
_iorm_priority4 = -1
_iorm_hint5 = -1
_iorm_priority5 = -1
_iorm_hint6 = -1
_iorm_priority6 = -1
_iorm_hint7 = -1
_iorm_priority7 = -1
_cell_iorm_pri_catidx = -1
_cell_iorm_pri_dbidx = -1
_cell_iorm_pri_cgidx = -1
_cell_iorm_enable = TRUE
_cell_iorm_max_io = 0
_cell_iorm_max_lio = 0
_cell_iorm_conc_writes = 0
_cell_iorm_deadline = 0
_cell_iorm_fake_dbs = 0
_cell_iorm_busy_util = 15
_cell_hard_disable = FALSE
_cell_raise_softassert_on_harderr = FALSE
_cell_enable_ossnet_checksum = 2
_cell_enable_skgxp_stats = TRUE
_skgxp_udp_use_tcb = TRUE
_skgxp_udp_use_tcb_client = TRUE
_cell_memory_tracing = TRUE
_cell_dmpsga_enabled = FALSE
_cell_enable_dynamic_credits = TRUE
_cell_num_ios_per_predjob = 10
_cell_num_pred_flashio_corrupt_retries = 1000
_cell_pred_polling_ctl_enabled = TRUE
_cell_pred_sim_block_byteord_conv = FALSE
_cell_max_kuty_failure_diagnostics = 0
_cell_print_all_params = FALSE
_cell_pred_disable_destbuf_refill = FALSE
_cell_smartio_passthru_enabled = FALSE
_cell_pred_no_predio_limit = FALSE
_cell_pred_enable_io_buffer_eviction = TRUE
_cell_pred_enable_dest_buffer_eviction = TRUE
_cell_pred_enable_flashio = TRUE
_cell_snapshot_bufsize = 1
_cell_snapshot_interval = 100
_cell_gen_time_stats_level = 1
_cell_gen_time_stats_timer_level = 0
_cell_force_split_gdisk = FALSE
_cell_testlevel = 0
_cell_max_receive_buffers_per_port = 600
_cell_num_8k_buffers = 10000
_cell_num_16k_buffers = 5000
_cell_num_32k_buffers = 5000
_cell_num_64k_buffers = 5000
_cell_max_receive_buffers_8k_port = 1000
_cell_max_receive_buffers_1mb_port = 50
_cell_crash_on_error = 0
_cell_crash_on_error_skip_n = 0
_cell_safefile_sync_enabled = FALSE
_cell_1mb_buffers_hugepage_support = TRUE
_skgxp_udp_interface_detection_time_secs = 1
_skgxp_gen_ant_ping_misscount = 8
_skgxp_gen_ant_ping_misscount_delay = 225
_disable_diskmon_tcp_monitor = FALSE
_disable_diskmon_subnet_manager_query = FALSE
_skgxp_min_zcpy_len = 2147483647
_skgxp_min_rpc_rcv_zcpy_len = 2147483647
_skgxp_zcpy_flags = 2147483647
_skgxp_ctx_flags1 = 0
_skgxp_ctx_flags1mask = 0
_skgxp_dynamic_protocol = 0
_skgxp_inets = 0
_skgxpg_last_parameter = 27
_skgxp_ant_options = 0
_libcell_enable_libcell_interrupts = 1
_cell_rcvport_hist_size = 0
_skgxp_gen_rpc_no_path_check_in_sec = 1
_skgxp_gen_rpc_timeout_in_sec = 300
_skgxp_gen_ant_off_rpc_timeout_in_sec = 10
_reconnect_to_cell_freq_in_sec = 2
_reconnect_to_cell_attempts = 7
_disconnect_to_cell_attempts = 2
_reconnect_controls_reset_interval = 60
_dskm_disable_reconnect_to_cell = FALSE
_cell_disable_resource_leak_check = FALSE
_cell_disable_ant_check_reid = FALSE
_cell_disable_proactive_drop = FALSE
_cell_server_event =
_cell_client_event =
_cell_offloadserver_event =
_cell_reserve_hugepage_memory_mb = 24
_cell_tolerates_max_backward_drift_microsecs = 300000
_cell_num_sched_log_entries = 8192
_cell_storage_index_columns = 8 (default = 0)
_cell_storage_index_partial_reads_threshold_percent = 85
_cell_storage_index_partial_rd_sectors = 512
_cell_enable_storage_index_for_loads = TRUE
_cell_enable_storage_index_for_writes = TRUE
_cell_storage_index_diag_mode = 0
_cell_storage_index_sizing_factor = 2
_cell_pred_max_smartio_sessions = 2320 (default = 0)
_cell_pred_max_core_exec_threads = 23 (default = 14)
_cell_core_exec_ctl_level = 0
_cell_pred_num_ios_toissue_flashobj = 2
_cell_max_topheap_references = 50
_cell_pred_max_cus_per_filter = 1
_cell_load_timezone_during_boot = TRUE
_cell_sendport_private_rqh_pool_size = 10
_cell_sendport_global_rqh_num_pools = 512
_cell_sendport_global_rqh_pool_maxincr = 150
_cell_capability_version = 0
_cell_iolat_stats_disable = FALSE
_cell_pred_mapelem_split_size = -1
_cell_perf_flags = 0
_cell_enable_sbuf_check = FALSE
_cell_disable_crash_dump_enhancement = FALSE
_cell_object_expiration_hours = 24
_cell_mutex_stats = 0
_cell_port_activity_threshold = 300000
_cell_ant_port_activity_threshold = 1800000
_cell_ant_port_noopen_threshold = 60000
_cell_in_lrg_testing = FALSE
_cell_write_simulate_hard_error_freq = 0
_cell_assert_on_flash_data_corruption = 0
_cell_flashcache_diag_reads_frequency = 0
_cell_flashcache_enable_coalescing = TRUE
_cell_flashcache_mdextent_chksum = 2
_cell_read_flash_data_verif_level = 3
_cell_flashcache_data_verif_aura1 = TRUE
_cell_flashcache_data_verif_aura2 = FALSE
_cell_read_flash_gdisk_verif_level = 3
_cell_max_retry_on_read_flash_gdisk_verif_err = 2
_cell_enable_read_verif_on_these_gdisks =
_cell_enable_read_verif_on_gdisk_first_N_MB = -1
_cell_flash_cache_sanity_checking = 0
_cell_fc_force_caching_for_nocache_ios = FALSE
_cellrsdef_fast_restart = 1
_cell_max_memory = 22171 (default = 0)
_cell_max_dump_objects = 300000
_cell_max_connections = 1500 (default = 0)
_cell_sga_lowmem_threshold_size = 1024 (default = 0)
_cell_nomem_threshold_enabled = TRUE
_cell_sga_lowmem_threshold_enabled = TRUE
_cell_disable_heap_summary = FALSE
_cell_flashcache_max_FDOM_outst_ios = 70
_cell_flashcache_dsync_jobs = 4
_cell_wait_gdisk_refcnt_timeout_in_sec = 600
_cell_gd_io_quiescent_timeout = 10
_cell_auto_close_fd_interval = 120
_cell_dump_sga_on_oom_exception = FALSE
_cell_quarantine_manager_disabled = FALSE
_cell_qm_disable_sql_step_quarantine = FALSE
_cell_qm_disable_disk_region_quarantine = FALSE
_cell_qm_db_quarantine_threshold = 3
_cell_qm_offload_quarantine_threshold = 3
_cell_thread_max_trace_file_size = -1
_cell_redolog_fast_ack = FALSE
_cell_disable_flashcache_db_blk_chksum = FALSE
_cell_disable_platform_decryption = FALSE
_cell_disable_flash_gdisk_db_blk_chksum = FALSE
_cell_auto_dump_errstack = TRUE
_cell_auto_dump_errstack_interval = 900
_cell_si_max_num_diag_mode_dumps = 20
_cell_fc_persistence_max_io_retry = 1
_cell_fc_slowflush_sleep_interval = 500
_cell_fc_persistence_state = 0
_cell_fc_md_shadow_paging_enabled = TRUE
_cell_fc_use_32k_cachelines = FALSE
_cell_fc_scan_tc_blksize = 4
_cell_fc_autocache_scans = TRUE
_cell_fc_scan_elgblty_threshold = 1
_cell_fc_scan_tc_threshold = 2
_cell_fc_scan_tc_cooling_threshold = 10
_cell_disable_crash_dump_enhancement = FALSE
_cell_object_expiration_hours = 24
_cell_mutex_stats = 0
_cell_port_activity_threshold = 300000
_cell_ant_port_activity_threshold = 1800000
_cell_ant_port_noopen_threshold = 60000
_cell_in_lrg_testing = FALSE
_cell_write_simulate_hard_error_freq = 0
_cell_assert_on_flash_data_corruption = 0
_cell_flashcache_diag_reads_frequency = 0
_cell_flashcache_enable_coalescing = TRUE
_cell_flashcache_mdextent_chksum = 2
_cell_read_flash_data_verif_level = 3
_cell_flashcache_data_verif_aura1 = TRUE
_cell_flashcache_data_verif_aura2 = FALSE
_cell_read_flash_gdisk_verif_level = 3
_cell_max_retry_on_read_flash_gdisk_verif_err = 2
_cell_enable_read_verif_on_these_gdisks =
_cell_enable_read_verif_on_gdisk_first_N_MB = -1
_cell_flash_cache_sanity_checking = 0
_cell_fc_force_caching_for_nocache_ios = FALSE
_cellrsdef_fast_restart = 1
_cell_max_memory = 22171 (default = 0)
_cell_max_dump_objects = 300000
_cell_max_connections = 1500 (default = 0)
_cell_sga_lowmem_threshold_size = 1024 (default = 0)
_cell_nomem_threshold_enabled = TRUE
_cell_sga_lowmem_threshold_enabled = TRUE
_cell_disable_heap_summary = FALSE
_cell_flashcache_max_FDOM_outst_ios = 70
_cell_flashcache_dsync_jobs = 4
_cell_wait_gdisk_refcnt_timeout_in_sec = 600
_cell_gd_io_quiescent_timeout = 10
_cell_auto_close_fd_interval = 120
_cell_dump_sga_on_oom_exception = FALSE
_cell_quarantine_manager_disabled = FALSE
_cell_qm_disable_sql_step_quarantine = FALSE
_cell_qm_disable_disk_region_quarantine = FALSE
_cell_qm_db_quarantine_threshold = 3
_cell_qm_offload_quarantine_threshold = 3
_cell_thread_max_trace_file_size = -1
_cell_redolog_fast_ack = FALSE
_cell_disable_flashcache_db_blk_chksum = FALSE
_cell_disable_platform_decryption = FALSE
_cell_disable_flash_gdisk_db_blk_chksum = FALSE
_cell_auto_dump_errstack = TRUE
_cell_auto_dump_errstack_interval = 900
_cell_si_max_num_diag_mode_dumps = 20
_cell_fc_persistence_max_io_retry = 1
_cell_fc_slowflush_sleep_interval = 500
_cell_fc_persistence_state = 0
_cell_fc_md_shadow_paging_enabled = TRUE
_cell_fc_use_32k_cachelines = FALSE
_cell_fc_scan_tc_blksize = 4
_cell_fc_autocache_scans = TRUE
_cell_fc_scan_elgblty_threshold = 1
_cell_fc_scan_tc_threshold = 2
_cell_fc_scan_tc_cooling_threshold = 10
_cell_fc_scan_tc_decay_zone_sz = 4
_cell_fc_scan_tc_num_decay_zones = 6
_cell_fc_scan_threshold_timeout = 1
_cell_fc_oltp_resv_pcntg = 50
_cell_fc_cache_unkept_tbls = TRUE
_cell_fc_num_hashbuckets = 0
_cell_fc_aura2_md_shadow_paging_enabled = FALSE
_cell_fc_bootstrap_timeout = 5000000
_cell_fc_cache_mirror_writes = 1
_cell_fc_dw_batch_size = 1
_cell_fc_lru_scan4freeable_depth = 128
_cell_fc_enable_opread = TRUE
_cell_fc_replacement_threshold = 94
_cell_fc_trim_threshold = 97
_cell_fc_lrg_compression_ratio = 5
_cell_fc_outstanding_trim_ios = 32
_cell_simulate_railroad_crashes = FALSE
_cell_fc_toresilver_limit_chdrs = 390000
_cell_qm_max_simulated_railroad_crashes = 2
_cell_latency_warning_threshold =
_cell_latency_threshold_check_interval = 360000
_cell_latency_threshold_print_warning = FALSE
_cell_si_expensive_debug_tracing = FALSE
_cell_si_lock_pool_num_locks = 1024
_cell_poor_perf_schedule_time = 5000
_cell_iohang_schedule_time = 500
_cell_assert_unsafe_allocmem = FALSE
_cell_fplib_fix_control = 0
Unable to lookup value for parameter _lost_cache_detect
_cell_num_vers_check_fail_messages = 0
_cell_qm_db_quarantine_time_threshold = 86400
_cell_pthread_stacksize = 10485760
_cell_flashlog_flags = 0
_cell_flashlog_max_active_table_size = 8191
_cell_secure_erase_power = 5
_cell_mpp_cpu_freq = 2
_ms_listener_port = 5043
_cell_mpp_threshold = 90
_cell_mpp_max_pushback = 50
_si_write_diag_disable = FALSE
_cell_max_cellsrvstat_sessions = 3
_cell_si_diag_mode_force = FALSE
_cell_tracefile_max_size = 1610612736
_cell_state_dump_options = 0
_cell_num_offload_processes = 0 (default = 1)
_cell_num_offload_groupmsg_threads = 0
_cell_in_debug_mode = FALSE
_cell_oflsrv_heartbeat_timeout_sec = 6
ERROR: _gentimestats_rut_thresh has invalid ossp_conf size of 8
_skgxp_diag_bucket_size = 32
_cell_diag_bucket_size = 131072 (default = 128)
Unable to lookup value for parameter _gentimestats_rut_disable_stkdmp
_ms_mprotect_corrupt_buf = TRUE
_cell_perf_action =
_cell_perf_tests =
_cell_fc_enable_resilvering = TRUE
_cell_netsend_out_jobq_count = 10
_cell_cache_out_jobq_count = 50
_cell_aio_ctx_count = 5
_cell_iops_perf_testing = FALSE
_cell_flashcache_dsync_jobs = 4
_cell_wait_gdisk_refcnt_timeout_in_sec = 600
_cell_gd_io_quiescent_timeout = 10
_cell_auto_close_fd_interval = 120
_cell_dump_sga_on_oom_exception = FALSE
_cell_quarantine_manager_disabled = FALSE
_cell_qm_disable_sql_step_quarantine = FALSE
_cell_qm_disable_disk_region_quarantine = FALSE
_cell_qm_db_quarantine_threshold = 3
_cell_qm_offload_quarantine_threshold = 3
_cell_thread_max_trace_file_size = -1
_cell_redolog_fast_ack = FALSE
_cell_disable_flashcache_db_blk_chksum = FALSE
_cell_disable_platform_decryption = FALSE
_cell_disable_flash_gdisk_db_blk_chksum = FALSE
_cell_auto_dump_errstack = TRUE
_cell_auto_dump_errstack_interval = 900
_cell_si_max_num_diag_mode_dumps = 20
_cell_fc_persistence_max_io_retry = 1
_cell_fc_slowflush_sleep_interval = 500
_cell_fc_persistence_state = 0
_cell_fc_md_shadow_paging_enabled = TRUE
_cell_fc_use_32k_cachelines = FALSE
_cell_fc_scan_tc_blksize = 4
_cell_fc_autocache_scans = TRUE
_cell_fc_scan_elgblty_threshold = 1
_cell_fc_scan_tc_threshold = 2
_cell_fc_scan_tc_cooling_threshold = 10
_cell_fc_scan_tc_decay_zone_sz = 4
_cell_fc_scan_tc_num_decay_zones = 6
_cell_fc_scan_threshold_timeout = 1
_cell_fc_oltp_resv_pcntg = 50
_cell_fc_cache_unkept_tbls = TRUE
_cell_fc_num_hashbuckets = 0
_cell_fc_aura2_md_shadow_paging_enabled = FALSE
_cell_fc_bootstrap_timeout = 5000000
_cell_fc_cache_mirror_writes = 1
_cell_fc_dw_batch_size = 1
_cell_fc_lru_scan4freeable_depth = 128
_cell_fc_enable_opread = TRUE
_cell_fc_replacement_threshold = 94
_cell_fc_trim_threshold = 97
_cell_fc_lrg_compression_ratio = 5
_cell_fc_outstanding_trim_ios = 32
_cell_simulate_railroad_crashes = FALSE
_cell_fc_toresilver_limit_chdrs = 390000
_cell_qm_max_simulated_railroad_crashes = 2
_cell_latency_warning_threshold =
_cell_latency_threshold_check_interval = 360000
_cell_latency_threshold_print_warning = FALSE
_cell_si_expensive_debug_tracing = FALSE
_cell_si_lock_pool_num_locks = 1024
_cell_poor_perf_schedule_time = 5000
_cell_iohang_schedule_time = 500
_cell_assert_unsafe_allocmem = FALSE
_cell_fplib_fix_control = 0
Unable to lookup value for parameter _lost_cache_detect
_cell_num_vers_check_fail_messages = 0
_cell_qm_db_quarantine_time_threshold = 86400
_cell_pthread_stacksize = 10485760
_cell_flashlog_flags = 0
_cell_flashlog_max_active_table_size = 8191
_cell_secure_erase_power = 5
_cell_mpp_cpu_freq = 2
_ms_listener_port = 5043
_cell_mpp_threshold = 90
_cell_mpp_max_pushback = 50
_si_write_diag_disable = FALSE
_cell_max_cellsrvstat_sessions = 3
_cell_si_diag_mode_force = FALSE
_cell_tracefile_max_size = 1610612736
_cell_state_dump_options = 0
_cell_num_offload_processes = 0 (default = 1)
_cell_num_offload_groupmsg_threads = 0
_cell_in_debug_mode = FALSE
_cell_oflsrv_heartbeat_timeout_sec = 6
ERROR: _gentimestats_rut_thresh has invalid ossp_conf size of 8
_skgxp_diag_bucket_size = 32
_cell_diag_bucket_size = 131072 (default = 128)
Unable to lookup value for parameter _gentimestats_rut_disable_stkdmp
_ms_mprotect_corrupt_buf = TRUE
_cell_perf_action =
_cell_perf_tests =
_cell_fc_enable_resilvering = TRUE
_cell_netsend_out_jobq_count = 10
_cell_cache_out_jobq_count = 50
_cell_aio_ctx_count = 5
_cell_iops_perf_testing = FALSE
_cell_netsend_returnjob_thres_for_8Kbuf = 500
_kse_snap_ring_size = 0
_kse_snap_ring_record_stack = FALSE
_cell_num_priv_fsa_8k_buffers = 0
_cell_num_dynamic_priv_fsa_8k_buffers = 60
_cell_num_dynamic_priv_fsa_cachegets = 50
_cell_num_dynamic_priv_fsa_bids = 50
_cell_dynamic_priv_fsa_8k_iops_threshold = 160000
_cell_assert_on_diagmode_catching_wrong_results = FALSE
_cell_cd_scrubbing_exclude_list =
_cell_receiveport_buf_alloc_incr = 10
_cell_num_qos_1_mb_recv_ports = 1
_cell_max_receive_buffers_qos_1mb_port = 10
_cell_oob_max_growth_percent = 20
_cell_oob_max_growth_qos = 250
_oss_enable_scrub_repair = TRUE
_cell_system_thread_max_trace_file_size = 6291456
_cell_timer_granularity_msec = 5
_cell_ffi_flush_interval = 2000
_cell_ffi_max_memory_usage_in_mb = 2000
_cell_netsend_queue_thresh = 500
_cell_port_frozen_threshold = 3600000
_dskm_test_fence_postpone = 0
_cell_rslv_poor_health = TRUE
_cell_hugetlbtext_relocate = TRUE
_cell_allow_write_cancellation_upon_hair_trigger = FALSE
_cell_enable_buffer_owner_hist = FALSE
Cellsrv Default Trace Size Limit 15042511
###################################
ZFS Tuning
##############
NFS parameter listen_backlog=128
ZFS52-2# ndd -set /dev/tcp tcp_conn_req_max_q 1024
ZFS52-2# ndd -set /dev/tcp tcp_conn_req_max_q0 8192
###################################
Exadata Network Interface Check
###################################
##-- Check bonded interface slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bond0/bonding/slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bond1/bonding/slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bond2/bonding/slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bondeth0/bonding/slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bondeth1/bonding/slaves
dcli -l root -g ~/dbs_group cat /sys/class/net/bondib0/bonding/slaves
##-- Check network interface status
dcli -l root -g ~/dbs_group ethtool eth0|egrep 'detected|Speed'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth1|egrep 'detected|Speed'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth2|egrep 'detected|Speed'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth3|egrep 'detected|Speed'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth4|egrep 'detected|Speed'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth5|egrep 'detected|Speed'|sort -k2
##-- Check supported network interface speed
dcli -l root -g ~/dbs_group ethtool eth0|egrep -A2 ' Supported link modes'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth1|egrep -A2 ' Supported link modes'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth2|egrep -A2 ' Supported link modes'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth3|egrep -A2 ' Supported link modes'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth4|egrep -A2 ' Supported link modes'|sort -k2
dcli -l root -g ~/dbs_group ethtool eth5|egrep -A2 ' Supported link modes'|sort -k2
##-- Check IP Address
dcli -l root -g ~/dbs_group ifconfig bond0|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig bond1|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig bond2|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig bondeth0|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig bondeth1|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig bondib0|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth0|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth1|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth2|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth3|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth4|grep 'inet addr'
dcli -l root -g ~/dbs_group ifconfig eth5|grep 'inet addr'
###############################
Exadata Unix Account Creation
###############################
dcli -l root -g ~/dbs_group mkdir -p /home/vgupta
dcli -l root -g ~/dbs_group useradd -u 24117 -g techdept -G dbapm -d /home/vgupta -s /bin/bash vgupta
## change user password expiry information
dcli -l root -g ~/dbs_group chage -E -1 -m 0 -M 99999 -W 7 vgupta
dcli -l root -g ~/dbs_group chown vgupta:vgupta /home/vgupta
dcli -l root -g ~/dbs_group "passwd vgupta <<EOF
password
password
EOF
"
#################
Exadata Tuning
#################
#------------------------------------------
# Increase vm.max_map_count to 200000
#------------------------------------------
# Max oracle process memory is allowd upto (_realfree_heap_pagesize_hint * vm.max_map_count ).
# Default value of vm.max_map_count is 65536. This can be changed on the fly
# Default value of _realfree_heap_pagesize_hint parameter is 65536. It requires instance bounce to change this value.
# Check
dcli -l root -g dbs_group cat /proc/sys/vm/max_map_count
dcli -l root -g dbs_group /sbin/sysctl vm.max_map_count
# Change
dcli -l root -g dbs_group 'echo 200000 > /proc/sys/vm/max_map_count'
dcli -l root -g dbs_group 'echo vm.max_map_count=200000 >> /etc/sysctl.conf'
##########################
Exadata Disk Scrubbing
##########################
# Check Disk scrubbing schedule
dcli -l root -g ~/cell_group ' grep hardDiskScrubInterval $OSSCONF/cell_disk_config.xml'
dcli -l root -g ~/cell_group ' grep hardDiskScrubStartTime $OSSCONF/cell_disk_config.xml'
# Stop disk scrubbing
dcli -l root -g ~/cell_group " cellcli -e ALTER CELL hardDiskScrubInterval=none "
# Schedule disk scrubbing time
dcli -l root -g ~/cell_group " \" cellcli -e ALTER CELL hardDiskScrubStartTime='2015-08-23T00:00:00-07:00' \" "
# Set disk scrubbing interval (Valid options are daily, weekly, biweekly and none). Time is in ms.
# 14 days = 1209600000 ms
# 7 days = 604800000 ms
# 1 day = 86400000 ms
dcli -l root -g ~/cell_group " cellcli -e ALTER CELL hardDiskScrubInterval=biweekly "
-------------------------------------------------------------------
-- ASM Disk Errors
-------------------------------------------------------------------
SELECT failgroup
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_00',read_errs+write_errs,0)) CD_00
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_01',read_errs+write_errs,0)) CD_01
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_02',read_errs+write_errs,0)) CD_02
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_03',read_errs+write_errs,0)) CD_03
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_04',read_errs+write_errs,0)) CD_04
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_05',read_errs+write_errs,0)) CD_05
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_06',read_errs+write_errs,0)) CD_06
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_07',read_errs+write_errs,0)) CD_07
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_08',read_errs+write_errs,0)) CD_08
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_09',read_errs+write_errs,0)) CD_09
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_10',read_errs+write_errs,0)) CD_10
, sum(DECODE(substr(name,INSTR(name,'_',-1,3)+1,5), 'CD_11',read_errs+write_errs,0)) CD_11
FROM gv$asm_disk_stat
where name like '%CD%'
GROUP BY failgroup
ORDER BY 1;
##########################
# Exadata Rsync backup
##########################
BACKUP_DIR=/u23/oraback/exadata/`hostname -s`/
rsync -a -v --one-file-system --delete /app ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /bin ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /boot ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /etc ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /lib ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /lib64 ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /mnt ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /opt ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /root ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /sbin ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /selinux ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /srv ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /usr ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /var ${BACKUP_DIR}
rsync -a -v --one-file-system --delete /u01 ${BACKUP_DIR} --exclude='/u01/patches/' --exclude='*.aud' --exclude='*.trc' --exclude='*.trm'
BACKUP_DIR=/u23/oraback/exadata/`hostname -s`/
rsync -a -v --one-file-system --delete /u01 ${BACKUP_DIR} --exclude='/u01/patches/' --exclude='*.aud' --exclude='*.trc' --exclude='*.trm'
Reference in New Issue Copy Permalink