WSO2 ESB SERVER AUTO MAINTAIN SCRIPT

Linux OS Topics
Post Reply
User avatar
nwclasantha
Posts: 75
Joined: Wed Apr 24, 2013 12:57 am
Location: Malabe

WSO2 ESB SERVER AUTO MAINTAIN SCRIPT

Post by nwclasantha » Tue Dec 29, 2015 10:31 am

Code: Select all

#!/bin/bash -e
#Developed By [email protected]
while :
    do
ESB_IP=192.168.200.86
SOAP_COMM_PORT=8290
HTTP_COMM_PORT=9453
#
SOAP_URI="http://$ESB_IP:$SOAP_COMM_PORT/services/echo?wsdl"
HTTPS_GUI="https://$ESB_IP:$HTTP_COMM_PORT/carbon/admin/login.jsp"
#
MEMINFO="/proc/meminfo"
PARK_LOC="/app"
JDK_BIN_LOC="$PARK_LOC/JAVADIR/jdk1.6.0_31/bin"
WSO2_HOME="$PARK_LOC/wso2esb-4.8.0"
THEDIR="$PARK_LOC/log/wso2-watchdog"
LOG="$PARK_LOC/log"
LOCKFILE="$PARK_LOC/log/is_ready"
#
LIMIT0=51
LIMIT1=90
LIMIT2=87
LIMIT3=89
LIMIT4=91
LIMIT5=99
#
MSG_HEADER="Maintain Log - Automated Mail From 2.86 ESB Server"
#
DATE=`date +%Y-%m-%d" "%H:%M:%S`
TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
#
mkdir -p $THEDIR
if [ "$USEDMEM" -gt 0 ]
 then
    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER	
    if [[ $USEDMEMPER -gt $LIMIT2 && $USEDMEMPER -lt $LIMIT3 ]] ; then	
            echo "Memory in the Range... True"
		if [ ! -f $LOCKFILE ]; then
         	     find $THEDIR/ -type f -name '*.zip' -exec rm {} \;
	             find $LOG/ -type f -name '*.zip' -exec rm {} \;
                 echo "$DATE | Memory: $USEDMEMPER%, (limit: $LIMIT2%) "
                 WSO2_PID=`ps -ef | grep ClassLoaderLogManager | grep -v grep | awk '{print $2}'`
                 echo "Creating JAVA Thread Dump Time-wise...."     
                 find $THEDIR/ -type f -name '*.dump' -exec rm {} \;               			   
		         $JDK_BIN_LOC/jstack -l $WSO2_PID > $THEDIR/WSO2_Threaddump_$(date +"%m-%d-%Y_%H-%M").dump
	             cd $THEDIR ; zip -r WSO2_Treaddump.zip *                     			   
		         tail -n500 $WSO2_HOME/repository/logs/wso2carbon.log > $LOG/wso2carbon.log               
		         cd $LOG ; zip -r wso2carbon.zip wso2carbon.log		         
                 echo $WSO2_PID > /dev/null
	             echo $USEDMEMPER > /dev/null
	             sleep 30
                 echo "JAVA Thread Dump Has Been Created...." 
		         touch $LOCKFILE
		fi
     else        
	    echo "Didn't Creating Thread Dump and Restarted.... "
     fi    
fi

    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER
	if [[ $USEDMEMPER -gt $LIMIT0 && $USEDMEMPER -lt $LIMIT1 ]] ; then  
            response_1=$(curl -k -s --head -w %{http_code} $SOAP_URI -o /dev/null)
            response_2=$(curl -k -s --head -w %{http_code} $HTTPS_GUI -o /dev/null)

            if [[ $response_1 -eq 200 && $response_2 -eq 200 ]] ; then
               echo "The ESB is Up & Running..."
            else
               echo "This URL Not Exist"
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory Before ESB Restart $USEDMEMPERLOG% | Memory Before ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               echo "killing all the Java PIDs at Once on Live ESB...."
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               echo "Waiting 20s to start ...."
               sleep 20
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               free && sync && echo 3 > /proc/sys/vm/drop_caches && free 
               echo "Stating WSO2_ESB Service...."               
               nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               RST01=`echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%"`
               echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.86" | mail -a $LOG/esb-free.log -a $LOG/wso2carbon.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected],[email protected]
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               sleep 15
            fi			
    fi
  
    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER	
    if [[ $USEDMEMPER -gt $LIMIT4 && $USEDMEMPER -lt $LIMIT5 ]] ; then	
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory Before ESB Restart $USEDMEMPERLOG% | Memory Before ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               echo "killing all the Java PIDs at Once on Live ESB...."
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               echo "Waiting 20s to start ...."
               sleep 20
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               free && sync && echo 3 > /proc/sys/vm/drop_caches && free 
               echo "Stating WSO2_ESB Service...."               
               nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log > /dev/null 2>&1 &
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               RST01=`echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%"`
               echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.86" | mail -a $LOG/esb-free.log -a $LOG/wso2carbon.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected],[email protected]
	        echo $USEDMEMPER > /dev/null
                tail -1 $LOG/esb-free.log   
                sleep 15
     fi    
find /app/log/is_ready -type f -mmin +20 -exec rm {} \; > /dev/null 2>&1
#nohup sh /app/ESB_Maintain_Script/wso_auto_maintain.sh > /dev/null 2>&1 &
sleep 10
done
User avatar
Neo
Site Admin
Site Admin
Posts: 2642
Joined: Wed Jul 15, 2009 2:07 am
Location: Colombo

Re: WSO2 ESB SERVER AUTO MAINTAIN SCRIPT

Post by Neo » Tue Mar 29, 2016 3:32 pm

Very valuable post. I have put the code inside CODE tags to improve readability.
User avatar
nwclasantha
Posts: 75
Joined: Wed Apr 24, 2013 12:57 am
Location: Malabe

Re: WSO2 ESB SERVER AUTO MAINTAIN SCRIPT

Post by nwclasantha » Fri May 13, 2016 6:14 pm

Code: Select all

#!/bin/bash -e
#Developed By [email protected]
while :
    do
ESB_IP=192.168.200.86
SOAP_COMM_PORT=8280
HTTP_COMM_PORT=9443
#
SOAP_URI="http://$ESB_IP:$SOAP_COMM_PORT/services/echo?wsdl"
HTTPS_GUI="https://$ESB_IP:$HTTP_COMM_PORT/carbon/admin/login.jsp"
#
MEMINFO="/proc/meminfo"
PARK_LOC="/app"
JDK_BIN_LOC="$PARK_LOC/JAVADIR/jdk1.6.0_31/bin"
WSO2_HOME="$PARK_LOC/wso2esb-4.8.0"
THEDIR="$PARK_LOC/log/wso2-watchdog"
LOG="$PARK_LOC/log"
LOCKFILE="$PARK_LOC/log/is_ready"
#
LIMIT0=51
LIMIT1=80
LIMIT2=75
LIMIT3=79
LIMIT4=81
LIMIT5=99
#
MSG_HEADER="Maintain Log - Automated Mail From 200.86 ESB Server"
#
DATE=`date +%Y-%m-%d" "%H:%M:%S`
TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
#
mkdir -p $THEDIR
if [ "$USEDMEM" -gt 0 ]
 then
    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER	
    if [[ $USEDMEMPER -gt $LIMIT2 && $USEDMEMPER -lt $LIMIT3 ]] ; then	
            echo "Memory in the Range... True"
		if [ ! -f $LOCKFILE ]; then
         	     find $THEDIR/ -type f -name '*.zip' -exec rm {} \;
	             find $LOG/ -type f -name '*.zip' -exec rm {} \;
                     echo "$DATE | Memory: $USEDMEMPER%, (limit: $LIMIT2%) "
                     WSO2_PID=`ps -ef | grep ClassLoaderLogManager | grep -v grep | awk '{print $2}'`
                     echo "Creating JAVA Thread Dump Time-wise...."     
                     find $THEDIR/ -type f -name '*.dump' -exec rm {} \;               			   
		     $JDK_BIN_LOC/jstack -l $WSO2_PID > $THEDIR/WSO2_Threaddump_$(date +"%m-%d-%Y_%H-%M").dump
	             cd $THEDIR
                     zip -r WSO2_Treaddump.zip *			   
		     tail -n500 $WSO2_HOME/repository/logs/wso2carbon.log > $LOG/wso2carbon.log               
		     cd $LOG
		     zip -r wso2carbon.zip wso2carbon.log
                     echo $WSO2_PID > /dev/null
	             echo $USEDMEMPER > /dev/null
	             sleep 30
                     echo "JAVA Thread Dump Has Been Created...." 
		     touch $LOCKFILE
		fi
     else        
	    echo "Didn't Creating Thread Dump and Restarted.... "
     fi    
fi
    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER
	if [[ $USEDMEMPER -gt $LIMIT0 && $USEDMEMPER -lt $LIMIT1 ]] ; then  
            response_1=$(curl -k -s --head -w %{http_code} $SOAP_URI -o /dev/null)
            response_2=$(curl -k -s --head -w %{http_code} $HTTPS_GUI -o /dev/null)

            if [[ $response_1 -eq 200 && $response_2 -eq 200 ]] ; then
               echo "The ESB is Up & Running..."
            else
               echo "This URL Not Exist"
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory Before ESB Restart $USEDMEMPERLOG% | Memory Before ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               echo "killing all the Java PIDs at Once on Live ESB...."
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               echo "Waiting 20s to start ...."
               sleep 20
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               #free && sync && echo 3 > /proc/sys/vm/drop_caches && free 
               echo "Stating WSO2_ESB Service...."               
               nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE ESB HEALTH STATUS WAS UNSUCCESSFUL | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               RST01=`echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%"`
               echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.86" | mail -a $LOG/esb-free.log -a $LOG/wso2carbon.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected]
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               sleep 15
            fi			
    fi
    USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
    USEDMEMPERLOG=$USEDMEMPER	
    if [[ $USEDMEMPER -gt $LIMIT4 && $USEDMEMPER -lt $LIMIT5 ]] ; then	
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory Before ESB Restart $USEDMEMPERLOG% | Memory Before ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log
               echo "killing all the Java PIDs at Once on Live ESB...."
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               echo "Waiting 20s to start ...."
               sleep 20
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               #free && sync && echo 3 > /proc/sys/vm/drop_caches && free 
               echo "Stating WSO2_ESB Service...."               
               nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log > /dev/null 2>&1 &
               TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
               USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
               USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
               echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%" >> $LOG/esb-free.log
               RST01=`echo "$DATE | Memory Limit Reached ($LIMIT%): Used Memory After ESB Restart $USEDMEMPERLOG% | Memory After ESB Restart: $USEDMEMPER%"`
               echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.86" | mail -a $LOG/esb-free.log -a $LOG/wso2carbon.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected]
	       echo $USEDMEMPER > /dev/null
               tail -1 $LOG/esb-free.log   
               sleep 15
     fi 
LOCKFILE_RST="/app/log/is_ready_rst"
MAX_TIME="0430"
MIN_TIME="0400"
     if [ ! -f $LOCKFILE_RST ]; then
              echo "Continue for time range calculation Process.."
     if ( now=$(date +%H%M) ; ! [[ $now < $MIN_TIME ]] && ! [[ $now > $MAX_TIME ]] ) ;
     then             
              echo "killing all the Java PIDs at Once on Live ESB...."
              kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
              echo "Waiting 20s to start ...."
              sleep 20
              kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
              #free && sync && echo 3 > /proc/sys/vm/drop_caches && free 
              echo "Stating WSO2_ESB Service...."               
              nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log > /dev/null 2>&1 &
              echo "Result = ESB Auto Restarted as a fresh copy to be running @ $MAX_TIME A.M. on Server 200.86" | mail -s "$MSG_HEADER" [email protected]
	      touch $LOCKFILE_RST              
     else
              echo "Out of time range ..."
     fi
     else
              echo " The $LOCKFILE_RST is already exsist ....."
     fi	
    RESULT_DB_CONN_TIMEOUT=`grep -H -R "Error in opening DBMS connection" $WSO2_HOME/logs/wso2carbon.log | cut -d: -f1`

            if [ -z "$RESULT_DB_CONN_TIMEOUT" ]
            then
               echo "Already Stopped ESB.."          
            else
               echo $RESULT_DB_CONN_TIMEOUT > /dev/null
               echo "killing all the Java PIDs at Once on Live ESB...."
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               echo "Waiting 20s to start ...."
               sleep 20
               kill -9 `ps -ef | grep wso2esb-4.8.0 | grep -v grep | awk '{print $2}'` > /dev/null 2>&1 
               #free && sync && echo 3 > /proc/sys/vm/drop_caches && free
               mv $WSO2_HOME/repository/logs/wso2carbon.log  $WSO2_HOME/repository/logs/wso2carbon$(date +"%m-%d-%Y_%H-%M").log               			   
               echo "Stating WSO2_ESB Service...."               
               nohup sh $WSO2_HOME/bin/wso2server.sh >> $WSO2_HOME/repository/logs/wso2carbon.log > /dev/null 2>&1 &
               echo "Result = Error in opening DBMS connection Problum Resolved on Server 200.86" | mail -s "$MSG_HEADER" [email protected]                
            fi
find /app/log/is_ready_rst -type f -mmin +120 -exec rm {} \; > /dev/null 2>&1	 
find /app/log/is_ready -type f -mmin +20 -exec rm {} \; > /dev/null 2>&1 &
#nohup sh /app/ESB_Maintain_Script/wso_auto_maintain.sh > /dev/null 2>&1 &
sleep 10
done

wso_auto_maintain.pdf
Post Reply

Return to “Linux”