Code: Select all
#!/bin/bash -e
#Developed By [email protected]
while :
do
MEMINFO="/proc/meminfo"
JDK_BIN_LOC="/app/JAVADIR/jdk1.7.0_79/bin"
JB_HOME="/app/jboss-as-7.1.3.Final"
BIN_DIR="$JB_HOME/bin"
LOG="/app/log"
THEDIR="$LOG/jboss-watchdog"
#
TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
#
JBOSS_IP=192.168.200.188
ESB_IP=192.168.200.190
JBOSS_PORT=9294
SOAP_COMM_PORT=8677
HTTP_COMM_PORT=443
JBOSS_URI="http://$JBOSS_IP:$JBOSS_PORT/cc-erp/private/loginSuccess.view.control"
SOAP_URI="http://$ESB_IP:$SOAP_COMM_PORT/services/echo?wsdl"
HTTPS_GUI="https://$ESB_IP:$HTTP_COMM_PORT/carbon/admin/login.jsp"
#
LOCKFILE="/app/log/is_ready"
#
LIMIT0=90
LIMIT1=85
LIMIT2=88
LIMIT_MIN=55
LIMIT_MAX=80
#
MSG_HEADER="Maintain Log - Automated Mail From 200.188 JBOSS Server"
#
DATE=`date +%Y-%m-%d" "%H:%M:%S`
mkdir -p $THEDIR
if [ "$USEDMEM" -gt 0 ]
then
USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
USEDMEMPERLOG=$USEDMEMPER
if [[ $USEDMEMPER -gt $LIMIT1 && $USEDMEMPER -lt $LIMIT2 ]]
then
echo "Memory in the Range... True"
if [ ! -f $LOCKFILE ]; then
find $THEDIR/ -type f -name '*.zip' -exec rm {} \;
find $LOG/ -type f -name '*.zip' -exec rm {} \;
echo "$DATE | Memory: $USEDMEMPER%, (limit: $LIMIT1%) | Creating Thread Dump ...."
JBOSS_PID=`ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'`
echo "Creating JAVA Thread Dump Time-wise...."
find $THEDIR/ -type f -name '*.dump' -exec rm {} \;
$JDK_BIN_LOC/jstack -l $JBOSS_PID > $THEDIR/Jboss_Threaddump_$(date +"%m-%d-%Y_%H-%M").dump
cd $THEDIR
zip -r Jboss_Treaddump.zip *
tail -n500 $JB_HOME/standalone/log/server.log > $LOG/Jboss_server.log
cd $LOG
zip -r Jboss_server.zip Jboss_server.log
echo $JBOSS_PID > /dev/null
echo $USEDMEMPER > /dev/null
sleep 30
echo "JAVA Thread Dump Has Been Created...."
touch $LOCKFILE
fi
else
echo "Out of Range ...FALSE"
echo "Didn't Creating Thread Dump and Restarted.."
fi
fi
#
USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
USEDMEMPERLOG=$USEDMEMPER
if [[ $USEDMEMPER -gt $LIMIT_MIN && $USEDMEMPER -lt $LIMIT_MAX ]] ; then
response_2=$(curl -k -s --head -w %{http_code} $JBOSS_URI -o /dev/null)
if [[ $response_2 -eq 405 ]] ; then
echo "The JBOSS is Up & Running..."
else
response_0=$(curl -k -s --head -w %{http_code} $SOAP_URI -o /dev/null)
response_1=$(curl -k -s --head -w %{http_code} $HTTPS_GUI -o /dev/null)
if [[ $response_0 -eq 200 && $response_1 -eq 200 ]] ; then
echo "The LB ESBs are Up & Running..."
echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Restarting Jboss...."
echo "killing all the java PIDs at Once on Live Jboss...."
RESULT_1=`ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'`
if [ -z "$RESULT_1" ]
then
echo -n $"Already Stopped jboss Service"
else
echo $RESULT_1 > /dev/null
$BIN_DIR/jboss-cli.sh --connect command=:shutdown > /dev/null 2>&1
sleep 15
fi
RESULT_2=`ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'`
if [ -z "$RESULT_2" ]
then
echo -n $"Already Stopped jboss.."
else
echo $RESULT_2 > /dev/null
kill -9 `ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'` > /dev/null 2>&1
echo "Waiting 10s to start ...."
sleep 10
fi
free && sync && echo 3 > /proc/sys/vm/drop_caches && free
echo "Stating Jboss Service..."
sh $BIN_DIR/jboss-start.sh
TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Memory After Jboss Restart: $USEDMEMPER%" >> $LOG/jboss-free.log
RST01=`echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Memory After Jboss Restart: $USEDMEMPER%"`
echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.188" | mail -a $LOG/jboss-free.log -a $LOG/Jboss_server.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected],[email protected]
echo $USEDMEMPER > /dev/null
tail -1 $LOG/jboss-free.log
fi
fi
fi
#
if [ "$USEDMEM" -gt 0 ]
then
USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
USEDMEMPERLOG=$USEDMEMPER
if [ $USEDMEMPER -lt $LIMIT0 ]; then
echo "$DATE | Memory: $USEDMEMPER%, (limit: $LIMIT0%) | Don't Restart Jboss..."
else
response_0=$(curl -k -s --head -w %{http_code} $SOAP_URI -o /dev/null)
response_1=$(curl -k -s --head -w %{http_code} $HTTPS_GUI -o /dev/null)
if [[ $response_0 -eq 200 && $response_1 -eq 200 ]] ; then
echo "The LB ESBs and JBOSS are Up & Running..."
echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Restarting Jboss...."
echo "killing all the java PIDs at Once on Live Jboss...."
RESULT_1=`ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'`
if [ -z "$RESULT_1" ]
then
echo -n $"Already Stopped jboss Service"
else
echo $RESULT_1 > /dev/null
$BIN_DIR/jboss-cli.sh --connect command=:shutdown > /dev/null 2>&1
sleep 15
fi
RESULT_2=`ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'`
if [ -z "$RESULT_2" ]
then
echo -n $"Already Stopped jboss.."
else
echo $RESULT_2 > /dev/null
kill -9 `ps -ef | grep jboss-logmanager-1.3.2.Final.jar | grep -v grep | awk '{print $2}'` > /dev/null 2>&1
echo "Waiting 10s to start ...."
sleep 10
fi
free && sync && echo 3 > /proc/sys/vm/drop_caches && free
echo "Stating Jboss Service..."
sh $BIN_DIR/jboss-start.sh
TOTAL=`cat $MEMINFO | grep MemTotal: | awk '{print $2}'`
USEDMEM=`cat $MEMINFO | grep Active: | awk '{print $2}'`
USEDMEMPER=$(($USEDMEM * 100 / $TOTAL))
echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Memory After Jboss Restart: $USEDMEMPER%" >> $LOG/jboss-free.log
RST01=`echo "$DATE | Memory Limit Reached ($LIMIT0%): Used Memory $USEDMEMPERLOG% | Memory After Jboss Restart: $USEDMEMPER%"`
echo "Result = $RST01 and JAVA Thread Dump Has Been Created @ Server 200.188" | mail -a $LOG/jboss-free.log -a $LOG/Jboss_server.zip -a $THEDIR/*.zip -s "$MSG_HEADER" [email protected],[email protected]
echo $USEDMEMPER > /dev/null
tail -1 $LOG/jboss-free.log
fi
fi
fi
find /app/log/is_ready -type f -mmin +120 -exec rm {} \; > /dev/null 2>&1
#nohup sh /app/Jboss_Maintain_Script/JBOSS_Auto_Maintain.sh > /dev/null 2>&1 &
sleep 10
done