环境说明： 公司是做在线教育的互联网企业，WEB架构为：前端使用LVS + Heartbeat做负载均衡，后端主要是Apache/Nginx + Tomcat，缓存有redis和Memcached,数据库使用的Oracle和Mysql。 脚本实现目的：
环境说明：
公司是做在线教育的互联网企业，WEB架构为：前端使用LVS + Heartbeat做负载均衡，后端主要是Apache/Nginx + Tomcat，缓存有redis和Memcached,数据库使用的Oracle和Mysql。
脚本实现目的：
通过在服务器本地运行脚本，当检测到服务不可用时，自动重启相关服务，并发邮件通知管理员。
通过脚本检测服务关键配置文件，若文件丢失，则根据备份自动恢复文件，并重启。
脚本思路：
通过脚本访问事先定义好的监控页面，一共访问三次，每次访问间隔几秒钟，若三次访问都失败，则认为服务故障，重启服务并发邮件。配置文件的备份则是通过重启脚本来完成，所有的服务都是通过配置文件的方式传递给脚本，以方便批量部署。
脚本内容：
#!/bin/bash #This shell-script is use for check tomcat and apache/nginx alive #Created in 2012-11-01 #Last changed in 2013-05-25 source ~/.bash_profile &>/dev/null dir='' dir=/tol/app dir2=/tol/script cd $dir2 dt2=`date +"%Y-%m-%d"` ls $dir2/logs &>/dev/null || mkdir -p $dir2/logs &>/dev/null log="$dir2/logs/tomcatwget-$dt2.log" host=`/sbin/ifconfig |grep "inet addr"|cut -d ':' -f2 |awk '{print $1}'|head -1` wgetfile=$dir2/tomcat_wgetfile conf=$dir2/tomcatwget.conf conf3=$dir2/web.conf sh_name=$0 function shijian () { dt=`date +"%Y-%m-%d-%H:%M:%S"` } function apache_restart () { #stop user=`ps -elf |grep -v grep|grep -v root|grep '/bin/httpd'|grep "$dir"|awk '{print $3}'|head -1` $dir/apache/bin/httpd -k stop &>/dev/null sleep 5 opid='' opid=`ps -elf |grep -v grep|grep "httpd"|grep "$dir"|awk '{print $4}'` if test -n "$opid" ; then kill -9 $opid fi #start /usr/bin/ipcs -s | grep "$user"| gawk '{ print $2 }' | xargs -n 1 ipcrm sem &> /dev/null $dir/apache/bin/httpd -k start &>/dev/null npid=`ps -elf |grep -v grep|grep "httpd"|grep root |grep "$dir"|awk '{print $4}'` shijian if test -z "$npid" ; then echo "The $dt $host apache restart fail by $sh_name" >> $log echo "The $dt $host apache restart fail by $sh_name" |mail -s "check $host apache restart fail" jiank else echo "The $dt $host apache is restart,the new pid=$npid" >> $log echo "The $dt $host apache is restart by $sh_name,the new pid=$npid" |mail -s "check $host apache restart" jiank fi } function nginx_restart () { #stop opid='' opid=`cat $dir/nginx/logs/nginx.pid` kill -QUIT $opid sleep 3 opid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'` if test ! -z "$opid" ; then kill -9 $opid fi #start $dir/nginx/sbin/nginx -c $dir/nginx/conf/nginx.conf npid='' npid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'` shijian if test -z "$npid" ; then echo "The $dt $host nginx restart fail by $sh_name" >> $log echo "The $dt $host nginx restart fail by $sh_name" |mail -s "check $host nginx restart fail" jiank else echo "The $dt $host nginx is restart,the new pid=$npid" >> $log echo "The $dt $host nginx is restart by $sh_name,the new pid=$npid" |mail -s "check $host nginx restart" jiank fi } function http_restart () { if test ! -f $conf3 ; then echo "The $conf3 is not exist" >> $log apache_restart sleep 3 nginx_restart else while read line do dir=`echo $line |awk -F\; '{print $1}'` softname=`echo $line |awk -F\; '{print $2}'` $softname\_restart done < $conf3 fi } function web_reboot () { http_restart sh $dir2/tomcat-restart.sh $name &>/dev/null exit 0 } #conf-check if test ! -f $conf ; then echo "The $conf is not exist" exit 0 fi shijian echo "$dt" >> $log echo "$host" >> $log ##Apache-check domain=`/usr/bin/head -1 $conf |awk -F\; '{print $1}'` if test `echo $domain |grep http` ; then html="$domain" else html=http://$domain/t-s.html fi cat /dev/null > $wgetfile /usr/bin/elinks -dump $html >> $wgetfile 2>&1 sleep 3 /usr/bin/elinks -dump $html >> $wgetfile 2>&1 sleep 3 /usr/bin/elinks -dump $html >> $wgetfile 2>&1 num='' num=`grep -i -c "ok" $wgetfile` if (( $num < 1 )) ; then shijian echo "The $dt $host $html wget error" >> $log echo "The $dt $host $html wget error" | mail -s "check the $host wget error" jiank http_restart elif (( $num < 3&&$num >= 1 )) ; then echo "The $host $html wget oknumber is $num" |mail -s "$host wget oknumber" jiank else echo "The $host $html wget success" >> $log fi while read line do shijian file1='' file1=`echo $line |awk -F\; '{print $2}'` file2='' file2=`echo $line |awk -F\; '{print $3}'` name='' name=`echo $line |awk -F\; '{print $4}'` port='' port=`echo $line |awk -F\; '{print $5}'` function pid () { pid='' pid=`ps -elf|grep java|grep "$dir"|grep $name|awk '{print $4}'` } function down () { `echo $line |awk -F\; '{print $6}'` sleep 3 pid if test -n "$pid" ; then kill -9 $pid fi rm -rf $dir/$name/work/Catalina/* rm -rf $dir/$name/temp/* } function up () { `echo $line |awk -F\; '{print $7}'` } #web.xml-check if test "$file1" ; then if test ! -f $file1 ; then shijian \cp $file1.bak $file1 && chown tomcat.tomcat $file1 &>/dev/null if test -f $file1 ; then echo "The $dt $file1 is lost and will be recovery" >> $log echo "The $dt $file1 is lost and will be recovery" | mail -s "check the $host $file1 lost" jiank web_reboot else echo "The $dt $file1 is lost and recovery fail" >> $log echo "The $dt $file1 is lost and recovery fail" | mail -s "check the $host $file1 lost" jiank fi else echo "The $dt $file1 is ok" >> $log fi fi #ROOT.xml-check if test "$file2" ; then if test ! -f $file2 ; then shijian \cp $file2.bak $file2 && chown tomcat.tomcat $file2 &>/dev/null echo "The $dt $file2 do not exist" >> $log echo "The $dt $file2 do not exist" | mail -s "check the $host $file2 lost" jiank down up pid if test -n "$pid" ; then echo "$dt $host $file2 is lost and restart by $sh_name" |mail -s "$host $file2 is lost" jiank else up echo "The $dt $host $file2 is lost and restart fail by $sh_name" >> $log echo "The $dt $host $file2 is lost and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiank fi else echo "The $dt $file2 is ok" >> $log fi fi #tomcat-check if test "$port" ; then if test `echo $port|grep http` ; then jsp="$port" else jsp=http://127.0.0.1:$port/t-s.jsp fi cat /dev/null > $wgetfile /usr/bin/elinks -dump $jsp >> $wgetfile 2>&1 sleep 5 /usr/bin/elinks -dump $jsp >> $wgetfile 2>&1 sleep 5 /usr/bin/elinks -dump $jsp >> $wgetfile 2>&1 num='' num=`grep -i -c "ok" $wgetfile` if (( $num < 1 )) ; then shijian echo "The $dt $host $name wget error 3 times and begin to restart" >> $log down up pid if test -n "$pid" ; then echo "$dt Restart $name success pid= " $pid >> $log echo "$dt $host $name wget error and restart by $sh_name" |mail -s "$host $name wget error" jiank else up echo "The $dt $host $name wget error and restart fail by $sh_name" >> $log echo "The $dt $host $name wget error and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiank fi elif (( $num < 3&&$num >= 1 )) ; then echo "The $host $name wget oknumber is $num" |mail -s "$host tomcat wget oknumber" jiank else echo "The $host $name wget success" >> $log fi fi done < $conf脚本配置文件：
127.0.0.1;/tol/htdocs/tomcata/ROOT/WEB-INF/web.xml;/tol/app/tomcata/conf/Catalina/localhost/ROOT.xml;tomcata;7010;/etc/init.d/tomcata stop;/etc/init.d/tomcata start; ;tol/htdocs/tomcatb/ROOT/WEB-INF/web.xml;/tol/app/tomcatb/conf/Catalina/localhost/ROOT.xml;tomcatb;7011;/etc/init.d/tomcatb stop;/etc/init.d/tomcatb start;