system-person-monitor.sh
监控脚本在crontab中的配置：
0,5,10,15,20,25,30,35,40,45,50,55 * * * * /usr/local/shell/alltest/system-person-monitor.sh > /dev/null 2>&1
脚本监控对象：
load、cpu利用率、内存利用率、ping、磁盘分区利用率、eth0或eth1数据流量
脚本原理：
Load
脚本会取w中load值并计算出它1min、5min、15min平均值，如果连续监测load平均值超过阀值的次数，超过警线次数就会连接报警平台。内容包括：ip地址、时间、超过阀值的load值。
cpu利用率
脚本会取vmstat中cpu空闲率并计算出cpu利用率。如果连续监测cpu利用率超过阀值的次数，超过警线次数就会连接报警平台。内容包括：ip地址、时间、cpu利用率。
内存利用率
脚本会取free取出内存使用量与内存总量，并计算出内存利用率。如果连续监测内存利用率超过阀值的次数，超过警线次数就会连接报警平台。报警内容包括：ip地址、时间、内存利用率。
磁盘分区利用率
脚本会取df -h及df -i中磁盘分区利用率（空间利用率和inodes利用率）最大的值, 如果这个最大值超过阀值就会连接报警平台。报警内容包括：ip地址、时间、磁盘利用率最大的值、磁盘利用率阀值。
eth0或eth1数据流量
脚本需要人为的设置选择监控eth0还是监控eth1，它会从sar -n DEV 1，提取出相应网口每秒进入的平均数据量及传出的平均数据量，计算并以KB/S为单位输出。如果计算出进入或者传出的数据大于或等于10000，会认为是脚本判断是错误，会把记录进入或传出的数据的变量归为0.如果进入或传出的数据大于阀值就会连接报警平台。报警内容包括：IP、时间、网卡接口、数据速率。
Ping测试
脚本会用ping -c 5向新浪主页发送5个数据包，如果延时超过阀值的次数超过警线次数就会连接报警平台。报警内容包括:IP、时间、延迟的时间。
以下是脚本代码
#!/bin/sh
msgid=xxxxxxx
#fazhi#
loadfazhi="50"
loadcishu="2"
cpufazhi="101"
cpucishu="2"
memfazhi="10000"
memcishu="3"
pingfazhi="300"
pingcishu="3"
diskfazhi="90"
netxuanzeeth0="yes"
netxuanzeeth1="no"
neteth0fazhi="8000"
neteth1fazhi="8000"
#####################################################################
#variable#
dir1="/usr/local/shell/alltest/system"
time1=`date +%Y%m%d"-"%T`
time2=`date +%Y%m%d`
ipis=`cat /etc/sysconfig/network-scripts/ifcfg-eth0 |grep "IPADDR"|awk -F "=" '{print$2}'`
#preparation#
mkdir -p $dir1/tmp
#load#
load=`w|grep load|awk -F "average:" '{print $2}'|sed 's/,//g'|sed 's/^.//g'|sed 's/ /\n/g'|awk 'BEGIN { a=0 }; { a=a+$0 }; END { printf "%.3f\n", a/NR }'`
echo $load
loadtmp=`echo $load |awk -F "." '{ print $1}'`
if [ $loadtmp -ge $loadfazhi ] ; then
echo "alarm load : $ipis $time1 load = $load!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history load : $ipis $time1 load = $load!" >> $dir1/$ipis.alltest.system.$time2.log
echo "1" >> $dir1/tmp/cpu-load.system.tmp
else
echo "history load : $ipis $time1 load = $load!" >> $dir1/$ipis.alltest.system.$time2.log
echo "0" >> $dir1/tmp/cpu-load.system.tmp
fi
for (( loadi = 1 ; loadi <= $loadcishu ; loadi++))
do
loada=`tac $dir1/tmp/cpu-load.system.tmp | sed -n "${loadi}p"`
loadb=$((loadb + loada))
done
if [ $loadb -ge $loadcishu ] ; then
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm load : $ipis $time1 load = $load!'"
wget --output-document=/dev/null " load : $ipis $time1 load = $load"
echo yes
else
echo no
fi
#cpu#
cpukong=`vmstat 1 2|tail -n 1 |awk '{print $15}'`
cpu=`expr 100 - $cpukong`
echo $cpu
if [ $cpu -ge $cpufazhi ] ; then
echo "alarm cpu : $ipis $time1 cpu used = $cpu%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history cpu : $ipis $time1 cpu used = $cpu%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "1" >> $dir1/tmp/cpu.system.tmp
else
echo "history cpu : $ipis $time1 cpu used = $cpu%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "0" >> $dir1/tmp/cpu.system.tmp
fi
for (( cpui = 1 ; cpui <= $cpucishu ; cpui++))
do
cpua=`tac $dir1/tmp/cpu.system.tmp | sed -n "${cpui}p"`
cpub=$((cpub + cpua))
done
if [ $cpub -ge $cpucishu ] ; then
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm cpu : $ipis $time1 cpu = $cpu%!'"
wget --output-document=/dev/null " cpu : $ipis $time1 cpu = $cpu%"
echo yes
else
echo no
fi
#memory#
mem=`free|grep Mem|awk '{print $3/$2*10000 }'|awk 'BEGIN { a=0 }; { a=a+$0 }; END { printf "%.2f\n", a/NR }'`
echo $mem
memtmp=`echo $mem |awk -F "." '{ print $1}'`
if [ $memtmp -ge $memfazhi ] ; then
echo "alarm memory : $ipis $time1 memory used = $mem%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history memory : $ipis $time1 memory used = $mem%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "1" >> $dir1/tmp/memory.system.tmp
else
echo "history memory : $ipis $time1 memory used = $mem%!" >> $dir1/$ipis.alltest.system.$time2.log
echo "0" >> $dir1/tmp/memory.system.tmp
fi
for (( memi = 1 ; memi <= $memcishu ; memi++))
do
mema=`tac $dir1/tmp/memory.system.tmp | sed -n "${memi}p"`
memb=$((memb + mema))
done
if [ $memb -ge $memcishu ] ; then
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm memory : $ipis $time1 memory = $mem!'"
wget --output-document=/dev/null " memory : $ipis $time1 memory = $mem"
echo yes
else
echo no
fi
#disk#
#disk space#
percent=`df -h | grep -v Used | grep -v boot | grep -v mnt | grep -v shm | awk '{if ( $1 ~/^[0-9]/ ) print $4 ; else print $5}'| grep -v ^$ | sed "s/%/ /g"`
diski=0
for i in $percent
do
if [ $i -gt $diski ]
then diski=$i
fi
done
diskweizhi=`df -h | sed "s/%/ /g" | awk -v a="$diski" '$4 == a || $5 == a' | awk '{if ( $1 !~/^[0-9]/ ) print $6"-"$4 ;else print $5"-"$3 }'`
echo space $diski
if [ $diski -ge $diskfazhi ] ; then
echo $diski
echo "alarm disk : $ipis $time1 max disk used is $diski%, $diskweizhi !" >> $dir1/$ipis.alltest.system.$time2.log
echo "history disk : $ipis $time1 max disk used is $diski%, $diskweizhi!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.120 "fetion-ops --mobile=13811193602 --pwd=nanyuan0301 --to=13426056563,13811193602 --msg-utf8='alarm disk : $ipis $time1 max disk used is $diski%, $diskweizhi !'"
wget --output-document=/dev/null " disk : $ipis $time1 max disk used is $diski%, $diskweizhi !"
unset diskweizhi
unset diski
unset i
unset percent
fi
#disk inode#
percent=`df -i | grep -v Used | grep -v boot | grep -v mnt | grep -v shm | awk '{if ( $1 ~/^[0-9]/ ) print $4 ; else print $5}'| grep -v ^$ | sed "s/%/ /g"`
diski=0
for i in $percent
do
if [ $i -gt $diski ]
then diski=$i
fi
done
echo inode $diski
diskweizhi=`df -i | sed "s/%/ /g" | awk -v a="$diski" '$4 == a || $5 == a' | awk '{if ( $1 !~/^[0-9]/ ) print $6"-"$4 ;else print $5"-"$3 }'`
if [ $diski -ge $diskfazhi ] ; then
echo $diski
echo "alarm disk : $ipis $time1 max inode used is $diski%, $diskweizhi !" >> $dir1/$ipis.alltest.system.$time2.log
echo "history disk : $ipis $time1 max inode used is $diski%, $diskweizhi!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm disk : $ipis $time1 max inode used is $diski%, $diskweizhi !'"
wget --output-document=/dev/null " disk : $ipis $time1 max inode used is $diski%, $diskweizhi "
unset diskweizhi
fi
#net#
if [ $netxuanzeeth0 = "yes" -o $netxuanzeeth1 = "yes" ];then
sar -n DEV 1 | tail -n 3 | head -n 2 > sar1.txt
fi
while [ $netxuanzeeth0 = "yes" ] ; do
neteth0in=`cat sar1.txt | head -n 1 | awk '{print $5}' | awk -F. '{print $1}'`
neteth0out=`cat sar1.txt | head -n 1 | awk '{print $6}' | awk -F. '{print $1}'`
neteth0inaver=`expr $neteth0in / 1024`
neteth0outaver=`expr $neteth0out / 1024`
echo $neteth0inaver $neteth0outaver
if [ $neteth0inaver -ge 100000 ] ; then
neteth0inaver=0
fi
if [ $neteth0inaver -ge $neteth0fazhi ] ; then
echo "alarm net : $ipis $time1 net eth0 in flux = $neteth0inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history net : $ipis $time1 net eth0 in flux = $neteth0inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='net alarm: $ipis $time1 eth0 in=$neteth0inaver Kb/s!'"
wget --output-document=/dev/null " alarm: $ipis $time1 eth0 in=$neteth0inaver Kb/s!"
else
echo "history net : $ipis $time1 net eth0 in flux = $neteth0inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
fi
if [ $neteth0outaver -ge 100000 ] ; then
neteth0outaver=0
fi
if [ $neteth0outaver -ge $neteth0fazhi ] ; then
echo "alarm net : $ipis $time1 net eth0 out flux = $neteth0outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history net : $ipis $time1 net eth0 out flux = $neteth0outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm net: $ipis $time1 net eth0 out flux=$neteth0outaver Kb/s!'"
wget --output-document=/dev/null " net: $ipis $time1 net eth0 out flux=$neteth0outaver Kb/s!"
else
echo "history net : $ipis $time1 net eth0 out flux = $neteth0outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
fi
netxuanzeeth0=done
done
while [ $netxuanzeeth1 = "yes" ] ; do
neteth1in=`cat sar1.txt | head -n 1 | awk '{print $5}' | awk -F. '{print $1}'`
neteth1out=`cat sar1.txt | head -n 1 | awk '{print $6}' | awk -F. '{print $1}'`
neteth1inaver=`expr $neteth1in / 1024`
neteth1outaver=`expr $neteth1out / 1024`
echo $neteth1inaver $neteth1outaver
if [ $neteth1inaver -ge 100000 ] ; then
neteth1inaver=0
fi
if [ $neteth1inaver -ge $neteth1fazhi ] ; then
echo "alarm net : $ipis $time1 net eth1 in flux = $neteth1inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history net : $ipis $time1 net eth1 in flux = $neteth1inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='net alarm: $ipis $time1 eth1 in=$neteth1inaver Kb/s!'"
wget --output-document=/dev/null " alarm: $ipis $time1 eth1 in=$neteth1inaver Kb/s!"
else
echo "history net : $ipis $time1 net eth1 in flux = $neteth1inaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
fi
if [ $neteth1outaver -ge 100000 ] ; then
neteth1outaver=0
fi
if [ $neteth1outaver -ge $neteth1fazhi ] ; then
echo "alarm net : $ipis $time1 net eth1 out flux = $neteth1outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history net : $ipis $time1 net eth1 out flux = $neteth1outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
#ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm net: $ipis $time1 net eth1 out flux=$neteth1outaver Kb/s!'"
wget --output-document=/dev/null " net: $ipis $time1 net eth1 out flux=$neteth1outaver Kb/s!"
else
echo "history net : $ipis $time1 net eth1 out flux = $neteth1outaver Kb/s!" >> $dir1/$ipis.alltest.system.$time2.log
fi
netxuanzeeth1=done
done
#ping
ping=`ping -c 5 | awk -F'=' '/time=/ {n++;sum+=$NF+0}END{print sum/n}'|awk -F "." '{print$1}'`
echo $ping
if [ $ping -ge $pingfazhi ] ; then
echo "alarm ping : $ipis $time1 ping = $ping ms!" >> $dir1/$ipis.alltest.system.$time2.log
echo "history ping : $ipis $time1 ping = $ping ms!" >> $dir1/$ipis.alltest.system.$time2.log
echo "1" >> $dir1/tmp/ping.system.tmp
else
echo "history ping : $ipis $time1 ping = $ping ms!" >> $dir1/$ipis.alltest.system.$time2.log
echo "0" >> $dir1/tmp/ping.system.tmp
fi
for (( pingi = 1 ; pingi <= $pingcishu ; pingi++))
do
pinga=`tac $dir1/tmp/ping.system.tmp | sed -n "${pingi}p"`
pingb=$((pingb + pinga))
done
if [ $pingb -ge $pingcishu ] ; then
# ssh 10.10.10.248 "fetion.sh phone="$msgid" msg='alarm ping : $ipis $time1 ping = $ping ms!'"
wget --output-document=/dev/null " ping : $ipis $time1 ping = $ping ms"
echo yes
else
echo no
fi