统计相关

1.统计单个IP 的访问

1）.awk 

  awk '{++count[$1]};END {for (i in count) print i,count[i]}' nginx-access_page.xywy.log | sort -nr -k 2 | more
 

2）. awk + sort + uniq

awk ‘{print $1}’ nginx-access_page.xywy.log | sort | uniq -c | sort -nr

2.统计IP段的访问

1）awk

awk '{split($1,ip,"."); net=ip[1]"."ip[2]"."ip[3] ; ++count[net]};END {for (i in count) print i ".0/24",count[i]} ' /data/logs/nginx/nginx-access_z.xywy.com.log | sort -nr -k 2 | more
 

2.统计流量带宽

awk '{total_flow+=$11}END{print "total_pv:",NR ; print  "total_flow:",total_flow/ 1024 /1024,"M" ; print "total_band" , total_flow / 1024 /1024 /86400 * 8,"Mbps"}' /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log

带宽保留两位小数

awk '{total_flow+=$10}END{print "total_pv:",NR ; print  "total_flow:",total_flow/ 1024 /1024,"M" ; printf "total_band" ; printf "%.2f" ,total_flow / 1024 /1024 /86400 * 8 ; print " Mbps"}' /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log

统计每小时带宽、访问量

for i in {00..23} ; do echo "2016:$i:"; grep "2016:$i" /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log | awk '{total_flow+=$10}END{print "total_pv:",NR ; print  "total_flow:",total_flow/ 1024 /1024,"M" ; printf "total_band" ; printf "%.2f" ,total_flow / 1024 /1024 /3600 * 8 ; print " Mbps"}' ; done
 

统计状态码的数量及百分比

for i in  {6..9}; do echo 2016032$i ; awk '{++count[$10]};END {print "Total:",NR ;for (i in count) print i,count[i]}' 2016032$i/3g.club.xywy.com-access_log | sort -nr -k 2  ; done
​

或者

grep -oP 'HTTP/1.." d+ ' /data/logs/nginx/www.xywy.com-access_log  | cut -d ' ' -f2 | sort | uniq -c

统计每小时访问量：

 for i in `seq -f '%02g' 0 23` ; do echo -ne  "2016:$i:  "; grep -w "2016:$i" ./20160702/page.xywy.com-access.log | wc -l ; done
 

统计所有日志下，指定时间段访问量最高的IP

 

for i in `ls` ; do  awk '/02:00:00/,/04:00:00/ {++count[$2]} ;END {for (s in count) if (count[s] > 2000) {print $1,s,count[s]}}' $i | sort -n -k 2 -r | head -5 ; done
 
for i in `ls` ; do  awk '/02/Nov/2016:02:00:00/,/02/Nov/2016:04:00:00/ {++count[$2]} ;END {for (s in count) if (count[s] > 3000) {print $1,s,count[s]}}' $i | sort -n -k 2 -r | head -5 ; done

3. 奇偶行合并

[fuzengjie@Mac ~/Downloads]$ seq 6
'1
2
3
4
5
6
[fuzengjie@Mac ~/Downloads]$ seq 6 | sed 'N;s/
//'
12
34
56

 

 

或者

[fuzengjie@Mac ~/Downloads]$ seq 6
1
2
3
4
5
6
[fuzengjie@Mac ~/Downloads]$ seq 6 | awk '(ORS=(i=!i)?"":RS)||1'
12
34
56
 

系统连接状态篇：

1.查看TCP连接状态

netstat -ant | awk '{print $6}' | sort | uniq -c | sort -rn

netstat -n | awk '/^tcp/ {++S[$NF]};END {for(a in S) print a, S[a]}' 或

netstat -n | awk '/^tcp/ {++state[$NF]}; END {for(key in state) print key," ",state[key]}'

netstat -n | awk '/^tcp/ {++arr[$NF]};END {for(k in arr) print k,"t",arr[k]}'

netstat -n |awk '/^tcp/ {print $NF}'|sort|uniq -c|sort -rn

netstat -ant | awk '{print $NF}' | grep -v '[a-z]' | sort | uniq -c

3.用tcpdump嗅探80端口的访问看看谁最高

tcpdump -i eth0 -tnn dst port 80 -c 1000 | awk -F"." '{print $1"."$2"."$3"."$4}' | sort | uniq -c | sort -nr |head -20

4.查找较多time_wait连接

netstat -n|grep TIME_WAIT|awk '{print $5}'|sort|uniq -c|sort -rn|head -n20

5.找查较多的SYN连接

netstat -an | grep SYN | awk '{print $5}' | awk -F: '{print $1}' | sort | uniq -c | sort -nr | more

6.根据端口列进程

netstat -ntlp | grep 80 | awk '{print $7}' | cut -d/ -f1

/usr/sbin/tcpdump -i eth0 -l -s 0 -w - dst port 80 | strings | grep -i user-agent | grep -i -E 'bot|crawler|slurp|spider'

网站日分析2(Squid篇）按域统计流量

zcat squid_access.log.tar.gz| awk '{print $10,$7}' |awk 'BEGIN{FS="[ /]"}{trfc[$4]+=$1}END{for(domain in trfc){printf "%st%dn",domain,trfc[domain]}}'

数据库篇

1.查看数据库执行的sql

/usr/sbin/tcpdump -i eth0 -s 0 -l -w - dst port 3306 | strings | egrep -i 'SELECT|UPDATE|DELETE|INSERT|SET|COMMIT|ROLLBACK|CREATE|DROP|ALTER|CALL'

系统Debug分析篇

1.调试命令

strace -p pid

2.跟踪指定进程的PID

gdb -p pid

awk ' {a=substr($2,1,3); b=substr($4,1,3); if ($a == $b ) print $1,$2,$3,$4}'  ne_02.txt  >> ne_02_01.txt

awk ' {a=substr($2,1,3); b=substr($4,1,3); if ($a != $b ) print $1,$2,$3,$4}'  ne_02.txt  >> ne_02_02.txt

awk '{split($2,a,"."); n=a[1]a[2]a[3];split($4,b,".");m=b[1]b[2]b[3] ; if((n == m))print $0 }' ne_02.txt

sed -n '/03/Nov/2015:12/,/03/Nov/2015:18/p' /data/logs/nginx/nginx-access_p.xywy.log | wc -l

sed -i  's/(XYWYSRV_REDIS[1-9]?_HOST_?R?).*/1                "172.16.207.27";/g'