shell实战之日志脱敏-2.0

cfg

# This is generated to be a configuration file.
# kay
# 06/01/2016

# This is a parameter for crontab and objective file.
interval=10
begin=1
end=13

# There are some log path 
# logpath=/root/shell/log/26;/root/shell/log/27;/root/shell/log/28
logpath=/home/pospadm/trc

# sh path
shpath=/home/pospadm/log-shell/log-secret/start.sh

# keylab path
keylabpath=/home/pospadm/log-shell/log-secret/key

collect.sh

#!/bin/bash -

# Used to collect value list
# kay
# Version: 1.0
# 06/01/2016

# key list
keylist=`cat ${keylabpath}`

# golobal value list
valuelist=""
index=0

# collect value by key in directory 
# @Parameter: directory file
collect_list()
{
    for key in ${keylist}
    do
        if grep -qr ${key} $1 
        then
            f0="/<"$key">/p"
            f1="<"$key">"
            f2="s/<"$key">/
"$key"=/gp"
            f3="s/</"$key">/
/gp"
            f4="/"$key"=/p"
            f5="s/"$key"=//gp"
            #f6="$!N;/^(.*)
key$/!P;D"
            #f6=":key;N;s/^(S+)((
.*)*)
key$/key2/M;$!bkey"

            #list=`sed -n ${f0} $1 | sed -n ${f2} | sed -n ${f3} | sed -n ${f4} | sed -n ${f5}`
            #list=`grep ${f1} $1 | sed -n ${f2} | sed -n ${f3} | sed -n ${f4} | sed -n ${f5}`
            list=`grep -r ${f1} $1 | sed -n ${f2} | sed -n ${f3} | sed -n ${f4} | sed -n ${f5}`
            #list=`grep -r ${f1} $2 | sed -n ${f2} | sed -n ${f3} | sed -n ${f4} | sed -n ${f5} | sort -k2n | sed ${f6}`
            #list=`grep -r ${f1} $2 | sed -n ${f2} | sed -n ${f3} | sed -n ${f4} | sed -n ${f5} | sed -r ${f6}`
        
            for l in ${list}
            do
                valuelist[index]=${l}
                ((index++))
            done
        fi    
    done
}

crontab.sh

#!/bin/bash -

# Used to do a crontab task
# kay
# Version: 1.0
# 06/01/2016

# @Name: docrontab
# @Parameter: specific crontab task
docrontab()
{
    cronfile="/tmp/crontab.${USER}"
    crontab -l > ${cronfile}
    query=`echo "$1" | sed 's/*/\*/g'`
    grep -q "${query}" ${cronfile} && echo "" ||
    {
            echo "$1" >> ${cronfile}
            crontab ${cronfile}
    }
    rm -f ${cronfile}
}

key

track_2
track_3 
pan 
exp_dt 
ic_dat 
pin_key 
track_key 
trm_key 
tpklmk 
zaklmk 
zpklmk 
pin_data

readme

本脚本使用方法说明：
1.配置环境变量
# vi ~/.bash_profile
export desen_path=/root/shell2
#source ~/.bash_profile
2.配置文件说明
文件cfg
interval为定时任务间隔时间
begin为查找的文件范围起始时间点
end为查找的文件范围的戒指时间点
logpath为要处理的日志路径，多个路径用;隔开
shpath为本脚本所在绝对路径
keylabpath为关键字字典文件所在绝对路径
3.关键字字典文件
里面的每一个关键字应该是日志内<></>内的内容
要替换的内容为标签内的内容
4.启动脚本
./start.sh
调试方法：
#nohup bash -x start.sh 1>tt.log &
5.停止脚本
#./stop.sh

start.sh

#!/bin/bash -

# This file is used to handle log desensitization
# kay
# Version: 1.0
# 05/31/2016

. ${HOME}/.bash_profile
. ${desen_path}/util.sh
. ${desen_path}/crontab.sh
. ${desen_path}/collect.sh

# write log
logfile="log.txt"
echo "["`date`"] start script." >> ${logfile}

# add crontab
task="*/"${interval}" * * * *   "${shpath}
docrontab "${task}"

dir1=""
dir2=""
# read config file and handle every file and directiory
for d in ${logpath}
do
    tmp="/tmp/find.ret"
    # search for all changed file between time begin and end
    ret=`find "${d}" -mmin +"${begin}" -mmin -"${end}" -type f`
     if [ "${ret}" != "" ]
    then
        echo ${ret} | xargs ls -rt > ${tmp} 
        file1=`sed -n '1p' ${tmp}` 
        file2=`sed -n '$p' ${tmp}` 
        rm -f ${tmp}
        dir1=`echo ${file1} | sed -r 's/^(/.*/)[^/]+/?/1/g'`
        dir2=`echo ${file2} | sed -r 's/^(/.*/)[^/]+/?/1/g'`
        collect_list ${dir1}
        if [ "${dir1}" != "${dir2}" ] 
        then
            collect_list ${dir2}
        fi 
    fi    
    
done

# filter the global list. delete the same value.
# "echo ${globallist[@]} | sort -k2n | sed ":1;N;s/^(S+)((
.*)*)
1$/12/M;$!b1"
valuelist=($(awk -vRS=' ' '!a[$1]++' <<< ${valuelist[@]}))

# hanlde a directory
handle_directory()
{
    for l in ${valuelist[@]}
    do
        cnb=""
        len=${#l}
        if [ ${len} -gt 8 ]
        then
            cnb="5-"$((${len} - 4))
        else
            cnb="1-"${len}
        fi
        mid=`echo ${l} | cut -nb ${cnb} | sed 's/*/\*/g'`
        h=`echo ${l} | sed 's/'${mid}'/*******/g'`
        f=`echo "${l}" | sed 's/*/\*/g'`

        #sed -i 's/'${f}'/'${h}'/g' ${c}
        grep -rl "${f}" $1 | xargs sed -i 's/'${f}'/'${h}'/g'
    done
}

if [ "${dir1}" != "" ] 
then
    handle_directory ${dir1}
    if [ "${dir1}" != "${dir2}" ] 
    then
        handle_directory ${dir2}
    fi
fi

echo "["`date`"] end script." >> ${logfile}

exit

stop.sh

#!/bin/bash -

# Used to stop script and crontab
# kay
# Version: 1.0
# 06/03/2016

crontab -r
ps -ef | grep 'start' | awk -F ' ' '{ print $2 }' | xargs kill -9

util.sh

#!/bin/bash -

# Read config
# kay
# Version: 1.0
# 06/01/2016

# configuration file path
config=${desen_path}/"cfg"

# config parameters
prefix=`grep -v "^#" ${config}`
interval=`echo "${prefix}" | grep  "interval" | awk -F "=" '{ print $2 } '`
begin=`echo "${prefix}" | grep  'begin' | awk -F '=' '{ print $2 } '`
end=`echo "${prefix}" | grep  'end' | awk -F '=' '{ print $2 } '`
logpath=`echo "${prefix}" | grep  'logpath' | awk -F '=' '{ print $2 }' | awk -F ';' '{ for(i=1;i<=NF;i++) print $i }'`
shpath=`echo "${prefix}" | grep  'shpath' | awk -F '=' '{ print $2 }'`
keylabpath=`echo "${prefix}" | grep  'keylabpath' | awk -F '=' '{ print $2 }'`