linux 的sed 正则+awk常用笔记

原始文件 ignores.sql

INSERT IGNORE into blacklists VALUES(0,1,'15284545381910','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'15284545802102','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'13114545615950','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'13944545488446','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'13864545749640','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'18684545056739','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'15774545787776','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
INSERT IGNORE into blacklists VALUES(0,1,'17104545150579','','XXXX','1914','2019-05-07 17:18:35','2019-05-07 17:18:35');
..................

 为拼SQL，需要吧第3个字段提出来和数据库做校验，看是否存在insert的遗漏。

思路如下

1：使用awk 取出第三个字段

[root@MYSQL03 tmp]# cat  ignores.sql  |awk -F',' '{print $3}'>/tmp/1.sql
[root@MYSQL03 tmp]# 
[root@MYSQL03 tmp]# head -1 1.sql
'15284545381910'

2： 使用sed 在每行末尾添加 ","

[root@MYSQL03 tmp]# sed -i 's/$/,/g' 1.sql 
[root@MYSQL03 tmp]# 
[root@MYSQL03 tmp]# head -1 1.sql 

'15280381910',
'15280802102',
'13110615950',

3： 使用sed 在每行最前加 select 

sed -i  's#^#select #' 1.sql    -----> # 等价于   注意不是每行都加，只是第一行

4：每行后面加 as ddd union all

[root@MYSQL03 tmp]# sed -i 's#$# as ddd union all #' 1.sql
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]# head -1 1.sql
select '15280381910', as ddd union all

5：文本最前加 " select *（  "     最后加  ");"

[root@MYSQL03 tmp]# sed -i '1i select * (' 1.sql
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]# head -3 1.sql
select * (
select '15280381910', as ddd union all
select '15280802102', as ddd union all

root@MYSQL03 tmp]# sed -i '$a )' 1.sql
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]# tail -3f 1.sql
select '6222802241001387574', as ddd union all
select '6217002920133324724', as ddd union all
)

[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]# sed -i '$a ;' 1.sql
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]#
[root@MYSQL03 tmp]# tail -3f 1.sql
select '6217002920133324724', as ddd union all
)
;

看总体效果

 select * (
select '15280381910', as ddd union all 
select '15280802102', as ddd union all 
select '13110615950', as ddd union all 
select '13940488446', as ddd union all 
select '13867749640', as ddd union all 
select '18682056739', as ddd union all 
select '15770787776', as ddd union all 
select '17100150579', as ddd union all 
select '18811192791', as ddd union all 
select '18813020407', as ddd union all 
select '13161981387', as ddd union all 
select '18215620709', as ddd union all 
select '13028190636', as ddd union all 
select '13176835035', as ddd union all 
select '15500014603', as ddd union all
select '6217002920133324724', as ddd union all 
)
;

完毕。

最后完成如下拼接：

select * （select '1312313131'  as ddd union all select '1232323' as ddd ..) as tmp  where ddd not in(select id from xx where id >xxx)

手工。