一.安装inception及使用
1安装基础环境:
inception是一个集审核、执行、备份及生成回滚语句于一身的MySQL自动化运维工具.
注意，Centos7一定要制定bison版本不高于2.6，否则Inception可能会出现编译错误，该下载速度十分缓慢，可以考虑长时间等待或者更换镜像源。
下载Inception安装包:
Inception源地址已没有源码，可以打开源地址，然后在右上角上面的fork处选择新的下载地址。
源码地址：https://github.com/mysql-inception/inception
https://github.com/bbotte/inception-mysql 下载地址
安装基础环境: [root@CentOS ~]# yum -y install cmake libncurses5-dev libssl-dev g++ bison gcc gcc-c++ openssl-devel ncurses-devel mysql pymysql [root@CentOS ]# yum remove bison -y 安装bison: [root@CentOS ~]#wget http://ftp.gnu.org/gnu/bison/bison-2.5.1.tar.gz [root@CentOS ~]# tar -zxvf bison-2.5.1.tar.gz [root@CentOS bison-2.5.1]# ./configure [root@CentOS bison-2.5.1]# vi ./lib/stdio.h 删除 这一行 _GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead"); [root@CentOS bison-2.5.1]# ./configure [root@CentOS bison-2.5.1]# make && make install [root@CentOS bison-2.5.1]# bison -V 安装inception： [root@CentOS bison-2.5.1]# cd /usr/local/ 将上述下载的包传上来 [root@CentOS local]# unzip inception-mysql-master.zip [root@CentOS local]# mv inception-mysql-master inception-master [root@CentOS local]# cd inception-master [root@CentOS inception-master]# sh inception_build.sh builddir Linux 修改配置文件: vim /etc/inc.cnf 创建此文件 [inception] general_log=1 general_log_file=inc.log port=6669 socket=/tmp/inc.socket character-set-client-handshake=0 character-set-server=utf8# 这里的数据库是填inception的库，也用作备份库 inception_remote_system_password=123456 inception_remote_system_user=root inception_remote_backup_port=3306 inception_remote_backup_host=127.0.0.1 inception_support_charset=utf8 inception_enable_nullable=0 inception_check_primary_key=1 inception_check_column_comment=1 inception_check_table_comment=1 inception_osc_min_table_size=1 inception_osc_bin_dir=/usr/bin inception_osc_chunk_time=0.1 inception_ddl_support=1 inception_enable_blob_type=1 inception_check_column_default_value=1 启动服务: [root@CentOS inception-master]# /usr/local/inception-master/builddir/mysql/bin/Inception --defaults-file=/etc/inc.cnf (python36env) [vagrant@CentOS devops]$ netstat -nltp 可以看到有6669的端口跑起来了 测试是否安装成功：输入后123456，进入数据库命令行 (python36env) [vagrant@CentOS devops]$ mysql -uroot -h127.0.0.1 -P 6669 登进inception管理控制台 MySQL [(none)]> inception get variables; 查看它所有变量/属性
+------------------------------------------+---------------------------------------------+
| autocommit | OFF |
| bind_address | * |
| character_set_system | utf8 |
| character_sets_dir | /usr/local/inception-master/share/charsets/ |
| connect_timeout | 10 |
| date_format | %Y-%m-%d |
| datetime_format | %Y-%m-%d %H:%i:%s |
| general_log | ON |
| general_log_file | inc.log |
| inception_check_autoincrement_datatype | ON |
| inception_check_autoincrement_init_value | ON |
| inception_check_autoincrement_name | ON |
| inception_check_column_comment | ON |
| inception_check_column_default_value | ON |
| inception_check_dml_limit | ON |
| inception_check_dml_orderby | ON |
| inception_check_dml_where | ON |
| inception_check_identifier | ON |
| inception_check_index_prefix | ON |
| inception_check_insert_field | ON |
| inception_check_primary_key | ON |
| inception_check_table_comment | ON |
| inception_check_timestamp_default | ON |
| inception_ddl_support | ON |
| inception_enable_autoincrement_unsigned | ON |
| inception_enable_blob_type | ON |
| inception_enable_column_charset | OFF |
| inception_enable_enum_set_bit | OFF |
| inception_enable_foreign_key | OFF |
| inception_enable_identifer_keyword | OFF |
| inception_enable_not_innodb | OFF |
| inception_enable_nullable | OFF |
| inception_enable_orderby_rand | OFF |
| inception_enable_partition_table | OFF |
| inception_enable_pk_columns_only_int | OFF |
| inception_enable_select_star | OFF |
| inception_enable_sql_statistic | ON |
| inception_max_char_length | 16 |
| inception_max_key_parts | 5 |
| inception_max_keys | 16 |
| inception_max_primary_key_parts | 5 |
| inception_max_update_rows | 10000 |
| inception_merge_alter_table | ON
。。。。。。
(python36env) [vagrant@CentOS devops]$ sudo su - root
[root@CentOS ~]# netstat -nltp
tcp6 0 0 :::6669 :::* LISTEN 1150/Incep
[root@CentOS ~]# kill -9 1150
使用后台方式跑:(公司中一般用supervisior去管理它)
[root@CentOS inception-master]# nohup /usr/local/inception-master/builddir/mysql/bin/Inception --defaults-file=/etc/inc.cnf &
inception使用规范及说明文档:
https://inception-document.readthedocs.io/zh_CN/latest/
2.inception简单使用:
可通过如下脚本调inception(在执行sql时，把sql作为一参数传给此脚本，此脚本去连inception作了一检测和执行操作)。
[root@CentOS ~]# mysql -uroot -p
MariaDB [(none)]> create database pro1;
MariaDB [(none)]> use pro1;
建表:
MariaDB [pro1]> create table if not exists `mytable1`(
-> `id` int unsigned auto_increment,
-> `myname` varchar(10) not null,
-> primary key(`id`)
-> )engine=innodb default charset=utf8;
MariaDB [pro1]> show tables;
+----------------+
| Tables_in_pro1 |
+----------------+
| mytable1 |
[root@CentOS ~]# vi test.py
#!/usr/bin/python #-*-coding: utf-8-*- ''' 建库建表语句 CREATE DATABASE pro1; CREATE TABLE IF NOT EXISTS `mytable1`( `id` INT UNSIGNED AUTO_INCREMENT, `myname` VARCHAR(10) NOT NULL, PRIMARY KEY ( `id` ) )ENGINE=InnoDB DEFAULT CHARSET=utf8; ''' import MySQLdb # 待审核/执行的sql语句（需包含目标数据库的地址、端口 等参数） sql='/* --user=root;--password=123456;--host=127.0.0.1;--port=3306;--enable-execute; */ inception_magic_start; use pro1; insert into mytable1 (myname) values ("xianyu1"),("xianyu2"); insert into mytable1 (myname) values ("xianyu1"),("xianyu2"); inception_magic_commit;' try: # inception的地址、端口等 conn=MySQLdb.connect(host='127.0.0.1',user='root',passwd='123456',db='',port=6669) cur=conn.cursor() ret=cur.execute(sql) result=cur.fetchall() num_fields = len(cur.description) field_names = [i[0] for i in cur.description] print (result) ''' for row in result: print row[0], "|",row[1],"|",row[2],"|",row[3],"|",row[4],"|", row[5],"|",row[6],"|",row[7],"|",row[8],"|",row[9],"|",row[10] ''' cur.close() conn.close() except Exception as e: print (e)
[root@CentOS ~]# yum install -y epel-release
[root@CentOS ~]# yum install -y python-pip
[root@CentOS ~]# pip install MySQL-python python3的话就用pymysql,那对应的如下test.py中也要改成pymysql
[root@CentOS ~]# python test.py 运行后返回如下结果了两条记录
((1L, 'CHECKED', 0L, 'Audit completed', 'None', 'use pro1', 0L, "'0_0_0'", 'None', '0', ''), (2L, 'CHECKED', 0L, 'Audit completed', 'None', 'insert into mytable1 (myname) values ("xianyu1"),("xianyu2")', 2L, "'0_0_1'", '127_0_0_1_3306_pro1', '0', ''), (3L, 'CHECKED', 0L, 'Audit completed', 'None', 'insert into mytable1 (myname) values ("xianyu1"),("xianyu2")', 2L, "'0_0_2'", '127_0_0_1_3306_pro1', '0', ''))
MariaDB [pro1]> select * from mytable1;表中也插入数据了
+----+---------+
| id | myname |
+----+---------+
| 1 | xianyu1 |
| 2 | xianyu2 |
| 3 | xianyu1 |
| 4 | xianyu2 |
inception自动备份功能需要:
1.mysql设置binlog需要如下配置
[root@CentOS ~]# vi /etc/my.cnf
[mysqld]
log-bin = mysql-bin
binlog_format = ROW
server-id = 1
[root@CentOS ~]# systemctl restart mariadb
MariaDB [(none)]> show databases;
+---------------------+
| Database |
+---------------------+
| information_schema |
| 127_0_0_1_3306_pro1 | 有备份库了.
| devops |
| inception |
| mysql |
| performance_schema |
| pro1 |
| test1 |
如图inception与djangog与mysql三个组件关系:
它们分别对应三个库:
二.启动sql上线项目基础包启动
(1)如下图把基础包拷贝到项目根目录中:
(2)用pycharm打开此sql项目基础包
(3)sqlweb/settings.py配置:
DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'sqlweb', 'USER': 'root', 'PASSWORD': '123456', 'HOST':'127.0.0.1', 'PORT':'3306', }, }
(5)数据迁移
(python36env) [vagrant@CentOS sqlweb]$ cd /vagrant/sqlweb (python36env) [vagrant@CentOS sqlweb]$ pip install pymysql (python36env) [vagrant@CentOS sqlweb]$ pip install django-rest-swagger (python36env) [vagrant@CentOS devops]$ mysql -uroot -p123456 MariaDB [(none)]> create database sqlweb; (python36env) [vagrant@CentOS sqlweb]$ python manage.py makemigrations (python36env) [vagrant@CentOS sqlweb]$ python manage.py migrate
(python36env) [vagrant@CentOS sqlweb]$ python manage.py createsuperuser
admin
admin@123.com
(6)pycharm远程连接sql项目
此时启动sql项目如下图无法打开， 原因没做映射:
但是用命令行却可以:
[vagrant@CentOS ~]$ curl http://127.0.0.1:8081/
a.解决：配置映射F:devopsdatavagrantfile
F:devopsdata>vagrant reload
效果如下图可以了，
且如下图中可登录的效果，这样基础包就跑起来了!
三.iview初始化--让sql项目与前端结合起来
把iview基础包运行起来。此基础包用法与element一样，只需在官网下载此包，并做基本配置---npm stall 装node库，然后运行起来即可了。
(1)
F:devopsdatafrontend>npm install --registry=https://registry.npm.taobao.org
(2)用webstorm打开frontend基础包
跨域代理配置:我所有的后端接口都是以api开头的，所以把api开头的都转换到127.0.0.1:8081这个地址。
(3)后端接口配置
F:devopsdatafrontend>npm run dev
报错:npm run dev 报错 iview TypeError [ERR_INVALID_CALLBACK]: Callback must be a function
然后找到webpack.dev.config.js打开
将这一行代码：fs.write(fd, buf, 0, buf.length, 0, function(err, written, buffer) {});
替换为fs.write(fd, buf, 0, 'utf-8', function(err, written, buffer) {});
保存然后再次执行 npm run dev 启动跳转到如下图
(4)在页面创建用户如图
登录不上报如下错:net::ERR_NAME_NOT_RESOLVED
解决:
(1)如下图，登录配置中告诉我们走的登录path是在utils中配置，所以改成如下
(2)C:WindowsSystem32driversetchosts做解析加上如下配置----我的没做
127.0.0.1 sqlweb
保存后如下图登成功了：
https://www.iviewui.com/ iviewui前端组件模版参考
四.sql 审核功能实现
1.写api接口,路由,视图--前端接口展示
(1)F:devopsdatafrontendsrcapisqlcheck.js
import axios from '../../libs/http' const autoSelects = '/api/sqlmng/autoselects/' const inceptionCheck = '/api/sqlmng/inceptioncheck/' export function GetSelectData(data) { return axios({ url: autoSelects, method: 'post', data: data }) } export function CheckSql(data) { return axios({ url: inceptionCheck, method: 'post', data: data }) }
(2)F:devopsdatafrontendsrcviewssqlcheck.vue
<template>
<div>
<Card>
<Row>
<Col span="12">
<Alert show-icon>输入要上线的SQL语句</Alert>
</br>
<div>
<Form class="step-form" ref="checkContent" :model="checkData" :rules="ruleCheckData" :label-width="100">
<FormItem label="SQL" prop="sql_content">
<Input v-model="checkData.sql_content" type="textarea" :autosize="{minRows: 10,maxRows: 20}" placeholder="请输入SQL" />
</FormItem>
<FormItem label="备注">
<Input v-model="checkData.remark" type="textarea" :autosize="{minRows: 2,maxRows: 5}" placeholder="请输入备注" />
</FormItem>
<FormItem label="操作">
<Row>
<Col span="12">
<center>
<Button type="primary" @click='handleCheckSql'>确定</Button>
</center>
</Col>
<Col span="12">
<center>
<Button>清空</Button>
</center>
</Col>
</Row>
</FormItem>
</Form>
</div>
</Col>
<Col span="12">
<Alert show-icon style='margin-left:12%'>
<Icon type="ios-lightbulb-outline" slot="icon"></Icon>
选择执行条件
</Alert>
<div style='margin-left:50px'>
</br>
<div>
<Form class="step-form" ref="checkConf" :model="checkData" :rules="ruleCheckData" :label-width="100">
<FormItem label="环境">
<RadioGroup v-model="checkData.env" @on-change="handleSelect">
<Radio label="prd">生产</Radio>
<Radio label="test">测试</Radio>
</RadioGroup>
</FormItem>
<FormItem label="数据库" prop="db">
<Select v-model="checkData.db" style="200px; margin-left:10px">
<Option v-for="item in dbList" :value="item.value" :key="item.value">{{ item.label }}</Option>
</Select>
</FormItem>
<FormItem label="执行人" prop="treater">
<Select v-model="checkData.treater" style="200px; margin-left:10px">
<Option v-for="item in transactorList" :value="item.value" :key="item.value">{{ item.label }}</Option>
</Select>
</FormItem>
</Form>
</div>
</div>
</Col>
</Row>
</Card>
</div>
</template>
<script>
import { GetSelectData, CheckSql } from '@/api/sql/check'
export default {
data () {
return {
checkData: {
sql_content: '',
remark: '',
env: 'prd',
db: '',
treater: '',
},
ruleCheckData: {
sql_content: [{ required: true, message: '请填写SQL', trigger: 'blur' }],
treater: [{ required: true, message: '请选择执行人', trigger: 'change' }],
db: [{ required: true, message: '请选择数据库', trigger: 'change', type: 'number' }],
},
dbList: [],
transactorList: [],
keyMap: {
'sql_content':'SQL',
'env':'环境',
'db':'数据库',
'treater':'执行人',
},
}
},
created () {
this.handleSelect(this.checkData.env)
},
methods: {
renderFunc (treater) {
this.$Notice.success({
title: 'SQL审核通过',
desc: 'SQL审核通过...',
render: h => {
return h('span', [
'请等待 ',
h('a', treater),
' 执行'
])
}
});
},
warning (title, msg) {
this.$Notice.warning({
title: title,
duration: 0,
desc: msg
});
},
handleSelect (e) {
GetSelectData({env:e})
.then(response => {
console.log(response)
const dbs = response.data.data.dbs
const managers = response.data.data.managers
this.dbList = []
dbs.map( (item) => {
this.dbList.push({
value:item.id,
label:item.name
})
})
this.transactorList = []
managers.map( (item) => {
this.transactorList.push({
value:item,
label:item,
})
})
})
.catch(error => {
console.log(error)
})
},
handleCheckSql () {
this.$refs.checkContent.validate((valid) => {
if (!valid) {
return
}
this.$refs.checkConf.validate((valid) => {
if (!valid) {
return
}
CheckSql(this.checkData)
.then(response => {
console.log(response)
let status = response.data.status
let msg = response.data.msg
if (status == 0){
this.renderFunc(this.checkData.treater)
} else if (status == -1 || status == -2){
this.warning('SQL审核不通过', msg)
}
})
.catch(error => {
console.log(error)
})
})
})
},
},
}
</script>
(3)src outer outer.js加入此段代码
刷新后如图效果:
2.sql实现sql语句简单审核--无判断
(1)sqlmgn/models.py:
#coding=utf-8 from django.db import models from django.contrib.auth.models import User, Group class Basemodel(models.Model): name = models.CharField(default='', null=True, blank=True, max_length=128, verbose_name='名字') createtime = models.DateTimeField(auto_now_add=True, verbose_name='创建时间') updatetime = models.DateTimeField(auto_now=True, verbose_name='修改时间') remark = models.TextField(default='', null=True, blank=True, verbose_name='备注') def __unicode__(self): return self.name class Meta: abstract = True ordering = ['-id'] class Dbconf(Basemodel): sql_content = models.TextField()
(2)sqlmng/urls.py
#coding=utf-8 from django.conf.urls import include, url from rest_framework.routers import DefaultRouter from .views import * # register的可选参数 base_name: 用来生成urls名字，如果viewset中没有包含queryset, base_name一定要有 router = DefaultRouter() router.register(r'dbconfs', DbViewSet, base_name='dbviewset') router.register(r'inceptioncheck', InceptionCheckView, base_name='Inceptioncheckview') urlpatterns = [ url(r'^', include(router.urls)), ]
(3)sqlmg/views.py
# Create your views here. from rest_framework.response import Response from .baseviews import BaseView from .serializers import DbSerializer from .models import * from utils.inception import table_structure class DbViewSet(BaseView): queryset = Dbconf.objects.all() serializer_class = DbSerializer search_fields = ['name', 'user', 'password','host', 'port', 'env'] class InceptionCheckView(BaseView): # 重写create--因为发任务是post发过来的,(注意view中的create就是对应前端请求的post方法(等价于post)。而序列化中的create方法是view中的create去写数据时调用它)。 def create(self, request, *args, **kwargs): # 接收前端数据 request_data = request.data print(request_data) sql_content = request_data.get('sql_content') db_addr = '--user=root; --password=123456; --host=127.0.0.1; --port=3306; --enable-check;' db_name = 'pro1' res = table_structure(db_addr, db_name, sql_content) print(res) return Response(self.ret)
(4)sqlmg/serializers.py
# -*- coding:utf-8 -*- from rest_framework import serializers from .models import * class DbSerializer(serializers.ModelSerializer): class Meta: model = Dbconf fields = '__all__'
(5)新建根项目下utils包/inception.py:
#coding=utf-8 import pymysql def table_structure(dbaddr, dbname, sqlcontent): status = 0 sql = '/* {} */ inception_magic_start; use {}; {} inception_magic_commit;'.format(dbaddr, dbname, sqlcontent) try: #conn = pymysql.connect(host='172.16.98.12',user='root',passwd='',db='',port=6669,use_unicode=True, charset="utf8") conn = pymysql.connect(host='127.0.0.1', user='root', passwd='', port=6669, db='', use_unicode=True, charset="utf8") # 连接inception cur = conn.cursor() cur.execute(sql) result = cur.fetchall() cur.close() conn.close() except pymysql.Error as e: status = -1 result = "Mysql Error {}: {}".format(e.args[0], e.args[1]) return {'result':result, 'status':status} def get_rollback(sql, dbname=''): conn = pymysql.connect(host = '127.0.0.1', port = 3306, user = 'root', passwd = '123456', db = dbname, charset = 'utf8') # 连接回滚库 conn.autocommit(True) cur = conn.cursor() cur.execute(sql) return cur.fetchall()
(5)src/view/sql/check.vue ----删除检查form功能修改成右图，否则登录不上
(python36env) [vagrant@CentOS sqlweb]$ python manage.py runserver 0.0.0.0:8081
运行后报错:
if int(self.server_version.split('.', 1)[0]) >= 5:
ValueError: invalid literal for int() with base 10: 'Inception2'
解决:
(python36env) [vagrant@CentOS sqlweb]$ vi /home/vagrant/python36env/lib/python3.6/site-packages/pymysql/connections.py 加入如下代码
try: if int(self.server_version.split('.', 1)[0]) >= 5: self.client_flag |= CLIENT.MULTI_RESULTS except: if self.server_version.split('.', 1)[0] == 'Inception2': self.client_flag |= CLIENT.MULTI_RESULTS
如下图效果:
如下图提交后出来：
上述效果中还没实现审核的功能，如有错误的sql语句它还不能处理 判断语句
3.sql审核判断功能
需求:dba跟我提需求，他需要在我们提交的sql中不允许有use,drop等这样的关键字出现！--实现:先获取sql内容，然后对它做正则匹配关键在内容中否。
我在让它调inception前匹配，先判断是否符合dba的规则，不符合则不让其调inception,
(1)sqlmng/views.py:
from django.shortcuts import render # Create your views here. from rest_framework.response import Response from .baseviews import BaseView from .serializers import DbSerializer from .models import * from utils.inception import table_structure from rest_framework.exceptions import ParseError import re class DbViewSet(BaseView): queryset = Dbconf.objects.all() serializer_class = DbSerializer search_fields = ['name', 'user', 'password','host', 'port', 'env'] class InceptionCheckView(BaseView): forbidden_word_list = ['drop ','use '] # 重写create--因为发任务是post发过来的,(注意view中的create就是对应前端请求的post方法(等价于post)。而序列化中的create方法是view中的create去写数据时调用它)。 def create(self, request, *args, **kwargs): # 接收前端数据 request_data = request.data print(request_data) sql_content = request_data.get('sql_content')
#匹配关键字在不在sql中，不在就抛异常: forbidden_words = [fword for fword in self.forbidden_word_list if re.search(re.compile(fword, re.I), sql_content)] if forbidden_words: raise ParseError({'禁用': forbidden_words})
db_addr = '--user=root; --password=123456; --host=127.0.0.1; --port=3306; --enable-check;' db_name = 'pro1' sql_review = table_structure(db_addr, db_name, sql_content) print(sql_review) result, status = sql_review.get('result'), sql_review.get('status') # 判断检测语法错误，有则返回 if status == -1 or len(result) == 1: # 兼容2种版本的抛错 raise ParseError({self.connect_error: result}) success_sqls = [] exception_sqls = [] for sql_result in result: error_message = sql_result[4] if error_message == 'None' or re.findall('Warning', error_message): success_sqls.append(sql_result) else: exception_sqls.append(error_message) if exception_sqls: raise ParseError({self.exception_sqls: exception_sqls}) return Response(self.ret)
点确定后效果如图:
b.
11
22