CMDB

CMDB - 发布系统

完全自己开发一套发布系统
表设计

• 环境，
• 主机 ， -> salt-id
• 代码地址、就是包 -> 地址
• 应用 -> app
  -记录日志 -> 时间，事件

SaltStack
SaltStack 采用 C/S模式
master和minion之间的通信用到了zeromq消息队列 ，每个minion 有一个salt_id 是绝对唯一的
Master与Minion之间通过ZeroMq进行消息传递，使用了ZeroMq的发布-订阅模式，连接方式包括tcp，ipc

废话少说，直接上图：

 

发布系统流程图.jpg

表设计
models.py

from django.db import models

# Create your models here.
class Use_Env(models.Model):
    name = models.CharField(max_length=32, blank=True, null=True, verbose_name='环境名')

    def __str__(self):
        return self.name

    class Meta:
        verbose_name_plural = '环境表'

class Host(models.Model):
    hostname = models.CharField(max_length=32, blank=True, null=True, verbose_name="salt_id")
    ip = models.CharField(max_length=64, blank=True, null=True, verbose_name='IP')

    def __str__(self):
        return self.hostname

    class Meta:
        verbose_name_plural = "主机表"


class Record_Log(models.Model):
    timestamp = models.CharField(max_length=64, blank=True, null=True, verbose_name='时间')
    project = models.ForeignKey(to='App', blank=True, null=True, verbose_name='项目', related_name='proj')
    package = models.ManyToManyField(to='Package', blank=True, null=True, verbose_name='包', related_name='pack')
    env = models.ForeignKey(to='Use_Env', blank=True, null=True, verbose_name='环境', related_name='env')

    def __str__(self):
        return self.timestamp

    class Meta:
        verbose_name_plural = '记录日志'


class App(models.Model):
    name = models.CharField(max_length=32, blank=True, null=True, verbose_name='应用名')
    path = models.CharField(max_length=64, blank=True, null=True, verbose_name='应用路径')
    environment = models.ForeignKey(to='Use_Env', blank=True, null=True, verbose_name='环境')
    hosts = models.ForeignKey(to='Host', blank=True, null=True, verbose_name='对应主机', related_name='apphost')
    # _script = models.CharField(max_length=32, blank=True, null=True, verbose_name='部署脚本')
    package = models.ForeignKey(to='Package', blank=True, null=True, verbose_name='代码', related_name='apppack')
    _app = models.ForeignKey(to='App', blank=True, null=True, verbose_name='上级应用')

    def __str__(self):
        return self.name

    class Meta:
        verbose_name_plural = '项目表'


class Package(models.Model):
    name = models.CharField(max_length=64, blank=True, null=True, verbose_name='包名/版本号')
    pack_path = models.CharField(max_length=64, blank=True, null=True, verbose_name='代码路径/地址')
    project = models.ForeignKey(to='App', blank=True, null=True, verbose_name='所属项目', related_name='packapp')

    def __str__(self):
        return self.name

    class Meta:
        verbose_name_plural = '代码'

迁移，创建数据库
python manage.py makemigrations
python manage.py migrate

伪代码

views.py

def pubilsh(request):
    if request.method == 'GET':
        env = models.Use_Env.object.all()
        return render(request,'fabu.html', locals())
    else:
        env = request.POST.get('env')
        app = request.POST.get('app')
        obj_list = models.App.objects.filter(name=app,environment__name=env)  # 跨表查询

        # 拿到对应的主机组   代码 -> 地址
        # 循环 主机组  推送代码
        app_name = ORM 查表
        host_list = [{'id':'salt-id','path':'/data/app/www/abc'},]   # 这里是通过数据库取到的
        package = 'svn://xxxx'   # svn 地址

template

fabu.html

<form method='post'>
    <label>应用：</label>
    <input type="text" name="app"> # name -> key  ,框是 -> values
        <select class="form-control" id="nubers" name="env">   # name="env" -> key
            {% for i in env %}
                <option value="{{ i.name }}"> {{ i.name }}</option>  # value="{{ i.name }}" -> values
            {% endfor %}
        </select>
    <input type="submit" value="提交">
</form>

自动化管理平台 -> 必须是 和salt-master 安装在同一台机上 ，使用salt原生的API

第一步 在 自动化管理平台 里面下载代码 （可打包） 通过 subporcsess 执行命令
# from subprocess import Popen, PIPE
import os
path = os.getcwd() + r'/project_path/'
subprocsess -> 执行命令
# cd path
# mkdir app_name
# svn co $package
# tar 打包

create.sh

#!/bin/bash
cd path
mkdir $app_name && cd $app_name
svn co $package
tar 打包



subprocess.call(['cd',  '-l'])


from subprocess import Popen, PIPE 

p = subprocess.Popen('sh create.sh', stdout=PIPE, shell=True)   

第二步 推送 salt stack -> state.sls # 状态管理
写 state.sls 规则的yml文件
通过 Python 代码 salt-api 调用 state 触发推送

第三部 执行远程端代码 -> cmd.run cd 路径 python xxx

django celery 被封装成了 djcelery

就要学会如何使用

celery.py
form __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings

os.environ.setdefault('DJANGO_SETTIONG_MODULE', '项目名称.settiongs')
app = Celery('项目名称')

app.config_from_object('django.conf:settings')

app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

@app.task(bind=True)
def debug_task(self):
    print ('Request:  {0!x}'.format(self.request))

settings.py

# 文件最后添加

import djcelery
from celery.schedules improt crontab
from datetime import timedelta
djcelery.setup_loader()

CELERY_TIMEZONE = TIME_ZONE
BROKER_URL='redis://:'             # redis 地址 发送端口
CELERY_RESULT_BACKEND = 'redis://:'      # redis 接收端口
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Africa/Nairobi'
CELERY_IMPORTS = ['应用名目录下的.task',]   # 应用名目录下的.task ，主要看有没有task.py文件
CELERY_MAX_TASKS_PRR_CHILD = 3
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'

task.py

form __future__ import absolute_import, unicode_literals
import time
import requests
from celery import shared_task
from django.views.decorators.csrf import csrf_exempt, csrf_protect
from django.shortcuts import render, HttpResponse,redirect



@shared_task:
def add(x, y):
    return x+y   # 定义自己的推送代码


@shared_task:
def add(x, y):
    return x*y


@shared_task:
def xsun(numbers):
    print (sum(numbers))
    return sum(numbers)    

urls.py

urlpatterns = [
           url(r'^celery/', views.celery_status),   # 必须要写的路由      
          ]

views.py

def celery_status(request):
    import datetime
    import json
    if request.method == 'GET':
        if request.GET.get('x') and request.GET.get('y'):
            if request.GET.get('after'):
                ctime = datetime.datetime.now()
                utc_ctime = datetime.datetime.utcfromtimestamp(ctime.timestamp())
                s1 = datetime.timedelta(seconds=int(request.GET.get('after'))*60)
                ctime_x = utc_ctime + s1
            
            year = request.GET.get('year')
            mouth = request.GET.get('month')
            day = reuqest.GET.get('day')
            hour = request.GET.get('hour')
            minute = request.GET.get('minute')

            if year and mouth and day and hour and minute:
                ctime = datetime.datetime(year=int(year), month=int(mouth)),
                                           day=int(day), hour=int(hour), minute=int(minute))
                # 把当前的本地时间转换成 UTC 时间
                ctime_x = datetime.datetime.utcfromtimestamp(ctime.timestamp)
            
            if ctime_x:
                #  最核心的代码
                ret = add.apply_async(args=[int(request.GET.get('x')), int(request.GET.get('y'))], eta=ctime_x)
                num = ret.id
 
            if request.GET.get('cancel'):
                async = AsyncResult(id=request.GET.get('cancel'), app=app)
                async.revoke(terminate=True)
                cancel_tag = '取消成功'
 
            if request.GET.get('stop'):
                async = AsyncResult(id=request.GET.get('stop'), app=app)       
                async.revoke()
                stop_tag='中止成功'
            return render(request, 'celery.html', locals())
        else:
            ret = request.POST.get('id','')
            data = ""
            if ret:
                async = AsyncResult(id=ret,app=app)
                if async.successful():
                    data = "执行成功，数据是: " + str( async.get() ) 
                    async.forget()
                elif async.failed():
                    data='执行失败'
                elif async.status == 'PBNDING':
                    data = "等待被执行"
                elif async.status == 'RBTPY' :
                    data = '任务异常正常重试'
                elif async.status == 'STARTBD':
                    data = "任务正在执行"
                else:
                    data = "未知"
        retrun render(request, 'celery.html', locals())

celery 需要在命令行里单独启动 terminal

celery worker -A 发布 -l debug

templates
celery.html

<form method="post">
    {% csrf_token %}
    id: <input type="text" name='id'>
    结果: <input type="text" value="{{ data }}">
    <input type="submit" value="提交">
</form>

<br>    # 空行
<hr>    # 分割线

<form method="get">
    x:<input type="text" name="x">
    +
    y:<input type="text" name="y">
    <br>
    年: <input type="text" name="year">
    月: <input type="text" name="month">
    日: <input type="text" name="day">
    时: <input type="text" name="hour">
    分: <input type="text" name="minute">
    <br>
    几分钟后： <inpur type="text" name="after">
    <br>
    取消这个任务: <input type="text"  name="cancel">
    结果: <input type="text" value="{{ cancel_tag }}">
    <br>
    中止这个任务: <input type="text" name="stop">
    结果: <input type="text" value="{{ stop_tag }}">
    <br>
    <hr>
    结果: <input type="text" value="{{ stop_tag }}"
    <br>
    <hr>
    结果： <input type="text" value="{{ num }}">
    <input type="submit" value="提交">

</form>

发布代码

第一步 在 自动化管理平台 里面下载代码 （可打包） 通过 subporcsess 执行命令

# from subprocess import Popen, PIPE 
import os 
path = os.getcwd() + r'/project_path/'
subprocsess -> 执行命令
                 # cd path
                 # mkdir $app_name && cd $app_name
                 # svn co $package
                 # tar 打包

create.sh

#!/bin/bash
cd path
mkdir $app_name && cd $app_name
svn co $package
tar 打包

from subprocess import Popen, PIPE
path = os.getcwd() + r'/project_path/'

拼接

cd path && mkdir $app_name && cd $app_name && svn co $package

cmd = 'cd {0} && mkdir {1} && cd {1} && svn co {2}'.format(path, app_name, package)

第二步 推送 salt stack -> state.sls # 状态管理
写 state.sls 规则的yml文件
通过 Python 代码 salt-api 调用 state 触发推送

第三部 执行远程端代码 -> cmd.run cd 路径 python xxx

class MainSalt(object):          # salt 代码
    def __init__(self, tgt='*')
        self.local = sc.LocalClient()
        self.tgt = tgt

    def get_cache_returns(self, func):
        while not self.local.get_cache_returns(func):
            time.sleep(1)
        return self.local.get_cache_returns(func)

  
    def cmd_run(self, run_func):
        if not isinstance(run_func, list):
            raise TypeError(AttributeError)
        cmd_id = self.local.cmd_async(self.tgt, 'cmd.run', run_func)
        ret_cmd = self.get_cache_returns(cmd_id)
        return ret_cmd
       
    def state(self, salt_fun, tag=''):
           
        if tag:    
            disk_id = self.local.cmd_async(self.tgt, 'state.sls', [salt_fun, tag]) 
                  # 实际上 把信息塞入ZeroMq 返回一个ID 
        else: 
            disk_id = self.local.cmd_async(self.tgt, 'state.sls', [salt_fun,])
        ret_disk_data = self.get_cache_returns(disk_id)
        return ret_disk_data

    def push_package(self, pillar_dic):
        tag = 'pillar={0}'.format(json.dumps(pillar_dic))
        salt_fun = 'test'   # test.sls   就是状态管理里面的这个文件 加载
        return self.state(salt_fun, tag)



def sub_run(cmd):
    retrue subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)    

def pubilsh(request):
    if request.method == 'GET':
        env = models.Use_Env.object.all()
        return render(request,'fabu.html', locals())
    else:
        env = request.POST.get('env')
        app = request.POST.get('app')
        obj_list = models.App.objects.filter(name=app,environment__name=env)  # 跨表查询

        # 拿到对应的主机组   代码 -> 地址
        # 循环 主机组  推送代码
        app_name = ORM 查表
        host_list = [{'id':'salt-id','path':'/data/app/www/abc'},]   # 这里是通过数据库取到的
        package = 'svn://xxxx'   # svn 地址


        for i in host_list:
            cmd = 'cd {0} && mkdir {1} && cd {1} && svn co {2}'.format(path, app_name, package)
            ret = sub_run(cmd)   

            m_salt = MainSalt(host.get('id'))
            pillar_dic = {
                         'path':i.get('path')+ '/' + app_name,
                         'app' : app_name
                    }
            ret = m_salt.push_package(pillar_dic)
            
            #第三部
            m_salt.cmd_run('cd {0} && python manage.py runserver 8080'.format(itme.get('path') + '/' + app_name))
        

satlstack 管理

top.sls

base:
  '*': 
    - jar_package

test.sls

test_ci:
  file.recurse:
    - name: {{ pillar['path'] }}
    - source: salt://project_path/{{ pillar['app'] }}   # project_path需要做个软连接
    - user: root
    - dir_mode: 755
    - file_mode: 644
    - template: jinja
    - makedirs: True
    - include_enpty: True