一 CMDB简介
1.1 什么是CMDB？
CMDB（资产管理系统）是所有运维工具的数据基础
1.2 CMDB包含的功能
用户管理，记录测试，开发，运维人员的用户表
业务线管理，需要记录业务的详情
项目管理，指定此项目用属于哪条业务线，以及项目详情
应用管理，指定此应用的开发人员，属于哪个项目，和代码地址，部署目录，部署集群，依赖的应用，软件等信息
主机管理，包括云主机，物理机，主机属于哪个集群，运行着哪些软件，主机管理员，连接哪些网络设备，云主机的资源池，存储等相关信息
主机变更管理，主机的一些信息变更，例如管理员，所属集群等信息更改，连接的网络变更等
网络设备管理，主要记录网络设备的详细信息，及网络设备连接的上级设备
IP管理，IP属于哪个主机，哪个网段, 是否被占用等
1.3 实现的四种方式
1.3.1 Agent实现方式
Agent方式，可以将服务器上面的Agent程序作定时任务，定时将资产信息提交到指定API录入数据库
其本质上就是在各个服务器上执行
subprocess.getoutput()命令，然后将每台机器上执行的结果，返回给主机API，然后主机API收到这些数据之后，放入到数据库中，最终通过web界面展现给用户
#linux
import subprocess
import re
res = subprocess.getoutput("ifconfig")
print(res)
ip=re.findall('inet (.*?) netmask',res)
print(ip)
# windows
import subprocess
import re
res=subprocess.getoutput('ipconfig')
print(res)
ip=re.findall('IPv4 地址 . . . . . . . . . . . . : (.*)',res)
print(ip)
优点：速度快
缺点：需要为每台服务器部署一个Agent程序
使用crontab定时执行python脚本
# 1 进入创建crontab定时任务
crontab -e
# 2 写入任务（每分钟执行一次test.py）
* * * * * python3 test.py
# 3 编写test.py
with open('a.txt','a') as f:
f.write('hello world')
# 4 查看定时任务
crontab -l
1.3.2 ssh实现方式 （基于Paramiko模块）
中控机通过Paramiko（py模块）登录到各个服务器上，然后执行命令的方式去获取各个服务器上的信息
优点：无Agent
缺点：速度慢
如果在服务器较少的情况下，可应用此方法
import paramiko
import re
#创建SSH对象
ssh = paramiko.SSHClient()
# 允许连接不在know_hosts文件中的主机
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy)
# 连接服务器
ssh.connect(hostname='101.133.225.166',port=22,username='root',password='')
# 执行命令
stdin,stdout,stderr = ssh.exec_command('ifconfig')
# 获取命令结果
result = stdout.read().decode('utf-8')
print(result)
ip=re.findall('inet (.*?) netmask',result)
print(ip)
# 关闭连接
ssh.close()
1.3.3 saltstack方式
此方案本质上和第二种方案大致是差不多的流程，中控机发送命令给服务器执行。服务器将结果放入另一个队列中，中控机获取将服务信息发送到API进而录入数据库。
执行流程:
第一步: 由管理员录入资产(主机名,SN等信息),通过后台管理,录入数据库
第二步: salt-master从数据库获取未采集资产信息的服务器
第三步: salt-master发送命令给salt-minion执行
第四步: salt-master拿到执行结果
第五步: 将结果发送给API
第六步: API将其写入数据库解释:
salt-master可以理解为主人
salt-minion可以理解为奴隶
优点：快，开发成本低
缺点：依赖于第三方工具
salstack的安装和配置
1.安装和配置
master端：
"""
1. 安装salt-master
yum install salt-master
2. 修改配置文件：/etc/salt/master
interface: 0.0.0.0 # 表示Master的IP
3. 启动
service salt-master start
"""
slave端：
"""
1. 安装salt-minion
yum install salt-minion
2. 修改配置文件 /etc/salt/minion
master: 10.211.55.4 # master的地址
或
master:
- 10.211.55.4
- 10.211.55.5
random_master: True
id: c2.salt.com # 客户端在salt-master中显示的唯一ID
3. 启动
service salt-minion start
2.授权
salt-key -L # 查看已授权和未授权的slave
salt-key -a salve_id # 接受指定id的salve
salt-key -r salve_id # 拒绝指定id的salve
salt-key -d salve_id # 删除指定id的salve
3.执行命令
在master服务器上对salve进行远程操作
salt 'c2.salt.com' cmd.run 'ifconfig'
# 基于API的方式
import salt.client
local = salt.client.LocalClient()
result = local.cmd('c2.salt.com', 'cmd.run', ['ifconfig'])
1.3.4 Puppet（ruby语言开发）（了解）
每隔30分钟，通过RPC消息队列将执行的结果返回给用户
二 三种方案客户端编写
2.1 目录结构划分
autoclient # 项目名
-bin # 启动文件路径
-start.py # 启动文件
-config # 配置文件路径
-cert # 私钥
-custom_settings.py # 用户自定义配置
-files # 测试数据文件
-board.out
-cpuinfo.out
-disk.out
-memory.out
-nic.out
-lib # 库文件夹
-conf # 配置信息文件夹
-config.py # 配置类
-global_settings.py # 全局常量配置
-convert.py # 公共方法
-src # 源文件
-plugins # 插件
-__init__.py # 初始化文件
-basic.py
-board.py
-cpu.py
-disk.py
-memory.py
-nic.py
script.py # 脚本文件
client.py # 客户端类
tests # 测试文件夹
# 总结：bin，config，files，lib，src几个文件夹
2.2 仿django配置文件
custom_settings.py
# 用户配置
PORT = 22
USER = 'lqz'
global_settings.py
#### 全局配置
PORT = 22
USER = 'root'
config.py
from config import custom_settings
from . import global_settings
class Settings():
def __init__(self):
#### 全局配置
for key in dir(global_settings):
if key.isupper():
#### 获取key所对应的值
v = getattr(global_settings, key)
#### 设置key以及值到当前的setting对象
setattr(self, key, v)
#### 自定制配置
for key in dir(custom_settings):
if key.isupper():
#### 获取key所对应的值
v = getattr(custom_settings, key)
#### 设置key以及值到当前的setting对象
setattr(self, key, v)
settings = Settings()
2.3 可插拔式配置
custom_settings.py
### 可插拔式的采集,注释掉某个就不会执行
PLUGINS_DICT = {
'basic':'src.plugins.basic.Basic',
'board':'src.plugins.board.Board',
'cpu':'src.plugins.cpu.Cpu',
'disk':'src.plugins.disk.Disk',
'nic':'src.plugins.nic.Nic',
'memory':'src.plugins.memory.Memory',
}
src/plugins/__init__.py
import traceback
from lib.conf.config import settings
import importlib
import subprocess
### 管理插件信息的类
class PluginsManager(object):
def __init__(self, hostname=None):
pass
### 读取配置文件中的pluginsdict， 并执行对应模块中的process方法
def execute(self):
response = {}
for k, v in self.plugins_dict.items():
ret = {"status":None, 'data':None}
'''
k: board,...
v: src.plugins.board.Board 字符串
'''
try:
# 1. 导入模块路径
moudle_path, class_name = v.rsplit('.', 1)
# 2. 导入这个路径
moudle_name = importlib.import_module(moudle_path)
# 3. 导入对应模块下的类
classobj = getattr(moudle_name, class_name)
# 4. 执行类下面对应的process方法
res = classobj().process()
except Exception as e:
pass
return response
src/plugins/cpu.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
from lib.conf.config import settings
class Cpu(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
print('cpu print')
src/plugins/disk.py
import os
from lib.conf.config import settings
class Cpu(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
print('disk print')
2.4 冗余代码抽取
继承方式
把函数当参数传入函数中:
在src/plugins/init.py中写，__隐藏，调用execute的时候，把函数地址和命令传入
import traceback
from lib.conf.config import settings
import importlib
import subprocess
### 管理插件信息的类
class PluginsManager(object):
def __init__(self, hostname=None):
self.plugins_dict = settings.PLUGINS_DICT
self.hostname = hostname # 采集客户端的地址
self.debug = settings.DEBUG
if settings.MODE == 'ssh': # ssh方式才需要端口，用户名，密码，这些应该放到配置文件中
self.port = settings.SSH_PORT
self.name = settings.SSH_USERNAME
self.pwd = settings.SSH_PASSWORD
### 读取配置文件中的pluginsdict， 并执行对应模块中的process方法
def execute(self):
response = {}
for k, v in self.plugins_dict.items():
ret = {"status":None, 'data':None}
'''
k: board,...
v: src.plugins.board.Board 字符串
'''
try:
# 1. 导入模块路径
moudle_path, class_name = v.rsplit('.', 1)
# 2. 导入这个路径
moudle_name = importlib.import_module(moudle_path)
# 3. 导入对应模块下的类
classobj = getattr(moudle_name, class_name)
# 4. 执行类下面对应的process方法
res = classobj().process(self.__cmd_run, self.debug)
ret['status'] = 10000
ret['data'] = res
except Exception as e:
ret['status'] = 10001
ret['data']= "[%s] 采集 [%s] 出错了， 错误信息是：%s" % (self.hostname if self.hostname else "Agent", k, str(traceback.format_exc()))
response[k] = ret
return response
def __cmd_run(self, cmd):
if settings.MODE == 'agent':
return self.__cmd_agent(cmd)
elif settings.MODE == 'ssh':
return self.__cmd_ssh(cmd)
elif settings.MODE == 'salt':
return self.__cmd_salt(cmd)
else:
print("只支持的模式有：agent/ssh/salt")
def __cmd_agent(self, cmd):
res = subprocess.getoutput(cmd)
return res
def __cmd_ssh(self, cmd):
import paramiko
# 创建SSH对象
ssh = paramiko.SSHClient()
# 允许连接不在know_hosts文件中的主机
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 连接服务器
ssh.connect(hostname=self.hostname, port=self.port, username=self.name, password=self.pwd)
# 执行命令
stdin, stdout, stderr = ssh.exec_command(cmd)
# 获取命令结果
result = stdout.read()
# 关闭连接
ssh.close()
return result
def __cmd_salt(self, cmd):
command = "salt %s cmd.run %s" % (self.hostname, cmd)
res = subprocess.getoutput(command)
return res
在cpu.py disk.py中编写
class Cpu(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/cpuinfo.out'), 'r', encoding='utf-8').read()
else:
output = command_func("cat /proc/cpuinfo")
return self.parse(output)
2.5 解析数据（以主板为例）
# sudo dmidecode -t1 https://ipcmen.com/dmidecode
# 可以获取BIOS，系统，主板，处理器，内存，缓存等 序列号、电脑厂商、串口信息以及其它系统配件信息
res = '''
SMBIOS 2.7 present.
Handle 0x0001, DMI type 1, 27 bytes
System Information
Manufacturer: Parallels Software International Inc.
Product Name: Parallels Virtual Platform
Version: None
Serial Number: Parallels-1A 1B CB 3B 64 66 4B 13 86 B0 86 FF 7E 2B 20 30
UUID: 3BCB1B1A-6664-134B-86B0-86FF7E2B2030
Wake-up Type: Power Switch
SKU Number: Undefined
Family: Parallels VM
'''
key_map = {
"Manufacturer" : 'manufacturer',
"Product Name" : 'product_name',
"Serial Number": 'sn'
}
result = {}
data = res.strip().split('
')
# print(data)
for k in data:
v = (k.strip().split(':'))
if len(v) == 2:
if v[0] in key_map:
result[key_map[v[0]]] = v[1].strip()
print(result)
'''
result = {
'manufacturer' : 'Parallels Software International Inc.' ,
'product_name' : 'Parallels Virtual Platform',
'sn' : 'Parallels-1A 1B CB 3B 64 66 4B 13 86 B0 86 FF 7E 2B 20 30'
}
'''
2.6 代码整合
plugins
-__init__.py
-basic.py
-board.py
-cpu.py
-disk.py
-memory.py
-nic.py
#__init__.py
import traceback
from lib.conf.config import settings
import importlib
import subprocess
### 管理插件信息的类
class PluginsManager(object):
def __init__(self, hostname=None):
self.plugins_dict = settings.PLUGINS_DICT
self.hostname = hostname
self.debug = settings.DEBUG
if settings.MODE == 'ssh':
self.port = settings.SSH_PORT
self.name = settings.SSH_USERNAME
self.pwd = settings.SSH_PASSWORD
### 读取配置文件中的pluginsdict， 并执行对应模块中的process方法
def execute(self):
response = {}
for k, v in self.plugins_dict.items():
ret = {"status":None, 'data':None}
'''
k: board,...
v: src.plugins.board.Board 字符串
'''
try:
# 1. 导入模块路径
moudle_path, class_name = v.rsplit('.', 1)
# 2. 导入这个路径
moudle_name = importlib.import_module(moudle_path)
# 3. 导入对应模块下的类
classobj = getattr(moudle_name, class_name)
# 4. 执行类下面对应的process方法
res = classobj().process(self.__cmd_run, self.debug)
ret['status'] = 10000
ret['data'] = res
except Exception as e:
ret['status'] = 10001
ret['data']= "[%s] 采集 [%s] 出错了， 错误信息是：%s" % (self.hostname if self.hostname else "Agent", k, str(traceback.format_exc()))
response[k] = ret
return response
def __cmd_run(self, cmd):
if settings.MODE == 'agent':
return self.__cmd_agent(cmd)
elif settings.MODE == 'ssh':
return self.__cmd_ssh(cmd)
elif settings.MODE == 'salt':
return self.__cmd_salt(cmd)
else:
print("只支持的模式有：agent/ssh/salt")
def __cmd_agent(self, cmd):
res = subprocess.getoutput(cmd)
return res
def __cmd_ssh(self, cmd):
import paramiko
# 创建SSH对象
ssh = paramiko.SSHClient()
# 允许连接不在know_hosts文件中的主机
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 连接服务器
ssh.connect(hostname=self.hostname, port=self.port, username=self.name, password=self.pwd)
# 执行命令
stdin, stdout, stderr = ssh.exec_command(cmd)
# 获取命令结果
result = stdout.read()
# 关闭连接
ssh.close()
return result
def __cmd_salt(self, cmd):
command = "salt %s cmd.run %s" % (self.hostname, cmd)
res = subprocess.getoutput(command)
return res
# basic.py
class Basic(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = {
'os_platform': "linux",
'os_version': "CentOS release 6.6 (Final)
Kernel
on an m",
'hostname': 'c2000.com'
}
else:
output = {
'os_platform': command_func("uname").strip(),
'os_version': command_func("cat /etc/issue").strip().split('
')[0],
'hostname': command_func("hostname").strip(),
}
return output
# board.py
import os
from lib.conf.config import settings
class Board(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/board.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo dmidecode -t1")
return self.parse(output)
def parse(self, content):
result = {}
key_map = {
'Manufacturer': 'manufacturer',
'Product Name': 'model',
'Serial Number': 'sn',
}
for item in content.split('
'):
row_data = item.strip().split(':')
if len(row_data) == 2:
if row_data[0] in key_map:
result[key_map[row_data[0]]] = row_data[1].strip() if row_data[1] else row_data[1]
return result
# cpu.py
import os
from lib.conf.config import settings
class Cpu(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/cpuinfo.out'), 'r', encoding='utf-8').read()
else:
output = command_func("cat /proc/cpuinfo")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {'cpu_count': 0, 'cpu_physical_count': 0, 'cpu_model': ''}
cpu_physical_set = set()
content = content.strip()
for item in content.split('
'):
for row_line in item.split('
'):
key, value = row_line.split(':')
key = key.strip()
if key == 'processor':
response['cpu_count'] += 1
elif key == 'physical id':
cpu_physical_set.add(value)
elif key == 'model name':
if not response['cpu_model']:
response['cpu_model'] = value
response['cpu_physical_count'] = len(cpu_physical_set)
return response
#disk.py
import re
import os
from lib.conf.config import settings
class Disk(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/disk.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo MegaCli -PDList -aALL")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {}
result = []
for row_line in content.split("
"):
result.append(row_line)
for item in result:
temp_dict = {}
for row in item.split('
'):
if not row.strip():
continue
if len(row.split(':')) != 2:
continue
key, value = row.split(':')
name = self.mega_patter_match(key)
if name:
if key == 'Raw Size':
raw_size = re.search('(d+.d+)', value.strip())
if raw_size:
temp_dict[name] = raw_size.group()
else:
raw_size = '0'
else:
temp_dict[name] = value.strip()
if temp_dict:
response[temp_dict['slot']] = temp_dict
return response
@staticmethod
def mega_patter_match(needle):
grep_pattern = {'Slot': 'slot', 'Raw Size': 'capacity', 'Inquiry': 'model', 'PD Type': 'pd_type'}
for key, value in grep_pattern.items():
if needle.startswith(key):
return value
return False
# memory.py
import os
from lib import convert
from lib.conf.config import settings
class Memory(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/memory.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo dmidecode -q -t 17 2>/dev/null")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
ram_dict = {}
key_map = {
'Size': 'capacity',
'Locator': 'slot',
'Type': 'model',
'Speed': 'speed',
'Manufacturer': 'manufacturer',
'Serial Number': 'sn',
}
devices = content.split('Memory Device')
for item in devices:
item = item.strip()
if not item:
continue
if item.startswith('#'):
continue
segment = {}
lines = item.split('
')
for line in lines:
if not line.strip():
continue
if len(line.split(':')):
key, value = line.split(':')
else:
key = line.split(':')[0]
value = ""
if key in key_map:
if key == 'Size':
segment[key_map['Size']] = convert.convert_mb_to_gb(value, 0)
else:
segment[key_map[key.strip()]] = value.strip()
ram_dict[segment['slot']] = segment
return ram_dict
#nic.py 网络接口控制器
import os
import re
from lib.conf.config import settings
class Nic(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/nic.out'), 'r', encoding='utf-8').read()
interfaces_info = self._interfaces_ip(output)
else:
interfaces_info = self.linux_interfaces(command_func)
self.standard(interfaces_info)
return interfaces_info
def linux_interfaces(self, command_func):
'''
Obtain interface information for *NIX/BSD variants
'''
ifaces = dict()
ip_path = 'ip'
if ip_path:
cmd1 = command_func('sudo {0} link show'.format(ip_path))
cmd2 = command_func('sudo {0} addr show'.format(ip_path))
ifaces = self._interfaces_ip(cmd1 + '
' + cmd2)
return ifaces
def which(self, exe):
def _is_executable_file_or_link(exe):
# check for os.X_OK doesn't suffice because directory may executable
return (os.access(exe, os.X_OK) and
(os.path.isfile(exe) or os.path.islink(exe)))
if exe:
if _is_executable_file_or_link(exe):
# executable in cwd or fullpath
return exe
# default path based on busybox's default
default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
search_path = os.environ.get('PATH', default_path)
path_ext = os.environ.get('PATHEXT', '.EXE')
ext_list = path_ext.split(';')
search_path = search_path.split(os.pathsep)
if True:
# Add any dirs in the default_path which are not in search_path. If
# there was no PATH variable found in os.environ, then this will be
# a no-op. This ensures that all dirs in the default_path are
# searched, which lets salt.utils.which() work well when invoked by
# salt-call running from cron (which, depending on platform, may
# have a severely limited PATH).
search_path.extend(
[
x for x in default_path.split(os.pathsep)
if x not in search_path
]
)
for path in search_path:
full_path = os.path.join(path, exe)
if _is_executable_file_or_link(full_path):
return full_path
return None
def _number_of_set_bits_to_ipv4_netmask(self, set_bits): # pylint: disable=C0103
'''
Returns an IPv4 netmask from the integer representation of that mask.
Ex. 0xffffff00 -> '255.255.255.0'
'''
return self.cidr_to_ipv4_netmask(self._number_of_set_bits(set_bits))
def cidr_to_ipv4_netmask(self, cidr_bits):
'''
Returns an IPv4 netmask
'''
try:
cidr_bits = int(cidr_bits)
if not 1 <= cidr_bits <= 32:
return ''
except ValueError:
return ''
netmask = ''
for idx in range(4):
if idx:
netmask += '.'
if cidr_bits >= 8:
netmask += '255'
cidr_bits -= 8
else:
netmask += '{0:d}'.format(256 - (2 ** (8 - cidr_bits)))
cidr_bits = 0
return netmask
def _number_of_set_bits(self, x):
'''
Returns the number of bits that are set in a 32bit int
'''
# Taken from http://stackoverflow.com/a/4912729. Many thanks!
x -= (x >> 1) & 0x55555555
x = ((x >> 2) & 0x33333333) + (x & 0x33333333)
x = ((x >> 4) + x) & 0x0f0f0f0f
x += x >> 8
x += x >> 16
return x & 0x0000003f
def _interfaces_ip(self, out):
'''
Uses ip to return a dictionary of interfaces with various information about
each (up/down state, ip address, netmask, and hwaddr)
'''
ret = dict()
right_keys = ['name', 'hwaddr', 'up', 'netmask', 'ipaddrs']
def parse_network(value, cols):
'''
Return a tuple of ip, netmask, broadcast
based on the current set of cols
'''
brd = None
if '/' in value: # we have a CIDR in this address
ip, cidr = value.split('/') # pylint: disable=C0103
else:
ip = value # pylint: disable=C0103
cidr = 32
if type_ == 'inet':
mask = self.cidr_to_ipv4_netmask(int(cidr))
if 'brd' in cols:
brd = cols[cols.index('brd') + 1]
return (ip, mask, brd)
groups = re.compile('
?
\d').split(out)
for group in groups:
iface = None
data = dict()
for line in group.splitlines():
if ' ' not in line:
continue
match = re.match(r'^d*:s+([w.-]+)(?:@)?([w.-]+)?:s+<(.+)>', line)
if match:
iface, parent, attrs = match.groups()
if 'UP' in attrs.split(','):
data['up'] = True
else:
data['up'] = False
if parent and parent in right_keys:
data[parent] = parent
continue
cols = line.split()
if len(cols) >= 2:
type_, value = tuple(cols[0:2])
iflabel = cols[-1:][0]
if type_ in ('inet',):
if 'secondary' not in cols:
ipaddr, netmask, broadcast = parse_network(value, cols)
if type_ == 'inet':
if 'inet' not in data:
data['inet'] = list()
addr_obj = dict()
addr_obj['address'] = ipaddr
addr_obj['netmask'] = netmask
addr_obj['broadcast'] = broadcast
data['inet'].append(addr_obj)
else:
if 'secondary' not in data:
data['secondary'] =