cmdb客户端采集数据的完善
Posted huangxuanya
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了cmdb客户端采集数据的完善相关的知识，希望对你有一定的参考价值。
file文件自己去拷贝（这里不提供）
custom_settings.py
import os BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 获取到根路径 MODE = ‘agent‘ SSH_USERNAME = ‘root‘ SSH_PASSWORD = ‘123‘ SHH_PORT = 22 DEBUG = True #代码调试，如果为True就会读取file中文件的内容，如果不是会执行linux命令 PLUGINS_DICT = ##新增的basic,nic要在这里添加才可以采集到数据 ‘basic‘:‘src.plugins.basic.Basic‘, ‘board‘: ‘src.plugins.board.Board‘, ‘disk‘: ‘src.plugins.disk.Disk‘, ‘memory‘: ‘src.plugins.memory.Memory‘, ‘cpu‘:‘src.plugins.cpu.Cpu‘, ‘nic‘:‘src.plugins.nic.Nic‘
convert.py
convert.py
#!/usr/bin/env python # -*- coding:utf-8 -*- def convert_to_int(value,default=0): try: result = int(value) except Exception as e: result = default return result def convert_mb_to_gb(value,default=0): try: value = value.strip(‘MB‘) result = int(value) except Exception as e: result = default return result
读取file文件中board.out的代码测试
s3.py
res = ‘‘‘ SMBios 2.7 present. Handle 0x0001, DMI type 1, 27 bytes System Information Manufacturer: Parallels Software International Inc. Product Name: Parallels Virtual Platform Version: None Serial Number: Parallels-1A 1B CB 3B 64 66 4B 13 86 B0 86 FF 7E 2B 20 30 UUID: 3BCB1B1A-6664-134B-86B0-86FF7E2B2030 Wake-up Type: Power Switch SKU Number: Undefined Family: Parallels VM ‘‘‘ ‘‘‘ result = ‘manufacturer‘ : ‘Parallels Software International Inc.‘ , ‘product_name‘ : ‘Parallels Virtual Platform‘, ‘sn‘ : ‘Parallels-1A 1B CB 3B 64 66 4B 13 86 B0 86 FF 7E 2B 20 30‘ ‘‘‘ key_map = "Manufacturer" : ‘manufacturer‘, "Product Name" : ‘product_name‘, "Serial Number": ‘sn‘ result = data = res.strip().split(‘\\n‘) # print(data) for k in data: v = (k.strip().split(‘:‘)) if len(v) == 2: if v[0] in key_map: result[key_map[v[0]]] = v[1].strip() print(result)
修改了一些代码
from lib.config.config import settings import traceback import importlib import subprocess #管理插件信息的类 class PluginsManger(object): def __init__(self, hostname=None): self.plugins_dict = settings.PLUGINS_DICT self.hostname = hostname self.debug = settings.DEBUG if settings.MODE == ‘ssh‘: self.port = settings.SSH_PORT self.username = settings.SSH_USERNAME self.pwd = settings.SSH_PASSWORD #读取配置文件中的pluginsdict, 并执行对应模块中的process方法 def execute(self): response = for k,v in self.plugins_dict.items(): ret = ‘status‘:None, ‘data‘:None ‘‘‘ k: board,... v: src.plugins.board.Board 字符串 ‘‘‘ #修改 try: #1.导入模块路径 moudle_path,class_name = v.rsplit(‘.‘, 1) #2.导入这个路径 moudle_name = importlib.import_module(moudle_path) #3.导入对应的类 classobj = getattr(moudle_name, class_name) #4.执行类下面对应的process方法 res = classobj().process(self.__cmd_run, self.debug) ret[‘status‘] = 10000 ret[‘data‘] = res except Exception as e: ret[‘status‘] = 10001 ret[‘data‘] = "[%s] 采集 [%s] 出错了， 错误信息是：%s" % ( self.hostname if self.hostname else "Agent", k, str(traceback.format_exc())) response[k] = ret return response def __cmd_run(self,cmd): if settings.MODE == ‘agent‘: return self.__cmd_agent(cmd) elif settings.MODE == ‘ssh‘: return self.__cmd_ssh(cmd) elif settings.MODE == ‘salt‘: return self.__cmd_salt(cmd) else: print(‘只支持的模式，agent/ssh/salt‘) def __cmd_agent(self,cmd): res = subprocess.getoutput(cmd) return res def __cmd_ssh(self, cmd): import paramiko # 创建SSH对象 ssh = paramiko.SSHClient() # 允许连接不在know_hosts文件中的主机 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 连接服务器 ssh.connect(hostname=self.hostname, port= self.port, username=self.username, password=self.pwd) # 执行命令 stdin, stdout, stderr = ssh.exec_command(cmd) # 获取命令结果 result = stdout.read() # 关闭连接 ssh.close() return result def __cmd_salt(self,cmd): command = "salt %s cmd.run %s" %(self.hostname,cmd) res = subprocess.getoutput(command) return res
增加了basic采集数据
#修改一 class Basic(object): def __init__(self): pass @classmethod def initial(cls): return cls() def process(self, command_func, debug): if debug: output = ‘os_platform‘: "linux", ‘os_version‘: "CentOS release 6.6 (Final)\\nKernel \\r on an \\m", ‘hostname‘: ‘c2.com‘ else: output = ‘os_platform‘: command_func("uname").strip(), ‘os_version‘: command_func("cat /etc/issue").strip().split(‘\\n‘)[0], ‘hostname‘: command_func("hostname").strip(), return output
修改了board,cpu,disk,memory代码
from lib.config.config import settings import subprocess import os class Board(object): # 修改二(1) def __init__(self): pass #修改二（2） @classmethod def initial(cls): return cls() def process(self, command_func, debug): if debug: output = open(os.path.join(settings.BASEDIR, ‘files/board.out‘), ‘r‘, encoding=‘utf8‘).read() else: #修改三命令 output = command_func(‘sudo dmidecode -t1‘) #修改四 # data = self.parse_output(output) # return data return self.parse(output) def parse(self, content): key_map = "Manufacturer": ‘manufacturer‘, "Product Name": ‘product_name‘, "Serial Number": ‘sn‘ result = #修改五 # data = res.strip().split(‘\\n‘) # for k in data: # v = (k.strip().split(‘:‘)) # if len(v) == 2: # if v[0] in key_map: # result[key_map[v[0]]] = v[1].strip() # return result for item in content.split(‘\\n‘): row_data = item.strip().split(‘:‘) if len(row_data) == 2: if row_data[0] in key_map: result[key_map[row_data[0]]] = row_data[1].strip() if row_data[1] else row_data[1] return result
import os from lib.config.config import settings class Cpu(object): def __init__(self): pass @classmethod def initial(cls): return cls() def process(self,command_func, debug): if debug: output = open(os.path.join(settings.BASEDIR, ‘files/cpuinfo.out‘), ‘r‘,encoding=‘utf-8‘).read() else: output = command_func("cat /proc/cpuinfo") return self.parse(output) def parse(self, content): """ 解析shell命令返回结果 :param content: shell 命令结果 :return:解析后的结果 """ response = ‘cpu_count‘: 0, ‘cpu_physical_count‘: 0, ‘cpu_model‘: ‘‘ cpu_physical_set = set() content = content.strip() for item in content.split(‘\\n\\n‘): for row_line in item.split(‘\\n‘): key, value = row_line.split(‘:‘) key = key.strip() if key == ‘processor‘: response[‘cpu_count‘] += 1 elif key == ‘physical id‘: cpu_physical_set.add(value) elif key == ‘model name‘: if not response[‘cpu_model‘]: response[‘cpu_model‘] = value response[‘cpu_physical_count‘] = len(cpu_physical_set) return response
from lib.config.config import settings import subprocess import os import re class Disk(object): #修改一 def __init__(self): pass @classmethod def initial(cls): return cls() #修改二 def process(self, command_func,debug): if debug: output = open(os.path.join(settings.BASEDIR, ‘files/disk.out‘), ‘r‘, encoding=‘utf-8‘).read() else: output = command_func(‘sudo MegaCli -PDList -aALL‘) return self.parse(output) def parse(self, content): """ 解析shell命令返回结果 :param content: shell 命令结果 :return:解析后的结果 """ response = result = [] for row_line in content.split(‘\\n\\n\\n\\n‘): result.append(row_line) for item in result: temp_dict = for row in item.split(‘\\n‘): if not row.strip(): continue if len(row.split(‘:‘)) !=2: continue key,value = row.split(‘:‘) name = self.mega_patter_match(key) if name: if key == ‘Raw Size‘: raw_size = re.search(‘(\\d+\\.\\d+)‘, value.strip()) if raw_size: temp_dict[name] = raw_size.group() else: raw_size = ‘0‘ else: temp_dict[name] = value.strip() if temp_dict: response[temp_dict[‘slot‘]] = temp_dict return response @staticmethod def mega_patter_match(needle): grep_pattern = ‘Slot‘: ‘slot‘, ‘Raw Size‘: ‘capacity‘, ‘Inquiry‘: ‘model‘, ‘PD Type‘: ‘pd_type‘ for key,value in grep_pattern.items(): if needle.startswith(key): return value return False
import os from lib import convert from lib.config.config import settings class Memory(object): def __init__(self): pass @classmethod def initial(cls): return cls() def process(self,command_func, debug): if debug: output = open(os.path.join(settings.BASEDIR, ‘files/memory.out‘), ‘r‘, encoding=‘utf-8‘).read() else: output = command_func("sudo dmidecode -q -t 17 2>/dev/null") return self.parse(output) def parse(self, content): """ 解析shell命令返回结果 :param content: shell 命令结果 :return:解析后的结果 """ ram_dict = key_map = ‘Size‘: ‘capacity‘, ‘Locator‘: ‘slot‘, ‘Type‘: ‘model‘, ‘Speed‘: ‘speed‘, ‘Manufacturer‘: ‘manufacturer‘, ‘Serial Number‘: ‘sn‘, devices = content.split(‘Memory Device‘) for item in devices: item = item.strip() if not item: continue if item.startswith(‘#‘): continue segment = lines = item.split(‘\\n\\t‘) for line in lines: if not line.strip(): continue if len(line.split(‘:‘)): key, value = line.split(‘:‘) else: key = line.split(‘:‘)[0] value = "" if key in key_map: if key == ‘Size‘: segment[key_map[‘Size‘]] = convert.convert_mb_to_gb(value, 0) else: segment[key_map[key.strip()]] = value.strip() ram_dict[segment[‘slot‘]] = segment return ram_dict
增加了nic.py
#!/usr/bin/env python # -*- coding:utf-8 -*- import os import re from lib.config.config import settings class Nic(object): def __init__(self): pass @classmethod def initial(cls): return cls() def process(self, command_func, debug): if debug: output = open(os.path.join(settings.BASEDIR, ‘files/nic.out‘), ‘r‘, encoding=‘utf-8‘).read() interfaces_info = self._interfaces_ip(output) else: interfaces_info = self.linux_interfaces(command_func) self.standard(interfaces_info) return interfaces_info def linux_interfaces(self, command_func): ‘‘‘ Obtain interface information for *NIX/BSD variants ‘‘‘ ifaces = dict() ip_path = ‘ip‘ if ip_path: cmd1 = command_func(‘sudo 0 link show‘.format(ip_path)) cmd2 = command_func(‘sudo 0 addr show‘.format(ip_path)) ifaces = self._interfaces_ip(cmd1 + ‘\\n‘ + cmd2) return ifaces def which(self, exe): def _is_executable_file_or_link(exe): # check for os.X_OK doesn‘t suffice because directory may executable return (os.access(exe, os.X_OK) and (os.path.isfile(exe) or os.path.islink(exe))) if exe: if _is_executable_file_or_link(exe): # executable in cwd or fullpath return exe # default path based on busybox‘s default default_path = ‘/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin‘ search_path = os.environ.get(‘PATH‘, default_path) path_ext = os.environ.get(‘PATHEXT‘, ‘.EXE‘) ext_list = path_ext.split(‘;‘) search_path = search_path.split(os.pathsep) if True: # Add any dirs in the default_path which are not in search_path. If # there was no PATH variable found in os.environ, then this will be # a no-op. This ensures that all dirs in the default_path are # searched, which lets salt.utils.which() work well when invoked by # salt-call running from cron (which, depending on platform, may # have a severely limited PATH). search_path.extend( [ x for x in default_path.split(os.pathsep) if x not in search_path ] ) for path in search_path: full_path = os.path.join(path, exe) if _is_executable_file_or_link(full_path): return full_path return None def _number_of_set_bits_to_ipv4_netmask(self, set_bits): # pylint: disable=C0103 ‘‘‘ Returns an IPv4 netmask from the integer representation of that mask. Ex. 0xffffff00 -> ‘255.255.255.0‘ ‘‘‘ return self.cidr_to_ipv4_netmask(self._number_of_set_bits(set_bits)) def cidr_to_ipv4_netmask(self, cidr_bits): ‘‘‘ Returns an IPv4 netmask ‘‘‘ try: cidr_bits = int(cidr_bits) if not 1 <= cidr_bits <= 32: return ‘‘ except ValueError: return ‘‘ netmask = ‘‘ for idx in range(4): if idx: netmask += ‘.‘ if cidr_bits >= 8: netmask += ‘255‘ cidr_bits -= 8 else: netmask += ‘0:d‘.format(256 - (2 ** (8 - cidr_bits))) cidr_bits = 0 return netmask def _number_of_set_bits(self, x): ‘‘‘ Returns the number of bits that are set in a 32bit int ‘‘‘ # Taken from http://stackoverflow.com/a/4912729. Many thanks! x -= (x >> 1) & 0x55555555 x = ((x >> 2) & 0x33333333) + (x & 0x33333333) x = ((x >> 4) + x) & 0x0f0f0f0f x += x >> 8 x += x >> 16 return x & 0x0000003f def _interfaces_ip(self, out): ‘‘‘ Uses ip to return a dictionary of interfaces with various information about each (up/down state, ip address, netmask, and hwaddr) ‘‘‘ ret = dict() right_keys = [‘name‘, ‘hwaddr‘, ‘up‘, ‘netmask‘, ‘ipaddrs‘] def parse_network(value, cols): ‘‘‘ Return a tuple of ip, netmask, broadcast based on the current set of cols ‘‘‘ brd = None if ‘/‘ in value: # we have a CIDR in this address ip, cidr = value.split(‘/‘) # pylint: disable=C0103 else: ip = value # pylint: disable=C0103 cidr = 32 if type_ == ‘inet‘: mask = self.cidr_to_ipv4_netmask(int(cidr)) if ‘brd‘ in cols: brd = cols[cols.index(‘brd‘) + 1] return (ip, mask, brd) groups = re.compile(‘\\r?\\n\\\\d‘).split(out) for group in groups: iface = None data = dict() for line in group.splitlines(): if ‘ ‘ not in line: continue match = re.match(r‘^\\d*:\\s+([\\w.\\-]+)(?:@)?([\\w.\\-]+)?:\\s+<(.+)>‘, line) if match: iface, parent, attrs = match.groups() if ‘UP‘ in attrs.split(‘,‘): data[‘up‘] = True else: data[‘up‘] = False if parent and parent in right_keys: data[parent] = parent continue cols = line.split() if len(cols) >= 2: type_, value = tuple(cols[0:2]) iflabel = cols[-1:][0] if type_ in (‘inet‘,): if ‘secondary‘ not in cols: ipaddr, netmask, broadcast = parse_network(value, cols) if type_ == ‘inet‘: if ‘inet‘ not in data: data[‘inet‘] = list() addr_obj = dict() addr_obj[‘address‘] = ipaddr addr_obj[‘netmask‘] = netmask addr_obj[‘broadcast‘] = broadcast data[‘inet‘].append(addr_obj) else: if ‘secondary‘ not in data: data[‘secondary‘] = list() ip_, mask, brd = parse_network(value, cols) data[‘secondary‘].append( ‘type‘: type_, ‘address‘: ip_, ‘netmask‘: mask, ‘broadcast‘: brd, ) del ip_, mask, brd elif type_.startswith(‘link‘): data[‘hwaddr‘] = value if iface: if iface.startswith(‘pan‘) or iface.startswith(‘lo‘) or iface.startswith(‘v‘): del iface, data else: ret[iface] = data del iface, data return ret def standard(self, interfaces_info): for key, value in interfaces_info.items(): ipaddrs = set() netmask = set() if not ‘inet‘ in value: value[‘ipaddrs‘] = ‘‘ value[‘netmask‘] = ‘‘ else: for item in value[‘inet‘]: ipaddrs.add(item[‘address‘]) netmask.add(item[‘netmask‘]) value[‘ipaddrs‘] = ‘/‘.join(ipaddrs) value[‘netmask‘] = ‘/‘.join(netmask) del value[‘inet‘]
启动文件进行测试
from src.plugins import PluginsManger if __name__ == ‘__main__‘: res = PluginsManger().execute() for k,v in res.items(): print(k,v)
以上是关于cmdb客户端采集数据的完善的主要内容，如果未能解决你的问题，请参考以下文章
CMDB3 完善采集端代码(ssh方案的多线程采集), 异常处理, 服务端目录结构的设计(django的app), API数据分析比对入库