『Python』 爬取 WooYun 论坛所有漏洞条目的相关信息

Posted 2020-07-13 Anka9080 ‘s Blog

 

每个漏洞条目包含：

乌云ID，漏洞标题，漏洞所属厂商，白帽子，漏洞类型，厂商或平台给的Rank值

 

主要是做数据分析使用：
可以分析某厂商的各类型漏洞的统计；
或者对白帽子的能力进行分析.....

数据更新时间：2016/5/27
漏洞条目：104796条

 

数据截图如下：

 

数据网盘链接：

链接：http://pan.baidu.com/s/1bpDNKOv 密码：6y57

 

爬虫脚本：

# coding:utf-8
# author: anka9080
# version: 1.0  py3

import sys,re,time,socket
from requests import get
from queue import Queue, Empty
from threading import Thread

# 全局变量
COUNT = 1
START_URL = \'http://wooyun.org/bugs\'
ID_DETAILS = []
ALL_ID = []
Failed_ID = []
PROXIES = []

HEADERS = {
	"Accept": "text/html,application/xhtml+xml,application/xml,application/json;q=0.9,image/webp,*/*;q=0.8",
	"Accept-Encoding": "gzip, deflate, sdch",
	"Accept-Language": "zh-CN,zh;q=0.8",
	"Cache-Control": "max-age=0",
	"Connection": "keep-alive",
	"DNT": "1",
	"Host": "wooyun.org",
	"Upgrade-Insecure-Requests": "1",
	"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2716.0 Safari/537.36"
}

class WooYunSpider(Thread):
	"""docstring for WooYunSpider"""
	def __init__(self,queue):
		Thread.__init__(self)
		self.pattern1 = re.compile(r\'title>(.*?)\\| WooYun.*?keywords" content="(.*?),(.*?),(.*?),wooyun\',re.S)  # 匹配模式在 compile 的时候指定
		self.pattern2 = re.compile(r"漏洞Rank：(\\d{1,3})")
		self.queue = queue
		self.start() # 执行 run()

	def run(self):
		"每次读取 queue 的一条"
		global COUNT,RES_LOG,ERR_LOG
		while(1):
			try:
				id = self.queue.get(block = False)
				r = get(\'http://wooyun.org/bugs/\' + id,headers = HEADERS)
				html = r.text
			except Empty:
				break
			except Exception as e:
				msg = \'[ - Socket_Excpt ] 链接被拒绝，再次添加到队列：\' + id
				print(msg)
				ERR_LOG.write(msg+\'\\n\')
				self.queue.put(id)  # 访问失败则把这个 URL从新加入队列
			else:
				title,comp,author,bug_type,rank = self.get_detail(html,id)
				detail = id+\'----\'+title+\'----\'+comp+\'----\'+author+\'----\'+bug_type+\'----\'+rank
				try: # 写入文件可能会诱发 gbk 编码异常，这里保存 id 到 failed
					RES_LOG.write(detail + \'\\n\')
				except Exception as e:
					Failed_ID.append(id)
					msg = \'[ - Encode_Excpt ] 字符编码异常：\' + id
					print(msg)
					ERR_LOG.write(msg+\'\\n\')
				ID_DETAILS.append(detail)
			# time.sleep(1)

			print(\'[ - info ] id: {}  count: {}  time: {:.2f}s\'.format(id,COUNT,time.time() - start))
			COUNT += 1

	# 由 缺陷编号 获得对应的 厂商 和 漏洞类型信息
	def get_detail(self,html,id):
		global ERR_LOG
		try:
			# print(html)
			res = self.pattern1.search(html)
			title = res.group(1).strip()
			comp = res.group(2).strip()
			author = res.group(3).strip()
			bug_type = res.group(4).strip()
		except Exception as e:
			msg = \'[ - Detail_Excpt ] 未解析出 标题等相关信息：\' + id
			print(msg)
			ERR_LOG.write(msg+\'\\n\')
			Failed_ID.append(id)
			title,comp,author,bug_type,rank = \'Null\',\'Null\',\'Null\',\'Null\',\'Null\'
		else:
			try:
				res2 = self.pattern2.search(html)  # 若厂商暂无回应则 rank 为 Null
				rank = res2.group(1).strip()
			except Exception as e:
				msg = \'[ - Rank_Excpt ] 未解析出 Rank：\' + id
				print(msg)
				ERR_LOG.write(msg+\'\\n\')
				rank = \'Null\'
			
		finally:
			try:
				print (title,comp,author,bug_type,rank)
			except Exception as e:
				msg = \'[ - Print_Excpt ] 字符编码异常：\' + id +\'::\'+ str(e)
				print(msg)
				ERR_LOG.write(msg+\'\\n\')
			return title,comp,author,bug_type,rank




class ThreadPool(object):
	def __init__(self,thread_num,id_file):
		self.queue = Queue() # 需要执行的队列
		self.threads = [] # 多线程列表
		self.add_task(id_file)
		self.init_threads(thread_num)
		

	def add_task(self,id_file):
		with open(id_file) as input:
			for id in input.readlines():
				self.queue.put(id.strip())			

	def init_threads(self,thread_num):
		for i in range(thread_num):
			print (\'[ - info :] loading threading ---> \',i)
			# time.sleep(1)
			self.threads.append(WooYunSpider(self.queue)) # threads 列表装的是 爬虫线程

	def wait(self):
		for t in self.threads:
			if t.isAlive():
				t.join()


def test():
	url = \'http://wooyun.org/bugs/wooyun-2016-0177647\'
	r = get(url,headers = HEADERS)
	html = r.text
	# print type(html)
	# keywords" content="(.*?),(.*?),(.*?),wooyun  ====> 厂商，白帽子，类型
	pattern1 = re.compile(r\'title>(.*?)\\| WooYun\')
	pattern2 = re.compile(r\'keywords" content="(.*?),(.*?),(.*?),wooyun\')
	pattern3 = re.compile(r\'漏洞Rank：(\\d{1,3})\')
	for x in range(500):
		res = pattern1.search(html)
		# print (res.group(1))
		res = pattern2.search(html)
		# print (res.group(1),res.group(2),res.group(3))
		res = pattern3.search(html)
		# print (res.group(1))
		x += 1
		print(x)
	# rank = res.group(4).strip()

	# print html

def test2():
	url = \'http://wooyun.org/bugs/wooyun-2016-0177647\'
	r = get(url,headers = HEADERS)
	html = r.text
	pattern = re.compile(r\'title>(.*?)\\| WooYun.*?keywords" content="(.*?),(.*?),(.*?),wooyun.*?漏洞Rank：(\\d{1,3})\',re.S)
	for x in range(500):
		res = pattern.search(html)
		# print (res.group(1),res.group(2),res.group(3),res.group(4),res.group(5))
		x += 1
		print(x)
# 保存结果
def save2file(filename,filename_failed_id):
	with open(filename,\'w\') as output:
		for item in ID_DETAILS:
			try: # 写入文件可能会诱发 gbk 编码异常，这里忽略
				output.write(item + \'\\n\')
			except Exception as e:
				pass
			
	with open(filename_failed_id,\'w\') as output:
		output.write(\'\\n\'.join(Failed_ID))

if __name__ == \'__main__\':

	socket.setdefaulttimeout(1)
	start = time.time()

	# test()

	# 日志记录
	ERR_LOG = open(\'err_log.txt\',\'w\')
	RES_LOG = open(\'res_log.txt\',\'w\')
	id_file = \'id_0526.txt\'
	# id_file = \'id_test.txt\'
	tp = ThreadPool(20,id_file)
	tp.wait()

	save2file(\'id_details.txt\',\'failed_id.txt\')

	end = time.time()
	print (\'[ - info ] cost time :{:.2f}s\'.format(end - start))