bokeyuan_python文章爬去入mongodb读取--LOWBIPROGRAMMER

Posted 2021-01-28 Xcsg

# -*- coding: utf-8 -*-

import requests,os

from lxml import etree

from pymongo import *

class Boke(object):

def __init__(self):

self.url ="https://www.cnblogs.com/cate/python/"

self.headers={\'user-agent\':\'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.90 Safari/537.36 2345Explorer/9.3.2.17331\'}

def get_data(self,url):

response = requests.get(url,headers=self.headers)

return response.content

def xml_data(self,data):

html = etree.HTML(data)

mes = html.xpath("//div[@class=\'post_item\']")

for i in mes:

dict={}

info_url = i.xpath("./div[@class=\'post_item_body\']/h3/a/@href")[0]

self.info_data(info_url)

dict[\'url\'] = info_url

self.write_dbs(dict)

def info_data(self,data):

path = "f:/woc/"

if not os.path.exists(path):

os.makedirs(path)

mes = self.get_data(data)

html = etree.HTML(mes)

list = html.xpath("//div[@id=\'topics\']/div[@class=\'post\']")

# print(list)

for x in list:

dictlist = {}

title = x.xpath("./h1[@class=\'postTitle\']/a/text()")[0]

info = x.xpath("./div[@class=\'postBody\']//text()")

dictlist[\'title\'] = title

dictlist[\'info\'] = info

self.write1_dbs(dictlist)

def dbs(self):

connect = MongoClient(\'127.0.0.1\',27017)

conn = connect[\'boke\']

conn1 =conn[\'zhu\']

conn2 =conn[\'info\']

return conn1,conn2

def write_dbs(self,data):

conn1,conn2 = self.dbs()

conn1.insert_one(data)

result=conn1.find()

for i in result:

print(i)

def write1_dbs(self,data):

conn1, conn2 = self.dbs()

conn2.insert_one(data)

result = conn2.find()

for i in result:

print(i)

def run(self):

url = self.url

data = self.get_data(url)

self.xml_data(data)

if __name__ == \'__main__\':

boke = Boke()

boke.run()