抓取小猪短租列表内容并保存在mongodb里

Posted 2020-10-05

import pymongo
import requests
from bs4 import BeautifulSoup
client = pymongo.MongoClient(‘localhost‘,27017)
xiaozhu = client[‘xiazhu‘]
sheet_line = xiaozhu[‘sheet_line‘]
def get_page_within(pages):
    for page_numb in range(1,pages+1):
        web_data = requests.get(‘http://bj.xiaozhu.com/search-duanzufang-p{}-0/‘.format(page_numb))
        soup = BeautifulSoup(web_data.text,‘lxml‘)
        titles = soup.select(‘span.result_title‘)
        prices = soup.select(‘span.result_price > i‘)
        for title,price in zip(titles,prices):
            data = {
                ‘title‘:title.get_text(),
                ‘price‘:int(price.get_text())
            }
            sheet_line.insert_one(data)
get_page_within(3)
for item in sheet_line.find({‘price‘:{‘$lt‘:500}}):
    print(item)