静态网页爬虫-安居客

Posted 2021-06-04 柴立不阿

python+request+xpath 12小时1w3s条数据 动态验证码

难度：★★

"""
@version: python3.6
@author: ‘achai‘
@software: PyCharm
@file: demo_5.py
@time: 2020/12/26 22:03
"""
import urllib.request
from lxml import etree
import xlwt
import re
import time
import random

book = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = book.add_sheet('housePrice')
sheet.write(0, 0, '经度')
sheet.write(0, 1, '纬度')
sheet.write(0, 2, '毛坯或者精装')
sheet.write(0, 3, '容积率')
sheet.write(0, 4, '绿化率')
sheet.write(0, 5, '物业费')
sheet.write(0, 6, '电梯')
sheet.write(0, 7, '低层或者高层')
sheet.write(0, 8, '室')
sheet.write(0, 9, '厅')
sheet.write(0, 10, '卫')
sheet.write(0, 11, '面积')
sheet.write(0, 12, '建造年')
sheet.write(0, 13, '房价')
sheet.write(0, 14, '小区名称')
sheet.write(0, 15, '小区链接')
n = 1

page_headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
    "Cookie": "aQQ_ajkguid=A789826A-7A5C-311C-565E-05CDD916C93C; id58=e87rkF/E7Q5+DzArBOW3Ag==; 58tj_uuid=a6d317a6-9b64-4a76-b92b-5af34c96bde3; als=0; wmda_new_uuid=1; wmda_uuid=a1e5f2b84628e0abf6f31c71bfa07655; wmda_visited_projects=%3B6289197098934; isp=true; ctid=28; _ga=GA1.2.646930384.1606826267; sessid=B47733BE-C6A8-8FEE-E0D8-FB445F012E3B; twe=2; wmda_session_id_6289197098934=1608990327081-f825170b-4ec7-d5d4; obtain_by=1; new_session=1; init_refer=; new_uv=3; _gid=GA1.2.1955540.1608990327; _gat=1; xxzl_cid=d869ee07e4404df6b994853a3eba70ee; xzuid=8b5f7a17-6587-4c55-bcce-02b0664b781f"
}

def getAllurl(baseUrl):
    listall=[]
    for i in range(0,50):
        i=i+1
        #https://sjz.anjuke.com/sale/yuhuaf/p2/#filtersort
        fullUrl=baseUrl+'p'+str(i)+'/'

        #提取每一页的所有链接
        req = urllib.request.Request(fullUrl, headers=page_headers)
        data = urllib.request.urlopen(req).read().decode('utf-8', 'ignore')
        html = etree.HTML(data)  # 将页面转换成文档树
        # 提取网页所有链接
        house_url = html.xpath('//div[@class="house-details"]/div[@class="house-title"]/a/@href')
        listall.extend(house_url)
        print('该页所有链接提取完成！',i)
    return listall

def getMsg(Ourl):
    try:
        print(Ourl)
        req = urllib.request.Request(Ourl, headers=page_headers)
        data = urllib.request.urlopen(req).read().decode('utf-8', 'ignore')
        html_1 = etree.HTML(data)  # 将页面转换成文档树
        # 坐标提取
        coord = html_1.xpath('//meta[@data-n-head="ssr"][3]/@content')
        num_list_new = map(lambda x: str(x), coord)  # 将列表转换为字符串
        coord_1 = ",".join(num_list_new)
        data_coord = re.findall(r"\d+\.?\d*", coord_1)
        lan = data_coord[0]
        lat = data_coord[1]
        # 毛坯或者精装
        data_decoration = html_1.xpath(
            '//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[2]/div[2]/text()')

        # 容积率
        data_plotRatio = html_1.xpath('//*[@id="community"]/div[1]/div[2]/div[2]/p/text()')

        # 绿化率
        greeningRate = html_1.xpath('//*[@id="community"]/div[1]/div[3]/div[1]/p[2]/text()')
        # 将score_list列表转换为以“，”为分隔符的字符串
        name_score_list_string = ",".join(greeningRate)
        # 提取数字
        data_greeningRate = re.findall(r"\d+\.?\d*", name_score_list_string)

        # 物业费 1.2 元/平米/月
        propertyFee = html_1.xpath('//*[@id="community"]/div[1]/div[2]/div[1]/p[2]/text()')
        # 将score_list列表转换为以“，”为分隔符的字符串
        fee_list_string = ",".join(propertyFee)
        # 提取数字
        data_propertyFee = re.findall(r"\d+\.?\d*", fee_list_string)

        # 电梯//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[1]/span[3]
        data_elevator = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[1]/span/text()')

        # 低层或者高层
        data_floor = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[1]/div[2]/text()')

        #  户型-几室
        data_jishi = html_1.xpath(
            '//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[1]/div[1]/i[1]/text()')
        data_jiting = html_1.xpath(
            '//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[1]/div[1]/i[2]/text()')
        data_jiwei = html_1.xpath(
            '//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[1]/div[1]/i[3]/text()')

        # 面积
        house_area = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[2]/div[1]/i/text()')

        # 建造时间
        house_time = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[3]/div[3]/div[2]/text()')
        # 将score_list列表转换为以“，”为分隔符的字符串
        fee_list_string = ",".join(house_time)
        # 提取数字
        house_time_2 = re.findall(r"\d+\.?\d*", fee_list_string)

        # 房价
        house_price = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[2]/div[1]/span[1]/text()')

        # 小区名称
        name = html_1.xpath('//*[@id="__layout"]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[2]/div[1]/a[1]/text()')
        name_1 = " , ".join(name)
        houseName = ''.join(re.findall('[\u4e00-\u9fa5]', name_1))

        # 用此条件判断安居客网站是否出现了验证码。如果house_area为空值，则输出提示，否则继续执行
        if len(house_area):

            global n
            sheet.write(n, 0, lan)
            sheet.write(n, 1, lat)
            sheet.write(n, 2, data_decoration)
            sheet.write(n, 3, data_plotRatio)
            sheet.write(n, 4, data_greeningRate)
            sheet.write(n, 5, data_propertyFee)
            sheet.write(n, 6, data_elevator)
            sheet.write(n, 7, data_floor)
            sheet.write(n, 8, data_jishi)
            sheet.write(n, 9, data_jiting)
            sheet.write(n, 10, data_jiwei)
            sheet.write(n, 11, house_area)
            sheet.write(n, 12, house_time_2)
            sheet.write(n, 13, house_price)
            sheet.write(n, 14, houseName)
            sheet.write(n, 15, Ourl)
            n = n + 1
            book.save('anjuke_sjz.xls')

    except:
        print("不好！！安居客出现了验证码！！！！！！！！！！！！！！！")
        time.sleep(30)  # 30秒后执行程序

if __name__ == '__main__':
    url_start = "https://sjz.anjuke.com/sale/kfqsjz/"
    allurl=getAllurl(url_start)
    print('共获取'+str(len(allurl)))
    print('所有链接提取完成！')
    k = 1
    for i in allurl:
        getMsg(i)
        print('正在写入第'+str(k)+'条数据')
        k = k + 1
        sleep_time = random.uniform(0.8, 1.5)  # random.uniform(x, y) 方法将随机生成一个实数，它在 [x,y] 范围内
        # 网页操作太频繁的话会被冻结，这个时候可以用time库里面的sleep操作
        time.sleep(sleep_time)  # 暂停给定秒数后执行程序

---

不足：以上代码还未能解决动态验证码，如图所示。