第一次完美完成xpath 构建不完全href 猎聘网

Posted 2020-10-28 mayunji

import requests
from lxml import etree
from multiprocessing import Pool
headers = {
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/55.0.2883.87 Safari/537.36‘
}
def getLinks(url):
    wb_data = requests.get(url,headers=headers)
    soup = etree.HTML(wb_data.text)
    links = soup.xpath(‘//div[@class="job-info"]‘)
    for link in links:
        href = link.xpath(‘h3/a/@href‘)[0]
        getinfo(href)

def getinfo(url):
    try:
        wb_data = requests.get(url,headers=headers)
        soup = etree.HTML(wb_data.text)
        requires = soup.xpath(‘//div[@class="job-title-left"]‘)
        for require in requires:
            a = require.xpath(‘div[1]/span[3]/text()‘)
            print(a)
    except:
        href = ‘https://www.liepin.com‘ + url
        getinfo(href)

if __name__ == ‘__main__‘:
    urls = [‘https://www.liepin.com/zhaopin/?pubTime=&ckid=e49b6fdfb6a698ed&fromSearchBtn=2&compkind=&isAnalysis=&init=-1&searchType=1&dqs=&industryType=&jobKind=&sortFlag=15&degradeFlag=0&industries=&salary=&compscale=&clean_condition=&key=python%E7%88%AC%E8%99%AB&headckid=e49b6fdfb6a698ed&d_pageSize=40&siTag=tF7woi2y6F2s2RnHW3wfUw~fA9rXquZc5IkJpXC-Ycixw&d_headId=ca30a54749a469a7967ac218f6204031&d_ckId=ca30a54749a469a7967ac218f6204031&d_sfrom=search_prime&d_curPage=2&curPage={0}‘
.format(str(i))
            for i in range(0,4)]
    for url in urls:
        getLinks(url)