爬取福州地区职位数据

Posted 2021-03-05 hr1347114782

从网页源代码中找到我们需要爬取的标签且是html结构，爬取目标为class=’job_titile‘和class=‘job_content‘

import requests
from bs4 import BeautifulSoup
import bs4
import pandas as pd #引用工具
url = ‘http://fj.huatu.com/zt/2019zwb/diqu/1.html‘
def job(s):#定义函数
    try:
        headers = {‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36‘}
        r=requests.get(s,timeout=30,headers=headers)#发送get请求
        r.raise_for_status()
        r.encoding=r.apparent_encoding#统一编码
        soup=BeautifulSoup(r.text,‘lxml‘)#使用BeautifulSoup库
        return soup
    except:
        return ""#出现错误，返回空字符串
soup=job(url)
a=[]#定义两个空列表
b=[]
for link1 in soup.find_all(‘div‘,class_=‘job_content‘): #使用find_all函数查找标签
    b.append(link1.get_text())
for link2 in soup.find_all(‘div‘,class_=‘job_title‘):
    a.append(link2.get_text().strip())
data=pd.DataFrame([a,b],index=["目录","职位"])#使用DataFrame可视化
print("福州地区职位数据:","
")#使用print函数打印
print(data)

爬取结果如图