6通过xpath获取网页数据

Posted 2020-10-28 toloy

from urllib import request
from lxml import etree
# 请求的url
url = "http://www.dfenqi.cn/Product/Index"
# 请求的头文件
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/64.0.3282.186 Safari/537.36"
}
# 创建请求对象
req = request.Request(url,headers = headers)
# 创建处理器对象
httpHandler = request.HTTPHandler()
# 创建opener
opener = request.build_opener(httpHandler)
# 发送请求
response = opener.open(req)
# 读取源文件
html = response.read().decode(‘utf-8‘)
# 创建xpath关系
xpath = "//div[@class=‘liebiao‘]/ul/li/p/text()"
# 获取属性值列表
# xpath = "//div[@class=‘liebiao‘]/ul/li/p/@class"
# 将html转换成可解析对象
selector = etree.HTML(html)
# 返回xpath查询列表
goodsList = selector.xpath(xpath)
# 显示商品标题
for goods in goodsList:
    print(goods)